dsh-eval-guard 0.2.3 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +91 -13
- package/assets/commit0/instance.json +30 -0
- package/assets/commit0/workspace.zip +0 -0
- package/assets/featurebench/hidden/test_regression.py +690 -0
- package/assets/featurebench/instance.json +16 -0
- package/assets/featurebench/workspace.zip +0 -0
- package/assets/swe-bench-pro/hidden/test.patch +36 -0
- package/assets/swe-bench-pro/instance.json +10 -0
- package/assets/swe-bench-pro/workspace.zip +0 -0
- package/eval/README.md +18 -0
- package/eval/arena/arena/README.md +21 -0
- package/eval/arena/arena/bt.py +104 -0
- package/eval/arena/arena/clock.py +15 -0
- package/eval/arena/arena/core.py +265 -0
- package/eval/arena/arena/run.cmd +24 -0
- package/eval/arena/arena/run.sh +11 -0
- package/eval/arena/arena/schedule.py +79 -0
- package/eval/arena/arena/tournament.py +183 -0
- package/eval/arena/bomberman/bots/README.md +16 -0
- package/eval/arena/bomberman/bots/deepseek-v4.1-flash.cpp +447 -0
- package/eval/arena/bomberman/bots/doubao-seed-2.1.cpp +542 -0
- package/eval/arena/bomberman/bots/glm-5.3.cpp +406 -0
- package/eval/arena/bomberman/bots/gpt-6-astra.cpp +768 -0
- package/eval/arena/bomberman/bots/gpt-6-sol.cpp +444 -0
- package/eval/arena/bomberman/bots/kimi-k3.cpp +432 -0
- package/eval/arena/bomberman/bots/mimo-v2.6-pro.cpp +529 -0
- package/eval/arena/bomberman/bots/opus-5.5.cpp +548 -0
- package/eval/arena/bomberman/bots/qwen-3.8-max.cpp +836 -0
- package/eval/arena/bomberman/tournament/README.md +11 -0
- package/eval/arena/bomberman/tournament/bt.py +56 -0
- package/eval/arena/bomberman/tournament/evaluate.py +199 -0
- package/eval/arena/bomberman/tournament/run.py +79 -0
- package/eval/arena/bomberman/tournament/schedule-2to10.json.gz +0 -0
- package/eval/arena/bomberman/workspace/MANIFEST.json +13 -0
- package/eval/arena/bomberman/workspace/README.md +29 -0
- package/eval/arena/bomberman/workspace/RULES.md +100 -0
- package/eval/arena/bomberman/workspace/baseline.cpp +240 -0
- package/eval/arena/bomberman/workspace/clock.py +25 -0
- package/eval/arena/bomberman/workspace/engine/arena.py +198 -0
- package/eval/arena/bomberman/workspace/engine/build.py +35 -0
- package/eval/arena/bomberman/workspace/engine/engine.py +256 -0
- package/eval/arena/bomberman/workspace/engine/verify_replay.py +24 -0
- package/eval/arena/bomberman/workspace/run.cmd +28 -0
- package/eval/arena/bomberman/workspace/run.sh +16 -0
- package/eval/arena/bomberman/workspace/viewer/index.html +42 -0
- package/eval/arena/ctf/bots/README.md +18 -0
- package/eval/arena/ctf/bots/deepseek-v4.1-flash.cpp +601 -0
- package/eval/arena/ctf/bots/doubao-seed-2.1-turbo.cpp +320 -0
- package/eval/arena/ctf/bots/fable-5.1.cpp +634 -0
- package/eval/arena/ctf/bots/glm-5.3.cpp +689 -0
- package/eval/arena/ctf/bots/gpt-6-astra.cpp +657 -0
- package/eval/arena/ctf/bots/gpt-6.1-sol.cpp +725 -0
- package/eval/arena/ctf/bots/kimi-k3.cpp +667 -0
- package/eval/arena/ctf/bots/mimo-v2.6-pro.cpp +533 -0
- package/eval/arena/ctf/bots/opus-5.5.cpp +645 -0
- package/eval/arena/ctf/bots/qwen-3.8-max.cpp +726 -0
- package/eval/arena/ctf/bots/sonnet-5.5.cpp +569 -0
- package/eval/arena/ctf/tournament/README.md +15 -0
- package/eval/arena/ctf/tournament/formal-2to8.json +31 -0
- package/eval/arena/ctf/tournament/formal-9to15.json +31 -0
- package/eval/arena/ctf/workspace/README.md +41 -0
- package/eval/arena/ctf/workspace/RULES.md +119 -0
- package/eval/arena/ctf/workspace/arena.py +186 -0
- package/eval/arena/ctf/workspace/baseline.cpp +257 -0
- package/eval/arena/ctf/workspace/clock.py +15 -0
- package/eval/arena/ctf/workspace/core.py +265 -0
- package/eval/arena/ctf/workspace/engine.py +389 -0
- package/eval/arena/ctf/workspace/run.cmd +24 -0
- package/eval/arena/ctf/workspace/run.sh +11 -0
- package/eval/arena/ctf/workspace/viewer.html +71 -0
- package/eval/arena/land/bots/README.md +17 -0
- package/eval/arena/land/bots/deepseek-4.1f@workbuddy#1/bot.cpp +608 -0
- package/eval/arena/land/bots/glm-5.3@workbuddy#1/bot.cpp +521 -0
- package/eval/arena/land/bots/gpt-6-astra@codex#1/bot.cpp +373 -0
- package/eval/arena/land/bots/gpt-6-sol@codex#1/bot.cpp +341 -0
- package/eval/arena/land/bots/kimi-k3@workbuddy#1/bot.cpp +492 -0
- package/eval/arena/land/bots/mimo-2.6-pro@zcode#1/bot.cpp +652 -0
- package/eval/arena/land/bots/mystery-model@zcode#1/bot.cpp +466 -0
- package/eval/arena/land/bots/opus-5.5@claude-code#1/bot.cpp +630 -0
- package/eval/arena/land/bots/sonnet-5@claude-code#1/bot.cpp +619 -0
- package/eval/arena/land/tournament/make_replay_page.py +17 -0
- package/eval/arena/land/tournament/rate.py +290 -0
- package/eval/arena/land/tournament/round_robin.py +172 -0
- package/eval/arena/land/tournament/sparring.cpp +356 -0
- package/eval/arena/land/workspace/RULES.md +114 -0
- package/eval/arena/land/workspace/TASK.md +46 -0
- package/eval/arena/land/workspace/anchors/.build/greedy_f2e6d2dea5ca +0 -0
- package/eval/arena/land/workspace/anchors/greedy.cpp +194 -0
- package/eval/arena/land/workspace/anchors/random.cpp +80 -0
- package/eval/arena/land/workspace/clock.py +64 -0
- package/eval/arena/land/workspace/engine/build.py +33 -0
- package/eval/arena/land/workspace/engine/game.py +273 -0
- package/eval/arena/land/workspace/engine/referee.py +308 -0
- package/eval/arena/land/workspace/starter/.build/bot_be32b45b4699 +0 -0
- package/eval/arena/land/workspace/starter/bot.cpp +82 -0
- package/eval/arena/land/workspace/viewer/viewer.html +210 -0
- package/eval/arena/tank/bots/README.md +24 -0
- package/eval/arena/tank/bots/half1/deepseek-v4.1-flash.cpp +367 -0
- package/eval/arena/tank/bots/half1/doubao-seed-evolving.cpp +402 -0
- package/eval/arena/tank/bots/half1/gemini-4-argon.cpp +1132 -0
- package/eval/arena/tank/bots/half1/glm-5.3.cpp +551 -0
- package/eval/arena/tank/bots/half1/gpt-6-astra.cpp +233 -0
- package/eval/arena/tank/bots/half1/gpt-6.1-sol-ultrafast.cpp +350 -0
- package/eval/arena/tank/bots/half1/gpt-6.1-sol.cpp +78 -0
- package/eval/arena/tank/bots/half1/grok-4.7.cpp +1099 -0
- package/eval/arena/tank/bots/half1/haiku-5.5.cpp +210 -0
- package/eval/arena/tank/bots/half1/kimi-k3.cpp +353 -0
- package/eval/arena/tank/bots/half1/mimo-v2.6-pro.cpp +336 -0
- package/eval/arena/tank/bots/half1/muse-wonder-full.cpp +560 -0
- package/eval/arena/tank/bots/half1/opus-5.5.cpp +435 -0
- package/eval/arena/tank/bots/half1/qwen-3.8-max.cpp +537 -0
- package/eval/arena/tank/bots/half2/deepseek-v4.1-flash.cpp +425 -0
- package/eval/arena/tank/bots/half2/doubao-seed-evolving.cpp +308 -0
- package/eval/arena/tank/bots/half2/gemini-4-argon.cpp +1390 -0
- package/eval/arena/tank/bots/half2/glm-5.3.cpp +651 -0
- package/eval/arena/tank/bots/half2/gpt-6-astra.cpp +241 -0
- package/eval/arena/tank/bots/half2/gpt-6.1-sol-ultrafast.cpp +356 -0
- package/eval/arena/tank/bots/half2/gpt-6.1-sol.cpp +87 -0
- package/eval/arena/tank/bots/half2/grok-4.7.cpp +1205 -0
- package/eval/arena/tank/bots/half2/haiku-5.5.cpp +210 -0
- package/eval/arena/tank/bots/half2/kimi-k3.cpp +487 -0
- package/eval/arena/tank/bots/half2/mimo-v2.6-pro.cpp +336 -0
- package/eval/arena/tank/bots/half2/muse-wonder-full.cpp +560 -0
- package/eval/arena/tank/bots/half2/opus-5.5.cpp +476 -0
- package/eval/arena/tank/bots/half2/qwen-3.8-max.cpp +392 -0
- package/eval/arena/tank/tournament/README.md +24 -0
- package/eval/arena/tank/tournament/bt.py +218 -0
- package/eval/arena/tank/tournament/final.json +202 -0
- package/eval/arena/tank/tournament/halftime.json +146 -0
- package/eval/arena/tank/tournament/progress.json +911 -0
- package/eval/arena/tank/tournament/schedule.py +100 -0
- package/eval/arena/tank/tournament/sparring.cpp +928 -0
- package/eval/arena/tank/tournament/tournament.py +1426 -0
- package/eval/arena/tank/workspace/README.md +169 -0
- package/eval/arena/tank/workspace/RULES.md +443 -0
- package/eval/arena/tank/workspace/arena.py +862 -0
- package/eval/arena/tank/workspace/baseline.cpp +259 -0
- package/eval/arena/tank/workspace/clock.py +103 -0
- package/eval/arena/tank/workspace/core.py +712 -0
- package/eval/arena/tank/workspace/engine.py +573 -0
- package/eval/arena/tank/workspace/history/.lock +0 -0
- package/eval/arena/tank/workspace/history/20261017-193610-08715287f91f-bot.cpp +109 -0
- package/eval/arena/tank/workspace/history/20261017-193619-f00b2d5635fa-gpt-6-astra.cpp +241 -0
- package/eval/arena/tank/workspace/history/20261017-193643-63319e0a2b1a-gpt-6_1-sol-ultrafast.cpp +356 -0
- package/eval/arena/tank/workspace/history/20261017-193647-41c1341d0a8a-gemini-4-argon.cpp +1390 -0
- package/eval/arena/tank/workspace/history/20261017-193652-e34c0f45539a-gpt-6_1-sol.cpp +87 -0
- package/eval/arena/tank/workspace/history/20261017-193654-c026348ce96e-opus-5_5.cpp +476 -0
- package/eval/arena/tank/workspace/history/20261017-193655-4a21366e94d1-muse-wonder-full.cpp +560 -0
- package/eval/arena/tank/workspace/history/20261017-193657-dbc4e9969c52-mimo-v2_6-pro.cpp +336 -0
- package/eval/arena/tank/workspace/history/20261017-193703-723cd036eb31-grok-4_7.cpp +1205 -0
- package/eval/arena/tank/workspace/history/20261017-193707-1ebbb227be68-deepseek-v4_1-flash.cpp +425 -0
- package/eval/arena/tank/workspace/history/20261017-193709-aeb0e7202ad5-doubao-seed-evolving.cpp +308 -0
- package/eval/arena/tank/workspace/history/20261017-193710-b21ebeb9bc76-kimi-k3.cpp +487 -0
- package/eval/arena/tank/workspace/history/20261017-193712-54cd86531905-haiku-5_5.cpp +210 -0
- package/eval/arena/tank/workspace/history/20261017-193714-efdb8e35c0bf-qwen-3_8-max.cpp +392 -0
- package/eval/arena/tank/workspace/history/20261017-193715-681423c23fdb-glm-5_3.cpp +651 -0
- package/eval/arena/tank/workspace/history/20261017-193717-1a8b3c75181d-sparring.cpp +928 -0
- package/eval/arena/tank/workspace/history/20261017-193720-1e0803a541e8-baseline.cpp +259 -0
- package/eval/arena/tank/workspace/history/log.jsonl +103 -0
- package/eval/arena/tank/workspace/judge/cycles.hpp +53 -0
- package/eval/arena/tank/workspace/judge/fast.cpp +337 -0
- package/eval/arena/tank/workspace/judge/referee.cpp +125 -0
- package/eval/arena/tank/workspace/judge/speed.cpp +99 -0
- package/eval/arena/tank/workspace/run.cmd +31 -0
- package/eval/arena/tank/workspace/run.sh +23 -0
- package/eval/arena/tank/workspace/tank.hpp +1762 -0
- package/eval/arena/tank/workspace/tank_main.cpp +215 -0
- package/eval/arena/tank/workspace/viewer.html +1150 -0
- package/eval/arena/warehouse/workspace/SPEC.md +130 -0
- package/eval/arena/warehouse/workspace/TASK.md +71 -0
- package/eval/arena/warehouse/workspace/baseline/common.h +88 -0
- package/eval/arena/warehouse/workspace/baseline/main.cpp +27 -0
- package/eval/arena/warehouse/workspace/baseline/old_stuff.cpp +87 -0
- package/eval/arena/warehouse/workspace/baseline/sim.cpp +427 -0
- package/eval/arena/warehouse/workspace/baseline/util.cpp +220 -0
- package/eval/arena/warehouse/workspace/clock.py +42 -0
- package/eval/arena/warehouse/workspace/src/common.h +88 -0
- package/eval/arena/warehouse/workspace/src/main.cpp +27 -0
- package/eval/arena/warehouse/workspace/src/old_stuff.cpp +87 -0
- package/eval/arena/warehouse/workspace/src/sim.cpp +427 -0
- package/eval/arena/warehouse/workspace/src/util.cpp +220 -0
- package/eval/arena/warehouse/workspace/tools/bench.py +119 -0
- package/eval/arena/warehouse/workspace/tools/check_input.py +161 -0
- package/eval/arena/warehouse/workspace/tools/gen.py +132 -0
- package/eval/arena/warehouse/workspace/tools/measure.py +107 -0
- package/eval/arena/warehouse/workspace/workloads/public/pub_crowded.in +1420 -0
- package/eval/arena/warehouse/workspace/workloads/public/pub_crowded.out +9162 -0
- package/eval/arena/warehouse/workspace/workloads/public/pub_dynamic.in +1465 -0
- package/eval/arena/warehouse/workspace/workloads/public/pub_dynamic.out +3439 -0
- package/eval/arena/warehouse/workspace/workloads/public/pub_flood.in +2757 -0
- package/eval/arena/warehouse/workspace/workloads/public/pub_flood.out +3126 -0
- package/eval/arena/warehouse/workspace/workloads/public/pub_small.in +731 -0
- package/eval/arena/warehouse/workspace/workloads/public/pub_small.out +2578 -0
- package/eval/arena/warehouse/workspace/workloads/public/pub_wide.in +528 -0
- package/eval/arena/warehouse/workspace/workloads/public/pub_wide.out +2026 -0
- package/eval/bomberman.py +20 -0
- package/eval/commit0.py +21 -0
- package/eval/common.py +61 -0
- package/eval/ctf.py +20 -0
- package/eval/featurebench.py +25 -0
- package/eval/land.py +20 -0
- package/eval/legacy/bomberman-prices.json +4 -0
- package/eval/legacy/bomberman.py +161 -0
- package/eval/legacy/commit0.py +91 -0
- package/eval/legacy/ctf-prices.json +4 -0
- package/eval/legacy/ctf.py +178 -0
- package/eval/legacy/featurebench.py +103 -0
- package/eval/legacy/land-prices.json +4 -0
- package/eval/legacy/land.py +177 -0
- package/eval/legacy/swe_bench_pro.py +116 -0
- package/eval/legacy/tank-prices.json +3 -0
- package/eval/legacy/tank.py +408 -0
- package/eval/legacy/warehouse-prices.json +4 -0
- package/eval/legacy/warehouse.py +365 -0
- package/eval/swe-bench-pro.py +25 -0
- package/eval/tank.py +20 -0
- package/eval/warehouse.py +20 -0
- package/lib/client/index.js +376 -63
- package/lib/client/index.js.map +1 -1
- package/lib/index.js +684 -27
- package/package.json +2 -1
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""赛程(1 对 1):每张图上,要打的每一对都换边各打一局。
|
|
2
|
+
|
|
3
|
+
和往期 公共/竞技场/schedule.py 的区别:往期两人局是"每张图随机两两分组",一张图上每个程序只碰到一个对手;
|
|
4
|
+
这里是每张图上全部配对都打,所以任何两个程序的交手次数、坐两个座位的次数、遇到的地图都完全相同。
|
|
5
|
+
|
|
6
|
+
make_schedule(names, maps, seed) 全配对
|
|
7
|
+
make_schedule(names, maps, seed, pairs=[[a, b], ...]) 只打指定的这些对(比如"下半场版本 对 固定对手池")
|
|
8
|
+
make_schedule(names, maps, seed, map_seeds=[种子, ...]) 地图不抽,按给定的种子依次用(和另一场比赛用同样的图时)
|
|
9
|
+
|
|
10
|
+
地图风格按图的序号轮换(第 k 张图的风格是 k % 风格数),不靠种子取余,所以每种风格的图数最多差 1。
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import hashlib
|
|
15
|
+
import itertools
|
|
16
|
+
import json
|
|
17
|
+
import random
|
|
18
|
+
from collections import Counter
|
|
19
|
+
|
|
20
|
+
SEED_LOW, SEED_HIGH = 1_000_000, 2 ** 31 # 正式赛的地图种子从这个范围里抽,避开选手本地常用的小种子
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def all_pairs(names):
|
|
24
|
+
return [list(p) for p in itertools.combinations(sorted(names), 2)]
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def make_schedule(names, maps, seed, pairs=None, n_styles=4, avoid_seeds=(), map_seeds=None):
|
|
28
|
+
"""返回局列表。每局:{"id", "map", "map_seed", "style", "seats": [0 号位的名字, 1 号位的名字]}。
|
|
29
|
+
|
|
30
|
+
同一张图上的局排在一起("map" 相同),图内各对的先后顺序打乱;每一对的两局紧挨着。
|
|
31
|
+
map_seeds 给了就不抽地图:第 m 张图用 map_seeds[m](要和另一场比赛用同样的图时用;这时不看 avoid_seeds)。
|
|
32
|
+
风格仍按图的序号轮换,所以"第 m 张图"在两场比赛里种子和风格都相同。"""
|
|
33
|
+
names = sorted(names)
|
|
34
|
+
if len(set(names)) != len(names):
|
|
35
|
+
raise ValueError("参赛名字有重复")
|
|
36
|
+
if pairs is None:
|
|
37
|
+
pairs = all_pairs(names)
|
|
38
|
+
else:
|
|
39
|
+
seen = set()
|
|
40
|
+
for a, b in pairs:
|
|
41
|
+
if a == b or a not in names or b not in names:
|
|
42
|
+
raise ValueError(f"配对不对:{a} 对 {b}")
|
|
43
|
+
key = tuple(sorted((a, b)))
|
|
44
|
+
if key in seen:
|
|
45
|
+
raise ValueError(f"配对重复:{a} 对 {b}")
|
|
46
|
+
seen.add(key)
|
|
47
|
+
pairs = [sorted(p) for p in pairs]
|
|
48
|
+
if maps < 1 or not pairs:
|
|
49
|
+
raise ValueError("至少要有一张图、一对选手")
|
|
50
|
+
if map_seeds is not None:
|
|
51
|
+
map_seeds = [int(s) for s in map_seeds]
|
|
52
|
+
if len(map_seeds) < maps:
|
|
53
|
+
raise ValueError(f"指定的地图种子只有 {len(map_seeds)} 个,不够 {maps} 张图")
|
|
54
|
+
if len(set(map_seeds[:maps])) != maps:
|
|
55
|
+
raise ValueError("指定的地图种子有重复")
|
|
56
|
+
rng = random.Random(seed)
|
|
57
|
+
used = set(int(s) for s in avoid_seeds)
|
|
58
|
+
games = []
|
|
59
|
+
for m in range(maps):
|
|
60
|
+
if map_seeds is not None:
|
|
61
|
+
map_seed = map_seeds[m]
|
|
62
|
+
else:
|
|
63
|
+
map_seed = rng.randrange(SEED_LOW, SEED_HIGH)
|
|
64
|
+
while map_seed in used:
|
|
65
|
+
map_seed = rng.randrange(SEED_LOW, SEED_HIGH)
|
|
66
|
+
used.add(map_seed)
|
|
67
|
+
order = [list(p) for p in pairs]
|
|
68
|
+
rng.shuffle(order)
|
|
69
|
+
for a, b in order:
|
|
70
|
+
if rng.random() < 0.5:
|
|
71
|
+
a, b = b, a
|
|
72
|
+
for seats in ([a, b], [b, a]):
|
|
73
|
+
games.append({"id": f"m{m:03d}-g{len(games) + 1:05d}", "map": m, "map_seed": map_seed,
|
|
74
|
+
"style": m % n_styles, "seats": seats})
|
|
75
|
+
return games
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def audit(games):
|
|
79
|
+
"""赛程是否均衡:每对的局数、每个程序坐两个座位的次数、每种风格的图数。"""
|
|
80
|
+
pair = Counter(tuple(sorted(g["seats"])) for g in games)
|
|
81
|
+
ordered = Counter(tuple(g["seats"]) for g in games)
|
|
82
|
+
seat = Counter((n, i) for g in games for i, n in enumerate(g["seats"]))
|
|
83
|
+
names = sorted({n for g in games for n in g["seats"]})
|
|
84
|
+
styles = Counter()
|
|
85
|
+
for m, style in {(g["map"], g["style"]) for g in games}:
|
|
86
|
+
styles[style] += 1
|
|
87
|
+
maps = len({g["map"] for g in games})
|
|
88
|
+
return {"games": len(games), "maps": maps, "pairs": len(pair),
|
|
89
|
+
"games_per_pair_min": min(pair.values()), "games_per_pair_max": max(pair.values()),
|
|
90
|
+
"swap_balanced": all(ordered[a, b] == ordered[b, a] for a, b in pair),
|
|
91
|
+
"seat_balanced": all(seat[n, 0] == seat[n, 1] for n in names),
|
|
92
|
+
"games_per_program": {n: seat[n, 0] + seat[n, 1] for n in names},
|
|
93
|
+
"maps_per_style": {str(k): v for k, v in sorted(styles.items())},
|
|
94
|
+
"distinct_map_seeds": len({g["map_seed"] for g in games}) == maps}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def games_hash(games):
|
|
98
|
+
"""赛程的指纹:续跑前用它确认赛程没有被改过。"""
|
|
99
|
+
raw = json.dumps(games, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
100
|
+
return hashlib.sha256(raw).hexdigest()
|