aiblueprint-cli 1.4.108 → 1.4.110
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/agents-config/claude-config/mods/agent4everything/.claude-plugin/plugin.json +5 -0
- package/agents-config/claude-config/mods/agent4everything/hooks/hooks.json +1 -0
- package/agents-config/claude-config/mods/agent4everything/hooks/register.ts +96 -0
- package/agents-config/claude-config/mods/aiblueprint/.claude-plugin/plugin.json +71 -0
- package/agents-config/claude-config/mods/aiblueprint/hooks/hooks.json +1 -0
- package/agents-config/claude-config/mods/aiblueprint/hooks/register.tsx +309 -0
- package/agents-config/claude-config/mods/aiblueprint/tests/band.test.tsx +74 -0
- package/agents-config/claude-config/mods/aiblueprint/types/index.d.ts +14 -0
- package/agents-config/claude-config/scripts/.claude/skills/fix-on-my-computer/SKILL.md +8 -74
- package/agents-config/claude-config/scripts/statusline/src/index.ts +1 -0
- package/agents-config/scripts/.claude/skills/fix-on-my-computer/SKILL.md +8 -74
- package/agents-config/scripts/statusline/src/index.ts +1 -0
- package/agents-config/skills/agents-manager/SKILL.md +15 -453
- package/agents-config/skills/apex/SKILL.md +14 -110
- package/agents-config/skills/app-icon/SKILL.md +22 -117
- package/agents-config/skills/appstore-connect/SKILL.md +22 -85
- package/agents-config/skills/arena/LICENSE +21 -0
- package/agents-config/skills/arena/SKILL.md +371 -0
- package/agents-config/skills/arena/agents/openai.yaml +9 -0
- package/agents-config/skills/arena/assets/codex-icon.svg +22 -0
- package/agents-config/skills/arena/bracket.py +1235 -0
- package/agents-config/skills/arena/rubric.md +79 -0
- package/agents-config/skills/arena/strategies.json +205 -0
- package/agents-config/skills/audit-memories/SKILL.md +13 -139
- package/agents-config/skills/audit-skills/SKILL.md +6 -65
- package/agents-config/skills/environments-manager/SKILL.md +14 -238
- package/agents-config/skills/grill-me/SKILL.md +8 -22
- package/agents-config/skills/hooks-manager/SKILL.md +8 -54
- package/agents-config/skills/rules-manager/SKILL.md +11 -192
- package/agents-config/skills/skill-manager/SKILL.md +19 -158
- package/agents-config/skills/skill-manager/references/description-recommandation.md +15 -85
- package/agents-config/skills/test-audit/CAMPAIGN.md +117 -0
- package/agents-config/skills/test-audit/LICENSE +23 -0
- package/agents-config/skills/test-audit/SKILL.md +169 -0
- package/agents-config/skills/test-audit/agents/openai.yaml +10 -0
- package/agents-config/skills/test-audit/assets/codex-icon.svg +1 -0
- package/agents-config/skills/use-style/SKILL.md +8 -48
- package/dist/cli.js +125 -78
- package/package.json +1 -1
|
@@ -0,0 +1,1235 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""bracket.py: the tournament state machine behind /arena.
|
|
3
|
+
|
|
4
|
+
The whole arena lives in one JSON file (arena.json), so the orchestrator never
|
|
5
|
+
loses track of who is alive, who fought whom and what happens next, even if its
|
|
6
|
+
own context gets compacted halfway through a 100-agent run.
|
|
7
|
+
|
|
8
|
+
python3 bracket.py plan --agents 100 # rounds, sub-agent calls and waves. Writes nothing.
|
|
9
|
+
python3 bracket.py init --agents 100 --seed 7 --task-file task.md [--baseline-file old.md]
|
|
10
|
+
python3 bracket.py init --quick --task "..." # 16 agents
|
|
11
|
+
python3 bracket.py next # what to do now, and the exact command for it
|
|
12
|
+
python3 bracket.py prompts <phase> # write the sub-agent briefs, list the jobs left, in waves
|
|
13
|
+
python3 bracket.py check <phase> # which outputs are still missing
|
|
14
|
+
python3 bracket.py pairings # this round's matches, including the bye
|
|
15
|
+
python3 bracket.py collect # read every judge verdict and record the winners
|
|
16
|
+
python3 bracket.py record <match_id> <winner_id> [--reason "..."]
|
|
17
|
+
python3 bracket.py advance # close the round, eliminate the losers, pair the survivors
|
|
18
|
+
python3 bracket.py status # alive and eliminated, per round
|
|
19
|
+
python3 bracket.py winner # the survivor, what it beat, the attacks it survived
|
|
20
|
+
python3 bracket.py card <agent_id> # one competitor's strategy card
|
|
21
|
+
|
|
22
|
+
Phases, in order: spawn, then per round attack, defend, judge; then final (only when
|
|
23
|
+
there is a rejected answer to beat). Every command takes --dir; without it, the run
|
|
24
|
+
named in .arena/LATEST is used.
|
|
25
|
+
|
|
26
|
+
Python 3.8+, standard library only.
|
|
27
|
+
"""
|
|
28
|
+
import argparse
|
|
29
|
+
import json
|
|
30
|
+
import math
|
|
31
|
+
import os
|
|
32
|
+
import random
|
|
33
|
+
import re
|
|
34
|
+
import sys
|
|
35
|
+
import time
|
|
36
|
+
|
|
37
|
+
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
38
|
+
SELF = os.path.abspath(__file__)
|
|
39
|
+
STRATEGIES_PATH = os.path.join(HERE, "strategies.json")
|
|
40
|
+
SKILL_PATH = os.path.join(HERE, "SKILL.md")
|
|
41
|
+
RUBRIC_PATH = os.path.join(HERE, "rubric.md")
|
|
42
|
+
|
|
43
|
+
DEFAULT_AGENTS = 100
|
|
44
|
+
QUICK_AGENTS = 16
|
|
45
|
+
DEFAULT_WAVE = 10 # Claude Code runs at most 10 tool calls at once by default
|
|
46
|
+
ROOT = ".arena"
|
|
47
|
+
LATEST = "LATEST"
|
|
48
|
+
STATE_FILE = "arena.json"
|
|
49
|
+
PHASES = ("spawn", "attack", "defend", "judge", "final")
|
|
50
|
+
TEMPLATES = ("competitor", "attacker", "defender", "judge", "final")
|
|
51
|
+
CALLS_PER_MATCH = 5 # 2 attacks, 2 defenses, 1 judge
|
|
52
|
+
|
|
53
|
+
# Mirrors the table in rubric.md. tests/test_bracket.py checks they agree.
|
|
54
|
+
WEIGHTS = (
|
|
55
|
+
("correctness", 30),
|
|
56
|
+
("completeness", 25),
|
|
57
|
+
("specificity", 15),
|
|
58
|
+
("robustness", 20),
|
|
59
|
+
("clarity", 10),
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class ArenaError(Exception):
|
|
64
|
+
pass
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# ---------------------------------------------------------------- the cards
|
|
68
|
+
|
|
69
|
+
def load_strategies(path=STRATEGIES_PATH):
|
|
70
|
+
with open(path, encoding="utf-8") as fh:
|
|
71
|
+
data = json.load(fh)
|
|
72
|
+
for key in ("reasoning", "workflows", "strategies"):
|
|
73
|
+
items = data.get(key) or []
|
|
74
|
+
if not items:
|
|
75
|
+
raise ArenaError("strategies.json has no '%s' entries" % key)
|
|
76
|
+
ids = [it["id"] for it in items]
|
|
77
|
+
if len(set(ids)) != len(ids):
|
|
78
|
+
raise ArenaError("strategies.json has a duplicate id in '%s'" % key)
|
|
79
|
+
for it in items:
|
|
80
|
+
if not it.get("name") or not it.get("how"):
|
|
81
|
+
raise ArenaError("strategies.json entry '%s' needs a name and a how" % it["id"])
|
|
82
|
+
return data
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def combo_count(data):
|
|
86
|
+
return len(data["reasoning"]) * len(data["workflows"]) * len(data["strategies"])
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _edge_colour(edges, k):
|
|
90
|
+
"""Properly edge-colour a bipartite (multi)graph with k colours, then balance it.
|
|
91
|
+
|
|
92
|
+
edges are (row, col) pairs and k must be at least the largest degree. Proper
|
|
93
|
+
means no two edges at the same row, or at the same col, share a colour. Every
|
|
94
|
+
bipartite graph has one (Konig), found here with alternating-path swaps. Then
|
|
95
|
+
de Werra's swaps even the colour classes out until each is used floor or ceil
|
|
96
|
+
of len(edges) / k times.
|
|
97
|
+
"""
|
|
98
|
+
colour = [None] * len(edges)
|
|
99
|
+
at = ({}, {})
|
|
100
|
+
|
|
101
|
+
def slot(side, node):
|
|
102
|
+
return at[side].setdefault(node, {})
|
|
103
|
+
|
|
104
|
+
def put(e, c):
|
|
105
|
+
u, v = edges[e]
|
|
106
|
+
colour[e] = c
|
|
107
|
+
slot(0, u)[c] = e
|
|
108
|
+
slot(1, v)[c] = e
|
|
109
|
+
|
|
110
|
+
def take(e):
|
|
111
|
+
u, v = edges[e]
|
|
112
|
+
del slot(0, u)[colour[e]]
|
|
113
|
+
del slot(1, v)[colour[e]]
|
|
114
|
+
|
|
115
|
+
def walk(side, node, first, other):
|
|
116
|
+
path, c = [], first
|
|
117
|
+
while True:
|
|
118
|
+
e = slot(side, node).get(c)
|
|
119
|
+
if e is None or e in path:
|
|
120
|
+
return path
|
|
121
|
+
path.append(e)
|
|
122
|
+
node = edges[e][1 - side]
|
|
123
|
+
side = 1 - side
|
|
124
|
+
c = other if c == first else first
|
|
125
|
+
|
|
126
|
+
def swap(path, a, b):
|
|
127
|
+
new = {e: (b if colour[e] == a else a) for e in path}
|
|
128
|
+
for e in path:
|
|
129
|
+
take(e)
|
|
130
|
+
for e in path:
|
|
131
|
+
put(e, new[e])
|
|
132
|
+
|
|
133
|
+
for e, (u, v) in enumerate(edges):
|
|
134
|
+
free_u = [c for c in range(k) if c not in slot(0, u)]
|
|
135
|
+
free_v = set(c for c in range(k) if c not in slot(1, v))
|
|
136
|
+
c = next((c for c in free_u if c in free_v), None)
|
|
137
|
+
if c is None:
|
|
138
|
+
a, b = free_u[0], min(free_v)
|
|
139
|
+
swap(walk(1, v, a, b), a, b)
|
|
140
|
+
c = a
|
|
141
|
+
put(e, c)
|
|
142
|
+
|
|
143
|
+
while True:
|
|
144
|
+
count = [0] * k
|
|
145
|
+
for c in colour:
|
|
146
|
+
count[c] += 1
|
|
147
|
+
hi = max(range(k), key=lambda c: (count[c], -c))
|
|
148
|
+
lo = min(range(k), key=lambda c: (count[c], c))
|
|
149
|
+
if count[hi] - count[lo] <= 1:
|
|
150
|
+
return colour
|
|
151
|
+
swapped = False
|
|
152
|
+
for side in (0, 1):
|
|
153
|
+
for node in sorted(at[side]):
|
|
154
|
+
s = at[side][node]
|
|
155
|
+
if hi in s and lo not in s:
|
|
156
|
+
path = walk(side, node, hi, lo)
|
|
157
|
+
if sum(colour[e] == hi for e in path) > sum(colour[e] == lo for e in path):
|
|
158
|
+
swap(path, hi, lo)
|
|
159
|
+
swapped = True
|
|
160
|
+
break
|
|
161
|
+
if swapped:
|
|
162
|
+
break
|
|
163
|
+
if not swapped:
|
|
164
|
+
raise ArenaError("could not balance the strategy cards")
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def deal(n, seed, data):
|
|
168
|
+
"""Deal n distinct (reasoning, workflow, strategy) index triples, one per agent.
|
|
169
|
+
|
|
170
|
+
Deterministic for a given seed and strategies.json. Guarantees:
|
|
171
|
+
- no card is dealt twice
|
|
172
|
+
- every reasoning mode, workflow and strategy is dealt as evenly as possible
|
|
173
|
+
(any two counts differ by at most 1), so 100 agents use all of them
|
|
174
|
+
- while every reasoning mode and every workflow has no more agents than there
|
|
175
|
+
are strategies (up to 144 agents with the shipped cards): no two agents share
|
|
176
|
+
a reasoning mode and a workflow, a reasoning mode and a strategy, or a
|
|
177
|
+
workflow and a strategy. Any two agents differ in at least two of the three.
|
|
178
|
+
|
|
179
|
+
How: agent slot i gets reasoning i mod R and workflow (i + i // lcm(R, W)) mod W,
|
|
180
|
+
which never repeats a workflow inside a reasoning mode and stays balanced. The
|
|
181
|
+
strategies are then a balanced proper edge colouring of that reasoning x
|
|
182
|
+
workflow grid. The seed shuffles every list's labels and who gets which card.
|
|
183
|
+
"""
|
|
184
|
+
R, W, S = len(data["reasoning"]), len(data["workflows"]), len(data["strategies"])
|
|
185
|
+
total = R * W * S
|
|
186
|
+
if n < 1 or n > total:
|
|
187
|
+
raise ArenaError("can deal between 1 and %d cards, not %d" % (total, n))
|
|
188
|
+
rng = random.Random("arena-deal:%s" % seed)
|
|
189
|
+
pr, pw, ps = list(range(R)), list(range(W)), list(range(S))
|
|
190
|
+
rng.shuffle(pr)
|
|
191
|
+
rng.shuffle(pw)
|
|
192
|
+
rng.shuffle(ps)
|
|
193
|
+
lcm = R * W // math.gcd(R, W)
|
|
194
|
+
cells = [(i % R, (i + i // lcm) % W) for i in range(n)]
|
|
195
|
+
deg_r = max(sum(1 for r, _ in cells if r == x) for x in range(R))
|
|
196
|
+
deg_w = max(sum(1 for _, w in cells if w == x) for x in range(W))
|
|
197
|
+
if deg_r <= S and deg_w <= S:
|
|
198
|
+
order = list(range(n))
|
|
199
|
+
rng.shuffle(order)
|
|
200
|
+
colours = _edge_colour([cells[i] for i in order], S)
|
|
201
|
+
strategy_of = {order[j]: colours[j] for j in range(n)}
|
|
202
|
+
triples = [(r, w, strategy_of[i]) for i, (r, w) in enumerate(cells)]
|
|
203
|
+
else:
|
|
204
|
+
# Past that size a repeat pair is unavoidable. Stay balanced and never repeat a card.
|
|
205
|
+
count, pairs_rs, pairs_ws, used, triples = [0] * S, {}, {}, set(), []
|
|
206
|
+
for r, w in cells:
|
|
207
|
+
low = min(count)
|
|
208
|
+
s = min((c for c in range(S) if (r, w, c) not in used),
|
|
209
|
+
key=lambda c: (count[c] - low, pairs_rs.get((r, c), 0), pairs_ws.get((w, c), 0), c))
|
|
210
|
+
used.add((r, w, s))
|
|
211
|
+
count[s] += 1
|
|
212
|
+
pairs_rs[(r, s)] = pairs_rs.get((r, s), 0) + 1
|
|
213
|
+
pairs_ws[(w, s)] = pairs_ws.get((w, s), 0) + 1
|
|
214
|
+
triples.append((r, w, s))
|
|
215
|
+
dealt = [(pr[r], pw[w], ps[s]) for r, w, s in triples]
|
|
216
|
+
rng.shuffle(dealt)
|
|
217
|
+
return dealt
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def agent_ids(n):
|
|
221
|
+
width = max(3, len(str(n)))
|
|
222
|
+
return ["a%0*d" % (width, i) for i in range(1, n + 1)]
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def bracket_sizes(n):
|
|
226
|
+
"""Alive count at the start of each round, ending at 1. 100 -> 50 -> 25 -> 13 -> 7 -> 4 -> 2 -> 1."""
|
|
227
|
+
sizes = [n]
|
|
228
|
+
while sizes[-1] > 1:
|
|
229
|
+
sizes.append((sizes[-1] + 1) // 2)
|
|
230
|
+
return sizes
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
# ---------------------------------------------------------------- paths
|
|
234
|
+
|
|
235
|
+
def spawn_out(d, aid):
|
|
236
|
+
return os.path.join(d, "r0", aid + ".md")
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def attack_out(d, rnd, mid, attacker):
|
|
240
|
+
return os.path.join(d, "r%d" % rnd, "%s.%s.attack.md" % (mid, attacker))
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def defense_out(d, rnd, mid, aid):
|
|
244
|
+
return os.path.join(d, "r%d" % rnd, "%s.%s.defense.md" % (mid, aid))
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def revised_out(d, rnd, mid, aid):
|
|
248
|
+
return os.path.join(d, "r%d" % rnd, "%s.%s.solution.md" % (mid, aid))
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def verdict_out(d, rnd, mid):
|
|
252
|
+
return os.path.join(d, "r%d" % rnd, "%s.verdict.json" % mid)
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def final_out(d):
|
|
256
|
+
return os.path.join(d, "final.verdict.json")
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def prompt_path(d, rnd, job_id):
|
|
260
|
+
sub = "final" if rnd is None else "r%d" % rnd
|
|
261
|
+
return os.path.join(d, "prompts", sub, job_id + ".md")
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _has_output(path):
|
|
265
|
+
return os.path.isfile(path) and os.path.getsize(path) > 0
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
# ---------------------------------------------------------------- the bracket
|
|
269
|
+
|
|
270
|
+
def new_state(n, seed, data, run_dir, wave=DEFAULT_WAVE, has_baseline=False):
|
|
271
|
+
run_dir = os.path.abspath(run_dir)
|
|
272
|
+
ids = agent_ids(n)
|
|
273
|
+
agents = {}
|
|
274
|
+
for aid, (r, w, s) in zip(ids, deal(n, seed, data)):
|
|
275
|
+
agents[aid] = {
|
|
276
|
+
"card": {
|
|
277
|
+
"reasoning": dict(data["reasoning"][r]),
|
|
278
|
+
"workflow": dict(data["workflows"][w]),
|
|
279
|
+
"strategy": dict(data["strategies"][s]),
|
|
280
|
+
},
|
|
281
|
+
"solution": spawn_out(run_dir, aid),
|
|
282
|
+
"alive": True,
|
|
283
|
+
"eliminated_in": None,
|
|
284
|
+
"eliminated_by": None,
|
|
285
|
+
"byes": 0,
|
|
286
|
+
}
|
|
287
|
+
state = {
|
|
288
|
+
"version": 1,
|
|
289
|
+
"created": time.strftime("%Y-%m-%d %H:%M:%S"),
|
|
290
|
+
"seed": seed,
|
|
291
|
+
"agents_n": n,
|
|
292
|
+
"cards_available": combo_count(data),
|
|
293
|
+
"wave": wave,
|
|
294
|
+
"dir": run_dir,
|
|
295
|
+
"has_baseline": bool(has_baseline),
|
|
296
|
+
"agents": agents,
|
|
297
|
+
"rounds": [],
|
|
298
|
+
"champion": None,
|
|
299
|
+
"final": None,
|
|
300
|
+
}
|
|
301
|
+
open_round(state, ids)
|
|
302
|
+
return state
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def pair_round(state, rnd, alive):
|
|
306
|
+
"""Seeded pairing. An odd pool gives one bye, to an agent with the fewest byes so far.
|
|
307
|
+
Where it can, it pairs agents with different reasoning modes, so every attack comes
|
|
308
|
+
from a genuinely different angle. The first-listed side is random, which is the
|
|
309
|
+
order the judge reads them in."""
|
|
310
|
+
agents = state["agents"]
|
|
311
|
+
rng = random.Random("arena-pair:%s:%d" % (state["seed"], rnd))
|
|
312
|
+
pool = sorted(alive)
|
|
313
|
+
rng.shuffle(pool)
|
|
314
|
+
bye = None
|
|
315
|
+
if len(pool) % 2:
|
|
316
|
+
fewest = min(agents[a]["byes"] for a in pool)
|
|
317
|
+
bye = next(a for a in pool if agents[a]["byes"] == fewest)
|
|
318
|
+
pool.remove(bye)
|
|
319
|
+
width = max(2, len(str(len(pool) // 2)))
|
|
320
|
+
matches = []
|
|
321
|
+
while pool:
|
|
322
|
+
a = pool.pop(0)
|
|
323
|
+
mode = agents[a]["card"]["reasoning"]["id"]
|
|
324
|
+
j = next((k for k, b in enumerate(pool) if agents[b]["card"]["reasoning"]["id"] != mode), 0)
|
|
325
|
+
b = pool.pop(j)
|
|
326
|
+
if rng.random() < 0.5:
|
|
327
|
+
a, b = b, a
|
|
328
|
+
matches.append({
|
|
329
|
+
"id": "r%d-m%0*d" % (rnd, width, len(matches) + 1),
|
|
330
|
+
"a": a, "b": b,
|
|
331
|
+
"winner": None, "loser": None, "reason": None,
|
|
332
|
+
"survived": [], "scores": None, "source": None,
|
|
333
|
+
})
|
|
334
|
+
return matches, bye
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def open_round(state, alive):
|
|
338
|
+
"""Open the next round, or crown the champion when one agent is left."""
|
|
339
|
+
alive = sorted(alive)
|
|
340
|
+
if len(alive) == 1:
|
|
341
|
+
state["champion"] = alive[0]
|
|
342
|
+
if state.get("has_baseline") and not state.get("final"):
|
|
343
|
+
rng = random.Random("arena-final:%s" % state["seed"])
|
|
344
|
+
x = "champion" if rng.random() < 0.5 else "baseline"
|
|
345
|
+
state["final"] = {"X": x, "Y": "baseline" if x == "champion" else "champion", "result": None}
|
|
346
|
+
return None
|
|
347
|
+
rnd = len(state["rounds"]) + 1
|
|
348
|
+
matches, bye = pair_round(state, rnd, alive)
|
|
349
|
+
rd = {"n": rnd, "alive": alive, "matches": matches, "bye": bye, "closed": False}
|
|
350
|
+
state["rounds"].append(rd)
|
|
351
|
+
return rd
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def current_round(state):
|
|
355
|
+
if state["champion"] or not state["rounds"]:
|
|
356
|
+
return None
|
|
357
|
+
rd = state["rounds"][-1]
|
|
358
|
+
return None if rd["closed"] else rd
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def norm_agent(state, raw):
|
|
362
|
+
raw = str(raw).strip().lower()
|
|
363
|
+
if raw in state["agents"]:
|
|
364
|
+
return raw
|
|
365
|
+
digits = raw.lstrip("a")
|
|
366
|
+
if digits.isdigit():
|
|
367
|
+
for aid in state["agents"]:
|
|
368
|
+
if int(aid[1:]) == int(digits):
|
|
369
|
+
return aid
|
|
370
|
+
raise ArenaError("no agent called '%s'" % raw)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def find_match(state, mid):
|
|
374
|
+
rd = current_round(state)
|
|
375
|
+
if rd is None:
|
|
376
|
+
raise ArenaError("no round is open" + (" (the arena has a champion)" if state["champion"] else ""))
|
|
377
|
+
raw = str(mid).strip().lower()
|
|
378
|
+
for m in rd["matches"]:
|
|
379
|
+
if m["id"] == raw:
|
|
380
|
+
return rd, m
|
|
381
|
+
tail = raw.split("-m")[-1].lstrip("m")
|
|
382
|
+
if tail.isdigit():
|
|
383
|
+
for m in rd["matches"]:
|
|
384
|
+
if int(m["id"].split("-m")[-1]) == int(tail):
|
|
385
|
+
return rd, m
|
|
386
|
+
raise ArenaError("no match '%s' in round %d (run pairings)" % (mid, rd["n"]))
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def record(state, mid, winner, reason=None, survived=None, scores=None, source="manual"):
|
|
390
|
+
rd, m = find_match(state, mid)
|
|
391
|
+
winner = norm_agent(state, winner)
|
|
392
|
+
if winner not in (m["a"], m["b"]):
|
|
393
|
+
raise ArenaError("%s is not in %s (%s vs %s)" % (winner, m["id"], m["a"], m["b"]))
|
|
394
|
+
previous = m["winner"]
|
|
395
|
+
m["winner"] = winner
|
|
396
|
+
m["loser"] = m["b"] if winner == m["a"] else m["a"]
|
|
397
|
+
m["reason"] = reason or None
|
|
398
|
+
m["survived"] = list(survived or [])
|
|
399
|
+
m["scores"] = scores
|
|
400
|
+
m["source"] = source
|
|
401
|
+
return m, previous
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def advance(state):
|
|
405
|
+
"""Close the current round: eliminate every loser, carry the winners' revised
|
|
406
|
+
solutions forward, give the bye its free pass, and open the next round."""
|
|
407
|
+
rd = current_round(state)
|
|
408
|
+
if rd is None:
|
|
409
|
+
raise ArenaError("no round is open" + (" (the arena has a champion)" if state["champion"] else ""))
|
|
410
|
+
unrecorded = [m["id"] for m in rd["matches"] if not m["winner"]]
|
|
411
|
+
if unrecorded:
|
|
412
|
+
raise ArenaError("round %d has %d unrecorded match(es): %s"
|
|
413
|
+
% (rd["n"], len(unrecorded), ", ".join(unrecorded)))
|
|
414
|
+
survivors = []
|
|
415
|
+
for m in rd["matches"]:
|
|
416
|
+
loser = state["agents"][m["loser"]]
|
|
417
|
+
loser["alive"] = False
|
|
418
|
+
loser["eliminated_in"] = rd["n"]
|
|
419
|
+
loser["eliminated_by"] = m["winner"]
|
|
420
|
+
for aid in (m["a"], m["b"]):
|
|
421
|
+
rev = revised_out(state["dir"], rd["n"], m["id"], aid)
|
|
422
|
+
if _has_output(rev):
|
|
423
|
+
state["agents"][aid]["solution"] = rev
|
|
424
|
+
survivors.append(m["winner"])
|
|
425
|
+
if rd["bye"]:
|
|
426
|
+
state["agents"][rd["bye"]]["byes"] += 1
|
|
427
|
+
survivors.append(rd["bye"])
|
|
428
|
+
rd["closed"] = True
|
|
429
|
+
open_round(state, survivors)
|
|
430
|
+
return survivors
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def rounds_played(state):
|
|
434
|
+
return sum(1 for rd in state["rounds"] if rd["closed"])
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def alive_ids(state):
|
|
438
|
+
return sorted(a for a, v in state["agents"].items() if v["alive"])
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
# ---------------------------------------------------------------- verdicts
|
|
442
|
+
|
|
443
|
+
def weighted_total(scores):
|
|
444
|
+
"""0 to 100 from five 0 to 10 scores, weighted as in rubric.md. None if any is missing."""
|
|
445
|
+
if not isinstance(scores, dict):
|
|
446
|
+
return None
|
|
447
|
+
total = 0.0
|
|
448
|
+
for key, weight in WEIGHTS:
|
|
449
|
+
v = scores.get(key)
|
|
450
|
+
if isinstance(v, bool):
|
|
451
|
+
return None
|
|
452
|
+
try:
|
|
453
|
+
v = float(v)
|
|
454
|
+
except (TypeError, ValueError):
|
|
455
|
+
return None
|
|
456
|
+
total += max(0.0, min(10.0, v)) * weight
|
|
457
|
+
return round(total / 10.0, 2)
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _truthy(v):
|
|
461
|
+
if isinstance(v, str):
|
|
462
|
+
return v.strip().lower() in ("true", "yes", "1")
|
|
463
|
+
return bool(v)
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def extract_json(text):
|
|
467
|
+
text = text.strip()
|
|
468
|
+
try:
|
|
469
|
+
return json.loads(text)
|
|
470
|
+
except ValueError:
|
|
471
|
+
pass
|
|
472
|
+
i, j = text.find("{"), text.rfind("}")
|
|
473
|
+
if i != -1 and j > i:
|
|
474
|
+
try:
|
|
475
|
+
return json.loads(text[i:j + 1])
|
|
476
|
+
except ValueError:
|
|
477
|
+
return None
|
|
478
|
+
return None
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def decide(verdict, a, b):
|
|
482
|
+
"""Apply the rubric's arithmetic to a judge's verdict.
|
|
483
|
+
|
|
484
|
+
The winner is the higher weighted total. A fatal solution cannot beat a
|
|
485
|
+
non-fatal one. On an exact tie: fewer standing attacks, then higher
|
|
486
|
+
correctness, then the judge's own pick. If the judge's pick disagrees with its
|
|
487
|
+
own scores, the scores win and the note says so.
|
|
488
|
+
Returns (winner, {a: total, b: total}, note).
|
|
489
|
+
"""
|
|
490
|
+
if not isinstance(verdict, dict):
|
|
491
|
+
raise ArenaError("verdict is not a JSON object")
|
|
492
|
+
raw_scores = verdict.get("scores") or {}
|
|
493
|
+
scores = {str(k).strip().lower(): v for k, v in raw_scores.items()} if isinstance(raw_scores, dict) else {}
|
|
494
|
+
sa, sb = scores.get(a), scores.get(b)
|
|
495
|
+
pick_raw = str(verdict.get("winner") or "").strip().lower()
|
|
496
|
+
in_a = re.search(r"\b%s\b" % re.escape(a), pick_raw) is not None
|
|
497
|
+
in_b = re.search(r"\b%s\b" % re.escape(b), pick_raw) is not None
|
|
498
|
+
pick = a if (in_a and not in_b) else b if (in_b and not in_a) else None
|
|
499
|
+
ta, tb = weighted_total(sa), weighted_total(sb)
|
|
500
|
+
totals = {a: ta, b: tb}
|
|
501
|
+
if ta is None or tb is None:
|
|
502
|
+
if pick:
|
|
503
|
+
return pick, totals, "scores incomplete, used the judge's pick"
|
|
504
|
+
raise ArenaError("verdict has neither complete scores nor a clear winner")
|
|
505
|
+
fa, fb = _truthy(sa.get("fatal")), _truthy(sb.get("fatal"))
|
|
506
|
+
if fa != fb:
|
|
507
|
+
winner = b if fa else a
|
|
508
|
+
elif ta != tb:
|
|
509
|
+
winner = a if ta > tb else b
|
|
510
|
+
else:
|
|
511
|
+
standing = verdict.get("standing") if isinstance(verdict.get("standing"), dict) else {}
|
|
512
|
+
standing = {str(k).strip().lower(): v for k, v in standing.items()}
|
|
513
|
+
na = len(standing.get(a) or []) if isinstance(standing.get(a), list) else None
|
|
514
|
+
nb = len(standing.get(b) or []) if isinstance(standing.get(b), list) else None
|
|
515
|
+
ca, cb = float(sa.get("correctness", 0)), float(sb.get("correctness", 0))
|
|
516
|
+
if na is not None and nb is not None and na != nb:
|
|
517
|
+
winner = a if na < nb else b
|
|
518
|
+
elif ca != cb:
|
|
519
|
+
winner = a if ca > cb else b
|
|
520
|
+
elif pick:
|
|
521
|
+
winner = pick
|
|
522
|
+
else:
|
|
523
|
+
raise ArenaError("exact tie and the judge did not pick a winner")
|
|
524
|
+
note = ""
|
|
525
|
+
if pick and pick != winner:
|
|
526
|
+
note = "judge picked %s but its own scores favour %s, so the scores win" % (pick, winner)
|
|
527
|
+
return winner, totals, note
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def collect(state):
|
|
531
|
+
"""Record every verdict the judges have written for the open round (or the final
|
|
532
|
+
check). An unreadable verdict is set aside as <name>.unreadable so the judge job
|
|
533
|
+
shows up as missing again and gets re-run."""
|
|
534
|
+
d = state["dir"]
|
|
535
|
+
results = []
|
|
536
|
+
if state["champion"]:
|
|
537
|
+
fin = state.get("final")
|
|
538
|
+
if not fin or fin.get("result"):
|
|
539
|
+
return results
|
|
540
|
+
path = final_out(d)
|
|
541
|
+
if not _has_output(path):
|
|
542
|
+
return [("final", "missing", "")]
|
|
543
|
+
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
544
|
+
v = extract_json(fh.read())
|
|
545
|
+
try:
|
|
546
|
+
if isinstance(v, dict) and isinstance(v.get("scores"), dict):
|
|
547
|
+
v = dict(v, scores={str(k).strip().lower(): s for k, s in v["scores"].items()})
|
|
548
|
+
w, totals, note = decide(v, "x", "y")
|
|
549
|
+
except ArenaError as e:
|
|
550
|
+
os.replace(path, path + ".unreadable")
|
|
551
|
+
return [("final", "unreadable", str(e))]
|
|
552
|
+
better = fin[w.upper()]
|
|
553
|
+
fin["result"] = {
|
|
554
|
+
"better": better,
|
|
555
|
+
"champion_total": totals["x" if fin["X"] == "champion" else "y"],
|
|
556
|
+
"baseline_total": totals["x" if fin["X"] == "baseline" else "y"],
|
|
557
|
+
"reason": str(v.get("reason") or "")[:400],
|
|
558
|
+
"fixed": [str(x) for x in (v.get("fixed") or [])][:10] if isinstance(v.get("fixed"), list) else [],
|
|
559
|
+
}
|
|
560
|
+
return [("final", "recorded", ("%s is better. %s" % (better, note)).strip())]
|
|
561
|
+
rd = current_round(state)
|
|
562
|
+
if rd is None:
|
|
563
|
+
return results
|
|
564
|
+
for m in rd["matches"]:
|
|
565
|
+
if m["winner"]:
|
|
566
|
+
continue
|
|
567
|
+
path = verdict_out(d, rd["n"], m["id"])
|
|
568
|
+
if not _has_output(path):
|
|
569
|
+
results.append((m["id"], "missing", ""))
|
|
570
|
+
continue
|
|
571
|
+
with open(path, encoding="utf-8", errors="replace") as fh:
|
|
572
|
+
v = extract_json(fh.read())
|
|
573
|
+
try:
|
|
574
|
+
winner, totals, note = decide(v, m["a"], m["b"])
|
|
575
|
+
except ArenaError as e:
|
|
576
|
+
os.replace(path, path + ".unreadable")
|
|
577
|
+
results.append((m["id"], "unreadable", str(e)))
|
|
578
|
+
continue
|
|
579
|
+
survived = v.get("survived") if isinstance(v.get("survived"), list) else []
|
|
580
|
+
record(state, m["id"], winner,
|
|
581
|
+
reason=str(v.get("reason") or "")[:400],
|
|
582
|
+
survived=[str(x) for x in survived][:10],
|
|
583
|
+
scores=totals, source="judge")
|
|
584
|
+
loser = m["loser"]
|
|
585
|
+
line = "%s beat %s, %s to %s" % (winner, loser, _fmt(totals[winner]), _fmt(totals[loser]))
|
|
586
|
+
results.append((m["id"], "recorded", line + (". " + note if note else "")))
|
|
587
|
+
return results
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def _fmt(x):
|
|
591
|
+
return "?" if x is None else ("%g" % x)
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
# ---------------------------------------------------------------- jobs and prompts
|
|
595
|
+
|
|
596
|
+
def phase_jobs(state, phase):
|
|
597
|
+
"""Every sub-agent job for a phase, with its prompt file and the outputs it must write.
|
|
598
|
+
Matches that already have a recorded winner are skipped."""
|
|
599
|
+
if phase not in PHASES:
|
|
600
|
+
raise ArenaError("unknown phase '%s' (one of: %s)" % (phase, ", ".join(PHASES)))
|
|
601
|
+
d = state["dir"]
|
|
602
|
+
jobs = []
|
|
603
|
+
if phase == "spawn":
|
|
604
|
+
for aid in sorted(state["agents"]):
|
|
605
|
+
jobs.append({"id": aid + ".spawn", "kind": "competitor", "round": 0, "agent": aid,
|
|
606
|
+
"outputs": [spawn_out(d, aid)]})
|
|
607
|
+
elif phase == "final":
|
|
608
|
+
fin = state.get("final")
|
|
609
|
+
if state["champion"] and fin and not fin.get("result"):
|
|
610
|
+
jobs.append({"id": "final.judge", "kind": "final", "round": None, "outputs": [final_out(d)]})
|
|
611
|
+
else:
|
|
612
|
+
rd = current_round(state)
|
|
613
|
+
if rd is None:
|
|
614
|
+
return []
|
|
615
|
+
n = rd["n"]
|
|
616
|
+
for m in rd["matches"]:
|
|
617
|
+
if m["winner"]:
|
|
618
|
+
continue
|
|
619
|
+
a, b = m["a"], m["b"]
|
|
620
|
+
if phase == "attack":
|
|
621
|
+
for me, opp in ((a, b), (b, a)):
|
|
622
|
+
jobs.append({"id": "%s.%s.attack" % (m["id"], me), "kind": "attacker", "round": n,
|
|
623
|
+
"agent": me, "opponent": opp, "match": m["id"],
|
|
624
|
+
"outputs": [attack_out(d, n, m["id"], me)]})
|
|
625
|
+
elif phase == "defend":
|
|
626
|
+
for me, opp in ((a, b), (b, a)):
|
|
627
|
+
jobs.append({"id": "%s.%s.defend" % (m["id"], me), "kind": "defender", "round": n,
|
|
628
|
+
"agent": me, "opponent": opp, "match": m["id"],
|
|
629
|
+
"outputs": [defense_out(d, n, m["id"], me), revised_out(d, n, m["id"], me)]})
|
|
630
|
+
else:
|
|
631
|
+
jobs.append({"id": "%s.judge" % m["id"], "kind": "judge", "round": n,
|
|
632
|
+
"match": m["id"], "a": a, "b": b,
|
|
633
|
+
"outputs": [verdict_out(d, n, m["id"])]})
|
|
634
|
+
for j in jobs:
|
|
635
|
+
j["prompt"] = prompt_path(d, j["round"], j["id"])
|
|
636
|
+
return jobs
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
def missing_jobs(jobs):
|
|
640
|
+
return [j for j in jobs if not all(_has_output(p) for p in j["outputs"])]
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
_TEMPLATE_RE = re.compile(r"<!-- template:([a-z]+) -->\s*```text\n(.*?)\n```\s*<!-- /template:\1 -->", re.S)
|
|
644
|
+
_PLACEHOLDER_RE = re.compile(r"\{\{(\w+)\}\}")
|
|
645
|
+
|
|
646
|
+
|
|
647
|
+
def load_templates(path=SKILL_PATH):
|
|
648
|
+
with open(path, encoding="utf-8") as fh:
|
|
649
|
+
text = fh.read()
|
|
650
|
+
found = {m.group(1): m.group(2) for m in _TEMPLATE_RE.finditer(text)}
|
|
651
|
+
missing = [t for t in TEMPLATES if t not in found]
|
|
652
|
+
if missing:
|
|
653
|
+
raise ArenaError("SKILL.md is missing prompt templates: %s" % ", ".join(missing))
|
|
654
|
+
return found
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def fill(template, values):
|
|
658
|
+
"""One pass, so a task that itself contains {{braces}} is left exactly as written."""
|
|
659
|
+
wanted = set(_PLACEHOLDER_RE.findall(template))
|
|
660
|
+
missing = sorted(wanted - set(values))
|
|
661
|
+
if missing:
|
|
662
|
+
raise ArenaError("no value for placeholder(s): %s" % ", ".join(missing))
|
|
663
|
+
return _PLACEHOLDER_RE.sub(lambda m: str(values[m.group(1)]), template)
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def read_task(state):
|
|
667
|
+
"""The task exactly as init stored it (init adds one trailing newline to the file)."""
|
|
668
|
+
with open(os.path.join(state["dir"], "task.md"), encoding="utf-8") as fh:
|
|
669
|
+
return fh.read().rstrip("\n")
|
|
670
|
+
|
|
671
|
+
|
|
672
|
+
def _card_values(card):
|
|
673
|
+
return {
|
|
674
|
+
"reasoning_name": card["reasoning"]["name"], "reasoning_how": card["reasoning"]["how"],
|
|
675
|
+
"workflow_name": card["workflow"]["name"], "workflow_how": card["workflow"]["how"],
|
|
676
|
+
"strategy_name": card["strategy"]["name"], "strategy_how": card["strategy"]["how"],
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def render_job(state, job, templates, task):
|
|
681
|
+
d = state["dir"]
|
|
682
|
+
agents = state["agents"]
|
|
683
|
+
v = {"task": task, "arena_dir": d, "rubric": os.path.join(d, "rubric.md"), "n": state["agents_n"]}
|
|
684
|
+
kind = job["kind"]
|
|
685
|
+
if kind == "competitor":
|
|
686
|
+
aid = job["agent"]
|
|
687
|
+
v.update(_card_values(agents[aid]["card"]))
|
|
688
|
+
if state.get("has_baseline"):
|
|
689
|
+
v["baseline_note"] = (
|
|
690
|
+
"The user already got an answer to this task and was NOT satisfied with it. It is at "
|
|
691
|
+
"%s. Read it first and work out exactly why it fell short. Then beat it. Do not just "
|
|
692
|
+
"polish it: the winner of this arena is compared with that answer at the end."
|
|
693
|
+
% os.path.join(d, "baseline.md"))
|
|
694
|
+
else:
|
|
695
|
+
v["baseline_note"] = "There is no earlier answer to beat. Start from the task."
|
|
696
|
+
v.update(agent=aid, out=job["outputs"][0])
|
|
697
|
+
elif kind == "attacker":
|
|
698
|
+
me, opp = job["agent"], job["opponent"]
|
|
699
|
+
v.update(_card_values(agents[me]["card"]))
|
|
700
|
+
v.update(agent=me, target=opp, match=job["match"], round=job["round"],
|
|
701
|
+
own_solution=agents[me]["solution"], target_solution=agents[opp]["solution"],
|
|
702
|
+
out=job["outputs"][0])
|
|
703
|
+
elif kind == "defender":
|
|
704
|
+
me, opp = job["agent"], job["opponent"]
|
|
705
|
+
v.update(_card_values(agents[me]["card"]))
|
|
706
|
+
v.update(agent=me, attacker=opp, match=job["match"], round=job["round"],
|
|
707
|
+
own_solution=agents[me]["solution"],
|
|
708
|
+
attacks=attack_out(d, job["round"], job["match"], opp),
|
|
709
|
+
defense_out=job["outputs"][0], solution_out=job["outputs"][1])
|
|
710
|
+
elif kind == "judge":
|
|
711
|
+
n, mid, a, b = job["round"], job["match"], job["a"], job["b"]
|
|
712
|
+
v.update(match=mid, round=n, first=a, second=b,
|
|
713
|
+
first_solution=revised_out(d, n, mid, a), second_solution=revised_out(d, n, mid, b),
|
|
714
|
+
first_attacks=attack_out(d, n, mid, b), second_attacks=attack_out(d, n, mid, a),
|
|
715
|
+
first_defense=defense_out(d, n, mid, a), second_defense=defense_out(d, n, mid, b),
|
|
716
|
+
out=job["outputs"][0])
|
|
717
|
+
elif kind == "final":
|
|
718
|
+
v.update(rounds=rounds_played(state),
|
|
719
|
+
x_solution=os.path.join(d, "final", "X.md"),
|
|
720
|
+
y_solution=os.path.join(d, "final", "Y.md"),
|
|
721
|
+
out=job["outputs"][0])
|
|
722
|
+
return fill(templates[kind], v)
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
def write_prompts(state, phase):
|
|
726
|
+
jobs = phase_jobs(state, phase)
|
|
727
|
+
if not jobs:
|
|
728
|
+
return jobs
|
|
729
|
+
templates = load_templates()
|
|
730
|
+
task = read_task(state)
|
|
731
|
+
for j in jobs:
|
|
732
|
+
for p in j["outputs"] + [j["prompt"]]:
|
|
733
|
+
os.makedirs(os.path.dirname(p), exist_ok=True)
|
|
734
|
+
body = render_job(state, j, templates, task)
|
|
735
|
+
with open(j["prompt"], "w", encoding="utf-8") as fh:
|
|
736
|
+
fh.write(body + "\n")
|
|
737
|
+
if phase == "spawn":
|
|
738
|
+
for aid in state["agents"]:
|
|
739
|
+
os.makedirs(os.path.join(state["dir"], "scratch", aid), exist_ok=True)
|
|
740
|
+
if phase == "final":
|
|
741
|
+
# Blind copies: the final judge sees X and Y, never which one is the champion.
|
|
742
|
+
fin = state["final"]
|
|
743
|
+
sources = {"champion": state["agents"][state["champion"]]["solution"],
|
|
744
|
+
"baseline": os.path.join(state["dir"], "baseline.md")}
|
|
745
|
+
os.makedirs(os.path.join(state["dir"], "final"), exist_ok=True)
|
|
746
|
+
for label in ("X", "Y"):
|
|
747
|
+
with open(sources[fin[label]], encoding="utf-8", errors="replace") as src, \
|
|
748
|
+
open(os.path.join(state["dir"], "final", label + ".md"), "w", encoding="utf-8") as dst:
|
|
749
|
+
dst.write(src.read())
|
|
750
|
+
return jobs
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
def next_action(state):
|
|
754
|
+
"""(phase, jobs_left, jobs_total). phase is one of spawn, attack, defend, judge,
|
|
755
|
+
final, collect, advance, done."""
|
|
756
|
+
spawn = phase_jobs(state, "spawn")
|
|
757
|
+
left = missing_jobs(spawn)
|
|
758
|
+
if left:
|
|
759
|
+
return "spawn", left, spawn
|
|
760
|
+
if state["champion"]:
|
|
761
|
+
fin = state.get("final")
|
|
762
|
+
if fin and not fin.get("result"):
|
|
763
|
+
jobs = phase_jobs(state, "final")
|
|
764
|
+
left = missing_jobs(jobs)
|
|
765
|
+
return ("final", left, jobs) if left else ("collect", [], jobs)
|
|
766
|
+
return "done", [], []
|
|
767
|
+
for phase in ("attack", "defend", "judge"):
|
|
768
|
+
jobs = phase_jobs(state, phase)
|
|
769
|
+
left = missing_jobs(jobs)
|
|
770
|
+
if left:
|
|
771
|
+
return phase, left, jobs
|
|
772
|
+
rd = current_round(state)
|
|
773
|
+
if any(not m["winner"] for m in rd["matches"]):
|
|
774
|
+
return "collect", [], []
|
|
775
|
+
return "advance", [], []
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
# ---------------------------------------------------------------- plan
|
|
779
|
+
|
|
780
|
+
def plan_rows(n, wave=DEFAULT_WAVE):
|
|
781
|
+
rows = []
|
|
782
|
+
for rnd, alive in enumerate(bracket_sizes(n)[:-1], start=1):
|
|
783
|
+
m = alive // 2
|
|
784
|
+
waves = 2 * math.ceil(2 * m / wave) + math.ceil(m / wave)
|
|
785
|
+
rows.append({"round": rnd, "alive": alive, "matches": m, "bye": alive % 2,
|
|
786
|
+
"calls": CALLS_PER_MATCH * m, "waves": waves})
|
|
787
|
+
return rows
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
def plan_totals(n, wave=DEFAULT_WAVE):
|
|
791
|
+
rows = plan_rows(n, wave)
|
|
792
|
+
calls = n + sum(r["calls"] for r in rows)
|
|
793
|
+
waves = math.ceil(n / wave) + sum(r["waves"] for r in rows)
|
|
794
|
+
return {"agents": n, "rounds": len(rows), "calls": calls, "waves": waves, "rows": rows}
|
|
795
|
+
|
|
796
|
+
|
|
797
|
+
# ---------------------------------------------------------------- state io
|
|
798
|
+
|
|
799
|
+
def run_dir_from(args):
|
|
800
|
+
if getattr(args, "dir", None):
|
|
801
|
+
return os.path.abspath(args.dir)
|
|
802
|
+
latest = os.path.join(ROOT, LATEST)
|
|
803
|
+
if os.path.isfile(latest):
|
|
804
|
+
with open(latest, encoding="utf-8") as fh:
|
|
805
|
+
return fh.read().strip()
|
|
806
|
+
raise ArenaError("no arena here. Run init first, or pass --dir")
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def load_state(args):
|
|
810
|
+
d = run_dir_from(args)
|
|
811
|
+
path = os.path.join(d, STATE_FILE)
|
|
812
|
+
if not os.path.isfile(path):
|
|
813
|
+
raise ArenaError("no %s in %s" % (STATE_FILE, d))
|
|
814
|
+
with open(path, encoding="utf-8") as fh:
|
|
815
|
+
return json.load(fh)
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
def save_state(state):
|
|
819
|
+
path = os.path.join(state["dir"], STATE_FILE)
|
|
820
|
+
tmp = path + ".tmp"
|
|
821
|
+
with open(tmp, "w", encoding="utf-8") as fh:
|
|
822
|
+
json.dump(state, fh, indent=1)
|
|
823
|
+
fh.write("\n")
|
|
824
|
+
os.replace(tmp, path)
|
|
825
|
+
|
|
826
|
+
|
|
827
|
+
def _cmd(*parts):
|
|
828
|
+
return 'python3 "%s" %s' % (SELF, " ".join(parts))
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def _card_line(card):
|
|
832
|
+
return "%s + %s + %s" % (card["reasoning"]["name"], card["workflow"]["name"], card["strategy"]["name"])
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
# ---------------------------------------------------------------- commands
|
|
836
|
+
|
|
837
|
+
def _agents_arg(args):
|
|
838
|
+
if args.quick and args.agents is not None:
|
|
839
|
+
raise ArenaError("pass --quick or --agents, not both")
|
|
840
|
+
if args.quick:
|
|
841
|
+
return QUICK_AGENTS
|
|
842
|
+
return DEFAULT_AGENTS if args.agents is None else args.agents
|
|
843
|
+
|
|
844
|
+
|
|
845
|
+
def cmd_plan(args):
|
|
846
|
+
n = _agents_arg(args)
|
|
847
|
+
if n < 1:
|
|
848
|
+
raise ArenaError("--agents must be at least 1")
|
|
849
|
+
t = plan_totals(n, args.wave)
|
|
850
|
+
if args.json:
|
|
851
|
+
print(json.dumps(t, indent=1))
|
|
852
|
+
return 0
|
|
853
|
+
print("%d agents, %d rounds, waves of %d" % (n, t["rounds"], args.wave))
|
|
854
|
+
print("")
|
|
855
|
+
print(" round alive matches bye sub-agent calls waves")
|
|
856
|
+
print(" spawn %5d %7s %3s %15d %5d" % (n, "-", "-", n, math.ceil(n / args.wave)))
|
|
857
|
+
for r in t["rows"]:
|
|
858
|
+
print(" %5d %5d %7d %3s %15d %5d"
|
|
859
|
+
% (r["round"], r["alive"], r["matches"], "yes" if r["bye"] else "-", r["calls"], r["waves"]))
|
|
860
|
+
print(" total %15d %5d" % (t["calls"], t["waves"]))
|
|
861
|
+
print("")
|
|
862
|
+
print(" alive per round: %s" % " -> ".join(str(s) for s in bracket_sizes(n)))
|
|
863
|
+
print(" plus 1 call for the final check when there is a rejected answer to beat")
|
|
864
|
+
return 0
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
def cmd_init(args):
|
|
868
|
+
n = _agents_arg(args)
|
|
869
|
+
if args.wave < 1:
|
|
870
|
+
raise ArenaError("--wave must be at least 1")
|
|
871
|
+
data = load_strategies()
|
|
872
|
+
if n < 1 or n > combo_count(data):
|
|
873
|
+
raise ArenaError("--agents must be between 1 and %d (the number of distinct cards)" % combo_count(data))
|
|
874
|
+
if args.task_file:
|
|
875
|
+
with open(args.task_file, encoding="utf-8") as fh:
|
|
876
|
+
task = fh.read()
|
|
877
|
+
else:
|
|
878
|
+
task = args.task or ""
|
|
879
|
+
task = task.strip()
|
|
880
|
+
if not task:
|
|
881
|
+
raise ArenaError("the task is empty. Pass --task-file or --task")
|
|
882
|
+
baseline = None
|
|
883
|
+
if args.baseline_file:
|
|
884
|
+
with open(args.baseline_file, encoding="utf-8") as fh:
|
|
885
|
+
baseline = fh.read().strip()
|
|
886
|
+
if not baseline:
|
|
887
|
+
raise ArenaError("the baseline file is empty")
|
|
888
|
+
seed = args.seed if args.seed is not None else random.SystemRandom().randrange(1, 1000000)
|
|
889
|
+
default_dir = not args.dir
|
|
890
|
+
d = os.path.abspath(args.dir or os.path.join(ROOT, "run-%s-s%s" % (time.strftime("%Y%m%d-%H%M%S"), seed)))
|
|
891
|
+
if os.path.exists(os.path.join(d, STATE_FILE)):
|
|
892
|
+
raise ArenaError("%s already has an arena in it. Pick another --dir" % d)
|
|
893
|
+
os.makedirs(d, exist_ok=True)
|
|
894
|
+
with open(os.path.join(d, "task.md"), "w", encoding="utf-8") as fh:
|
|
895
|
+
fh.write(task + "\n")
|
|
896
|
+
if baseline:
|
|
897
|
+
with open(os.path.join(d, "baseline.md"), "w", encoding="utf-8") as fh:
|
|
898
|
+
fh.write(baseline + "\n")
|
|
899
|
+
# The run gets its own copy of the rubric: frozen for this run, and inside the
|
|
900
|
+
# working directory, so judges can read it without a permission prompt each.
|
|
901
|
+
with open(RUBRIC_PATH, encoding="utf-8") as src, \
|
|
902
|
+
open(os.path.join(d, "rubric.md"), "w", encoding="utf-8") as dst:
|
|
903
|
+
dst.write(src.read())
|
|
904
|
+
state = new_state(n, seed, data, d, wave=args.wave, has_baseline=bool(baseline))
|
|
905
|
+
save_state(state)
|
|
906
|
+
if default_dir:
|
|
907
|
+
os.makedirs(ROOT, exist_ok=True)
|
|
908
|
+
with open(os.path.join(ROOT, LATEST), "w", encoding="utf-8") as fh:
|
|
909
|
+
fh.write(d + "\n")
|
|
910
|
+
t = plan_totals(n, args.wave)
|
|
911
|
+
print("arena ready: %s" % d)
|
|
912
|
+
print("seed %s. %d agents, %d distinct cards dealt from %d, no repeats."
|
|
913
|
+
% (seed, n, n, combo_count(data)))
|
|
914
|
+
print("%d rounds (%s). %d sub-agent calls in %d waves of %d%s."
|
|
915
|
+
% (t["rounds"], " -> ".join(str(s) for s in bracket_sizes(n)), t["calls"] + (1 if baseline else 0),
|
|
916
|
+
t["waves"] + (1 if baseline else 0), args.wave, ", including the final check" if baseline else ""))
|
|
917
|
+
print("next: %s" % _cmd("next"))
|
|
918
|
+
return 0
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def cmd_next(args):
|
|
922
|
+
state = load_state(args)
|
|
923
|
+
phase, left, jobs = next_action(state)
|
|
924
|
+
rd = current_round(state)
|
|
925
|
+
total_rounds = len(bracket_sizes(state["agents_n"])) - 1
|
|
926
|
+
if phase == "spawn":
|
|
927
|
+
print("spawn: %d competitors, then %d round(s)" % (state["agents_n"], total_rounds))
|
|
928
|
+
elif rd:
|
|
929
|
+
print("round %d of %d, %d alive" % (rd["n"], total_rounds, len(rd["alive"])))
|
|
930
|
+
elif state["champion"]:
|
|
931
|
+
print("champion: %s after %d round(s)" % (state["champion"], rounds_played(state)))
|
|
932
|
+
if phase in PHASES:
|
|
933
|
+
print("NEXT: %s. %d of %d job(s) to run." % (phase, len(left), len(jobs)))
|
|
934
|
+
print(" 1. %s" % _cmd("prompts", phase))
|
|
935
|
+
print(" 2. launch the jobs it lists, one wave of %d at a time" % state["wave"])
|
|
936
|
+
print(" 3. %s" % _cmd("next"))
|
|
937
|
+
elif phase == "collect":
|
|
938
|
+
print("NEXT: collect. Every verdict for this step is on disk.")
|
|
939
|
+
print(" %s" % _cmd("collect"))
|
|
940
|
+
elif phase == "advance":
|
|
941
|
+
print("NEXT: advance. Every match in round %d has a winner." % rd["n"])
|
|
942
|
+
print(" %s" % _cmd("advance"))
|
|
943
|
+
else:
|
|
944
|
+
print("DONE. %s" % _cmd("winner"))
|
|
945
|
+
return 0
|
|
946
|
+
|
|
947
|
+
|
|
948
|
+
def cmd_prompts(args):
|
|
949
|
+
state = load_state(args)
|
|
950
|
+
jobs = write_prompts(state, args.phase)
|
|
951
|
+
if not jobs:
|
|
952
|
+
print("nothing to run for %s right now. %s" % (args.phase, _cmd("next")))
|
|
953
|
+
return 0
|
|
954
|
+
left = missing_jobs(jobs)
|
|
955
|
+
w = state["wave"]
|
|
956
|
+
rd = current_round(state)
|
|
957
|
+
where = "round 0" if args.phase == "spawn" else ("final" if args.phase == "final" else "round %d" % rd["n"])
|
|
958
|
+
waves = [left[i:i + w] for i in range(0, len(left), w)]
|
|
959
|
+
print("%s, %s: %d job(s), %d still to run, %d wave(s) of up to %d"
|
|
960
|
+
% (where, args.phase, len(jobs), len(left), len(waves), w))
|
|
961
|
+
for k, wave in enumerate(waves, start=1):
|
|
962
|
+
print("wave %d" % k)
|
|
963
|
+
for j in wave:
|
|
964
|
+
print(" %-26s %s" % (j["id"], j["prompt"]))
|
|
965
|
+
if left:
|
|
966
|
+
print("")
|
|
967
|
+
print("Each job is one Agent call: subagent_type general-purpose, description \"arena <job id>\",")
|
|
968
|
+
print("prompt \"Read <prompt path> and follow it exactly. It is your whole brief.\"")
|
|
969
|
+
print("A wave is one message with all of its Agent calls. Let it finish before the next wave.")
|
|
970
|
+
print("then: %s" % _cmd("next"))
|
|
971
|
+
return 0
|
|
972
|
+
|
|
973
|
+
|
|
974
|
+
def cmd_check(args):
|
|
975
|
+
state = load_state(args)
|
|
976
|
+
jobs = phase_jobs(state, args.phase)
|
|
977
|
+
left = missing_jobs(jobs)
|
|
978
|
+
print("%s: %d of %d job(s) done" % (args.phase, len(jobs) - len(left), len(jobs)))
|
|
979
|
+
for j in left:
|
|
980
|
+
gone = [p for p in j["outputs"] if not _has_output(p)]
|
|
981
|
+
print(" missing %-26s %s" % (j["id"], ", ".join(gone)))
|
|
982
|
+
return 1 if left else 0
|
|
983
|
+
|
|
984
|
+
|
|
985
|
+
def cmd_pairings(args):
|
|
986
|
+
state = load_state(args)
|
|
987
|
+
rd = current_round(state)
|
|
988
|
+
if rd is None:
|
|
989
|
+
if state["champion"]:
|
|
990
|
+
print("No matches left. %s is the last one standing." % state["champion"])
|
|
991
|
+
return 0
|
|
992
|
+
raise ArenaError("no round is open")
|
|
993
|
+
if args.json:
|
|
994
|
+
print(json.dumps(rd, indent=1))
|
|
995
|
+
return 0
|
|
996
|
+
agents = state["agents"]
|
|
997
|
+
print("round %d: %d alive, %d match(es), bye: %s"
|
|
998
|
+
% (rd["n"], len(rd["alive"]), len(rd["matches"]), rd["bye"] or "none"))
|
|
999
|
+
for m in rd["matches"]:
|
|
1000
|
+
ra = agents[m["a"]]["card"]["reasoning"]["name"]
|
|
1001
|
+
rb = agents[m["b"]]["card"]["reasoning"]["name"]
|
|
1002
|
+
res = ("winner %s" % m["winner"]) if m["winner"] else "open"
|
|
1003
|
+
print(" %s %s vs %s (%s vs %s) %s" % (m["id"], m["a"], m["b"], ra, rb, res))
|
|
1004
|
+
if rd["bye"]:
|
|
1005
|
+
print(" bye %s goes through without a match" % rd["bye"])
|
|
1006
|
+
return 0
|
|
1007
|
+
|
|
1008
|
+
|
|
1009
|
+
def cmd_record(args):
|
|
1010
|
+
state = load_state(args)
|
|
1011
|
+
m, previous = record(state, args.match_id, args.winner_id, reason=args.reason,
|
|
1012
|
+
survived=args.survived, source="manual")
|
|
1013
|
+
save_state(state)
|
|
1014
|
+
extra = " (was %s)" % previous if previous and previous != m["winner"] else ""
|
|
1015
|
+
print("%s: %s beat %s%s" % (m["id"], m["winner"], m["loser"], extra))
|
|
1016
|
+
return 0
|
|
1017
|
+
|
|
1018
|
+
|
|
1019
|
+
def cmd_collect(args):
|
|
1020
|
+
state = load_state(args)
|
|
1021
|
+
results = collect(state)
|
|
1022
|
+
save_state(state)
|
|
1023
|
+
if not results:
|
|
1024
|
+
print("nothing to collect. %s" % _cmd("next"))
|
|
1025
|
+
return 0
|
|
1026
|
+
bad = 0
|
|
1027
|
+
for mid, what, note in results:
|
|
1028
|
+
if what != "recorded":
|
|
1029
|
+
bad += 1
|
|
1030
|
+
print(" %-10s %-10s %s" % (mid, what, note))
|
|
1031
|
+
rd = current_round(state)
|
|
1032
|
+
if rd:
|
|
1033
|
+
done = sum(1 for m in rd["matches"] if m["winner"])
|
|
1034
|
+
print("round %d: %d of %d match(es) recorded" % (rd["n"], done, len(rd["matches"])))
|
|
1035
|
+
if bad:
|
|
1036
|
+
print("%d verdict(s) missing or unreadable. Re-run those judges: %s" % (bad, _cmd("next")))
|
|
1037
|
+
return 1
|
|
1038
|
+
print("then: %s" % _cmd("next"))
|
|
1039
|
+
return 0
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
def cmd_advance(args):
|
|
1043
|
+
state = load_state(args)
|
|
1044
|
+
rd = current_round(state)
|
|
1045
|
+
closing = rd["n"] if rd else None
|
|
1046
|
+
survivors = advance(state)
|
|
1047
|
+
save_state(state)
|
|
1048
|
+
print("round %s closed: %d through, %d eliminated"
|
|
1049
|
+
% (closing, len(survivors), len(rd["matches"])))
|
|
1050
|
+
if state["champion"]:
|
|
1051
|
+
print("CHAMPION: %s. %s" % (state["champion"], _cmd("next")))
|
|
1052
|
+
else:
|
|
1053
|
+
nr = state["rounds"][-1]
|
|
1054
|
+
print("round %d open: %d match(es)%s. %s"
|
|
1055
|
+
% (nr["n"], len(nr["matches"]), ", bye %s" % nr["bye"] if nr["bye"] else "", _cmd("next")))
|
|
1056
|
+
return 0
|
|
1057
|
+
|
|
1058
|
+
|
|
1059
|
+
def status_rows(state):
|
|
1060
|
+
rows = []
|
|
1061
|
+
for rd in state["rounds"]:
|
|
1062
|
+
recorded = sum(1 for m in rd["matches"] if m["winner"])
|
|
1063
|
+
rows.append({"round": rd["n"], "alive": len(rd["alive"]), "matches": len(rd["matches"]),
|
|
1064
|
+
"bye": rd["bye"], "eliminated": recorded,
|
|
1065
|
+
"state": "closed" if rd["closed"] else "open, %d of %d recorded" % (recorded, len(rd["matches"]))})
|
|
1066
|
+
return rows
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def cmd_status(args):
|
|
1070
|
+
state = load_state(args)
|
|
1071
|
+
rows = status_rows(state)
|
|
1072
|
+
alive = alive_ids(state)
|
|
1073
|
+
if args.json:
|
|
1074
|
+
print(json.dumps({"dir": state["dir"], "seed": state["seed"], "agents": state["agents_n"],
|
|
1075
|
+
"alive": len(alive), "eliminated": state["agents_n"] - len(alive),
|
|
1076
|
+
"champion": state["champion"], "rounds": rows}, indent=1))
|
|
1077
|
+
return 0
|
|
1078
|
+
print("arena %s" % state["dir"])
|
|
1079
|
+
print("seed %s, %d agents, waves of %d, %d distinct cards available"
|
|
1080
|
+
% (state["seed"], state["agents_n"], state["wave"], state["cards_available"]))
|
|
1081
|
+
print(" round alive matches bye eliminated state")
|
|
1082
|
+
for r in rows:
|
|
1083
|
+
print(" %5d %5d %7d %-6s %10d %s"
|
|
1084
|
+
% (r["round"], r["alive"], r["matches"], r["bye"] or "-", r["eliminated"], r["state"]))
|
|
1085
|
+
print("now: %d alive, %d eliminated%s"
|
|
1086
|
+
% (len(alive), state["agents_n"] - len(alive),
|
|
1087
|
+
", champion %s" % state["champion"] if state["champion"] else ""))
|
|
1088
|
+
return 0
|
|
1089
|
+
|
|
1090
|
+
|
|
1091
|
+
def winner_report(state):
|
|
1092
|
+
champ = state["champion"]
|
|
1093
|
+
if not champ:
|
|
1094
|
+
return None
|
|
1095
|
+
agent = state["agents"][champ]
|
|
1096
|
+
path = []
|
|
1097
|
+
survived = []
|
|
1098
|
+
for rd in state["rounds"]:
|
|
1099
|
+
if rd["bye"] == champ:
|
|
1100
|
+
path.append({"round": rd["n"], "bye": True})
|
|
1101
|
+
for m in rd["matches"]:
|
|
1102
|
+
if champ in (m["a"], m["b"]):
|
|
1103
|
+
opp = m["b"] if m["a"] == champ else m["a"]
|
|
1104
|
+
sc = m.get("scores") or {}
|
|
1105
|
+
path.append({"round": rd["n"], "bye": False, "beat": opp, "match": m["id"],
|
|
1106
|
+
"score": sc.get(champ), "opponent_score": sc.get(opp),
|
|
1107
|
+
"reason": m.get("reason"), "survived": m.get("survived") or []})
|
|
1108
|
+
survived.extend(m.get("survived") or [])
|
|
1109
|
+
return {"champion": champ, "card": agent["card"], "card_line": _card_line(agent["card"]),
|
|
1110
|
+
"solution": agent["solution"], "rounds": rounds_played(state), "agents": state["agents_n"],
|
|
1111
|
+
"matches_won": sum(1 for p in path if not p["bye"]), "path": path,
|
|
1112
|
+
"attacks_survived": survived, "final": (state.get("final") or {}).get("result")}
|
|
1113
|
+
|
|
1114
|
+
|
|
1115
|
+
def cmd_winner(args):
|
|
1116
|
+
state = load_state(args)
|
|
1117
|
+
rep = winner_report(state)
|
|
1118
|
+
if rep is None:
|
|
1119
|
+
rd = current_round(state)
|
|
1120
|
+
print("no winner yet: round %s, %d alive. %s"
|
|
1121
|
+
% (rd["n"] if rd else "?", len(alive_ids(state)), _cmd("next")))
|
|
1122
|
+
return 1
|
|
1123
|
+
if args.json:
|
|
1124
|
+
print(json.dumps(rep, indent=1))
|
|
1125
|
+
return 0
|
|
1126
|
+
print("CHAMPION %s" % rep["champion"])
|
|
1127
|
+
print("card %s" % rep["card_line"])
|
|
1128
|
+
print("solution %s" % rep["solution"])
|
|
1129
|
+
print("rounds %d played, %d agents in, 1 left, %d match(es) won"
|
|
1130
|
+
% (rep["rounds"], rep["agents"], rep["matches_won"]))
|
|
1131
|
+
for p in rep["path"]:
|
|
1132
|
+
if p["bye"]:
|
|
1133
|
+
print(" round %d bye" % p["round"])
|
|
1134
|
+
continue
|
|
1135
|
+
print(" round %d beat %s, %s to %s. %s"
|
|
1136
|
+
% (p["round"], p["beat"], _fmt(p["score"]), _fmt(p["opponent_score"]), p["reason"] or ""))
|
|
1137
|
+
for s in p["survived"]:
|
|
1138
|
+
print(" survived: %s" % s)
|
|
1139
|
+
fin = rep["final"]
|
|
1140
|
+
if fin:
|
|
1141
|
+
verdict = ("the winner beats it" if fin["better"] == "champion"
|
|
1142
|
+
else "the answer you rejected scored higher. Say so")
|
|
1143
|
+
print("vs the answer you rejected: winner %s, rejected %s, %s. %s"
|
|
1144
|
+
% (_fmt(fin["champion_total"]), _fmt(fin["baseline_total"]), verdict, fin["reason"]))
|
|
1145
|
+
return 0
|
|
1146
|
+
|
|
1147
|
+
|
|
1148
|
+
def cmd_card(args):
|
|
1149
|
+
state = load_state(args)
|
|
1150
|
+
aid = norm_agent(state, args.agent_id)
|
|
1151
|
+
a = state["agents"][aid]
|
|
1152
|
+
c = a["card"]
|
|
1153
|
+
print("%s %s" % (aid, "alive" if a["alive"] else "eliminated in round %s by %s"
|
|
1154
|
+
% (a["eliminated_in"], a["eliminated_by"])))
|
|
1155
|
+
for part in ("reasoning", "workflow", "strategy"):
|
|
1156
|
+
print(" %-9s %s. %s" % (part, c[part]["name"], c[part]["how"]))
|
|
1157
|
+
print(" solution %s" % a["solution"])
|
|
1158
|
+
return 0
|
|
1159
|
+
|
|
1160
|
+
|
|
1161
|
+
def build_parser():
|
|
1162
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
1163
|
+
common.add_argument("--dir", help="the arena run directory (default: the one in .arena/LATEST)")
|
|
1164
|
+
p = argparse.ArgumentParser(prog="bracket.py", description="Tournament state for /arena.")
|
|
1165
|
+
sub = p.add_subparsers(dest="cmd")
|
|
1166
|
+
sub.required = True
|
|
1167
|
+
|
|
1168
|
+
s = sub.add_parser("plan", help="rounds, sub-agent calls and waves for N agents")
|
|
1169
|
+
s.add_argument("--agents", type=int)
|
|
1170
|
+
s.add_argument("--quick", action="store_true")
|
|
1171
|
+
s.add_argument("--wave", type=int, default=DEFAULT_WAVE)
|
|
1172
|
+
s.add_argument("--json", action="store_true")
|
|
1173
|
+
s.set_defaults(func=cmd_plan)
|
|
1174
|
+
|
|
1175
|
+
s = sub.add_parser("init", parents=[common], help="deal the cards and write arena.json")
|
|
1176
|
+
s.add_argument("--agents", type=int, help="number of competitors (default %d)" % DEFAULT_AGENTS)
|
|
1177
|
+
s.add_argument("--quick", action="store_true", help="%d competitors" % QUICK_AGENTS)
|
|
1178
|
+
s.add_argument("--seed", type=int, help="fixes the cards and the pairings (default: random, recorded)")
|
|
1179
|
+
s.add_argument("--task-file", help="the task, word for word, as every competitor will get it")
|
|
1180
|
+
s.add_argument("--task", help="the task as a string, instead of --task-file")
|
|
1181
|
+
s.add_argument("--baseline-file", help="the answer the user was not satisfied with")
|
|
1182
|
+
s.add_argument("--wave", type=int, default=DEFAULT_WAVE, help="parallel sub-agents per wave (default 10)")
|
|
1183
|
+
s.set_defaults(func=cmd_init)
|
|
1184
|
+
|
|
1185
|
+
s = sub.add_parser("next", parents=[common], help="what to do now")
|
|
1186
|
+
s.set_defaults(func=cmd_next)
|
|
1187
|
+
|
|
1188
|
+
for name, fn, hlp in (("prompts", cmd_prompts, "write the briefs for a phase and list the jobs left"),
|
|
1189
|
+
("check", cmd_check, "list the jobs whose outputs are missing")):
|
|
1190
|
+
s = sub.add_parser(name, parents=[common], help=hlp)
|
|
1191
|
+
s.add_argument("phase", choices=PHASES)
|
|
1192
|
+
s.set_defaults(func=fn)
|
|
1193
|
+
|
|
1194
|
+
s = sub.add_parser("pairings", parents=[common], help="this round's matches, including the bye")
|
|
1195
|
+
s.add_argument("--json", action="store_true")
|
|
1196
|
+
s.set_defaults(func=cmd_pairings)
|
|
1197
|
+
|
|
1198
|
+
s = sub.add_parser("record", parents=[common], help="record a match result by hand")
|
|
1199
|
+
s.add_argument("match_id")
|
|
1200
|
+
s.add_argument("winner_id")
|
|
1201
|
+
s.add_argument("--reason")
|
|
1202
|
+
s.add_argument("--survived", action="append", help="an attack the winner survived (repeatable)")
|
|
1203
|
+
s.set_defaults(func=cmd_record)
|
|
1204
|
+
|
|
1205
|
+
s = sub.add_parser("collect", parents=[common], help="record every verdict the judges wrote")
|
|
1206
|
+
s.set_defaults(func=cmd_collect)
|
|
1207
|
+
|
|
1208
|
+
s = sub.add_parser("advance", parents=[common], help="close the round and pair the survivors")
|
|
1209
|
+
s.set_defaults(func=cmd_advance)
|
|
1210
|
+
|
|
1211
|
+
s = sub.add_parser("status", parents=[common], help="alive and eliminated, per round")
|
|
1212
|
+
s.add_argument("--json", action="store_true")
|
|
1213
|
+
s.set_defaults(func=cmd_status)
|
|
1214
|
+
|
|
1215
|
+
s = sub.add_parser("winner", parents=[common], help="the survivor and how it got there")
|
|
1216
|
+
s.add_argument("--json", action="store_true")
|
|
1217
|
+
s.set_defaults(func=cmd_winner)
|
|
1218
|
+
|
|
1219
|
+
s = sub.add_parser("card", parents=[common], help="one competitor's strategy card")
|
|
1220
|
+
s.add_argument("agent_id")
|
|
1221
|
+
s.set_defaults(func=cmd_card)
|
|
1222
|
+
return p
|
|
1223
|
+
|
|
1224
|
+
|
|
1225
|
+
def main(argv=None):
|
|
1226
|
+
args = build_parser().parse_args(argv)
|
|
1227
|
+
try:
|
|
1228
|
+
return args.func(args)
|
|
1229
|
+
except ArenaError as e:
|
|
1230
|
+
print("error: %s" % e, file=sys.stderr)
|
|
1231
|
+
return 2
|
|
1232
|
+
|
|
1233
|
+
|
|
1234
|
+
if __name__ == "__main__":
|
|
1235
|
+
sys.exit(main())
|