aiblueprint-cli 1.4.108 → 1.4.110

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +10 -0
  2. package/agents-config/claude-config/mods/agent4everything/.claude-plugin/plugin.json +5 -0
  3. package/agents-config/claude-config/mods/agent4everything/hooks/hooks.json +1 -0
  4. package/agents-config/claude-config/mods/agent4everything/hooks/register.ts +96 -0
  5. package/agents-config/claude-config/mods/aiblueprint/.claude-plugin/plugin.json +71 -0
  6. package/agents-config/claude-config/mods/aiblueprint/hooks/hooks.json +1 -0
  7. package/agents-config/claude-config/mods/aiblueprint/hooks/register.tsx +309 -0
  8. package/agents-config/claude-config/mods/aiblueprint/tests/band.test.tsx +74 -0
  9. package/agents-config/claude-config/mods/aiblueprint/types/index.d.ts +14 -0
  10. package/agents-config/claude-config/scripts/.claude/skills/fix-on-my-computer/SKILL.md +8 -74
  11. package/agents-config/claude-config/scripts/statusline/src/index.ts +1 -0
  12. package/agents-config/scripts/.claude/skills/fix-on-my-computer/SKILL.md +8 -74
  13. package/agents-config/scripts/statusline/src/index.ts +1 -0
  14. package/agents-config/skills/agents-manager/SKILL.md +15 -453
  15. package/agents-config/skills/apex/SKILL.md +14 -110
  16. package/agents-config/skills/app-icon/SKILL.md +22 -117
  17. package/agents-config/skills/appstore-connect/SKILL.md +22 -85
  18. package/agents-config/skills/arena/LICENSE +21 -0
  19. package/agents-config/skills/arena/SKILL.md +371 -0
  20. package/agents-config/skills/arena/agents/openai.yaml +9 -0
  21. package/agents-config/skills/arena/assets/codex-icon.svg +22 -0
  22. package/agents-config/skills/arena/bracket.py +1235 -0
  23. package/agents-config/skills/arena/rubric.md +79 -0
  24. package/agents-config/skills/arena/strategies.json +205 -0
  25. package/agents-config/skills/audit-memories/SKILL.md +13 -139
  26. package/agents-config/skills/audit-skills/SKILL.md +6 -65
  27. package/agents-config/skills/environments-manager/SKILL.md +14 -238
  28. package/agents-config/skills/grill-me/SKILL.md +8 -22
  29. package/agents-config/skills/hooks-manager/SKILL.md +8 -54
  30. package/agents-config/skills/rules-manager/SKILL.md +11 -192
  31. package/agents-config/skills/skill-manager/SKILL.md +19 -158
  32. package/agents-config/skills/skill-manager/references/description-recommandation.md +15 -85
  33. package/agents-config/skills/test-audit/CAMPAIGN.md +117 -0
  34. package/agents-config/skills/test-audit/LICENSE +23 -0
  35. package/agents-config/skills/test-audit/SKILL.md +169 -0
  36. package/agents-config/skills/test-audit/agents/openai.yaml +10 -0
  37. package/agents-config/skills/test-audit/assets/codex-icon.svg +1 -0
  38. package/agents-config/skills/use-style/SKILL.md +8 -48
  39. package/dist/cli.js +125 -78
  40. package/package.json +1 -1
@@ -0,0 +1,1235 @@
1
+ #!/usr/bin/env python3
2
+ """bracket.py: the tournament state machine behind /arena.
3
+
4
+ The whole arena lives in one JSON file (arena.json), so the orchestrator never
5
+ loses track of who is alive, who fought whom and what happens next, even if its
6
+ own context gets compacted halfway through a 100-agent run.
7
+
8
+ python3 bracket.py plan --agents 100 # rounds, sub-agent calls and waves. Writes nothing.
9
+ python3 bracket.py init --agents 100 --seed 7 --task-file task.md [--baseline-file old.md]
10
+ python3 bracket.py init --quick --task "..." # 16 agents
11
+ python3 bracket.py next # what to do now, and the exact command for it
12
+ python3 bracket.py prompts <phase> # write the sub-agent briefs, list the jobs left, in waves
13
+ python3 bracket.py check <phase> # which outputs are still missing
14
+ python3 bracket.py pairings # this round's matches, including the bye
15
+ python3 bracket.py collect # read every judge verdict and record the winners
16
+ python3 bracket.py record <match_id> <winner_id> [--reason "..."]
17
+ python3 bracket.py advance # close the round, eliminate the losers, pair the survivors
18
+ python3 bracket.py status # alive and eliminated, per round
19
+ python3 bracket.py winner # the survivor, what it beat, the attacks it survived
20
+ python3 bracket.py card <agent_id> # one competitor's strategy card
21
+
22
+ Phases, in order: spawn, then per round attack, defend, judge; then final (only when
23
+ there is a rejected answer to beat). Every command takes --dir; without it, the run
24
+ named in .arena/LATEST is used.
25
+
26
+ Python 3.8+, standard library only.
27
+ """
28
+ import argparse
29
+ import json
30
+ import math
31
+ import os
32
+ import random
33
+ import re
34
+ import sys
35
+ import time
36
+
37
+ HERE = os.path.dirname(os.path.abspath(__file__))
38
+ SELF = os.path.abspath(__file__)
39
+ STRATEGIES_PATH = os.path.join(HERE, "strategies.json")
40
+ SKILL_PATH = os.path.join(HERE, "SKILL.md")
41
+ RUBRIC_PATH = os.path.join(HERE, "rubric.md")
42
+
43
+ DEFAULT_AGENTS = 100
44
+ QUICK_AGENTS = 16
45
+ DEFAULT_WAVE = 10 # Claude Code runs at most 10 tool calls at once by default
46
+ ROOT = ".arena"
47
+ LATEST = "LATEST"
48
+ STATE_FILE = "arena.json"
49
+ PHASES = ("spawn", "attack", "defend", "judge", "final")
50
+ TEMPLATES = ("competitor", "attacker", "defender", "judge", "final")
51
+ CALLS_PER_MATCH = 5 # 2 attacks, 2 defenses, 1 judge
52
+
53
+ # Mirrors the table in rubric.md. tests/test_bracket.py checks they agree.
54
+ WEIGHTS = (
55
+ ("correctness", 30),
56
+ ("completeness", 25),
57
+ ("specificity", 15),
58
+ ("robustness", 20),
59
+ ("clarity", 10),
60
+ )
61
+
62
+
63
+ class ArenaError(Exception):
64
+ pass
65
+
66
+
67
+ # ---------------------------------------------------------------- the cards
68
+
69
+ def load_strategies(path=STRATEGIES_PATH):
70
+ with open(path, encoding="utf-8") as fh:
71
+ data = json.load(fh)
72
+ for key in ("reasoning", "workflows", "strategies"):
73
+ items = data.get(key) or []
74
+ if not items:
75
+ raise ArenaError("strategies.json has no '%s' entries" % key)
76
+ ids = [it["id"] for it in items]
77
+ if len(set(ids)) != len(ids):
78
+ raise ArenaError("strategies.json has a duplicate id in '%s'" % key)
79
+ for it in items:
80
+ if not it.get("name") or not it.get("how"):
81
+ raise ArenaError("strategies.json entry '%s' needs a name and a how" % it["id"])
82
+ return data
83
+
84
+
85
+ def combo_count(data):
86
+ return len(data["reasoning"]) * len(data["workflows"]) * len(data["strategies"])
87
+
88
+
89
+ def _edge_colour(edges, k):
90
+ """Properly edge-colour a bipartite (multi)graph with k colours, then balance it.
91
+
92
+ edges are (row, col) pairs and k must be at least the largest degree. Proper
93
+ means no two edges at the same row, or at the same col, share a colour. Every
94
+ bipartite graph has one (Konig), found here with alternating-path swaps. Then
95
+ de Werra's swaps even the colour classes out until each is used floor or ceil
96
+ of len(edges) / k times.
97
+ """
98
+ colour = [None] * len(edges)
99
+ at = ({}, {})
100
+
101
+ def slot(side, node):
102
+ return at[side].setdefault(node, {})
103
+
104
+ def put(e, c):
105
+ u, v = edges[e]
106
+ colour[e] = c
107
+ slot(0, u)[c] = e
108
+ slot(1, v)[c] = e
109
+
110
+ def take(e):
111
+ u, v = edges[e]
112
+ del slot(0, u)[colour[e]]
113
+ del slot(1, v)[colour[e]]
114
+
115
+ def walk(side, node, first, other):
116
+ path, c = [], first
117
+ while True:
118
+ e = slot(side, node).get(c)
119
+ if e is None or e in path:
120
+ return path
121
+ path.append(e)
122
+ node = edges[e][1 - side]
123
+ side = 1 - side
124
+ c = other if c == first else first
125
+
126
+ def swap(path, a, b):
127
+ new = {e: (b if colour[e] == a else a) for e in path}
128
+ for e in path:
129
+ take(e)
130
+ for e in path:
131
+ put(e, new[e])
132
+
133
+ for e, (u, v) in enumerate(edges):
134
+ free_u = [c for c in range(k) if c not in slot(0, u)]
135
+ free_v = set(c for c in range(k) if c not in slot(1, v))
136
+ c = next((c for c in free_u if c in free_v), None)
137
+ if c is None:
138
+ a, b = free_u[0], min(free_v)
139
+ swap(walk(1, v, a, b), a, b)
140
+ c = a
141
+ put(e, c)
142
+
143
+ while True:
144
+ count = [0] * k
145
+ for c in colour:
146
+ count[c] += 1
147
+ hi = max(range(k), key=lambda c: (count[c], -c))
148
+ lo = min(range(k), key=lambda c: (count[c], c))
149
+ if count[hi] - count[lo] <= 1:
150
+ return colour
151
+ swapped = False
152
+ for side in (0, 1):
153
+ for node in sorted(at[side]):
154
+ s = at[side][node]
155
+ if hi in s and lo not in s:
156
+ path = walk(side, node, hi, lo)
157
+ if sum(colour[e] == hi for e in path) > sum(colour[e] == lo for e in path):
158
+ swap(path, hi, lo)
159
+ swapped = True
160
+ break
161
+ if swapped:
162
+ break
163
+ if not swapped:
164
+ raise ArenaError("could not balance the strategy cards")
165
+
166
+
167
+ def deal(n, seed, data):
168
+ """Deal n distinct (reasoning, workflow, strategy) index triples, one per agent.
169
+
170
+ Deterministic for a given seed and strategies.json. Guarantees:
171
+ - no card is dealt twice
172
+ - every reasoning mode, workflow and strategy is dealt as evenly as possible
173
+ (any two counts differ by at most 1), so 100 agents use all of them
174
+ - while every reasoning mode and every workflow has no more agents than there
175
+ are strategies (up to 144 agents with the shipped cards): no two agents share
176
+ a reasoning mode and a workflow, a reasoning mode and a strategy, or a
177
+ workflow and a strategy. Any two agents differ in at least two of the three.
178
+
179
+ How: agent slot i gets reasoning i mod R and workflow (i + i // lcm(R, W)) mod W,
180
+ which never repeats a workflow inside a reasoning mode and stays balanced. The
181
+ strategies are then a balanced proper edge colouring of that reasoning x
182
+ workflow grid. The seed shuffles every list's labels and who gets which card.
183
+ """
184
+ R, W, S = len(data["reasoning"]), len(data["workflows"]), len(data["strategies"])
185
+ total = R * W * S
186
+ if n < 1 or n > total:
187
+ raise ArenaError("can deal between 1 and %d cards, not %d" % (total, n))
188
+ rng = random.Random("arena-deal:%s" % seed)
189
+ pr, pw, ps = list(range(R)), list(range(W)), list(range(S))
190
+ rng.shuffle(pr)
191
+ rng.shuffle(pw)
192
+ rng.shuffle(ps)
193
+ lcm = R * W // math.gcd(R, W)
194
+ cells = [(i % R, (i + i // lcm) % W) for i in range(n)]
195
+ deg_r = max(sum(1 for r, _ in cells if r == x) for x in range(R))
196
+ deg_w = max(sum(1 for _, w in cells if w == x) for x in range(W))
197
+ if deg_r <= S and deg_w <= S:
198
+ order = list(range(n))
199
+ rng.shuffle(order)
200
+ colours = _edge_colour([cells[i] for i in order], S)
201
+ strategy_of = {order[j]: colours[j] for j in range(n)}
202
+ triples = [(r, w, strategy_of[i]) for i, (r, w) in enumerate(cells)]
203
+ else:
204
+ # Past that size a repeat pair is unavoidable. Stay balanced and never repeat a card.
205
+ count, pairs_rs, pairs_ws, used, triples = [0] * S, {}, {}, set(), []
206
+ for r, w in cells:
207
+ low = min(count)
208
+ s = min((c for c in range(S) if (r, w, c) not in used),
209
+ key=lambda c: (count[c] - low, pairs_rs.get((r, c), 0), pairs_ws.get((w, c), 0), c))
210
+ used.add((r, w, s))
211
+ count[s] += 1
212
+ pairs_rs[(r, s)] = pairs_rs.get((r, s), 0) + 1
213
+ pairs_ws[(w, s)] = pairs_ws.get((w, s), 0) + 1
214
+ triples.append((r, w, s))
215
+ dealt = [(pr[r], pw[w], ps[s]) for r, w, s in triples]
216
+ rng.shuffle(dealt)
217
+ return dealt
218
+
219
+
220
+ def agent_ids(n):
221
+ width = max(3, len(str(n)))
222
+ return ["a%0*d" % (width, i) for i in range(1, n + 1)]
223
+
224
+
225
+ def bracket_sizes(n):
226
+ """Alive count at the start of each round, ending at 1. 100 -> 50 -> 25 -> 13 -> 7 -> 4 -> 2 -> 1."""
227
+ sizes = [n]
228
+ while sizes[-1] > 1:
229
+ sizes.append((sizes[-1] + 1) // 2)
230
+ return sizes
231
+
232
+
233
+ # ---------------------------------------------------------------- paths
234
+
235
+ def spawn_out(d, aid):
236
+ return os.path.join(d, "r0", aid + ".md")
237
+
238
+
239
+ def attack_out(d, rnd, mid, attacker):
240
+ return os.path.join(d, "r%d" % rnd, "%s.%s.attack.md" % (mid, attacker))
241
+
242
+
243
+ def defense_out(d, rnd, mid, aid):
244
+ return os.path.join(d, "r%d" % rnd, "%s.%s.defense.md" % (mid, aid))
245
+
246
+
247
+ def revised_out(d, rnd, mid, aid):
248
+ return os.path.join(d, "r%d" % rnd, "%s.%s.solution.md" % (mid, aid))
249
+
250
+
251
+ def verdict_out(d, rnd, mid):
252
+ return os.path.join(d, "r%d" % rnd, "%s.verdict.json" % mid)
253
+
254
+
255
+ def final_out(d):
256
+ return os.path.join(d, "final.verdict.json")
257
+
258
+
259
+ def prompt_path(d, rnd, job_id):
260
+ sub = "final" if rnd is None else "r%d" % rnd
261
+ return os.path.join(d, "prompts", sub, job_id + ".md")
262
+
263
+
264
+ def _has_output(path):
265
+ return os.path.isfile(path) and os.path.getsize(path) > 0
266
+
267
+
268
+ # ---------------------------------------------------------------- the bracket
269
+
270
+ def new_state(n, seed, data, run_dir, wave=DEFAULT_WAVE, has_baseline=False):
271
+ run_dir = os.path.abspath(run_dir)
272
+ ids = agent_ids(n)
273
+ agents = {}
274
+ for aid, (r, w, s) in zip(ids, deal(n, seed, data)):
275
+ agents[aid] = {
276
+ "card": {
277
+ "reasoning": dict(data["reasoning"][r]),
278
+ "workflow": dict(data["workflows"][w]),
279
+ "strategy": dict(data["strategies"][s]),
280
+ },
281
+ "solution": spawn_out(run_dir, aid),
282
+ "alive": True,
283
+ "eliminated_in": None,
284
+ "eliminated_by": None,
285
+ "byes": 0,
286
+ }
287
+ state = {
288
+ "version": 1,
289
+ "created": time.strftime("%Y-%m-%d %H:%M:%S"),
290
+ "seed": seed,
291
+ "agents_n": n,
292
+ "cards_available": combo_count(data),
293
+ "wave": wave,
294
+ "dir": run_dir,
295
+ "has_baseline": bool(has_baseline),
296
+ "agents": agents,
297
+ "rounds": [],
298
+ "champion": None,
299
+ "final": None,
300
+ }
301
+ open_round(state, ids)
302
+ return state
303
+
304
+
305
+ def pair_round(state, rnd, alive):
306
+ """Seeded pairing. An odd pool gives one bye, to an agent with the fewest byes so far.
307
+ Where it can, it pairs agents with different reasoning modes, so every attack comes
308
+ from a genuinely different angle. The first-listed side is random, which is the
309
+ order the judge reads them in."""
310
+ agents = state["agents"]
311
+ rng = random.Random("arena-pair:%s:%d" % (state["seed"], rnd))
312
+ pool = sorted(alive)
313
+ rng.shuffle(pool)
314
+ bye = None
315
+ if len(pool) % 2:
316
+ fewest = min(agents[a]["byes"] for a in pool)
317
+ bye = next(a for a in pool if agents[a]["byes"] == fewest)
318
+ pool.remove(bye)
319
+ width = max(2, len(str(len(pool) // 2)))
320
+ matches = []
321
+ while pool:
322
+ a = pool.pop(0)
323
+ mode = agents[a]["card"]["reasoning"]["id"]
324
+ j = next((k for k, b in enumerate(pool) if agents[b]["card"]["reasoning"]["id"] != mode), 0)
325
+ b = pool.pop(j)
326
+ if rng.random() < 0.5:
327
+ a, b = b, a
328
+ matches.append({
329
+ "id": "r%d-m%0*d" % (rnd, width, len(matches) + 1),
330
+ "a": a, "b": b,
331
+ "winner": None, "loser": None, "reason": None,
332
+ "survived": [], "scores": None, "source": None,
333
+ })
334
+ return matches, bye
335
+
336
+
337
+ def open_round(state, alive):
338
+ """Open the next round, or crown the champion when one agent is left."""
339
+ alive = sorted(alive)
340
+ if len(alive) == 1:
341
+ state["champion"] = alive[0]
342
+ if state.get("has_baseline") and not state.get("final"):
343
+ rng = random.Random("arena-final:%s" % state["seed"])
344
+ x = "champion" if rng.random() < 0.5 else "baseline"
345
+ state["final"] = {"X": x, "Y": "baseline" if x == "champion" else "champion", "result": None}
346
+ return None
347
+ rnd = len(state["rounds"]) + 1
348
+ matches, bye = pair_round(state, rnd, alive)
349
+ rd = {"n": rnd, "alive": alive, "matches": matches, "bye": bye, "closed": False}
350
+ state["rounds"].append(rd)
351
+ return rd
352
+
353
+
354
+ def current_round(state):
355
+ if state["champion"] or not state["rounds"]:
356
+ return None
357
+ rd = state["rounds"][-1]
358
+ return None if rd["closed"] else rd
359
+
360
+
361
+ def norm_agent(state, raw):
362
+ raw = str(raw).strip().lower()
363
+ if raw in state["agents"]:
364
+ return raw
365
+ digits = raw.lstrip("a")
366
+ if digits.isdigit():
367
+ for aid in state["agents"]:
368
+ if int(aid[1:]) == int(digits):
369
+ return aid
370
+ raise ArenaError("no agent called '%s'" % raw)
371
+
372
+
373
+ def find_match(state, mid):
374
+ rd = current_round(state)
375
+ if rd is None:
376
+ raise ArenaError("no round is open" + (" (the arena has a champion)" if state["champion"] else ""))
377
+ raw = str(mid).strip().lower()
378
+ for m in rd["matches"]:
379
+ if m["id"] == raw:
380
+ return rd, m
381
+ tail = raw.split("-m")[-1].lstrip("m")
382
+ if tail.isdigit():
383
+ for m in rd["matches"]:
384
+ if int(m["id"].split("-m")[-1]) == int(tail):
385
+ return rd, m
386
+ raise ArenaError("no match '%s' in round %d (run pairings)" % (mid, rd["n"]))
387
+
388
+
389
+ def record(state, mid, winner, reason=None, survived=None, scores=None, source="manual"):
390
+ rd, m = find_match(state, mid)
391
+ winner = norm_agent(state, winner)
392
+ if winner not in (m["a"], m["b"]):
393
+ raise ArenaError("%s is not in %s (%s vs %s)" % (winner, m["id"], m["a"], m["b"]))
394
+ previous = m["winner"]
395
+ m["winner"] = winner
396
+ m["loser"] = m["b"] if winner == m["a"] else m["a"]
397
+ m["reason"] = reason or None
398
+ m["survived"] = list(survived or [])
399
+ m["scores"] = scores
400
+ m["source"] = source
401
+ return m, previous
402
+
403
+
404
+ def advance(state):
405
+ """Close the current round: eliminate every loser, carry the winners' revised
406
+ solutions forward, give the bye its free pass, and open the next round."""
407
+ rd = current_round(state)
408
+ if rd is None:
409
+ raise ArenaError("no round is open" + (" (the arena has a champion)" if state["champion"] else ""))
410
+ unrecorded = [m["id"] for m in rd["matches"] if not m["winner"]]
411
+ if unrecorded:
412
+ raise ArenaError("round %d has %d unrecorded match(es): %s"
413
+ % (rd["n"], len(unrecorded), ", ".join(unrecorded)))
414
+ survivors = []
415
+ for m in rd["matches"]:
416
+ loser = state["agents"][m["loser"]]
417
+ loser["alive"] = False
418
+ loser["eliminated_in"] = rd["n"]
419
+ loser["eliminated_by"] = m["winner"]
420
+ for aid in (m["a"], m["b"]):
421
+ rev = revised_out(state["dir"], rd["n"], m["id"], aid)
422
+ if _has_output(rev):
423
+ state["agents"][aid]["solution"] = rev
424
+ survivors.append(m["winner"])
425
+ if rd["bye"]:
426
+ state["agents"][rd["bye"]]["byes"] += 1
427
+ survivors.append(rd["bye"])
428
+ rd["closed"] = True
429
+ open_round(state, survivors)
430
+ return survivors
431
+
432
+
433
+ def rounds_played(state):
434
+ return sum(1 for rd in state["rounds"] if rd["closed"])
435
+
436
+
437
+ def alive_ids(state):
438
+ return sorted(a for a, v in state["agents"].items() if v["alive"])
439
+
440
+
441
+ # ---------------------------------------------------------------- verdicts
442
+
443
+ def weighted_total(scores):
444
+ """0 to 100 from five 0 to 10 scores, weighted as in rubric.md. None if any is missing."""
445
+ if not isinstance(scores, dict):
446
+ return None
447
+ total = 0.0
448
+ for key, weight in WEIGHTS:
449
+ v = scores.get(key)
450
+ if isinstance(v, bool):
451
+ return None
452
+ try:
453
+ v = float(v)
454
+ except (TypeError, ValueError):
455
+ return None
456
+ total += max(0.0, min(10.0, v)) * weight
457
+ return round(total / 10.0, 2)
458
+
459
+
460
+ def _truthy(v):
461
+ if isinstance(v, str):
462
+ return v.strip().lower() in ("true", "yes", "1")
463
+ return bool(v)
464
+
465
+
466
+ def extract_json(text):
467
+ text = text.strip()
468
+ try:
469
+ return json.loads(text)
470
+ except ValueError:
471
+ pass
472
+ i, j = text.find("{"), text.rfind("}")
473
+ if i != -1 and j > i:
474
+ try:
475
+ return json.loads(text[i:j + 1])
476
+ except ValueError:
477
+ return None
478
+ return None
479
+
480
+
481
+ def decide(verdict, a, b):
482
+ """Apply the rubric's arithmetic to a judge's verdict.
483
+
484
+ The winner is the higher weighted total. A fatal solution cannot beat a
485
+ non-fatal one. On an exact tie: fewer standing attacks, then higher
486
+ correctness, then the judge's own pick. If the judge's pick disagrees with its
487
+ own scores, the scores win and the note says so.
488
+ Returns (winner, {a: total, b: total}, note).
489
+ """
490
+ if not isinstance(verdict, dict):
491
+ raise ArenaError("verdict is not a JSON object")
492
+ raw_scores = verdict.get("scores") or {}
493
+ scores = {str(k).strip().lower(): v for k, v in raw_scores.items()} if isinstance(raw_scores, dict) else {}
494
+ sa, sb = scores.get(a), scores.get(b)
495
+ pick_raw = str(verdict.get("winner") or "").strip().lower()
496
+ in_a = re.search(r"\b%s\b" % re.escape(a), pick_raw) is not None
497
+ in_b = re.search(r"\b%s\b" % re.escape(b), pick_raw) is not None
498
+ pick = a if (in_a and not in_b) else b if (in_b and not in_a) else None
499
+ ta, tb = weighted_total(sa), weighted_total(sb)
500
+ totals = {a: ta, b: tb}
501
+ if ta is None or tb is None:
502
+ if pick:
503
+ return pick, totals, "scores incomplete, used the judge's pick"
504
+ raise ArenaError("verdict has neither complete scores nor a clear winner")
505
+ fa, fb = _truthy(sa.get("fatal")), _truthy(sb.get("fatal"))
506
+ if fa != fb:
507
+ winner = b if fa else a
508
+ elif ta != tb:
509
+ winner = a if ta > tb else b
510
+ else:
511
+ standing = verdict.get("standing") if isinstance(verdict.get("standing"), dict) else {}
512
+ standing = {str(k).strip().lower(): v for k, v in standing.items()}
513
+ na = len(standing.get(a) or []) if isinstance(standing.get(a), list) else None
514
+ nb = len(standing.get(b) or []) if isinstance(standing.get(b), list) else None
515
+ ca, cb = float(sa.get("correctness", 0)), float(sb.get("correctness", 0))
516
+ if na is not None and nb is not None and na != nb:
517
+ winner = a if na < nb else b
518
+ elif ca != cb:
519
+ winner = a if ca > cb else b
520
+ elif pick:
521
+ winner = pick
522
+ else:
523
+ raise ArenaError("exact tie and the judge did not pick a winner")
524
+ note = ""
525
+ if pick and pick != winner:
526
+ note = "judge picked %s but its own scores favour %s, so the scores win" % (pick, winner)
527
+ return winner, totals, note
528
+
529
+
530
+ def collect(state):
531
+ """Record every verdict the judges have written for the open round (or the final
532
+ check). An unreadable verdict is set aside as <name>.unreadable so the judge job
533
+ shows up as missing again and gets re-run."""
534
+ d = state["dir"]
535
+ results = []
536
+ if state["champion"]:
537
+ fin = state.get("final")
538
+ if not fin or fin.get("result"):
539
+ return results
540
+ path = final_out(d)
541
+ if not _has_output(path):
542
+ return [("final", "missing", "")]
543
+ with open(path, encoding="utf-8", errors="replace") as fh:
544
+ v = extract_json(fh.read())
545
+ try:
546
+ if isinstance(v, dict) and isinstance(v.get("scores"), dict):
547
+ v = dict(v, scores={str(k).strip().lower(): s for k, s in v["scores"].items()})
548
+ w, totals, note = decide(v, "x", "y")
549
+ except ArenaError as e:
550
+ os.replace(path, path + ".unreadable")
551
+ return [("final", "unreadable", str(e))]
552
+ better = fin[w.upper()]
553
+ fin["result"] = {
554
+ "better": better,
555
+ "champion_total": totals["x" if fin["X"] == "champion" else "y"],
556
+ "baseline_total": totals["x" if fin["X"] == "baseline" else "y"],
557
+ "reason": str(v.get("reason") or "")[:400],
558
+ "fixed": [str(x) for x in (v.get("fixed") or [])][:10] if isinstance(v.get("fixed"), list) else [],
559
+ }
560
+ return [("final", "recorded", ("%s is better. %s" % (better, note)).strip())]
561
+ rd = current_round(state)
562
+ if rd is None:
563
+ return results
564
+ for m in rd["matches"]:
565
+ if m["winner"]:
566
+ continue
567
+ path = verdict_out(d, rd["n"], m["id"])
568
+ if not _has_output(path):
569
+ results.append((m["id"], "missing", ""))
570
+ continue
571
+ with open(path, encoding="utf-8", errors="replace") as fh:
572
+ v = extract_json(fh.read())
573
+ try:
574
+ winner, totals, note = decide(v, m["a"], m["b"])
575
+ except ArenaError as e:
576
+ os.replace(path, path + ".unreadable")
577
+ results.append((m["id"], "unreadable", str(e)))
578
+ continue
579
+ survived = v.get("survived") if isinstance(v.get("survived"), list) else []
580
+ record(state, m["id"], winner,
581
+ reason=str(v.get("reason") or "")[:400],
582
+ survived=[str(x) for x in survived][:10],
583
+ scores=totals, source="judge")
584
+ loser = m["loser"]
585
+ line = "%s beat %s, %s to %s" % (winner, loser, _fmt(totals[winner]), _fmt(totals[loser]))
586
+ results.append((m["id"], "recorded", line + (". " + note if note else "")))
587
+ return results
588
+
589
+
590
+ def _fmt(x):
591
+ return "?" if x is None else ("%g" % x)
592
+
593
+
594
+ # ---------------------------------------------------------------- jobs and prompts
595
+
596
+ def phase_jobs(state, phase):
597
+ """Every sub-agent job for a phase, with its prompt file and the outputs it must write.
598
+ Matches that already have a recorded winner are skipped."""
599
+ if phase not in PHASES:
600
+ raise ArenaError("unknown phase '%s' (one of: %s)" % (phase, ", ".join(PHASES)))
601
+ d = state["dir"]
602
+ jobs = []
603
+ if phase == "spawn":
604
+ for aid in sorted(state["agents"]):
605
+ jobs.append({"id": aid + ".spawn", "kind": "competitor", "round": 0, "agent": aid,
606
+ "outputs": [spawn_out(d, aid)]})
607
+ elif phase == "final":
608
+ fin = state.get("final")
609
+ if state["champion"] and fin and not fin.get("result"):
610
+ jobs.append({"id": "final.judge", "kind": "final", "round": None, "outputs": [final_out(d)]})
611
+ else:
612
+ rd = current_round(state)
613
+ if rd is None:
614
+ return []
615
+ n = rd["n"]
616
+ for m in rd["matches"]:
617
+ if m["winner"]:
618
+ continue
619
+ a, b = m["a"], m["b"]
620
+ if phase == "attack":
621
+ for me, opp in ((a, b), (b, a)):
622
+ jobs.append({"id": "%s.%s.attack" % (m["id"], me), "kind": "attacker", "round": n,
623
+ "agent": me, "opponent": opp, "match": m["id"],
624
+ "outputs": [attack_out(d, n, m["id"], me)]})
625
+ elif phase == "defend":
626
+ for me, opp in ((a, b), (b, a)):
627
+ jobs.append({"id": "%s.%s.defend" % (m["id"], me), "kind": "defender", "round": n,
628
+ "agent": me, "opponent": opp, "match": m["id"],
629
+ "outputs": [defense_out(d, n, m["id"], me), revised_out(d, n, m["id"], me)]})
630
+ else:
631
+ jobs.append({"id": "%s.judge" % m["id"], "kind": "judge", "round": n,
632
+ "match": m["id"], "a": a, "b": b,
633
+ "outputs": [verdict_out(d, n, m["id"])]})
634
+ for j in jobs:
635
+ j["prompt"] = prompt_path(d, j["round"], j["id"])
636
+ return jobs
637
+
638
+
639
+ def missing_jobs(jobs):
640
+ return [j for j in jobs if not all(_has_output(p) for p in j["outputs"])]
641
+
642
+
643
+ _TEMPLATE_RE = re.compile(r"<!-- template:([a-z]+) -->\s*```text\n(.*?)\n```\s*<!-- /template:\1 -->", re.S)
644
+ _PLACEHOLDER_RE = re.compile(r"\{\{(\w+)\}\}")
645
+
646
+
647
+ def load_templates(path=SKILL_PATH):
648
+ with open(path, encoding="utf-8") as fh:
649
+ text = fh.read()
650
+ found = {m.group(1): m.group(2) for m in _TEMPLATE_RE.finditer(text)}
651
+ missing = [t for t in TEMPLATES if t not in found]
652
+ if missing:
653
+ raise ArenaError("SKILL.md is missing prompt templates: %s" % ", ".join(missing))
654
+ return found
655
+
656
+
657
+ def fill(template, values):
658
+ """One pass, so a task that itself contains {{braces}} is left exactly as written."""
659
+ wanted = set(_PLACEHOLDER_RE.findall(template))
660
+ missing = sorted(wanted - set(values))
661
+ if missing:
662
+ raise ArenaError("no value for placeholder(s): %s" % ", ".join(missing))
663
+ return _PLACEHOLDER_RE.sub(lambda m: str(values[m.group(1)]), template)
664
+
665
+
666
+ def read_task(state):
667
+ """The task exactly as init stored it (init adds one trailing newline to the file)."""
668
+ with open(os.path.join(state["dir"], "task.md"), encoding="utf-8") as fh:
669
+ return fh.read().rstrip("\n")
670
+
671
+
672
+ def _card_values(card):
673
+ return {
674
+ "reasoning_name": card["reasoning"]["name"], "reasoning_how": card["reasoning"]["how"],
675
+ "workflow_name": card["workflow"]["name"], "workflow_how": card["workflow"]["how"],
676
+ "strategy_name": card["strategy"]["name"], "strategy_how": card["strategy"]["how"],
677
+ }
678
+
679
+
680
+ def render_job(state, job, templates, task):
681
+ d = state["dir"]
682
+ agents = state["agents"]
683
+ v = {"task": task, "arena_dir": d, "rubric": os.path.join(d, "rubric.md"), "n": state["agents_n"]}
684
+ kind = job["kind"]
685
+ if kind == "competitor":
686
+ aid = job["agent"]
687
+ v.update(_card_values(agents[aid]["card"]))
688
+ if state.get("has_baseline"):
689
+ v["baseline_note"] = (
690
+ "The user already got an answer to this task and was NOT satisfied with it. It is at "
691
+ "%s. Read it first and work out exactly why it fell short. Then beat it. Do not just "
692
+ "polish it: the winner of this arena is compared with that answer at the end."
693
+ % os.path.join(d, "baseline.md"))
694
+ else:
695
+ v["baseline_note"] = "There is no earlier answer to beat. Start from the task."
696
+ v.update(agent=aid, out=job["outputs"][0])
697
+ elif kind == "attacker":
698
+ me, opp = job["agent"], job["opponent"]
699
+ v.update(_card_values(agents[me]["card"]))
700
+ v.update(agent=me, target=opp, match=job["match"], round=job["round"],
701
+ own_solution=agents[me]["solution"], target_solution=agents[opp]["solution"],
702
+ out=job["outputs"][0])
703
+ elif kind == "defender":
704
+ me, opp = job["agent"], job["opponent"]
705
+ v.update(_card_values(agents[me]["card"]))
706
+ v.update(agent=me, attacker=opp, match=job["match"], round=job["round"],
707
+ own_solution=agents[me]["solution"],
708
+ attacks=attack_out(d, job["round"], job["match"], opp),
709
+ defense_out=job["outputs"][0], solution_out=job["outputs"][1])
710
+ elif kind == "judge":
711
+ n, mid, a, b = job["round"], job["match"], job["a"], job["b"]
712
+ v.update(match=mid, round=n, first=a, second=b,
713
+ first_solution=revised_out(d, n, mid, a), second_solution=revised_out(d, n, mid, b),
714
+ first_attacks=attack_out(d, n, mid, b), second_attacks=attack_out(d, n, mid, a),
715
+ first_defense=defense_out(d, n, mid, a), second_defense=defense_out(d, n, mid, b),
716
+ out=job["outputs"][0])
717
+ elif kind == "final":
718
+ v.update(rounds=rounds_played(state),
719
+ x_solution=os.path.join(d, "final", "X.md"),
720
+ y_solution=os.path.join(d, "final", "Y.md"),
721
+ out=job["outputs"][0])
722
+ return fill(templates[kind], v)
723
+
724
+
725
+ def write_prompts(state, phase):
726
+ jobs = phase_jobs(state, phase)
727
+ if not jobs:
728
+ return jobs
729
+ templates = load_templates()
730
+ task = read_task(state)
731
+ for j in jobs:
732
+ for p in j["outputs"] + [j["prompt"]]:
733
+ os.makedirs(os.path.dirname(p), exist_ok=True)
734
+ body = render_job(state, j, templates, task)
735
+ with open(j["prompt"], "w", encoding="utf-8") as fh:
736
+ fh.write(body + "\n")
737
+ if phase == "spawn":
738
+ for aid in state["agents"]:
739
+ os.makedirs(os.path.join(state["dir"], "scratch", aid), exist_ok=True)
740
+ if phase == "final":
741
+ # Blind copies: the final judge sees X and Y, never which one is the champion.
742
+ fin = state["final"]
743
+ sources = {"champion": state["agents"][state["champion"]]["solution"],
744
+ "baseline": os.path.join(state["dir"], "baseline.md")}
745
+ os.makedirs(os.path.join(state["dir"], "final"), exist_ok=True)
746
+ for label in ("X", "Y"):
747
+ with open(sources[fin[label]], encoding="utf-8", errors="replace") as src, \
748
+ open(os.path.join(state["dir"], "final", label + ".md"), "w", encoding="utf-8") as dst:
749
+ dst.write(src.read())
750
+ return jobs
751
+
752
+
753
+ def next_action(state):
754
+ """(phase, jobs_left, jobs_total). phase is one of spawn, attack, defend, judge,
755
+ final, collect, advance, done."""
756
+ spawn = phase_jobs(state, "spawn")
757
+ left = missing_jobs(spawn)
758
+ if left:
759
+ return "spawn", left, spawn
760
+ if state["champion"]:
761
+ fin = state.get("final")
762
+ if fin and not fin.get("result"):
763
+ jobs = phase_jobs(state, "final")
764
+ left = missing_jobs(jobs)
765
+ return ("final", left, jobs) if left else ("collect", [], jobs)
766
+ return "done", [], []
767
+ for phase in ("attack", "defend", "judge"):
768
+ jobs = phase_jobs(state, phase)
769
+ left = missing_jobs(jobs)
770
+ if left:
771
+ return phase, left, jobs
772
+ rd = current_round(state)
773
+ if any(not m["winner"] for m in rd["matches"]):
774
+ return "collect", [], []
775
+ return "advance", [], []
776
+
777
+
778
+ # ---------------------------------------------------------------- plan
779
+
780
+ def plan_rows(n, wave=DEFAULT_WAVE):
781
+ rows = []
782
+ for rnd, alive in enumerate(bracket_sizes(n)[:-1], start=1):
783
+ m = alive // 2
784
+ waves = 2 * math.ceil(2 * m / wave) + math.ceil(m / wave)
785
+ rows.append({"round": rnd, "alive": alive, "matches": m, "bye": alive % 2,
786
+ "calls": CALLS_PER_MATCH * m, "waves": waves})
787
+ return rows
788
+
789
+
790
+ def plan_totals(n, wave=DEFAULT_WAVE):
791
+ rows = plan_rows(n, wave)
792
+ calls = n + sum(r["calls"] for r in rows)
793
+ waves = math.ceil(n / wave) + sum(r["waves"] for r in rows)
794
+ return {"agents": n, "rounds": len(rows), "calls": calls, "waves": waves, "rows": rows}
795
+
796
+
797
+ # ---------------------------------------------------------------- state io
798
+
799
+ def run_dir_from(args):
800
+ if getattr(args, "dir", None):
801
+ return os.path.abspath(args.dir)
802
+ latest = os.path.join(ROOT, LATEST)
803
+ if os.path.isfile(latest):
804
+ with open(latest, encoding="utf-8") as fh:
805
+ return fh.read().strip()
806
+ raise ArenaError("no arena here. Run init first, or pass --dir")
807
+
808
+
809
+ def load_state(args):
810
+ d = run_dir_from(args)
811
+ path = os.path.join(d, STATE_FILE)
812
+ if not os.path.isfile(path):
813
+ raise ArenaError("no %s in %s" % (STATE_FILE, d))
814
+ with open(path, encoding="utf-8") as fh:
815
+ return json.load(fh)
816
+
817
+
818
+ def save_state(state):
819
+ path = os.path.join(state["dir"], STATE_FILE)
820
+ tmp = path + ".tmp"
821
+ with open(tmp, "w", encoding="utf-8") as fh:
822
+ json.dump(state, fh, indent=1)
823
+ fh.write("\n")
824
+ os.replace(tmp, path)
825
+
826
+
827
+ def _cmd(*parts):
828
+ return 'python3 "%s" %s' % (SELF, " ".join(parts))
829
+
830
+
831
+ def _card_line(card):
832
+ return "%s + %s + %s" % (card["reasoning"]["name"], card["workflow"]["name"], card["strategy"]["name"])
833
+
834
+
835
+ # ---------------------------------------------------------------- commands
836
+
837
+ def _agents_arg(args):
838
+ if args.quick and args.agents is not None:
839
+ raise ArenaError("pass --quick or --agents, not both")
840
+ if args.quick:
841
+ return QUICK_AGENTS
842
+ return DEFAULT_AGENTS if args.agents is None else args.agents
843
+
844
+
845
+ def cmd_plan(args):
846
+ n = _agents_arg(args)
847
+ if n < 1:
848
+ raise ArenaError("--agents must be at least 1")
849
+ t = plan_totals(n, args.wave)
850
+ if args.json:
851
+ print(json.dumps(t, indent=1))
852
+ return 0
853
+ print("%d agents, %d rounds, waves of %d" % (n, t["rounds"], args.wave))
854
+ print("")
855
+ print(" round alive matches bye sub-agent calls waves")
856
+ print(" spawn %5d %7s %3s %15d %5d" % (n, "-", "-", n, math.ceil(n / args.wave)))
857
+ for r in t["rows"]:
858
+ print(" %5d %5d %7d %3s %15d %5d"
859
+ % (r["round"], r["alive"], r["matches"], "yes" if r["bye"] else "-", r["calls"], r["waves"]))
860
+ print(" total %15d %5d" % (t["calls"], t["waves"]))
861
+ print("")
862
+ print(" alive per round: %s" % " -> ".join(str(s) for s in bracket_sizes(n)))
863
+ print(" plus 1 call for the final check when there is a rejected answer to beat")
864
+ return 0
865
+
866
+
867
+ def cmd_init(args):
868
+ n = _agents_arg(args)
869
+ if args.wave < 1:
870
+ raise ArenaError("--wave must be at least 1")
871
+ data = load_strategies()
872
+ if n < 1 or n > combo_count(data):
873
+ raise ArenaError("--agents must be between 1 and %d (the number of distinct cards)" % combo_count(data))
874
+ if args.task_file:
875
+ with open(args.task_file, encoding="utf-8") as fh:
876
+ task = fh.read()
877
+ else:
878
+ task = args.task or ""
879
+ task = task.strip()
880
+ if not task:
881
+ raise ArenaError("the task is empty. Pass --task-file or --task")
882
+ baseline = None
883
+ if args.baseline_file:
884
+ with open(args.baseline_file, encoding="utf-8") as fh:
885
+ baseline = fh.read().strip()
886
+ if not baseline:
887
+ raise ArenaError("the baseline file is empty")
888
+ seed = args.seed if args.seed is not None else random.SystemRandom().randrange(1, 1000000)
889
+ default_dir = not args.dir
890
+ d = os.path.abspath(args.dir or os.path.join(ROOT, "run-%s-s%s" % (time.strftime("%Y%m%d-%H%M%S"), seed)))
891
+ if os.path.exists(os.path.join(d, STATE_FILE)):
892
+ raise ArenaError("%s already has an arena in it. Pick another --dir" % d)
893
+ os.makedirs(d, exist_ok=True)
894
+ with open(os.path.join(d, "task.md"), "w", encoding="utf-8") as fh:
895
+ fh.write(task + "\n")
896
+ if baseline:
897
+ with open(os.path.join(d, "baseline.md"), "w", encoding="utf-8") as fh:
898
+ fh.write(baseline + "\n")
899
+ # The run gets its own copy of the rubric: frozen for this run, and inside the
900
+ # working directory, so judges can read it without a permission prompt each.
901
+ with open(RUBRIC_PATH, encoding="utf-8") as src, \
902
+ open(os.path.join(d, "rubric.md"), "w", encoding="utf-8") as dst:
903
+ dst.write(src.read())
904
+ state = new_state(n, seed, data, d, wave=args.wave, has_baseline=bool(baseline))
905
+ save_state(state)
906
+ if default_dir:
907
+ os.makedirs(ROOT, exist_ok=True)
908
+ with open(os.path.join(ROOT, LATEST), "w", encoding="utf-8") as fh:
909
+ fh.write(d + "\n")
910
+ t = plan_totals(n, args.wave)
911
+ print("arena ready: %s" % d)
912
+ print("seed %s. %d agents, %d distinct cards dealt from %d, no repeats."
913
+ % (seed, n, n, combo_count(data)))
914
+ print("%d rounds (%s). %d sub-agent calls in %d waves of %d%s."
915
+ % (t["rounds"], " -> ".join(str(s) for s in bracket_sizes(n)), t["calls"] + (1 if baseline else 0),
916
+ t["waves"] + (1 if baseline else 0), args.wave, ", including the final check" if baseline else ""))
917
+ print("next: %s" % _cmd("next"))
918
+ return 0
919
+
920
+
921
+ def cmd_next(args):
922
+ state = load_state(args)
923
+ phase, left, jobs = next_action(state)
924
+ rd = current_round(state)
925
+ total_rounds = len(bracket_sizes(state["agents_n"])) - 1
926
+ if phase == "spawn":
927
+ print("spawn: %d competitors, then %d round(s)" % (state["agents_n"], total_rounds))
928
+ elif rd:
929
+ print("round %d of %d, %d alive" % (rd["n"], total_rounds, len(rd["alive"])))
930
+ elif state["champion"]:
931
+ print("champion: %s after %d round(s)" % (state["champion"], rounds_played(state)))
932
+ if phase in PHASES:
933
+ print("NEXT: %s. %d of %d job(s) to run." % (phase, len(left), len(jobs)))
934
+ print(" 1. %s" % _cmd("prompts", phase))
935
+ print(" 2. launch the jobs it lists, one wave of %d at a time" % state["wave"])
936
+ print(" 3. %s" % _cmd("next"))
937
+ elif phase == "collect":
938
+ print("NEXT: collect. Every verdict for this step is on disk.")
939
+ print(" %s" % _cmd("collect"))
940
+ elif phase == "advance":
941
+ print("NEXT: advance. Every match in round %d has a winner." % rd["n"])
942
+ print(" %s" % _cmd("advance"))
943
+ else:
944
+ print("DONE. %s" % _cmd("winner"))
945
+ return 0
946
+
947
+
948
+ def cmd_prompts(args):
949
+ state = load_state(args)
950
+ jobs = write_prompts(state, args.phase)
951
+ if not jobs:
952
+ print("nothing to run for %s right now. %s" % (args.phase, _cmd("next")))
953
+ return 0
954
+ left = missing_jobs(jobs)
955
+ w = state["wave"]
956
+ rd = current_round(state)
957
+ where = "round 0" if args.phase == "spawn" else ("final" if args.phase == "final" else "round %d" % rd["n"])
958
+ waves = [left[i:i + w] for i in range(0, len(left), w)]
959
+ print("%s, %s: %d job(s), %d still to run, %d wave(s) of up to %d"
960
+ % (where, args.phase, len(jobs), len(left), len(waves), w))
961
+ for k, wave in enumerate(waves, start=1):
962
+ print("wave %d" % k)
963
+ for j in wave:
964
+ print(" %-26s %s" % (j["id"], j["prompt"]))
965
+ if left:
966
+ print("")
967
+ print("Each job is one Agent call: subagent_type general-purpose, description \"arena <job id>\",")
968
+ print("prompt \"Read <prompt path> and follow it exactly. It is your whole brief.\"")
969
+ print("A wave is one message with all of its Agent calls. Let it finish before the next wave.")
970
+ print("then: %s" % _cmd("next"))
971
+ return 0
972
+
973
+
974
+ def cmd_check(args):
975
+ state = load_state(args)
976
+ jobs = phase_jobs(state, args.phase)
977
+ left = missing_jobs(jobs)
978
+ print("%s: %d of %d job(s) done" % (args.phase, len(jobs) - len(left), len(jobs)))
979
+ for j in left:
980
+ gone = [p for p in j["outputs"] if not _has_output(p)]
981
+ print(" missing %-26s %s" % (j["id"], ", ".join(gone)))
982
+ return 1 if left else 0
983
+
984
+
985
+ def cmd_pairings(args):
986
+ state = load_state(args)
987
+ rd = current_round(state)
988
+ if rd is None:
989
+ if state["champion"]:
990
+ print("No matches left. %s is the last one standing." % state["champion"])
991
+ return 0
992
+ raise ArenaError("no round is open")
993
+ if args.json:
994
+ print(json.dumps(rd, indent=1))
995
+ return 0
996
+ agents = state["agents"]
997
+ print("round %d: %d alive, %d match(es), bye: %s"
998
+ % (rd["n"], len(rd["alive"]), len(rd["matches"]), rd["bye"] or "none"))
999
+ for m in rd["matches"]:
1000
+ ra = agents[m["a"]]["card"]["reasoning"]["name"]
1001
+ rb = agents[m["b"]]["card"]["reasoning"]["name"]
1002
+ res = ("winner %s" % m["winner"]) if m["winner"] else "open"
1003
+ print(" %s %s vs %s (%s vs %s) %s" % (m["id"], m["a"], m["b"], ra, rb, res))
1004
+ if rd["bye"]:
1005
+ print(" bye %s goes through without a match" % rd["bye"])
1006
+ return 0
1007
+
1008
+
1009
+ def cmd_record(args):
1010
+ state = load_state(args)
1011
+ m, previous = record(state, args.match_id, args.winner_id, reason=args.reason,
1012
+ survived=args.survived, source="manual")
1013
+ save_state(state)
1014
+ extra = " (was %s)" % previous if previous and previous != m["winner"] else ""
1015
+ print("%s: %s beat %s%s" % (m["id"], m["winner"], m["loser"], extra))
1016
+ return 0
1017
+
1018
+
1019
+ def cmd_collect(args):
1020
+ state = load_state(args)
1021
+ results = collect(state)
1022
+ save_state(state)
1023
+ if not results:
1024
+ print("nothing to collect. %s" % _cmd("next"))
1025
+ return 0
1026
+ bad = 0
1027
+ for mid, what, note in results:
1028
+ if what != "recorded":
1029
+ bad += 1
1030
+ print(" %-10s %-10s %s" % (mid, what, note))
1031
+ rd = current_round(state)
1032
+ if rd:
1033
+ done = sum(1 for m in rd["matches"] if m["winner"])
1034
+ print("round %d: %d of %d match(es) recorded" % (rd["n"], done, len(rd["matches"])))
1035
+ if bad:
1036
+ print("%d verdict(s) missing or unreadable. Re-run those judges: %s" % (bad, _cmd("next")))
1037
+ return 1
1038
+ print("then: %s" % _cmd("next"))
1039
+ return 0
1040
+
1041
+
1042
+ def cmd_advance(args):
1043
+ state = load_state(args)
1044
+ rd = current_round(state)
1045
+ closing = rd["n"] if rd else None
1046
+ survivors = advance(state)
1047
+ save_state(state)
1048
+ print("round %s closed: %d through, %d eliminated"
1049
+ % (closing, len(survivors), len(rd["matches"])))
1050
+ if state["champion"]:
1051
+ print("CHAMPION: %s. %s" % (state["champion"], _cmd("next")))
1052
+ else:
1053
+ nr = state["rounds"][-1]
1054
+ print("round %d open: %d match(es)%s. %s"
1055
+ % (nr["n"], len(nr["matches"]), ", bye %s" % nr["bye"] if nr["bye"] else "", _cmd("next")))
1056
+ return 0
1057
+
1058
+
1059
+ def status_rows(state):
1060
+ rows = []
1061
+ for rd in state["rounds"]:
1062
+ recorded = sum(1 for m in rd["matches"] if m["winner"])
1063
+ rows.append({"round": rd["n"], "alive": len(rd["alive"]), "matches": len(rd["matches"]),
1064
+ "bye": rd["bye"], "eliminated": recorded,
1065
+ "state": "closed" if rd["closed"] else "open, %d of %d recorded" % (recorded, len(rd["matches"]))})
1066
+ return rows
1067
+
1068
+
1069
+ def cmd_status(args):
1070
+ state = load_state(args)
1071
+ rows = status_rows(state)
1072
+ alive = alive_ids(state)
1073
+ if args.json:
1074
+ print(json.dumps({"dir": state["dir"], "seed": state["seed"], "agents": state["agents_n"],
1075
+ "alive": len(alive), "eliminated": state["agents_n"] - len(alive),
1076
+ "champion": state["champion"], "rounds": rows}, indent=1))
1077
+ return 0
1078
+ print("arena %s" % state["dir"])
1079
+ print("seed %s, %d agents, waves of %d, %d distinct cards available"
1080
+ % (state["seed"], state["agents_n"], state["wave"], state["cards_available"]))
1081
+ print(" round alive matches bye eliminated state")
1082
+ for r in rows:
1083
+ print(" %5d %5d %7d %-6s %10d %s"
1084
+ % (r["round"], r["alive"], r["matches"], r["bye"] or "-", r["eliminated"], r["state"]))
1085
+ print("now: %d alive, %d eliminated%s"
1086
+ % (len(alive), state["agents_n"] - len(alive),
1087
+ ", champion %s" % state["champion"] if state["champion"] else ""))
1088
+ return 0
1089
+
1090
+
1091
+ def winner_report(state):
1092
+ champ = state["champion"]
1093
+ if not champ:
1094
+ return None
1095
+ agent = state["agents"][champ]
1096
+ path = []
1097
+ survived = []
1098
+ for rd in state["rounds"]:
1099
+ if rd["bye"] == champ:
1100
+ path.append({"round": rd["n"], "bye": True})
1101
+ for m in rd["matches"]:
1102
+ if champ in (m["a"], m["b"]):
1103
+ opp = m["b"] if m["a"] == champ else m["a"]
1104
+ sc = m.get("scores") or {}
1105
+ path.append({"round": rd["n"], "bye": False, "beat": opp, "match": m["id"],
1106
+ "score": sc.get(champ), "opponent_score": sc.get(opp),
1107
+ "reason": m.get("reason"), "survived": m.get("survived") or []})
1108
+ survived.extend(m.get("survived") or [])
1109
+ return {"champion": champ, "card": agent["card"], "card_line": _card_line(agent["card"]),
1110
+ "solution": agent["solution"], "rounds": rounds_played(state), "agents": state["agents_n"],
1111
+ "matches_won": sum(1 for p in path if not p["bye"]), "path": path,
1112
+ "attacks_survived": survived, "final": (state.get("final") or {}).get("result")}
1113
+
1114
+
1115
+ def cmd_winner(args):
1116
+ state = load_state(args)
1117
+ rep = winner_report(state)
1118
+ if rep is None:
1119
+ rd = current_round(state)
1120
+ print("no winner yet: round %s, %d alive. %s"
1121
+ % (rd["n"] if rd else "?", len(alive_ids(state)), _cmd("next")))
1122
+ return 1
1123
+ if args.json:
1124
+ print(json.dumps(rep, indent=1))
1125
+ return 0
1126
+ print("CHAMPION %s" % rep["champion"])
1127
+ print("card %s" % rep["card_line"])
1128
+ print("solution %s" % rep["solution"])
1129
+ print("rounds %d played, %d agents in, 1 left, %d match(es) won"
1130
+ % (rep["rounds"], rep["agents"], rep["matches_won"]))
1131
+ for p in rep["path"]:
1132
+ if p["bye"]:
1133
+ print(" round %d bye" % p["round"])
1134
+ continue
1135
+ print(" round %d beat %s, %s to %s. %s"
1136
+ % (p["round"], p["beat"], _fmt(p["score"]), _fmt(p["opponent_score"]), p["reason"] or ""))
1137
+ for s in p["survived"]:
1138
+ print(" survived: %s" % s)
1139
+ fin = rep["final"]
1140
+ if fin:
1141
+ verdict = ("the winner beats it" if fin["better"] == "champion"
1142
+ else "the answer you rejected scored higher. Say so")
1143
+ print("vs the answer you rejected: winner %s, rejected %s, %s. %s"
1144
+ % (_fmt(fin["champion_total"]), _fmt(fin["baseline_total"]), verdict, fin["reason"]))
1145
+ return 0
1146
+
1147
+
1148
+ def cmd_card(args):
1149
+ state = load_state(args)
1150
+ aid = norm_agent(state, args.agent_id)
1151
+ a = state["agents"][aid]
1152
+ c = a["card"]
1153
+ print("%s %s" % (aid, "alive" if a["alive"] else "eliminated in round %s by %s"
1154
+ % (a["eliminated_in"], a["eliminated_by"])))
1155
+ for part in ("reasoning", "workflow", "strategy"):
1156
+ print(" %-9s %s. %s" % (part, c[part]["name"], c[part]["how"]))
1157
+ print(" solution %s" % a["solution"])
1158
+ return 0
1159
+
1160
+
1161
+ def build_parser():
1162
+ common = argparse.ArgumentParser(add_help=False)
1163
+ common.add_argument("--dir", help="the arena run directory (default: the one in .arena/LATEST)")
1164
+ p = argparse.ArgumentParser(prog="bracket.py", description="Tournament state for /arena.")
1165
+ sub = p.add_subparsers(dest="cmd")
1166
+ sub.required = True
1167
+
1168
+ s = sub.add_parser("plan", help="rounds, sub-agent calls and waves for N agents")
1169
+ s.add_argument("--agents", type=int)
1170
+ s.add_argument("--quick", action="store_true")
1171
+ s.add_argument("--wave", type=int, default=DEFAULT_WAVE)
1172
+ s.add_argument("--json", action="store_true")
1173
+ s.set_defaults(func=cmd_plan)
1174
+
1175
+ s = sub.add_parser("init", parents=[common], help="deal the cards and write arena.json")
1176
+ s.add_argument("--agents", type=int, help="number of competitors (default %d)" % DEFAULT_AGENTS)
1177
+ s.add_argument("--quick", action="store_true", help="%d competitors" % QUICK_AGENTS)
1178
+ s.add_argument("--seed", type=int, help="fixes the cards and the pairings (default: random, recorded)")
1179
+ s.add_argument("--task-file", help="the task, word for word, as every competitor will get it")
1180
+ s.add_argument("--task", help="the task as a string, instead of --task-file")
1181
+ s.add_argument("--baseline-file", help="the answer the user was not satisfied with")
1182
+ s.add_argument("--wave", type=int, default=DEFAULT_WAVE, help="parallel sub-agents per wave (default 10)")
1183
+ s.set_defaults(func=cmd_init)
1184
+
1185
+ s = sub.add_parser("next", parents=[common], help="what to do now")
1186
+ s.set_defaults(func=cmd_next)
1187
+
1188
+ for name, fn, hlp in (("prompts", cmd_prompts, "write the briefs for a phase and list the jobs left"),
1189
+ ("check", cmd_check, "list the jobs whose outputs are missing")):
1190
+ s = sub.add_parser(name, parents=[common], help=hlp)
1191
+ s.add_argument("phase", choices=PHASES)
1192
+ s.set_defaults(func=fn)
1193
+
1194
+ s = sub.add_parser("pairings", parents=[common], help="this round's matches, including the bye")
1195
+ s.add_argument("--json", action="store_true")
1196
+ s.set_defaults(func=cmd_pairings)
1197
+
1198
+ s = sub.add_parser("record", parents=[common], help="record a match result by hand")
1199
+ s.add_argument("match_id")
1200
+ s.add_argument("winner_id")
1201
+ s.add_argument("--reason")
1202
+ s.add_argument("--survived", action="append", help="an attack the winner survived (repeatable)")
1203
+ s.set_defaults(func=cmd_record)
1204
+
1205
+ s = sub.add_parser("collect", parents=[common], help="record every verdict the judges wrote")
1206
+ s.set_defaults(func=cmd_collect)
1207
+
1208
+ s = sub.add_parser("advance", parents=[common], help="close the round and pair the survivors")
1209
+ s.set_defaults(func=cmd_advance)
1210
+
1211
+ s = sub.add_parser("status", parents=[common], help="alive and eliminated, per round")
1212
+ s.add_argument("--json", action="store_true")
1213
+ s.set_defaults(func=cmd_status)
1214
+
1215
+ s = sub.add_parser("winner", parents=[common], help="the survivor and how it got there")
1216
+ s.add_argument("--json", action="store_true")
1217
+ s.set_defaults(func=cmd_winner)
1218
+
1219
+ s = sub.add_parser("card", parents=[common], help="one competitor's strategy card")
1220
+ s.add_argument("agent_id")
1221
+ s.set_defaults(func=cmd_card)
1222
+ return p
1223
+
1224
+
1225
+ def main(argv=None):
1226
+ args = build_parser().parse_args(argv)
1227
+ try:
1228
+ return args.func(args)
1229
+ except ArenaError as e:
1230
+ print("error: %s" % e, file=sys.stderr)
1231
+ return 2
1232
+
1233
+
1234
+ if __name__ == "__main__":
1235
+ sys.exit(main())