aiblueprint-cli 1.4.109 → 1.4.110
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/agents-config/skills/arena/LICENSE +21 -0
- package/agents-config/skills/arena/SKILL.md +371 -0
- package/agents-config/skills/arena/agents/openai.yaml +9 -0
- package/agents-config/skills/arena/assets/codex-icon.svg +22 -0
- package/agents-config/skills/arena/bracket.py +1235 -0
- package/agents-config/skills/arena/rubric.md +79 -0
- package/agents-config/skills/arena/strategies.json +205 -0
- package/dist/cli.js +10 -6
- package/package.json +1 -1
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# The arena rubric
|
|
2
|
+
|
|
3
|
+
Every match in the arena is scored against this page. The judge scores both solutions, `bracket.py`
|
|
4
|
+
does the arithmetic, and the higher weighted total goes through. The loser is out.
|
|
5
|
+
|
|
6
|
+
## The five criteria
|
|
7
|
+
|
|
8
|
+
Each criterion is scored 0 to 10. The weight is how much of the 100-point total it is worth.
|
|
9
|
+
|
|
10
|
+
| criterion | weight | the question |
|
|
11
|
+
| --- | --- | --- |
|
|
12
|
+
| Correctness | 30 | Is it right? No false claims, no logic errors, no bugs, nothing that would mislead the user. |
|
|
13
|
+
| Completeness | 25 | Does it meet every requirement the task actually states? Judged against the task text, not against what the judge would have liked. |
|
|
14
|
+
| Specificity | 15 | Could the user act on it right now without guessing? Exact steps, values, names, code. |
|
|
15
|
+
| Robustness | 20 | Does it hold up against the attacks raised in this match? Fixed, correctly rebutted, or still standing. |
|
|
16
|
+
| Clarity | 10 | Is it easy to read and use, at a length that fits the task? |
|
|
17
|
+
|
|
18
|
+
Weighted total = (correctness x 30 + completeness x 25 + specificity x 15 + robustness x 20 + clarity x 10) / 10.
|
|
19
|
+
That gives a number from 0 to 100.
|
|
20
|
+
|
|
21
|
+
## Anchors
|
|
22
|
+
|
|
23
|
+
Use the whole scale. A 7 is not a polite default.
|
|
24
|
+
|
|
25
|
+
**Correctness**
|
|
26
|
+
- 10: nothing wrong that you can find after checking it yourself.
|
|
27
|
+
- 7: minor slips that do not change the outcome for the user.
|
|
28
|
+
- 4: at least one real error the user would trip on.
|
|
29
|
+
- 0 to 2: wrong at the core, or it would cause harm if used.
|
|
30
|
+
|
|
31
|
+
**Completeness**
|
|
32
|
+
- 10: every stated requirement is met, fully.
|
|
33
|
+
- 7: every requirement is touched, one is thin.
|
|
34
|
+
- 4: a stated requirement is missing.
|
|
35
|
+
- 0 to 2: it answers a different question from the one asked.
|
|
36
|
+
|
|
37
|
+
**Specificity**
|
|
38
|
+
- 10: the user can act on every part of it immediately.
|
|
39
|
+
- 7: mostly concrete, with one or two places that need a guess.
|
|
40
|
+
- 4: a lot of "consider", "ensure" and "it depends" without the actual answer.
|
|
41
|
+
- 0 to 2: generic advice that would fit any task.
|
|
42
|
+
|
|
43
|
+
**Robustness**
|
|
44
|
+
- 10: every attack in this match is fixed in the revised solution or correctly rebutted, and nothing new is broken.
|
|
45
|
+
- 7: one MINOR attack still standing.
|
|
46
|
+
- 4: a MAJOR attack still standing, or a fix that broke something else.
|
|
47
|
+
- 0 to 2: a FATAL attack still standing.
|
|
48
|
+
|
|
49
|
+
A missing attack file means the opponent raised nothing. Score robustness on the flaws you found yourself.
|
|
50
|
+
|
|
51
|
+
**Clarity**
|
|
52
|
+
- 10: the shape makes it obvious how to use it. No padding.
|
|
53
|
+
- 7: fine, with some padding or one confusing section.
|
|
54
|
+
- 4: the user has to dig for the answer.
|
|
55
|
+
- 0 to 2: hard to follow at all.
|
|
56
|
+
|
|
57
|
+
## The fatal rule
|
|
58
|
+
|
|
59
|
+
Mark a solution `fatal` only when you have verified a flaw that makes it wrong or unusable for the
|
|
60
|
+
task: code that cannot work, a false central claim, a hard constraint broken, the wrong question
|
|
61
|
+
answered. A fatal solution cannot beat a solution that is not fatal, whatever the totals say. If both
|
|
62
|
+
are fatal, the totals decide.
|
|
63
|
+
|
|
64
|
+
## Ties
|
|
65
|
+
|
|
66
|
+
There are no draws. On an exact tie, the solution with fewer attacks still standing wins. If that is
|
|
67
|
+
also level, the higher correctness score wins. If that is also level, pick the one you would hand to
|
|
68
|
+
the user, and say why in the reason.
|
|
69
|
+
|
|
70
|
+
## What the judge does not reward
|
|
71
|
+
|
|
72
|
+
- **Length.** Longer is not better. A tight answer that meets every requirement beats a long one
|
|
73
|
+
that meets the same requirements.
|
|
74
|
+
- **Confidence.** A defense that says "fixed" is not proof. Check the revised solution.
|
|
75
|
+
- **The approach.** The judge never sees the strategy cards and does not guess at them. It scores
|
|
76
|
+
the work, not the method that produced it.
|
|
77
|
+
- **Talking about quality.** "This robust, comprehensive solution" earns nothing. The solution has to
|
|
78
|
+
be robust and comprehensive.
|
|
79
|
+
- **Agreement with the judge's own taste** where the task does not ask for it.
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 1,
|
|
3
|
+
"about": "The strategy cards for /arena. Every competitor gets the same task and exactly one card: one reasoning mode, one workflow, one strategy. 15 x 12 x 12 = 2,160 distinct cards. bracket.py deals them with no repeats and spreads them evenly, so a 100-agent arena uses every reasoning mode, every workflow and every strategy. Edit freely: add an entry and the dealer picks it up. Ids must stay unique within a list.",
|
|
4
|
+
"reasoning": [
|
|
5
|
+
{
|
|
6
|
+
"id": "first-principles",
|
|
7
|
+
"name": "First principles",
|
|
8
|
+
"how": "Strip the task down to the facts and requirements nobody can argue with. Rebuild the answer from those alone, and keep a convention only if it earns its place again."
|
|
9
|
+
},
|
|
10
|
+
{
|
|
11
|
+
"id": "inversion",
|
|
12
|
+
"name": "Inversion",
|
|
13
|
+
"how": "Ask what would make this answer fail. List every way it could, then build the answer that makes each of those failures impossible."
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"id": "analogy",
|
|
17
|
+
"name": "Analogy",
|
|
18
|
+
"how": "Find a problem in a different field that has already been solved and has the same shape. Port its solution over, then fix every place where the analogy breaks."
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"id": "adversarial",
|
|
22
|
+
"name": "Adversarial",
|
|
23
|
+
"how": "Picture the harshest expert reviewer trying to tear the answer apart. Write their attacks down first and build the answer backwards from them."
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
"id": "constraint-first",
|
|
27
|
+
"name": "Constraint first",
|
|
28
|
+
"how": "List every hard constraint, stated and implied: time, budget, tools, format, audience, platform. Solve strictly inside that box and let the tightest constraint drive the design."
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"id": "worked-example",
|
|
32
|
+
"name": "Worked example",
|
|
33
|
+
"how": "Solve one concrete, realistic instance end to end first, with real values. Generalise only from what that example actually taught you."
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"id": "socratic",
|
|
37
|
+
"name": "Socratic",
|
|
38
|
+
"how": "Interrogate the task with the questions a sharp expert would ask before starting. Answer each one explicitly and let the answers decide the shape of the solution."
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
"id": "contrarian",
|
|
42
|
+
"name": "Contrarian",
|
|
43
|
+
"how": "Write down the obvious default answer, then argue seriously against it. Keep only what survives the argument, even if that takes you the unpopular way."
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"id": "systems-thinking",
|
|
47
|
+
"name": "Systems thinking",
|
|
48
|
+
"how": "Map the moving parts, how they affect each other, and what happens second and third order. Optimise the whole system, not just the one part the task points at."
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"id": "decomposition",
|
|
52
|
+
"name": "Decomposition",
|
|
53
|
+
"how": "Break the task into independent sub-problems. Solve each one completely, then integrate them and test the seams where they meet."
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"id": "working-backwards",
|
|
57
|
+
"name": "Working backwards",
|
|
58
|
+
"how": "Describe the finished result precisely, as if it already exists. Then work backwards, step by step, to where the user is right now."
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
"id": "probabilistic",
|
|
62
|
+
"name": "Probabilistic",
|
|
63
|
+
"how": "Treat every claim and choice as uncertain. Weigh what is likely, prefer what holds up across most scenarios, and state your confidence where it changes the decision."
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"id": "dialectical",
|
|
67
|
+
"name": "Dialectical",
|
|
68
|
+
"how": "State the strongest version of one answer, then the strongest opposing answer, then write the synthesis that keeps what is true in both."
|
|
69
|
+
},
|
|
70
|
+
{
|
|
71
|
+
"id": "evidence-first",
|
|
72
|
+
"name": "Evidence first",
|
|
73
|
+
"how": "Ground every claim in something checkable: the task text, the files it points to, the numbers given, documentation. Anything you cannot check gets labelled as an assumption."
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
"id": "expert-panel",
|
|
77
|
+
"name": "Expert panel",
|
|
78
|
+
"how": "Answer the task as three different experts would, picking roles that fit it. Take the strongest idea from each and settle every point where they disagree."
|
|
79
|
+
}
|
|
80
|
+
],
|
|
81
|
+
"workflows": [
|
|
82
|
+
{
|
|
83
|
+
"id": "draft-critique-rewrite",
|
|
84
|
+
"name": "Draft, critique, rewrite",
|
|
85
|
+
"how": "Write a complete first draft. Critique it line by line against the task as if someone else wrote it. Then rewrite it from scratch using the critique."
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
"id": "outline-first",
|
|
89
|
+
"name": "Outline first",
|
|
90
|
+
"how": "Write a skeleton outline, check the outline against every requirement in the task, fix it, and only then fill it in."
|
|
91
|
+
},
|
|
92
|
+
{
|
|
93
|
+
"id": "test-first",
|
|
94
|
+
"name": "Test first",
|
|
95
|
+
"how": "Before writing the answer, write the checks it must pass: acceptance tests for code, pass or fail criteria for anything else. Then write the answer and run every check against it."
|
|
96
|
+
},
|
|
97
|
+
{
|
|
98
|
+
"id": "research-then-synthesise",
|
|
99
|
+
"name": "Research, then synthesise",
|
|
100
|
+
"how": "Gather every relevant fact first: the task, any files or context it points to, what is known about the domain. Write nothing until the research is done, then synthesise."
|
|
101
|
+
},
|
|
102
|
+
{
|
|
103
|
+
"id": "three-drafts",
|
|
104
|
+
"name": "Three drafts, pick one",
|
|
105
|
+
"how": "Write three short, genuinely different approaches. Pick the strongest on the task's own terms and develop only that one fully."
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
"id": "requirements-checklist",
|
|
109
|
+
"name": "Requirements checklist",
|
|
110
|
+
"how": "Pull every requirement out of the task into a numbered list. Solve, then tick each item off and say exactly where it is met."
|
|
111
|
+
},
|
|
112
|
+
{
|
|
113
|
+
"id": "iterative-deepening",
|
|
114
|
+
"name": "Iterative deepening",
|
|
115
|
+
"how": "Write a one-paragraph answer first. Then expand the parts that matter most in two more passes, each one deeper than the last."
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
"id": "build-then-break",
|
|
119
|
+
"name": "Build, then break",
|
|
120
|
+
"how": "Build the answer. Then throw five hostile inputs or scenarios at it, write down what breaks, and patch every break."
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
"id": "smallest-version-first",
|
|
124
|
+
"name": "Smallest version first",
|
|
125
|
+
"how": "Produce the smallest complete version that actually works. Then extend it one step at a time, keeping it working after every step."
|
|
126
|
+
},
|
|
127
|
+
{
|
|
128
|
+
"id": "options-matrix",
|
|
129
|
+
"name": "Options matrix",
|
|
130
|
+
"how": "List the realistic options, score them in a small matrix against the criteria that matter for this task, choose one, and write up the choice."
|
|
131
|
+
},
|
|
132
|
+
{
|
|
133
|
+
"id": "open-questions-first",
|
|
134
|
+
"name": "Open questions first",
|
|
135
|
+
"how": "List every open question the task leaves. Resolve each one with reasoning or a stated assumption, then answer."
|
|
136
|
+
},
|
|
137
|
+
{
|
|
138
|
+
"id": "write-then-restructure",
|
|
139
|
+
"name": "Write, then restructure",
|
|
140
|
+
"how": "Write freely and fast. Then outline what you actually wrote, fix the structure, and rewrite it to the new outline."
|
|
141
|
+
}
|
|
142
|
+
],
|
|
143
|
+
"strategies": [
|
|
144
|
+
{
|
|
145
|
+
"id": "simplest",
|
|
146
|
+
"name": "Simplest thing that works",
|
|
147
|
+
"how": "Pick the least complicated answer that fully does the job. Every extra part has to justify itself."
|
|
148
|
+
},
|
|
149
|
+
{
|
|
150
|
+
"id": "maximal-rigour",
|
|
151
|
+
"name": "Maximal rigour",
|
|
152
|
+
"how": "Be exhaustive and exact. Check every step, verify every claim, and leave no hand-waving anywhere."
|
|
153
|
+
},
|
|
154
|
+
{
|
|
155
|
+
"id": "user-empathy",
|
|
156
|
+
"name": "User empathy first",
|
|
157
|
+
"how": "Start from the person who will use this: their situation, their skill level, and what they will actually do next. Shape everything around that."
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
"id": "edge-cases-first",
|
|
161
|
+
"name": "Edge cases first",
|
|
162
|
+
"how": "Hunt the edge cases, failure modes and weird inputs first, and design the answer around surviving them."
|
|
163
|
+
},
|
|
164
|
+
{
|
|
165
|
+
"id": "speed",
|
|
166
|
+
"name": "Speed",
|
|
167
|
+
"how": "Optimise for the fastest route to something usable today. Cut anything that does not move the result."
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
"id": "defensive",
|
|
171
|
+
"name": "Defensive",
|
|
172
|
+
"how": "Assume the inputs are bad and the conditions are hostile. Make the answer fail safely and loudly rather than quietly."
|
|
173
|
+
},
|
|
174
|
+
{
|
|
175
|
+
"id": "clarity",
|
|
176
|
+
"name": "Clarity above all",
|
|
177
|
+
"how": "Make it the easiest answer to read and act on: plain words, obvious structure, nothing the reader has to decode."
|
|
178
|
+
},
|
|
179
|
+
{
|
|
180
|
+
"id": "completeness",
|
|
181
|
+
"name": "Completeness",
|
|
182
|
+
"how": "Leave out nothing the task asks for or clearly implies. Cover every requirement explicitly."
|
|
183
|
+
},
|
|
184
|
+
{
|
|
185
|
+
"id": "fewest-moving-parts",
|
|
186
|
+
"name": "Fewest moving parts",
|
|
187
|
+
"how": "Remove things until it breaks, then put the last one back. Fewer steps, fewer dependencies, fewer words."
|
|
188
|
+
},
|
|
189
|
+
{
|
|
190
|
+
"id": "explicit-trade-offs",
|
|
191
|
+
"name": "Explicit trade-offs",
|
|
192
|
+
"how": "Name the real trade-offs, choose for the user's most likely constraints, and say plainly what that choice gives up."
|
|
193
|
+
},
|
|
194
|
+
{
|
|
195
|
+
"id": "built-to-last",
|
|
196
|
+
"name": "Built to last",
|
|
197
|
+
"how": "Optimise for how this holds up in six months: easy to maintain, easy to extend, no traps for the next person."
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
"id": "concrete-specifics",
|
|
201
|
+
"name": "Concrete specifics",
|
|
202
|
+
"how": "Exact numbers, names, commands, file paths and steps. Zero generic advice."
|
|
203
|
+
}
|
|
204
|
+
]
|
|
205
|
+
}
|
package/dist/cli.js
CHANGED
|
@@ -30139,6 +30139,16 @@ __exportCjs(exports_commander, {
|
|
|
30139
30139
|
InvalidArgumentError: (value) => $InvalidArgumentError = value,
|
|
30140
30140
|
InvalidOptionArgumentError: (value) => $InvalidOptionArgumentError = value
|
|
30141
30141
|
});
|
|
30142
|
+
|
|
30143
|
+
// node_modules/ansi-escapes/index.js
|
|
30144
|
+
var exports_ansi_escapes = {};
|
|
30145
|
+
__exportCjs(exports_ansi_escapes, {
|
|
30146
|
+
default: () => $default
|
|
30147
|
+
}, {
|
|
30148
|
+
default: (value2) => $default = value2
|
|
30149
|
+
});
|
|
30150
|
+
|
|
30151
|
+
// node_modules/commander/index.js
|
|
30142
30152
|
var { Argument } = require_argument();
|
|
30143
30153
|
var { Command } = require_command();
|
|
30144
30154
|
var { CommanderError, InvalidArgumentError } = require_error();
|
|
@@ -30172,12 +30182,6 @@ var {
|
|
|
30172
30182
|
} = exports_commander;
|
|
30173
30183
|
|
|
30174
30184
|
// node_modules/ansi-escapes/index.js
|
|
30175
|
-
var exports_ansi_escapes = {};
|
|
30176
|
-
__exportCjs(exports_ansi_escapes, {
|
|
30177
|
-
default: () => $default
|
|
30178
|
-
}, {
|
|
30179
|
-
default: (value2) => $default = value2
|
|
30180
|
-
});
|
|
30181
30185
|
var ansiEscapes = exports_ansi_escapes;
|
|
30182
30186
|
var $default = ansiEscapes;
|
|
30183
30187
|
var ESC = "\x1B[";
|