catherd-cli 1.1.1 → 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -14
- package/catalog/ATTRIBUTION.md +36 -0
- package/catalog/models.json +30 -0
- package/catalog/scores.json +3006 -194
- package/catalog/sources.json +87 -0
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/bin/catherd-mcp +1 -1
- package/plugin/skills/catherd/SKILL.md +2 -2
- package/src/adapters/claude-code/models.ts +2 -1
- package/src/domain/bars.ts +119 -0
- package/src/domain/calibration.ts +125 -0
- package/src/domain/catalog.ts +210 -23
- package/src/domain/failover.ts +14 -1
- package/src/domain/profile-rules.ts +108 -18
- package/src/domain/sources.ts +144 -0
- package/src/entry/catalog-command.ts +163 -8
- package/src/entry/init-command.ts +72 -2
- package/src/entry/mcp/lane-tools.ts +1 -1
- package/src/entry/mcp/server.ts +13 -3
- package/src/entry/mcp/setup-tools.ts +24 -2
- package/src/entry/profile-command.ts +10 -4
- package/src/entry/tui/commands.ts +20 -2
- package/src/entry/tui/effects.ts +50 -1
- package/src/entry/tui/fixtures.ts +31 -3
- package/src/entry/tui/profile-edits.ts +17 -9
- package/src/entry/tui/profile-tree.ts +2 -2
- package/src/entry/tui/state.ts +4 -0
- package/src/entry/tui/views/profile-actions.ts +9 -1
- package/src/entry/tui/views/profiles.tsx +81 -14
- package/src/entry/tui/views/save-dialog.tsx +12 -4
- package/src/infra/env.ts +2 -0
- package/src/infra/jev-client.ts +88 -40
- package/src/infra/paths.ts +2 -0
- package/src/infra/sources/arena.ts +87 -0
- package/src/infra/sources/artificial-analysis.ts +112 -0
- package/src/infra/sources/cache.ts +124 -0
- package/src/infra/sources/csv.ts +38 -0
- package/src/infra/sources/epoch.ts +82 -0
- package/src/infra/sources/http.ts +75 -0
- package/src/infra/sources/litellm.ts +58 -0
- package/src/infra/sources/models-dev.ts +59 -0
- package/src/infra/sources/openrouter-endpoints.ts +52 -0
- package/src/infra/sources/openrouter-models.ts +60 -0
- package/src/infra/sources/rows.ts +38 -0
- package/src/infra/sources/vectara.ts +69 -0
- package/src/infra/sources/zip.ts +48 -0
- package/src/services/catalog-refresh.ts +200 -0
- package/src/services/catalog-service.ts +83 -23
- package/src/services/credentials.ts +69 -0
- package/src/services/doctor-sources.ts +76 -0
- package/src/services/doctor.ts +10 -0
- package/src/services/jev-service.ts +14 -41
- package/src/services/ports.ts +6 -0
- package/src/services/profile-service.ts +8 -6
- package/src/services/provenance.ts +115 -0
- package/src/services/routing-service.ts +7 -2
- package/src/services/run-evidence.ts +86 -0
- package/src/services/source-derive.ts +349 -0
- package/src/services/source-sync.ts +305 -0
- package/src/services/standins.ts +204 -0
- package/src/services/treat-likes.ts +84 -0
package/catalog/scores.json
CHANGED
|
@@ -1,15 +1,2624 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schema": 1,
|
|
3
|
-
"version": "2026-09-
|
|
3
|
+
"version": "2026-09-28",
|
|
4
4
|
"benchmarks": {
|
|
5
|
-
"repo_code": {
|
|
6
|
-
|
|
5
|
+
"repo_code": {
|
|
6
|
+
"benchmark": "DeepSWE",
|
|
7
|
+
"version": "1.1"
|
|
8
|
+
},
|
|
9
|
+
"terminal": {
|
|
10
|
+
"benchmark": "Terminal-Bench",
|
|
11
|
+
"version": "4.0"
|
|
12
|
+
},
|
|
7
13
|
"honesty": {
|
|
8
14
|
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
9
15
|
"version": "GPT-6 system card appendix 11.6.4.2"
|
|
16
|
+
},
|
|
17
|
+
"agentic": {
|
|
18
|
+
"benchmark": "Arena agent, net improvement",
|
|
19
|
+
"version": "lmarena-ai/leaderboard-dataset"
|
|
20
|
+
},
|
|
21
|
+
"steer": {
|
|
22
|
+
"benchmark": "Arena agent steerability",
|
|
23
|
+
"version": "lmarena-ai/leaderboard-dataset"
|
|
24
|
+
},
|
|
25
|
+
"frontend": {
|
|
26
|
+
"benchmark": "Arena WebDev rating",
|
|
27
|
+
"version": "lmarena-ai/leaderboard-dataset"
|
|
10
28
|
}
|
|
11
29
|
},
|
|
12
30
|
"scores": [
|
|
31
|
+
{
|
|
32
|
+
"rung": "claude-fable-5-1#high",
|
|
33
|
+
"dim": "terminal",
|
|
34
|
+
"value": 55.8,
|
|
35
|
+
"benchmark": "Terminal-Bench",
|
|
36
|
+
"version": "4.0",
|
|
37
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
38
|
+
"date": "2026-09-22",
|
|
39
|
+
"confidence": "adjacent",
|
|
40
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"rung": "claude-fable-5-1#high",
|
|
44
|
+
"dim": "agentic",
|
|
45
|
+
"value": 0.1384,
|
|
46
|
+
"benchmark": "Arena agent, net improvement",
|
|
47
|
+
"version": "2026-09-27",
|
|
48
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
49
|
+
"date": "2026-09-27",
|
|
50
|
+
"confidence": "adjacent",
|
|
51
|
+
"source": "arena",
|
|
52
|
+
"note": "arena has it at max; carried to this effort"
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"rung": "claude-fable-5-1#high",
|
|
56
|
+
"dim": "steer",
|
|
57
|
+
"value": 0.0832,
|
|
58
|
+
"benchmark": "Arena agent steerability",
|
|
59
|
+
"version": "2026-09-27",
|
|
60
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
61
|
+
"date": "2026-09-27",
|
|
62
|
+
"confidence": "adjacent",
|
|
63
|
+
"source": "arena",
|
|
64
|
+
"note": "arena has it at max; carried to this effort"
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
"rung": "claude-fable-5-1#high",
|
|
68
|
+
"dim": "frontend",
|
|
69
|
+
"value": 1751.4868,
|
|
70
|
+
"benchmark": "Arena WebDev rating",
|
|
71
|
+
"version": "2026-09-25",
|
|
72
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
73
|
+
"date": "2026-09-25",
|
|
74
|
+
"confidence": "adjacent",
|
|
75
|
+
"source": "arena",
|
|
76
|
+
"note": "arena has it at max; carried to this effort"
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
"rung": "claude-fable-5-1#low",
|
|
80
|
+
"dim": "terminal",
|
|
81
|
+
"value": 55.8,
|
|
82
|
+
"benchmark": "Terminal-Bench",
|
|
83
|
+
"version": "4.0",
|
|
84
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
85
|
+
"date": "2026-09-22",
|
|
86
|
+
"confidence": "adjacent",
|
|
87
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"rung": "claude-fable-5-1#low",
|
|
91
|
+
"dim": "agentic",
|
|
92
|
+
"value": 0.1384,
|
|
93
|
+
"benchmark": "Arena agent, net improvement",
|
|
94
|
+
"version": "2026-09-27",
|
|
95
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
96
|
+
"date": "2026-09-27",
|
|
97
|
+
"confidence": "adjacent",
|
|
98
|
+
"source": "arena",
|
|
99
|
+
"note": "arena has it at max; carried to this effort"
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
"rung": "claude-fable-5-1#low",
|
|
103
|
+
"dim": "steer",
|
|
104
|
+
"value": 0.0832,
|
|
105
|
+
"benchmark": "Arena agent steerability",
|
|
106
|
+
"version": "2026-09-27",
|
|
107
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
108
|
+
"date": "2026-09-27",
|
|
109
|
+
"confidence": "adjacent",
|
|
110
|
+
"source": "arena",
|
|
111
|
+
"note": "arena has it at max; carried to this effort"
|
|
112
|
+
},
|
|
113
|
+
{
|
|
114
|
+
"rung": "claude-fable-5-1#low",
|
|
115
|
+
"dim": "frontend",
|
|
116
|
+
"value": 1751.4868,
|
|
117
|
+
"benchmark": "Arena WebDev rating",
|
|
118
|
+
"version": "2026-09-25",
|
|
119
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
120
|
+
"date": "2026-09-25",
|
|
121
|
+
"confidence": "adjacent",
|
|
122
|
+
"source": "arena",
|
|
123
|
+
"note": "arena has it at max; carried to this effort"
|
|
124
|
+
},
|
|
125
|
+
{
|
|
126
|
+
"rung": "claude-fable-5-1#max",
|
|
127
|
+
"dim": "repo_code",
|
|
128
|
+
"value": 67.4,
|
|
129
|
+
"benchmark": "DeepSWE",
|
|
130
|
+
"version": "1.1",
|
|
131
|
+
"url": "https://codingfleet.com/blog/deepswe-v11-leaderboard-2026/",
|
|
132
|
+
"date": "2026-09-25",
|
|
133
|
+
"confidence": "inferred",
|
|
134
|
+
"note": "a 5-trial mean from a secondary source that names no effort"
|
|
135
|
+
},
|
|
136
|
+
{
|
|
137
|
+
"rung": "claude-fable-5-1#max",
|
|
138
|
+
"dim": "terminal",
|
|
139
|
+
"value": 55.8,
|
|
140
|
+
"benchmark": "Terminal-Bench",
|
|
141
|
+
"version": "4.0",
|
|
142
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
143
|
+
"date": "2026-09-22",
|
|
144
|
+
"confidence": "verified"
|
|
145
|
+
},
|
|
146
|
+
{
|
|
147
|
+
"rung": "claude-fable-5-1#max",
|
|
148
|
+
"dim": "agentic",
|
|
149
|
+
"value": 0.1384,
|
|
150
|
+
"benchmark": "Arena agent, net improvement",
|
|
151
|
+
"version": "2026-09-27",
|
|
152
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
153
|
+
"date": "2026-09-27",
|
|
154
|
+
"confidence": "measured",
|
|
155
|
+
"source": "arena"
|
|
156
|
+
},
|
|
157
|
+
{
|
|
158
|
+
"rung": "claude-fable-5-1#max",
|
|
159
|
+
"dim": "steer",
|
|
160
|
+
"value": 0.0832,
|
|
161
|
+
"benchmark": "Arena agent steerability",
|
|
162
|
+
"version": "2026-09-27",
|
|
163
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
164
|
+
"date": "2026-09-27",
|
|
165
|
+
"confidence": "measured",
|
|
166
|
+
"source": "arena"
|
|
167
|
+
},
|
|
168
|
+
{
|
|
169
|
+
"rung": "claude-fable-5-1#max",
|
|
170
|
+
"dim": "frontend",
|
|
171
|
+
"value": 1751.4868,
|
|
172
|
+
"benchmark": "Arena WebDev rating",
|
|
173
|
+
"version": "2026-09-25",
|
|
174
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
175
|
+
"date": "2026-09-25",
|
|
176
|
+
"confidence": "measured",
|
|
177
|
+
"source": "arena"
|
|
178
|
+
},
|
|
179
|
+
{
|
|
180
|
+
"rung": "claude-fable-5-1#medium",
|
|
181
|
+
"dim": "terminal",
|
|
182
|
+
"value": 55.8,
|
|
183
|
+
"benchmark": "Terminal-Bench",
|
|
184
|
+
"version": "4.0",
|
|
185
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
186
|
+
"date": "2026-09-22",
|
|
187
|
+
"confidence": "adjacent",
|
|
188
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
189
|
+
},
|
|
190
|
+
{
|
|
191
|
+
"rung": "claude-fable-5-1#medium",
|
|
192
|
+
"dim": "agentic",
|
|
193
|
+
"value": 0.1384,
|
|
194
|
+
"benchmark": "Arena agent, net improvement",
|
|
195
|
+
"version": "2026-09-27",
|
|
196
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
197
|
+
"date": "2026-09-27",
|
|
198
|
+
"confidence": "adjacent",
|
|
199
|
+
"source": "arena",
|
|
200
|
+
"note": "arena has it at max; carried to this effort"
|
|
201
|
+
},
|
|
202
|
+
{
|
|
203
|
+
"rung": "claude-fable-5-1#medium",
|
|
204
|
+
"dim": "steer",
|
|
205
|
+
"value": 0.0832,
|
|
206
|
+
"benchmark": "Arena agent steerability",
|
|
207
|
+
"version": "2026-09-27",
|
|
208
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
209
|
+
"date": "2026-09-27",
|
|
210
|
+
"confidence": "adjacent",
|
|
211
|
+
"source": "arena",
|
|
212
|
+
"note": "arena has it at max; carried to this effort"
|
|
213
|
+
},
|
|
214
|
+
{
|
|
215
|
+
"rung": "claude-fable-5-1#medium",
|
|
216
|
+
"dim": "frontend",
|
|
217
|
+
"value": 1751.4868,
|
|
218
|
+
"benchmark": "Arena WebDev rating",
|
|
219
|
+
"version": "2026-09-25",
|
|
220
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
221
|
+
"date": "2026-09-25",
|
|
222
|
+
"confidence": "adjacent",
|
|
223
|
+
"source": "arena",
|
|
224
|
+
"note": "arena has it at max; carried to this effort"
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
"rung": "claude-fable-5-1#xhigh",
|
|
228
|
+
"dim": "terminal",
|
|
229
|
+
"value": 55.8,
|
|
230
|
+
"benchmark": "Terminal-Bench",
|
|
231
|
+
"version": "4.0",
|
|
232
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
233
|
+
"date": "2026-09-22",
|
|
234
|
+
"confidence": "adjacent",
|
|
235
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
236
|
+
},
|
|
237
|
+
{
|
|
238
|
+
"rung": "claude-fable-5-1#xhigh",
|
|
239
|
+
"dim": "agentic",
|
|
240
|
+
"value": 0.1384,
|
|
241
|
+
"benchmark": "Arena agent, net improvement",
|
|
242
|
+
"version": "2026-09-27",
|
|
243
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
244
|
+
"date": "2026-09-27",
|
|
245
|
+
"confidence": "adjacent",
|
|
246
|
+
"source": "arena",
|
|
247
|
+
"note": "arena has it at max; carried to this effort"
|
|
248
|
+
},
|
|
249
|
+
{
|
|
250
|
+
"rung": "claude-fable-5-1#xhigh",
|
|
251
|
+
"dim": "steer",
|
|
252
|
+
"value": 0.0832,
|
|
253
|
+
"benchmark": "Arena agent steerability",
|
|
254
|
+
"version": "2026-09-27",
|
|
255
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
256
|
+
"date": "2026-09-27",
|
|
257
|
+
"confidence": "adjacent",
|
|
258
|
+
"source": "arena",
|
|
259
|
+
"note": "arena has it at max; carried to this effort"
|
|
260
|
+
},
|
|
261
|
+
{
|
|
262
|
+
"rung": "claude-fable-5-1#xhigh",
|
|
263
|
+
"dim": "frontend",
|
|
264
|
+
"value": 1751.4868,
|
|
265
|
+
"benchmark": "Arena WebDev rating",
|
|
266
|
+
"version": "2026-09-25",
|
|
267
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
268
|
+
"date": "2026-09-25",
|
|
269
|
+
"confidence": "adjacent",
|
|
270
|
+
"source": "arena",
|
|
271
|
+
"note": "arena has it at max; carried to this effort"
|
|
272
|
+
},
|
|
273
|
+
{
|
|
274
|
+
"rung": "claude-haiku-4-5#default",
|
|
275
|
+
"dim": "frontend",
|
|
276
|
+
"value": 1337.5807,
|
|
277
|
+
"benchmark": "WebDev Arena (Epoch AI)",
|
|
278
|
+
"version": "2026-09-28",
|
|
279
|
+
"url": "https://arena.ai/leaderboard",
|
|
280
|
+
"date": "2026-09-28",
|
|
281
|
+
"confidence": "calibrated",
|
|
282
|
+
"source": "epoch",
|
|
283
|
+
"effortAssumed": true,
|
|
284
|
+
"fit": {
|
|
285
|
+
"source": "epoch",
|
|
286
|
+
"field": "webdev",
|
|
287
|
+
"a": 0.9655664507608555,
|
|
288
|
+
"b": 54.04359475979754,
|
|
289
|
+
"r2": 0.9519443341786422,
|
|
290
|
+
"n": 61
|
|
291
|
+
}
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
"rung": "claude-opus-5-5#high",
|
|
295
|
+
"dim": "repo_code",
|
|
296
|
+
"value": 74.2,
|
|
297
|
+
"benchmark": "DeepSWE",
|
|
298
|
+
"version": "1.1",
|
|
299
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
300
|
+
"date": "2026-09-22",
|
|
301
|
+
"confidence": "adjacent",
|
|
302
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
303
|
+
},
|
|
304
|
+
{
|
|
305
|
+
"rung": "claude-opus-5-5#high",
|
|
306
|
+
"dim": "terminal",
|
|
307
|
+
"value": 66.4,
|
|
308
|
+
"benchmark": "Terminal-Bench",
|
|
309
|
+
"version": "4.0",
|
|
310
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
311
|
+
"date": "2026-09-22",
|
|
312
|
+
"confidence": "adjacent",
|
|
313
|
+
"note": "Terminal-Bench has it at xhigh; carried to this effort"
|
|
314
|
+
},
|
|
315
|
+
{
|
|
316
|
+
"rung": "claude-opus-5-5#high",
|
|
317
|
+
"dim": "agentic",
|
|
318
|
+
"value": 0.1215,
|
|
319
|
+
"benchmark": "Arena agent, net improvement",
|
|
320
|
+
"version": "2026-09-27",
|
|
321
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
322
|
+
"date": "2026-09-27",
|
|
323
|
+
"confidence": "measured",
|
|
324
|
+
"source": "arena"
|
|
325
|
+
},
|
|
326
|
+
{
|
|
327
|
+
"rung": "claude-opus-5-5#high",
|
|
328
|
+
"dim": "steer",
|
|
329
|
+
"value": 0.145,
|
|
330
|
+
"benchmark": "Arena agent steerability",
|
|
331
|
+
"version": "2026-09-27",
|
|
332
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
333
|
+
"date": "2026-09-27",
|
|
334
|
+
"confidence": "measured",
|
|
335
|
+
"source": "arena"
|
|
336
|
+
},
|
|
337
|
+
{
|
|
338
|
+
"rung": "claude-opus-5-5#high",
|
|
339
|
+
"dim": "frontend",
|
|
340
|
+
"value": 1826.7307,
|
|
341
|
+
"benchmark": "Arena WebDev rating",
|
|
342
|
+
"version": "2026-09-25",
|
|
343
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
344
|
+
"date": "2026-09-25",
|
|
345
|
+
"confidence": "adjacent",
|
|
346
|
+
"source": "arena",
|
|
347
|
+
"note": "arena has it at max; carried to this effort"
|
|
348
|
+
},
|
|
349
|
+
{
|
|
350
|
+
"rung": "claude-opus-5-5#low",
|
|
351
|
+
"dim": "repo_code",
|
|
352
|
+
"value": 74.2,
|
|
353
|
+
"benchmark": "DeepSWE",
|
|
354
|
+
"version": "1.1",
|
|
355
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
356
|
+
"date": "2026-09-22",
|
|
357
|
+
"confidence": "adjacent",
|
|
358
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
359
|
+
},
|
|
360
|
+
{
|
|
361
|
+
"rung": "claude-opus-5-5#low",
|
|
362
|
+
"dim": "terminal",
|
|
363
|
+
"value": 66.4,
|
|
364
|
+
"benchmark": "Terminal-Bench",
|
|
365
|
+
"version": "4.0",
|
|
366
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
367
|
+
"date": "2026-09-22",
|
|
368
|
+
"confidence": "adjacent",
|
|
369
|
+
"note": "Terminal-Bench has it at xhigh; carried to this effort"
|
|
370
|
+
},
|
|
371
|
+
{
|
|
372
|
+
"rung": "claude-opus-5-5#low",
|
|
373
|
+
"dim": "agentic",
|
|
374
|
+
"value": 0.1215,
|
|
375
|
+
"benchmark": "Arena agent, net improvement",
|
|
376
|
+
"version": "2026-09-27",
|
|
377
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
378
|
+
"date": "2026-09-27",
|
|
379
|
+
"confidence": "adjacent",
|
|
380
|
+
"source": "arena",
|
|
381
|
+
"note": "arena has it at high; carried to this effort"
|
|
382
|
+
},
|
|
383
|
+
{
|
|
384
|
+
"rung": "claude-opus-5-5#low",
|
|
385
|
+
"dim": "steer",
|
|
386
|
+
"value": 0.145,
|
|
387
|
+
"benchmark": "Arena agent steerability",
|
|
388
|
+
"version": "2026-09-27",
|
|
389
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
390
|
+
"date": "2026-09-27",
|
|
391
|
+
"confidence": "adjacent",
|
|
392
|
+
"source": "arena",
|
|
393
|
+
"note": "arena has it at high; carried to this effort"
|
|
394
|
+
},
|
|
395
|
+
{
|
|
396
|
+
"rung": "claude-opus-5-5#low",
|
|
397
|
+
"dim": "frontend",
|
|
398
|
+
"value": 1826.7307,
|
|
399
|
+
"benchmark": "Arena WebDev rating",
|
|
400
|
+
"version": "2026-09-25",
|
|
401
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
402
|
+
"date": "2026-09-25",
|
|
403
|
+
"confidence": "adjacent",
|
|
404
|
+
"source": "arena",
|
|
405
|
+
"note": "arena has it at max; carried to this effort"
|
|
406
|
+
},
|
|
407
|
+
{
|
|
408
|
+
"rung": "claude-opus-5-5#max",
|
|
409
|
+
"dim": "repo_code",
|
|
410
|
+
"value": 74.2,
|
|
411
|
+
"benchmark": "DeepSWE",
|
|
412
|
+
"version": "1.1",
|
|
413
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
414
|
+
"date": "2026-09-22",
|
|
415
|
+
"confidence": "verified",
|
|
416
|
+
"note": "vendor-run, mean of 5 trials (system card 8.3)"
|
|
417
|
+
},
|
|
418
|
+
{
|
|
419
|
+
"rung": "claude-opus-5-5#max",
|
|
420
|
+
"dim": "terminal",
|
|
421
|
+
"value": 64.8,
|
|
422
|
+
"benchmark": "Terminal-Bench",
|
|
423
|
+
"version": "4.0",
|
|
424
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
425
|
+
"date": "2026-09-22",
|
|
426
|
+
"confidence": "verified"
|
|
427
|
+
},
|
|
428
|
+
{
|
|
429
|
+
"rung": "claude-opus-5-5#max",
|
|
430
|
+
"dim": "agentic",
|
|
431
|
+
"value": 0.1215,
|
|
432
|
+
"benchmark": "Arena agent, net improvement",
|
|
433
|
+
"version": "2026-09-27",
|
|
434
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
435
|
+
"date": "2026-09-27",
|
|
436
|
+
"confidence": "adjacent",
|
|
437
|
+
"source": "arena",
|
|
438
|
+
"note": "arena has it at high; carried to this effort"
|
|
439
|
+
},
|
|
440
|
+
{
|
|
441
|
+
"rung": "claude-opus-5-5#max",
|
|
442
|
+
"dim": "steer",
|
|
443
|
+
"value": 0.145,
|
|
444
|
+
"benchmark": "Arena agent steerability",
|
|
445
|
+
"version": "2026-09-27",
|
|
446
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
447
|
+
"date": "2026-09-27",
|
|
448
|
+
"confidence": "adjacent",
|
|
449
|
+
"source": "arena",
|
|
450
|
+
"note": "arena has it at high; carried to this effort"
|
|
451
|
+
},
|
|
452
|
+
{
|
|
453
|
+
"rung": "claude-opus-5-5#max",
|
|
454
|
+
"dim": "frontend",
|
|
455
|
+
"value": 1826.7307,
|
|
456
|
+
"benchmark": "Arena WebDev rating",
|
|
457
|
+
"version": "2026-09-25",
|
|
458
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
459
|
+
"date": "2026-09-25",
|
|
460
|
+
"confidence": "measured",
|
|
461
|
+
"source": "arena"
|
|
462
|
+
},
|
|
463
|
+
{
|
|
464
|
+
"rung": "claude-opus-5-5#medium",
|
|
465
|
+
"dim": "repo_code",
|
|
466
|
+
"value": 74.2,
|
|
467
|
+
"benchmark": "DeepSWE",
|
|
468
|
+
"version": "1.1",
|
|
469
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
470
|
+
"date": "2026-09-22",
|
|
471
|
+
"confidence": "adjacent",
|
|
472
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
473
|
+
},
|
|
474
|
+
{
|
|
475
|
+
"rung": "claude-opus-5-5#medium",
|
|
476
|
+
"dim": "terminal",
|
|
477
|
+
"value": 66.4,
|
|
478
|
+
"benchmark": "Terminal-Bench",
|
|
479
|
+
"version": "4.0",
|
|
480
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
481
|
+
"date": "2026-09-22",
|
|
482
|
+
"confidence": "adjacent",
|
|
483
|
+
"note": "Terminal-Bench has it at xhigh; carried to this effort"
|
|
484
|
+
},
|
|
485
|
+
{
|
|
486
|
+
"rung": "claude-opus-5-5#medium",
|
|
487
|
+
"dim": "agentic",
|
|
488
|
+
"value": 0.1215,
|
|
489
|
+
"benchmark": "Arena agent, net improvement",
|
|
490
|
+
"version": "2026-09-27",
|
|
491
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
492
|
+
"date": "2026-09-27",
|
|
493
|
+
"confidence": "adjacent",
|
|
494
|
+
"source": "arena",
|
|
495
|
+
"note": "arena has it at high; carried to this effort"
|
|
496
|
+
},
|
|
497
|
+
{
|
|
498
|
+
"rung": "claude-opus-5-5#medium",
|
|
499
|
+
"dim": "steer",
|
|
500
|
+
"value": 0.145,
|
|
501
|
+
"benchmark": "Arena agent steerability",
|
|
502
|
+
"version": "2026-09-27",
|
|
503
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
504
|
+
"date": "2026-09-27",
|
|
505
|
+
"confidence": "adjacent",
|
|
506
|
+
"source": "arena",
|
|
507
|
+
"note": "arena has it at high; carried to this effort"
|
|
508
|
+
},
|
|
509
|
+
{
|
|
510
|
+
"rung": "claude-opus-5-5#medium",
|
|
511
|
+
"dim": "frontend",
|
|
512
|
+
"value": 1826.7307,
|
|
513
|
+
"benchmark": "Arena WebDev rating",
|
|
514
|
+
"version": "2026-09-25",
|
|
515
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
516
|
+
"date": "2026-09-25",
|
|
517
|
+
"confidence": "adjacent",
|
|
518
|
+
"source": "arena",
|
|
519
|
+
"note": "arena has it at max; carried to this effort"
|
|
520
|
+
},
|
|
521
|
+
{
|
|
522
|
+
"rung": "claude-opus-5-5#xhigh",
|
|
523
|
+
"dim": "repo_code",
|
|
524
|
+
"value": 74.2,
|
|
525
|
+
"benchmark": "DeepSWE",
|
|
526
|
+
"version": "1.1",
|
|
527
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
528
|
+
"date": "2026-09-22",
|
|
529
|
+
"confidence": "adjacent",
|
|
530
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
531
|
+
},
|
|
532
|
+
{
|
|
533
|
+
"rung": "claude-opus-5-5#xhigh",
|
|
534
|
+
"dim": "terminal",
|
|
535
|
+
"value": 66.4,
|
|
536
|
+
"benchmark": "Terminal-Bench",
|
|
537
|
+
"version": "4.0",
|
|
538
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
539
|
+
"date": "2026-09-22",
|
|
540
|
+
"confidence": "verified",
|
|
541
|
+
"note": "Claude Code --bare, 5 trials (system card 8.5)"
|
|
542
|
+
},
|
|
543
|
+
{
|
|
544
|
+
"rung": "claude-opus-5-5#xhigh",
|
|
545
|
+
"dim": "agentic",
|
|
546
|
+
"value": 0.1215,
|
|
547
|
+
"benchmark": "Arena agent, net improvement",
|
|
548
|
+
"version": "2026-09-27",
|
|
549
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
550
|
+
"date": "2026-09-27",
|
|
551
|
+
"confidence": "adjacent",
|
|
552
|
+
"source": "arena",
|
|
553
|
+
"note": "arena has it at high; carried to this effort"
|
|
554
|
+
},
|
|
555
|
+
{
|
|
556
|
+
"rung": "claude-opus-5-5#xhigh",
|
|
557
|
+
"dim": "steer",
|
|
558
|
+
"value": 0.145,
|
|
559
|
+
"benchmark": "Arena agent steerability",
|
|
560
|
+
"version": "2026-09-27",
|
|
561
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
562
|
+
"date": "2026-09-27",
|
|
563
|
+
"confidence": "adjacent",
|
|
564
|
+
"source": "arena",
|
|
565
|
+
"note": "arena has it at high; carried to this effort"
|
|
566
|
+
},
|
|
567
|
+
{
|
|
568
|
+
"rung": "claude-opus-5-5#xhigh",
|
|
569
|
+
"dim": "frontend",
|
|
570
|
+
"value": 1826.7307,
|
|
571
|
+
"benchmark": "Arena WebDev rating",
|
|
572
|
+
"version": "2026-09-25",
|
|
573
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
574
|
+
"date": "2026-09-25",
|
|
575
|
+
"confidence": "adjacent",
|
|
576
|
+
"source": "arena",
|
|
577
|
+
"note": "arena has it at max; carried to this effort"
|
|
578
|
+
},
|
|
579
|
+
{
|
|
580
|
+
"rung": "claude-sonnet-5#high",
|
|
581
|
+
"dim": "repo_code",
|
|
582
|
+
"value": 54,
|
|
583
|
+
"benchmark": "DeepSWE",
|
|
584
|
+
"version": "1.1",
|
|
585
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
586
|
+
"date": "2026-09-22",
|
|
587
|
+
"confidence": "adjacent",
|
|
588
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
589
|
+
},
|
|
590
|
+
{
|
|
591
|
+
"rung": "claude-sonnet-5#high",
|
|
592
|
+
"dim": "agentic",
|
|
593
|
+
"value": 0.0482,
|
|
594
|
+
"benchmark": "Arena agent, net improvement",
|
|
595
|
+
"version": "2026-09-27",
|
|
596
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
597
|
+
"date": "2026-09-27",
|
|
598
|
+
"confidence": "measured",
|
|
599
|
+
"source": "arena"
|
|
600
|
+
},
|
|
601
|
+
{
|
|
602
|
+
"rung": "claude-sonnet-5#high",
|
|
603
|
+
"dim": "steer",
|
|
604
|
+
"value": 0.0744,
|
|
605
|
+
"benchmark": "Arena agent steerability",
|
|
606
|
+
"version": "2026-09-27",
|
|
607
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
608
|
+
"date": "2026-09-27",
|
|
609
|
+
"confidence": "measured",
|
|
610
|
+
"source": "arena"
|
|
611
|
+
},
|
|
612
|
+
{
|
|
613
|
+
"rung": "claude-sonnet-5#high",
|
|
614
|
+
"dim": "frontend",
|
|
615
|
+
"value": 1538.9132,
|
|
616
|
+
"benchmark": "Arena WebDev rating",
|
|
617
|
+
"version": "2026-09-25",
|
|
618
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
619
|
+
"date": "2026-09-25",
|
|
620
|
+
"confidence": "measured",
|
|
621
|
+
"source": "arena"
|
|
622
|
+
},
|
|
623
|
+
{
|
|
624
|
+
"rung": "claude-sonnet-5#low",
|
|
625
|
+
"dim": "repo_code",
|
|
626
|
+
"value": 54,
|
|
627
|
+
"benchmark": "DeepSWE",
|
|
628
|
+
"version": "1.1",
|
|
629
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
630
|
+
"date": "2026-09-22",
|
|
631
|
+
"confidence": "adjacent",
|
|
632
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
633
|
+
},
|
|
634
|
+
{
|
|
635
|
+
"rung": "claude-sonnet-5#low",
|
|
636
|
+
"dim": "agentic",
|
|
637
|
+
"value": 0.0482,
|
|
638
|
+
"benchmark": "Arena agent, net improvement",
|
|
639
|
+
"version": "2026-09-27",
|
|
640
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
641
|
+
"date": "2026-09-27",
|
|
642
|
+
"confidence": "adjacent",
|
|
643
|
+
"source": "arena",
|
|
644
|
+
"note": "arena has it at high; carried to this effort"
|
|
645
|
+
},
|
|
646
|
+
{
|
|
647
|
+
"rung": "claude-sonnet-5#low",
|
|
648
|
+
"dim": "steer",
|
|
649
|
+
"value": 0.0744,
|
|
650
|
+
"benchmark": "Arena agent steerability",
|
|
651
|
+
"version": "2026-09-27",
|
|
652
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
653
|
+
"date": "2026-09-27",
|
|
654
|
+
"confidence": "adjacent",
|
|
655
|
+
"source": "arena",
|
|
656
|
+
"note": "arena has it at high; carried to this effort"
|
|
657
|
+
},
|
|
658
|
+
{
|
|
659
|
+
"rung": "claude-sonnet-5#low",
|
|
660
|
+
"dim": "frontend",
|
|
661
|
+
"value": 1538.9132,
|
|
662
|
+
"benchmark": "Arena WebDev rating",
|
|
663
|
+
"version": "2026-09-25",
|
|
664
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
665
|
+
"date": "2026-09-25",
|
|
666
|
+
"confidence": "adjacent",
|
|
667
|
+
"source": "arena",
|
|
668
|
+
"note": "arena has it at high; carried to this effort"
|
|
669
|
+
},
|
|
670
|
+
{
|
|
671
|
+
"rung": "claude-sonnet-5#max",
|
|
672
|
+
"dim": "repo_code",
|
|
673
|
+
"value": 54,
|
|
674
|
+
"benchmark": "DeepSWE",
|
|
675
|
+
"version": "1.1",
|
|
676
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
677
|
+
"date": "2026-09-22",
|
|
678
|
+
"confidence": "verified"
|
|
679
|
+
},
|
|
680
|
+
{
|
|
681
|
+
"rung": "claude-sonnet-5#max",
|
|
682
|
+
"dim": "agentic",
|
|
683
|
+
"value": 0.0482,
|
|
684
|
+
"benchmark": "Arena agent, net improvement",
|
|
685
|
+
"version": "2026-09-27",
|
|
686
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
687
|
+
"date": "2026-09-27",
|
|
688
|
+
"confidence": "adjacent",
|
|
689
|
+
"source": "arena",
|
|
690
|
+
"note": "arena has it at high; carried to this effort"
|
|
691
|
+
},
|
|
692
|
+
{
|
|
693
|
+
"rung": "claude-sonnet-5#max",
|
|
694
|
+
"dim": "steer",
|
|
695
|
+
"value": 0.0744,
|
|
696
|
+
"benchmark": "Arena agent steerability",
|
|
697
|
+
"version": "2026-09-27",
|
|
698
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
699
|
+
"date": "2026-09-27",
|
|
700
|
+
"confidence": "adjacent",
|
|
701
|
+
"source": "arena",
|
|
702
|
+
"note": "arena has it at high; carried to this effort"
|
|
703
|
+
},
|
|
704
|
+
{
|
|
705
|
+
"rung": "claude-sonnet-5#max",
|
|
706
|
+
"dim": "frontend",
|
|
707
|
+
"value": 1538.9132,
|
|
708
|
+
"benchmark": "Arena WebDev rating",
|
|
709
|
+
"version": "2026-09-25",
|
|
710
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
711
|
+
"date": "2026-09-25",
|
|
712
|
+
"confidence": "adjacent",
|
|
713
|
+
"source": "arena",
|
|
714
|
+
"note": "arena has it at high; carried to this effort"
|
|
715
|
+
},
|
|
716
|
+
{
|
|
717
|
+
"rung": "claude-sonnet-5#medium",
|
|
718
|
+
"dim": "repo_code",
|
|
719
|
+
"value": 54,
|
|
720
|
+
"benchmark": "DeepSWE",
|
|
721
|
+
"version": "1.1",
|
|
722
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
723
|
+
"date": "2026-09-22",
|
|
724
|
+
"confidence": "adjacent",
|
|
725
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
726
|
+
},
|
|
727
|
+
{
|
|
728
|
+
"rung": "claude-sonnet-5#medium",
|
|
729
|
+
"dim": "agentic",
|
|
730
|
+
"value": 0.0482,
|
|
731
|
+
"benchmark": "Arena agent, net improvement",
|
|
732
|
+
"version": "2026-09-27",
|
|
733
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
734
|
+
"date": "2026-09-27",
|
|
735
|
+
"confidence": "adjacent",
|
|
736
|
+
"source": "arena",
|
|
737
|
+
"note": "arena has it at high; carried to this effort"
|
|
738
|
+
},
|
|
739
|
+
{
|
|
740
|
+
"rung": "claude-sonnet-5#medium",
|
|
741
|
+
"dim": "steer",
|
|
742
|
+
"value": 0.0744,
|
|
743
|
+
"benchmark": "Arena agent steerability",
|
|
744
|
+
"version": "2026-09-27",
|
|
745
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
746
|
+
"date": "2026-09-27",
|
|
747
|
+
"confidence": "adjacent",
|
|
748
|
+
"source": "arena",
|
|
749
|
+
"note": "arena has it at high; carried to this effort"
|
|
750
|
+
},
|
|
751
|
+
{
|
|
752
|
+
"rung": "claude-sonnet-5#medium",
|
|
753
|
+
"dim": "frontend",
|
|
754
|
+
"value": 1538.9132,
|
|
755
|
+
"benchmark": "Arena WebDev rating",
|
|
756
|
+
"version": "2026-09-25",
|
|
757
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
758
|
+
"date": "2026-09-25",
|
|
759
|
+
"confidence": "adjacent",
|
|
760
|
+
"source": "arena",
|
|
761
|
+
"note": "arena has it at high; carried to this effort"
|
|
762
|
+
},
|
|
763
|
+
{
|
|
764
|
+
"rung": "claude-sonnet-5#xhigh",
|
|
765
|
+
"dim": "repo_code",
|
|
766
|
+
"value": 54,
|
|
767
|
+
"benchmark": "DeepSWE",
|
|
768
|
+
"version": "1.1",
|
|
769
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
770
|
+
"date": "2026-09-22",
|
|
771
|
+
"confidence": "adjacent",
|
|
772
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
773
|
+
},
|
|
774
|
+
{
|
|
775
|
+
"rung": "claude-sonnet-5#xhigh",
|
|
776
|
+
"dim": "agentic",
|
|
777
|
+
"value": 0.0482,
|
|
778
|
+
"benchmark": "Arena agent, net improvement",
|
|
779
|
+
"version": "2026-09-27",
|
|
780
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
781
|
+
"date": "2026-09-27",
|
|
782
|
+
"confidence": "adjacent",
|
|
783
|
+
"source": "arena",
|
|
784
|
+
"note": "arena has it at high; carried to this effort"
|
|
785
|
+
},
|
|
786
|
+
{
|
|
787
|
+
"rung": "claude-sonnet-5#xhigh",
|
|
788
|
+
"dim": "steer",
|
|
789
|
+
"value": 0.0744,
|
|
790
|
+
"benchmark": "Arena agent steerability",
|
|
791
|
+
"version": "2026-09-27",
|
|
792
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
793
|
+
"date": "2026-09-27",
|
|
794
|
+
"confidence": "adjacent",
|
|
795
|
+
"source": "arena",
|
|
796
|
+
"note": "arena has it at high; carried to this effort"
|
|
797
|
+
},
|
|
798
|
+
{
|
|
799
|
+
"rung": "claude-sonnet-5#xhigh",
|
|
800
|
+
"dim": "frontend",
|
|
801
|
+
"value": 1538.9132,
|
|
802
|
+
"benchmark": "Arena WebDev rating",
|
|
803
|
+
"version": "2026-09-25",
|
|
804
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
805
|
+
"date": "2026-09-25",
|
|
806
|
+
"confidence": "adjacent",
|
|
807
|
+
"source": "arena",
|
|
808
|
+
"note": "arena has it at high; carried to this effort"
|
|
809
|
+
},
|
|
810
|
+
{
|
|
811
|
+
"rung": "gpt-5.6-luna#high",
|
|
812
|
+
"dim": "repo_code",
|
|
813
|
+
"value": 67,
|
|
814
|
+
"benchmark": "DeepSWE",
|
|
815
|
+
"version": "1.1",
|
|
816
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
817
|
+
"date": "2026-09-22",
|
|
818
|
+
"confidence": "adjacent",
|
|
819
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
820
|
+
},
|
|
821
|
+
{
|
|
822
|
+
"rung": "gpt-5.6-luna#high",
|
|
823
|
+
"dim": "honesty",
|
|
824
|
+
"value": 21.8,
|
|
825
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
826
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
827
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
828
|
+
"date": "2026-09-22",
|
|
829
|
+
"confidence": "adjacent",
|
|
830
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
831
|
+
},
|
|
832
|
+
{
|
|
833
|
+
"rung": "gpt-5.6-luna#high",
|
|
834
|
+
"dim": "agentic",
|
|
835
|
+
"value": -0.0075,
|
|
836
|
+
"benchmark": "Arena agent, net improvement",
|
|
837
|
+
"version": "2026-09-27",
|
|
838
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
839
|
+
"date": "2026-09-27",
|
|
840
|
+
"confidence": "adjacent",
|
|
841
|
+
"source": "arena",
|
|
842
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
843
|
+
},
|
|
844
|
+
{
|
|
845
|
+
"rung": "gpt-5.6-luna#high",
|
|
846
|
+
"dim": "steer",
|
|
847
|
+
"value": 0.0185,
|
|
848
|
+
"benchmark": "Arena agent steerability",
|
|
849
|
+
"version": "2026-09-27",
|
|
850
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
851
|
+
"date": "2026-09-27",
|
|
852
|
+
"confidence": "adjacent",
|
|
853
|
+
"source": "arena",
|
|
854
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
855
|
+
},
|
|
856
|
+
{
|
|
857
|
+
"rung": "gpt-5.6-luna#high",
|
|
858
|
+
"dim": "frontend",
|
|
859
|
+
"value": 1519.8875,
|
|
860
|
+
"benchmark": "Arena WebDev rating",
|
|
861
|
+
"version": "2026-09-25",
|
|
862
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
863
|
+
"date": "2026-09-25",
|
|
864
|
+
"confidence": "adjacent",
|
|
865
|
+
"source": "arena",
|
|
866
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
867
|
+
},
|
|
868
|
+
{
|
|
869
|
+
"rung": "gpt-5.6-luna#low",
|
|
870
|
+
"dim": "repo_code",
|
|
871
|
+
"value": 67,
|
|
872
|
+
"benchmark": "DeepSWE",
|
|
873
|
+
"version": "1.1",
|
|
874
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
875
|
+
"date": "2026-09-22",
|
|
876
|
+
"confidence": "adjacent",
|
|
877
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
878
|
+
},
|
|
879
|
+
{
|
|
880
|
+
"rung": "gpt-5.6-luna#low",
|
|
881
|
+
"dim": "honesty",
|
|
882
|
+
"value": 21.8,
|
|
883
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
884
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
885
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
886
|
+
"date": "2026-09-22",
|
|
887
|
+
"confidence": "adjacent",
|
|
888
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
889
|
+
},
|
|
890
|
+
{
|
|
891
|
+
"rung": "gpt-5.6-luna#low",
|
|
892
|
+
"dim": "agentic",
|
|
893
|
+
"value": -0.0075,
|
|
894
|
+
"benchmark": "Arena agent, net improvement",
|
|
895
|
+
"version": "2026-09-27",
|
|
896
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
897
|
+
"date": "2026-09-27",
|
|
898
|
+
"confidence": "adjacent",
|
|
899
|
+
"source": "arena",
|
|
900
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
901
|
+
},
|
|
902
|
+
{
|
|
903
|
+
"rung": "gpt-5.6-luna#low",
|
|
904
|
+
"dim": "steer",
|
|
905
|
+
"value": 0.0185,
|
|
906
|
+
"benchmark": "Arena agent steerability",
|
|
907
|
+
"version": "2026-09-27",
|
|
908
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
909
|
+
"date": "2026-09-27",
|
|
910
|
+
"confidence": "adjacent",
|
|
911
|
+
"source": "arena",
|
|
912
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
913
|
+
},
|
|
914
|
+
{
|
|
915
|
+
"rung": "gpt-5.6-luna#low",
|
|
916
|
+
"dim": "frontend",
|
|
917
|
+
"value": 1519.8875,
|
|
918
|
+
"benchmark": "Arena WebDev rating",
|
|
919
|
+
"version": "2026-09-25",
|
|
920
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
921
|
+
"date": "2026-09-25",
|
|
922
|
+
"confidence": "adjacent",
|
|
923
|
+
"source": "arena",
|
|
924
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
925
|
+
},
|
|
926
|
+
{
|
|
927
|
+
"rung": "gpt-5.6-luna#max",
|
|
928
|
+
"dim": "repo_code",
|
|
929
|
+
"value": 67,
|
|
930
|
+
"benchmark": "DeepSWE",
|
|
931
|
+
"version": "1.1",
|
|
932
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
933
|
+
"date": "2026-09-22",
|
|
934
|
+
"confidence": "verified"
|
|
935
|
+
},
|
|
936
|
+
{
|
|
937
|
+
"rung": "gpt-5.6-luna#max",
|
|
938
|
+
"dim": "honesty",
|
|
939
|
+
"value": 21.8,
|
|
940
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
941
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
942
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
943
|
+
"date": "2026-09-22",
|
|
944
|
+
"confidence": "verified"
|
|
945
|
+
},
|
|
946
|
+
{
|
|
947
|
+
"rung": "gpt-5.6-luna#max",
|
|
948
|
+
"dim": "agentic",
|
|
949
|
+
"value": -0.0075,
|
|
950
|
+
"benchmark": "Arena agent, net improvement",
|
|
951
|
+
"version": "2026-09-27",
|
|
952
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
953
|
+
"date": "2026-09-27",
|
|
954
|
+
"confidence": "adjacent",
|
|
955
|
+
"source": "arena",
|
|
956
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
957
|
+
},
|
|
958
|
+
{
|
|
959
|
+
"rung": "gpt-5.6-luna#max",
|
|
960
|
+
"dim": "steer",
|
|
961
|
+
"value": 0.0185,
|
|
962
|
+
"benchmark": "Arena agent steerability",
|
|
963
|
+
"version": "2026-09-27",
|
|
964
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
965
|
+
"date": "2026-09-27",
|
|
966
|
+
"confidence": "adjacent",
|
|
967
|
+
"source": "arena",
|
|
968
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
969
|
+
},
|
|
970
|
+
{
|
|
971
|
+
"rung": "gpt-5.6-luna#max",
|
|
972
|
+
"dim": "frontend",
|
|
973
|
+
"value": 1519.8875,
|
|
974
|
+
"benchmark": "Arena WebDev rating",
|
|
975
|
+
"version": "2026-09-25",
|
|
976
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
977
|
+
"date": "2026-09-25",
|
|
978
|
+
"confidence": "adjacent",
|
|
979
|
+
"source": "arena",
|
|
980
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
981
|
+
},
|
|
982
|
+
{
|
|
983
|
+
"rung": "gpt-5.6-luna#medium",
|
|
984
|
+
"dim": "repo_code",
|
|
985
|
+
"value": 67,
|
|
986
|
+
"benchmark": "DeepSWE",
|
|
987
|
+
"version": "1.1",
|
|
988
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
989
|
+
"date": "2026-09-22",
|
|
990
|
+
"confidence": "adjacent",
|
|
991
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
992
|
+
},
|
|
993
|
+
{
|
|
994
|
+
"rung": "gpt-5.6-luna#medium",
|
|
995
|
+
"dim": "honesty",
|
|
996
|
+
"value": 21.8,
|
|
997
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
998
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
999
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1000
|
+
"date": "2026-09-22",
|
|
1001
|
+
"confidence": "adjacent",
|
|
1002
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
1003
|
+
},
|
|
1004
|
+
{
|
|
1005
|
+
"rung": "gpt-5.6-luna#medium",
|
|
1006
|
+
"dim": "agentic",
|
|
1007
|
+
"value": -0.0075,
|
|
1008
|
+
"benchmark": "Arena agent, net improvement",
|
|
1009
|
+
"version": "2026-09-27",
|
|
1010
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1011
|
+
"date": "2026-09-27",
|
|
1012
|
+
"confidence": "adjacent",
|
|
1013
|
+
"source": "arena",
|
|
1014
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1015
|
+
},
|
|
1016
|
+
{
|
|
1017
|
+
"rung": "gpt-5.6-luna#medium",
|
|
1018
|
+
"dim": "steer",
|
|
1019
|
+
"value": 0.0185,
|
|
1020
|
+
"benchmark": "Arena agent steerability",
|
|
1021
|
+
"version": "2026-09-27",
|
|
1022
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1023
|
+
"date": "2026-09-27",
|
|
1024
|
+
"confidence": "adjacent",
|
|
1025
|
+
"source": "arena",
|
|
1026
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1027
|
+
},
|
|
1028
|
+
{
|
|
1029
|
+
"rung": "gpt-5.6-luna#medium",
|
|
1030
|
+
"dim": "frontend",
|
|
1031
|
+
"value": 1519.8875,
|
|
1032
|
+
"benchmark": "Arena WebDev rating",
|
|
1033
|
+
"version": "2026-09-25",
|
|
1034
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1035
|
+
"date": "2026-09-25",
|
|
1036
|
+
"confidence": "adjacent",
|
|
1037
|
+
"source": "arena",
|
|
1038
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1039
|
+
},
|
|
1040
|
+
{
|
|
1041
|
+
"rung": "gpt-5.6-luna#none",
|
|
1042
|
+
"dim": "repo_code",
|
|
1043
|
+
"value": 67,
|
|
1044
|
+
"benchmark": "DeepSWE",
|
|
1045
|
+
"version": "1.1",
|
|
1046
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
1047
|
+
"date": "2026-09-22",
|
|
1048
|
+
"confidence": "adjacent",
|
|
1049
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
1050
|
+
},
|
|
1051
|
+
{
|
|
1052
|
+
"rung": "gpt-5.6-luna#none",
|
|
1053
|
+
"dim": "honesty",
|
|
1054
|
+
"value": 21.8,
|
|
1055
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
1056
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
1057
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1058
|
+
"date": "2026-09-22",
|
|
1059
|
+
"confidence": "adjacent",
|
|
1060
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
1061
|
+
},
|
|
1062
|
+
{
|
|
1063
|
+
"rung": "gpt-5.6-luna#none",
|
|
1064
|
+
"dim": "agentic",
|
|
1065
|
+
"value": -0.0075,
|
|
1066
|
+
"benchmark": "Arena agent, net improvement",
|
|
1067
|
+
"version": "2026-09-27",
|
|
1068
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1069
|
+
"date": "2026-09-27",
|
|
1070
|
+
"confidence": "adjacent",
|
|
1071
|
+
"source": "arena",
|
|
1072
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1073
|
+
},
|
|
1074
|
+
{
|
|
1075
|
+
"rung": "gpt-5.6-luna#none",
|
|
1076
|
+
"dim": "steer",
|
|
1077
|
+
"value": 0.0185,
|
|
1078
|
+
"benchmark": "Arena agent steerability",
|
|
1079
|
+
"version": "2026-09-27",
|
|
1080
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1081
|
+
"date": "2026-09-27",
|
|
1082
|
+
"confidence": "adjacent",
|
|
1083
|
+
"source": "arena",
|
|
1084
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1085
|
+
},
|
|
1086
|
+
{
|
|
1087
|
+
"rung": "gpt-5.6-luna#none",
|
|
1088
|
+
"dim": "frontend",
|
|
1089
|
+
"value": 1519.8875,
|
|
1090
|
+
"benchmark": "Arena WebDev rating",
|
|
1091
|
+
"version": "2026-09-25",
|
|
1092
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1093
|
+
"date": "2026-09-25",
|
|
1094
|
+
"confidence": "adjacent",
|
|
1095
|
+
"source": "arena",
|
|
1096
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1097
|
+
},
|
|
1098
|
+
{
|
|
1099
|
+
"rung": "gpt-5.6-luna#xhigh",
|
|
1100
|
+
"dim": "repo_code",
|
|
1101
|
+
"value": 67,
|
|
1102
|
+
"benchmark": "DeepSWE",
|
|
1103
|
+
"version": "1.1",
|
|
1104
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
1105
|
+
"date": "2026-09-22",
|
|
1106
|
+
"confidence": "adjacent",
|
|
1107
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
1108
|
+
},
|
|
1109
|
+
{
|
|
1110
|
+
"rung": "gpt-5.6-luna#xhigh",
|
|
1111
|
+
"dim": "honesty",
|
|
1112
|
+
"value": 21.8,
|
|
1113
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
1114
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
1115
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1116
|
+
"date": "2026-09-22",
|
|
1117
|
+
"confidence": "adjacent",
|
|
1118
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
1119
|
+
},
|
|
1120
|
+
{
|
|
1121
|
+
"rung": "gpt-5.6-luna#xhigh",
|
|
1122
|
+
"dim": "agentic",
|
|
1123
|
+
"value": -0.0075,
|
|
1124
|
+
"benchmark": "Arena agent, net improvement",
|
|
1125
|
+
"version": "2026-09-27",
|
|
1126
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1127
|
+
"date": "2026-09-27",
|
|
1128
|
+
"confidence": "measured",
|
|
1129
|
+
"source": "arena"
|
|
1130
|
+
},
|
|
1131
|
+
{
|
|
1132
|
+
"rung": "gpt-5.6-luna#xhigh",
|
|
1133
|
+
"dim": "steer",
|
|
1134
|
+
"value": 0.0185,
|
|
1135
|
+
"benchmark": "Arena agent steerability",
|
|
1136
|
+
"version": "2026-09-27",
|
|
1137
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1138
|
+
"date": "2026-09-27",
|
|
1139
|
+
"confidence": "measured",
|
|
1140
|
+
"source": "arena"
|
|
1141
|
+
},
|
|
1142
|
+
{
|
|
1143
|
+
"rung": "gpt-5.6-luna#xhigh",
|
|
1144
|
+
"dim": "frontend",
|
|
1145
|
+
"value": 1519.8875,
|
|
1146
|
+
"benchmark": "Arena WebDev rating",
|
|
1147
|
+
"version": "2026-09-25",
|
|
1148
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1149
|
+
"date": "2026-09-25",
|
|
1150
|
+
"confidence": "measured",
|
|
1151
|
+
"source": "arena"
|
|
1152
|
+
},
|
|
1153
|
+
{
|
|
1154
|
+
"rung": "gpt-5.6-sol#high",
|
|
1155
|
+
"dim": "repo_code",
|
|
1156
|
+
"value": 73,
|
|
1157
|
+
"benchmark": "DeepSWE",
|
|
1158
|
+
"version": "1.1",
|
|
1159
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
1160
|
+
"date": "2026-09-22",
|
|
1161
|
+
"confidence": "adjacent",
|
|
1162
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
1163
|
+
},
|
|
1164
|
+
{
|
|
1165
|
+
"rung": "gpt-5.6-sol#high",
|
|
1166
|
+
"dim": "terminal",
|
|
1167
|
+
"value": 37.3,
|
|
1168
|
+
"benchmark": "Terminal-Bench",
|
|
1169
|
+
"version": "4.0",
|
|
1170
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
1171
|
+
"date": "2026-09-22",
|
|
1172
|
+
"confidence": "adjacent",
|
|
1173
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
1174
|
+
},
|
|
1175
|
+
{
|
|
1176
|
+
"rung": "gpt-5.6-sol#high",
|
|
1177
|
+
"dim": "honesty",
|
|
1178
|
+
"value": 22.5,
|
|
1179
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
1180
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
1181
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1182
|
+
"date": "2026-09-22",
|
|
1183
|
+
"confidence": "adjacent",
|
|
1184
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
1185
|
+
},
|
|
1186
|
+
{
|
|
1187
|
+
"rung": "gpt-5.6-sol#high",
|
|
1188
|
+
"dim": "agentic",
|
|
1189
|
+
"value": 0.0621,
|
|
1190
|
+
"benchmark": "Arena agent, net improvement",
|
|
1191
|
+
"version": "2026-09-27",
|
|
1192
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1193
|
+
"date": "2026-09-27",
|
|
1194
|
+
"confidence": "adjacent",
|
|
1195
|
+
"source": "arena",
|
|
1196
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1197
|
+
},
|
|
1198
|
+
{
|
|
1199
|
+
"rung": "gpt-5.6-sol#high",
|
|
1200
|
+
"dim": "steer",
|
|
1201
|
+
"value": 0.0656,
|
|
1202
|
+
"benchmark": "Arena agent steerability",
|
|
1203
|
+
"version": "2026-09-27",
|
|
1204
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1205
|
+
"date": "2026-09-27",
|
|
1206
|
+
"confidence": "adjacent",
|
|
1207
|
+
"source": "arena",
|
|
1208
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1209
|
+
},
|
|
1210
|
+
{
|
|
1211
|
+
"rung": "gpt-5.6-sol#high",
|
|
1212
|
+
"dim": "frontend",
|
|
1213
|
+
"value": 1617.3525,
|
|
1214
|
+
"benchmark": "Arena WebDev rating",
|
|
1215
|
+
"version": "2026-09-25",
|
|
1216
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1217
|
+
"date": "2026-09-25",
|
|
1218
|
+
"confidence": "adjacent",
|
|
1219
|
+
"source": "arena",
|
|
1220
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1221
|
+
},
|
|
1222
|
+
{
|
|
1223
|
+
"rung": "gpt-5.6-sol#low",
|
|
1224
|
+
"dim": "repo_code",
|
|
1225
|
+
"value": 73,
|
|
1226
|
+
"benchmark": "DeepSWE",
|
|
1227
|
+
"version": "1.1",
|
|
1228
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
1229
|
+
"date": "2026-09-22",
|
|
1230
|
+
"confidence": "adjacent",
|
|
1231
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
1232
|
+
},
|
|
1233
|
+
{
|
|
1234
|
+
"rung": "gpt-5.6-sol#low",
|
|
1235
|
+
"dim": "terminal",
|
|
1236
|
+
"value": 37.3,
|
|
1237
|
+
"benchmark": "Terminal-Bench",
|
|
1238
|
+
"version": "4.0",
|
|
1239
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
1240
|
+
"date": "2026-09-22",
|
|
1241
|
+
"confidence": "adjacent",
|
|
1242
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
1243
|
+
},
|
|
1244
|
+
{
|
|
1245
|
+
"rung": "gpt-5.6-sol#low",
|
|
1246
|
+
"dim": "honesty",
|
|
1247
|
+
"value": 22.5,
|
|
1248
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
1249
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
1250
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1251
|
+
"date": "2026-09-22",
|
|
1252
|
+
"confidence": "adjacent",
|
|
1253
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
1254
|
+
},
|
|
1255
|
+
{
|
|
1256
|
+
"rung": "gpt-5.6-sol#low",
|
|
1257
|
+
"dim": "agentic",
|
|
1258
|
+
"value": 0.0621,
|
|
1259
|
+
"benchmark": "Arena agent, net improvement",
|
|
1260
|
+
"version": "2026-09-27",
|
|
1261
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1262
|
+
"date": "2026-09-27",
|
|
1263
|
+
"confidence": "adjacent",
|
|
1264
|
+
"source": "arena",
|
|
1265
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1266
|
+
},
|
|
1267
|
+
{
|
|
1268
|
+
"rung": "gpt-5.6-sol#low",
|
|
1269
|
+
"dim": "steer",
|
|
1270
|
+
"value": 0.0656,
|
|
1271
|
+
"benchmark": "Arena agent steerability",
|
|
1272
|
+
"version": "2026-09-27",
|
|
1273
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1274
|
+
"date": "2026-09-27",
|
|
1275
|
+
"confidence": "adjacent",
|
|
1276
|
+
"source": "arena",
|
|
1277
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1278
|
+
},
|
|
1279
|
+
{
|
|
1280
|
+
"rung": "gpt-5.6-sol#low",
|
|
1281
|
+
"dim": "frontend",
|
|
1282
|
+
"value": 1617.3525,
|
|
1283
|
+
"benchmark": "Arena WebDev rating",
|
|
1284
|
+
"version": "2026-09-25",
|
|
1285
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1286
|
+
"date": "2026-09-25",
|
|
1287
|
+
"confidence": "adjacent",
|
|
1288
|
+
"source": "arena",
|
|
1289
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1290
|
+
},
|
|
1291
|
+
{
|
|
1292
|
+
"rung": "gpt-5.6-sol#max",
|
|
1293
|
+
"dim": "repo_code",
|
|
1294
|
+
"value": 73,
|
|
1295
|
+
"benchmark": "DeepSWE",
|
|
1296
|
+
"version": "1.1",
|
|
1297
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
1298
|
+
"date": "2026-09-22",
|
|
1299
|
+
"confidence": "verified"
|
|
1300
|
+
},
|
|
1301
|
+
{
|
|
1302
|
+
"rung": "gpt-5.6-sol#max",
|
|
1303
|
+
"dim": "terminal",
|
|
1304
|
+
"value": 37.3,
|
|
1305
|
+
"benchmark": "Terminal-Bench",
|
|
1306
|
+
"version": "4.0",
|
|
1307
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
1308
|
+
"date": "2026-09-22",
|
|
1309
|
+
"confidence": "secondary"
|
|
1310
|
+
},
|
|
1311
|
+
{
|
|
1312
|
+
"rung": "gpt-5.6-sol#max",
|
|
1313
|
+
"dim": "honesty",
|
|
1314
|
+
"value": 22.5,
|
|
1315
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
1316
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
1317
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1318
|
+
"date": "2026-09-22",
|
|
1319
|
+
"confidence": "verified"
|
|
1320
|
+
},
|
|
1321
|
+
{
|
|
1322
|
+
"rung": "gpt-5.6-sol#max",
|
|
1323
|
+
"dim": "agentic",
|
|
1324
|
+
"value": 0.0621,
|
|
1325
|
+
"benchmark": "Arena agent, net improvement",
|
|
1326
|
+
"version": "2026-09-27",
|
|
1327
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1328
|
+
"date": "2026-09-27",
|
|
1329
|
+
"confidence": "adjacent",
|
|
1330
|
+
"source": "arena",
|
|
1331
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1332
|
+
},
|
|
1333
|
+
{
|
|
1334
|
+
"rung": "gpt-5.6-sol#max",
|
|
1335
|
+
"dim": "steer",
|
|
1336
|
+
"value": 0.0656,
|
|
1337
|
+
"benchmark": "Arena agent steerability",
|
|
1338
|
+
"version": "2026-09-27",
|
|
1339
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1340
|
+
"date": "2026-09-27",
|
|
1341
|
+
"confidence": "adjacent",
|
|
1342
|
+
"source": "arena",
|
|
1343
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1344
|
+
},
|
|
1345
|
+
{
|
|
1346
|
+
"rung": "gpt-5.6-sol#max",
|
|
1347
|
+
"dim": "frontend",
|
|
1348
|
+
"value": 1617.3525,
|
|
1349
|
+
"benchmark": "Arena WebDev rating",
|
|
1350
|
+
"version": "2026-09-25",
|
|
1351
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1352
|
+
"date": "2026-09-25",
|
|
1353
|
+
"confidence": "adjacent",
|
|
1354
|
+
"source": "arena",
|
|
1355
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1356
|
+
},
|
|
1357
|
+
{
|
|
1358
|
+
"rung": "gpt-5.6-sol#medium",
|
|
1359
|
+
"dim": "repo_code",
|
|
1360
|
+
"value": 73,
|
|
1361
|
+
"benchmark": "DeepSWE",
|
|
1362
|
+
"version": "1.1",
|
|
1363
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
1364
|
+
"date": "2026-09-22",
|
|
1365
|
+
"confidence": "adjacent",
|
|
1366
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
1367
|
+
},
|
|
1368
|
+
{
|
|
1369
|
+
"rung": "gpt-5.6-sol#medium",
|
|
1370
|
+
"dim": "terminal",
|
|
1371
|
+
"value": 37.3,
|
|
1372
|
+
"benchmark": "Terminal-Bench",
|
|
1373
|
+
"version": "4.0",
|
|
1374
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
1375
|
+
"date": "2026-09-22",
|
|
1376
|
+
"confidence": "adjacent",
|
|
1377
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
1378
|
+
},
|
|
1379
|
+
{
|
|
1380
|
+
"rung": "gpt-5.6-sol#medium",
|
|
1381
|
+
"dim": "honesty",
|
|
1382
|
+
"value": 22.5,
|
|
1383
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
1384
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
1385
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1386
|
+
"date": "2026-09-22",
|
|
1387
|
+
"confidence": "adjacent",
|
|
1388
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
1389
|
+
},
|
|
1390
|
+
{
|
|
1391
|
+
"rung": "gpt-5.6-sol#medium",
|
|
1392
|
+
"dim": "agentic",
|
|
1393
|
+
"value": 0.0621,
|
|
1394
|
+
"benchmark": "Arena agent, net improvement",
|
|
1395
|
+
"version": "2026-09-27",
|
|
1396
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1397
|
+
"date": "2026-09-27",
|
|
1398
|
+
"confidence": "adjacent",
|
|
1399
|
+
"source": "arena",
|
|
1400
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1401
|
+
},
|
|
1402
|
+
{
|
|
1403
|
+
"rung": "gpt-5.6-sol#medium",
|
|
1404
|
+
"dim": "steer",
|
|
1405
|
+
"value": 0.0656,
|
|
1406
|
+
"benchmark": "Arena agent steerability",
|
|
1407
|
+
"version": "2026-09-27",
|
|
1408
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1409
|
+
"date": "2026-09-27",
|
|
1410
|
+
"confidence": "adjacent",
|
|
1411
|
+
"source": "arena",
|
|
1412
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1413
|
+
},
|
|
1414
|
+
{
|
|
1415
|
+
"rung": "gpt-5.6-sol#medium",
|
|
1416
|
+
"dim": "frontend",
|
|
1417
|
+
"value": 1617.3525,
|
|
1418
|
+
"benchmark": "Arena WebDev rating",
|
|
1419
|
+
"version": "2026-09-25",
|
|
1420
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1421
|
+
"date": "2026-09-25",
|
|
1422
|
+
"confidence": "adjacent",
|
|
1423
|
+
"source": "arena",
|
|
1424
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1425
|
+
},
|
|
1426
|
+
{
|
|
1427
|
+
"rung": "gpt-5.6-sol#none",
|
|
1428
|
+
"dim": "repo_code",
|
|
1429
|
+
"value": 73,
|
|
1430
|
+
"benchmark": "DeepSWE",
|
|
1431
|
+
"version": "1.1",
|
|
1432
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
1433
|
+
"date": "2026-09-22",
|
|
1434
|
+
"confidence": "adjacent",
|
|
1435
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
1436
|
+
},
|
|
1437
|
+
{
|
|
1438
|
+
"rung": "gpt-5.6-sol#none",
|
|
1439
|
+
"dim": "terminal",
|
|
1440
|
+
"value": 37.3,
|
|
1441
|
+
"benchmark": "Terminal-Bench",
|
|
1442
|
+
"version": "4.0",
|
|
1443
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
1444
|
+
"date": "2026-09-22",
|
|
1445
|
+
"confidence": "adjacent",
|
|
1446
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
1447
|
+
},
|
|
1448
|
+
{
|
|
1449
|
+
"rung": "gpt-5.6-sol#none",
|
|
1450
|
+
"dim": "honesty",
|
|
1451
|
+
"value": 22.5,
|
|
1452
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
1453
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
1454
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1455
|
+
"date": "2026-09-22",
|
|
1456
|
+
"confidence": "adjacent",
|
|
1457
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
1458
|
+
},
|
|
1459
|
+
{
|
|
1460
|
+
"rung": "gpt-5.6-sol#none",
|
|
1461
|
+
"dim": "agentic",
|
|
1462
|
+
"value": 0.0621,
|
|
1463
|
+
"benchmark": "Arena agent, net improvement",
|
|
1464
|
+
"version": "2026-09-27",
|
|
1465
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1466
|
+
"date": "2026-09-27",
|
|
1467
|
+
"confidence": "adjacent",
|
|
1468
|
+
"source": "arena",
|
|
1469
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1470
|
+
},
|
|
1471
|
+
{
|
|
1472
|
+
"rung": "gpt-5.6-sol#none",
|
|
1473
|
+
"dim": "steer",
|
|
1474
|
+
"value": 0.0656,
|
|
1475
|
+
"benchmark": "Arena agent steerability",
|
|
1476
|
+
"version": "2026-09-27",
|
|
1477
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1478
|
+
"date": "2026-09-27",
|
|
1479
|
+
"confidence": "adjacent",
|
|
1480
|
+
"source": "arena",
|
|
1481
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1482
|
+
},
|
|
1483
|
+
{
|
|
1484
|
+
"rung": "gpt-5.6-sol#none",
|
|
1485
|
+
"dim": "frontend",
|
|
1486
|
+
"value": 1617.3525,
|
|
1487
|
+
"benchmark": "Arena WebDev rating",
|
|
1488
|
+
"version": "2026-09-25",
|
|
1489
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1490
|
+
"date": "2026-09-25",
|
|
1491
|
+
"confidence": "adjacent",
|
|
1492
|
+
"source": "arena",
|
|
1493
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1494
|
+
},
|
|
1495
|
+
{
|
|
1496
|
+
"rung": "gpt-5.6-sol#ultra",
|
|
1497
|
+
"dim": "repo_code",
|
|
1498
|
+
"value": 73,
|
|
1499
|
+
"benchmark": "DeepSWE",
|
|
1500
|
+
"version": "1.1",
|
|
1501
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
1502
|
+
"date": "2026-09-22",
|
|
1503
|
+
"confidence": "adjacent",
|
|
1504
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
1505
|
+
},
|
|
1506
|
+
{
|
|
1507
|
+
"rung": "gpt-5.6-sol#ultra",
|
|
1508
|
+
"dim": "terminal",
|
|
1509
|
+
"value": 37.3,
|
|
1510
|
+
"benchmark": "Terminal-Bench",
|
|
1511
|
+
"version": "4.0",
|
|
1512
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
1513
|
+
"date": "2026-09-22",
|
|
1514
|
+
"confidence": "adjacent",
|
|
1515
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
1516
|
+
},
|
|
1517
|
+
{
|
|
1518
|
+
"rung": "gpt-5.6-sol#ultra",
|
|
1519
|
+
"dim": "honesty",
|
|
1520
|
+
"value": 22.5,
|
|
1521
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
1522
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
1523
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1524
|
+
"date": "2026-09-22",
|
|
1525
|
+
"confidence": "adjacent",
|
|
1526
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
1527
|
+
},
|
|
1528
|
+
{
|
|
1529
|
+
"rung": "gpt-5.6-sol#ultra",
|
|
1530
|
+
"dim": "agentic",
|
|
1531
|
+
"value": 0.0621,
|
|
1532
|
+
"benchmark": "Arena agent, net improvement",
|
|
1533
|
+
"version": "2026-09-27",
|
|
1534
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1535
|
+
"date": "2026-09-27",
|
|
1536
|
+
"confidence": "adjacent",
|
|
1537
|
+
"source": "arena",
|
|
1538
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1539
|
+
},
|
|
1540
|
+
{
|
|
1541
|
+
"rung": "gpt-5.6-sol#ultra",
|
|
1542
|
+
"dim": "steer",
|
|
1543
|
+
"value": 0.0656,
|
|
1544
|
+
"benchmark": "Arena agent steerability",
|
|
1545
|
+
"version": "2026-09-27",
|
|
1546
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1547
|
+
"date": "2026-09-27",
|
|
1548
|
+
"confidence": "adjacent",
|
|
1549
|
+
"source": "arena",
|
|
1550
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1551
|
+
},
|
|
1552
|
+
{
|
|
1553
|
+
"rung": "gpt-5.6-sol#ultra",
|
|
1554
|
+
"dim": "frontend",
|
|
1555
|
+
"value": 1617.3525,
|
|
1556
|
+
"benchmark": "Arena WebDev rating",
|
|
1557
|
+
"version": "2026-09-25",
|
|
1558
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1559
|
+
"date": "2026-09-25",
|
|
1560
|
+
"confidence": "adjacent",
|
|
1561
|
+
"source": "arena",
|
|
1562
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1563
|
+
},
|
|
1564
|
+
{
|
|
1565
|
+
"rung": "gpt-5.6-sol#xhigh",
|
|
1566
|
+
"dim": "repo_code",
|
|
1567
|
+
"value": 73,
|
|
1568
|
+
"benchmark": "DeepSWE",
|
|
1569
|
+
"version": "1.1",
|
|
1570
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
1571
|
+
"date": "2026-09-22",
|
|
1572
|
+
"confidence": "adjacent",
|
|
1573
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
1574
|
+
},
|
|
1575
|
+
{
|
|
1576
|
+
"rung": "gpt-5.6-sol#xhigh",
|
|
1577
|
+
"dim": "terminal",
|
|
1578
|
+
"value": 37.3,
|
|
1579
|
+
"benchmark": "Terminal-Bench",
|
|
1580
|
+
"version": "4.0",
|
|
1581
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
1582
|
+
"date": "2026-09-22",
|
|
1583
|
+
"confidence": "adjacent",
|
|
1584
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
1585
|
+
},
|
|
1586
|
+
{
|
|
1587
|
+
"rung": "gpt-5.6-sol#xhigh",
|
|
1588
|
+
"dim": "honesty",
|
|
1589
|
+
"value": 22.5,
|
|
1590
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
1591
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
1592
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1593
|
+
"date": "2026-09-22",
|
|
1594
|
+
"confidence": "adjacent",
|
|
1595
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
1596
|
+
},
|
|
1597
|
+
{
|
|
1598
|
+
"rung": "gpt-5.6-sol#xhigh",
|
|
1599
|
+
"dim": "agentic",
|
|
1600
|
+
"value": 0.0621,
|
|
1601
|
+
"benchmark": "Arena agent, net improvement",
|
|
1602
|
+
"version": "2026-09-27",
|
|
1603
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1604
|
+
"date": "2026-09-27",
|
|
1605
|
+
"confidence": "measured",
|
|
1606
|
+
"source": "arena"
|
|
1607
|
+
},
|
|
1608
|
+
{
|
|
1609
|
+
"rung": "gpt-5.6-sol#xhigh",
|
|
1610
|
+
"dim": "steer",
|
|
1611
|
+
"value": 0.0656,
|
|
1612
|
+
"benchmark": "Arena agent steerability",
|
|
1613
|
+
"version": "2026-09-27",
|
|
1614
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1615
|
+
"date": "2026-09-27",
|
|
1616
|
+
"confidence": "measured",
|
|
1617
|
+
"source": "arena"
|
|
1618
|
+
},
|
|
1619
|
+
{
|
|
1620
|
+
"rung": "gpt-5.6-sol#xhigh",
|
|
1621
|
+
"dim": "frontend",
|
|
1622
|
+
"value": 1617.3525,
|
|
1623
|
+
"benchmark": "Arena WebDev rating",
|
|
1624
|
+
"version": "2026-09-25",
|
|
1625
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1626
|
+
"date": "2026-09-25",
|
|
1627
|
+
"confidence": "measured",
|
|
1628
|
+
"source": "arena"
|
|
1629
|
+
},
|
|
1630
|
+
{
|
|
1631
|
+
"rung": "gpt-5.6-terra#high",
|
|
1632
|
+
"dim": "agentic",
|
|
1633
|
+
"value": 0.0071,
|
|
1634
|
+
"benchmark": "Arena agent, net improvement",
|
|
1635
|
+
"version": "2026-09-27",
|
|
1636
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1637
|
+
"date": "2026-09-27",
|
|
1638
|
+
"confidence": "adjacent",
|
|
1639
|
+
"source": "arena",
|
|
1640
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1641
|
+
},
|
|
1642
|
+
{
|
|
1643
|
+
"rung": "gpt-5.6-terra#high",
|
|
1644
|
+
"dim": "steer",
|
|
1645
|
+
"value": 0.0423,
|
|
1646
|
+
"benchmark": "Arena agent steerability",
|
|
1647
|
+
"version": "2026-09-27",
|
|
1648
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1649
|
+
"date": "2026-09-27",
|
|
1650
|
+
"confidence": "adjacent",
|
|
1651
|
+
"source": "arena",
|
|
1652
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1653
|
+
},
|
|
1654
|
+
{
|
|
1655
|
+
"rung": "gpt-5.6-terra#high",
|
|
1656
|
+
"dim": "frontend",
|
|
1657
|
+
"value": 1520.5493,
|
|
1658
|
+
"benchmark": "Arena WebDev rating",
|
|
1659
|
+
"version": "2026-09-25",
|
|
1660
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1661
|
+
"date": "2026-09-25",
|
|
1662
|
+
"confidence": "adjacent",
|
|
1663
|
+
"source": "arena",
|
|
1664
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1665
|
+
},
|
|
1666
|
+
{
|
|
1667
|
+
"rung": "gpt-5.6-terra#low",
|
|
1668
|
+
"dim": "agentic",
|
|
1669
|
+
"value": 0.0071,
|
|
1670
|
+
"benchmark": "Arena agent, net improvement",
|
|
1671
|
+
"version": "2026-09-27",
|
|
1672
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1673
|
+
"date": "2026-09-27",
|
|
1674
|
+
"confidence": "adjacent",
|
|
1675
|
+
"source": "arena",
|
|
1676
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1677
|
+
},
|
|
1678
|
+
{
|
|
1679
|
+
"rung": "gpt-5.6-terra#low",
|
|
1680
|
+
"dim": "steer",
|
|
1681
|
+
"value": 0.0423,
|
|
1682
|
+
"benchmark": "Arena agent steerability",
|
|
1683
|
+
"version": "2026-09-27",
|
|
1684
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1685
|
+
"date": "2026-09-27",
|
|
1686
|
+
"confidence": "adjacent",
|
|
1687
|
+
"source": "arena",
|
|
1688
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1689
|
+
},
|
|
1690
|
+
{
|
|
1691
|
+
"rung": "gpt-5.6-terra#low",
|
|
1692
|
+
"dim": "frontend",
|
|
1693
|
+
"value": 1520.5493,
|
|
1694
|
+
"benchmark": "Arena WebDev rating",
|
|
1695
|
+
"version": "2026-09-25",
|
|
1696
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1697
|
+
"date": "2026-09-25",
|
|
1698
|
+
"confidence": "adjacent",
|
|
1699
|
+
"source": "arena",
|
|
1700
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1701
|
+
},
|
|
1702
|
+
{
|
|
1703
|
+
"rung": "gpt-5.6-terra#max",
|
|
1704
|
+
"dim": "agentic",
|
|
1705
|
+
"value": 0.0071,
|
|
1706
|
+
"benchmark": "Arena agent, net improvement",
|
|
1707
|
+
"version": "2026-09-27",
|
|
1708
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1709
|
+
"date": "2026-09-27",
|
|
1710
|
+
"confidence": "adjacent",
|
|
1711
|
+
"source": "arena",
|
|
1712
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1713
|
+
},
|
|
1714
|
+
{
|
|
1715
|
+
"rung": "gpt-5.6-terra#max",
|
|
1716
|
+
"dim": "steer",
|
|
1717
|
+
"value": 0.0423,
|
|
1718
|
+
"benchmark": "Arena agent steerability",
|
|
1719
|
+
"version": "2026-09-27",
|
|
1720
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1721
|
+
"date": "2026-09-27",
|
|
1722
|
+
"confidence": "adjacent",
|
|
1723
|
+
"source": "arena",
|
|
1724
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1725
|
+
},
|
|
1726
|
+
{
|
|
1727
|
+
"rung": "gpt-5.6-terra#max",
|
|
1728
|
+
"dim": "frontend",
|
|
1729
|
+
"value": 1520.5493,
|
|
1730
|
+
"benchmark": "Arena WebDev rating",
|
|
1731
|
+
"version": "2026-09-25",
|
|
1732
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1733
|
+
"date": "2026-09-25",
|
|
1734
|
+
"confidence": "adjacent",
|
|
1735
|
+
"source": "arena",
|
|
1736
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1737
|
+
},
|
|
1738
|
+
{
|
|
1739
|
+
"rung": "gpt-5.6-terra#medium",
|
|
1740
|
+
"dim": "agentic",
|
|
1741
|
+
"value": 0.0071,
|
|
1742
|
+
"benchmark": "Arena agent, net improvement",
|
|
1743
|
+
"version": "2026-09-27",
|
|
1744
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1745
|
+
"date": "2026-09-27",
|
|
1746
|
+
"confidence": "adjacent",
|
|
1747
|
+
"source": "arena",
|
|
1748
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1749
|
+
},
|
|
1750
|
+
{
|
|
1751
|
+
"rung": "gpt-5.6-terra#medium",
|
|
1752
|
+
"dim": "steer",
|
|
1753
|
+
"value": 0.0423,
|
|
1754
|
+
"benchmark": "Arena agent steerability",
|
|
1755
|
+
"version": "2026-09-27",
|
|
1756
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1757
|
+
"date": "2026-09-27",
|
|
1758
|
+
"confidence": "adjacent",
|
|
1759
|
+
"source": "arena",
|
|
1760
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1761
|
+
},
|
|
1762
|
+
{
|
|
1763
|
+
"rung": "gpt-5.6-terra#medium",
|
|
1764
|
+
"dim": "frontend",
|
|
1765
|
+
"value": 1520.5493,
|
|
1766
|
+
"benchmark": "Arena WebDev rating",
|
|
1767
|
+
"version": "2026-09-25",
|
|
1768
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1769
|
+
"date": "2026-09-25",
|
|
1770
|
+
"confidence": "adjacent",
|
|
1771
|
+
"source": "arena",
|
|
1772
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1773
|
+
},
|
|
1774
|
+
{
|
|
1775
|
+
"rung": "gpt-5.6-terra#none",
|
|
1776
|
+
"dim": "agentic",
|
|
1777
|
+
"value": 0.0071,
|
|
1778
|
+
"benchmark": "Arena agent, net improvement",
|
|
1779
|
+
"version": "2026-09-27",
|
|
1780
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1781
|
+
"date": "2026-09-27",
|
|
1782
|
+
"confidence": "adjacent",
|
|
1783
|
+
"source": "arena",
|
|
1784
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1785
|
+
},
|
|
1786
|
+
{
|
|
1787
|
+
"rung": "gpt-5.6-terra#none",
|
|
1788
|
+
"dim": "steer",
|
|
1789
|
+
"value": 0.0423,
|
|
1790
|
+
"benchmark": "Arena agent steerability",
|
|
1791
|
+
"version": "2026-09-27",
|
|
1792
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1793
|
+
"date": "2026-09-27",
|
|
1794
|
+
"confidence": "adjacent",
|
|
1795
|
+
"source": "arena",
|
|
1796
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1797
|
+
},
|
|
1798
|
+
{
|
|
1799
|
+
"rung": "gpt-5.6-terra#none",
|
|
1800
|
+
"dim": "frontend",
|
|
1801
|
+
"value": 1520.5493,
|
|
1802
|
+
"benchmark": "Arena WebDev rating",
|
|
1803
|
+
"version": "2026-09-25",
|
|
1804
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1805
|
+
"date": "2026-09-25",
|
|
1806
|
+
"confidence": "adjacent",
|
|
1807
|
+
"source": "arena",
|
|
1808
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1809
|
+
},
|
|
1810
|
+
{
|
|
1811
|
+
"rung": "gpt-5.6-terra#ultra",
|
|
1812
|
+
"dim": "agentic",
|
|
1813
|
+
"value": 0.0071,
|
|
1814
|
+
"benchmark": "Arena agent, net improvement",
|
|
1815
|
+
"version": "2026-09-27",
|
|
1816
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1817
|
+
"date": "2026-09-27",
|
|
1818
|
+
"confidence": "adjacent",
|
|
1819
|
+
"source": "arena",
|
|
1820
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1821
|
+
},
|
|
1822
|
+
{
|
|
1823
|
+
"rung": "gpt-5.6-terra#ultra",
|
|
1824
|
+
"dim": "steer",
|
|
1825
|
+
"value": 0.0423,
|
|
1826
|
+
"benchmark": "Arena agent steerability",
|
|
1827
|
+
"version": "2026-09-27",
|
|
1828
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1829
|
+
"date": "2026-09-27",
|
|
1830
|
+
"confidence": "adjacent",
|
|
1831
|
+
"source": "arena",
|
|
1832
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1833
|
+
},
|
|
1834
|
+
{
|
|
1835
|
+
"rung": "gpt-5.6-terra#ultra",
|
|
1836
|
+
"dim": "frontend",
|
|
1837
|
+
"value": 1520.5493,
|
|
1838
|
+
"benchmark": "Arena WebDev rating",
|
|
1839
|
+
"version": "2026-09-25",
|
|
1840
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1841
|
+
"date": "2026-09-25",
|
|
1842
|
+
"confidence": "adjacent",
|
|
1843
|
+
"source": "arena",
|
|
1844
|
+
"note": "arena has it at xhigh; carried to this effort"
|
|
1845
|
+
},
|
|
1846
|
+
{
|
|
1847
|
+
"rung": "gpt-5.6-terra#xhigh",
|
|
1848
|
+
"dim": "agentic",
|
|
1849
|
+
"value": 0.0071,
|
|
1850
|
+
"benchmark": "Arena agent, net improvement",
|
|
1851
|
+
"version": "2026-09-27",
|
|
1852
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1853
|
+
"date": "2026-09-27",
|
|
1854
|
+
"confidence": "measured",
|
|
1855
|
+
"source": "arena"
|
|
1856
|
+
},
|
|
1857
|
+
{
|
|
1858
|
+
"rung": "gpt-5.6-terra#xhigh",
|
|
1859
|
+
"dim": "steer",
|
|
1860
|
+
"value": 0.0423,
|
|
1861
|
+
"benchmark": "Arena agent steerability",
|
|
1862
|
+
"version": "2026-09-27",
|
|
1863
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1864
|
+
"date": "2026-09-27",
|
|
1865
|
+
"confidence": "measured",
|
|
1866
|
+
"source": "arena"
|
|
1867
|
+
},
|
|
1868
|
+
{
|
|
1869
|
+
"rung": "gpt-5.6-terra#xhigh",
|
|
1870
|
+
"dim": "frontend",
|
|
1871
|
+
"value": 1520.5493,
|
|
1872
|
+
"benchmark": "Arena WebDev rating",
|
|
1873
|
+
"version": "2026-09-25",
|
|
1874
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1875
|
+
"date": "2026-09-25",
|
|
1876
|
+
"confidence": "measured",
|
|
1877
|
+
"source": "arena"
|
|
1878
|
+
},
|
|
1879
|
+
{
|
|
1880
|
+
"rung": "gpt-6-astra#high",
|
|
1881
|
+
"dim": "repo_code",
|
|
1882
|
+
"value": 74,
|
|
1883
|
+
"benchmark": "DeepSWE",
|
|
1884
|
+
"version": "1.1",
|
|
1885
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
1886
|
+
"date": "2026-09-22",
|
|
1887
|
+
"confidence": "adjacent",
|
|
1888
|
+
"note": "DeepSWE has it at xhigh; carried to this effort"
|
|
1889
|
+
},
|
|
1890
|
+
{
|
|
1891
|
+
"rung": "gpt-6-astra#high",
|
|
1892
|
+
"dim": "terminal",
|
|
1893
|
+
"value": 57.9,
|
|
1894
|
+
"benchmark": "Terminal-Bench",
|
|
1895
|
+
"version": "4.0",
|
|
1896
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
1897
|
+
"date": "2026-09-22",
|
|
1898
|
+
"confidence": "secondary",
|
|
1899
|
+
"note": "OpenAI-reported, as quoted in the Opus 5.5 system card"
|
|
1900
|
+
},
|
|
1901
|
+
{
|
|
1902
|
+
"rung": "gpt-6-astra#high",
|
|
1903
|
+
"dim": "honesty",
|
|
1904
|
+
"value": 98.5,
|
|
1905
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
1906
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
1907
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1908
|
+
"date": "2026-09-22",
|
|
1909
|
+
"confidence": "adjacent",
|
|
1910
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
1911
|
+
},
|
|
1912
|
+
{
|
|
1913
|
+
"rung": "gpt-6-astra#high",
|
|
1914
|
+
"dim": "agentic",
|
|
1915
|
+
"value": 0.1031,
|
|
1916
|
+
"benchmark": "Arena agent, net improvement",
|
|
1917
|
+
"version": "2026-09-27",
|
|
1918
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1919
|
+
"date": "2026-09-27",
|
|
1920
|
+
"confidence": "adjacent",
|
|
1921
|
+
"source": "arena",
|
|
1922
|
+
"note": "arena has it at max; carried to this effort"
|
|
1923
|
+
},
|
|
1924
|
+
{
|
|
1925
|
+
"rung": "gpt-6-astra#high",
|
|
1926
|
+
"dim": "steer",
|
|
1927
|
+
"value": -0.021,
|
|
1928
|
+
"benchmark": "Arena agent steerability",
|
|
1929
|
+
"version": "2026-09-27",
|
|
1930
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1931
|
+
"date": "2026-09-27",
|
|
1932
|
+
"confidence": "adjacent",
|
|
1933
|
+
"source": "arena",
|
|
1934
|
+
"note": "arena has it at max; carried to this effort"
|
|
1935
|
+
},
|
|
1936
|
+
{
|
|
1937
|
+
"rung": "gpt-6-astra#high",
|
|
1938
|
+
"dim": "frontend",
|
|
1939
|
+
"value": 1791.6526,
|
|
1940
|
+
"benchmark": "Arena WebDev rating",
|
|
1941
|
+
"version": "2026-09-25",
|
|
1942
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1943
|
+
"date": "2026-09-25",
|
|
1944
|
+
"confidence": "adjacent",
|
|
1945
|
+
"source": "arena",
|
|
1946
|
+
"note": "arena has it at max; carried to this effort"
|
|
1947
|
+
},
|
|
1948
|
+
{
|
|
1949
|
+
"rung": "gpt-6-astra#low",
|
|
1950
|
+
"dim": "repo_code",
|
|
1951
|
+
"value": 74,
|
|
1952
|
+
"benchmark": "DeepSWE",
|
|
1953
|
+
"version": "1.1",
|
|
1954
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
1955
|
+
"date": "2026-09-22",
|
|
1956
|
+
"confidence": "adjacent",
|
|
1957
|
+
"note": "DeepSWE has it at xhigh; carried to this effort"
|
|
1958
|
+
},
|
|
1959
|
+
{
|
|
1960
|
+
"rung": "gpt-6-astra#low",
|
|
1961
|
+
"dim": "terminal",
|
|
1962
|
+
"value": 57.9,
|
|
1963
|
+
"benchmark": "Terminal-Bench",
|
|
1964
|
+
"version": "4.0",
|
|
1965
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
1966
|
+
"date": "2026-09-22",
|
|
1967
|
+
"confidence": "adjacent",
|
|
1968
|
+
"note": "Terminal-Bench has it at high; carried to this effort"
|
|
1969
|
+
},
|
|
1970
|
+
{
|
|
1971
|
+
"rung": "gpt-6-astra#low",
|
|
1972
|
+
"dim": "honesty",
|
|
1973
|
+
"value": 98.5,
|
|
1974
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
1975
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
1976
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
1977
|
+
"date": "2026-09-22",
|
|
1978
|
+
"confidence": "adjacent",
|
|
1979
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
1980
|
+
},
|
|
1981
|
+
{
|
|
1982
|
+
"rung": "gpt-6-astra#low",
|
|
1983
|
+
"dim": "agentic",
|
|
1984
|
+
"value": 0.1031,
|
|
1985
|
+
"benchmark": "Arena agent, net improvement",
|
|
1986
|
+
"version": "2026-09-27",
|
|
1987
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
1988
|
+
"date": "2026-09-27",
|
|
1989
|
+
"confidence": "adjacent",
|
|
1990
|
+
"source": "arena",
|
|
1991
|
+
"note": "arena has it at max; carried to this effort"
|
|
1992
|
+
},
|
|
1993
|
+
{
|
|
1994
|
+
"rung": "gpt-6-astra#low",
|
|
1995
|
+
"dim": "steer",
|
|
1996
|
+
"value": -0.021,
|
|
1997
|
+
"benchmark": "Arena agent steerability",
|
|
1998
|
+
"version": "2026-09-27",
|
|
1999
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2000
|
+
"date": "2026-09-27",
|
|
2001
|
+
"confidence": "adjacent",
|
|
2002
|
+
"source": "arena",
|
|
2003
|
+
"note": "arena has it at max; carried to this effort"
|
|
2004
|
+
},
|
|
2005
|
+
{
|
|
2006
|
+
"rung": "gpt-6-astra#low",
|
|
2007
|
+
"dim": "frontend",
|
|
2008
|
+
"value": 1791.6526,
|
|
2009
|
+
"benchmark": "Arena WebDev rating",
|
|
2010
|
+
"version": "2026-09-25",
|
|
2011
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2012
|
+
"date": "2026-09-25",
|
|
2013
|
+
"confidence": "adjacent",
|
|
2014
|
+
"source": "arena",
|
|
2015
|
+
"note": "arena has it at max; carried to this effort"
|
|
2016
|
+
},
|
|
2017
|
+
{
|
|
2018
|
+
"rung": "gpt-6-astra#max",
|
|
2019
|
+
"dim": "repo_code",
|
|
2020
|
+
"value": 74,
|
|
2021
|
+
"benchmark": "DeepSWE",
|
|
2022
|
+
"version": "1.1",
|
|
2023
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
2024
|
+
"date": "2026-09-22",
|
|
2025
|
+
"confidence": "adjacent",
|
|
2026
|
+
"note": "DeepSWE has it at xhigh; carried to this effort"
|
|
2027
|
+
},
|
|
2028
|
+
{
|
|
2029
|
+
"rung": "gpt-6-astra#max",
|
|
2030
|
+
"dim": "terminal",
|
|
2031
|
+
"value": 57.9,
|
|
2032
|
+
"benchmark": "Terminal-Bench",
|
|
2033
|
+
"version": "4.0",
|
|
2034
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
2035
|
+
"date": "2026-09-22",
|
|
2036
|
+
"confidence": "adjacent",
|
|
2037
|
+
"note": "Terminal-Bench has it at high; carried to this effort"
|
|
2038
|
+
},
|
|
2039
|
+
{
|
|
2040
|
+
"rung": "gpt-6-astra#max",
|
|
2041
|
+
"dim": "honesty",
|
|
2042
|
+
"value": 98.5,
|
|
2043
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2044
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2045
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
2046
|
+
"date": "2026-09-22",
|
|
2047
|
+
"confidence": "verified"
|
|
2048
|
+
},
|
|
2049
|
+
{
|
|
2050
|
+
"rung": "gpt-6-astra#max",
|
|
2051
|
+
"dim": "agentic",
|
|
2052
|
+
"value": 0.1031,
|
|
2053
|
+
"benchmark": "Arena agent, net improvement",
|
|
2054
|
+
"version": "2026-09-27",
|
|
2055
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2056
|
+
"date": "2026-09-27",
|
|
2057
|
+
"confidence": "measured",
|
|
2058
|
+
"source": "arena"
|
|
2059
|
+
},
|
|
2060
|
+
{
|
|
2061
|
+
"rung": "gpt-6-astra#max",
|
|
2062
|
+
"dim": "steer",
|
|
2063
|
+
"value": -0.021,
|
|
2064
|
+
"benchmark": "Arena agent steerability",
|
|
2065
|
+
"version": "2026-09-27",
|
|
2066
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2067
|
+
"date": "2026-09-27",
|
|
2068
|
+
"confidence": "measured",
|
|
2069
|
+
"source": "arena"
|
|
2070
|
+
},
|
|
2071
|
+
{
|
|
2072
|
+
"rung": "gpt-6-astra#max",
|
|
2073
|
+
"dim": "frontend",
|
|
2074
|
+
"value": 1791.6526,
|
|
2075
|
+
"benchmark": "Arena WebDev rating",
|
|
2076
|
+
"version": "2026-09-25",
|
|
2077
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2078
|
+
"date": "2026-09-25",
|
|
2079
|
+
"confidence": "measured",
|
|
2080
|
+
"source": "arena"
|
|
2081
|
+
},
|
|
2082
|
+
{
|
|
2083
|
+
"rung": "gpt-6-astra#medium",
|
|
2084
|
+
"dim": "repo_code",
|
|
2085
|
+
"value": 74,
|
|
2086
|
+
"benchmark": "DeepSWE",
|
|
2087
|
+
"version": "1.1",
|
|
2088
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
2089
|
+
"date": "2026-09-22",
|
|
2090
|
+
"confidence": "adjacent",
|
|
2091
|
+
"note": "DeepSWE has it at xhigh; carried to this effort"
|
|
2092
|
+
},
|
|
2093
|
+
{
|
|
2094
|
+
"rung": "gpt-6-astra#medium",
|
|
2095
|
+
"dim": "terminal",
|
|
2096
|
+
"value": 57.9,
|
|
2097
|
+
"benchmark": "Terminal-Bench",
|
|
2098
|
+
"version": "4.0",
|
|
2099
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
2100
|
+
"date": "2026-09-22",
|
|
2101
|
+
"confidence": "adjacent",
|
|
2102
|
+
"note": "Terminal-Bench has it at high; carried to this effort"
|
|
2103
|
+
},
|
|
2104
|
+
{
|
|
2105
|
+
"rung": "gpt-6-astra#medium",
|
|
2106
|
+
"dim": "honesty",
|
|
2107
|
+
"value": 98.5,
|
|
2108
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2109
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2110
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
2111
|
+
"date": "2026-09-22",
|
|
2112
|
+
"confidence": "adjacent",
|
|
2113
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
2114
|
+
},
|
|
2115
|
+
{
|
|
2116
|
+
"rung": "gpt-6-astra#medium",
|
|
2117
|
+
"dim": "agentic",
|
|
2118
|
+
"value": 0.1031,
|
|
2119
|
+
"benchmark": "Arena agent, net improvement",
|
|
2120
|
+
"version": "2026-09-27",
|
|
2121
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2122
|
+
"date": "2026-09-27",
|
|
2123
|
+
"confidence": "adjacent",
|
|
2124
|
+
"source": "arena",
|
|
2125
|
+
"note": "arena has it at max; carried to this effort"
|
|
2126
|
+
},
|
|
2127
|
+
{
|
|
2128
|
+
"rung": "gpt-6-astra#medium",
|
|
2129
|
+
"dim": "steer",
|
|
2130
|
+
"value": -0.021,
|
|
2131
|
+
"benchmark": "Arena agent steerability",
|
|
2132
|
+
"version": "2026-09-27",
|
|
2133
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2134
|
+
"date": "2026-09-27",
|
|
2135
|
+
"confidence": "adjacent",
|
|
2136
|
+
"source": "arena",
|
|
2137
|
+
"note": "arena has it at max; carried to this effort"
|
|
2138
|
+
},
|
|
2139
|
+
{
|
|
2140
|
+
"rung": "gpt-6-astra#medium",
|
|
2141
|
+
"dim": "frontend",
|
|
2142
|
+
"value": 1791.6526,
|
|
2143
|
+
"benchmark": "Arena WebDev rating",
|
|
2144
|
+
"version": "2026-09-25",
|
|
2145
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2146
|
+
"date": "2026-09-25",
|
|
2147
|
+
"confidence": "adjacent",
|
|
2148
|
+
"source": "arena",
|
|
2149
|
+
"note": "arena has it at max; carried to this effort"
|
|
2150
|
+
},
|
|
2151
|
+
{
|
|
2152
|
+
"rung": "gpt-6-astra#ultra",
|
|
2153
|
+
"dim": "repo_code",
|
|
2154
|
+
"value": 74,
|
|
2155
|
+
"benchmark": "DeepSWE",
|
|
2156
|
+
"version": "1.1",
|
|
2157
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
2158
|
+
"date": "2026-09-22",
|
|
2159
|
+
"confidence": "adjacent",
|
|
2160
|
+
"note": "DeepSWE has it at xhigh; carried to this effort"
|
|
2161
|
+
},
|
|
2162
|
+
{
|
|
2163
|
+
"rung": "gpt-6-astra#ultra",
|
|
2164
|
+
"dim": "terminal",
|
|
2165
|
+
"value": 57.9,
|
|
2166
|
+
"benchmark": "Terminal-Bench",
|
|
2167
|
+
"version": "4.0",
|
|
2168
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
2169
|
+
"date": "2026-09-22",
|
|
2170
|
+
"confidence": "adjacent",
|
|
2171
|
+
"note": "Terminal-Bench has it at high; carried to this effort"
|
|
2172
|
+
},
|
|
2173
|
+
{
|
|
2174
|
+
"rung": "gpt-6-astra#ultra",
|
|
2175
|
+
"dim": "honesty",
|
|
2176
|
+
"value": 98.5,
|
|
2177
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2178
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2179
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
2180
|
+
"date": "2026-09-22",
|
|
2181
|
+
"confidence": "adjacent",
|
|
2182
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
2183
|
+
},
|
|
2184
|
+
{
|
|
2185
|
+
"rung": "gpt-6-astra#ultra",
|
|
2186
|
+
"dim": "agentic",
|
|
2187
|
+
"value": 0.1031,
|
|
2188
|
+
"benchmark": "Arena agent, net improvement",
|
|
2189
|
+
"version": "2026-09-27",
|
|
2190
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2191
|
+
"date": "2026-09-27",
|
|
2192
|
+
"confidence": "adjacent",
|
|
2193
|
+
"source": "arena",
|
|
2194
|
+
"note": "arena has it at max; carried to this effort"
|
|
2195
|
+
},
|
|
2196
|
+
{
|
|
2197
|
+
"rung": "gpt-6-astra#ultra",
|
|
2198
|
+
"dim": "steer",
|
|
2199
|
+
"value": -0.021,
|
|
2200
|
+
"benchmark": "Arena agent steerability",
|
|
2201
|
+
"version": "2026-09-27",
|
|
2202
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2203
|
+
"date": "2026-09-27",
|
|
2204
|
+
"confidence": "adjacent",
|
|
2205
|
+
"source": "arena",
|
|
2206
|
+
"note": "arena has it at max; carried to this effort"
|
|
2207
|
+
},
|
|
2208
|
+
{
|
|
2209
|
+
"rung": "gpt-6-astra#ultra",
|
|
2210
|
+
"dim": "frontend",
|
|
2211
|
+
"value": 1791.6526,
|
|
2212
|
+
"benchmark": "Arena WebDev rating",
|
|
2213
|
+
"version": "2026-09-25",
|
|
2214
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2215
|
+
"date": "2026-09-25",
|
|
2216
|
+
"confidence": "adjacent",
|
|
2217
|
+
"source": "arena",
|
|
2218
|
+
"note": "arena has it at max; carried to this effort"
|
|
2219
|
+
},
|
|
2220
|
+
{
|
|
2221
|
+
"rung": "gpt-6-astra#xhigh",
|
|
2222
|
+
"dim": "repo_code",
|
|
2223
|
+
"value": 74,
|
|
2224
|
+
"benchmark": "DeepSWE",
|
|
2225
|
+
"version": "1.1",
|
|
2226
|
+
"url": "https://deepswe.datacurve.ai/",
|
|
2227
|
+
"date": "2026-09-22",
|
|
2228
|
+
"confidence": "verified"
|
|
2229
|
+
},
|
|
2230
|
+
{
|
|
2231
|
+
"rung": "gpt-6-astra#xhigh",
|
|
2232
|
+
"dim": "terminal",
|
|
2233
|
+
"value": 57.9,
|
|
2234
|
+
"benchmark": "Terminal-Bench",
|
|
2235
|
+
"version": "4.0",
|
|
2236
|
+
"url": "https://www-cdn.anthropic.com/fc1b44717c85dc068bc6ba5024219938094694bd/Claude%20Opus%205.5%20System%20Card.pdf",
|
|
2237
|
+
"date": "2026-09-22",
|
|
2238
|
+
"confidence": "adjacent",
|
|
2239
|
+
"note": "Terminal-Bench has it at high; carried to this effort"
|
|
2240
|
+
},
|
|
2241
|
+
{
|
|
2242
|
+
"rung": "gpt-6-astra#xhigh",
|
|
2243
|
+
"dim": "honesty",
|
|
2244
|
+
"value": 98.5,
|
|
2245
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2246
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2247
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
2248
|
+
"date": "2026-09-22",
|
|
2249
|
+
"confidence": "adjacent",
|
|
2250
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
2251
|
+
},
|
|
2252
|
+
{
|
|
2253
|
+
"rung": "gpt-6-astra#xhigh",
|
|
2254
|
+
"dim": "agentic",
|
|
2255
|
+
"value": 0.1031,
|
|
2256
|
+
"benchmark": "Arena agent, net improvement",
|
|
2257
|
+
"version": "2026-09-27",
|
|
2258
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2259
|
+
"date": "2026-09-27",
|
|
2260
|
+
"confidence": "adjacent",
|
|
2261
|
+
"source": "arena",
|
|
2262
|
+
"note": "arena has it at max; carried to this effort"
|
|
2263
|
+
},
|
|
2264
|
+
{
|
|
2265
|
+
"rung": "gpt-6-astra#xhigh",
|
|
2266
|
+
"dim": "steer",
|
|
2267
|
+
"value": -0.021,
|
|
2268
|
+
"benchmark": "Arena agent steerability",
|
|
2269
|
+
"version": "2026-09-27",
|
|
2270
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2271
|
+
"date": "2026-09-27",
|
|
2272
|
+
"confidence": "adjacent",
|
|
2273
|
+
"source": "arena",
|
|
2274
|
+
"note": "arena has it at max; carried to this effort"
|
|
2275
|
+
},
|
|
2276
|
+
{
|
|
2277
|
+
"rung": "gpt-6-astra#xhigh",
|
|
2278
|
+
"dim": "frontend",
|
|
2279
|
+
"value": 1791.6526,
|
|
2280
|
+
"benchmark": "Arena WebDev rating",
|
|
2281
|
+
"version": "2026-09-25",
|
|
2282
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2283
|
+
"date": "2026-09-25",
|
|
2284
|
+
"confidence": "adjacent",
|
|
2285
|
+
"source": "arena",
|
|
2286
|
+
"note": "arena has it at max; carried to this effort"
|
|
2287
|
+
},
|
|
2288
|
+
{
|
|
2289
|
+
"rung": "gpt-6-luna#high",
|
|
2290
|
+
"dim": "repo_code",
|
|
2291
|
+
"value": 66.6,
|
|
2292
|
+
"benchmark": "DeepSWE",
|
|
2293
|
+
"version": "1.1",
|
|
2294
|
+
"url": "https://kingy.ai/blog/gpt-6-sol-luna-specs-benchmarks-pricing-comparison/",
|
|
2295
|
+
"date": "2026-09-22",
|
|
2296
|
+
"confidence": "adjacent",
|
|
2297
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
2298
|
+
},
|
|
2299
|
+
{
|
|
2300
|
+
"rung": "gpt-6-luna#high",
|
|
2301
|
+
"dim": "terminal",
|
|
2302
|
+
"value": 13,
|
|
2303
|
+
"benchmark": "Terminal-Bench",
|
|
2304
|
+
"version": "4.0",
|
|
2305
|
+
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
2306
|
+
"date": "2026-09-22",
|
|
2307
|
+
"confidence": "adjacent",
|
|
2308
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
2309
|
+
},
|
|
2310
|
+
{
|
|
2311
|
+
"rung": "gpt-6-luna#high",
|
|
2312
|
+
"dim": "honesty",
|
|
2313
|
+
"value": 71.3,
|
|
2314
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2315
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2316
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
2317
|
+
"date": "2026-09-22",
|
|
2318
|
+
"confidence": "adjacent",
|
|
2319
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
2320
|
+
},
|
|
2321
|
+
{
|
|
2322
|
+
"rung": "gpt-6-luna#high",
|
|
2323
|
+
"dim": "frontend",
|
|
2324
|
+
"value": 1592.6467,
|
|
2325
|
+
"benchmark": "Arena WebDev rating",
|
|
2326
|
+
"version": "2026-09-25",
|
|
2327
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2328
|
+
"date": "2026-09-25",
|
|
2329
|
+
"confidence": "adjacent",
|
|
2330
|
+
"source": "arena",
|
|
2331
|
+
"note": "arena has it at max; carried to this effort"
|
|
2332
|
+
},
|
|
2333
|
+
{
|
|
2334
|
+
"rung": "gpt-6-luna#low",
|
|
2335
|
+
"dim": "repo_code",
|
|
2336
|
+
"value": 66.6,
|
|
2337
|
+
"benchmark": "DeepSWE",
|
|
2338
|
+
"version": "1.1",
|
|
2339
|
+
"url": "https://kingy.ai/blog/gpt-6-sol-luna-specs-benchmarks-pricing-comparison/",
|
|
2340
|
+
"date": "2026-09-22",
|
|
2341
|
+
"confidence": "adjacent",
|
|
2342
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
2343
|
+
},
|
|
2344
|
+
{
|
|
2345
|
+
"rung": "gpt-6-luna#low",
|
|
2346
|
+
"dim": "terminal",
|
|
2347
|
+
"value": 13,
|
|
2348
|
+
"benchmark": "Terminal-Bench",
|
|
2349
|
+
"version": "4.0",
|
|
2350
|
+
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
2351
|
+
"date": "2026-09-22",
|
|
2352
|
+
"confidence": "adjacent",
|
|
2353
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
2354
|
+
},
|
|
2355
|
+
{
|
|
2356
|
+
"rung": "gpt-6-luna#low",
|
|
2357
|
+
"dim": "honesty",
|
|
2358
|
+
"value": 71.3,
|
|
2359
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2360
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2361
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
2362
|
+
"date": "2026-09-22",
|
|
2363
|
+
"confidence": "adjacent",
|
|
2364
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
2365
|
+
},
|
|
2366
|
+
{
|
|
2367
|
+
"rung": "gpt-6-luna#low",
|
|
2368
|
+
"dim": "frontend",
|
|
2369
|
+
"value": 1592.6467,
|
|
2370
|
+
"benchmark": "Arena WebDev rating",
|
|
2371
|
+
"version": "2026-09-25",
|
|
2372
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2373
|
+
"date": "2026-09-25",
|
|
2374
|
+
"confidence": "adjacent",
|
|
2375
|
+
"source": "arena",
|
|
2376
|
+
"note": "arena has it at max; carried to this effort"
|
|
2377
|
+
},
|
|
2378
|
+
{
|
|
2379
|
+
"rung": "gpt-6-luna#max",
|
|
2380
|
+
"dim": "repo_code",
|
|
2381
|
+
"value": 66.6,
|
|
2382
|
+
"benchmark": "DeepSWE",
|
|
2383
|
+
"version": "1.1",
|
|
2384
|
+
"url": "https://kingy.ai/blog/gpt-6-sol-luna-specs-benchmarks-pricing-comparison/",
|
|
2385
|
+
"date": "2026-09-22",
|
|
2386
|
+
"confidence": "secondary"
|
|
2387
|
+
},
|
|
2388
|
+
{
|
|
2389
|
+
"rung": "gpt-6-luna#max",
|
|
2390
|
+
"dim": "terminal",
|
|
2391
|
+
"value": 13,
|
|
2392
|
+
"benchmark": "Terminal-Bench",
|
|
2393
|
+
"version": "4.0",
|
|
2394
|
+
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
2395
|
+
"date": "2026-09-22",
|
|
2396
|
+
"confidence": "secondary"
|
|
2397
|
+
},
|
|
2398
|
+
{
|
|
2399
|
+
"rung": "gpt-6-luna#max",
|
|
2400
|
+
"dim": "honesty",
|
|
2401
|
+
"value": 71.3,
|
|
2402
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2403
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2404
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
2405
|
+
"date": "2026-09-22",
|
|
2406
|
+
"confidence": "verified"
|
|
2407
|
+
},
|
|
2408
|
+
{
|
|
2409
|
+
"rung": "gpt-6-luna#max",
|
|
2410
|
+
"dim": "frontend",
|
|
2411
|
+
"value": 1592.6467,
|
|
2412
|
+
"benchmark": "Arena WebDev rating",
|
|
2413
|
+
"version": "2026-09-25",
|
|
2414
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2415
|
+
"date": "2026-09-25",
|
|
2416
|
+
"confidence": "measured",
|
|
2417
|
+
"source": "arena"
|
|
2418
|
+
},
|
|
2419
|
+
{
|
|
2420
|
+
"rung": "gpt-6-luna#medium",
|
|
2421
|
+
"dim": "repo_code",
|
|
2422
|
+
"value": 66.6,
|
|
2423
|
+
"benchmark": "DeepSWE",
|
|
2424
|
+
"version": "1.1",
|
|
2425
|
+
"url": "https://kingy.ai/blog/gpt-6-sol-luna-specs-benchmarks-pricing-comparison/",
|
|
2426
|
+
"date": "2026-09-22",
|
|
2427
|
+
"confidence": "adjacent",
|
|
2428
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
2429
|
+
},
|
|
2430
|
+
{
|
|
2431
|
+
"rung": "gpt-6-luna#medium",
|
|
2432
|
+
"dim": "terminal",
|
|
2433
|
+
"value": 13,
|
|
2434
|
+
"benchmark": "Terminal-Bench",
|
|
2435
|
+
"version": "4.0",
|
|
2436
|
+
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
2437
|
+
"date": "2026-09-22",
|
|
2438
|
+
"confidence": "adjacent",
|
|
2439
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
2440
|
+
},
|
|
2441
|
+
{
|
|
2442
|
+
"rung": "gpt-6-luna#medium",
|
|
2443
|
+
"dim": "honesty",
|
|
2444
|
+
"value": 71.3,
|
|
2445
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2446
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2447
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
2448
|
+
"date": "2026-09-22",
|
|
2449
|
+
"confidence": "adjacent",
|
|
2450
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
2451
|
+
},
|
|
2452
|
+
{
|
|
2453
|
+
"rung": "gpt-6-luna#medium",
|
|
2454
|
+
"dim": "frontend",
|
|
2455
|
+
"value": 1592.6467,
|
|
2456
|
+
"benchmark": "Arena WebDev rating",
|
|
2457
|
+
"version": "2026-09-25",
|
|
2458
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2459
|
+
"date": "2026-09-25",
|
|
2460
|
+
"confidence": "adjacent",
|
|
2461
|
+
"source": "arena",
|
|
2462
|
+
"note": "arena has it at max; carried to this effort"
|
|
2463
|
+
},
|
|
2464
|
+
{
|
|
2465
|
+
"rung": "gpt-6-luna#none",
|
|
2466
|
+
"dim": "repo_code",
|
|
2467
|
+
"value": 66.6,
|
|
2468
|
+
"benchmark": "DeepSWE",
|
|
2469
|
+
"version": "1.1",
|
|
2470
|
+
"url": "https://kingy.ai/blog/gpt-6-sol-luna-specs-benchmarks-pricing-comparison/",
|
|
2471
|
+
"date": "2026-09-22",
|
|
2472
|
+
"confidence": "adjacent",
|
|
2473
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
2474
|
+
},
|
|
2475
|
+
{
|
|
2476
|
+
"rung": "gpt-6-luna#none",
|
|
2477
|
+
"dim": "terminal",
|
|
2478
|
+
"value": 13,
|
|
2479
|
+
"benchmark": "Terminal-Bench",
|
|
2480
|
+
"version": "4.0",
|
|
2481
|
+
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
2482
|
+
"date": "2026-09-22",
|
|
2483
|
+
"confidence": "adjacent",
|
|
2484
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
2485
|
+
},
|
|
2486
|
+
{
|
|
2487
|
+
"rung": "gpt-6-luna#none",
|
|
2488
|
+
"dim": "honesty",
|
|
2489
|
+
"value": 71.3,
|
|
2490
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2491
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2492
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
2493
|
+
"date": "2026-09-22",
|
|
2494
|
+
"confidence": "adjacent",
|
|
2495
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
2496
|
+
},
|
|
2497
|
+
{
|
|
2498
|
+
"rung": "gpt-6-luna#none",
|
|
2499
|
+
"dim": "frontend",
|
|
2500
|
+
"value": 1592.6467,
|
|
2501
|
+
"benchmark": "Arena WebDev rating",
|
|
2502
|
+
"version": "2026-09-25",
|
|
2503
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2504
|
+
"date": "2026-09-25",
|
|
2505
|
+
"confidence": "adjacent",
|
|
2506
|
+
"source": "arena",
|
|
2507
|
+
"note": "arena has it at max; carried to this effort"
|
|
2508
|
+
},
|
|
2509
|
+
{
|
|
2510
|
+
"rung": "gpt-6-luna#xhigh",
|
|
2511
|
+
"dim": "repo_code",
|
|
2512
|
+
"value": 66.6,
|
|
2513
|
+
"benchmark": "DeepSWE",
|
|
2514
|
+
"version": "1.1",
|
|
2515
|
+
"url": "https://kingy.ai/blog/gpt-6-sol-luna-specs-benchmarks-pricing-comparison/",
|
|
2516
|
+
"date": "2026-09-22",
|
|
2517
|
+
"confidence": "adjacent",
|
|
2518
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
2519
|
+
},
|
|
2520
|
+
{
|
|
2521
|
+
"rung": "gpt-6-luna#xhigh",
|
|
2522
|
+
"dim": "terminal",
|
|
2523
|
+
"value": 13,
|
|
2524
|
+
"benchmark": "Terminal-Bench",
|
|
2525
|
+
"version": "4.0",
|
|
2526
|
+
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
2527
|
+
"date": "2026-09-22",
|
|
2528
|
+
"confidence": "adjacent",
|
|
2529
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
2530
|
+
},
|
|
2531
|
+
{
|
|
2532
|
+
"rung": "gpt-6-luna#xhigh",
|
|
2533
|
+
"dim": "honesty",
|
|
2534
|
+
"value": 71.3,
|
|
2535
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2536
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2537
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
2538
|
+
"date": "2026-09-22",
|
|
2539
|
+
"confidence": "adjacent",
|
|
2540
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
2541
|
+
},
|
|
2542
|
+
{
|
|
2543
|
+
"rung": "gpt-6-luna#xhigh",
|
|
2544
|
+
"dim": "frontend",
|
|
2545
|
+
"value": 1592.6467,
|
|
2546
|
+
"benchmark": "Arena WebDev rating",
|
|
2547
|
+
"version": "2026-09-25",
|
|
2548
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2549
|
+
"date": "2026-09-25",
|
|
2550
|
+
"confidence": "adjacent",
|
|
2551
|
+
"source": "arena",
|
|
2552
|
+
"note": "arena has it at max; carried to this effort"
|
|
2553
|
+
},
|
|
2554
|
+
{
|
|
2555
|
+
"rung": "gpt-6-sol#high",
|
|
2556
|
+
"dim": "repo_code",
|
|
2557
|
+
"value": 65.3,
|
|
2558
|
+
"benchmark": "DeepSWE",
|
|
2559
|
+
"version": "1.1",
|
|
2560
|
+
"url": "https://kingy.ai/blog/gpt-6-sol-luna-specs-benchmarks-pricing-comparison/",
|
|
2561
|
+
"date": "2026-09-22",
|
|
2562
|
+
"confidence": "secondary"
|
|
2563
|
+
},
|
|
2564
|
+
{
|
|
2565
|
+
"rung": "gpt-6-sol#high",
|
|
2566
|
+
"dim": "terminal",
|
|
2567
|
+
"value": 43,
|
|
2568
|
+
"benchmark": "Terminal-Bench",
|
|
2569
|
+
"version": "4.0",
|
|
2570
|
+
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
2571
|
+
"date": "2026-09-22",
|
|
2572
|
+
"confidence": "adjacent",
|
|
2573
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
2574
|
+
},
|
|
2575
|
+
{
|
|
2576
|
+
"rung": "gpt-6-sol#high",
|
|
2577
|
+
"dim": "honesty",
|
|
2578
|
+
"value": 95.1,
|
|
2579
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2580
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2581
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
2582
|
+
"date": "2026-09-22",
|
|
2583
|
+
"confidence": "adjacent",
|
|
2584
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
2585
|
+
},
|
|
2586
|
+
{
|
|
2587
|
+
"rung": "gpt-6-sol#high",
|
|
2588
|
+
"dim": "agentic",
|
|
2589
|
+
"value": 0.0818,
|
|
2590
|
+
"benchmark": "Arena agent, net improvement",
|
|
2591
|
+
"version": "2026-09-27",
|
|
2592
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2593
|
+
"date": "2026-09-27",
|
|
2594
|
+
"confidence": "adjacent",
|
|
2595
|
+
"source": "arena",
|
|
2596
|
+
"note": "arena has it at max; carried to this effort"
|
|
2597
|
+
},
|
|
2598
|
+
{
|
|
2599
|
+
"rung": "gpt-6-sol#high",
|
|
2600
|
+
"dim": "steer",
|
|
2601
|
+
"value": 0.1335,
|
|
2602
|
+
"benchmark": "Arena agent steerability",
|
|
2603
|
+
"version": "2026-09-27",
|
|
2604
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2605
|
+
"date": "2026-09-27",
|
|
2606
|
+
"confidence": "adjacent",
|
|
2607
|
+
"source": "arena",
|
|
2608
|
+
"note": "arena has it at max; carried to this effort"
|
|
2609
|
+
},
|
|
2610
|
+
{
|
|
2611
|
+
"rung": "gpt-6-sol#high",
|
|
2612
|
+
"dim": "frontend",
|
|
2613
|
+
"value": 1680.9725,
|
|
2614
|
+
"benchmark": "Arena WebDev rating",
|
|
2615
|
+
"version": "2026-09-25",
|
|
2616
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2617
|
+
"date": "2026-09-25",
|
|
2618
|
+
"confidence": "adjacent",
|
|
2619
|
+
"source": "arena",
|
|
2620
|
+
"note": "arena has it at max; carried to this effort"
|
|
2621
|
+
},
|
|
13
2622
|
{
|
|
14
2623
|
"rung": "gpt-6-sol#low",
|
|
15
2624
|
"dim": "repo_code",
|
|
@@ -21,34 +2630,62 @@
|
|
|
21
2630
|
"confidence": "secondary"
|
|
22
2631
|
},
|
|
23
2632
|
{
|
|
24
|
-
"rung": "gpt-6-sol#
|
|
25
|
-
"dim": "
|
|
26
|
-
"value":
|
|
27
|
-
"benchmark": "
|
|
28
|
-
"version": "
|
|
29
|
-
"url": "https://
|
|
2633
|
+
"rung": "gpt-6-sol#low",
|
|
2634
|
+
"dim": "terminal",
|
|
2635
|
+
"value": 43,
|
|
2636
|
+
"benchmark": "Terminal-Bench",
|
|
2637
|
+
"version": "4.0",
|
|
2638
|
+
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
30
2639
|
"date": "2026-09-22",
|
|
31
|
-
"confidence": "
|
|
2640
|
+
"confidence": "adjacent",
|
|
2641
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
32
2642
|
},
|
|
33
2643
|
{
|
|
34
|
-
"rung": "gpt-6-sol#
|
|
35
|
-
"dim": "
|
|
36
|
-
"value":
|
|
37
|
-
"benchmark": "
|
|
38
|
-
"version": "
|
|
39
|
-
"url": "https://
|
|
2644
|
+
"rung": "gpt-6-sol#low",
|
|
2645
|
+
"dim": "honesty",
|
|
2646
|
+
"value": 95.1,
|
|
2647
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2648
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2649
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
40
2650
|
"date": "2026-09-22",
|
|
41
|
-
"confidence": "
|
|
2651
|
+
"confidence": "adjacent",
|
|
2652
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
42
2653
|
},
|
|
43
2654
|
{
|
|
44
|
-
"rung": "gpt-6-sol#
|
|
45
|
-
"dim": "
|
|
46
|
-
"value":
|
|
47
|
-
"benchmark": "
|
|
48
|
-
"version": "
|
|
49
|
-
"url": "https://
|
|
50
|
-
"date": "2026-09-
|
|
51
|
-
"confidence": "
|
|
2655
|
+
"rung": "gpt-6-sol#low",
|
|
2656
|
+
"dim": "agentic",
|
|
2657
|
+
"value": 0.0818,
|
|
2658
|
+
"benchmark": "Arena agent, net improvement",
|
|
2659
|
+
"version": "2026-09-27",
|
|
2660
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2661
|
+
"date": "2026-09-27",
|
|
2662
|
+
"confidence": "adjacent",
|
|
2663
|
+
"source": "arena",
|
|
2664
|
+
"note": "arena has it at max; carried to this effort"
|
|
2665
|
+
},
|
|
2666
|
+
{
|
|
2667
|
+
"rung": "gpt-6-sol#low",
|
|
2668
|
+
"dim": "steer",
|
|
2669
|
+
"value": 0.1335,
|
|
2670
|
+
"benchmark": "Arena agent steerability",
|
|
2671
|
+
"version": "2026-09-27",
|
|
2672
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2673
|
+
"date": "2026-09-27",
|
|
2674
|
+
"confidence": "adjacent",
|
|
2675
|
+
"source": "arena",
|
|
2676
|
+
"note": "arena has it at max; carried to this effort"
|
|
2677
|
+
},
|
|
2678
|
+
{
|
|
2679
|
+
"rung": "gpt-6-sol#low",
|
|
2680
|
+
"dim": "frontend",
|
|
2681
|
+
"value": 1680.9725,
|
|
2682
|
+
"benchmark": "Arena WebDev rating",
|
|
2683
|
+
"version": "2026-09-25",
|
|
2684
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2685
|
+
"date": "2026-09-25",
|
|
2686
|
+
"confidence": "adjacent",
|
|
2687
|
+
"source": "arena",
|
|
2688
|
+
"note": "arena has it at max; carried to this effort"
|
|
52
2689
|
},
|
|
53
2690
|
{
|
|
54
2691
|
"rung": "gpt-6-sol#max",
|
|
@@ -81,53 +2718,42 @@
|
|
|
81
2718
|
"confidence": "verified"
|
|
82
2719
|
},
|
|
83
2720
|
{
|
|
84
|
-
"rung": "gpt-6-sol#
|
|
85
|
-
"dim": "
|
|
86
|
-
"value":
|
|
87
|
-
"benchmark": "
|
|
88
|
-
"version": "
|
|
89
|
-
"url": "https://
|
|
90
|
-
"date": "2026-09-
|
|
91
|
-
"confidence": "
|
|
92
|
-
"
|
|
93
|
-
},
|
|
94
|
-
{
|
|
95
|
-
"rung": "gpt-6-sol#high",
|
|
96
|
-
"dim": "honesty",
|
|
97
|
-
"value": 95.1,
|
|
98
|
-
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
99
|
-
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
100
|
-
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
101
|
-
"date": "2026-09-22",
|
|
102
|
-
"confidence": "inferred",
|
|
103
|
-
"note": "measured at max effort; carried to this effort"
|
|
2721
|
+
"rung": "gpt-6-sol#max",
|
|
2722
|
+
"dim": "agentic",
|
|
2723
|
+
"value": 0.0818,
|
|
2724
|
+
"benchmark": "Arena agent, net improvement",
|
|
2725
|
+
"version": "2026-09-27",
|
|
2726
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2727
|
+
"date": "2026-09-27",
|
|
2728
|
+
"confidence": "measured",
|
|
2729
|
+
"source": "arena"
|
|
104
2730
|
},
|
|
105
2731
|
{
|
|
106
|
-
"rung": "gpt-6-sol#
|
|
107
|
-
"dim": "
|
|
108
|
-
"value":
|
|
109
|
-
"benchmark": "
|
|
110
|
-
"version": "
|
|
111
|
-
"url": "https://
|
|
112
|
-
"date": "2026-09-
|
|
113
|
-
"confidence": "
|
|
114
|
-
"
|
|
2732
|
+
"rung": "gpt-6-sol#max",
|
|
2733
|
+
"dim": "steer",
|
|
2734
|
+
"value": 0.1335,
|
|
2735
|
+
"benchmark": "Arena agent steerability",
|
|
2736
|
+
"version": "2026-09-27",
|
|
2737
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2738
|
+
"date": "2026-09-27",
|
|
2739
|
+
"confidence": "measured",
|
|
2740
|
+
"source": "arena"
|
|
115
2741
|
},
|
|
116
2742
|
{
|
|
117
|
-
"rung": "gpt-6-
|
|
118
|
-
"dim": "
|
|
119
|
-
"value":
|
|
120
|
-
"benchmark": "
|
|
121
|
-
"version": "
|
|
122
|
-
"url": "https://
|
|
123
|
-
"date": "2026-09-
|
|
124
|
-
"confidence": "
|
|
125
|
-
"
|
|
2743
|
+
"rung": "gpt-6-sol#max",
|
|
2744
|
+
"dim": "frontend",
|
|
2745
|
+
"value": 1680.9725,
|
|
2746
|
+
"benchmark": "Arena WebDev rating",
|
|
2747
|
+
"version": "2026-09-25",
|
|
2748
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2749
|
+
"date": "2026-09-25",
|
|
2750
|
+
"confidence": "measured",
|
|
2751
|
+
"source": "arena"
|
|
126
2752
|
},
|
|
127
2753
|
{
|
|
128
|
-
"rung": "gpt-6-
|
|
2754
|
+
"rung": "gpt-6-sol#medium",
|
|
129
2755
|
"dim": "repo_code",
|
|
130
|
-
"value":
|
|
2756
|
+
"value": 56.6,
|
|
131
2757
|
"benchmark": "DeepSWE",
|
|
132
2758
|
"version": "1.1",
|
|
133
2759
|
"url": "https://kingy.ai/blog/gpt-6-sol-luna-specs-benchmarks-pricing-comparison/",
|
|
@@ -135,232 +2761,418 @@
|
|
|
135
2761
|
"confidence": "secondary"
|
|
136
2762
|
},
|
|
137
2763
|
{
|
|
138
|
-
"rung": "gpt-6-
|
|
2764
|
+
"rung": "gpt-6-sol#medium",
|
|
139
2765
|
"dim": "terminal",
|
|
140
|
-
"value":
|
|
2766
|
+
"value": 43,
|
|
141
2767
|
"benchmark": "Terminal-Bench",
|
|
142
2768
|
"version": "4.0",
|
|
143
2769
|
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
144
2770
|
"date": "2026-09-22",
|
|
145
|
-
"confidence": "
|
|
2771
|
+
"confidence": "adjacent",
|
|
2772
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
146
2773
|
},
|
|
147
2774
|
{
|
|
148
|
-
"rung": "gpt-6-
|
|
2775
|
+
"rung": "gpt-6-sol#medium",
|
|
149
2776
|
"dim": "honesty",
|
|
150
|
-
"value":
|
|
2777
|
+
"value": 95.1,
|
|
151
2778
|
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
152
2779
|
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
153
2780
|
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
154
2781
|
"date": "2026-09-22",
|
|
155
|
-
"confidence": "
|
|
2782
|
+
"confidence": "adjacent",
|
|
2783
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
156
2784
|
},
|
|
157
2785
|
{
|
|
158
|
-
"rung": "gpt-6-
|
|
159
|
-
"dim": "
|
|
160
|
-
"value":
|
|
161
|
-
"benchmark": "
|
|
162
|
-
"version": "
|
|
163
|
-
"url": "https://
|
|
164
|
-
"date": "2026-09-
|
|
165
|
-
"confidence": "
|
|
166
|
-
"
|
|
2786
|
+
"rung": "gpt-6-sol#medium",
|
|
2787
|
+
"dim": "agentic",
|
|
2788
|
+
"value": 0.0818,
|
|
2789
|
+
"benchmark": "Arena agent, net improvement",
|
|
2790
|
+
"version": "2026-09-27",
|
|
2791
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2792
|
+
"date": "2026-09-27",
|
|
2793
|
+
"confidence": "adjacent",
|
|
2794
|
+
"source": "arena",
|
|
2795
|
+
"note": "arena has it at max; carried to this effort"
|
|
167
2796
|
},
|
|
168
2797
|
{
|
|
169
|
-
"rung": "gpt-6-
|
|
2798
|
+
"rung": "gpt-6-sol#medium",
|
|
2799
|
+
"dim": "steer",
|
|
2800
|
+
"value": 0.1335,
|
|
2801
|
+
"benchmark": "Arena agent steerability",
|
|
2802
|
+
"version": "2026-09-27",
|
|
2803
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2804
|
+
"date": "2026-09-27",
|
|
2805
|
+
"confidence": "adjacent",
|
|
2806
|
+
"source": "arena",
|
|
2807
|
+
"note": "arena has it at max; carried to this effort"
|
|
2808
|
+
},
|
|
2809
|
+
{
|
|
2810
|
+
"rung": "gpt-6-sol#medium",
|
|
2811
|
+
"dim": "frontend",
|
|
2812
|
+
"value": 1680.9725,
|
|
2813
|
+
"benchmark": "Arena WebDev rating",
|
|
2814
|
+
"version": "2026-09-25",
|
|
2815
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2816
|
+
"date": "2026-09-25",
|
|
2817
|
+
"confidence": "adjacent",
|
|
2818
|
+
"source": "arena",
|
|
2819
|
+
"note": "arena has it at max; carried to this effort"
|
|
2820
|
+
},
|
|
2821
|
+
{
|
|
2822
|
+
"rung": "gpt-6-sol#none",
|
|
170
2823
|
"dim": "repo_code",
|
|
171
|
-
"value":
|
|
2824
|
+
"value": 37.2,
|
|
172
2825
|
"benchmark": "DeepSWE",
|
|
173
2826
|
"version": "1.1",
|
|
174
|
-
"url": "https://
|
|
2827
|
+
"url": "https://kingy.ai/blog/gpt-6-sol-luna-specs-benchmarks-pricing-comparison/",
|
|
175
2828
|
"date": "2026-09-22",
|
|
176
|
-
"confidence": "
|
|
2829
|
+
"confidence": "adjacent",
|
|
2830
|
+
"note": "DeepSWE has it at low; carried to this effort"
|
|
177
2831
|
},
|
|
178
2832
|
{
|
|
179
|
-
"rung": "gpt-6-
|
|
2833
|
+
"rung": "gpt-6-sol#none",
|
|
180
2834
|
"dim": "terminal",
|
|
181
|
-
"value":
|
|
2835
|
+
"value": 43,
|
|
182
2836
|
"benchmark": "Terminal-Bench",
|
|
183
2837
|
"version": "4.0",
|
|
184
|
-
"url": "https://
|
|
2838
|
+
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
185
2839
|
"date": "2026-09-22",
|
|
186
|
-
"confidence": "
|
|
187
|
-
"note": "
|
|
2840
|
+
"confidence": "adjacent",
|
|
2841
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
188
2842
|
},
|
|
189
2843
|
{
|
|
190
|
-
"rung": "gpt-6-
|
|
2844
|
+
"rung": "gpt-6-sol#none",
|
|
191
2845
|
"dim": "honesty",
|
|
192
|
-
"value":
|
|
2846
|
+
"value": 95.1,
|
|
193
2847
|
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
194
2848
|
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
195
2849
|
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
196
2850
|
"date": "2026-09-22",
|
|
197
|
-
"confidence": "
|
|
2851
|
+
"confidence": "adjacent",
|
|
2852
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
198
2853
|
},
|
|
199
2854
|
{
|
|
200
|
-
"rung": "gpt-6-
|
|
201
|
-
"dim": "
|
|
202
|
-
"value":
|
|
203
|
-
"benchmark": "
|
|
204
|
-
"version": "
|
|
205
|
-
"url": "https://
|
|
206
|
-
"date": "2026-09-
|
|
207
|
-
"confidence": "
|
|
208
|
-
"
|
|
2855
|
+
"rung": "gpt-6-sol#none",
|
|
2856
|
+
"dim": "agentic",
|
|
2857
|
+
"value": 0.0818,
|
|
2858
|
+
"benchmark": "Arena agent, net improvement",
|
|
2859
|
+
"version": "2026-09-27",
|
|
2860
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2861
|
+
"date": "2026-09-27",
|
|
2862
|
+
"confidence": "adjacent",
|
|
2863
|
+
"source": "arena",
|
|
2864
|
+
"note": "arena has it at max; carried to this effort"
|
|
209
2865
|
},
|
|
210
2866
|
{
|
|
211
|
-
"rung": "gpt-
|
|
2867
|
+
"rung": "gpt-6-sol#none",
|
|
2868
|
+
"dim": "steer",
|
|
2869
|
+
"value": 0.1335,
|
|
2870
|
+
"benchmark": "Arena agent steerability",
|
|
2871
|
+
"version": "2026-09-27",
|
|
2872
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2873
|
+
"date": "2026-09-27",
|
|
2874
|
+
"confidence": "adjacent",
|
|
2875
|
+
"source": "arena",
|
|
2876
|
+
"note": "arena has it at max; carried to this effort"
|
|
2877
|
+
},
|
|
2878
|
+
{
|
|
2879
|
+
"rung": "gpt-6-sol#none",
|
|
2880
|
+
"dim": "frontend",
|
|
2881
|
+
"value": 1680.9725,
|
|
2882
|
+
"benchmark": "Arena WebDev rating",
|
|
2883
|
+
"version": "2026-09-25",
|
|
2884
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2885
|
+
"date": "2026-09-25",
|
|
2886
|
+
"confidence": "adjacent",
|
|
2887
|
+
"source": "arena",
|
|
2888
|
+
"note": "arena has it at max; carried to this effort"
|
|
2889
|
+
},
|
|
2890
|
+
{
|
|
2891
|
+
"rung": "gpt-6-sol#ultra",
|
|
212
2892
|
"dim": "repo_code",
|
|
213
|
-
"value":
|
|
2893
|
+
"value": 68.8,
|
|
214
2894
|
"benchmark": "DeepSWE",
|
|
215
2895
|
"version": "1.1",
|
|
216
|
-
"url": "https://
|
|
2896
|
+
"url": "https://x.com/OpenAIDevs/status/2102461464279912495",
|
|
217
2897
|
"date": "2026-09-22",
|
|
218
|
-
"confidence": "
|
|
2898
|
+
"confidence": "adjacent",
|
|
2899
|
+
"note": "DeepSWE has it at max; carried to this effort"
|
|
219
2900
|
},
|
|
220
2901
|
{
|
|
221
|
-
"rung": "gpt-
|
|
2902
|
+
"rung": "gpt-6-sol#ultra",
|
|
222
2903
|
"dim": "terminal",
|
|
223
|
-
"value":
|
|
2904
|
+
"value": 43,
|
|
224
2905
|
"benchmark": "Terminal-Bench",
|
|
225
2906
|
"version": "4.0",
|
|
226
|
-
"url": "https://
|
|
2907
|
+
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
227
2908
|
"date": "2026-09-22",
|
|
228
|
-
"confidence": "
|
|
2909
|
+
"confidence": "adjacent",
|
|
2910
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
229
2911
|
},
|
|
230
2912
|
{
|
|
231
|
-
"rung": "gpt-
|
|
2913
|
+
"rung": "gpt-6-sol#ultra",
|
|
232
2914
|
"dim": "honesty",
|
|
233
|
-
"value":
|
|
2915
|
+
"value": 95.1,
|
|
234
2916
|
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
235
2917
|
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
236
2918
|
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
237
2919
|
"date": "2026-09-22",
|
|
238
|
-
"confidence": "
|
|
2920
|
+
"confidence": "adjacent",
|
|
2921
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
239
2922
|
},
|
|
240
2923
|
{
|
|
241
|
-
"rung": "gpt-
|
|
242
|
-
"dim": "
|
|
243
|
-
"value":
|
|
244
|
-
"benchmark": "
|
|
245
|
-
"version": "
|
|
246
|
-
"url": "https://
|
|
247
|
-
"date": "2026-09-
|
|
248
|
-
"confidence": "
|
|
2924
|
+
"rung": "gpt-6-sol#ultra",
|
|
2925
|
+
"dim": "agentic",
|
|
2926
|
+
"value": 0.0818,
|
|
2927
|
+
"benchmark": "Arena agent, net improvement",
|
|
2928
|
+
"version": "2026-09-27",
|
|
2929
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2930
|
+
"date": "2026-09-27",
|
|
2931
|
+
"confidence": "adjacent",
|
|
2932
|
+
"source": "arena",
|
|
2933
|
+
"note": "arena has it at max; carried to this effort"
|
|
249
2934
|
},
|
|
250
2935
|
{
|
|
251
|
-
"rung": "gpt-
|
|
252
|
-
"dim": "
|
|
253
|
-
"value":
|
|
254
|
-
"benchmark": "
|
|
255
|
-
"version": "
|
|
256
|
-
"url": "https://
|
|
257
|
-
"date": "2026-09-
|
|
258
|
-
"confidence": "
|
|
2936
|
+
"rung": "gpt-6-sol#ultra",
|
|
2937
|
+
"dim": "steer",
|
|
2938
|
+
"value": 0.1335,
|
|
2939
|
+
"benchmark": "Arena agent steerability",
|
|
2940
|
+
"version": "2026-09-27",
|
|
2941
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2942
|
+
"date": "2026-09-27",
|
|
2943
|
+
"confidence": "adjacent",
|
|
2944
|
+
"source": "arena",
|
|
2945
|
+
"note": "arena has it at max; carried to this effort"
|
|
259
2946
|
},
|
|
260
2947
|
{
|
|
261
|
-
"rung": "
|
|
2948
|
+
"rung": "gpt-6-sol#ultra",
|
|
2949
|
+
"dim": "frontend",
|
|
2950
|
+
"value": 1680.9725,
|
|
2951
|
+
"benchmark": "Arena WebDev rating",
|
|
2952
|
+
"version": "2026-09-25",
|
|
2953
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2954
|
+
"date": "2026-09-25",
|
|
2955
|
+
"confidence": "adjacent",
|
|
2956
|
+
"source": "arena",
|
|
2957
|
+
"note": "arena has it at max; carried to this effort"
|
|
2958
|
+
},
|
|
2959
|
+
{
|
|
2960
|
+
"rung": "gpt-6-sol#xhigh",
|
|
262
2961
|
"dim": "repo_code",
|
|
263
|
-
"value":
|
|
2962
|
+
"value": 66.6,
|
|
264
2963
|
"benchmark": "DeepSWE",
|
|
265
2964
|
"version": "1.1",
|
|
266
|
-
"url": "https://
|
|
2965
|
+
"url": "https://kingy.ai/blog/gpt-6-sol-luna-specs-benchmarks-pricing-comparison/",
|
|
267
2966
|
"date": "2026-09-22",
|
|
268
|
-
"confidence": "
|
|
269
|
-
"note": "vendor-run, mean of 5 trials (system card 8.3)"
|
|
2967
|
+
"confidence": "secondary"
|
|
270
2968
|
},
|
|
271
2969
|
{
|
|
272
|
-
"rung": "
|
|
2970
|
+
"rung": "gpt-6-sol#xhigh",
|
|
273
2971
|
"dim": "terminal",
|
|
274
|
-
"value":
|
|
2972
|
+
"value": 43,
|
|
275
2973
|
"benchmark": "Terminal-Bench",
|
|
276
2974
|
"version": "4.0",
|
|
277
|
-
"url": "https://
|
|
2975
|
+
"url": "https://artificialanalysis.ai/articles/gpt-6-sol-and-luna-push-the-cost-efficiency-frontier",
|
|
278
2976
|
"date": "2026-09-22",
|
|
279
|
-
"confidence": "
|
|
280
|
-
"note": "
|
|
2977
|
+
"confidence": "adjacent",
|
|
2978
|
+
"note": "Terminal-Bench has it at max; carried to this effort"
|
|
281
2979
|
},
|
|
282
2980
|
{
|
|
283
|
-
"rung": "
|
|
284
|
-
"dim": "
|
|
285
|
-
"value":
|
|
286
|
-
"benchmark": "
|
|
287
|
-
"version": "4.
|
|
288
|
-
"url": "https://
|
|
2981
|
+
"rung": "gpt-6-sol#xhigh",
|
|
2982
|
+
"dim": "honesty",
|
|
2983
|
+
"value": 95.1,
|
|
2984
|
+
"benchmark": "OpenAI Broken Search Tool, 100 minus the failure rate",
|
|
2985
|
+
"version": "GPT-6 system card appendix 11.6.4.2",
|
|
2986
|
+
"url": "https://deploymentsafety.openai.com/gpt-6-astra/sec:appendix-sol-luna",
|
|
289
2987
|
"date": "2026-09-22",
|
|
290
|
-
"confidence": "
|
|
2988
|
+
"confidence": "adjacent",
|
|
2989
|
+
"note": "OpenAI Broken Search Tool, 100 minus the failure rate has it at max; carried to this effort"
|
|
291
2990
|
},
|
|
292
2991
|
{
|
|
293
|
-
"rung": "
|
|
294
|
-
"dim": "
|
|
295
|
-
"value":
|
|
296
|
-
"benchmark": "
|
|
297
|
-
"version": "
|
|
298
|
-
"url": "https://
|
|
299
|
-
"date": "2026-09-
|
|
300
|
-
"confidence": "
|
|
2992
|
+
"rung": "gpt-6-sol#xhigh",
|
|
2993
|
+
"dim": "agentic",
|
|
2994
|
+
"value": 0.0818,
|
|
2995
|
+
"benchmark": "Arena agent, net improvement",
|
|
2996
|
+
"version": "2026-09-27",
|
|
2997
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
2998
|
+
"date": "2026-09-27",
|
|
2999
|
+
"confidence": "adjacent",
|
|
3000
|
+
"source": "arena",
|
|
3001
|
+
"note": "arena has it at max; carried to this effort"
|
|
301
3002
|
},
|
|
302
3003
|
{
|
|
303
|
-
"rung": "
|
|
304
|
-
"dim": "
|
|
305
|
-
"value":
|
|
306
|
-
"benchmark": "
|
|
307
|
-
"version": "
|
|
308
|
-
"url": "https://
|
|
309
|
-
"date": "2026-09-
|
|
310
|
-
"confidence": "
|
|
311
|
-
"
|
|
3004
|
+
"rung": "gpt-6-sol#xhigh",
|
|
3005
|
+
"dim": "steer",
|
|
3006
|
+
"value": 0.1335,
|
|
3007
|
+
"benchmark": "Arena agent steerability",
|
|
3008
|
+
"version": "2026-09-27",
|
|
3009
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
3010
|
+
"date": "2026-09-27",
|
|
3011
|
+
"confidence": "adjacent",
|
|
3012
|
+
"source": "arena",
|
|
3013
|
+
"note": "arena has it at max; carried to this effort"
|
|
312
3014
|
},
|
|
313
3015
|
{
|
|
314
|
-
"rung": "
|
|
315
|
-
"dim": "
|
|
316
|
-
"value":
|
|
317
|
-
"benchmark": "
|
|
318
|
-
"version": "
|
|
319
|
-
"url": "https://
|
|
320
|
-
"date": "2026-09-
|
|
321
|
-
"confidence": "
|
|
3016
|
+
"rung": "gpt-6-sol#xhigh",
|
|
3017
|
+
"dim": "frontend",
|
|
3018
|
+
"value": 1680.9725,
|
|
3019
|
+
"benchmark": "Arena WebDev rating",
|
|
3020
|
+
"version": "2026-09-25",
|
|
3021
|
+
"url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset",
|
|
3022
|
+
"date": "2026-09-25",
|
|
3023
|
+
"confidence": "adjacent",
|
|
3024
|
+
"source": "arena",
|
|
3025
|
+
"note": "arena has it at max; carried to this effort"
|
|
322
3026
|
}
|
|
323
3027
|
],
|
|
324
3028
|
"treatLike": {
|
|
325
|
-
"claude-opus-5-5#low": { "like": "claude-opus-5-5#xhigh", "note": "only xhigh and max are published" },
|
|
326
|
-
"claude-opus-5-5#medium": { "like": "claude-opus-5-5#xhigh", "note": "only xhigh and max are published" },
|
|
327
|
-
"claude-opus-5-5#high": { "like": "claude-opus-5-5#xhigh", "note": "only xhigh and max are published" },
|
|
328
3029
|
"opencode-go/kimi-k3#max": {
|
|
329
3030
|
"like": "gpt-6-sol#medium",
|
|
330
3031
|
"note": "the default profile's OpenCode Go stand-in for Sol; no DeepSWE 1.1, Terminal-Bench 4.0 or honesty score is published for it"
|
|
3032
|
+
},
|
|
3033
|
+
"gpt-6-luna#none": {
|
|
3034
|
+
"like": "gpt-5.6-luna#none",
|
|
3035
|
+
"note": "no Arena agent score is published for GPT-6 Luna; GPT-5.6 Luna at the same effort stands in for agentic and steer (the nearest stand-in, spec 1.2 §6.3)"
|
|
3036
|
+
},
|
|
3037
|
+
"gpt-6-luna#low": {
|
|
3038
|
+
"like": "gpt-5.6-luna#low",
|
|
3039
|
+
"note": "no Arena agent score is published for GPT-6 Luna; GPT-5.6 Luna at the same effort stands in for agentic and steer (the nearest stand-in, spec 1.2 §6.3)"
|
|
3040
|
+
},
|
|
3041
|
+
"gpt-6-luna#medium": {
|
|
3042
|
+
"like": "gpt-5.6-luna#medium",
|
|
3043
|
+
"note": "no Arena agent score is published for GPT-6 Luna; GPT-5.6 Luna at the same effort stands in for agentic and steer (the nearest stand-in, spec 1.2 §6.3)"
|
|
3044
|
+
},
|
|
3045
|
+
"gpt-6-luna#high": {
|
|
3046
|
+
"like": "gpt-5.6-luna#high",
|
|
3047
|
+
"note": "no Arena agent score is published for GPT-6 Luna; GPT-5.6 Luna at the same effort stands in for agentic and steer (the nearest stand-in, spec 1.2 §6.3)"
|
|
3048
|
+
},
|
|
3049
|
+
"gpt-6-luna#xhigh": {
|
|
3050
|
+
"like": "gpt-5.6-luna#xhigh",
|
|
3051
|
+
"note": "no Arena agent score is published for GPT-6 Luna; GPT-5.6 Luna at the same effort stands in for agentic and steer (the nearest stand-in, spec 1.2 §6.3)"
|
|
3052
|
+
},
|
|
3053
|
+
"gpt-6-luna#max": {
|
|
3054
|
+
"like": "gpt-5.6-luna#max",
|
|
3055
|
+
"note": "no Arena agent score is published for GPT-6 Luna; GPT-5.6 Luna at the same effort stands in for agentic and steer (the nearest stand-in, spec 1.2 §6.3)"
|
|
331
3056
|
}
|
|
332
3057
|
},
|
|
333
3058
|
"bars": {
|
|
334
3059
|
"repo_code": {
|
|
335
|
-
"copy": {
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
"
|
|
3060
|
+
"copy": {
|
|
3061
|
+
"repo_code": 60.95
|
|
3062
|
+
},
|
|
3063
|
+
"build": {
|
|
3064
|
+
"repo_code": 66.6
|
|
3065
|
+
},
|
|
3066
|
+
"logic": {
|
|
3067
|
+
"repo_code": 67,
|
|
3068
|
+
"honesty": 80.82,
|
|
3069
|
+
"agentic": 0.08606
|
|
3070
|
+
},
|
|
3071
|
+
"hard": {
|
|
3072
|
+
"repo_code": 70.9,
|
|
3073
|
+
"honesty": 95.1,
|
|
3074
|
+
"agentic": 0.1077
|
|
3075
|
+
}
|
|
339
3076
|
},
|
|
340
3077
|
"terminal": {
|
|
341
|
-
"copy": {
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
"
|
|
3078
|
+
"copy": {
|
|
3079
|
+
"terminal": 40.15
|
|
3080
|
+
},
|
|
3081
|
+
"build": {
|
|
3082
|
+
"terminal": 55.8
|
|
3083
|
+
},
|
|
3084
|
+
"logic": {
|
|
3085
|
+
"terminal": 57.06,
|
|
3086
|
+
"honesty": 80.82,
|
|
3087
|
+
"agentic": 0.08606
|
|
3088
|
+
},
|
|
3089
|
+
"hard": {
|
|
3090
|
+
"terminal": 61.35,
|
|
3091
|
+
"honesty": 95.1,
|
|
3092
|
+
"agentic": 0.1077
|
|
3093
|
+
}
|
|
345
3094
|
},
|
|
346
3095
|
"ui": {
|
|
347
|
-
"copy": {
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
3096
|
+
"copy": {
|
|
3097
|
+
"frontend": 1539,
|
|
3098
|
+
"repo_code": 60.95
|
|
3099
|
+
},
|
|
3100
|
+
"build": {
|
|
3101
|
+
"frontend": 1617,
|
|
3102
|
+
"repo_code": 66.6
|
|
3103
|
+
},
|
|
3104
|
+
"logic": {
|
|
3105
|
+
"frontend": 1668,
|
|
3106
|
+
"repo_code": 67,
|
|
3107
|
+
"honesty": 80.82,
|
|
3108
|
+
"agentic": 0.08606
|
|
3109
|
+
},
|
|
3110
|
+
"hard": {
|
|
3111
|
+
"frontend": 1751,
|
|
3112
|
+
"repo_code": 70.9,
|
|
3113
|
+
"honesty": 95.1,
|
|
3114
|
+
"agentic": 0.1077
|
|
3115
|
+
}
|
|
351
3116
|
},
|
|
352
3117
|
"prose": {
|
|
353
|
-
"copy": {
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
"
|
|
3118
|
+
"copy": {
|
|
3119
|
+
"repo_code": 60.95
|
|
3120
|
+
},
|
|
3121
|
+
"build": {
|
|
3122
|
+
"repo_code": 66.6
|
|
3123
|
+
},
|
|
3124
|
+
"logic": {
|
|
3125
|
+
"repo_code": 67,
|
|
3126
|
+
"honesty": 80.82
|
|
3127
|
+
},
|
|
3128
|
+
"hard": {
|
|
3129
|
+
"repo_code": 70.9,
|
|
3130
|
+
"honesty": 95.1
|
|
3131
|
+
}
|
|
357
3132
|
},
|
|
358
3133
|
"research": {
|
|
359
|
-
"copy": {
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
"
|
|
3134
|
+
"copy": {
|
|
3135
|
+
"repo_code": 60.95
|
|
3136
|
+
},
|
|
3137
|
+
"build": {
|
|
3138
|
+
"repo_code": 66.6
|
|
3139
|
+
},
|
|
3140
|
+
"logic": {
|
|
3141
|
+
"repo_code": 67,
|
|
3142
|
+
"honesty": 80.82
|
|
3143
|
+
},
|
|
3144
|
+
"hard": {
|
|
3145
|
+
"repo_code": 70.9,
|
|
3146
|
+
"honesty": 95.1
|
|
3147
|
+
}
|
|
363
3148
|
}
|
|
364
3149
|
},
|
|
365
|
-
"barsWhy":
|
|
3150
|
+
"barsWhy": {
|
|
3151
|
+
"repo_code": {
|
|
3152
|
+
"copy": "the 25th percentile of 11 rungs measured or better on DeepSWE 1.1 (2026-09-28): 60.95",
|
|
3153
|
+
"build": "the median of 11 rungs measured or better on DeepSWE 1.1 (2026-09-28): 66.6",
|
|
3154
|
+
"logic": "the 60th percentile of 11 rungs measured or better on DeepSWE 1.1 (2026-09-28): 67",
|
|
3155
|
+
"hard": "the 75th percentile of 11 rungs measured or better on DeepSWE 1.1 (2026-09-28): 70.9"
|
|
3156
|
+
},
|
|
3157
|
+
"terminal": {
|
|
3158
|
+
"copy": "the 25th percentile of 7 rungs measured or better on Terminal-Bench 4.0 (2026-09-28): 40.15",
|
|
3159
|
+
"build": "the median of 7 rungs measured or better on Terminal-Bench 4.0 (2026-09-28): 55.8",
|
|
3160
|
+
"logic": "the 60th percentile of 7 rungs measured or better on Terminal-Bench 4.0 (2026-09-28): 57.06",
|
|
3161
|
+
"hard": "the 75th percentile of 7 rungs measured or better on Terminal-Bench 4.0 (2026-09-28): 61.35"
|
|
3162
|
+
},
|
|
3163
|
+
"honesty": {
|
|
3164
|
+
"logic": "the 60th percentile of 5 rungs measured or better on OpenAI Broken Search Tool, 100 minus the failure rate GPT-6 system card appendix 11.6.4.2 (2026-09-28): 80.82",
|
|
3165
|
+
"hard": "the 75th percentile of 5 rungs measured or better on OpenAI Broken Search Tool, 100 minus the failure rate GPT-6 system card appendix 11.6.4.2 (2026-09-28): 95.1"
|
|
3166
|
+
},
|
|
3167
|
+
"agentic": {
|
|
3168
|
+
"logic": "the 60th percentile of 8 rungs measured or better on Arena agent, net improvement lmarena-ai/leaderboard-dataset (2026-09-28): 0.08606",
|
|
3169
|
+
"hard": "the 75th percentile of 8 rungs measured or better on Arena agent, net improvement lmarena-ai/leaderboard-dataset (2026-09-28): 0.1077"
|
|
3170
|
+
},
|
|
3171
|
+
"frontend": {
|
|
3172
|
+
"copy": "the 25th percentile of 9 rungs measured or better on Arena WebDev rating lmarena-ai/leaderboard-dataset (2026-09-28): 1539",
|
|
3173
|
+
"build": "the median of 9 rungs measured or better on Arena WebDev rating lmarena-ai/leaderboard-dataset (2026-09-28): 1617",
|
|
3174
|
+
"logic": "the 60th percentile of 9 rungs measured or better on Arena WebDev rating lmarena-ai/leaderboard-dataset (2026-09-28): 1668",
|
|
3175
|
+
"hard": "the 75th percentile of 9 rungs measured or better on Arena WebDev rating lmarena-ai/leaderboard-dataset (2026-09-28): 1751"
|
|
3176
|
+
}
|
|
3177
|
+
}
|
|
366
3178
|
}
|