@lucasschirm/devinp 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/benchlm-scores.json +69 -41
- package/package.json +1 -1
package/data/benchlm-scores.json
CHANGED
|
@@ -1,206 +1,231 @@
|
|
|
1
1
|
{
|
|
2
|
-
"updated": "2026-09-
|
|
2
|
+
"updated": "2026-09-29",
|
|
3
3
|
"source": "https://benchlm.ai/api/data/leaderboard?category=coding&limit=200",
|
|
4
4
|
"models": {
|
|
5
5
|
"claude-fable-5": {
|
|
6
|
-
"score":
|
|
6
|
+
"score": 73.1,
|
|
7
7
|
"benchName": "Claude Fable 5",
|
|
8
8
|
"supported": true
|
|
9
9
|
},
|
|
10
10
|
"claude-fable-5.1": {
|
|
11
|
-
"score":
|
|
11
|
+
"score": 79.99,
|
|
12
12
|
"benchName": "Claude Fable 5.1",
|
|
13
13
|
"supported": true
|
|
14
14
|
},
|
|
15
15
|
"claude-haiku-4.5": {
|
|
16
|
-
"score": 19.
|
|
16
|
+
"score": 19.61,
|
|
17
17
|
"benchName": "Claude Haiku 4.5",
|
|
18
18
|
"supported": true
|
|
19
19
|
},
|
|
20
20
|
"claude-opus-4.5": {
|
|
21
|
-
"score":
|
|
21
|
+
"score": 42.56,
|
|
22
22
|
"benchName": "Claude Opus 4.5",
|
|
23
23
|
"supported": false
|
|
24
24
|
},
|
|
25
25
|
"claude-opus-4.6": {
|
|
26
|
-
"score":
|
|
26
|
+
"score": 49.68,
|
|
27
27
|
"benchName": "Claude Opus 4.6",
|
|
28
28
|
"supported": true
|
|
29
29
|
},
|
|
30
30
|
"claude-opus-4.7": {
|
|
31
|
-
"score":
|
|
31
|
+
"score": 58.1,
|
|
32
32
|
"benchName": "Claude Opus 4.7",
|
|
33
33
|
"supported": true
|
|
34
34
|
},
|
|
35
35
|
"claude-opus-4.8": {
|
|
36
|
-
"score":
|
|
36
|
+
"score": 62.37,
|
|
37
37
|
"benchName": "Claude Opus 4.8",
|
|
38
38
|
"supported": true
|
|
39
39
|
},
|
|
40
40
|
"claude-opus-5": {
|
|
41
|
-
"score": 72.
|
|
41
|
+
"score": 72.18,
|
|
42
42
|
"benchName": "Claude Opus 5",
|
|
43
43
|
"supported": true
|
|
44
44
|
},
|
|
45
45
|
"claude-opus-5.5": {
|
|
46
|
-
"score":
|
|
46
|
+
"score": 83.6,
|
|
47
47
|
"benchName": "Claude Opus 5.5",
|
|
48
48
|
"supported": true
|
|
49
49
|
},
|
|
50
50
|
"claude-sonnet-4.6": {
|
|
51
|
-
"score":
|
|
51
|
+
"score": 47.04,
|
|
52
52
|
"benchName": "Claude Sonnet 4.6",
|
|
53
53
|
"supported": true
|
|
54
54
|
},
|
|
55
55
|
"claude-sonnet-5": {
|
|
56
|
-
"score":
|
|
56
|
+
"score": 59.96,
|
|
57
57
|
"benchName": "Claude Sonnet 5",
|
|
58
58
|
"supported": true
|
|
59
59
|
},
|
|
60
|
+
"claude-sonnet-5.5": {
|
|
61
|
+
"score": 85.12,
|
|
62
|
+
"benchName": "Claude Sonnet 5.5",
|
|
63
|
+
"supported": true
|
|
64
|
+
},
|
|
65
|
+
"deepseek-v4-pro": {
|
|
66
|
+
"score": 49.36,
|
|
67
|
+
"benchName": "DeepSeek V4 Pro 0813",
|
|
68
|
+
"supported": true
|
|
69
|
+
},
|
|
70
|
+
"deepseek-v4.1-flash": {
|
|
71
|
+
"score": 49.63,
|
|
72
|
+
"benchName": "DeepSeek V4.1 Flash",
|
|
73
|
+
"supported": false
|
|
74
|
+
},
|
|
60
75
|
"gemini-3-flash": {
|
|
61
|
-
"score": 36.
|
|
76
|
+
"score": 36.38,
|
|
62
77
|
"benchName": "Gemini 3 Flash",
|
|
63
78
|
"supported": true
|
|
64
79
|
},
|
|
65
80
|
"gemini-3.1-pro": {
|
|
66
|
-
"score":
|
|
81
|
+
"score": 42.44,
|
|
67
82
|
"benchName": "Gemini 3.1 Pro",
|
|
68
83
|
"supported": true
|
|
69
84
|
},
|
|
70
85
|
"gemini-3.5-flash": {
|
|
71
|
-
"score":
|
|
86
|
+
"score": 52.43,
|
|
72
87
|
"benchName": "Gemini 3.5 Flash",
|
|
73
88
|
"supported": true
|
|
74
89
|
},
|
|
75
90
|
"gemini-3.6-flash": {
|
|
76
|
-
"score":
|
|
91
|
+
"score": 53.78,
|
|
77
92
|
"benchName": "Gemini 3.6 Flash",
|
|
78
93
|
"supported": true
|
|
79
94
|
},
|
|
80
95
|
"gemini-3.7-flash": {
|
|
81
|
-
"score": 59.
|
|
96
|
+
"score": 59.48,
|
|
82
97
|
"benchName": "Gemini 3.7 Flash",
|
|
83
98
|
"supported": true
|
|
84
99
|
},
|
|
85
100
|
"gemini-3.8-flash": {
|
|
86
|
-
"score": 64.
|
|
101
|
+
"score": 64.16,
|
|
87
102
|
"benchName": "Gemini 3.8 Flash",
|
|
88
103
|
"supported": true
|
|
89
104
|
},
|
|
90
105
|
"glm-5.2": {
|
|
91
|
-
"score":
|
|
106
|
+
"score": 56.29,
|
|
92
107
|
"benchName": "GLM-5.2",
|
|
93
108
|
"supported": true
|
|
94
109
|
},
|
|
95
110
|
"glm-5.3": {
|
|
96
|
-
"score":
|
|
111
|
+
"score": 56.84,
|
|
97
112
|
"benchName": "GLM-5.3",
|
|
98
113
|
"supported": true
|
|
99
114
|
},
|
|
100
115
|
"glm-5.3-flash": {
|
|
101
|
-
"score":
|
|
116
|
+
"score": 52.54,
|
|
102
117
|
"benchName": "GLM-5.3-Flash",
|
|
103
118
|
"supported": true
|
|
104
119
|
},
|
|
105
120
|
"gpt-5.1": {
|
|
106
|
-
"score":
|
|
121
|
+
"score": 38.86,
|
|
107
122
|
"benchName": "GPT-5.1",
|
|
108
123
|
"supported": true
|
|
109
124
|
},
|
|
110
125
|
"gpt-5.2": {
|
|
111
|
-
"score":
|
|
126
|
+
"score": 39.7,
|
|
112
127
|
"benchName": "GPT-5.2",
|
|
113
128
|
"supported": true
|
|
114
129
|
},
|
|
115
130
|
"gpt-5.3-codex": {
|
|
116
|
-
"score":
|
|
131
|
+
"score": 55.5,
|
|
117
132
|
"benchName": "GPT-5.3 Codex",
|
|
118
133
|
"supported": true
|
|
119
134
|
},
|
|
120
135
|
"gpt-5.4": {
|
|
121
|
-
"score":
|
|
136
|
+
"score": 49.83,
|
|
122
137
|
"benchName": "GPT-5.4",
|
|
123
138
|
"supported": true
|
|
124
139
|
},
|
|
125
140
|
"gpt-5.4-mini": {
|
|
126
|
-
"score":
|
|
141
|
+
"score": 36.92,
|
|
127
142
|
"benchName": "GPT-5.4 mini",
|
|
128
143
|
"supported": true
|
|
129
144
|
},
|
|
130
145
|
"gpt-5.5": {
|
|
131
|
-
"score":
|
|
146
|
+
"score": 62.83,
|
|
132
147
|
"benchName": "GPT-5.5",
|
|
133
148
|
"supported": true
|
|
134
149
|
},
|
|
135
150
|
"gpt-5.6-luna": {
|
|
136
|
-
"score":
|
|
151
|
+
"score": 63.3,
|
|
137
152
|
"benchName": "GPT-5.6 Luna",
|
|
138
153
|
"supported": true
|
|
139
154
|
},
|
|
140
155
|
"gpt-5.6-sol": {
|
|
141
|
-
"score":
|
|
156
|
+
"score": 70.53,
|
|
142
157
|
"benchName": "GPT-5.6 Sol",
|
|
143
158
|
"supported": true
|
|
144
159
|
},
|
|
145
160
|
"gpt-5.6-terra": {
|
|
146
|
-
"score":
|
|
161
|
+
"score": 64.41,
|
|
147
162
|
"benchName": "GPT-5.6 Terra",
|
|
148
163
|
"supported": true
|
|
149
164
|
},
|
|
150
165
|
"gpt-6-astra": {
|
|
151
|
-
"score": 74.
|
|
166
|
+
"score": 74.18,
|
|
152
167
|
"benchName": "GPT-6 Astra",
|
|
153
168
|
"supported": true
|
|
154
169
|
},
|
|
155
170
|
"gpt-6-luna": {
|
|
156
|
-
"score":
|
|
171
|
+
"score": 51.37,
|
|
157
172
|
"benchName": "GPT-6 Luna",
|
|
158
173
|
"supported": true
|
|
159
174
|
},
|
|
160
175
|
"gpt-6-sol": {
|
|
161
|
-
"score":
|
|
176
|
+
"score": 62.99,
|
|
162
177
|
"benchName": "GPT-6 Sol",
|
|
163
178
|
"supported": true
|
|
164
179
|
},
|
|
180
|
+
"gpt-6.1-sol": {
|
|
181
|
+
"score": 66.97,
|
|
182
|
+
"benchName": "GPT-6.1 Sol",
|
|
183
|
+
"supported": true
|
|
184
|
+
},
|
|
165
185
|
"grok-4.5": {
|
|
166
|
-
"score": 58.
|
|
186
|
+
"score": 58.07,
|
|
167
187
|
"benchName": "Grok 4.5",
|
|
168
188
|
"supported": true
|
|
169
189
|
},
|
|
170
190
|
"grok-4.6": {
|
|
171
|
-
"score": 62
|
|
191
|
+
"score": 61.62,
|
|
172
192
|
"benchName": "Grok 4.6",
|
|
173
193
|
"supported": true
|
|
174
194
|
},
|
|
175
195
|
"inkling": {
|
|
176
|
-
"score":
|
|
196
|
+
"score": 36.56,
|
|
177
197
|
"benchName": "Inkling",
|
|
178
198
|
"supported": true
|
|
179
199
|
},
|
|
180
200
|
"kimi-k2.6": {
|
|
181
|
-
"score":
|
|
201
|
+
"score": 46.36,
|
|
182
202
|
"benchName": "Kimi K2.6",
|
|
183
203
|
"supported": true
|
|
184
204
|
},
|
|
205
|
+
"kimi-k2.7": {
|
|
206
|
+
"score": 44.45,
|
|
207
|
+
"benchName": "Kimi K2.7 Code",
|
|
208
|
+
"supported": true
|
|
209
|
+
},
|
|
185
210
|
"kimi-k3": {
|
|
186
|
-
"score":
|
|
211
|
+
"score": 61.59,
|
|
187
212
|
"benchName": "Kimi K3",
|
|
188
213
|
"supported": true
|
|
189
214
|
},
|
|
190
215
|
"nemotron-3-ultra": {
|
|
191
|
-
"score":
|
|
216
|
+
"score": 22.78,
|
|
192
217
|
"benchName": "Nemotron 3 Ultra",
|
|
193
218
|
"supported": true
|
|
194
219
|
}
|
|
195
220
|
},
|
|
196
221
|
"unmatched": [
|
|
197
222
|
"Apodex 1.1",
|
|
223
|
+
"Celeris-1",
|
|
198
224
|
"Claude 3 Opus",
|
|
199
225
|
"Command A+",
|
|
200
226
|
"DeepSeek V3",
|
|
201
227
|
"DeepSeek V3.2",
|
|
202
228
|
"DeepSeek V3.2 (Thinking)",
|
|
203
|
-
"DeepSeek V4 Pro 0813",
|
|
204
229
|
"GLM-4.6",
|
|
205
230
|
"GLM-4.7",
|
|
206
231
|
"GLM-5",
|
|
@@ -241,7 +266,6 @@
|
|
|
241
266
|
"Inkling-Small",
|
|
242
267
|
"K-Exaone",
|
|
243
268
|
"Kimi K2.5",
|
|
244
|
-
"Kimi K2.7 Code",
|
|
245
269
|
"LFM2.5-2.6B",
|
|
246
270
|
"Laguna M.1",
|
|
247
271
|
"Laguna XS.2",
|
|
@@ -255,6 +279,8 @@
|
|
|
255
279
|
"MiMo-V2-Pro",
|
|
256
280
|
"MiMo-V2.5",
|
|
257
281
|
"MiMo-V2.5-Pro",
|
|
282
|
+
"MiMo-V2.6-Flash",
|
|
283
|
+
"MiMo-V2.6-Pro",
|
|
258
284
|
"MiniMax M2.5",
|
|
259
285
|
"MiniMax M2.7",
|
|
260
286
|
"MiniMax M3",
|
|
@@ -272,6 +298,7 @@
|
|
|
272
298
|
"Nemotron 3.5 Lightning 30B A3B NVFP4",
|
|
273
299
|
"Ornith-1.5-35B-A3B",
|
|
274
300
|
"Ornith-1.5-397B",
|
|
301
|
+
"Quasar 438B",
|
|
275
302
|
"Qwen 3.6 Max (preview)",
|
|
276
303
|
"Qwen3.5 Flash",
|
|
277
304
|
"Qwen3.5-122B-A10B",
|
|
@@ -286,6 +313,7 @@
|
|
|
286
313
|
"Qwen3.8-27B",
|
|
287
314
|
"Qwen3.8-Flash-Next",
|
|
288
315
|
"Step 3.7 Flash",
|
|
316
|
+
"Step 5 Preview",
|
|
289
317
|
"Trinity-Large-Thinking",
|
|
290
318
|
"o1",
|
|
291
319
|
"o1-preview"
|