@lucasschirm/devinp 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/benchlm-scores.json +65 -80
- package/package.json +1 -1
package/data/benchlm-scores.json
CHANGED
|
@@ -1,219 +1,189 @@
|
|
|
1
1
|
{
|
|
2
|
-
"updated": "2026-10-
|
|
2
|
+
"updated": "2026-10-25",
|
|
3
3
|
"source": "https://benchlm.ai/api/data/leaderboard?category=coding&limit=200",
|
|
4
4
|
"models": {
|
|
5
5
|
"claude-fable-5": {
|
|
6
|
-
"score":
|
|
6
|
+
"score": 71.36,
|
|
7
7
|
"benchName": "Claude Fable 5",
|
|
8
8
|
"supported": true
|
|
9
9
|
},
|
|
10
10
|
"claude-fable-5.1": {
|
|
11
|
-
"score":
|
|
11
|
+
"score": 76.99,
|
|
12
12
|
"benchName": "Claude Fable 5.1",
|
|
13
13
|
"supported": true
|
|
14
14
|
},
|
|
15
|
-
"claude-haiku-
|
|
16
|
-
"score":
|
|
17
|
-
"benchName": "Claude Haiku
|
|
15
|
+
"claude-haiku-5.5": {
|
|
16
|
+
"score": 52,
|
|
17
|
+
"benchName": "Claude Haiku 5.5",
|
|
18
18
|
"supported": true
|
|
19
19
|
},
|
|
20
|
-
"claude-opus-4.5": {
|
|
21
|
-
"score": 42.64,
|
|
22
|
-
"benchName": "Claude Opus 4.5",
|
|
23
|
-
"supported": false
|
|
24
|
-
},
|
|
25
20
|
"claude-opus-4.6": {
|
|
26
|
-
"score":
|
|
21
|
+
"score": 46.37,
|
|
27
22
|
"benchName": "Claude Opus 4.6",
|
|
28
23
|
"supported": true
|
|
29
24
|
},
|
|
30
25
|
"claude-opus-4.7": {
|
|
31
|
-
"score":
|
|
26
|
+
"score": 55.47,
|
|
32
27
|
"benchName": "Claude Opus 4.7",
|
|
33
28
|
"supported": true
|
|
34
29
|
},
|
|
35
30
|
"claude-opus-4.8": {
|
|
36
|
-
"score":
|
|
31
|
+
"score": 60.69,
|
|
37
32
|
"benchName": "Claude Opus 4.8",
|
|
38
33
|
"supported": true
|
|
39
34
|
},
|
|
40
35
|
"claude-opus-5": {
|
|
41
|
-
"score":
|
|
36
|
+
"score": 70.8,
|
|
42
37
|
"benchName": "Claude Opus 5",
|
|
43
38
|
"supported": true
|
|
44
39
|
},
|
|
45
40
|
"claude-opus-5.5": {
|
|
46
|
-
"score":
|
|
41
|
+
"score": 82.78,
|
|
47
42
|
"benchName": "Claude Opus 5.5",
|
|
48
43
|
"supported": true
|
|
49
44
|
},
|
|
50
45
|
"claude-sonnet-4.6": {
|
|
51
|
-
"score":
|
|
46
|
+
"score": 44.81,
|
|
52
47
|
"benchName": "Claude Sonnet 4.6",
|
|
53
48
|
"supported": true
|
|
54
49
|
},
|
|
55
50
|
"claude-sonnet-5": {
|
|
56
|
-
"score":
|
|
51
|
+
"score": 57.52,
|
|
57
52
|
"benchName": "Claude Sonnet 5",
|
|
58
53
|
"supported": true
|
|
59
54
|
},
|
|
60
55
|
"claude-sonnet-5.5": {
|
|
61
|
-
"score":
|
|
56
|
+
"score": 84.44,
|
|
62
57
|
"benchName": "Claude Sonnet 5.5",
|
|
63
58
|
"supported": true
|
|
64
59
|
},
|
|
65
60
|
"deepseek-v4-pro": {
|
|
66
|
-
"score":
|
|
61
|
+
"score": 47.42,
|
|
67
62
|
"benchName": "DeepSeek V4 Pro 0813",
|
|
68
63
|
"supported": true
|
|
69
64
|
},
|
|
70
65
|
"deepseek-v4.1-flash": {
|
|
71
|
-
"score":
|
|
72
|
-
"benchName": "DeepSeek V4.1 Flash",
|
|
73
|
-
"supported": false
|
|
74
|
-
},
|
|
75
|
-
"gemini-3-flash": {
|
|
76
|
-
"score": 36.32,
|
|
77
|
-
"benchName": "Gemini 3 Flash",
|
|
78
|
-
"supported": true
|
|
79
|
-
},
|
|
80
|
-
"gemini-3.1-pro": {
|
|
81
|
-
"score": 42.32,
|
|
82
|
-
"benchName": "Gemini 3.1 Pro",
|
|
66
|
+
"score": 61.05,
|
|
67
|
+
"benchName": "DeepSeek V4.1 Flash 1017",
|
|
83
68
|
"supported": true
|
|
84
69
|
},
|
|
85
70
|
"gemini-3.5-flash": {
|
|
86
|
-
"score":
|
|
71
|
+
"score": 50.29,
|
|
87
72
|
"benchName": "Gemini 3.5 Flash",
|
|
88
73
|
"supported": true
|
|
89
74
|
},
|
|
90
75
|
"gemini-3.6-flash": {
|
|
91
|
-
"score":
|
|
76
|
+
"score": 51.14,
|
|
92
77
|
"benchName": "Gemini 3.6 Flash",
|
|
93
78
|
"supported": true
|
|
94
79
|
},
|
|
95
80
|
"gemini-3.7-flash": {
|
|
96
|
-
"score":
|
|
81
|
+
"score": 57.71,
|
|
97
82
|
"benchName": "Gemini 3.7 Flash",
|
|
98
83
|
"supported": true
|
|
99
84
|
},
|
|
100
85
|
"gemini-3.8-flash": {
|
|
101
|
-
"score":
|
|
86
|
+
"score": 60.76,
|
|
102
87
|
"benchName": "Gemini 3.8 Flash",
|
|
103
88
|
"supported": true
|
|
104
89
|
},
|
|
90
|
+
"gemini-4-argon": {
|
|
91
|
+
"score": 73.11,
|
|
92
|
+
"benchName": "Gemini 4 Argon",
|
|
93
|
+
"supported": true
|
|
94
|
+
},
|
|
105
95
|
"glm-5.2": {
|
|
106
|
-
"score":
|
|
96
|
+
"score": 53.72,
|
|
107
97
|
"benchName": "GLM-5.2",
|
|
108
98
|
"supported": true
|
|
109
99
|
},
|
|
110
100
|
"glm-5.3": {
|
|
111
|
-
"score": 56.
|
|
101
|
+
"score": 56.19,
|
|
112
102
|
"benchName": "GLM-5.3",
|
|
113
103
|
"supported": true
|
|
114
104
|
},
|
|
115
105
|
"glm-5.3-flash": {
|
|
116
|
-
"score":
|
|
106
|
+
"score": 47.9,
|
|
117
107
|
"benchName": "GLM-5.3-Flash",
|
|
118
108
|
"supported": true
|
|
119
109
|
},
|
|
120
|
-
"gpt-5.1": {
|
|
121
|
-
"score": 38.92,
|
|
122
|
-
"benchName": "GPT-5.1",
|
|
123
|
-
"supported": true
|
|
124
|
-
},
|
|
125
|
-
"gpt-5.2": {
|
|
126
|
-
"score": 39.68,
|
|
127
|
-
"benchName": "GPT-5.2",
|
|
128
|
-
"supported": true
|
|
129
|
-
},
|
|
130
|
-
"gpt-5.3-codex": {
|
|
131
|
-
"score": 56.08,
|
|
132
|
-
"benchName": "GPT-5.3 Codex",
|
|
133
|
-
"supported": true
|
|
134
|
-
},
|
|
135
110
|
"gpt-5.4": {
|
|
136
|
-
"score":
|
|
111
|
+
"score": 47.89,
|
|
137
112
|
"benchName": "GPT-5.4",
|
|
138
113
|
"supported": true
|
|
139
114
|
},
|
|
140
115
|
"gpt-5.4-mini": {
|
|
141
|
-
"score":
|
|
116
|
+
"score": 35.19,
|
|
142
117
|
"benchName": "GPT-5.4 mini",
|
|
143
118
|
"supported": true
|
|
144
119
|
},
|
|
145
120
|
"gpt-5.5": {
|
|
146
|
-
"score": 62.
|
|
121
|
+
"score": 62.06,
|
|
147
122
|
"benchName": "GPT-5.5",
|
|
148
123
|
"supported": true
|
|
149
124
|
},
|
|
150
125
|
"gpt-5.6-luna": {
|
|
151
|
-
"score":
|
|
126
|
+
"score": 61.52,
|
|
152
127
|
"benchName": "GPT-5.6 Luna",
|
|
153
128
|
"supported": true
|
|
154
129
|
},
|
|
155
130
|
"gpt-5.6-sol": {
|
|
156
|
-
"score":
|
|
131
|
+
"score": 67.39,
|
|
157
132
|
"benchName": "GPT-5.6 Sol",
|
|
158
133
|
"supported": true
|
|
159
134
|
},
|
|
160
135
|
"gpt-5.6-terra": {
|
|
161
|
-
"score":
|
|
136
|
+
"score": 61.77,
|
|
162
137
|
"benchName": "GPT-5.6 Terra",
|
|
163
138
|
"supported": true
|
|
164
139
|
},
|
|
165
140
|
"gpt-6-astra": {
|
|
166
|
-
"score":
|
|
141
|
+
"score": 75.67,
|
|
167
142
|
"benchName": "GPT-6 Astra",
|
|
168
143
|
"supported": true
|
|
169
144
|
},
|
|
170
145
|
"gpt-6-luna": {
|
|
171
|
-
"score":
|
|
146
|
+
"score": 53.39,
|
|
172
147
|
"benchName": "GPT-6 Luna",
|
|
173
148
|
"supported": true
|
|
174
149
|
},
|
|
175
150
|
"gpt-6-sol": {
|
|
176
|
-
"score":
|
|
151
|
+
"score": 66.43,
|
|
177
152
|
"benchName": "GPT-6 Sol",
|
|
178
153
|
"supported": true
|
|
179
154
|
},
|
|
180
155
|
"gpt-6.1-sol": {
|
|
181
|
-
"score":
|
|
156
|
+
"score": 71.05,
|
|
182
157
|
"benchName": "GPT-6.1 Sol",
|
|
183
158
|
"supported": true
|
|
184
159
|
},
|
|
185
|
-
"grok-4.5": {
|
|
186
|
-
"score": 58.11,
|
|
187
|
-
"benchName": "Grok 4.5",
|
|
188
|
-
"supported": true
|
|
189
|
-
},
|
|
190
160
|
"grok-4.6": {
|
|
191
|
-
"score":
|
|
161
|
+
"score": 58.5,
|
|
192
162
|
"benchName": "Grok 4.6",
|
|
193
163
|
"supported": true
|
|
194
164
|
},
|
|
195
|
-
"inkling": {
|
|
196
|
-
"score": 36.41,
|
|
197
|
-
"benchName": "Inkling",
|
|
198
|
-
"supported": true
|
|
199
|
-
},
|
|
200
165
|
"kimi-k2.6": {
|
|
201
|
-
"score":
|
|
166
|
+
"score": 43.29,
|
|
202
167
|
"benchName": "Kimi K2.6",
|
|
203
168
|
"supported": true
|
|
204
169
|
},
|
|
205
170
|
"kimi-k2.7": {
|
|
206
|
-
"score":
|
|
171
|
+
"score": 41.83,
|
|
207
172
|
"benchName": "Kimi K2.7 Code",
|
|
208
173
|
"supported": true
|
|
209
174
|
},
|
|
210
175
|
"kimi-k3": {
|
|
211
|
-
"score":
|
|
176
|
+
"score": 58.88,
|
|
212
177
|
"benchName": "Kimi K3",
|
|
213
178
|
"supported": true
|
|
214
179
|
},
|
|
180
|
+
"muse-wonder": {
|
|
181
|
+
"score": 70.13,
|
|
182
|
+
"benchName": "Muse Wonder",
|
|
183
|
+
"supported": true
|
|
184
|
+
},
|
|
215
185
|
"nemotron-3-ultra": {
|
|
216
|
-
"score":
|
|
186
|
+
"score": 21.92,
|
|
217
187
|
"benchName": "Nemotron 3 Ultra",
|
|
218
188
|
"supported": true
|
|
219
189
|
}
|
|
@@ -222,6 +192,8 @@
|
|
|
222
192
|
"Apodex 1.1",
|
|
223
193
|
"Celeris-1",
|
|
224
194
|
"Claude 3 Opus",
|
|
195
|
+
"Claude Haiku 4.5",
|
|
196
|
+
"Claude Opus 4.5",
|
|
225
197
|
"Command A+",
|
|
226
198
|
"DeepSeek V3",
|
|
227
199
|
"DeepSeek V3.2",
|
|
@@ -237,17 +209,21 @@
|
|
|
237
209
|
"GPT-4o mini",
|
|
238
210
|
"GPT-5 (medium)",
|
|
239
211
|
"GPT-5 mini",
|
|
212
|
+
"GPT-5.1",
|
|
240
213
|
"GPT-5.1-Codex",
|
|
214
|
+
"GPT-5.2",
|
|
241
215
|
"GPT-5.2-Codex",
|
|
216
|
+
"GPT-5.3 Codex",
|
|
242
217
|
"GPT-5.4 nano",
|
|
243
218
|
"GPT-OSS 120B",
|
|
244
219
|
"GPT-OSS 20B",
|
|
245
220
|
"Gemini 1.5 Pro",
|
|
246
221
|
"Gemini 2.5 Pro",
|
|
222
|
+
"Gemini 3 Flash",
|
|
247
223
|
"Gemini 3 Pro",
|
|
248
224
|
"Gemini 3.1 Flash-Lite",
|
|
225
|
+
"Gemini 3.1 Pro",
|
|
249
226
|
"Gemini 3.5 Flash-Lite",
|
|
250
|
-
"Gemini 4 Argon",
|
|
251
227
|
"Gemma 3 27B",
|
|
252
228
|
"Gemma 4 12B",
|
|
253
229
|
"Gemma 4 26B A4B",
|
|
@@ -259,11 +235,13 @@
|
|
|
259
235
|
"Grok 4.1 Fast (Reasoning)",
|
|
260
236
|
"Grok 4.20",
|
|
261
237
|
"Grok 4.3",
|
|
238
|
+
"Grok 4.5",
|
|
262
239
|
"Grok Build 0.1",
|
|
263
240
|
"Grok Code Fast 1",
|
|
264
241
|
"Hy3",
|
|
265
242
|
"Hy3 Preview",
|
|
266
243
|
"Hy4 preview",
|
|
244
|
+
"Inkling",
|
|
267
245
|
"Inkling-Small",
|
|
268
246
|
"K-Exaone",
|
|
269
247
|
"Kimi K2.5",
|
|
@@ -272,6 +250,7 @@
|
|
|
272
250
|
"Laguna XS.2",
|
|
273
251
|
"Ling 2.6 Flash",
|
|
274
252
|
"Ling 3.0 Flash",
|
|
253
|
+
"Ling 3.1 Flash",
|
|
275
254
|
"Llama 4 Maverick",
|
|
276
255
|
"Llama 4 Scout",
|
|
277
256
|
"Mercury 2",
|
|
@@ -285,16 +264,20 @@
|
|
|
285
264
|
"MiniMax M2.5",
|
|
286
265
|
"MiniMax M2.7",
|
|
287
266
|
"MiniMax M3",
|
|
267
|
+
"MiniMax M3.1 Flash",
|
|
288
268
|
"Ministral 3 14B",
|
|
289
269
|
"Ministral 3 3B",
|
|
290
270
|
"Ministral 3 8B",
|
|
291
271
|
"Mistral Large 3",
|
|
272
|
+
"Mistral Large 4",
|
|
292
273
|
"Mistral Medium 3.5 128B",
|
|
293
274
|
"Mistral Small 4",
|
|
294
275
|
"Muse Glimmer 30B",
|
|
295
276
|
"Muse Spark",
|
|
296
277
|
"Muse Spark 1.1",
|
|
297
278
|
"Muse Spark 1.2",
|
|
279
|
+
"Muse Spark Open",
|
|
280
|
+
"Muse Wonder Full",
|
|
298
281
|
"Nemotron 3 Nano Omni 30B A3B",
|
|
299
282
|
"Nemotron 3.5 Lightning 30B A3B NVFP4",
|
|
300
283
|
"Ornith-1.5-35B-A3B",
|
|
@@ -313,6 +296,8 @@
|
|
|
313
296
|
"Qwen3.8 Max",
|
|
314
297
|
"Qwen3.8-27B",
|
|
315
298
|
"Qwen3.8-Flash-Next",
|
|
299
|
+
"Sage-A1.5",
|
|
300
|
+
"Spark-X2.6-4B",
|
|
316
301
|
"Step 3.7 Flash",
|
|
317
302
|
"Step 5 Preview",
|
|
318
303
|
"Trinity-Large-Thinking",
|