@lucasschirm/devinp 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/data/benchlm-scores.json +65 -79
- package/package.json +1 -1
package/data/benchlm-scores.json
CHANGED
|
@@ -1,219 +1,189 @@
|
|
|
1
1
|
{
|
|
2
|
-
"updated": "2026-
|
|
2
|
+
"updated": "2026-10-25",
|
|
3
3
|
"source": "https://benchlm.ai/api/data/leaderboard?category=coding&limit=200",
|
|
4
4
|
"models": {
|
|
5
5
|
"claude-fable-5": {
|
|
6
|
-
"score":
|
|
6
|
+
"score": 71.36,
|
|
7
7
|
"benchName": "Claude Fable 5",
|
|
8
8
|
"supported": true
|
|
9
9
|
},
|
|
10
10
|
"claude-fable-5.1": {
|
|
11
|
-
"score":
|
|
11
|
+
"score": 76.99,
|
|
12
12
|
"benchName": "Claude Fable 5.1",
|
|
13
13
|
"supported": true
|
|
14
14
|
},
|
|
15
|
-
"claude-haiku-
|
|
16
|
-
"score":
|
|
17
|
-
"benchName": "Claude Haiku
|
|
15
|
+
"claude-haiku-5.5": {
|
|
16
|
+
"score": 52,
|
|
17
|
+
"benchName": "Claude Haiku 5.5",
|
|
18
18
|
"supported": true
|
|
19
19
|
},
|
|
20
|
-
"claude-opus-4.5": {
|
|
21
|
-
"score": 42.56,
|
|
22
|
-
"benchName": "Claude Opus 4.5",
|
|
23
|
-
"supported": false
|
|
24
|
-
},
|
|
25
20
|
"claude-opus-4.6": {
|
|
26
|
-
"score":
|
|
21
|
+
"score": 46.37,
|
|
27
22
|
"benchName": "Claude Opus 4.6",
|
|
28
23
|
"supported": true
|
|
29
24
|
},
|
|
30
25
|
"claude-opus-4.7": {
|
|
31
|
-
"score":
|
|
26
|
+
"score": 55.47,
|
|
32
27
|
"benchName": "Claude Opus 4.7",
|
|
33
28
|
"supported": true
|
|
34
29
|
},
|
|
35
30
|
"claude-opus-4.8": {
|
|
36
|
-
"score":
|
|
31
|
+
"score": 60.69,
|
|
37
32
|
"benchName": "Claude Opus 4.8",
|
|
38
33
|
"supported": true
|
|
39
34
|
},
|
|
40
35
|
"claude-opus-5": {
|
|
41
|
-
"score":
|
|
36
|
+
"score": 70.8,
|
|
42
37
|
"benchName": "Claude Opus 5",
|
|
43
38
|
"supported": true
|
|
44
39
|
},
|
|
45
40
|
"claude-opus-5.5": {
|
|
46
|
-
"score":
|
|
41
|
+
"score": 82.78,
|
|
47
42
|
"benchName": "Claude Opus 5.5",
|
|
48
43
|
"supported": true
|
|
49
44
|
},
|
|
50
45
|
"claude-sonnet-4.6": {
|
|
51
|
-
"score":
|
|
46
|
+
"score": 44.81,
|
|
52
47
|
"benchName": "Claude Sonnet 4.6",
|
|
53
48
|
"supported": true
|
|
54
49
|
},
|
|
55
50
|
"claude-sonnet-5": {
|
|
56
|
-
"score":
|
|
51
|
+
"score": 57.52,
|
|
57
52
|
"benchName": "Claude Sonnet 5",
|
|
58
53
|
"supported": true
|
|
59
54
|
},
|
|
60
55
|
"claude-sonnet-5.5": {
|
|
61
|
-
"score":
|
|
56
|
+
"score": 84.44,
|
|
62
57
|
"benchName": "Claude Sonnet 5.5",
|
|
63
58
|
"supported": true
|
|
64
59
|
},
|
|
65
60
|
"deepseek-v4-pro": {
|
|
66
|
-
"score":
|
|
61
|
+
"score": 47.42,
|
|
67
62
|
"benchName": "DeepSeek V4 Pro 0813",
|
|
68
63
|
"supported": true
|
|
69
64
|
},
|
|
70
65
|
"deepseek-v4.1-flash": {
|
|
71
|
-
"score":
|
|
72
|
-
"benchName": "DeepSeek V4.1 Flash",
|
|
73
|
-
"supported": false
|
|
74
|
-
},
|
|
75
|
-
"gemini-3-flash": {
|
|
76
|
-
"score": 36.38,
|
|
77
|
-
"benchName": "Gemini 3 Flash",
|
|
78
|
-
"supported": true
|
|
79
|
-
},
|
|
80
|
-
"gemini-3.1-pro": {
|
|
81
|
-
"score": 42.44,
|
|
82
|
-
"benchName": "Gemini 3.1 Pro",
|
|
66
|
+
"score": 61.05,
|
|
67
|
+
"benchName": "DeepSeek V4.1 Flash 1017",
|
|
83
68
|
"supported": true
|
|
84
69
|
},
|
|
85
70
|
"gemini-3.5-flash": {
|
|
86
|
-
"score":
|
|
71
|
+
"score": 50.29,
|
|
87
72
|
"benchName": "Gemini 3.5 Flash",
|
|
88
73
|
"supported": true
|
|
89
74
|
},
|
|
90
75
|
"gemini-3.6-flash": {
|
|
91
|
-
"score":
|
|
76
|
+
"score": 51.14,
|
|
92
77
|
"benchName": "Gemini 3.6 Flash",
|
|
93
78
|
"supported": true
|
|
94
79
|
},
|
|
95
80
|
"gemini-3.7-flash": {
|
|
96
|
-
"score":
|
|
81
|
+
"score": 57.71,
|
|
97
82
|
"benchName": "Gemini 3.7 Flash",
|
|
98
83
|
"supported": true
|
|
99
84
|
},
|
|
100
85
|
"gemini-3.8-flash": {
|
|
101
|
-
"score":
|
|
86
|
+
"score": 60.76,
|
|
102
87
|
"benchName": "Gemini 3.8 Flash",
|
|
103
88
|
"supported": true
|
|
104
89
|
},
|
|
90
|
+
"gemini-4-argon": {
|
|
91
|
+
"score": 73.11,
|
|
92
|
+
"benchName": "Gemini 4 Argon",
|
|
93
|
+
"supported": true
|
|
94
|
+
},
|
|
105
95
|
"glm-5.2": {
|
|
106
|
-
"score":
|
|
96
|
+
"score": 53.72,
|
|
107
97
|
"benchName": "GLM-5.2",
|
|
108
98
|
"supported": true
|
|
109
99
|
},
|
|
110
100
|
"glm-5.3": {
|
|
111
|
-
"score": 56.
|
|
101
|
+
"score": 56.19,
|
|
112
102
|
"benchName": "GLM-5.3",
|
|
113
103
|
"supported": true
|
|
114
104
|
},
|
|
115
105
|
"glm-5.3-flash": {
|
|
116
|
-
"score":
|
|
106
|
+
"score": 47.9,
|
|
117
107
|
"benchName": "GLM-5.3-Flash",
|
|
118
108
|
"supported": true
|
|
119
109
|
},
|
|
120
|
-
"gpt-5.1": {
|
|
121
|
-
"score": 38.86,
|
|
122
|
-
"benchName": "GPT-5.1",
|
|
123
|
-
"supported": true
|
|
124
|
-
},
|
|
125
|
-
"gpt-5.2": {
|
|
126
|
-
"score": 39.7,
|
|
127
|
-
"benchName": "GPT-5.2",
|
|
128
|
-
"supported": true
|
|
129
|
-
},
|
|
130
|
-
"gpt-5.3-codex": {
|
|
131
|
-
"score": 55.5,
|
|
132
|
-
"benchName": "GPT-5.3 Codex",
|
|
133
|
-
"supported": true
|
|
134
|
-
},
|
|
135
110
|
"gpt-5.4": {
|
|
136
|
-
"score":
|
|
111
|
+
"score": 47.89,
|
|
137
112
|
"benchName": "GPT-5.4",
|
|
138
113
|
"supported": true
|
|
139
114
|
},
|
|
140
115
|
"gpt-5.4-mini": {
|
|
141
|
-
"score":
|
|
116
|
+
"score": 35.19,
|
|
142
117
|
"benchName": "GPT-5.4 mini",
|
|
143
118
|
"supported": true
|
|
144
119
|
},
|
|
145
120
|
"gpt-5.5": {
|
|
146
|
-
"score": 62.
|
|
121
|
+
"score": 62.06,
|
|
147
122
|
"benchName": "GPT-5.5",
|
|
148
123
|
"supported": true
|
|
149
124
|
},
|
|
150
125
|
"gpt-5.6-luna": {
|
|
151
|
-
"score":
|
|
126
|
+
"score": 61.52,
|
|
152
127
|
"benchName": "GPT-5.6 Luna",
|
|
153
128
|
"supported": true
|
|
154
129
|
},
|
|
155
130
|
"gpt-5.6-sol": {
|
|
156
|
-
"score":
|
|
131
|
+
"score": 67.39,
|
|
157
132
|
"benchName": "GPT-5.6 Sol",
|
|
158
133
|
"supported": true
|
|
159
134
|
},
|
|
160
135
|
"gpt-5.6-terra": {
|
|
161
|
-
"score":
|
|
136
|
+
"score": 61.77,
|
|
162
137
|
"benchName": "GPT-5.6 Terra",
|
|
163
138
|
"supported": true
|
|
164
139
|
},
|
|
165
140
|
"gpt-6-astra": {
|
|
166
|
-
"score":
|
|
141
|
+
"score": 75.67,
|
|
167
142
|
"benchName": "GPT-6 Astra",
|
|
168
143
|
"supported": true
|
|
169
144
|
},
|
|
170
145
|
"gpt-6-luna": {
|
|
171
|
-
"score":
|
|
146
|
+
"score": 53.39,
|
|
172
147
|
"benchName": "GPT-6 Luna",
|
|
173
148
|
"supported": true
|
|
174
149
|
},
|
|
175
150
|
"gpt-6-sol": {
|
|
176
|
-
"score":
|
|
151
|
+
"score": 66.43,
|
|
177
152
|
"benchName": "GPT-6 Sol",
|
|
178
153
|
"supported": true
|
|
179
154
|
},
|
|
180
155
|
"gpt-6.1-sol": {
|
|
181
|
-
"score":
|
|
156
|
+
"score": 71.05,
|
|
182
157
|
"benchName": "GPT-6.1 Sol",
|
|
183
158
|
"supported": true
|
|
184
159
|
},
|
|
185
|
-
"grok-4.5": {
|
|
186
|
-
"score": 58.07,
|
|
187
|
-
"benchName": "Grok 4.5",
|
|
188
|
-
"supported": true
|
|
189
|
-
},
|
|
190
160
|
"grok-4.6": {
|
|
191
|
-
"score":
|
|
161
|
+
"score": 58.5,
|
|
192
162
|
"benchName": "Grok 4.6",
|
|
193
163
|
"supported": true
|
|
194
164
|
},
|
|
195
|
-
"inkling": {
|
|
196
|
-
"score": 36.56,
|
|
197
|
-
"benchName": "Inkling",
|
|
198
|
-
"supported": true
|
|
199
|
-
},
|
|
200
165
|
"kimi-k2.6": {
|
|
201
|
-
"score":
|
|
166
|
+
"score": 43.29,
|
|
202
167
|
"benchName": "Kimi K2.6",
|
|
203
168
|
"supported": true
|
|
204
169
|
},
|
|
205
170
|
"kimi-k2.7": {
|
|
206
|
-
"score":
|
|
171
|
+
"score": 41.83,
|
|
207
172
|
"benchName": "Kimi K2.7 Code",
|
|
208
173
|
"supported": true
|
|
209
174
|
},
|
|
210
175
|
"kimi-k3": {
|
|
211
|
-
"score":
|
|
176
|
+
"score": 58.88,
|
|
212
177
|
"benchName": "Kimi K3",
|
|
213
178
|
"supported": true
|
|
214
179
|
},
|
|
180
|
+
"muse-wonder": {
|
|
181
|
+
"score": 70.13,
|
|
182
|
+
"benchName": "Muse Wonder",
|
|
183
|
+
"supported": true
|
|
184
|
+
},
|
|
215
185
|
"nemotron-3-ultra": {
|
|
216
|
-
"score":
|
|
186
|
+
"score": 21.92,
|
|
217
187
|
"benchName": "Nemotron 3 Ultra",
|
|
218
188
|
"supported": true
|
|
219
189
|
}
|
|
@@ -222,6 +192,8 @@
|
|
|
222
192
|
"Apodex 1.1",
|
|
223
193
|
"Celeris-1",
|
|
224
194
|
"Claude 3 Opus",
|
|
195
|
+
"Claude Haiku 4.5",
|
|
196
|
+
"Claude Opus 4.5",
|
|
225
197
|
"Command A+",
|
|
226
198
|
"DeepSeek V3",
|
|
227
199
|
"DeepSeek V3.2",
|
|
@@ -237,15 +209,20 @@
|
|
|
237
209
|
"GPT-4o mini",
|
|
238
210
|
"GPT-5 (medium)",
|
|
239
211
|
"GPT-5 mini",
|
|
212
|
+
"GPT-5.1",
|
|
240
213
|
"GPT-5.1-Codex",
|
|
214
|
+
"GPT-5.2",
|
|
241
215
|
"GPT-5.2-Codex",
|
|
216
|
+
"GPT-5.3 Codex",
|
|
242
217
|
"GPT-5.4 nano",
|
|
243
218
|
"GPT-OSS 120B",
|
|
244
219
|
"GPT-OSS 20B",
|
|
245
220
|
"Gemini 1.5 Pro",
|
|
246
221
|
"Gemini 2.5 Pro",
|
|
222
|
+
"Gemini 3 Flash",
|
|
247
223
|
"Gemini 3 Pro",
|
|
248
224
|
"Gemini 3.1 Flash-Lite",
|
|
225
|
+
"Gemini 3.1 Pro",
|
|
249
226
|
"Gemini 3.5 Flash-Lite",
|
|
250
227
|
"Gemma 3 27B",
|
|
251
228
|
"Gemma 4 12B",
|
|
@@ -258,11 +235,13 @@
|
|
|
258
235
|
"Grok 4.1 Fast (Reasoning)",
|
|
259
236
|
"Grok 4.20",
|
|
260
237
|
"Grok 4.3",
|
|
238
|
+
"Grok 4.5",
|
|
261
239
|
"Grok Build 0.1",
|
|
262
240
|
"Grok Code Fast 1",
|
|
263
241
|
"Hy3",
|
|
264
242
|
"Hy3 Preview",
|
|
265
243
|
"Hy4 preview",
|
|
244
|
+
"Inkling",
|
|
266
245
|
"Inkling-Small",
|
|
267
246
|
"K-Exaone",
|
|
268
247
|
"Kimi K2.5",
|
|
@@ -271,6 +250,7 @@
|
|
|
271
250
|
"Laguna XS.2",
|
|
272
251
|
"Ling 2.6 Flash",
|
|
273
252
|
"Ling 3.0 Flash",
|
|
253
|
+
"Ling 3.1 Flash",
|
|
274
254
|
"Llama 4 Maverick",
|
|
275
255
|
"Llama 4 Scout",
|
|
276
256
|
"Mercury 2",
|
|
@@ -284,16 +264,20 @@
|
|
|
284
264
|
"MiniMax M2.5",
|
|
285
265
|
"MiniMax M2.7",
|
|
286
266
|
"MiniMax M3",
|
|
267
|
+
"MiniMax M3.1 Flash",
|
|
287
268
|
"Ministral 3 14B",
|
|
288
269
|
"Ministral 3 3B",
|
|
289
270
|
"Ministral 3 8B",
|
|
290
271
|
"Mistral Large 3",
|
|
272
|
+
"Mistral Large 4",
|
|
291
273
|
"Mistral Medium 3.5 128B",
|
|
292
274
|
"Mistral Small 4",
|
|
293
275
|
"Muse Glimmer 30B",
|
|
294
276
|
"Muse Spark",
|
|
295
277
|
"Muse Spark 1.1",
|
|
296
278
|
"Muse Spark 1.2",
|
|
279
|
+
"Muse Spark Open",
|
|
280
|
+
"Muse Wonder Full",
|
|
297
281
|
"Nemotron 3 Nano Omni 30B A3B",
|
|
298
282
|
"Nemotron 3.5 Lightning 30B A3B NVFP4",
|
|
299
283
|
"Ornith-1.5-35B-A3B",
|
|
@@ -312,6 +296,8 @@
|
|
|
312
296
|
"Qwen3.8 Max",
|
|
313
297
|
"Qwen3.8-27B",
|
|
314
298
|
"Qwen3.8-Flash-Next",
|
|
299
|
+
"Sage-A1.5",
|
|
300
|
+
"Spark-X2.6-4B",
|
|
315
301
|
"Step 3.7 Flash",
|
|
316
302
|
"Step 5 Preview",
|
|
317
303
|
"Trinity-Large-Thinking",
|