@lucasschirm/devinp 0.1.14 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/data/benchlm-scores.json +67 -38
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -101,7 +101,7 @@ devinp --json # emit normalized rows as JSON
|
|
|
101
101
|
|
|
102
102
|
```
|
|
103
103
|
│ # │ Model │ Ctx │ Input │ Cached │ Output │ Bench % │ Value │ Sidekick i/c/o │
|
|
104
|
-
│
|
|
104
|
+
│ 10 │ Gemini 3.8 Flash│ 1.05M │ $0.75 │ $0.08 │ $3.75 │ 55.3% │ 14.75 │ — │
|
|
105
105
|
```
|
|
106
106
|
|
|
107
107
|
- Prices are USD per 1M tokens; columns are heat-colored green → red.
|
package/data/benchlm-scores.json
CHANGED
|
@@ -1,195 +1,221 @@
|
|
|
1
1
|
{
|
|
2
|
-
"updated": "2026-
|
|
2
|
+
"updated": "2026-11-13",
|
|
3
3
|
"source": "https://benchlm.ai/api/data/leaderboard?category=coding&limit=200",
|
|
4
4
|
"models": {
|
|
5
5
|
"claude-fable-5": {
|
|
6
|
-
"score":
|
|
6
|
+
"score": 67.82,
|
|
7
7
|
"benchName": "Claude Fable 5",
|
|
8
8
|
"supported": true
|
|
9
9
|
},
|
|
10
10
|
"claude-fable-5.1": {
|
|
11
|
-
"score":
|
|
11
|
+
"score": 73.98,
|
|
12
12
|
"benchName": "Claude Fable 5.1",
|
|
13
13
|
"supported": true
|
|
14
14
|
},
|
|
15
|
+
"claude-fable-5.2": {
|
|
16
|
+
"score": 89.83,
|
|
17
|
+
"benchName": "Claude Fable 5.2",
|
|
18
|
+
"supported": true
|
|
19
|
+
},
|
|
15
20
|
"claude-haiku-5.5": {
|
|
16
|
-
"score":
|
|
21
|
+
"score": 59.02,
|
|
17
22
|
"benchName": "Claude Haiku 5.5",
|
|
18
23
|
"supported": true
|
|
19
24
|
},
|
|
20
25
|
"claude-opus-4.6": {
|
|
21
|
-
"score": 46.
|
|
26
|
+
"score": 46.03,
|
|
22
27
|
"benchName": "Claude Opus 4.6",
|
|
23
28
|
"supported": true
|
|
24
29
|
},
|
|
25
30
|
"claude-opus-4.7": {
|
|
26
|
-
"score":
|
|
31
|
+
"score": 53.66,
|
|
27
32
|
"benchName": "Claude Opus 4.7",
|
|
28
33
|
"supported": true
|
|
29
34
|
},
|
|
30
35
|
"claude-opus-4.8": {
|
|
31
|
-
"score":
|
|
36
|
+
"score": 55.77,
|
|
32
37
|
"benchName": "Claude Opus 4.8",
|
|
33
38
|
"supported": true
|
|
34
39
|
},
|
|
35
40
|
"claude-opus-5": {
|
|
36
|
-
"score":
|
|
41
|
+
"score": 66.83,
|
|
37
42
|
"benchName": "Claude Opus 5",
|
|
38
43
|
"supported": true
|
|
39
44
|
},
|
|
40
45
|
"claude-opus-5.5": {
|
|
41
|
-
"score": 82.
|
|
46
|
+
"score": 82.67,
|
|
42
47
|
"benchName": "Claude Opus 5.5",
|
|
43
48
|
"supported": true
|
|
44
49
|
},
|
|
45
50
|
"claude-sonnet-4.6": {
|
|
46
|
-
"score": 44.
|
|
51
|
+
"score": 44.82,
|
|
47
52
|
"benchName": "Claude Sonnet 4.6",
|
|
48
53
|
"supported": true
|
|
49
54
|
},
|
|
50
55
|
"claude-sonnet-5": {
|
|
51
|
-
"score":
|
|
56
|
+
"score": 51.68,
|
|
52
57
|
"benchName": "Claude Sonnet 5",
|
|
53
58
|
"supported": true
|
|
54
59
|
},
|
|
55
60
|
"claude-sonnet-5.5": {
|
|
56
|
-
"score":
|
|
61
|
+
"score": 81.04,
|
|
57
62
|
"benchName": "Claude Sonnet 5.5",
|
|
58
63
|
"supported": true
|
|
59
64
|
},
|
|
60
65
|
"deepseek-v4-pro": {
|
|
61
|
-
"score":
|
|
66
|
+
"score": 46.49,
|
|
62
67
|
"benchName": "DeepSeek V4 Pro 0813",
|
|
63
68
|
"supported": true
|
|
64
69
|
},
|
|
65
70
|
"deepseek-v4.1-flash": {
|
|
66
|
-
"score":
|
|
71
|
+
"score": 59.74,
|
|
67
72
|
"benchName": "DeepSeek V4.1 Flash 1017",
|
|
68
73
|
"supported": true
|
|
69
74
|
},
|
|
70
75
|
"gemini-3.5-flash": {
|
|
71
|
-
"score": 50.
|
|
76
|
+
"score": 50.15,
|
|
72
77
|
"benchName": "Gemini 3.5 Flash",
|
|
73
78
|
"supported": true
|
|
74
79
|
},
|
|
75
80
|
"gemini-3.6-flash": {
|
|
76
|
-
"score":
|
|
81
|
+
"score": 50.87,
|
|
77
82
|
"benchName": "Gemini 3.6 Flash",
|
|
78
83
|
"supported": true
|
|
79
84
|
},
|
|
80
85
|
"gemini-3.7-flash": {
|
|
81
|
-
"score":
|
|
86
|
+
"score": 54.74,
|
|
82
87
|
"benchName": "Gemini 3.7 Flash",
|
|
83
88
|
"supported": true
|
|
84
89
|
},
|
|
85
90
|
"gemini-3.8-flash": {
|
|
86
|
-
"score":
|
|
91
|
+
"score": 55.32,
|
|
87
92
|
"benchName": "Gemini 3.8 Flash",
|
|
88
93
|
"supported": true
|
|
89
94
|
},
|
|
90
95
|
"gemini-4-argon": {
|
|
91
|
-
"score":
|
|
96
|
+
"score": 71.88,
|
|
92
97
|
"benchName": "Gemini 4 Argon",
|
|
93
98
|
"supported": true
|
|
94
99
|
},
|
|
100
|
+
"gemini-4-carbon": {
|
|
101
|
+
"score": 83.97,
|
|
102
|
+
"benchName": "Gemini 4 Carbon",
|
|
103
|
+
"supported": true
|
|
104
|
+
},
|
|
105
|
+
"gemini-4-hydrogen": {
|
|
106
|
+
"score": 75.86,
|
|
107
|
+
"benchName": "Gemini 4 Hydrogen",
|
|
108
|
+
"supported": true
|
|
109
|
+
},
|
|
95
110
|
"glm-5.2": {
|
|
96
|
-
"score":
|
|
111
|
+
"score": 52.18,
|
|
97
112
|
"benchName": "GLM-5.2",
|
|
98
113
|
"supported": true
|
|
99
114
|
},
|
|
100
115
|
"glm-5.3": {
|
|
101
|
-
"score":
|
|
116
|
+
"score": 53.71,
|
|
102
117
|
"benchName": "GLM-5.3",
|
|
103
118
|
"supported": true
|
|
104
119
|
},
|
|
105
120
|
"glm-5.3-flash": {
|
|
106
|
-
"score":
|
|
121
|
+
"score": 45.49,
|
|
107
122
|
"benchName": "GLM-5.3-Flash",
|
|
108
123
|
"supported": true
|
|
109
124
|
},
|
|
110
125
|
"gpt-5.4": {
|
|
111
|
-
"score":
|
|
126
|
+
"score": 48.82,
|
|
112
127
|
"benchName": "GPT-5.4",
|
|
113
128
|
"supported": true
|
|
114
129
|
},
|
|
115
130
|
"gpt-5.4-mini": {
|
|
116
|
-
"score":
|
|
131
|
+
"score": 34.05,
|
|
117
132
|
"benchName": "GPT-5.4 mini",
|
|
118
133
|
"supported": true
|
|
119
134
|
},
|
|
120
135
|
"gpt-5.5": {
|
|
121
|
-
"score":
|
|
136
|
+
"score": 57.72,
|
|
122
137
|
"benchName": "GPT-5.5",
|
|
123
138
|
"supported": true
|
|
124
139
|
},
|
|
125
140
|
"gpt-5.6-luna": {
|
|
126
|
-
"score":
|
|
141
|
+
"score": 57.66,
|
|
127
142
|
"benchName": "GPT-5.6 Luna",
|
|
128
143
|
"supported": true
|
|
129
144
|
},
|
|
130
145
|
"gpt-5.6-sol": {
|
|
131
|
-
"score":
|
|
146
|
+
"score": 62.46,
|
|
132
147
|
"benchName": "GPT-5.6 Sol",
|
|
133
148
|
"supported": true
|
|
134
149
|
},
|
|
135
150
|
"gpt-5.6-terra": {
|
|
136
|
-
"score":
|
|
151
|
+
"score": 57.19,
|
|
137
152
|
"benchName": "GPT-5.6 Terra",
|
|
138
153
|
"supported": true
|
|
139
154
|
},
|
|
140
155
|
"gpt-6-astra": {
|
|
141
|
-
"score":
|
|
156
|
+
"score": 78.73,
|
|
142
157
|
"benchName": "GPT-6 Astra",
|
|
143
158
|
"supported": true
|
|
144
159
|
},
|
|
145
160
|
"gpt-6-luna": {
|
|
146
|
-
"score":
|
|
161
|
+
"score": 52.04,
|
|
147
162
|
"benchName": "GPT-6 Luna",
|
|
148
163
|
"supported": true
|
|
149
164
|
},
|
|
150
165
|
"gpt-6-sol": {
|
|
151
|
-
"score":
|
|
166
|
+
"score": 67.31,
|
|
152
167
|
"benchName": "GPT-6 Sol",
|
|
153
168
|
"supported": true
|
|
154
169
|
},
|
|
155
170
|
"gpt-6.1-sol": {
|
|
156
|
-
"score":
|
|
171
|
+
"score": 73.39,
|
|
157
172
|
"benchName": "GPT-6.1 Sol",
|
|
158
173
|
"supported": true
|
|
159
174
|
},
|
|
175
|
+
"gpt-6.1-terra": {
|
|
176
|
+
"score": 69.8,
|
|
177
|
+
"benchName": "GPT-6.1 Terra",
|
|
178
|
+
"supported": true
|
|
179
|
+
},
|
|
160
180
|
"grok-4.6": {
|
|
161
|
-
"score":
|
|
181
|
+
"score": 54.63,
|
|
162
182
|
"benchName": "Grok 4.6",
|
|
163
183
|
"supported": true
|
|
164
184
|
},
|
|
165
185
|
"kimi-k2.6": {
|
|
166
|
-
"score": 43.
|
|
186
|
+
"score": 43.12,
|
|
167
187
|
"benchName": "Kimi K2.6",
|
|
168
188
|
"supported": true
|
|
169
189
|
},
|
|
170
190
|
"kimi-k2.7": {
|
|
171
|
-
"score": 41.
|
|
191
|
+
"score": 41.85,
|
|
172
192
|
"benchName": "Kimi K2.7 Code",
|
|
173
193
|
"supported": true
|
|
174
194
|
},
|
|
175
195
|
"kimi-k3": {
|
|
176
|
-
"score":
|
|
196
|
+
"score": 56.24,
|
|
177
197
|
"benchName": "Kimi K3",
|
|
178
198
|
"supported": true
|
|
179
199
|
},
|
|
200
|
+
"kimi-k3.1-pro": {
|
|
201
|
+
"score": 61.03,
|
|
202
|
+
"benchName": "Kimi K3.1 Pro",
|
|
203
|
+
"supported": true
|
|
204
|
+
},
|
|
180
205
|
"muse-wonder": {
|
|
181
|
-
"score":
|
|
206
|
+
"score": 64.99,
|
|
182
207
|
"benchName": "Muse Wonder",
|
|
183
208
|
"supported": true
|
|
184
209
|
},
|
|
185
210
|
"nemotron-3-ultra": {
|
|
186
|
-
"score":
|
|
211
|
+
"score": 23.6,
|
|
187
212
|
"benchName": "Nemotron 3 Ultra",
|
|
188
213
|
"supported": true
|
|
189
214
|
}
|
|
190
215
|
},
|
|
191
216
|
"unmatched": [
|
|
192
217
|
"Apodex 1.1",
|
|
218
|
+
"Beam 1.0 501B-A23B",
|
|
193
219
|
"Celeris-1",
|
|
194
220
|
"Claude 3 Opus",
|
|
195
221
|
"Claude Haiku 4.5",
|
|
@@ -242,10 +268,13 @@
|
|
|
242
268
|
"Hy3 Preview",
|
|
243
269
|
"Hy4 preview",
|
|
244
270
|
"Inkling",
|
|
271
|
+
"Inkling 2 Small",
|
|
245
272
|
"Inkling-Small",
|
|
246
273
|
"K-Exaone",
|
|
247
274
|
"Kimi K2.5",
|
|
275
|
+
"Kimi K2.5 (Reasoning)",
|
|
248
276
|
"LFM2.5-2.6B",
|
|
277
|
+
"LFM3-1.7B",
|
|
249
278
|
"Laguna M.1",
|
|
250
279
|
"Laguna XS.2",
|
|
251
280
|
"Ling 2.6 Flash",
|