@aggc/or-info 0.2.6 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/mcp/server.mjs +118 -8
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -7,6 +7,7 @@ to make informed decisions about which model to use.
|
|
|
7
7
|
|
|
8
8
|
[](https://www.npmjs.com/package/@aggc/or-info)
|
|
9
9
|
[](https://github.com/jmtrs/or-info/actions/workflows/ci.yml)
|
|
10
|
+
[](https://smithery.ai/servers/aggc/or-info)
|
|
10
11
|
[](LICENSE)
|
|
11
12
|
|
|
12
13
|
## Install
|
package/mcp/server.mjs
CHANGED
|
@@ -6,6 +6,26 @@ import { getElo, getAllElo, loadLeaderboard } from '../lib/lmarena.mjs';
|
|
|
6
6
|
import { rankModels } from '../lib/scorer.mjs';
|
|
7
7
|
import { getApiKey } from '../lib/secrets.mjs';
|
|
8
8
|
|
|
9
|
+
const MODEL_SUMMARY_SCHEMA = {
|
|
10
|
+
type: 'object',
|
|
11
|
+
properties: {
|
|
12
|
+
id: { type: 'string' },
|
|
13
|
+
name: { type: 'string' },
|
|
14
|
+
input_per_m: { type: ['number', 'null'], description: 'Input price per 1M tokens (USD)' },
|
|
15
|
+
output_per_m: { type: ['number', 'null'], description: 'Output price per 1M tokens (USD)' },
|
|
16
|
+
image_per_m: { type: ['number', 'null'] },
|
|
17
|
+
cache_read_per_m: { type: ['number', 'null'] },
|
|
18
|
+
context_length: { type: ['integer', 'null'] },
|
|
19
|
+
features: { type: 'array', items: { type: 'string' } },
|
|
20
|
+
modality: { type: ['string', 'null'] },
|
|
21
|
+
tokenizer: { type: ['string', 'null'] },
|
|
22
|
+
max_output_tokens: { type: ['integer', 'null'] },
|
|
23
|
+
supported_parameters: { type: 'array', items: { type: 'string' } },
|
|
24
|
+
},
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const ELO_SCHEMA = { type: ['object', 'null'], description: 'LMArena ELO entry or null when not tracked' };
|
|
28
|
+
|
|
9
29
|
const TOOLS = [
|
|
10
30
|
{
|
|
11
31
|
name: 'get_model_info',
|
|
@@ -17,6 +37,16 @@ const TOOLS = [
|
|
|
17
37
|
},
|
|
18
38
|
required: ['model_id'],
|
|
19
39
|
},
|
|
40
|
+
outputSchema: {
|
|
41
|
+
type: 'object',
|
|
42
|
+
properties: { ...MODEL_SUMMARY_SCHEMA.properties, lmarena_elo: ELO_SCHEMA },
|
|
43
|
+
},
|
|
44
|
+
annotations: {
|
|
45
|
+
title: 'Get model info',
|
|
46
|
+
readOnlyHint: true,
|
|
47
|
+
idempotentHint: true,
|
|
48
|
+
openWorldHint: true,
|
|
49
|
+
},
|
|
20
50
|
},
|
|
21
51
|
{
|
|
22
52
|
name: 'list_models',
|
|
@@ -30,6 +60,20 @@ const TOOLS = [
|
|
|
30
60
|
free_only: { type: 'boolean', description: 'Return only free models' },
|
|
31
61
|
},
|
|
32
62
|
},
|
|
63
|
+
outputSchema: {
|
|
64
|
+
type: 'object',
|
|
65
|
+
properties: {
|
|
66
|
+
total: { type: 'integer' },
|
|
67
|
+
models: { type: 'array', items: MODEL_SUMMARY_SCHEMA },
|
|
68
|
+
},
|
|
69
|
+
required: ['total', 'models'],
|
|
70
|
+
},
|
|
71
|
+
annotations: {
|
|
72
|
+
title: 'List models',
|
|
73
|
+
readOnlyHint: true,
|
|
74
|
+
idempotentHint: true,
|
|
75
|
+
openWorldHint: true,
|
|
76
|
+
},
|
|
33
77
|
},
|
|
34
78
|
{
|
|
35
79
|
name: 'get_benchmarks',
|
|
@@ -41,6 +85,20 @@ const TOOLS = [
|
|
|
41
85
|
},
|
|
42
86
|
required: ['model_id'],
|
|
43
87
|
},
|
|
88
|
+
outputSchema: {
|
|
89
|
+
type: 'object',
|
|
90
|
+
properties: {
|
|
91
|
+
model_id: { type: 'string' },
|
|
92
|
+
lmarena_elo: ELO_SCHEMA,
|
|
93
|
+
},
|
|
94
|
+
required: ['model_id'],
|
|
95
|
+
},
|
|
96
|
+
annotations: {
|
|
97
|
+
title: 'Get benchmarks',
|
|
98
|
+
readOnlyHint: true,
|
|
99
|
+
idempotentHint: true,
|
|
100
|
+
openWorldHint: true,
|
|
101
|
+
},
|
|
44
102
|
},
|
|
45
103
|
{
|
|
46
104
|
name: 'compare_models',
|
|
@@ -53,6 +111,20 @@ const TOOLS = [
|
|
|
53
111
|
},
|
|
54
112
|
required: ['model_a', 'model_b'],
|
|
55
113
|
},
|
|
114
|
+
outputSchema: {
|
|
115
|
+
type: 'object',
|
|
116
|
+
properties: {
|
|
117
|
+
a: { type: 'object', properties: { ...MODEL_SUMMARY_SCHEMA.properties, lmarena_elo: ELO_SCHEMA } },
|
|
118
|
+
b: { type: 'object', properties: { ...MODEL_SUMMARY_SCHEMA.properties, lmarena_elo: ELO_SCHEMA } },
|
|
119
|
+
},
|
|
120
|
+
required: ['a', 'b'],
|
|
121
|
+
},
|
|
122
|
+
annotations: {
|
|
123
|
+
title: 'Compare models',
|
|
124
|
+
readOnlyHint: true,
|
|
125
|
+
idempotentHint: true,
|
|
126
|
+
openWorldHint: true,
|
|
127
|
+
},
|
|
56
128
|
},
|
|
57
129
|
{
|
|
58
130
|
name: 'best_for_task',
|
|
@@ -73,11 +145,46 @@ const TOOLS = [
|
|
|
73
145
|
},
|
|
74
146
|
required: ['task'],
|
|
75
147
|
},
|
|
148
|
+
outputSchema: {
|
|
149
|
+
type: 'object',
|
|
150
|
+
properties: {
|
|
151
|
+
task: { type: 'string' },
|
|
152
|
+
results: {
|
|
153
|
+
type: 'array',
|
|
154
|
+
items: {
|
|
155
|
+
type: 'object',
|
|
156
|
+
properties: { ...MODEL_SUMMARY_SCHEMA.properties, score: { type: 'number' }, lmarena_elo: ELO_SCHEMA },
|
|
157
|
+
},
|
|
158
|
+
},
|
|
159
|
+
},
|
|
160
|
+
required: ['task', 'results'],
|
|
161
|
+
},
|
|
162
|
+
annotations: {
|
|
163
|
+
title: 'Best models for task',
|
|
164
|
+
readOnlyHint: true,
|
|
165
|
+
idempotentHint: true,
|
|
166
|
+
openWorldHint: true,
|
|
167
|
+
},
|
|
76
168
|
},
|
|
77
169
|
{
|
|
78
170
|
name: 'refresh_cache',
|
|
79
171
|
description: 'Force-refresh the local cache: OpenRouter model catalog + LMArena ELO data',
|
|
80
172
|
inputSchema: { type: 'object', properties: {} },
|
|
173
|
+
outputSchema: {
|
|
174
|
+
type: 'object',
|
|
175
|
+
properties: {
|
|
176
|
+
refreshed: { type: 'boolean' },
|
|
177
|
+
models_count: { type: 'integer' },
|
|
178
|
+
elo_entries: { type: 'integer' },
|
|
179
|
+
},
|
|
180
|
+
required: ['refreshed', 'models_count', 'elo_entries'],
|
|
181
|
+
},
|
|
182
|
+
annotations: {
|
|
183
|
+
title: 'Refresh cache',
|
|
184
|
+
readOnlyHint: false,
|
|
185
|
+
idempotentHint: true,
|
|
186
|
+
openWorldHint: true,
|
|
187
|
+
},
|
|
81
188
|
},
|
|
82
189
|
];
|
|
83
190
|
|
|
@@ -99,8 +206,11 @@ function safeModelSummary(model) {
|
|
|
99
206
|
};
|
|
100
207
|
}
|
|
101
208
|
|
|
102
|
-
function
|
|
103
|
-
return
|
|
209
|
+
function result(obj) {
|
|
210
|
+
return {
|
|
211
|
+
content: [{ type: 'text', text: JSON.stringify(obj, null, 2) }],
|
|
212
|
+
structuredContent: obj,
|
|
213
|
+
};
|
|
104
214
|
}
|
|
105
215
|
|
|
106
216
|
function errorContent(msg) {
|
|
@@ -117,7 +227,7 @@ async function handleTool(name, args) {
|
|
|
117
227
|
const model = findModel(models, model_id);
|
|
118
228
|
if (!model) return errorContent(`Model not found: ${model_id}`);
|
|
119
229
|
const elo = await getElo(model_id);
|
|
120
|
-
return
|
|
230
|
+
return result({ ...safeModelSummary(model), lmarena_elo: elo ?? null });
|
|
121
231
|
}
|
|
122
232
|
|
|
123
233
|
if (name === 'list_models') {
|
|
@@ -135,14 +245,14 @@ async function handleTool(name, args) {
|
|
|
135
245
|
else models.sort((a, b) => a.id.localeCompare(b.id));
|
|
136
246
|
|
|
137
247
|
models = models.slice(0, limit);
|
|
138
|
-
return
|
|
248
|
+
return result({ total: models.length, models: models.map(safeModelSummary) });
|
|
139
249
|
}
|
|
140
250
|
|
|
141
251
|
if (name === 'get_benchmarks') {
|
|
142
252
|
const { model_id } = args;
|
|
143
253
|
if (!model_id || typeof model_id !== 'string') return errorContent('model_id is required');
|
|
144
254
|
const elo = await getElo(model_id);
|
|
145
|
-
return
|
|
255
|
+
return result({ model_id, lmarena_elo: elo ?? null });
|
|
146
256
|
}
|
|
147
257
|
|
|
148
258
|
if (name === 'compare_models') {
|
|
@@ -157,7 +267,7 @@ async function handleTool(name, args) {
|
|
|
157
267
|
const mB = findModel(models, model_b);
|
|
158
268
|
if (!mA) return errorContent(`Model not found: ${model_a}`);
|
|
159
269
|
if (!mB) return errorContent(`Model not found: ${model_b}`);
|
|
160
|
-
return
|
|
270
|
+
return result({ a: { ...safeModelSummary(mA), lmarena_elo: eloA }, b: { ...safeModelSummary(mB), lmarena_elo: eloB } });
|
|
161
271
|
}
|
|
162
272
|
|
|
163
273
|
if (name === 'best_for_task') {
|
|
@@ -167,7 +277,7 @@ async function handleTool(name, args) {
|
|
|
167
277
|
|
|
168
278
|
const [models, allElo] = await Promise.all([fetchModels({ apiKey: key }), getAllElo()]);
|
|
169
279
|
const ranked = rankModels(models, allElo, { task, maxPricePerMOutput: maxPrice, limit });
|
|
170
|
-
return
|
|
280
|
+
return result({ task, results: ranked.map((r) => ({ ...safeModelSummary(r.model), score: r.score, lmarena_elo: r.eloEntry })) });
|
|
171
281
|
}
|
|
172
282
|
|
|
173
283
|
if (name === 'refresh_cache') {
|
|
@@ -175,7 +285,7 @@ async function handleTool(name, args) {
|
|
|
175
285
|
fetchModels({ force: true, apiKey: key }),
|
|
176
286
|
loadLeaderboard({ force: true }),
|
|
177
287
|
]);
|
|
178
|
-
return
|
|
288
|
+
return result({ refreshed: true, models_count: models.length, elo_entries: elo.length });
|
|
179
289
|
}
|
|
180
290
|
|
|
181
291
|
return errorContent(`Unknown tool: ${name}`);
|