@agentic.artists/modelshortlist 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.local.example +6 -0
- package/ATTRIBUTION.md +25 -0
- package/LICENSE +21 -0
- package/README.md +240 -0
- package/bin/modelshortlist.js +3 -0
- package/config/aliases.json +3 -0
- package/lib/artificial-analysis.js +104 -0
- package/lib/catalog.js +117 -0
- package/lib/http.js +68 -0
- package/lib/match.js +100 -0
- package/lib/openrouter.js +276 -0
- package/mcp/server.js +369 -0
- package/package.json +56 -0
- package/scripts/setup.js +179 -0
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
import { fetchJson } from './http.js'
|
|
2
|
+
|
|
3
|
+
const MODELS_URL = 'https://openrouter.ai/api/v1/models'
|
|
4
|
+
const ZDR_URL = 'https://openrouter.ai/api/v1/endpoints/zdr'
|
|
5
|
+
|
|
6
|
+
function requireApiKey() {
|
|
7
|
+
const apiKey = process.env.OPENROUTER_API_KEY
|
|
8
|
+
if (!apiKey) {
|
|
9
|
+
throw new Error('OPENROUTER_API_KEY is not configured')
|
|
10
|
+
}
|
|
11
|
+
return apiKey
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function authHeaders() {
|
|
15
|
+
return {
|
|
16
|
+
authorization: `Bearer ${requireApiKey()}`,
|
|
17
|
+
accept: 'application/json',
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export async function fetchOpenRouterModels() {
|
|
22
|
+
const { body } = await fetchJson(MODELS_URL, { headers: authHeaders() })
|
|
23
|
+
|
|
24
|
+
if (!Array.isArray(body?.data)) {
|
|
25
|
+
throw new Error('Unexpected OpenRouter models response: data is not an array')
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
return body.data
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export async function fetchOpenRouterZdrEndpoints() {
|
|
32
|
+
const { body } = await fetchJson(ZDR_URL, { headers: authHeaders() })
|
|
33
|
+
|
|
34
|
+
if (!Array.isArray(body?.data)) {
|
|
35
|
+
throw new Error('Unexpected OpenRouter ZDR response: data is not an array')
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
return body.data
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function finiteNumbers(values) {
|
|
42
|
+
return values
|
|
43
|
+
.filter((value) => value !== null && value !== undefined && value !== '')
|
|
44
|
+
.map(Number)
|
|
45
|
+
.filter(Number.isFinite)
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function minOrNull(values) {
|
|
49
|
+
const nums = finiteNumbers(values)
|
|
50
|
+
return nums.length ? Math.min(...nums) : null
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function maxOrNull(values) {
|
|
54
|
+
const nums = finiteNumbers(values)
|
|
55
|
+
return nums.length ? Math.max(...nums) : null
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function medianOrNull(values) {
|
|
59
|
+
const nums = finiteNumbers(values).sort((a, b) => a - b)
|
|
60
|
+
if (!nums.length) return null
|
|
61
|
+
const mid = Math.floor(nums.length / 2)
|
|
62
|
+
return nums.length % 2 ? nums[mid] : (nums[mid - 1] + nums[mid]) / 2
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function pricePerMillion(value) {
|
|
66
|
+
if (value === null || value === undefined || value === '') return null
|
|
67
|
+
const n = Number(value)
|
|
68
|
+
return Number.isFinite(n) ? n * 1_000_000 : null
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function supportsToolUse(parameters = []) {
|
|
72
|
+
const set = new Set(parameters.map((v) => String(v).toLowerCase()))
|
|
73
|
+
return set.has('tools') || set.has('tool_choice')
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function numberOrNull(value) {
|
|
77
|
+
if (value === null || value === undefined || value === '') return null
|
|
78
|
+
const n = Number(value)
|
|
79
|
+
return Number.isFinite(n) ? n : null
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function compactZdrEndpoint(row) {
|
|
83
|
+
return {
|
|
84
|
+
provider: row.provider_name ?? null,
|
|
85
|
+
context_length: numberOrNull(row.context_length),
|
|
86
|
+
max_prompt_tokens: numberOrNull(row.max_prompt_tokens),
|
|
87
|
+
max_completion_tokens: numberOrNull(row.max_completion_tokens),
|
|
88
|
+
supports_tools: supportsToolUse(row.supported_parameters),
|
|
89
|
+
supported_parameters: (row.supported_parameters ?? []).map(String).sort(),
|
|
90
|
+
pricing_usd_per_1m_tokens: {
|
|
91
|
+
input: pricePerMillion(row.pricing?.prompt),
|
|
92
|
+
output: pricePerMillion(row.pricing?.completion),
|
|
93
|
+
},
|
|
94
|
+
performance: {
|
|
95
|
+
latency_p50_seconds: numberOrNull(row.latency_last_30m?.p50),
|
|
96
|
+
throughput_p50_tokens_per_second:
|
|
97
|
+
numberOrNull(row.throughput_last_30m?.p50),
|
|
98
|
+
uptime_last_1d_percent: numberOrNull(row.uptime_last_1d),
|
|
99
|
+
},
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export function groupZdrEndpointsByModel(endpoints) {
|
|
104
|
+
const grouped = new Map()
|
|
105
|
+
|
|
106
|
+
for (const endpoint of endpoints) {
|
|
107
|
+
if (!endpoint?.model_id) continue
|
|
108
|
+
const list = grouped.get(endpoint.model_id) ?? []
|
|
109
|
+
list.push(endpoint)
|
|
110
|
+
grouped.set(endpoint.model_id, list)
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
return [...grouped.entries()].map(([modelId, rows]) => {
|
|
114
|
+
const endpointOptions = rows.map(compactZdrEndpoint)
|
|
115
|
+
const providers = [...new Set(rows.map((row) => row.provider_name).filter(Boolean))]
|
|
116
|
+
const toolOptions = endpointOptions.filter((row) => row.supports_tools)
|
|
117
|
+
const toolProviders = [...new Set(toolOptions.map((row) => row.provider).filter(Boolean))]
|
|
118
|
+
const parameterSets = rows.map(
|
|
119
|
+
(row) => new Set((row.supported_parameters ?? []).map(String)),
|
|
120
|
+
)
|
|
121
|
+
const union = [...new Set(rows.flatMap((row) => row.supported_parameters ?? []))].sort()
|
|
122
|
+
const intersection = parameterSets.length
|
|
123
|
+
? [...parameterSets[0]].filter((value) =>
|
|
124
|
+
parameterSets.every((set) => set.has(value)),
|
|
125
|
+
).sort()
|
|
126
|
+
: []
|
|
127
|
+
|
|
128
|
+
return {
|
|
129
|
+
model_id: modelId,
|
|
130
|
+
model_name: rows.find((row) => row.model_name)?.model_name ?? modelId.split('/').pop(),
|
|
131
|
+
author: modelId.includes('/') ? modelId.split('/')[0] : null,
|
|
132
|
+
zdr: true,
|
|
133
|
+
zdr_endpoint_count: rows.length,
|
|
134
|
+
zdr_providers: providers.sort(),
|
|
135
|
+
context: {
|
|
136
|
+
min_endpoint_context_length: minOrNull(endpointOptions.map((row) => row.context_length)),
|
|
137
|
+
max_endpoint_context_length: maxOrNull(endpointOptions.map((row) => row.context_length)),
|
|
138
|
+
max_tool_capable_context_length:
|
|
139
|
+
maxOrNull(toolOptions.map((row) => row.context_length)),
|
|
140
|
+
max_prompt_tokens: maxOrNull(endpointOptions.map((row) => row.max_prompt_tokens)),
|
|
141
|
+
max_completion_tokens:
|
|
142
|
+
maxOrNull(endpointOptions.map((row) => row.max_completion_tokens)),
|
|
143
|
+
},
|
|
144
|
+
capabilities: {
|
|
145
|
+
supports_tools_on_any_zdr_endpoint: toolProviders.length > 0,
|
|
146
|
+
tool_capable_zdr_providers: toolProviders.sort(),
|
|
147
|
+
supported_parameters_union: union,
|
|
148
|
+
supported_parameters_all_zdr_endpoints: intersection,
|
|
149
|
+
},
|
|
150
|
+
openrouter_zdr_pricing_usd_per_1m_tokens: {
|
|
151
|
+
lowest_input:
|
|
152
|
+
minOrNull(endpointOptions.map((row) => row.pricing_usd_per_1m_tokens.input)),
|
|
153
|
+
median_input:
|
|
154
|
+
medianOrNull(endpointOptions.map((row) => row.pricing_usd_per_1m_tokens.input)),
|
|
155
|
+
lowest_output:
|
|
156
|
+
minOrNull(endpointOptions.map((row) => row.pricing_usd_per_1m_tokens.output)),
|
|
157
|
+
median_output:
|
|
158
|
+
medianOrNull(endpointOptions.map((row) => row.pricing_usd_per_1m_tokens.output)),
|
|
159
|
+
},
|
|
160
|
+
openrouter_zdr_performance: {
|
|
161
|
+
best_latency_p50_seconds:
|
|
162
|
+
minOrNull(endpointOptions.map((row) => row.performance.latency_p50_seconds)),
|
|
163
|
+
median_endpoint_latency_p50_seconds:
|
|
164
|
+
medianOrNull(endpointOptions.map((row) => row.performance.latency_p50_seconds)),
|
|
165
|
+
best_throughput_p50_tokens_per_second:
|
|
166
|
+
maxOrNull(endpointOptions.map((row) => row.performance.throughput_p50_tokens_per_second)),
|
|
167
|
+
median_endpoint_throughput_p50_tokens_per_second:
|
|
168
|
+
medianOrNull(endpointOptions.map((row) => row.performance.throughput_p50_tokens_per_second)),
|
|
169
|
+
best_uptime_last_1d_percent:
|
|
170
|
+
maxOrNull(endpointOptions.map((row) => row.performance.uptime_last_1d_percent)),
|
|
171
|
+
worst_uptime_last_1d_percent:
|
|
172
|
+
minOrNull(endpointOptions.map((row) => row.performance.uptime_last_1d_percent)),
|
|
173
|
+
},
|
|
174
|
+
zdr_endpoint_options: endpointOptions,
|
|
175
|
+
}
|
|
176
|
+
})
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function compactCatalogModel(row) {
|
|
180
|
+
const parameters = (row.supported_parameters ?? []).map(String).sort()
|
|
181
|
+
const contextLength = numberOrNull(row.top_provider?.context_length ?? row.context_length)
|
|
182
|
+
|
|
183
|
+
return {
|
|
184
|
+
model_id: row.id,
|
|
185
|
+
model_name: row.name ?? row.id?.split('/').pop() ?? row.id,
|
|
186
|
+
author: row.id?.includes('/') ? row.id.split('/')[0] : null,
|
|
187
|
+
openrouter_catalog: {
|
|
188
|
+
context_length: contextLength,
|
|
189
|
+
max_completion_tokens: numberOrNull(row.top_provider?.max_completion_tokens),
|
|
190
|
+
supports_tools: supportsToolUse(parameters),
|
|
191
|
+
supported_parameters: parameters,
|
|
192
|
+
pricing_usd_per_1m_tokens: {
|
|
193
|
+
input: pricePerMillion(row.pricing?.prompt),
|
|
194
|
+
output: pricePerMillion(row.pricing?.completion),
|
|
195
|
+
},
|
|
196
|
+
created: numberOrNull(row.created),
|
|
197
|
+
canonical_slug: row.canonical_slug ?? row.id ?? null,
|
|
198
|
+
},
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export function buildOpenRouterCatalog(modelRows, zdrEndpointRows = []) {
|
|
203
|
+
const zdrByModel = new Map(
|
|
204
|
+
groupZdrEndpointsByModel(zdrEndpointRows).map((model) => [model.model_id, model]),
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
return modelRows
|
|
208
|
+
.filter((row) => row?.id)
|
|
209
|
+
.map((row) => {
|
|
210
|
+
const model = compactCatalogModel(row)
|
|
211
|
+
const zdr = zdrByModel.get(model.model_id)
|
|
212
|
+
return {
|
|
213
|
+
...model,
|
|
214
|
+
zdr: Boolean(zdr),
|
|
215
|
+
zdr_endpoint_count: zdr?.zdr_endpoint_count ?? 0,
|
|
216
|
+
zdr_providers: zdr?.zdr_providers ?? [],
|
|
217
|
+
zdr_context: zdr?.context ?? null,
|
|
218
|
+
zdr_capabilities: zdr?.capabilities ?? null,
|
|
219
|
+
openrouter_zdr_pricing_usd_per_1m_tokens:
|
|
220
|
+
zdr?.openrouter_zdr_pricing_usd_per_1m_tokens ?? null,
|
|
221
|
+
openrouter_zdr_performance: zdr?.openrouter_zdr_performance ?? null,
|
|
222
|
+
zdr_endpoint_options: zdr?.zdr_endpoint_options ?? [],
|
|
223
|
+
}
|
|
224
|
+
})
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
export function modelCatalogSatisfies(model, constraints = {}) {
|
|
228
|
+
const catalog = model.openrouter_catalog ?? {}
|
|
229
|
+
if (constraints.tools === true && !catalog.supports_tools) return false
|
|
230
|
+
if (
|
|
231
|
+
constraints.minContext != null &&
|
|
232
|
+
(catalog.context_length == null || catalog.context_length < constraints.minContext)
|
|
233
|
+
) {
|
|
234
|
+
return false
|
|
235
|
+
}
|
|
236
|
+
if (
|
|
237
|
+
constraints.maxInputPrice != null &&
|
|
238
|
+
(catalog.pricing_usd_per_1m_tokens?.input == null ||
|
|
239
|
+
catalog.pricing_usd_per_1m_tokens.input > constraints.maxInputPrice)
|
|
240
|
+
) {
|
|
241
|
+
return false
|
|
242
|
+
}
|
|
243
|
+
if (
|
|
244
|
+
constraints.maxOutputPrice != null &&
|
|
245
|
+
(catalog.pricing_usd_per_1m_tokens?.output == null ||
|
|
246
|
+
catalog.pricing_usd_per_1m_tokens.output > constraints.maxOutputPrice)
|
|
247
|
+
) {
|
|
248
|
+
return false
|
|
249
|
+
}
|
|
250
|
+
return true
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
export function endpointSatisfies(endpoint, constraints = {}) {
|
|
254
|
+
if (constraints.tools === true && !endpoint.supports_tools) return false
|
|
255
|
+
if (
|
|
256
|
+
constraints.minContext != null &&
|
|
257
|
+
(endpoint.context_length == null || endpoint.context_length < constraints.minContext)
|
|
258
|
+
) {
|
|
259
|
+
return false
|
|
260
|
+
}
|
|
261
|
+
if (
|
|
262
|
+
constraints.maxInputPrice != null &&
|
|
263
|
+
(endpoint.pricing_usd_per_1m_tokens?.input == null ||
|
|
264
|
+
endpoint.pricing_usd_per_1m_tokens.input > constraints.maxInputPrice)
|
|
265
|
+
) {
|
|
266
|
+
return false
|
|
267
|
+
}
|
|
268
|
+
if (
|
|
269
|
+
constraints.maxOutputPrice != null &&
|
|
270
|
+
(endpoint.pricing_usd_per_1m_tokens?.output == null ||
|
|
271
|
+
endpoint.pricing_usd_per_1m_tokens.output > constraints.maxOutputPrice)
|
|
272
|
+
) {
|
|
273
|
+
return false
|
|
274
|
+
}
|
|
275
|
+
return true
|
|
276
|
+
}
|
package/mcp/server.js
ADDED
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import fs from 'node:fs'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
import { fileURLToPath } from 'node:url'
|
|
5
|
+
import { McpServer } from '@modelcontextprotocol/server'
|
|
6
|
+
import { serveStdio } from '@modelcontextprotocol/server/stdio'
|
|
7
|
+
import * as z from 'zod/v4'
|
|
8
|
+
import { getMergedCatalog } from '../lib/catalog.js'
|
|
9
|
+
import { endpointSatisfies, modelCatalogSatisfies } from '../lib/openrouter.js'
|
|
10
|
+
|
|
11
|
+
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
|
|
12
|
+
loadLocalEnv(path.join(ROOT, '.env.local'))
|
|
13
|
+
|
|
14
|
+
function loadLocalEnv(filePath) {
|
|
15
|
+
if (!fs.existsSync(filePath)) return
|
|
16
|
+
const text = fs.readFileSync(filePath, 'utf8')
|
|
17
|
+
for (const rawLine of text.split(/\r?\n/)) {
|
|
18
|
+
const line = rawLine.trim()
|
|
19
|
+
if (!line || line.startsWith('#')) continue
|
|
20
|
+
const equals = line.indexOf('=')
|
|
21
|
+
if (equals <= 0) continue
|
|
22
|
+
const key = line.slice(0, equals).trim()
|
|
23
|
+
let value = line.slice(equals + 1).trim()
|
|
24
|
+
if (
|
|
25
|
+
(value.startsWith('"') && value.endsWith('"')) ||
|
|
26
|
+
(value.startsWith("'") && value.endsWith("'"))
|
|
27
|
+
) {
|
|
28
|
+
value = value.slice(1, -1)
|
|
29
|
+
}
|
|
30
|
+
if (process.env[key] == null) process.env[key] = value
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function requireLocalSecrets() {
|
|
35
|
+
const missing = [
|
|
36
|
+
'ARTIFICIAL_ANALYSIS_API_KEY',
|
|
37
|
+
'OPENROUTER_API_KEY',
|
|
38
|
+
].filter((key) => !process.env[key])
|
|
39
|
+
|
|
40
|
+
if (missing.length) {
|
|
41
|
+
throw new Error(
|
|
42
|
+
`Missing ${missing.join(', ')}. Create ${path.join(ROOT, '.env.local')} from .env.local.example.`,
|
|
43
|
+
)
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function num(value) {
|
|
48
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : null
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function min(values) {
|
|
52
|
+
const nums = values.map(num).filter((value) => value != null)
|
|
53
|
+
return nums.length ? Math.min(...nums) : null
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function max(values) {
|
|
57
|
+
const nums = values.map(num).filter((value) => value != null)
|
|
58
|
+
return nums.length ? Math.max(...nums) : null
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function compactCandidate(model, { requiresZdr = false, eligibleZdrEndpoints = [] } = {}) {
|
|
62
|
+
const aa = model.artificial_analysis
|
|
63
|
+
const zdrEndpoints = requiresZdr ? eligibleZdrEndpoints : model.zdr_endpoint_options ?? []
|
|
64
|
+
|
|
65
|
+
return {
|
|
66
|
+
model_id: model.model_id,
|
|
67
|
+
model_name: model.model_name,
|
|
68
|
+
creator: aa?.creator ?? model.author ?? null,
|
|
69
|
+
release_date: aa?.release_date ?? null,
|
|
70
|
+
artificial_analysis: aa
|
|
71
|
+
? {
|
|
72
|
+
intelligence_index: aa.evaluations?.intelligence_index ?? null,
|
|
73
|
+
coding_index: aa.evaluations?.coding_index ?? null,
|
|
74
|
+
agentic_index: aa.evaluations?.agentic_index ?? null,
|
|
75
|
+
price_usd_per_1m_tokens: aa.pricing_usd_per_1m_tokens ?? null,
|
|
76
|
+
median_performance: aa.performance ?? null,
|
|
77
|
+
}
|
|
78
|
+
: null,
|
|
79
|
+
openrouter: {
|
|
80
|
+
catalog: model.openrouter_catalog,
|
|
81
|
+
zdr: {
|
|
82
|
+
available: model.zdr === true,
|
|
83
|
+
requirement_applied: requiresZdr,
|
|
84
|
+
requirement_met: requiresZdr ? zdrEndpoints.length > 0 : null,
|
|
85
|
+
endpoint_count: model.zdr_endpoint_count ?? 0,
|
|
86
|
+
providers: model.zdr_providers ?? [],
|
|
87
|
+
eligible_endpoint_count: requiresZdr ? zdrEndpoints.length : null,
|
|
88
|
+
eligible_providers: requiresZdr
|
|
89
|
+
? [...new Set(zdrEndpoints.map((e) => e.provider).filter(Boolean))].sort()
|
|
90
|
+
: null,
|
|
91
|
+
eligible_max_context_length: requiresZdr
|
|
92
|
+
? max(zdrEndpoints.map((e) => e.context_length))
|
|
93
|
+
: null,
|
|
94
|
+
eligible_max_completion_tokens: requiresZdr
|
|
95
|
+
? max(zdrEndpoints.map((e) => e.max_completion_tokens))
|
|
96
|
+
: null,
|
|
97
|
+
eligible_lowest_input_price_usd_per_1m_tokens: requiresZdr
|
|
98
|
+
? min(zdrEndpoints.map((e) => e.pricing_usd_per_1m_tokens?.input))
|
|
99
|
+
: null,
|
|
100
|
+
eligible_lowest_output_price_usd_per_1m_tokens: requiresZdr
|
|
101
|
+
? min(zdrEndpoints.map((e) => e.pricing_usd_per_1m_tokens?.output))
|
|
102
|
+
: null,
|
|
103
|
+
eligible_best_latency_p50_seconds: requiresZdr
|
|
104
|
+
? min(zdrEndpoints.map((e) => e.performance?.latency_p50_seconds))
|
|
105
|
+
: null,
|
|
106
|
+
eligible_best_throughput_p50_tokens_per_second: requiresZdr
|
|
107
|
+
? max(zdrEndpoints.map((e) => e.performance?.throughput_p50_tokens_per_second))
|
|
108
|
+
: null,
|
|
109
|
+
eligible_worst_uptime_last_1d_percent: requiresZdr
|
|
110
|
+
? min(zdrEndpoints.map((e) => e.performance?.uptime_last_1d_percent))
|
|
111
|
+
: null,
|
|
112
|
+
},
|
|
113
|
+
},
|
|
114
|
+
reconciliation: model.reconciliation,
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function filterCandidates(models, args) {
|
|
119
|
+
const query = args.query?.trim().toLowerCase() || null
|
|
120
|
+
const creator = args.creator?.trim().toLowerCase() || null
|
|
121
|
+
const requiresZdr = args.requires_zdr === true
|
|
122
|
+
const constraints = {
|
|
123
|
+
tools: args.requires_tools === true,
|
|
124
|
+
minContext: args.min_context ?? null,
|
|
125
|
+
maxInputPrice: args.max_input_price ?? null,
|
|
126
|
+
maxOutputPrice: args.max_output_price ?? null,
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const results = []
|
|
130
|
+
for (const model of models) {
|
|
131
|
+
if (query) {
|
|
132
|
+
const haystack = [
|
|
133
|
+
model.model_id,
|
|
134
|
+
model.model_name,
|
|
135
|
+
model.artificial_analysis?.name,
|
|
136
|
+
model.artificial_analysis?.slug,
|
|
137
|
+
].filter(Boolean).join(' ').toLowerCase()
|
|
138
|
+
if (!haystack.includes(query)) continue
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
if (creator) {
|
|
142
|
+
const creatorText = `${model.author ?? ''} ${model.artificial_analysis?.creator ?? ''}`.toLowerCase()
|
|
143
|
+
if (!creatorText.includes(creator)) continue
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (requiresZdr) {
|
|
147
|
+
const eligibleZdrEndpoints = (model.zdr_endpoint_options ?? []).filter((endpoint) =>
|
|
148
|
+
endpointSatisfies(endpoint, constraints),
|
|
149
|
+
)
|
|
150
|
+
if (eligibleZdrEndpoints.length === 0) continue
|
|
151
|
+
results.push(compactCandidate(model, { requiresZdr: true, eligibleZdrEndpoints }))
|
|
152
|
+
continue
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
if (!modelCatalogSatisfies(model, constraints)) continue
|
|
156
|
+
results.push(compactCandidate(model, { requiresZdr: false }))
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return results.sort((a, b) => a.model_id.localeCompare(b.model_id))
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const recommendationSchema = z.object({
|
|
163
|
+
use_case: z.string().min(3).describe(
|
|
164
|
+
'What the user needs the model to do. Preserve their actual workload and priorities in detail.',
|
|
165
|
+
),
|
|
166
|
+
requires_zdr: z.boolean().default(false).describe(
|
|
167
|
+
'Set true only when the user explicitly requires Zero Data Retention (ZDR). Otherwise leave false so all OpenRouter models remain eligible.',
|
|
168
|
+
),
|
|
169
|
+
requires_tools: z.boolean().default(false).describe(
|
|
170
|
+
'True when the workload requires tool/function calling.',
|
|
171
|
+
),
|
|
172
|
+
min_context: z.number().int().positive().optional().describe(
|
|
173
|
+
'Minimum context window required.',
|
|
174
|
+
),
|
|
175
|
+
max_input_price: z.number().nonnegative().optional().describe(
|
|
176
|
+
'Maximum acceptable OpenRouter input price in USD per 1M tokens.',
|
|
177
|
+
),
|
|
178
|
+
max_output_price: z.number().nonnegative().optional().describe(
|
|
179
|
+
'Maximum acceptable OpenRouter output price in USD per 1M tokens.',
|
|
180
|
+
),
|
|
181
|
+
creator: z.string().optional().describe('Optional model creator/provider family filter.'),
|
|
182
|
+
query: z.string().optional().describe('Optional model-name substring filter.'),
|
|
183
|
+
limit: z.number().int().min(1).max(100).default(50),
|
|
184
|
+
force_refresh: z.boolean().default(false).describe(
|
|
185
|
+
'Force fresh upstream API requests. Use sparingly because AA Free has a quota.',
|
|
186
|
+
),
|
|
187
|
+
})
|
|
188
|
+
|
|
189
|
+
function createServer() {
|
|
190
|
+
const server = new McpServer({
|
|
191
|
+
name: 'modelshortlist',
|
|
192
|
+
version: '0.2.1',
|
|
193
|
+
})
|
|
194
|
+
|
|
195
|
+
server.registerTool(
|
|
196
|
+
'recommend_models',
|
|
197
|
+
{
|
|
198
|
+
title: 'Recommend AI models',
|
|
199
|
+
description:
|
|
200
|
+
'Use this whenever the user asks which AI model to use for a workload. It considers the full current OpenRouter model catalog and adds Artificial Analysis benchmark data when confidently matched. Zero Data Retention is optional and must only be enforced when the user explicitly requires ZDR. After calling this tool, YOU must make the recommendation based on the user use case; do not simply pick the first model or a single benchmark winner. Clearly attribute Artificial Analysis benchmark metrics and OpenRouter catalog/endpoint data.',
|
|
201
|
+
inputSchema: recommendationSchema,
|
|
202
|
+
annotations: {
|
|
203
|
+
readOnlyHint: true,
|
|
204
|
+
destructiveHint: false,
|
|
205
|
+
openWorldHint: true,
|
|
206
|
+
},
|
|
207
|
+
},
|
|
208
|
+
async (args) => {
|
|
209
|
+
try {
|
|
210
|
+
requireLocalSecrets()
|
|
211
|
+
const catalog = await getMergedCatalog({ forceRefresh: args.force_refresh })
|
|
212
|
+
const candidates = filterCandidates(catalog.models, args).slice(0, args.limit)
|
|
213
|
+
const requiresZdr = args.requires_zdr === true
|
|
214
|
+
|
|
215
|
+
const selectionInstructions = [
|
|
216
|
+
requiresZdr
|
|
217
|
+
? 'ZDR is an explicit hard requirement. Only recommend candidates with a current ZDR endpoint that satisfies all hard constraints on that same endpoint.'
|
|
218
|
+
: 'ZDR is not a requirement for this request. Do not exclude a model because it lacks a ZDR endpoint; treat ZDR availability as optional metadata only.',
|
|
219
|
+
requiresZdr
|
|
220
|
+
? 'If the selected model is called through OpenRouter, enforce provider.zdr=true in the actual inference request.'
|
|
221
|
+
: 'Do not add provider.zdr=true merely because ModelShortlist reports ZDR availability; only enforce it when the user requests ZDR.',
|
|
222
|
+
'Choose weights based on the stated use case rather than using a universal scoring formula.',
|
|
223
|
+
'For coding work, Artificial Analysis coding_index and agentic_index are usually more relevant than intelligence_index alone when those metrics are available.',
|
|
224
|
+
'For high-volume workloads, weigh OpenRouter pricing and relevant performance evidence heavily.',
|
|
225
|
+
'For interactive workloads, weigh latency when endpoint-level latency data is available.',
|
|
226
|
+
'Models without a confident Artificial Analysis match remain eligible; do not invent missing benchmark scores.',
|
|
227
|
+
'Distinguish Artificial Analysis benchmark/performance data from OpenRouter model catalog and endpoint data.',
|
|
228
|
+
'Recommend one primary model and normally two alternatives with explicit tradeoffs.',
|
|
229
|
+
]
|
|
230
|
+
|
|
231
|
+
const payload = {
|
|
232
|
+
use_case: args.use_case,
|
|
233
|
+
zdr_required: requiresZdr,
|
|
234
|
+
constraint_basis: requiresZdr
|
|
235
|
+
? 'current OpenRouter ZDR endpoints; hard constraints must be satisfied by the same endpoint'
|
|
236
|
+
: 'current OpenRouter model catalog; ZDR is not used as an eligibility filter',
|
|
237
|
+
selection_instructions: selectionInstructions,
|
|
238
|
+
generated_at: catalog.generated_at,
|
|
239
|
+
candidate_count: candidates.length,
|
|
240
|
+
candidates,
|
|
241
|
+
diagnostics: {
|
|
242
|
+
openrouter_model_count: catalog.diagnostics.openrouter_model_count,
|
|
243
|
+
openrouter_zdr_model_count: catalog.diagnostics.openrouter_zdr_model_count,
|
|
244
|
+
matched_model_count: catalog.diagnostics.matched_model_count,
|
|
245
|
+
unmatched_model_count: catalog.diagnostics.unmatched_model_count,
|
|
246
|
+
ambiguous_model_count: catalog.diagnostics.ambiguous_model_count,
|
|
247
|
+
aa_rate_limit: catalog.diagnostics.aa?.rateLimit ?? null,
|
|
248
|
+
},
|
|
249
|
+
attribution: {
|
|
250
|
+
artificial_analysis: 'https://artificialanalysis.ai/',
|
|
251
|
+
openrouter: 'https://openrouter.ai/',
|
|
252
|
+
},
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
return {
|
|
256
|
+
content: [{ type: 'text', text: JSON.stringify(payload) }],
|
|
257
|
+
}
|
|
258
|
+
} catch (error) {
|
|
259
|
+
return {
|
|
260
|
+
isError: true,
|
|
261
|
+
content: [{ type: 'text', text: `ModelShortlist error: ${error.message}` }],
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
},
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
server.registerTool(
|
|
268
|
+
'compare_models',
|
|
269
|
+
{
|
|
270
|
+
title: 'Compare specific AI models',
|
|
271
|
+
description:
|
|
272
|
+
'Use this after model discovery for a focused comparison of specific OpenRouter model IDs. It returns general OpenRouter catalog data, current ZDR availability, and Artificial Analysis metrics when confidently matched. ZDR is not assumed to be required.',
|
|
273
|
+
inputSchema: z.object({
|
|
274
|
+
model_ids: z.array(z.string()).min(2).max(12),
|
|
275
|
+
requires_zdr: z.boolean().default(false).describe(
|
|
276
|
+
'Set true only when the user explicitly requires Zero Data Retention.',
|
|
277
|
+
),
|
|
278
|
+
force_refresh: z.boolean().default(false),
|
|
279
|
+
}),
|
|
280
|
+
annotations: {
|
|
281
|
+
readOnlyHint: true,
|
|
282
|
+
destructiveHint: false,
|
|
283
|
+
openWorldHint: true,
|
|
284
|
+
},
|
|
285
|
+
},
|
|
286
|
+
async ({ model_ids, requires_zdr, force_refresh }) => {
|
|
287
|
+
try {
|
|
288
|
+
requireLocalSecrets()
|
|
289
|
+
const catalog = await getMergedCatalog({ forceRefresh: force_refresh })
|
|
290
|
+
const wanted = new Set(model_ids)
|
|
291
|
+
const models = catalog.models
|
|
292
|
+
.filter((model) => wanted.has(model.model_id))
|
|
293
|
+
.map((model) => compactCandidate(model, {
|
|
294
|
+
requiresZdr: requires_zdr === true,
|
|
295
|
+
eligibleZdrEndpoints: model.zdr_endpoint_options ?? [],
|
|
296
|
+
}))
|
|
297
|
+
const found = new Set(models.map((model) => model.model_id))
|
|
298
|
+
const missing = model_ids.filter((id) => !found.has(id))
|
|
299
|
+
|
|
300
|
+
return {
|
|
301
|
+
content: [{
|
|
302
|
+
type: 'text',
|
|
303
|
+
text: JSON.stringify({
|
|
304
|
+
generated_at: catalog.generated_at,
|
|
305
|
+
zdr_required: requires_zdr === true,
|
|
306
|
+
models,
|
|
307
|
+
missing_model_ids: missing,
|
|
308
|
+
response_instruction:
|
|
309
|
+
'Attribute Artificial Analysis benchmark/performance metrics to Artificial Analysis and OpenRouter catalog/endpoint data to OpenRouter. Do not treat ZDR as required unless zdr_required is true.',
|
|
310
|
+
attribution: {
|
|
311
|
+
artificial_analysis: 'https://artificialanalysis.ai/',
|
|
312
|
+
openrouter: 'https://openrouter.ai/',
|
|
313
|
+
},
|
|
314
|
+
}),
|
|
315
|
+
}],
|
|
316
|
+
}
|
|
317
|
+
} catch (error) {
|
|
318
|
+
return {
|
|
319
|
+
isError: true,
|
|
320
|
+
content: [{ type: 'text', text: `ModelShortlist error: ${error.message}` }],
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
},
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
server.registerTool(
|
|
327
|
+
'modelshortlist_status',
|
|
328
|
+
{
|
|
329
|
+
title: 'Check ModelShortlist status',
|
|
330
|
+
description:
|
|
331
|
+
'Use this to diagnose OpenRouter catalog coverage, model matching coverage, ZDR availability, or Artificial Analysis quota state.',
|
|
332
|
+
inputSchema: z.object({
|
|
333
|
+
include_unmatched: z.boolean().default(false),
|
|
334
|
+
force_refresh: z.boolean().default(false),
|
|
335
|
+
}),
|
|
336
|
+
annotations: {
|
|
337
|
+
readOnlyHint: true,
|
|
338
|
+
destructiveHint: false,
|
|
339
|
+
openWorldHint: true,
|
|
340
|
+
},
|
|
341
|
+
},
|
|
342
|
+
async ({ include_unmatched, force_refresh }) => {
|
|
343
|
+
try {
|
|
344
|
+
requireLocalSecrets()
|
|
345
|
+
const catalog = await getMergedCatalog({ forceRefresh: force_refresh })
|
|
346
|
+
const payload = {
|
|
347
|
+
generated_at: catalog.generated_at,
|
|
348
|
+
diagnostics: catalog.diagnostics,
|
|
349
|
+
unmatched: include_unmatched ? catalog.unmatched : undefined,
|
|
350
|
+
ambiguous: include_unmatched ? catalog.ambiguous : undefined,
|
|
351
|
+
cache: catalog.cache,
|
|
352
|
+
}
|
|
353
|
+
return {
|
|
354
|
+
content: [{ type: 'text', text: JSON.stringify(payload) }],
|
|
355
|
+
}
|
|
356
|
+
} catch (error) {
|
|
357
|
+
return {
|
|
358
|
+
isError: true,
|
|
359
|
+
content: [{ type: 'text', text: `ModelShortlist error: ${error.message}` }],
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
},
|
|
363
|
+
)
|
|
364
|
+
|
|
365
|
+
return server
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
void serveStdio(createServer)
|
|
369
|
+
console.error('ModelShortlist MCP server running on stdio')
|
package/package.json
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@agentic.artists/modelshortlist",
|
|
3
|
+
"version": "0.2.1",
|
|
4
|
+
"description": "A local BYOK MCP server that helps AI assistants shortlist models using OpenRouter catalog data and Artificial Analysis benchmarks, with optional ZDR filtering.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"mcpName": "io.github.AgenticArtists/modelshortlist",
|
|
8
|
+
"bin": {
|
|
9
|
+
"modelshortlist": "bin/modelshortlist.js"
|
|
10
|
+
},
|
|
11
|
+
"files": [
|
|
12
|
+
"bin/",
|
|
13
|
+
"mcp/",
|
|
14
|
+
"lib/",
|
|
15
|
+
"config/",
|
|
16
|
+
"scripts/setup.js",
|
|
17
|
+
".env.local.example",
|
|
18
|
+
"ATTRIBUTION.md"
|
|
19
|
+
],
|
|
20
|
+
"publishConfig": {
|
|
21
|
+
"access": "public"
|
|
22
|
+
},
|
|
23
|
+
"repository": {
|
|
24
|
+
"type": "git",
|
|
25
|
+
"url": "git+https://github.com/AgenticArtists/ModelShortlist.git"
|
|
26
|
+
},
|
|
27
|
+
"homepage": "https://modelshortlist.com",
|
|
28
|
+
"bugs": {
|
|
29
|
+
"url": "https://github.com/AgenticArtists/ModelShortlist/issues"
|
|
30
|
+
},
|
|
31
|
+
"keywords": [
|
|
32
|
+
"mcp",
|
|
33
|
+
"model-context-protocol",
|
|
34
|
+
"openrouter",
|
|
35
|
+
"artificial-analysis",
|
|
36
|
+
"llm",
|
|
37
|
+
"model-selection",
|
|
38
|
+
"model-recommendation",
|
|
39
|
+
"zdr",
|
|
40
|
+
"ai"
|
|
41
|
+
],
|
|
42
|
+
"engines": {
|
|
43
|
+
"node": ">=20"
|
|
44
|
+
},
|
|
45
|
+
"scripts": {
|
|
46
|
+
"setup": "node scripts/setup.js",
|
|
47
|
+
"test": "node --test",
|
|
48
|
+
"check": "node --check bin/modelshortlist.js && node --check lib/artificial-analysis.js && node --check lib/openrouter.js && node --check lib/match.js && node --check lib/catalog.js && node --check lib/http.js && node --check mcp/server.js && node --check scripts/setup.js",
|
|
49
|
+
"pack:check": "npm pack --dry-run",
|
|
50
|
+
"mcp": "node mcp/server.js"
|
|
51
|
+
},
|
|
52
|
+
"dependencies": {
|
|
53
|
+
"@modelcontextprotocol/server": "2.0.0",
|
|
54
|
+
"zod": "4.5.4"
|
|
55
|
+
}
|
|
56
|
+
}
|