@agentic.artists/modelshortlist 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.local.example +6 -0
- package/ATTRIBUTION.md +25 -0
- package/LICENSE +21 -0
- package/README.md +240 -0
- package/bin/modelshortlist.js +3 -0
- package/config/aliases.json +3 -0
- package/lib/artificial-analysis.js +104 -0
- package/lib/catalog.js +117 -0
- package/lib/http.js +68 -0
- package/lib/match.js +100 -0
- package/lib/openrouter.js +276 -0
- package/mcp/server.js +369 -0
- package/package.json +56 -0
- package/scripts/setup.js +179 -0
package/ATTRIBUTION.md
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Attribution and upstream data
|
|
2
|
+
|
|
3
|
+
ModelShortlist is open-source software. It does not bundle or redistribute the underlying datasets it queries.
|
|
4
|
+
|
|
5
|
+
## Artificial Analysis
|
|
6
|
+
|
|
7
|
+
ModelShortlist uses Artificial Analysis API data for model benchmarks, pricing, and performance information.
|
|
8
|
+
|
|
9
|
+
Artificial Analysis: https://artificialanalysis.ai/
|
|
10
|
+
|
|
11
|
+
Users supply their own Artificial Analysis API key and are responsible for complying with Artificial Analysis terms, plan limits, attribution requirements, and restrictions on external use or redistribution.
|
|
12
|
+
|
|
13
|
+
## OpenRouter
|
|
14
|
+
|
|
15
|
+
ModelShortlist uses OpenRouter's model catalog API for current model availability, capabilities, context limits, and pricing. It also uses OpenRouter's ZDR endpoint API to identify current Zero Data Retention options and endpoint-level provider/performance information when ZDR is relevant to the user's request.
|
|
16
|
+
|
|
17
|
+
OpenRouter: https://openrouter.ai/
|
|
18
|
+
|
|
19
|
+
Users supply their own OpenRouter API key and are responsible for complying with OpenRouter and applicable provider terms.
|
|
20
|
+
|
|
21
|
+
## Project relationship
|
|
22
|
+
|
|
23
|
+
ModelShortlist is not affiliated with, sponsored by, or endorsed by Artificial Analysis or OpenRouter.
|
|
24
|
+
|
|
25
|
+
The MIT License applies only to the ModelShortlist source code. It does not grant rights to third-party data, APIs, trademarks, or services.
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 AgenticArtists
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
# ModelShortlist
|
|
2
|
+
|
|
3
|
+
[](https://github.com/AgenticArtists/ModelShortlist/actions/workflows/ci.yml)
|
|
4
|
+
|
|
5
|
+
**Stop guessing which AI model to use.**
|
|
6
|
+
|
|
7
|
+
ModelShortlist is a local, bring-your-own-key MCP server that gives your AI assistant current model-selection context from the **full OpenRouter model catalog** plus **Artificial Analysis benchmarks**. Zero Data Retention (ZDR) is available as an optional hard constraint when you explicitly require it.
|
|
8
|
+
|
|
9
|
+
No hosted service. No account. No deployment. Your API keys are supplied locally and used only to call the upstream services directly.
|
|
10
|
+
|
|
11
|
+
Website: [modelshortlist.com](https://modelshortlist.com)
|
|
12
|
+
|
|
13
|
+
## Why ModelShortlist
|
|
14
|
+
|
|
15
|
+
Model choice is no longer just "which model has the highest benchmark score?" The right answer depends on the workload, capabilities, cost, context, and any privacy requirements you actually have.
|
|
16
|
+
|
|
17
|
+
ModelShortlist helps your chat agent reason over:
|
|
18
|
+
|
|
19
|
+
- the current OpenRouter model catalog
|
|
20
|
+
- tool/function-calling support
|
|
21
|
+
- context and completion limits
|
|
22
|
+
- OpenRouter input/output pricing
|
|
23
|
+
- current ZDR endpoint availability when privacy requires it
|
|
24
|
+
- ZDR endpoint latency, throughput, uptime, and provider options when applicable
|
|
25
|
+
- Artificial Analysis Intelligence Index
|
|
26
|
+
- Artificial Analysis Coding Index
|
|
27
|
+
- Artificial Analysis Agentic Index
|
|
28
|
+
- Artificial Analysis pricing and median performance
|
|
29
|
+
|
|
30
|
+
By default, **ZDR is not an eligibility requirement**. ModelShortlist considers the full OpenRouter catalog. If you explicitly require ZDR, the tool switches to current OpenRouter ZDR endpoint data and requires all hard constraints to be satisfied by the same real ZDR endpoint.
|
|
31
|
+
|
|
32
|
+
The chat model makes the final recommendation based on your use case. ModelShortlist deliberately does not impose one universal ranking formula.
|
|
33
|
+
|
|
34
|
+
## Quick start
|
|
35
|
+
|
|
36
|
+
Requirements:
|
|
37
|
+
|
|
38
|
+
- Node.js 20+
|
|
39
|
+
- an Artificial Analysis API key
|
|
40
|
+
- an OpenRouter API key
|
|
41
|
+
- an MCP-capable chat client such as Hermes Desktop, Claude Code, Cursor, or VS Code/Copilot
|
|
42
|
+
|
|
43
|
+
The npm package is:
|
|
44
|
+
|
|
45
|
+
```text
|
|
46
|
+
@agentic.artists/modelshortlist
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### Fastest install: run from npm
|
|
50
|
+
|
|
51
|
+
Most stdio MCP hosts can launch ModelShortlist directly with `npx`.
|
|
52
|
+
|
|
53
|
+
```json
|
|
54
|
+
{
|
|
55
|
+
"mcpServers": {
|
|
56
|
+
"modelshortlist": {
|
|
57
|
+
"command": "npx",
|
|
58
|
+
"args": ["-y", "@agentic.artists/modelshortlist"],
|
|
59
|
+
"env": {
|
|
60
|
+
"ARTIFICIAL_ANALYSIS_API_KEY": "YOUR_KEY",
|
|
61
|
+
"OPENROUTER_API_KEY": "YOUR_KEY"
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
On Windows GUI clients where `npx` is not available on the app's PATH, use the full path to `npx.cmd` or use the local-clone setup below.
|
|
69
|
+
|
|
70
|
+
### Local clone + gitignored `.env.local`
|
|
71
|
+
|
|
72
|
+
This option keeps the API keys in a local gitignored file and generates client config with absolute Node/server paths.
|
|
73
|
+
|
|
74
|
+
#### Windows
|
|
75
|
+
|
|
76
|
+
```powershell
|
|
77
|
+
git clone https://github.com/AgenticArtists/ModelShortlist.git
|
|
78
|
+
cd ModelShortlist
|
|
79
|
+
npm.cmd install
|
|
80
|
+
npm.cmd run setup
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
#### macOS / Linux
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
git clone https://github.com/AgenticArtists/ModelShortlist.git
|
|
87
|
+
cd ModelShortlist
|
|
88
|
+
npm install
|
|
89
|
+
npm run setup
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
The setup command:
|
|
93
|
+
|
|
94
|
+
- asks for your two API keys with masked input
|
|
95
|
+
- stores them only in the gitignored `.env.local`
|
|
96
|
+
- prints ready-to-paste Hermes Desktop / Cursor and VS Code / Copilot MCP configs
|
|
97
|
+
- uses the exact Node executable that ran setup, avoiding GUI-client PATH issues
|
|
98
|
+
|
|
99
|
+
If PowerShell blocks `npm.ps1`, use `npm.cmd`; you do not need to change your execution policy.
|
|
100
|
+
|
|
101
|
+
## Hermes Desktop
|
|
102
|
+
|
|
103
|
+
Either use the npm config above or run the local setup command and paste the generated **Hermes Desktop / Cursor MCP config**.
|
|
104
|
+
|
|
105
|
+
In Hermes Desktop, open **Skills & Tools → MCP**, import the JSON, and save it. Hermes should discover three tools:
|
|
106
|
+
|
|
107
|
+
- `recommend_models`
|
|
108
|
+
- `compare_models`
|
|
109
|
+
- `modelshortlist_status`
|
|
110
|
+
|
|
111
|
+
Then start a normal chat and ask something like:
|
|
112
|
+
|
|
113
|
+
> I need the best-value model for a long-running autonomous coding agent. Tool calling is required and I need at least 100k context. Quality matters more than cost, but I care about value. What should I use?
|
|
114
|
+
|
|
115
|
+
If privacy matters, say so explicitly:
|
|
116
|
+
|
|
117
|
+
> Same workload, but ZDR is mandatory.
|
|
118
|
+
|
|
119
|
+
More client setup examples and manual configuration are in [LOCAL_MCP.md](./LOCAL_MCP.md).
|
|
120
|
+
|
|
121
|
+
## Example prompts
|
|
122
|
+
|
|
123
|
+
> What's the cheapest model I'd trust with repetitive coding subagents? Tool use is required.
|
|
124
|
+
|
|
125
|
+
> I need 200k context and tool use. What are my best current options?
|
|
126
|
+
|
|
127
|
+
> I need 200k context, tool use, and ZDR. What are my best current options?
|
|
128
|
+
|
|
129
|
+
> Is the premium frontier model actually worth the price for this coding workload?
|
|
130
|
+
|
|
131
|
+
> Best model for extracting structured data from thousands of documents while keeping output cost low?
|
|
132
|
+
|
|
133
|
+
> I need maximum autonomous coding performance under $10 per million output tokens. What should I use?
|
|
134
|
+
|
|
135
|
+
## MCP tools
|
|
136
|
+
|
|
137
|
+
### `recommend_models`
|
|
138
|
+
|
|
139
|
+
The primary tool. It accepts a workload plus hard constraints such as:
|
|
140
|
+
|
|
141
|
+
- ZDR required or not required
|
|
142
|
+
- tool calling required
|
|
143
|
+
- minimum context
|
|
144
|
+
- maximum input/output price
|
|
145
|
+
- creator/model filter
|
|
146
|
+
|
|
147
|
+
When ZDR is not required, it considers the full OpenRouter catalog. When ZDR is explicitly required, it filters against current ZDR endpoints and verifies hard constraints against the same endpoint. Artificial Analysis benchmark data is attached only when the model can be confidently reconciled; models without a confident benchmark match remain eligible with missing benchmark fields rather than being silently removed.
|
|
148
|
+
|
|
149
|
+
### `compare_models`
|
|
150
|
+
|
|
151
|
+
Returns current OpenRouter catalog information, ZDR availability, and Artificial Analysis benchmark information when available for a specific shortlist of OpenRouter model IDs. ZDR is not assumed to be required.
|
|
152
|
+
|
|
153
|
+
### `modelshortlist_status`
|
|
154
|
+
|
|
155
|
+
Shows OpenRouter catalog coverage, ZDR coverage, model matching coverage, ambiguous/unmatched records, cache state, and Artificial Analysis rate-limit metadata.
|
|
156
|
+
|
|
157
|
+
## How matching works
|
|
158
|
+
|
|
159
|
+
The Artificial Analysis Free API does not expose an OpenRouter model ID. ModelShortlist therefore reconciles models conservatively:
|
|
160
|
+
|
|
161
|
+
1. manually verified aliases
|
|
162
|
+
2. exact normalized name matches
|
|
163
|
+
3. otherwise the Artificial Analysis benchmark match remains unavailable
|
|
164
|
+
|
|
165
|
+
ModelShortlist does **not** fuzzy-match uncertain model variants. A missing benchmark is better than attaching benchmark data to the wrong model. An unmatched OpenRouter model can still be considered; it simply carries no Artificial Analysis metrics.
|
|
166
|
+
|
|
167
|
+
Verified aliases live in [`config/aliases.json`](./config/aliases.json).
|
|
168
|
+
|
|
169
|
+
## ZDR is optional
|
|
170
|
+
|
|
171
|
+
ModelShortlist tracks which models have current ZDR-capable OpenRouter endpoints, but it does **not** filter to them unless the user explicitly requires Zero Data Retention.
|
|
172
|
+
|
|
173
|
+
When ZDR is required, ModelShortlist checks current endpoint-level eligibility and hard constraints. If you later call the selected model through OpenRouter, enforce ZDR again in the actual inference request:
|
|
174
|
+
|
|
175
|
+
```json
|
|
176
|
+
{
|
|
177
|
+
"provider": {
|
|
178
|
+
"zdr": true,
|
|
179
|
+
"require_parameters": true
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
When ZDR is not required, do not add `provider.zdr=true` merely because a model happens to support it.
|
|
185
|
+
|
|
186
|
+
## Data sources and attribution
|
|
187
|
+
|
|
188
|
+
ModelShortlist uses data accessed with **your own API credentials**.
|
|
189
|
+
|
|
190
|
+
- Model catalog, capabilities, pricing, context, and ZDR endpoint metadata: [OpenRouter](https://openrouter.ai/)
|
|
191
|
+
- Benchmark and model-performance data: [Artificial Analysis](https://artificialanalysis.ai/)
|
|
192
|
+
|
|
193
|
+
ModelShortlist is not affiliated with or endorsed by Artificial Analysis or OpenRouter.
|
|
194
|
+
|
|
195
|
+
The ModelShortlist source code is licensed under the MIT License. Upstream data and APIs remain subject to their respective terms. In particular, Artificial Analysis API access may have restrictions on external use and redistribution. ModelShortlist does not bundle or host their dataset; each user accesses upstream data with their own credentials and is responsible for complying with the applicable terms.
|
|
196
|
+
|
|
197
|
+
See [ATTRIBUTION.md](./ATTRIBUTION.md) for more detail.
|
|
198
|
+
|
|
199
|
+
## Privacy and security
|
|
200
|
+
|
|
201
|
+
- `.env.local` is gitignored for the clone-based setup.
|
|
202
|
+
- API keys are loaded locally by the MCP process.
|
|
203
|
+
- The setup command masks API-key input.
|
|
204
|
+
- ModelShortlist does not operate a hosted backend.
|
|
205
|
+
- MCP tools are read-only.
|
|
206
|
+
- The server writes protocol traffic to stdout and diagnostic messages to stderr.
|
|
207
|
+
- No telemetry is built into ModelShortlist.
|
|
208
|
+
|
|
209
|
+
If you discover a security issue, see [SECURITY.md](./SECURITY.md).
|
|
210
|
+
|
|
211
|
+
## Development
|
|
212
|
+
|
|
213
|
+
Install dependencies and run validation:
|
|
214
|
+
|
|
215
|
+
```powershell
|
|
216
|
+
npm.cmd install
|
|
217
|
+
npm.cmd test
|
|
218
|
+
npm.cmd run check
|
|
219
|
+
npm.cmd run pack:check
|
|
220
|
+
```
|
|
221
|
+
|
|
222
|
+
Test the MCP process manually:
|
|
223
|
+
|
|
224
|
+
```powershell
|
|
225
|
+
npm.cmd run mcp
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
A healthy server prints:
|
|
229
|
+
|
|
230
|
+
```text
|
|
231
|
+
ModelShortlist MCP server running on stdio
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
and waits for an MCP client. Press `Ctrl+C` to stop it.
|
|
235
|
+
|
|
236
|
+
Contributions are welcome. See [CONTRIBUTING.md](./CONTRIBUTING.md).
|
|
237
|
+
|
|
238
|
+
## License
|
|
239
|
+
|
|
240
|
+
MIT. See [LICENSE](./LICENSE).
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { fetchJson } from './http.js'
|
|
2
|
+
|
|
3
|
+
const AA_BASE_URL = 'https://artificialanalysis.ai/api/v2'
|
|
4
|
+
|
|
5
|
+
function requireApiKey() {
|
|
6
|
+
const apiKey = process.env.ARTIFICIAL_ANALYSIS_API_KEY
|
|
7
|
+
if (!apiKey) {
|
|
8
|
+
throw new Error('ARTIFICIAL_ANALYSIS_API_KEY is not configured')
|
|
9
|
+
}
|
|
10
|
+
return apiKey
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export async function fetchArtificialAnalysisModels() {
|
|
14
|
+
const apiKey = requireApiKey()
|
|
15
|
+
const models = []
|
|
16
|
+
let page = 1
|
|
17
|
+
let hasMore = true
|
|
18
|
+
let tier = null
|
|
19
|
+
let intelligenceIndexVersion = null
|
|
20
|
+
let rateLimit = null
|
|
21
|
+
|
|
22
|
+
while (hasMore) {
|
|
23
|
+
if (page > 20) {
|
|
24
|
+
throw new Error('Artificial Analysis pagination exceeded safety limit')
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const url = new URL(`${AA_BASE_URL}/language/models/free`)
|
|
28
|
+
url.searchParams.set('page', String(page))
|
|
29
|
+
|
|
30
|
+
const { body, headers } = await fetchJson(url, {
|
|
31
|
+
headers: {
|
|
32
|
+
'x-api-key': apiKey,
|
|
33
|
+
accept: 'application/json',
|
|
34
|
+
},
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
if (!Array.isArray(body?.data)) {
|
|
38
|
+
throw new Error('Unexpected Artificial Analysis response: data is not an array')
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
models.push(...body.data)
|
|
42
|
+
tier = body.tier ?? headers.get('x-aa-tier') ?? tier
|
|
43
|
+
intelligenceIndexVersion =
|
|
44
|
+
body.intelligence_index_version ?? intelligenceIndexVersion
|
|
45
|
+
rateLimit = {
|
|
46
|
+
limit: numberOrNull(headers.get('x-ratelimit-limit')),
|
|
47
|
+
remaining: numberOrNull(headers.get('x-ratelimit-remaining')),
|
|
48
|
+
resetUnix: numberOrNull(headers.get('x-ratelimit-reset')),
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
hasMore = Boolean(body.pagination?.has_more)
|
|
52
|
+
page += 1
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return {
|
|
56
|
+
models,
|
|
57
|
+
meta: {
|
|
58
|
+
tier,
|
|
59
|
+
pagesFetched: page - 1,
|
|
60
|
+
intelligenceIndexVersion,
|
|
61
|
+
rateLimit,
|
|
62
|
+
},
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function numberOrNull(value) {
|
|
67
|
+
if (value == null || value === '') return null
|
|
68
|
+
const parsed = Number(value)
|
|
69
|
+
return Number.isFinite(parsed) ? parsed : null
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function compactArtificialAnalysisModel(model) {
|
|
73
|
+
return {
|
|
74
|
+
id: model.id ?? null,
|
|
75
|
+
slug: model.slug ?? null,
|
|
76
|
+
name: model.name ?? null,
|
|
77
|
+
creator: model.model_creator?.name ?? null,
|
|
78
|
+
release_date: model.release_date ?? null,
|
|
79
|
+
evaluations: {
|
|
80
|
+
intelligence_index:
|
|
81
|
+
model.evaluations?.artificial_analysis_intelligence_index ?? null,
|
|
82
|
+
coding_index:
|
|
83
|
+
model.evaluations?.artificial_analysis_coding_index ?? null,
|
|
84
|
+
agentic_index:
|
|
85
|
+
model.evaluations?.artificial_analysis_agentic_index ?? null,
|
|
86
|
+
},
|
|
87
|
+
pricing_usd_per_1m_tokens: {
|
|
88
|
+
input: model.pricing?.price_1m_input_tokens ?? null,
|
|
89
|
+
output: model.pricing?.price_1m_output_tokens ?? null,
|
|
90
|
+
cache_hit: model.pricing?.price_1m_cache_hit_tokens ?? null,
|
|
91
|
+
cache_write: model.pricing?.price_1m_cache_write_tokens ?? null,
|
|
92
|
+
},
|
|
93
|
+
performance: {
|
|
94
|
+
median_output_tokens_per_second:
|
|
95
|
+
model.performance?.median_output_tokens_per_second ?? null,
|
|
96
|
+
median_time_to_first_token_seconds:
|
|
97
|
+
model.performance?.median_time_to_first_token_seconds ?? null,
|
|
98
|
+
median_time_to_first_answer_token_seconds:
|
|
99
|
+
model.performance?.median_time_to_first_answer_token_seconds ?? null,
|
|
100
|
+
median_end_to_end_response_time_seconds:
|
|
101
|
+
model.performance?.median_end_to_end_response_time_seconds ?? null,
|
|
102
|
+
},
|
|
103
|
+
}
|
|
104
|
+
}
|
package/lib/catalog.js
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import aliasesFile from '../config/aliases.json' with { type: 'json' }
|
|
2
|
+
import {
|
|
3
|
+
compactArtificialAnalysisModel,
|
|
4
|
+
fetchArtificialAnalysisModels,
|
|
5
|
+
} from './artificial-analysis.js'
|
|
6
|
+
import {
|
|
7
|
+
buildOpenRouterCatalog,
|
|
8
|
+
fetchOpenRouterModels,
|
|
9
|
+
fetchOpenRouterZdrEndpoints,
|
|
10
|
+
} from './openrouter.js'
|
|
11
|
+
import { matchModels } from './match.js'
|
|
12
|
+
|
|
13
|
+
const DEFAULT_TTL_MS = 12 * 60 * 60 * 1000
|
|
14
|
+
let cached = null
|
|
15
|
+
|
|
16
|
+
function aliases() {
|
|
17
|
+
return Object.fromEntries(
|
|
18
|
+
Object.entries(aliasesFile).filter(([key]) => !key.startsWith('_')),
|
|
19
|
+
)
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export async function getMergedCatalog({ forceRefresh = false } = {}) {
|
|
23
|
+
const ttl = Number(process.env.MODEL_SELECTOR_CACHE_TTL_MS ?? DEFAULT_TTL_MS)
|
|
24
|
+
const now = Date.now()
|
|
25
|
+
|
|
26
|
+
if (
|
|
27
|
+
!forceRefresh &&
|
|
28
|
+
cached &&
|
|
29
|
+
Number.isFinite(ttl) &&
|
|
30
|
+
ttl > 0 &&
|
|
31
|
+
now - cached.cachedAt < ttl
|
|
32
|
+
) {
|
|
33
|
+
return { ...cached.value, cache: { status: 'hit', ttl_ms: ttl } }
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const [aaResult, openRouterRows, openRouterZdrEndpoints] = await Promise.all([
|
|
37
|
+
fetchArtificialAnalysisModels(),
|
|
38
|
+
fetchOpenRouterModels(),
|
|
39
|
+
fetchOpenRouterZdrEndpoints(),
|
|
40
|
+
])
|
|
41
|
+
|
|
42
|
+
const openRouterModels = buildOpenRouterCatalog(openRouterRows, openRouterZdrEndpoints)
|
|
43
|
+
const aaModels = aaResult.models.map(compactArtificialAnalysisModel)
|
|
44
|
+
const reconciliation = matchModels(openRouterModels, aaModels, aliases())
|
|
45
|
+
|
|
46
|
+
const matchedById = new Map(
|
|
47
|
+
reconciliation.matches.map((entry) => [entry.openRouter.model_id, entry]),
|
|
48
|
+
)
|
|
49
|
+
const unmatchedById = new Map(
|
|
50
|
+
reconciliation.unmatched.map((entry) => [entry.openRouter.model_id, entry]),
|
|
51
|
+
)
|
|
52
|
+
const ambiguousById = new Map(
|
|
53
|
+
reconciliation.ambiguous.map((entry) => [entry.openRouter.model_id, entry]),
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
const models = openRouterModels.map((openRouter) => {
|
|
57
|
+
const matched = matchedById.get(openRouter.model_id)
|
|
58
|
+
if (matched) {
|
|
59
|
+
return {
|
|
60
|
+
...openRouter,
|
|
61
|
+
artificial_analysis: matched.aa,
|
|
62
|
+
reconciliation: matched.match,
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const ambiguous = ambiguousById.get(openRouter.model_id)
|
|
67
|
+
if (ambiguous) {
|
|
68
|
+
return {
|
|
69
|
+
...openRouter,
|
|
70
|
+
artificial_analysis: null,
|
|
71
|
+
reconciliation: {
|
|
72
|
+
method: 'ambiguous',
|
|
73
|
+
confidence: 'none',
|
|
74
|
+
candidates: ambiguous.candidates,
|
|
75
|
+
},
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
...openRouter,
|
|
81
|
+
artificial_analysis: null,
|
|
82
|
+
reconciliation: {
|
|
83
|
+
method: 'unmatched',
|
|
84
|
+
confidence: 'none',
|
|
85
|
+
reason: unmatchedById.get(openRouter.model_id)?.reason ?? 'No Artificial Analysis match',
|
|
86
|
+
},
|
|
87
|
+
}
|
|
88
|
+
})
|
|
89
|
+
|
|
90
|
+
const value = {
|
|
91
|
+
generated_at: new Date().toISOString(),
|
|
92
|
+
models,
|
|
93
|
+
diagnostics: {
|
|
94
|
+
openrouter_model_count: openRouterModels.length,
|
|
95
|
+
openrouter_zdr_endpoint_count: openRouterZdrEndpoints.length,
|
|
96
|
+
openrouter_zdr_model_count: openRouterModels.filter((model) => model.zdr).length,
|
|
97
|
+
artificial_analysis_model_count: aaModels.length,
|
|
98
|
+
matched_model_count: reconciliation.matches.length,
|
|
99
|
+
unmatched_model_count: reconciliation.unmatched.length,
|
|
100
|
+
ambiguous_model_count: reconciliation.ambiguous.length,
|
|
101
|
+
aa: aaResult.meta,
|
|
102
|
+
},
|
|
103
|
+
unmatched: reconciliation.unmatched.map(({ openRouter, reason }) => ({
|
|
104
|
+
model_id: openRouter.model_id,
|
|
105
|
+
model_name: openRouter.model_name,
|
|
106
|
+
reason,
|
|
107
|
+
})),
|
|
108
|
+
ambiguous: reconciliation.ambiguous.map(({ openRouter, candidates }) => ({
|
|
109
|
+
model_id: openRouter.model_id,
|
|
110
|
+
model_name: openRouter.model_name,
|
|
111
|
+
candidates,
|
|
112
|
+
})),
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
cached = { cachedAt: now, value }
|
|
116
|
+
return { ...value, cache: { status: 'miss', ttl_ms: ttl } }
|
|
117
|
+
}
|
package/lib/http.js
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
const DEFAULT_TIMEOUT_MS = 15000
|
|
2
|
+
const DEFAULT_ATTEMPTS = 3
|
|
3
|
+
|
|
4
|
+
function sleep(ms) {
|
|
5
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export async function fetchJson(url, options = {}) {
|
|
9
|
+
const {
|
|
10
|
+
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
11
|
+
attempts = DEFAULT_ATTEMPTS,
|
|
12
|
+
...fetchOptions
|
|
13
|
+
} = options
|
|
14
|
+
|
|
15
|
+
let lastError
|
|
16
|
+
for (let attempt = 1; attempt <= attempts; attempt += 1) {
|
|
17
|
+
const controller = new AbortController()
|
|
18
|
+
const timeout = setTimeout(() => controller.abort(), timeoutMs)
|
|
19
|
+
|
|
20
|
+
try {
|
|
21
|
+
const response = await fetch(url, {
|
|
22
|
+
...fetchOptions,
|
|
23
|
+
signal: controller.signal,
|
|
24
|
+
})
|
|
25
|
+
|
|
26
|
+
const text = await response.text()
|
|
27
|
+
let body = null
|
|
28
|
+
try {
|
|
29
|
+
body = text ? JSON.parse(text) : null
|
|
30
|
+
} catch {
|
|
31
|
+
body = { raw: text.slice(0, 1000) }
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
if (response.ok) {
|
|
35
|
+
return { body, headers: response.headers, status: response.status }
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const retryable = response.status === 429 || response.status >= 500
|
|
39
|
+
const message =
|
|
40
|
+
body?.error ||
|
|
41
|
+
body?.message ||
|
|
42
|
+
`HTTP ${response.status} from ${new URL(url).hostname}`
|
|
43
|
+
|
|
44
|
+
const error = new Error(message)
|
|
45
|
+
error.status = response.status
|
|
46
|
+
error.responseBody = body
|
|
47
|
+
error.retryAfter = response.headers.get('retry-after')
|
|
48
|
+
lastError = error
|
|
49
|
+
|
|
50
|
+
if (!retryable || attempt === attempts) throw error
|
|
51
|
+
|
|
52
|
+
const retryAfterMs = Number(error.retryAfter) * 1000
|
|
53
|
+
const backoffMs = Number.isFinite(retryAfterMs) && retryAfterMs > 0
|
|
54
|
+
? Math.min(retryAfterMs, 10000)
|
|
55
|
+
: 400 * (2 ** (attempt - 1))
|
|
56
|
+
await sleep(backoffMs)
|
|
57
|
+
} catch (error) {
|
|
58
|
+
lastError = error
|
|
59
|
+
if (attempt === attempts) throw error
|
|
60
|
+
if (error?.status && error.status < 500 && error.status !== 429) throw error
|
|
61
|
+
await sleep(400 * (2 ** (attempt - 1)))
|
|
62
|
+
} finally {
|
|
63
|
+
clearTimeout(timeout)
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
throw lastError ?? new Error('Request failed')
|
|
68
|
+
}
|
package/lib/match.js
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
function ascii(value) {
|
|
2
|
+
return String(value ?? '')
|
|
3
|
+
.normalize('NFKD')
|
|
4
|
+
.replace(/[\u0300-\u036f]/g, '')
|
|
5
|
+
.toLowerCase()
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export function normalizeModelLabel(value) {
|
|
9
|
+
return ascii(value)
|
|
10
|
+
.replace(/&/g, ' and ')
|
|
11
|
+
.replace(/[^a-z0-9]+/g, '')
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function normalizedVariantsForAa(model) {
|
|
15
|
+
return new Set(
|
|
16
|
+
[
|
|
17
|
+
model.name,
|
|
18
|
+
model.slug,
|
|
19
|
+
model.creator && model.name ? `${model.creator} ${model.name}` : null,
|
|
20
|
+
]
|
|
21
|
+
.filter(Boolean)
|
|
22
|
+
.map(normalizeModelLabel)
|
|
23
|
+
.filter(Boolean),
|
|
24
|
+
)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function normalizedVariantsForOpenRouter(model) {
|
|
28
|
+
const slugPart = model.model_id?.includes('/')
|
|
29
|
+
? model.model_id.slice(model.model_id.indexOf('/') + 1)
|
|
30
|
+
: model.model_id
|
|
31
|
+
|
|
32
|
+
return new Set(
|
|
33
|
+
[model.model_name, slugPart, model.model_id]
|
|
34
|
+
.filter(Boolean)
|
|
35
|
+
.map(normalizeModelLabel)
|
|
36
|
+
.filter(Boolean),
|
|
37
|
+
)
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function matchModels(openRouterModels, aaModels, aliases = {}) {
|
|
41
|
+
const aaBySlug = new Map(aaModels.map((m) => [m.slug, m]))
|
|
42
|
+
const aaVariants = aaModels.map((model) => ({
|
|
43
|
+
model,
|
|
44
|
+
variants: normalizedVariantsForAa(model),
|
|
45
|
+
}))
|
|
46
|
+
|
|
47
|
+
const matches = []
|
|
48
|
+
const unmatched = []
|
|
49
|
+
const ambiguous = []
|
|
50
|
+
|
|
51
|
+
for (const openRouter of openRouterModels) {
|
|
52
|
+
const aliasSlug = aliases[openRouter.model_id]
|
|
53
|
+
if (aliasSlug) {
|
|
54
|
+
const aa = aaBySlug.get(aliasSlug)
|
|
55
|
+
if (aa) {
|
|
56
|
+
matches.push({
|
|
57
|
+
openRouter,
|
|
58
|
+
aa,
|
|
59
|
+
match: { method: 'alias', confidence: 'verified' },
|
|
60
|
+
})
|
|
61
|
+
continue
|
|
62
|
+
}
|
|
63
|
+
unmatched.push({
|
|
64
|
+
openRouter,
|
|
65
|
+
reason: `Alias points to missing AA slug: ${aliasSlug}`,
|
|
66
|
+
})
|
|
67
|
+
continue
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const orVariants = normalizedVariantsForOpenRouter(openRouter)
|
|
71
|
+
const candidates = aaVariants.filter(({ variants }) =>
|
|
72
|
+
[...orVariants].some((value) => variants.has(value)),
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
if (candidates.length === 1) {
|
|
76
|
+
matches.push({
|
|
77
|
+
openRouter,
|
|
78
|
+
aa: candidates[0].model,
|
|
79
|
+
match: { method: 'exact-normalized', confidence: 'high' },
|
|
80
|
+
})
|
|
81
|
+
} else if (candidates.length > 1) {
|
|
82
|
+
ambiguous.push({
|
|
83
|
+
openRouter,
|
|
84
|
+
candidates: candidates.map(({ model }) => ({
|
|
85
|
+
id: model.id,
|
|
86
|
+
slug: model.slug,
|
|
87
|
+
name: model.name,
|
|
88
|
+
creator: model.creator,
|
|
89
|
+
})),
|
|
90
|
+
})
|
|
91
|
+
} else {
|
|
92
|
+
unmatched.push({
|
|
93
|
+
openRouter,
|
|
94
|
+
reason: 'No exact normalized AA name/slug match',
|
|
95
|
+
})
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
return { matches, unmatched, ambiguous }
|
|
100
|
+
}
|