@aggc/or-info 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +294 -0
- package/bin/or-info.mjs +240 -0
- package/lib/cache.mjs +117 -0
- package/lib/formatter.mjs +178 -0
- package/lib/lmarena.mjs +174 -0
- package/lib/openrouter.mjs +125 -0
- package/lib/paths.mjs +53 -0
- package/lib/scorer.mjs +81 -0
- package/lib/secrets.mjs +41 -0
- package/mcp/server.mjs +213 -0
- package/package.json +51 -0
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
import chalk from 'chalk';
|
|
2
|
+
import { pricePerMillion, contextLength, maxOutput, modelTags, isFree } from './openrouter.mjs';
|
|
3
|
+
|
|
4
|
+
const DIM = chalk.dim;
|
|
5
|
+
const BOLD = chalk.bold;
|
|
6
|
+
const GREEN = chalk.green;
|
|
7
|
+
const YELLOW = chalk.yellow;
|
|
8
|
+
const CYAN = chalk.cyan;
|
|
9
|
+
const RED = chalk.red;
|
|
10
|
+
const BLUE = chalk.blue;
|
|
11
|
+
const MAGENTA = chalk.magenta;
|
|
12
|
+
|
|
13
|
+
function fmtPrice(n) {
|
|
14
|
+
if (n === null) return DIM('n/a');
|
|
15
|
+
if (n === 0) return GREEN('free');
|
|
16
|
+
if (n < 0.1) return GREEN(`$${n.toFixed(4)}`);
|
|
17
|
+
if (n < 1) return YELLOW(`$${n.toFixed(4)}`);
|
|
18
|
+
return RED(`$${n.toFixed(4)}`);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function fmtCtx(n) {
|
|
22
|
+
if (!n) return DIM('n/a');
|
|
23
|
+
if (n >= 1_000_000) return CYAN(`${(n / 1_000_000).toFixed(1)}M`);
|
|
24
|
+
if (n >= 1_000) return CYAN(`${Math.round(n / 1_000)}k`);
|
|
25
|
+
return CYAN(String(n));
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function fmtTags(tags) {
|
|
29
|
+
if (!tags.length) return '';
|
|
30
|
+
return tags.map((t) => DIM(`[${t}]`)).join(' ');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function pad(s, n, right = false) {
|
|
34
|
+
const str = String(s ?? '');
|
|
35
|
+
// Strip ANSI for length calculation
|
|
36
|
+
const visible = str.replace(/\x1b\[[0-9;]*m/g, '');
|
|
37
|
+
const diff = n - visible.length;
|
|
38
|
+
if (diff <= 0) return str;
|
|
39
|
+
return right ? ' '.repeat(diff) + str : str + ' '.repeat(diff);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function modelTable(models, { showTags = false } = {}) {
|
|
43
|
+
if (!models.length) return DIM('No models found.');
|
|
44
|
+
|
|
45
|
+
const rows = models.map((m) => {
|
|
46
|
+
const price = pricePerMillion(m);
|
|
47
|
+
const ctx = contextLength(m);
|
|
48
|
+
const tags = showTags ? modelTags(m) : [];
|
|
49
|
+
return {
|
|
50
|
+
id: m.id,
|
|
51
|
+
name: m.name ?? m.id,
|
|
52
|
+
input: fmtPrice(price.input),
|
|
53
|
+
output: fmtPrice(price.output),
|
|
54
|
+
ctx: fmtCtx(ctx),
|
|
55
|
+
tags: fmtTags(tags),
|
|
56
|
+
};
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
const idW = Math.min(50, Math.max(20, ...rows.map((r) => r.id.length)));
|
|
60
|
+
const nameW = Math.min(30, Math.max(10, ...rows.map((r) => r.name.replace(/\x1b\[[0-9;]*m/g, '').length)));
|
|
61
|
+
|
|
62
|
+
const header =
|
|
63
|
+
BOLD(pad('ID', idW)) + ' ' +
|
|
64
|
+
BOLD(pad('In /M', 10, true)) + ' ' +
|
|
65
|
+
BOLD(pad('Out /M', 10, true)) + ' ' +
|
|
66
|
+
BOLD(pad('Context', 8, true));
|
|
67
|
+
|
|
68
|
+
const sep = DIM('─'.repeat(idW + 2 + 10 + 2 + 10 + 2 + 8));
|
|
69
|
+
|
|
70
|
+
const lines = rows.map((r) => {
|
|
71
|
+
const line =
|
|
72
|
+
pad(r.id, idW) + ' ' +
|
|
73
|
+
pad(r.input, 10, true) + ' ' +
|
|
74
|
+
pad(r.output, 10, true) + ' ' +
|
|
75
|
+
pad(r.ctx, 8, true);
|
|
76
|
+
return r.tags ? line + ' ' + r.tags : line;
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
return [header, sep, ...lines].join('\n');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function modelDetail(model, bench = null) {
|
|
83
|
+
const price = pricePerMillion(model);
|
|
84
|
+
const ctx = contextLength(model);
|
|
85
|
+
const maxOut = maxOutput(model);
|
|
86
|
+
const tags = modelTags(model);
|
|
87
|
+
|
|
88
|
+
const lines = [
|
|
89
|
+
'',
|
|
90
|
+
BOLD(CYAN(model.name ?? model.id)),
|
|
91
|
+
DIM(model.id),
|
|
92
|
+
'',
|
|
93
|
+
BOLD('Pricing'),
|
|
94
|
+
` Input ${fmtPrice(price.input)} per 1M tokens`,
|
|
95
|
+
` Output ${fmtPrice(price.output)} per 1M tokens`,
|
|
96
|
+
];
|
|
97
|
+
|
|
98
|
+
if (price.image !== null) lines.push(` Image ${fmtPrice(price.image)} per 1M tokens`);
|
|
99
|
+
if (price.cacheRead !== null) lines.push(` Cache↓ ${fmtPrice(price.cacheRead)} per 1M tokens`);
|
|
100
|
+
if (price.cacheWrite !== null) lines.push(` Cache↑ ${fmtPrice(price.cacheWrite)} per 1M tokens`);
|
|
101
|
+
|
|
102
|
+
lines.push('');
|
|
103
|
+
lines.push(BOLD('Capacity'));
|
|
104
|
+
lines.push(` Context ${fmtCtx(ctx)}`);
|
|
105
|
+
if (maxOut) lines.push(` Max output ${fmtCtx(maxOut)}`);
|
|
106
|
+
|
|
107
|
+
if (tags.length) {
|
|
108
|
+
lines.push('');
|
|
109
|
+
lines.push(BOLD('Features'));
|
|
110
|
+
lines.push(' ' + tags.map((t) => BLUE(`[${t}]`)).join(' '));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const arch = model?.architecture;
|
|
114
|
+
if (arch?.modality) {
|
|
115
|
+
lines.push('');
|
|
116
|
+
lines.push(BOLD('Architecture'));
|
|
117
|
+
lines.push(` ${DIM(arch.modality)}`);
|
|
118
|
+
if (arch.tokenizer) lines.push(` Tokenizer: ${DIM(arch.tokenizer)}`);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (bench) {
|
|
122
|
+
lines.push('');
|
|
123
|
+
lines.push(BOLD('LMArena ELO ') + DIM('(lmarena.ai — human preference votes)'));
|
|
124
|
+
lines.push(` ELO ${YELLOW(String(bench.elo))} ${DIM(`±${Math.round((bench.eloUpper - bench.eloLower) / 2)}`)}`);
|
|
125
|
+
lines.push(` Rank ${YELLOW('#' + bench.rank)}`);
|
|
126
|
+
lines.push(` Votes ${DIM(bench.votes.toLocaleString())}`);
|
|
127
|
+
if (bench.updatedAt) lines.push(` ${DIM('Updated: ' + bench.updatedAt)}`);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
lines.push('');
|
|
131
|
+
return lines.join('\n');
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function comparison(modelA, benchA, modelB, benchB) {
|
|
135
|
+
const render = (model, bench) => modelDetail(model, bench).split('\n');
|
|
136
|
+
|
|
137
|
+
const linesA = render(modelA, benchA);
|
|
138
|
+
const linesB = render(modelB, benchB);
|
|
139
|
+
const maxLines = Math.max(linesA.length, linesB.length);
|
|
140
|
+
const colW = 52;
|
|
141
|
+
|
|
142
|
+
const result = [BOLD('── Comparison ──'), ''];
|
|
143
|
+
for (let i = 0; i < maxLines; i++) {
|
|
144
|
+
const a = linesA[i] ?? '';
|
|
145
|
+
const b = linesB[i] ?? '';
|
|
146
|
+
const aVis = a.replace(/\x1b\[[0-9;]*m/g, '');
|
|
147
|
+
const padding = Math.max(0, colW - aVis.length);
|
|
148
|
+
result.push(a + ' '.repeat(padding) + DIM('│') + ' ' + b);
|
|
149
|
+
}
|
|
150
|
+
return result.join('\n');
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export function topList(ranked) {
|
|
154
|
+
if (!ranked.length) return DIM('No results.');
|
|
155
|
+
|
|
156
|
+
const lines = [BOLD('── Top Models ──'), ''];
|
|
157
|
+
ranked.forEach(({ model, score, qualityScore, eloEntry }, i) => {
|
|
158
|
+
const price = pricePerMillion(model);
|
|
159
|
+
const medal = i === 0 ? YELLOW('#1') : i === 1 ? DIM('#2') : i === 2 ? DIM('#3') : DIM(`#${i + 1}`);
|
|
160
|
+
const eloStr = eloEntry ? DIM(`ELO ${eloEntry.elo}`) : DIM('no ELO');
|
|
161
|
+
lines.push(`${medal} ${BOLD(CYAN(model.id))}`);
|
|
162
|
+
lines.push(` Score ${YELLOW(score.toFixed(1))} │ Out ${fmtPrice(price.output)}/M │ ${eloStr}`);
|
|
163
|
+
lines.push('');
|
|
164
|
+
});
|
|
165
|
+
return lines.join('\n');
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export function statusReport(items) {
|
|
169
|
+
const lines = [BOLD('── Cache Status ──'), ''];
|
|
170
|
+
for (const { key, exists, ageMs, fresh, ttlMs } of items) {
|
|
171
|
+
const indicator = !exists ? RED('✗ missing') : fresh ? GREEN('✓ fresh') : YELLOW('⚠ stale');
|
|
172
|
+
const age = ageMs ? `${Math.round(ageMs / 60_000)}m ago` : '—';
|
|
173
|
+
const ttl = `TTL ${Math.round(ttlMs / 60_000)}m`;
|
|
174
|
+
lines.push(` ${pad(key, 12)} ${indicator} ${DIM(age)} ${DIM(ttl)}`);
|
|
175
|
+
}
|
|
176
|
+
lines.push('');
|
|
177
|
+
return lines.join('\n');
|
|
178
|
+
}
|
package/lib/lmarena.mjs
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { get, set, TTL } from './cache.mjs';
|
|
2
|
+
import { BENCHMARKS_CACHE } from './paths.mjs';
|
|
3
|
+
|
|
4
|
+
const HF_ROWS =
|
|
5
|
+
'https://datasets-server.huggingface.co/rows?dataset=lmarena-ai%2Fleaderboard-dataset&config=text&split=latest';
|
|
6
|
+
const PAGE = 100;
|
|
7
|
+
const RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 504]);
|
|
8
|
+
const MAX_FETCH_RETRIES = 2;
|
|
9
|
+
|
|
10
|
+
function retryDelayMs(retryAfter, attempt) {
|
|
11
|
+
const seconds = Number(retryAfter);
|
|
12
|
+
if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000;
|
|
13
|
+
|
|
14
|
+
const dateMs = Date.parse(retryAfter ?? '');
|
|
15
|
+
if (Number.isFinite(dateMs)) return Math.max(0, dateMs - Date.now());
|
|
16
|
+
|
|
17
|
+
return 500 * (attempt + 1);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async function sleep(ms) {
|
|
21
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// ── Fetch ──────────────────────────────────────────────────────────────────
|
|
25
|
+
|
|
26
|
+
async function fetchPage(offset) {
|
|
27
|
+
const url = `${HF_ROWS}&offset=${offset}&length=${PAGE}`;
|
|
28
|
+
for (let attempt = 0; attempt <= MAX_FETCH_RETRIES; attempt++) {
|
|
29
|
+
let res;
|
|
30
|
+
try {
|
|
31
|
+
res = await fetch(url, {
|
|
32
|
+
headers: { 'User-Agent': 'or-info-cli/0.1.0' },
|
|
33
|
+
signal: AbortSignal.timeout(15_000),
|
|
34
|
+
});
|
|
35
|
+
} catch (err) {
|
|
36
|
+
if (attempt === MAX_FETCH_RETRIES) {
|
|
37
|
+
throw new Error(`LMArena request failed: ${err.message}`);
|
|
38
|
+
}
|
|
39
|
+
await sleep(retryDelayMs(null, attempt));
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
if (res.ok) return res.json();
|
|
44
|
+
if (attempt === MAX_FETCH_RETRIES || !RETRYABLE_STATUSES.has(res.status)) {
|
|
45
|
+
throw new Error(`LMArena fetch ${res.status}: ${res.statusText}`);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
await sleep(retryDelayMs(res.headers.get('retry-after'), attempt));
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
throw new Error('LMArena request failed: exhausted retries');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Fetch all rows where category === 'overall'.
|
|
55
|
+
// The dataset is sorted so 'overall' rows appear first; we stop
|
|
56
|
+
// as soon as we see a different category.
|
|
57
|
+
async function fetchAllOverall() {
|
|
58
|
+
const entries = [];
|
|
59
|
+
let offset = 0;
|
|
60
|
+
|
|
61
|
+
while (true) {
|
|
62
|
+
const page = await fetchPage(offset);
|
|
63
|
+
const rows = page.rows ?? [];
|
|
64
|
+
if (!rows.length) break;
|
|
65
|
+
|
|
66
|
+
let sawOther = false;
|
|
67
|
+
for (const { row } of rows) {
|
|
68
|
+
if (row.category !== 'overall') { sawOther = true; break; }
|
|
69
|
+
entries.push({
|
|
70
|
+
lmarenaName: row.model_name,
|
|
71
|
+
elo: Math.round(row.rating),
|
|
72
|
+
eloLower: Math.round(row.rating_lower),
|
|
73
|
+
eloUpper: Math.round(row.rating_upper),
|
|
74
|
+
votes: Math.round(row.vote_count),
|
|
75
|
+
rank: Math.round(row.rank),
|
|
76
|
+
updatedAt: row.leaderboard_publish_date,
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
if (sawOther) break;
|
|
80
|
+
offset += PAGE;
|
|
81
|
+
if (offset >= (page.num_rows_total ?? Infinity)) break;
|
|
82
|
+
}
|
|
83
|
+
return entries;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ── Name normalisation ─────────────────────────────────────────────────────
|
|
87
|
+
|
|
88
|
+
// Normalise a name for fuzzy matching.
|
|
89
|
+
// Rules (applied in order):
|
|
90
|
+
// 1. Lowercase
|
|
91
|
+
// 2. Dots → hyphens (4.6 → 4-6)
|
|
92
|
+
// 3. Remove date suffixes like -20250929 or -2025-09-29
|
|
93
|
+
// 4. Collapse multiple hyphens
|
|
94
|
+
function normalise(name) {
|
|
95
|
+
return name
|
|
96
|
+
.toLowerCase()
|
|
97
|
+
.replace(/\./g, '-')
|
|
98
|
+
.replace(/-20\d{2}-?\d{2}-?\d{2,}(-.+)?$/, '')
|
|
99
|
+
.replace(/-{2,}/g, '-')
|
|
100
|
+
.trim();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Extract the "slug" portion of an OpenRouter model ID (after the provider /).
|
|
104
|
+
// openai/gpt-4o → gpt-4o
|
|
105
|
+
// anthropic/claude-opus-4.6 → claude-opus-4-6 (after normalise)
|
|
106
|
+
function orSlug(id) {
|
|
107
|
+
return normalise(id.includes('/') ? id.split('/').slice(1).join('/') : id);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Build a lookup Map from normalised LMArena name → entry.
|
|
111
|
+
// Also index by the last "segment" after the last '-' number group
|
|
112
|
+
// to help with partial matches.
|
|
113
|
+
function buildIndex(entries) {
|
|
114
|
+
const exact = new Map();
|
|
115
|
+
for (const e of entries) {
|
|
116
|
+
exact.set(normalise(e.lmarenaName), e);
|
|
117
|
+
}
|
|
118
|
+
return exact;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Find the best matching LMArena entry for an OpenRouter model ID.
|
|
122
|
+
// Returns the entry or null.
|
|
123
|
+
function match(orId, index) {
|
|
124
|
+
const slug = orSlug(orId);
|
|
125
|
+
|
|
126
|
+
// 1. Exact slug match
|
|
127
|
+
if (index.has(slug)) return index.get(slug);
|
|
128
|
+
|
|
129
|
+
// 2. The OpenRouter slug contains the LMArena key as a substring
|
|
130
|
+
// e.g. OR "deepseek-chat-v3-0324" ⊇ LMArena "v3-0324" (too short, skip < 8 chars)
|
|
131
|
+
for (const [key, entry] of index) {
|
|
132
|
+
if (key.length >= 8 && slug.includes(key)) return entry;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// 3. The LMArena key contains the OR slug
|
|
136
|
+
for (const [key, entry] of index) {
|
|
137
|
+
if (slug.length >= 8 && key.includes(slug)) return entry;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// 4. Full OR ID normalised (keeping provider prefix stripped differently)
|
|
141
|
+
const fullNorm = normalise(orId.replace('/', '-'));
|
|
142
|
+
if (index.has(fullNorm)) return index.get(fullNorm);
|
|
143
|
+
|
|
144
|
+
return null;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// ── Public API ─────────────────────────────────────────────────────────────
|
|
148
|
+
|
|
149
|
+
let _index = null;
|
|
150
|
+
|
|
151
|
+
export async function loadLeaderboard({ force = false } = {}) {
|
|
152
|
+
if (!force) {
|
|
153
|
+
const cached = await get(BENCHMARKS_CACHE, TTL.BENCHMARKS);
|
|
154
|
+
if (cached?.entries) {
|
|
155
|
+
_index = buildIndex(cached.entries);
|
|
156
|
+
return cached.entries;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
const entries = await fetchAllOverall();
|
|
161
|
+
await set(BENCHMARKS_CACHE, { entries, fetchedAt: Date.now() });
|
|
162
|
+
_index = buildIndex(entries);
|
|
163
|
+
return entries;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export async function getElo(orModelId, { force = false } = {}) {
|
|
167
|
+
if (!_index || force) await loadLeaderboard({ force });
|
|
168
|
+
return match(orModelId, _index);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export async function getAllElo({ force = false } = {}) {
|
|
172
|
+
const entries = await loadLeaderboard({ force });
|
|
173
|
+
return entries; // [{lmarenaName, elo, eloLower, eloUpper, votes, rank, updatedAt}]
|
|
174
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { get, set, TTL } from './cache.mjs';
|
|
2
|
+
import { MODELS_CACHE } from './paths.mjs';
|
|
3
|
+
|
|
4
|
+
const OR_BASE = 'https://openrouter.ai/api/v1';
|
|
5
|
+
const USER_AGENT = 'or-info-cli/0.1.0 (https://github.com/jmtrs/or-info)';
|
|
6
|
+
const RETRYABLE_STATUSES = new Set([429, 500, 502, 503, 504]);
|
|
7
|
+
const MAX_FETCH_RETRIES = 2;
|
|
8
|
+
|
|
9
|
+
function retryDelayMs(retryAfter, attempt) {
|
|
10
|
+
const seconds = Number(retryAfter);
|
|
11
|
+
if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000;
|
|
12
|
+
|
|
13
|
+
const dateMs = Date.parse(retryAfter ?? '');
|
|
14
|
+
if (Number.isFinite(dateMs)) return Math.max(0, dateMs - Date.now());
|
|
15
|
+
|
|
16
|
+
return 500 * (attempt + 1);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
async function sleep(ms) {
|
|
20
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
async function fetchJson(url, apiKey) {
|
|
24
|
+
const headers = { 'User-Agent': USER_AGENT };
|
|
25
|
+
if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`;
|
|
26
|
+
|
|
27
|
+
for (let attempt = 0; attempt <= MAX_FETCH_RETRIES; attempt++) {
|
|
28
|
+
let res;
|
|
29
|
+
try {
|
|
30
|
+
res = await fetch(url, { headers });
|
|
31
|
+
} catch (err) {
|
|
32
|
+
if (attempt === MAX_FETCH_RETRIES) {
|
|
33
|
+
throw new Error(`OpenRouter request failed: ${err.message}`);
|
|
34
|
+
}
|
|
35
|
+
await sleep(retryDelayMs(null, attempt));
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (res.ok) return res.json();
|
|
40
|
+
|
|
41
|
+
let message = res.statusText;
|
|
42
|
+
try {
|
|
43
|
+
const body = await res.json();
|
|
44
|
+
message = body?.error?.message || body?.message || message;
|
|
45
|
+
} catch {}
|
|
46
|
+
|
|
47
|
+
if (attempt === MAX_FETCH_RETRIES || !RETRYABLE_STATUSES.has(res.status)) {
|
|
48
|
+
throw new Error(`OpenRouter ${res.status}: ${message}`);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
await sleep(retryDelayMs(res.headers.get('retry-after'), attempt));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
throw new Error('OpenRouter request failed: exhausted retries');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export async function fetchModels({ apiKey, force = false } = {}) {
|
|
58
|
+
if (!force) {
|
|
59
|
+
const cached = await get(MODELS_CACHE, TTL.MODELS);
|
|
60
|
+
if (cached?.data) return cached.data;
|
|
61
|
+
}
|
|
62
|
+
const fresh = await fetchJson(`${OR_BASE}/models`, apiKey);
|
|
63
|
+
await set(MODELS_CACHE, fresh);
|
|
64
|
+
return fresh.data ?? [];
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function findModel(models, id) {
|
|
68
|
+
return models.find((m) => m?.id === id) ?? null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function pricePerMillion(model) {
|
|
72
|
+
const p = model?.pricing ?? {};
|
|
73
|
+
const toM = (v) => {
|
|
74
|
+
const n = Number(v);
|
|
75
|
+
return Number.isFinite(n) && n >= 0 ? n * 1_000_000 : null;
|
|
76
|
+
};
|
|
77
|
+
return {
|
|
78
|
+
input: toM(p.prompt),
|
|
79
|
+
output: toM(p.completion),
|
|
80
|
+
image: toM(p.image),
|
|
81
|
+
cacheRead: toM(p.input_cache_read),
|
|
82
|
+
cacheWrite: toM(p.input_cache_write),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function contextLength(model) {
|
|
87
|
+
const raw = model?.top_provider?.context_length ?? model?.context_length;
|
|
88
|
+
const n = Number(raw);
|
|
89
|
+
return Number.isFinite(n) && n > 0 ? n : null;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function maxOutput(model) {
|
|
93
|
+
const n = Number(model?.top_provider?.max_completion_tokens);
|
|
94
|
+
return Number.isFinite(n) && n > 0 ? n : null;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function supportsFeature(model, feature) {
|
|
98
|
+
const params = Array.isArray(model?.supported_parameters) ? model.supported_parameters : [];
|
|
99
|
+
const featureMap = {
|
|
100
|
+
reasoning: ['include_reasoning', 'reasoning'],
|
|
101
|
+
tools: ['tools', 'tool_choice'],
|
|
102
|
+
vision: () => (model?.architecture?.input_modalities ?? []).includes('image'),
|
|
103
|
+
structured: ['structured_outputs'],
|
|
104
|
+
};
|
|
105
|
+
const check = featureMap[feature];
|
|
106
|
+
if (typeof check === 'function') return check();
|
|
107
|
+
if (Array.isArray(check)) return check.some((f) => params.includes(f));
|
|
108
|
+
return params.includes(feature);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function modelTags(model) {
|
|
112
|
+
const tags = [];
|
|
113
|
+
if (supportsFeature(model, 'reasoning')) tags.push('reasoning');
|
|
114
|
+
if (supportsFeature(model, 'tools')) tags.push('tools');
|
|
115
|
+
if (supportsFeature(model, 'vision')) tags.push('vision');
|
|
116
|
+
if (supportsFeature(model, 'structured')) tags.push('structured');
|
|
117
|
+
const ctx = contextLength(model);
|
|
118
|
+
if (ctx && ctx >= 128_000) tags.push('long-context');
|
|
119
|
+
return tags;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export function isFree(model) {
|
|
123
|
+
const p = pricePerMillion(model);
|
|
124
|
+
return p.input === 0 && p.output === 0;
|
|
125
|
+
}
|
package/lib/paths.mjs
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { homedir } from 'node:os';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
|
|
4
|
+
function cleanEnvPath(value) {
|
|
5
|
+
return typeof value === 'string' && value.trim() ? value.trim() : null;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
export function resolveAppPaths({
|
|
9
|
+
platform = process.platform,
|
|
10
|
+
env = process.env,
|
|
11
|
+
homeDir = homedir(),
|
|
12
|
+
} = {}) {
|
|
13
|
+
const pathApi = platform === 'win32' ? path.win32 : path.posix;
|
|
14
|
+
const join = pathApi.join;
|
|
15
|
+
const resolve = pathApi.resolve;
|
|
16
|
+
const configOverride = cleanEnvPath(env.OR_INFO_CONFIG_DIR);
|
|
17
|
+
const cacheOverride = cleanEnvPath(env.OR_INFO_CACHE_DIR);
|
|
18
|
+
|
|
19
|
+
let defaultConfigRoot;
|
|
20
|
+
let defaultCacheRoot;
|
|
21
|
+
|
|
22
|
+
if (platform === 'win32') {
|
|
23
|
+
defaultConfigRoot = cleanEnvPath(env.APPDATA) ?? join(homeDir, 'AppData', 'Roaming');
|
|
24
|
+
defaultCacheRoot =
|
|
25
|
+
cleanEnvPath(env.LOCALAPPDATA) ??
|
|
26
|
+
cleanEnvPath(env.APPDATA) ??
|
|
27
|
+
join(homeDir, 'AppData', 'Local');
|
|
28
|
+
} else {
|
|
29
|
+
defaultConfigRoot = cleanEnvPath(env.XDG_CONFIG_HOME) ?? join(homeDir, '.config');
|
|
30
|
+
defaultCacheRoot = cleanEnvPath(env.XDG_CACHE_HOME) ?? join(homeDir, '.cache');
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const configDir = resolve(configOverride ?? join(defaultConfigRoot, 'or-info'));
|
|
34
|
+
const cacheDir = resolve(cacheOverride ?? join(defaultCacheRoot, 'or-info'));
|
|
35
|
+
const envFile = join(configDir, '.env');
|
|
36
|
+
|
|
37
|
+
return {
|
|
38
|
+
configDir,
|
|
39
|
+
cacheDir,
|
|
40
|
+
envFile,
|
|
41
|
+
files: {
|
|
42
|
+
modelsCache: join(cacheDir, 'models.json'),
|
|
43
|
+
benchmarksCache: join(cacheDir, 'benchmarks.json'),
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export const APP_PATHS = resolveAppPaths();
|
|
49
|
+
export const CONFIG_DIR = APP_PATHS.configDir;
|
|
50
|
+
export const CACHE_DIR = APP_PATHS.cacheDir;
|
|
51
|
+
export const ENV_FILE = APP_PATHS.envFile;
|
|
52
|
+
export const MODELS_CACHE = APP_PATHS.files.modelsCache;
|
|
53
|
+
export const BENCHMARKS_CACHE = APP_PATHS.files.benchmarksCache;
|
package/lib/scorer.mjs
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { pricePerMillion, supportsFeature } from './openrouter.mjs';
|
|
2
|
+
|
|
3
|
+
// ELO range observed on LMArena (2026): ~1050 (weak) to ~1500 (best)
|
|
4
|
+
const ELO_MIN = 1050;
|
|
5
|
+
const ELO_MAX = 1500;
|
|
6
|
+
|
|
7
|
+
function normaliseElo(elo) {
|
|
8
|
+
return Math.max(0, Math.min(100, ((elo - ELO_MIN) / (ELO_MAX - ELO_MIN)) * 100));
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
// Returns a penalty factor [0, 1] based on output price.
|
|
12
|
+
// Free models → 1.0 (no penalty)
|
|
13
|
+
// Very cheap (<$0.5/M) → near 1.0
|
|
14
|
+
// Expensive (>$20/M) → significantly penalised
|
|
15
|
+
function pricePenalty(outputPerM) {
|
|
16
|
+
if (outputPerM === null || outputPerM === 0) return 1.0;
|
|
17
|
+
// log-scale penalty: $1/M → 0.93, $5/M → 0.83, $20/M → 0.72
|
|
18
|
+
return Math.max(0.1, 1 - Math.log10(outputPerM + 1) * 0.15);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function requiresCapability(task) {
|
|
22
|
+
if (task === 'vision') return 'vision';
|
|
23
|
+
if (task === 'coding') return 'tools';
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Score a model for a task.
|
|
28
|
+
// Returns { score, qualityScore } or null if not eligible.
|
|
29
|
+
export function scoreForTask(model, eloEntry, task = 'general') {
|
|
30
|
+
const cap = requiresCapability(task);
|
|
31
|
+
if (cap && !supportsFeature(model, cap)) return null;
|
|
32
|
+
if (!eloEntry?.elo) return null;
|
|
33
|
+
|
|
34
|
+
const quality = normaliseElo(eloEntry.elo);
|
|
35
|
+
const price = pricePerMillion(model);
|
|
36
|
+
const penalty = task === 'cheap'
|
|
37
|
+
? pricePenalty(price.output) * 1.4 // aggressively favour cheap
|
|
38
|
+
: pricePenalty(price.output);
|
|
39
|
+
|
|
40
|
+
return {
|
|
41
|
+
score: Math.round(quality * penalty * 10) / 10,
|
|
42
|
+
qualityScore: Math.round(quality * 10) / 10,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function rankModels(models, allElo, { task = 'general', maxPricePerMOutput, limit = 5 } = {}) {
|
|
47
|
+
// Build a fast ELO lookup by OR model ID using the same normalisation
|
|
48
|
+
// as lmarena.mjs. We re-use getElo lazily per model here.
|
|
49
|
+
const scored = [];
|
|
50
|
+
|
|
51
|
+
for (const model of models) {
|
|
52
|
+
// Find this model's ELO entry (allElo is the raw entries array)
|
|
53
|
+
const eloEntry = allElo.find
|
|
54
|
+
? allElo.find((e) => _matchName(e.lmarenaName, model.id))
|
|
55
|
+
: null;
|
|
56
|
+
|
|
57
|
+
const result = scoreForTask(model, eloEntry, task);
|
|
58
|
+
if (!result) continue;
|
|
59
|
+
|
|
60
|
+
const price = pricePerMillion(model);
|
|
61
|
+
if (maxPricePerMOutput !== undefined && price.output !== null && price.output > maxPricePerMOutput) continue;
|
|
62
|
+
|
|
63
|
+
scored.push({ model, score: result.score, qualityScore: result.qualityScore, eloEntry });
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return scored.sort((a, b) => b.score - a.score).slice(0, limit);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Inline minimal name matching (mirrors lmarena.mjs logic without importing it)
|
|
70
|
+
function _norm(s) {
|
|
71
|
+
return s.toLowerCase().replace(/\./g, '-').replace(/-20\d{2}-?\d{2}-?\d{2,}(-.+)?$/, '').replace(/-{2,}/g, '-');
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function _matchName(lmarenaName, orId) {
|
|
75
|
+
const lm = _norm(lmarenaName);
|
|
76
|
+
const slug = _norm(orId.includes('/') ? orId.split('/').slice(1).join('/') : orId);
|
|
77
|
+
if (lm === slug) return true;
|
|
78
|
+
if (slug.length >= 8 && slug.includes(lm)) return true;
|
|
79
|
+
if (lm.length >= 8 && lm.includes(slug)) return true;
|
|
80
|
+
return false;
|
|
81
|
+
}
|
package/lib/secrets.mjs
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import { ENV_FILE } from './paths.mjs';
|
|
3
|
+
|
|
4
|
+
const ENV_LINE_RE = /^\s*([A-Za-z_][A-Za-z0-9_]*)=(.*)$/;
|
|
5
|
+
|
|
6
|
+
function unquote(v) {
|
|
7
|
+
const t = v.trim();
|
|
8
|
+
if ((t.startsWith('"') && t.endsWith('"')) || (t.startsWith("'") && t.endsWith("'"))) {
|
|
9
|
+
return t.slice(1, -1);
|
|
10
|
+
}
|
|
11
|
+
return t;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
async function parseEnvFile(file) {
|
|
15
|
+
try {
|
|
16
|
+
const text = await fs.readFile(file, 'utf8');
|
|
17
|
+
const out = {};
|
|
18
|
+
for (const raw of text.split(/\r?\n/)) {
|
|
19
|
+
const line = raw.trim();
|
|
20
|
+
if (!line || line.startsWith('#')) continue;
|
|
21
|
+
const m = line.match(ENV_LINE_RE);
|
|
22
|
+
if (m) out[m[1]] = unquote(m[2]);
|
|
23
|
+
}
|
|
24
|
+
return out;
|
|
25
|
+
} catch {
|
|
26
|
+
return {};
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function loadEnv() {
|
|
31
|
+
const fileVars = await parseEnvFile(ENV_FILE);
|
|
32
|
+
// Process env takes precedence over file so users can override with shell exports
|
|
33
|
+
return { ...fileVars, ...Object.fromEntries(
|
|
34
|
+
Object.entries(process.env).filter(([k]) => k.startsWith('OPENROUTER_'))
|
|
35
|
+
)};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export async function getApiKey() {
|
|
39
|
+
const env = await loadEnv();
|
|
40
|
+
return env.OPENROUTER_API_KEY || null;
|
|
41
|
+
}
|