@aggc/or-info 0.2.9 → 0.2.11

package/bin/or-info.mjs

@@ -108,7 +108,7 @@ program
 
     if (opts.json) {
       console.log(JSON.stringify({
-        model: model.id,
+        id: model.id,
         pricing: model.pricing,
         context_length: contextLength(model),
       }, null, 2));
@@ -158,6 +158,10 @@ program
     if (!mA) die(`Model not found: ${idA}`);
     if (!mB) die(`Model not found: ${idB}`);
 
+    if (mA.id === mB.id) {
+      console.log(chalk.yellow('Both model IDs resolve to the same model:') + ` ${mA.id}`);
+    }
+
     if (opts.json) {
       console.log(JSON.stringify({ a: { model: mA, elo: eloA }, b: { model: mB, elo: eloB } }, null, 2));
       return;
@@ -213,7 +217,8 @@ program
 
     process.stdout.write(chalk.dim('Refreshing LMArena ELO…'));
     const elo = await loadLeaderboard({ force: true });
-    console.log(chalk.green(` ✓  ${elo.length} entries`));
+    const eloCount = (elo.overall ?? []).length;
+    console.log(chalk.green(` ✓  ${eloCount} entries`));
   });
 
 // ── status ─────────────────────────────────────────────────────────────────

package/lib/lmarena.mjs

@@ -52,36 +52,52 @@ async function fetchPage(offset) {
   throw new Error('LMArena request failed: exhausted retries');
 }
 
-// Fetch all rows where category === 'overall'.
-// The dataset is sorted so 'overall' rows appear first; we stop
-// as soon as we see a different category.
-async function fetchAllOverall() {
-  const entries = [];
+// Categories we actually use. LMArena has ~25 but we only need 3.
+// This cuts pages from ~89 to ~12 and avoids HuggingFace 429s.
+const WANTED_CATEGORIES = new Set(['overall', 'coding', 'math']);
+
+// Fetch only the wanted category rows from the dataset.
+// The dataset is sorted by category so once we've moved past all
+// wanted categories we stop early.
+// Returns { overall: [...], coding: [...], math: [...] }
+async function fetchAllByCategory() {
+  const byCategory = {};
   let offset = 0;
+  let lastCat = null;
+  let passedAllWanted = false;
 
-  while (true) {
+  while (!passedAllWanted) {
     const page = await fetchPage(offset);
     const rows = page.rows ?? [];
     if (!rows.length) break;
 
-    let sawOther = false;
     for (const { row } of rows) {
-      if (row.category !== 'overall') { sawOther = true; break; }
-      entries.push({
-        lmarenaName: row.model_name,
-        elo: Math.round(row.rating),
-        eloLower: Math.round(row.rating_lower),
-        eloUpper: Math.round(row.rating_upper),
-        votes: Math.round(row.vote_count),
-        rank: Math.round(row.rank),
-        updatedAt: row.leaderboard_publish_date,
-      });
+      const cat = row.category;
+      lastCat = cat;
+      if (WANTED_CATEGORIES.has(cat)) {
+        if (!byCategory[cat]) byCategory[cat] = [];
+        byCategory[cat].push({
+          lmarenaName: row.model_name,
+          elo: Math.round(row.rating),
+          eloLower: Math.round(row.rating_lower),
+          eloUpper: Math.round(row.rating_upper),
+          votes: Math.round(row.vote_count),
+          rank: Math.round(row.rank),
+          updatedAt: row.leaderboard_publish_date,
+        });
+      }
+    }
+
+    // Stop early: all wanted categories found AND current page moved past them
+    const found = Object.keys(byCategory);
+    if (found.length === WANTED_CATEGORIES.size && !WANTED_CATEGORIES.has(lastCat)) {
+      passedAllWanted = true;
     }
-    if (sawOther) break;
+
     offset += PAGE;
     if (offset >= (page.num_rows_total ?? Infinity)) break;
   }
-  return entries;
+  return byCategory;
 }
 
 // ── Name normalisation ─────────────────────────────────────────────────────
@@ -109,8 +125,6 @@ function orSlug(id) {
 }
 
 // Build a lookup Map from normalised LMArena name → entry.
-// Also index by the last "segment" after the last '-' number group
-// to help with partial matches.
 function buildIndex(entries) {
   const exact = new Map();
   for (const e of entries) {
@@ -119,6 +133,15 @@ function buildIndex(entries) {
   return exact;
 }
 
+// Build a per-category index: { overall: Map<name, entry>, coding: Map<name, entry>, ... }
+function buildCategoryIndex(byCategory) {
+  const result = {};
+  for (const [cat, entries] of Object.entries(byCategory)) {
+    result[cat] = buildIndex(entries);
+  }
+  return result;
+}
+
 // Find the best matching LMArena entry for an OpenRouter model ID.
 // Returns the entry or null.
 function match(orId, index) {
@@ -147,29 +170,34 @@ function match(orId, index) {
 
 // ── Public API ─────────────────────────────────────────────────────────────
 
-let _index = null;
+let _categoryIndex = null; // { overall: Map<name, entry>, coding: Map<name, entry>, ... }
+let _byCategory = null;    // { overall: entry[], coding: entry[], ... }
 
 export async function loadLeaderboard({ force = false } = {}) {
   if (!force) {
     const cached = await get(BENCHMARKS_CACHE, TTL.BENCHMARKS);
-    if (cached?.entries) {
-      _index = buildIndex(cached.entries);
-      return cached.entries;
+    if (cached?.byCategory) {
+      _byCategory = cached.byCategory;
+      _categoryIndex = buildCategoryIndex(cached.byCategory);
+      return cached.byCategory;
     }
   }
 
-  const entries = await fetchAllOverall();
-  await set(BENCHMARKS_CACHE, { entries, fetchedAt: Date.now() });
-  _index = buildIndex(entries);
-  return entries;
+  const byCategory = await fetchAllByCategory();
+  await set(BENCHMARKS_CACHE, { byCategory, fetchedAt: Date.now() });
+  _byCategory = byCategory;
+  _categoryIndex = buildCategoryIndex(byCategory);
+  return byCategory;
 }
 
-export async function getElo(orModelId, { force = false } = {}) {
-  if (!_index || force) await loadLeaderboard({ force });
-  return match(orModelId, _index);
+export async function getElo(orModelId, { force = false, category = 'overall' } = {}) {
+  if (!_categoryIndex || force) await loadLeaderboard({ force });
+  const idx = _categoryIndex[category] ?? _categoryIndex.overall;
+  if (!idx) return null;
+  return match(orModelId, idx);
 }
 
 export async function getAllElo({ force = false } = {}) {
-  const entries = await loadLeaderboard({ force });
-  return entries; // [{lmarenaName, elo, eloLower, eloUpper, votes, rank, updatedAt}]
+  const byCategory = await loadLeaderboard({ force });
+  return byCategory; // { overall: entry[], coding: entry[], ... }
 }

package/lib/openrouter.mjs

@@ -100,7 +100,17 @@ export function supportsFeature(model, feature) {
   const featureMap = {
     reasoning: ['include_reasoning', 'reasoning'],
     tools: ['tools', 'tool_choice'],
-    vision: () => (model?.architecture?.input_modalities ?? []).includes('image'),
+    vision: () => {
+      // Prefer the canonical modality string (e.g. "text+image->text") because
+      // input_modalities is inconsistently populated by OpenRouter providers.
+      const modality = model?.architecture?.modality ?? '';
+      if (modality) {
+        return modality.split('->')[0].split('+').map((s) => s.trim()).includes('image');
+      }
+      const inputMods = model?.architecture?.input_modalities;
+      if (Array.isArray(inputMods)) return inputMods.includes('image');
+      return false;
+    },
     structured: ['structured_outputs'],
   };
   const check = featureMap[feature];

package/lib/scorer.mjs

@@ -43,15 +43,21 @@ export function scoreForTask(model, eloEntry, task = 'general') {
   };
 }
 
+const CATEGORY_FOR_TASK = { coding: 'coding', reasoning: 'math', vision: null };
+
 export function rankModels(models, allElo, { task = 'general', maxPricePerMOutput, limit = 5 } = {}) {
-  // Build a fast ELO lookup by OR model ID using the same normalisation
-  // as lmarena.mjs.  We re-use getElo lazily per model here.
+  // allElo may be a by-category map { overall: [...], coding: [...], ... }
+  // or a plain array (legacy). Select the right category for this task.
+  const category = CATEGORY_FOR_TASK[task] ?? 'overall';
+  const entries = Array.isArray(allElo)
+    ? allElo
+    : (allElo[category] ?? allElo.overall ?? []);
+
   const scored = [];
 
   for (const model of models) {
-    // Find this model's ELO entry (allElo is the raw entries array)
-    const eloEntry = allElo.find
-      ? allElo.find((e) => _matchName(e.lmarenaName, model.id))
+    const eloEntry = entries.find
+      ? entries.find((e) => _matchName(e.lmarenaName, model.id))
       : null;
 
     const result = scoreForTask(model, eloEntry, task);

package/mcp/server.mjs

@@ -320,7 +320,7 @@ async function handleTool(name, args) {
       fetchModels({ force: true, apiKey: key }),
       loadLeaderboard({ force: true }),
     ]);
-    return result({ refreshed: true, models_count: models.length, elo_entries: elo.length });
+    return result({ refreshed: true, models_count: models.length, elo_entries: (elo.overall ?? []).length });
   }
 
   return errorContent(`Unknown tool: ${name}`);
@@ -347,20 +347,45 @@ function wireHandlers(server) {
 }
 
 export async function startMcp() {
+  // Track in-flight tool calls so we don't exit while a response is still being written.
+  // Race condition: stdin EOF fires before the async handleTool completes, causing
+  // process.exit(0) to kill the process before the MCP SDK writes the response to stdout.
+  let pending = 0;
+  let stdinEnded = false;
+  let resolveWhenDone;
+  const donePromise = new Promise((res) => { resolveWhenDone = res; });
+
+  function checkDone() {
+    if (stdinEnded && pending === 0) resolveWhenDone();
+  }
+
   const server = makeServer();
-  wireHandlers(server);
+  server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: TOOLS }));
+  server.setRequestHandler(CallToolRequestSchema, async (req) => {
+    const { name, arguments: args } = req.params;
+    pending++;
+    try {
+      return await handleTool(name, args ?? {});
+    } catch (err) {
+      const safe = err.message?.replace(/sk-[a-zA-Z0-9-]+/g, '[REDACTED]') ?? 'Unexpected error';
+      return errorContent(safe);
+    } finally {
+      pending--;
+      // Defer checkDone by one tick so the SDK's response-write microtask runs first.
+      setImmediate(checkDone);
+    }
+  });
 
   const transport = new StdioServerTransport();
   await server.connect(transport);
 
-  // server.connect() returns immediately after wiring up the transport.
-  // Block here until stdin closes so the process stays alive while serving.
   if (!process.stdin.destroyed) {
-    await new Promise((resolve) => {
-      process.stdin.once('close', resolve);
-      process.stdin.once('end', resolve);
-    });
+    process.stdin.once('close', () => { stdinEnded = true; checkDone(); });
+    process.stdin.once('end', () => { stdinEnded = true; checkDone(); });
+    await donePromise;
   }
+  // One extra tick for any buffered stdout writes before the caller calls process.exit().
+  await new Promise((resolve) => setImmediate(resolve));
 }
 
 export async function startHttpMcp() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aggc/or-info",
-  "version": "0.2.9",
+  "version": "0.2.11",
   "description": "CLI + MCP server for OpenRouter models: prices, benchmarks, context and comparisons",
   "type": "module",
   "engines": {