@lucasschirm/devinp 0.1.0 → 0.1.2

package/README.md

@@ -1,7 +1,7 @@
 # devinp
 
 Pretty pricing table for [Devin](https://devin.ai) models — reads live prices
-from `devin models list`, joins SWE-bench Verified scores, and renders a
+from `devin models list`, joins BenchLM coding scores, and renders a
 sorted table in your terminal or a web page.
 
 Thinking-effort variants that share a price (`low`, `medium`, `high`, `xhigh`,
@@ -27,12 +27,12 @@ The package ships as a self-contained bundle — no dependencies are installed.
 ## Usage
 
 ```sh
-devinp                       # table sorted by best value (SWE % ÷ output price)
+devinp                       # table sorted by best value (Bench % ÷ output price)
 devinp -o=input              # cheapest input price first
 devinp -o=cached             # cheapest cached-input price first
 devinp -o=output             # cheapest output price first
-devinp -o=swe                # highest SWE-bench Verified score first
-devinp -o=value              # default: best SWE score per output dollar
+devinp -o=bench              # highest BenchLM coding score first
+devinp -o=value              # default: best Bench score per output dollar
 
 devinp -g=family             # section the table by model family
 devinp -g=price              # section by identical price tier
@@ -48,33 +48,35 @@ devinp --json                # emit normalized rows as JSON
 ### Reading the table
 
 ```
-│  # │ Model           │   Ctx │ Input │ Cached │ Output │  SWE % │ Value │ Sidekick i/c/o │
-│  1 │ Gemini 3 Flash  │ 1.05M │  $0.5 │  $0.05 │     $3 │  75.8% │ 25.27 │ —              │
+│  # │ Model           │   Ctx │ Input │ Cached │ Output │ Bench % │ Value │ Sidekick i/c/o │
+│  1 │ Gemini 3.8 Flash│ 1.05M │ $0.75 │  $0.08 │  $3.75 │  64.1%  │ 17.10 │ —              │
 ```
 
 - Prices are USD per 1M tokens; columns are heat-colored green → red.
-- **Value** = `SWE % ÷ output $/1M` — the score you buy per output dollar.
-  Free models show `∞`; models without a SWE score show `—` and sort last
-  under `-o=swe` / `-o=value`.
+- **Value** = `Bench % ÷ output $/1M` — the score you buy per output dollar.
+  Free models show `∞`; models without a Bench score show `—` and sort last
+  under `-o=bench` / `-o=value`.
 - Markers: `β` = beta, `●` = new, magenta name = free. `Sidekick` shows the
   partner-model pricing for Fusion rows.
 - Colors disable automatically when piping; `NO_COLOR` / `FORCE_COLOR` are
   respected.
 
-## SWE-bench scores
+## BenchLM scores
 
-Scores live in `data/swe-scores.json` and ship with the package. To refresh
+Scores live in `data/benchlm-scores.json` and ship with the package. To refresh
 them (maintainers, manual — not automated):
 
 ```sh
-npm run update-swe
+npm run update-benchlm
 ```
 
-This scrapes the embedded leaderboard JSON on
-[swebench.com](https://www.swebench.com) — the **Verified** board's
-`mini-SWE-agent` ("Bash Only") entries — and writes `data/swe-scores.json`.
-Leaderboard names that don't map to a family slug are reported under
-`unmatched`; add entries to `MANUAL_MAP` in `scripts/update-swe.js` to fix them.
+This fetches the [BenchLM](https://benchlm.ai/embed) leaderboard API
+(`api/data/leaderboard?category=coding&limit=200` — the limit is raised from
+the default 50 so the long tail of the board is included) and writes
+`data/benchlm-scores.json`. Model names are
+matched to Devin family slugs order-insensitively; names that still don't map
+are reported under `unmatched` — add entries to `MANUAL_MAP` in
+`scripts/update-benchlm.js` to fix them.
 
 ## Development
 

package/bin/command.js

@@ -54630,10 +54630,10 @@ function formatContext(tokens) {
 function priceKey(cost) {
   return [cost.input, cost.cached, cost.output, cost.sidekick?.input, cost.sidekick?.cached, cost.sidekick?.output].map((n) => n == null ? "-" : n).join("/");
 }
-function buildRows(data, sweScores = {}) {
+function buildRows(data, scores = {}) {
   const rows = /* @__PURE__ */ new Map();
   for (const family of data.families ?? []) {
-    const swe = sweScores.models?.[family.slug] ?? null;
+    const bench = scores.models?.[family.slug] ?? null;
     for (const variant of family.variants ?? []) {
       const cost = parseCostSummary(variant.cost_summary);
       const label = normalizeLabel(variant.label, family.family_label);
@@ -54658,26 +54658,26 @@ function buildRows(data, sweScores = {}) {
         isBeta: Boolean(variant.is_beta),
         isNew: Boolean(variant.is_new),
         variantCount: 1,
-        swe: swe ? { resolved: swe.resolved, sweName: swe.sweName } : null,
-        value: computeValue(swe?.resolved, cost.output)
+        bench: bench ? { score: bench.score, benchName: bench.benchName } : null,
+        value: computeValue(bench?.score, cost.output)
       });
     }
   }
   return [...rows.values()];
 }
-function computeValue(resolved, output) {
-  if (resolved == null) return null;
+function computeValue(score, output) {
+  if (score == null) return null;
   if (output == null) return null;
   if (output === 0) return Number.POSITIVE_INFINITY;
-  return resolved / output;
+  return score / output;
 }
 var priceAsc = (field) => (a2, b) => (a2[field] ?? Number.POSITIVE_INFINITY) - (b[field] ?? Number.POSITIVE_INFINITY) || (a2.output ?? Number.POSITIVE_INFINITY) - (b.output ?? Number.POSITIVE_INFINITY) || a2.label.localeCompare(b.label);
 var SORTS = {
   input: priceAsc("input"),
   cached: priceAsc("cached"),
   output: priceAsc("output"),
-  swe: (a2, b) => (b.swe?.resolved ?? -1) - (a2.swe?.resolved ?? -1) || (a2.output ?? Number.POSITIVE_INFINITY) - (b.output ?? Number.POSITIVE_INFINITY) || a2.label.localeCompare(b.label),
-  value: (a2, b) => (b.value ?? -1) - (a2.value ?? -1) || (b.swe?.resolved ?? -1) - (a2.swe?.resolved ?? -1) || a2.label.localeCompare(b.label)
+  bench: (a2, b) => (b.bench?.score ?? -1) - (a2.bench?.score ?? -1) || (a2.output ?? Number.POSITIVE_INFINITY) - (b.output ?? Number.POSITIVE_INFINITY) || a2.label.localeCompare(b.label),
+  value: (a2, b) => (b.value ?? -1) - (a2.value ?? -1) || (b.bench?.score ?? -1) - (a2.bench?.score ?? -1) || a2.label.localeCompare(b.label)
 };
 var SORT_IDS = Object.keys(SORTS);
 var GROUPS = {
@@ -54745,10 +54745,10 @@ function formatValue(c, row) {
   if (!Number.isFinite(row.value)) return c.magenta("\u221E");
   return c.bold.green(row.value.toFixed(2));
 }
-function formatSwe(c, row) {
-  if (row.swe == null) return c.dim("\u2014");
-  const text = `${row.swe.resolved}%`;
-  return row.swe.resolved >= 70 ? c.cyan.bold(text) : c.cyan(text);
+function formatBench(c, row) {
+  if (row.bench == null) return c.dim("\u2014");
+  const text = `${row.bench.score}%`;
+  return row.bench.score >= 70 ? c.cyan.bold(text) : c.cyan(text);
 }
 function formatLabel(c, row) {
   let text = row.free ? c.magenta(row.label) : c.bold(row.label);
@@ -54769,7 +54769,7 @@ function colorsEnabled(stream = process.stdout) {
   if (process.env.NO_COLOR) return false;
   return Boolean(stream.isTTY);
 }
-function renderTable(sections, { sweMeta, colors = colorsEnabled() } = {}) {
+function renderTable(sections, { benchMeta, colors = colorsEnabled() } = {}) {
   const c = colors ? ansis_default : plain;
   const allRows = sections.flatMap((s) => s.rows);
   const hasSidekick = allRows.some((r) => r.sidekick);
@@ -54785,7 +54785,7 @@ function renderTable(sections, { sweMeta, colors = colorsEnabled() } = {}) {
     { title: "Input", align: "right" },
     { title: "Cached", align: "right" },
     { title: "Output", align: "right" },
-    { title: "SWE %", align: "right" },
+    { title: "Bench %", align: "right" },
     { title: "Value", align: "right" }
   ];
   if (hasSidekick) columns.push({ title: "Sidekick i/c/o", align: "left" });
@@ -54803,7 +54803,7 @@ function renderTable(sections, { sweMeta, colors = colorsEnabled() } = {}) {
           formatPrice(c, row.input, ...ranges.input),
           formatPrice(c, row.cached, ...ranges.cached),
           formatPrice(c, row.output, ...ranges.output),
-          formatSwe(c, row),
+          formatBench(c, row),
           formatValue(c, row),
           ...hasSidekick ? [formatSidekick(c, row)] : []
         ]
@@ -54830,13 +54830,29 @@ function renderTable(sections, { sweMeta, colors = colorsEnabled() } = {}) {
   out.push(line("\u2570", "\u2534", "\u256F"));
   const legend = [
     c.dim("prices per 1M tokens"),
-    c.dim("value = SWE % \xF7 output $/1M"),
-    sweMeta?.updated ? c.dim(`SWE-bench Verified (mini-SWE-agent) \xB7 updated ${sweMeta.updated}`) : c.dim("SWE scores: not available")
+    c.dim("value = Bench % \xF7 output $/1M"),
+    benchMeta?.updated ? c.dim(`BenchLM (coding) \xB7 updated ${benchMeta.updated}`) : c.dim("Bench scores: not available")
   ];
   return `${out.join("\n")}
 ${legend.join(c.dim(" \xB7 "))}`;
 }
 
+// src/lib/scores.js
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+async function loadScores(root) {
+  const path = join(root, "data", "benchlm-scores.json");
+  try {
+    const parsed = JSON.parse(await readFile(path, "utf8"));
+    return {
+      models: parsed.models ?? {},
+      meta: { updated: parsed.updated, source: parsed.source, unmatched: parsed.unmatched ?? [] }
+    };
+  } catch {
+    return { models: {}, meta: {} };
+  }
+}
+
 // src/lib/server.js
 var import_fastify = __toESM(require_fastify(), 1);
 import { execFile as execFile3 } from "node:child_process";
@@ -54871,7 +54887,7 @@ var PAGE = `<!doctype html>
   td.name, th.name { text-align: left; font-weight: 600; }
   td.dim { color: #64748b; }
   td.free { color: #e879f9; }
-  td.swe { color: #22d3ee; font-weight: 600; }
+  td.bench { color: #22d3ee; font-weight: 600; }
   td.value { color: #4ade80; font-weight: 700; }
   tr.section td {
     color: #7dd3fc; font-weight: 700; text-align: left;
@@ -54882,12 +54898,12 @@ var PAGE = `<!doctype html>
 </head>
 <body>
   <h1>devinp</h1>
-  <div class="sub">Devin model prices \xB7 per 1M tokens \xB7 value = SWE % \xF7 output $/1M</div>
+  <div class="sub">Devin model prices \xB7 per 1M tokens \xB7 value = Bench % \xF7 output $/1M</div>
   <div class="controls">
     <label>sort
       <select id="order">
-        <option value="value">best value (SWE \xF7 output)</option>
-        <option value="swe">SWE-bench score</option>
+        <option value="value">best value (Bench \xF7 output)</option>
+        <option value="bench">BenchLM score</option>
         <option value="input">input price</option>
         <option value="cached">cached price</option>
         <option value="output">output price</option>
@@ -54906,7 +54922,7 @@ var PAGE = `<!doctype html>
   <table>
     <thead><tr>
       <th>#</th><th class="name">Model</th><th>Ctx</th><th>Input</th>
-      <th>Cached</th><th>Output</th><th>SWE %</th><th>Value</th><th id="sk"></th>
+      <th>Cached</th><th>Output</th><th>Bench %</th><th>Value</th><th id="sk"></th>
     </tr></thead>
     <tbody id="body"></tbody>
   </table>
@@ -54920,7 +54936,7 @@ const heat = (v, lo, hi) => v == null ? '<span class="dim">\u2014</span>'
 
 async function main() {
   const res = await fetch('/api/models');
-  const {rows, swe} = await res.json();
+  const {rows, bench} = await res.json();
   const range = (f) => { const vs = rows.map(r => r[f]).filter(v => v != null); return [Math.min(...vs), Math.max(...vs)]; };
   const RI = range('input'), RC = range('cached'), RO = range('output');
   const hasSk = rows.some(r => r.sidekick);
@@ -54930,8 +54946,8 @@ async function main() {
     input: (a, b) => (a.input ?? 1e9) - (b.input ?? 1e9) || (a.output ?? 1e9) - (b.output ?? 1e9) || a.label.localeCompare(b.label),
     cached: (a, b) => (a.cached ?? 1e9) - (b.cached ?? 1e9) || (a.output ?? 1e9) - (b.output ?? 1e9) || a.label.localeCompare(b.label),
     output: (a, b) => (a.output ?? 1e9) - (b.output ?? 1e9) || a.label.localeCompare(b.label),
-    swe: (a, b) => (b.swe?.resolved ?? -1) - (a.swe?.resolved ?? -1) || (a.output ?? 1e9) - (b.output ?? 1e9) || a.label.localeCompare(b.label),
-    value: (a, b) => (b.value ?? -1) - (a.value ?? -1) || (b.swe?.resolved ?? -1) - (a.swe?.resolved ?? -1) || a.label.localeCompare(b.label),
+    bench: (a, b) => (b.bench?.score ?? -1) - (a.bench?.score ?? -1) || (a.output ?? 1e9) - (b.output ?? 1e9) || a.label.localeCompare(b.label),
+    value: (a, b) => (b.value ?? -1) - (a.value ?? -1) || (b.bench?.score ?? -1) - (a.bench?.score ?? -1) || a.label.localeCompare(b.label),
   };
   const GROUPS = {
     family: {key: r => r.familySlug, head: r => r.familyLabel},
@@ -54967,7 +54983,7 @@ async function main() {
           + '<td>' + heat(r.input, ...RI) + '</td>'
           + '<td>' + heat(r.cached, ...RC) + '</td>'
           + '<td>' + heat(r.output, ...RO) + '</td>'
-          + '<td class="swe">' + (r.swe ? r.swe.resolved + '%' : '<span class="dim">\u2014</span>') + '</td>'
+          + '<td class="bench">' + (r.bench ? r.bench.score + '%' : '<span class="dim">\u2014</span>') + '</td>'
           + '<td class="value">' + (r.value == null ? '<span class="dim">\u2014</span>' : !isFinite(r.value) ? '\u221E' : r.value.toFixed(2)) + '</td>'
           + (hasSk ? '<td class="dim">' + (r.sidekick ? '$' + r.sidekick.input + '/$' + r.sidekick.cached + '/$' + r.sidekick.output : '\u2014') + '</td>' : '')
           + '</tr>';
@@ -54975,9 +54991,9 @@ async function main() {
     }
     document.getElementById('body').innerHTML = html;
     document.getElementById('count').textContent = n + ' rows';
-    document.getElementById('legend').textContent = swe.updated
-      ? 'SWE-bench Verified (mini-SWE-agent) \xB7 scores updated ' + swe.updated
-      : 'SWE scores not available';
+    document.getElementById('legend').textContent = bench.updated
+      ? 'BenchLM (coding) \xB7 scores updated ' + bench.updated
+      : 'Bench scores not available';
   }
   document.getElementById('order').onchange = render;
   document.getElementById('group').onchange = render;
@@ -54987,12 +55003,12 @@ main();
 </script>
 </body>
 </html>`;
-async function startServer({ rows, sweMeta, port, log }) {
+async function startServer({ rows, benchMeta, port, log }) {
   const app = (0, import_fastify.default)({ logger: false });
   app.get("/", async (_request, reply) => {
     reply.type("text/html").send(PAGE);
   });
-  app.get("/api/models", async () => ({ rows, swe: sweMeta ?? {} }));
+  app.get("/api/models", async () => ({ rows, bench: benchMeta ?? {} }));
   const address = await app.listen({ port, host: "127.0.0.1" });
   log(`devinp web \u2014 ${address}`);
   const opener = process.platform === "darwin" ? "open" : "xdg-open";
@@ -55008,14 +55024,14 @@ async function startServer({ rows, sweMeta, port, log }) {
 
 // src/lib/source.js
 import { execFile as execFile4 } from "node:child_process";
-import { readFile } from "node:fs/promises";
+import { readFile as readFile2 } from "node:fs/promises";
 import { promisify as promisify3 } from "node:util";
 var execFileAsync = promisify3(execFile4);
 async function loadModelData({ source } = {}) {
   if (source) {
     let raw;
     try {
-      raw = await readFile(source, "utf8");
+      raw = await readFile2(source, "utf8");
     } catch (error) {
       throw new Error(`Unable to get model prices: cannot read source file "${source}" (${error.code ?? error.message})`);
     }
@@ -55045,35 +55061,19 @@ async function loadModelData({ source } = {}) {
   }
 }
 
-// src/lib/swe.js
-import { readFile as readFile2 } from "node:fs/promises";
-import { join } from "node:path";
-async function loadSweScores(root) {
-  const path = join(root, "data", "swe-scores.json");
-  try {
-    const parsed = JSON.parse(await readFile2(path, "utf8"));
-    return {
-      models: parsed.models ?? {},
-      meta: { updated: parsed.updated, source: parsed.source, unmatched: parsed.unmatched ?? [] }
-    };
-  } catch {
-    return { models: {}, meta: {} };
-  }
-}
-
 // src/command.js
 var Devinp = class _Devinp extends import_core.Command {
   static summary = "Show Devin model prices in a pretty table";
   static description = [
     "Reads prices from `devin models list --format json` (or a saved snapshot via",
-    "--source), joins SWE-bench Verified scores from data/swe-scores.json, and",
+    "--source), joins BenchLM coding scores from data/benchlm-scores.json, and",
     "renders a sorted pricing table. Thinking-effort variants that share a price",
     "are collapsed into one row."
   ].join("\n");
   static examples = [
     "<%= config.bin %>",
     "<%= config.bin %> -o=input",
-    "<%= config.bin %> -o=swe -g=family",
+    "<%= config.bin %> -o=bench -g=family",
     "<%= config.bin %> --web --port 8080",
     "<%= config.bin %> --source sample/models.json"
   ];
@@ -55082,7 +55082,7 @@ var Devinp = class _Devinp extends import_core.Command {
       char: "o",
       options: SORT_IDS,
       default: "value",
-      summary: "sort by input | cached | output price, swe score, or value (SWE \xF7 output)"
+      summary: "sort by input | cached | output price, bench score, or value (Bench \xF7 output)"
     }),
     group: import_core.Flags.string({
       char: "g",
@@ -55115,18 +55115,18 @@ var Devinp = class _Devinp extends import_core.Command {
     } catch (error) {
       this.error(error instanceof Error ? error.message : String(error), { exit: 1 });
     }
-    const swe = await loadSweScores(this.config.root);
-    const rows = buildRows(data, swe);
+    const scores = await loadScores(this.config.root);
+    const rows = buildRows(data, scores);
     if (flags.json) {
       this.log(JSON.stringify(rows, null, 2));
       return;
     }
     if (flags.web) {
-      await startServer({ rows, sweMeta: swe.meta, port: flags.port, log: (m) => this.log(m) });
+      await startServer({ rows, benchMeta: scores.meta, port: flags.port, log: (m) => this.log(m) });
       return;
     }
     const sections = arrangeRows(rows, { order: flags.order, group: flags.group });
-    this.log(renderTable(sections, { sweMeta: swe.meta }));
+    this.log(renderTable(sections, { benchMeta: scores.meta }));
   }
 };
 export {

package/data/benchlm-scores.json

@@ -0,0 +1,293 @@
+{
+  "updated": "2026-09-22",
+  "source": "https://benchlm.ai/api/data/leaderboard?category=coding&limit=200",
+  "models": {
+    "claude-fable-5": {
+      "score": 74.47,
+      "benchName": "Claude Fable 5",
+      "supported": true
+    },
+    "claude-fable-5.1": {
+      "score": 81.29,
+      "benchName": "Claude Fable 5.1",
+      "supported": true
+    },
+    "claude-haiku-4.5": {
+      "score": 19.77,
+      "benchName": "Claude Haiku 4.5",
+      "supported": true
+    },
+    "claude-opus-4.5": {
+      "score": 46.1,
+      "benchName": "Claude Opus 4.5",
+      "supported": false
+    },
+    "claude-opus-4.6": {
+      "score": 51.32,
+      "benchName": "Claude Opus 4.6",
+      "supported": true
+    },
+    "claude-opus-4.7": {
+      "score": 59.77,
+      "benchName": "Claude Opus 4.7",
+      "supported": true
+    },
+    "claude-opus-4.8": {
+      "score": 63.58,
+      "benchName": "Claude Opus 4.8",
+      "supported": true
+    },
+    "claude-opus-5": {
+      "score": 72.91,
+      "benchName": "Claude Opus 5",
+      "supported": true
+    },
+    "claude-opus-5.5": {
+      "score": 86.85,
+      "benchName": "Claude Opus 5.5",
+      "supported": true
+    },
+    "claude-sonnet-4.6": {
+      "score": 48.52,
+      "benchName": "Claude Sonnet 4.6",
+      "supported": true
+    },
+    "claude-sonnet-5": {
+      "score": 61.05,
+      "benchName": "Claude Sonnet 5",
+      "supported": true
+    },
+    "gemini-3-flash": {
+      "score": 36.84,
+      "benchName": "Gemini 3 Flash",
+      "supported": true
+    },
+    "gemini-3.1-pro": {
+      "score": 43.36,
+      "benchName": "Gemini 3.1 Pro",
+      "supported": true
+    },
+    "gemini-3.5-flash": {
+      "score": 53.7,
+      "benchName": "Gemini 3.5 Flash",
+      "supported": true
+    },
+    "gemini-3.6-flash": {
+      "score": 54.73,
+      "benchName": "Gemini 3.6 Flash",
+      "supported": true
+    },
+    "gemini-3.7-flash": {
+      "score": 59.79,
+      "benchName": "Gemini 3.7 Flash",
+      "supported": true
+    },
+    "gemini-3.8-flash": {
+      "score": 64.14,
+      "benchName": "Gemini 3.8 Flash",
+      "supported": true
+    },
+    "glm-5.2": {
+      "score": 57.32,
+      "benchName": "GLM-5.2",
+      "supported": true
+    },
+    "glm-5.3": {
+      "score": 58.15,
+      "benchName": "GLM-5.3",
+      "supported": true
+    },
+    "glm-5.3-flash": {
+      "score": 51.6,
+      "benchName": "GLM-5.3-Flash",
+      "supported": true
+    },
+    "gpt-5.1": {
+      "score": 39.77,
+      "benchName": "GPT-5.1",
+      "supported": true
+    },
+    "gpt-5.2": {
+      "score": 41.57,
+      "benchName": "GPT-5.2",
+      "supported": true
+    },
+    "gpt-5.3-codex": {
+      "score": 56.8,
+      "benchName": "GPT-5.3 Codex",
+      "supported": true
+    },
+    "gpt-5.4": {
+      "score": 50.93,
+      "benchName": "GPT-5.4",
+      "supported": true
+    },
+    "gpt-5.4-mini": {
+      "score": 38.61,
+      "benchName": "GPT-5.4 mini",
+      "supported": true
+    },
+    "gpt-5.5": {
+      "score": 64.47,
+      "benchName": "GPT-5.5",
+      "supported": true
+    },
+    "gpt-5.6-luna": {
+      "score": 64.72,
+      "benchName": "GPT-5.6 Luna",
+      "supported": true
+    },
+    "gpt-5.6-sol": {
+      "score": 71.77,
+      "benchName": "GPT-5.6 Sol",
+      "supported": true
+    },
+    "gpt-5.6-terra": {
+      "score": 65.05,
+      "benchName": "GPT-5.6 Terra",
+      "supported": true
+    },
+    "gpt-6-astra": {
+      "score": 74.43,
+      "benchName": "GPT-6 Astra",
+      "supported": true
+    },
+    "gpt-6-luna": {
+      "score": 59.93,
+      "benchName": "GPT-6 Luna",
+      "supported": true
+    },
+    "gpt-6-sol": {
+      "score": 65.48,
+      "benchName": "GPT-6 Sol",
+      "supported": true
+    },
+    "grok-4.5": {
+      "score": 58.55,
+      "benchName": "Grok 4.5",
+      "supported": true
+    },
+    "grok-4.6": {
+      "score": 62.55,
+      "benchName": "Grok 4.6",
+      "supported": true
+    },
+    "inkling": {
+      "score": 37.23,
+      "benchName": "Inkling",
+      "supported": true
+    },
+    "kimi-k2.6": {
+      "score": 47.36,
+      "benchName": "Kimi K2.6",
+      "supported": true
+    },
+    "kimi-k3": {
+      "score": 63.31,
+      "benchName": "Kimi K3",
+      "supported": true
+    },
+    "nemotron-3-ultra": {
+      "score": 23.53,
+      "benchName": "Nemotron 3 Ultra",
+      "supported": true
+    }
+  },
+  "unmatched": [
+    "Apodex 1.1",
+    "Claude 3 Opus",
+    "Command A+",
+    "DeepSeek V3",
+    "DeepSeek V3.2",
+    "DeepSeek V3.2 (Thinking)",
+    "DeepSeek V4 Pro 0813",
+    "GLM-4.6",
+    "GLM-4.7",
+    "GLM-5",
+    "GLM-5.1",
+    "GLM-5V-Turbo",
+    "GPT-4 Turbo",
+    "GPT-4.1 mini",
+    "GPT-4.1 nano",
+    "GPT-4o mini",
+    "GPT-5 (medium)",
+    "GPT-5 mini",
+    "GPT-5.1-Codex",
+    "GPT-5.2-Codex",
+    "GPT-5.4 nano",
+    "GPT-OSS 120B",
+    "GPT-OSS 20B",
+    "Gemini 1.5 Pro",
+    "Gemini 2.5 Pro",
+    "Gemini 3 Pro",
+    "Gemini 3.1 Flash-Lite",
+    "Gemini 3.5 Flash-Lite",
+    "Gemma 3 27B",
+    "Gemma 4 12B",
+    "Gemma 4 26B A4B",
+    "Gemma 4 31B",
+    "Gemma 4 E2B",
+    "Gemma 4 E4B",
+    "Granite 4.2 8B",
+    "Grok 4 Fast (Reasoning)",
+    "Grok 4.1 Fast (Reasoning)",
+    "Grok 4.20",
+    "Grok 4.3",
+    "Grok Build 0.1",
+    "Grok Code Fast 1",
+    "Hy3",
+    "Hy3 Preview",
+    "Hy4 preview",
+    "Inkling-Small",
+    "K-Exaone",
+    "Kimi K2.5",
+    "Kimi K2.7 Code",
+    "LFM2.5-2.6B",
+    "Laguna M.1",
+    "Laguna XS.2",
+    "Ling 2.6 Flash",
+    "Ling 3.0 Flash",
+    "Llama 4 Maverick",
+    "Llama 4 Scout",
+    "Mercury 2",
+    "Mercury 2.5",
+    "MiMo-V2-Flash",
+    "MiMo-V2-Pro",
+    "MiMo-V2.5",
+    "MiMo-V2.5-Pro",
+    "MiniMax M2.5",
+    "MiniMax M2.7",
+    "MiniMax M3",
+    "Ministral 3 14B",
+    "Ministral 3 3B",
+    "Ministral 3 8B",
+    "Mistral Large 3",
+    "Mistral Medium 3.5 128B",
+    "Mistral Small 4",
+    "Muse Glimmer 30B",
+    "Muse Spark",
+    "Muse Spark 1.1",
+    "Muse Spark 1.2",
+    "Nemotron 3 Nano Omni 30B A3B",
+    "Nemotron 3.5 Lightning 30B A3B NVFP4",
+    "Ornith-1.5-35B-A3B",
+    "Ornith-1.5-397B",
+    "Qwen 3.6 Max (preview)",
+    "Qwen3.5 Flash",
+    "Qwen3.5-122B-A10B",
+    "Qwen3.5-27B",
+    "Qwen3.5-35B-A3B",
+    "Qwen3.6 Plus",
+    "Qwen3.6-27B",
+    "Qwen3.6-35B-A3B",
+    "Qwen3.7 Max",
+    "Qwen3.7 Plus",
+    "Qwen3.8 Max",
+    "Qwen3.8-27B",
+    "Qwen3.8-Flash-Next",
+    "Step 3.7 Flash",
+    "Trinity-Large-Thinking",
+    "o1",
+    "o1-preview"
+  ]
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lucasschirm/devinp",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Pretty pricing table for Devin models, powered by `devin models list`",
   "type": "module",
   "bin": {
@@ -23,7 +23,7 @@
     "devin",
     "pricing",
     "models",
-    "swe-bench",
+    "benchlm",
     "cli"
   ],
   "engines": {
@@ -35,7 +35,7 @@
     "prepack": "npm run build",
     "prepare": "npm run build",
     "test": "node --test",
-    "update-swe": "node scripts/update-swe.js"
+    "update-benchlm": "node scripts/update-benchlm.js"
   },
   "oclif": {
     "bin": "devinp",

package/data/swe-scores.json

@@ -1,73 +0,0 @@
-{
-  "updated": "2026-09-21",
-  "source": "https://www.swebench.com/index.html (Verified · mini-SWE-agent)",
-  "models": {
-    "gemini-3-flash": {
-      "resolved": 75.8,
-      "sweName": "Gemini 3 Flash (high)",
-      "checked": false,
-      "date": "2026-02-17"
-    },
-    "gpt-4.1": {
-      "resolved": 39.58,
-      "sweName": "GPT 4.1 (2025-04-14)",
-      "checked": true,
-      "date": "2025-07-26"
-    },
-    "gpt-5.1": {
-      "resolved": 66,
-      "sweName": "GPT 5.1 (2025-11-13) (medium)",
-      "checked": false,
-      "date": "2025-11-20"
-    },
-    "gpt-5.2": {
-      "resolved": 72.8,
-      "sweName": "GPT 5.2 (high)",
-      "checked": false,
-      "date": "2026-02-17"
-    }
-  },
-  "unmatched": [
-    "Claude 3.7 Sonnet (20250219)",
-    "Claude 4 Opus (20250514)",
-    "Claude 4 Sonnet (20250514)",
-    "Claude 4.5 Haiku (high)",
-    "Claude 4.5 Opus (20251101) (medium)",
-    "Claude 4.5 Opus (high)",
-    "Claude 4.5 Sonnet (20250929)",
-    "Claude 4.5 Sonnet (high)",
-    "Claude 4.6 Opus",
-    "DeepSeek V3.2 (high)",
-    "DeepSeek V3.2 Reasoner",
-    "Devstral (2512)",
-    "Devstral Small (2512)",
-    "GLM 4.5 (2025-08-22)",
-    "GLM 4.6 (T=1)",
-    "GLM 5 (high)",
-    "GPT 4.1 mini (2025-04-14)",
-    "GPT 4o (2024-11-20)",
-    "GPT 5 (2025-08-07) (medium)",
-    "GPT 5 mini",
-    "GPT 5 mini (2025-08-07) (medium)",
-    "GPT 5 nano (2025-08-07) (medium)",
-    "GPT 5.1 Codex (medium)",
-    "GPT 5.2 Codex",
-    "Gemini 2.0 Flash",
-    "Gemini 2.5 Flash (2025-04-17)",
-    "Gemini 2.5 Pro (2025-05-06)",
-    "Gemini 3 Pro",
-    "Gemini 3 Pro Preview (2025-11-18)",
-    "Kimi K2 Instruct",
-    "Kimi K2 Thinking",
-    "Kimi K2.5 (high)",
-    "Llama 4 Maverick Instruct",
-    "Llama 4 Scout Instruct",
-    "MiniMax M2",
-    "MiniMax M2.5 (high)",
-    "Qwen2.5-Coder 32B Instruct",
-    "Qwen3-Coder 480B/A35B Instruct",
-    "gpt-oss-120b",
-    "o3 (2025-04-16)",
-    "o4-mini (2025-04-16)"
-  ]
-}