@mijanlab/llmtest 1.0.5

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mijanur Rahman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,69 @@
+# LLMtest — Node.js Beta
+
+This beta ports the CLI runtime to Node.js so Python, pip, virtualenvs, and PyInstaller are not required.
+
+## Requirements
+
+- Node.js 18+
+- npm
+
+## One-line install (recommended)
+
+Checks for Node.js 18+, installs it automatically if missing, then installs `llmtest` globally via npm.
+
+macOS / Linux:
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/mijanlab/LLMtest/main/install.sh | bash
+```
+
+Windows (PowerShell):
+
+```powershell
+irm https://raw.githubusercontent.com/mijanlab/LLMtest/main/install.ps1 | iex
+```
+
+Then run:
+
+```bash
+llmtest
+```
+
+Or:
+
+```bash
+llmtest https://api.openai.com/v1 YOUR_API_KEY
+```
+
+## Manual npm installation
+
+If you already have Node.js 18+ installed:
+
+```bash
+npm i -g @mijanlab/llmtest@latest
+```
+
+## Install this ZIP locally (offline / pre-publish testing)
+
+```bash
+unzip LLMtest-beta.zip
+cd LLMtest-beta
+npm install -g .
+```
+
+## Included beta features
+
+- OpenAI-compatible `/models` discovery
+- Streaming `/chat/completions` benchmark
+- TTFT, total latency, and tokens/sec measurement
+- Concurrent model testing
+- Model filtering and run count
+- JSON, CSV, Markdown, and HTML reports
+- Interactive setup wizard
+- `llmtest --update`
+- `llmtest --uninstall`
+- `llmtest` and `llm-test` executable aliases
+
+## Beta note
+
+The current Python implementation on `main` is not modified by this ZIP. Validate provider compatibility and report parity before replacing the production branch.

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@mijanlab/llmtest",
+  "version": "1.0.5",
+  "description": "Fast zero-dependency CLI for benchmarking OpenAI-compatible LLM endpoints",
+  "type": "module",
+  "bin": {
+    "llmtest": "src/cli.js",
+    "llm-test": "src/cli.js"
+  },
+  "scripts": {
+    "test": "node --test",
+    "check": "node --check src/cli.js && node --check src/benchmark.js",
+    "pack:check": "npm pack --dry-run"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "files": [
+    "src/",
+    "README.md",
+    "LICENSE"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mijanlab/LLMtest.git"
+  },
+  "bugs": {
+    "url": "https://github.com/mijanlab/LLMtest/issues"
+  },
+  "homepage": "https://github.com/mijanlab/LLMtest#readme",
+  "keywords": [
+    "llm",
+    "benchmark",
+    "openai",
+    "latency",
+    "ttft",
+    "throughput",
+    "cli"
+  ],
+  "author": "Mijanur Rahman",
+  "license": "MIT"
+}

package/src/benchmark.js

@@ -0,0 +1,313 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+const useColor = Boolean(process.stdout.isTTY) && !process.env.NO_COLOR;
+export const C = {
+  reset: useColor ? '\x1b[0m' : '',
+  bold: useColor ? '\x1b[1m' : '',
+  dim: useColor ? '\x1b[2m' : '',
+  cyan: useColor ? '\x1b[36m' : '',
+  green: useColor ? '\x1b[32m' : '',
+  yellow: useColor ? '\x1b[33m' : '',
+  red: useColor ? '\x1b[31m' : '',
+  gray: useColor ? '\x1b[90m' : ''
+};
+
+export function normalizeUrls(rawEndpoint = '') {
+  const endpoint = String(rawEndpoint).trim().replace(/\/+$/, '');
+  if (!endpoint) return { modelsUrl: '', chatUrl: '' };
+  if (endpoint.endsWith('/chat/completions')) {
+    const base = endpoint.slice(0, -'/chat/completions'.length).replace(/\/+$/, '');
+    return { modelsUrl: `${base}/models`, chatUrl: endpoint };
+  }
+  if (endpoint.endsWith('/models')) {
+    const base = endpoint.slice(0, -'/models'.length).replace(/\/+$/, '');
+    return { modelsUrl: endpoint, chatUrl: `${base}/chat/completions` };
+  }
+  return { modelsUrl: `${endpoint}/models`, chatUrl: `${endpoint}/chat/completions` };
+}
+
+function headers(apiKey = '') {
+  const h = {
+    'content-type': 'application/json',
+    'http-referer': 'https://github.com/mijanlab/LLMtest',
+    'x-title': 'LLM Benchmark Suite'
+  };
+  if (apiKey) h.authorization = `Bearer ${apiKey}`;
+  return h;
+}
+
+async function fetchWithTimeout(url, options = {}, timeoutSeconds = 35) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), Math.max(1, timeoutSeconds) * 1000);
+  try {
+    return await fetch(url, { ...options, signal: controller.signal });
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export async function fetchAvailableModels(modelsUrl, apiKey = '', timeout = 15) {
+  try {
+    const response = await fetchWithTimeout(modelsUrl, { headers: headers(apiKey) }, timeout);
+    if (!response.ok) throw new Error(`HTTP ${response.status}: ${(await response.text()).slice(0, 200)}`);
+    const payload = await response.json();
+    const raw = Array.isArray(payload) ? payload : (payload?.data ?? payload?.models ?? []);
+    return raw.flatMap((item) => {
+      if (typeof item === 'string') return [{ id: item, name: item, is_free: false }];
+      if (!item || typeof item !== 'object') return [];
+      const id = item.id ?? item.name;
+      if (!id) return [];
+      const pricing = item.pricing ?? null;
+      const promptPrice = Number(pricing?.prompt ?? 1);
+      const completionPrice = Number(pricing?.completion ?? 1);
+      return [{
+        id,
+        name: item.name ?? id,
+        is_free: id.includes(':free') || Boolean(pricing && promptPrice === 0 && completionPrice === 0),
+        context_length: item.context_length ?? null
+      }];
+    });
+  } catch (error) {
+    console.error(` ${C.red}✖ Error discovering models:${C.reset} ${error.message}`);
+    return [];
+  }
+}
+
+export function estimateTokens(text = '') {
+  if (!text) return 0;
+  const tokens = text.match(/[\p{L}\p{N}_']+|[^\s\p{L}\p{N}_]/gu) ?? [];
+  return Math.max(1, tokens.length);
+}
+
+export function classifySkippableError(message = '') {
+  const value = message.toLowerCase();
+  const groups = [
+    [['402','payment required','insufficient_quota','insufficient_funds','insufficient funds','insufficient credit','out of credits','quota exceeded','billing','credits required'], 'Non-available fund / insufficient credits'],
+    [['429','rate limit','too many requests','rate_limit','throttled','quota_exceeded'], 'Rate limit exceeded (provider throttled)'],
+    [['403','forbidden','permission_denied','access denied','not allowed','restricted','region'], 'Provider restricted / access denied'],
+    [['not a chat model','unsupported model','invalid model','does not support chat','not found'], 'Unsupported model modality']
+  ];
+  for (const [needles, reason] of groups) {
+    if (needles.some((needle) => value.includes(needle))) return { skippable: true, reason };
+  }
+  return { skippable: false, reason: '' };
+}
+
+function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }
+
+function extractText(obj) {
+  const choice = obj?.choices?.[0];
+  if (!choice) return '';
+  const delta = choice.delta ?? {};
+  return delta.content ?? delta.reasoning ?? delta.reasoning_content ?? delta.thought ?? delta.text ?? choice.text ?? choice.message?.content ?? choice.message?.reasoning ?? '';
+}
+
+export async function testSingleModelStreaming({ chatUrl, modelId, apiKey = '', prompt, systemPrompt, maxTokens = 60, timeout = 35 }) {
+  const messages = [];
+  if (systemPrompt) messages.push({ role: 'system', content: systemPrompt });
+  messages.push({ role: 'user', content: prompt });
+
+  let payload = { model: modelId, messages, stream: true, stream_options: { include_usage: true }, temperature: 0.2, max_tokens: maxTokens };
+
+  for (let attempt = 0; attempt < 3; attempt += 1) {
+    const start = performance.now();
+    let first = null;
+    let last = null;
+    let chunks = 0;
+    let usage = {};
+    let text = '';
+    try {
+      let response = await fetchWithTimeout(chatUrl, { method: 'POST', headers: headers(apiKey), body: JSON.stringify(payload) }, timeout);
+      if (response.status === 400 && payload.stream_options) {
+        payload = { ...payload };
+        delete payload.stream_options;
+        response = await fetchWithTimeout(chatUrl, { method: 'POST', headers: headers(apiKey), body: JSON.stringify(payload) }, timeout);
+      }
+
+      if ([429, 503, 529].includes(response.status) && attempt < 2) {
+        const retryAfter = Number(response.headers.get('retry-after'));
+        await sleep(Math.max(Number.isFinite(retryAfter) ? retryAfter * 1000 : 0, 2000 * (attempt + 1)));
+        continue;
+      }
+      if (!response.ok) {
+        const body = (await response.text()).slice(0, 200);
+        let message = body;
+        try { message = JSON.parse(body)?.error?.message ?? body; } catch {}
+        return { ok: false, error: `HTTP ${response.status}: ${message}` };
+      }
+      if (!response.body) return { ok: false, error: 'Provider returned an empty response body' };
+
+      const decoder = new TextDecoder();
+      let buffer = '';
+      for await (const chunk of response.body) {
+        buffer += decoder.decode(chunk, { stream: true });
+        const lines = buffer.split(/\r?\n/);
+        buffer = lines.pop() ?? '';
+        for (const line of lines) {
+          if (!line.startsWith('data:')) continue;
+          const data = line.slice(5).trim();
+          if (!data || data === '[DONE]') continue;
+          let obj;
+          try { obj = JSON.parse(data); } catch { continue; }
+          if (obj?.error) return { ok: false, error: `Stream Error: ${obj.error.message ?? JSON.stringify(obj.error)}` };
+          if (obj?.usage) usage = obj.usage;
+          const piece = extractText(obj);
+          if (piece !== '' && piece != null) {
+            const now = performance.now();
+            if (first == null) first = now;
+            last = now;
+            chunks += 1;
+            text += String(piece);
+          }
+        }
+      }
+      const end = performance.now();
+      text = text.trim();
+      if (first == null) {
+        if (!text && attempt < 2) { await sleep(1000); continue; }
+        if (!text) return { ok: false, error: 'No response text received from model' };
+        first = end;
+        last = end;
+      }
+      const ttft = (first - start) / 1000;
+      const total = (end - start) / 1000;
+      const generation = Math.max((end - first) / 1000, ((last ?? end) - first) / 1000, 0.001);
+      const completionTokens = Number(usage?.completion_tokens) > 0 ? Number(usage.completion_tokens) : estimateTokens(text);
+      return {
+        ok: true,
+        ttft,
+        total,
+        generation,
+        chunks,
+        tps: completionTokens > 0 ? completionTokens / generation : null,
+        completion_tokens: completionTokens,
+        prompt_tokens: usage?.prompt_tokens ?? null,
+        total_tokens: usage?.total_tokens ?? null,
+        preview: text.length > 80 ? `${text.slice(0, 80)}...` : text,
+        full_text: text
+      };
+    } catch (error) {
+      if (attempt < 2 && (/429|503|timeout|aborted/i.test(error.message))) {
+        await sleep(2000 * (attempt + 1));
+        continue;
+      }
+      return { ok: false, error: error.name === 'AbortError' ? `Request timed out after ${timeout}s` : error.message };
+    }
+  }
+  return { ok: false, error: 'Maximum retries exceeded' };
+}
+
+function mean(values) { return values.length ? values.reduce((a, b) => a + b, 0) / values.length : null; }
+
+export async function benchmarkModel(options) {
+  const details = [];
+  for (let run = 1; run <= options.runs; run += 1) {
+    details.push({ ...(await testSingleModelStreaming(options)), run });
+  }
+  const good = details.filter((r) => r.ok);
+  const errors = [...new Set(details.filter((r) => !r.ok && r.error).map((r) => r.error))];
+  let status = 'FAIL';
+  let skipReason = null;
+  if (good.length === options.runs) status = 'PASS';
+  else if (good.length > 0) status = 'PARTIAL';
+  else if (errors.length) {
+    const classified = errors.map(classifySkippableError);
+    if (classified.every((r) => r.skippable)) {
+      status = 'SKIPPED';
+      skipReason = classified[0].reason;
+    }
+  }
+  return {
+    model: options.modelId,
+    runs: options.runs,
+    success: good.length,
+    failed: options.runs - good.length,
+    status,
+    skip_reason: skipReason,
+    avg_ttft: mean(good.map((r) => r.ttft).filter((v) => v != null)),
+    avg_total: mean(good.map((r) => r.total).filter((v) => v != null)),
+    avg_tps: mean(good.map((r) => r.tps).filter((v) => v != null)),
+    sample_preview: good.find((r) => r.preview)?.preview ?? null,
+    errors,
+    details
+  };
+}
+
+function fmtTTFT(value) { return value == null ? '—' : value < 1 ? `${Math.round(value * 1000)} ms` : `${value.toFixed(3)} s`; }
+function fmtTotal(value) { return value == null ? '—' : `${value.toFixed(3)} s`; }
+function fmtTPS(value) { return value == null ? '—' : `${value.toFixed(1)} tok/s`; }
+function pad(value, width) { const s = String(value); return s.length >= width ? s.slice(0, width) : s.padEnd(width); }
+
+const STATUS_ICON = { PASS: '🟢', PARTIAL: '⚪', SKIPPED: '⚪', FAIL: '🔴' };
+export function statusLabel(status) { return `${STATUS_ICON[status] ?? '⚪'} ${status}`; }
+
+export function printTable(results) {
+  console.log('\n#   Model ID                    Status         TTFT       Total      Speed        Output / Notes');
+  console.log('─'.repeat(114));
+  results.forEach((r, i) => {
+    const note = r.status === 'PASS' ? (r.sample_preview ?? '') : (r.skip_reason ?? r.errors?.join('; ') ?? 'Failed');
+    console.log(`${pad(i + 1, 3)} ${pad(r.model, 27)} ${pad(statusLabel(r.status), 14)} ${pad(fmtTTFT(r.avg_ttft), 10)} ${pad(fmtTotal(r.avg_total), 10)} ${pad(fmtTPS(r.avg_tps), 12)} ${String(note).replace(/\s+/g, ' ').slice(0, 34)}`);
+  });
+}
+
+export function printSummary(results, seconds) {
+  const passed = results.filter((r) => r.status === 'PASS');
+  const skipped = results.filter((r) => r.status === 'SKIPPED');
+  const failed = results.filter((r) => r.status === 'FAIL');
+  const fastest = passed.filter((r) => r.avg_ttft != null).sort((a,b) => a.avg_ttft - b.avg_ttft)[0];
+  const highest = passed.filter((r) => r.avg_tps != null).sort((a,b) => b.avg_tps - a.avg_tps)[0];
+  console.log(`\n${C.bold}📊 Benchmark Summary${C.reset}`);
+  console.log(` • Total Models Tested : ${results.length}`);
+  console.log(` • Working & Passed    : ${passed.length}/${results.length}`);
+  if (skipped.length) console.log(` • Skipped / Throttled : ${skipped.length}`);
+  if (failed.length) console.log(` • Failed Execution    : ${failed.length}`);
+  if (fastest) console.log(` • Fastest TTFT        : ${fastest.model} (${fmtTTFT(fastest.avg_ttft)})`);
+  if (highest) console.log(` • Highest Speed       : ${highest.model} (${fmtTPS(highest.avg_tps)})`);
+  if (seconds > 0) console.log(` • Total Time Elapsed  : ${seconds.toFixed(2)}s`);
+}
+
+function csvEscape(value) {
+  const s = String(value ?? '');
+  return /[",\n]/.test(s) ? `"${s.replaceAll('"', '""')}"` : s;
+}
+
+export function exportReports(results, endpoint, opts = {}) {
+  const created = [];
+  if (opts.json) {
+    fs.writeFileSync(opts.json, JSON.stringify({ endpoint, tested_at: Date.now() / 1000, results }, null, 2));
+    created.push(opts.json);
+  }
+  if (opts.csv) {
+    const rows = [['Model ID','Status','Success Runs','Total Runs','TTFT Seconds','Total Seconds','Tokens/Sec','Notes']];
+    for (const r of results) rows.push([r.model,r.status,r.success,r.runs,r.avg_ttft ?? '',r.avg_total ?? '',r.avg_tps ?? '',r.status === 'PASS' ? r.sample_preview ?? '' : r.skip_reason ?? r.errors?.join('; ') ?? '']);
+    fs.writeFileSync(opts.csv, rows.map((row) => row.map(csvEscape).join(',')).join('\n'));
+    created.push(opts.csv);
+  }
+  if (opts.md) {
+    const lines = [`# LLM Models Benchmark Report`, '', `- **Endpoint Tested**: \`${endpoint}\``, `- **Tested At**: ${new Date().toISOString()}`, '', '| # | Model ID | Status | TTFT | Total | Throughput |', '|---|---|---|---|---|---|'];
+    results.forEach((r, i) => lines.push(`| ${i + 1} | \`${r.model.replaceAll('|','\\|')}\` | ${r.status} | ${fmtTTFT(r.avg_ttft)} | ${fmtTotal(r.avg_total)} | ${fmtTPS(r.avg_tps)} |`));
+    fs.writeFileSync(opts.md, lines.join('\n'));
+    created.push(opts.md);
+  }
+  if (opts.html) {
+    const safe = (v) => String(v ?? '').replace(/[&<>"']/g, (ch) => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[ch]));
+    const rows = results.map((r) => `<tr><td>${safe(r.model)}</td><td>${safe(r.status)}</td><td>${safe(fmtTTFT(r.avg_ttft))}</td><td>${safe(fmtTotal(r.avg_total))}</td><td>${safe(fmtTPS(r.avg_tps))}</td></tr>`).join('');
+    const html = `<!doctype html><html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>llmtest report</title><style>
+body{font-family:system-ui;background:#090d0a;color:#e8f5eb;margin:40px}
+h1{color:#4ade80}
+code{color:#86efac}
+table{width:100%;border-collapse:collapse;margin-top:24px}
+th,td{padding:10px;border-bottom:1px solid #26352a;text-align:left}
+th{color:#86efac}
+.pdf-btn{position:fixed;top:24px;right:24px;background:#4ade80;color:#052e16;border:none;border-radius:8px;padding:10px 18px;font-size:14px;font-weight:600;cursor:pointer;box-shadow:0 2px 8px rgba(0,0,0,.4)}
+.pdf-btn:hover{background:#86efac}
+@media print{.pdf-btn{display:none}body{background:#fff;color:#111}h1,th{color:#111}th,td{border-color:#ccc}}
+</style></head><body>
+<button class="pdf-btn" onclick="window.print()">⬇ Download as PDF</button>
+<h1>⚡ llmtest</h1><p>Endpoint: <code>${safe(endpoint)}</code></p><p>Generated: ${safe(new Date().toISOString())}</p><table><thead><tr><th>Model</th><th>Status</th><th>TTFT</th><th>Total</th><th>Speed</th></tr></thead><tbody>${rows}</tbody></table></body></html>`;
+    fs.writeFileSync(opts.html, html);
+    created.push(opts.html);
+  }
+  return created.map((file) => path.resolve(file));
+}

package/src/cli.js

@@ -0,0 +1,187 @@
+#!/usr/bin/env node
+import fs from 'node:fs';
+import { pathToFileURL } from 'node:url';
+import { spawnSync } from 'node:child_process';
+import { createInterface } from 'node:readline/promises';
+import { stdin as input, stdout as output } from 'node:process';
+import {
+  C, normalizeUrls, fetchAvailableModels, benchmarkModel,
+  printTable, printSummary, exportReports, statusLabel
+} from './benchmark.js';
+
+const VERSION = '1.0.5';
+const banner = `\n${C.bold}LLM Speed & Latency Benchmark${C.reset}\n${C.cyan}    LLMTEST v${VERSION}${C.reset}\n${C.gray}-------------------------------${C.reset}`;
+
+function help() {
+  console.log(`\nUsage:\n  llmtest [endpoint] [key] [filter] [options]\n\nOptions:\n  -e, --endpoint <url>       API base URL\n  -k, --key <key>            API key\n  -f, --filter <text>        Model ID filter; use "free" for free models\n  -l, --limit <n>            Limit models (0 = all)\n  -r, --runs <n>             Runs per model (default: 1)\n  -c, --concurrency <n>      Concurrent models (default: 3)\n  -p, --prompt <text>        Evaluation prompt\n  -s, --system <text>        System prompt\n      --max-tokens <n>       Max response tokens (default: 60)\n      --timeout <sec>        Request timeout (default: 35)\n      --output-json <file>   JSON report path\n      --output-csv <file>    CSV report path\n      --output-md <file>     Markdown report path\n      --output-html <file>   HTML report path\n      --no-report            Disable report files\n      --report <json>        Render reports from an existing JSON file\n      --update               npm install -g @mijanlab/llmtest@latest\n      --uninstall            npm uninstall -g @mijanlab/llmtest\n  -v, --version              Show version\n  -h, --help                 Show help\n`);
+}
+
+function timestamp() {
+  return new Date().toISOString().replace(/[:T]/g, '-').slice(0, 19);
+}
+
+function parseArgs(argv) {
+  const opts = { endpoint:'', key:'', filter:'', limit:0, runs:1, concurrency:3, prompt:'Say hello and describe your purpose in one brief sentence.', system:'You are a helpful assistant.', maxTokens:60, timeout:35, json:'', csv:'', md:'', html:'', noReport:false, report:'', update:false, uninstall:false, open:false };
+  const pos = [];
+  const aliases = { '-e':'endpoint','--endpoint':'endpoint','-k':'key','--key':'key','-f':'filter','--filter':'filter','-l':'limit','--limit':'limit','-r':'runs','--runs':'runs','-c':'concurrency','--concurrency':'concurrency','-p':'prompt','--prompt':'prompt','-s':'system','--system':'system','--max-tokens':'maxTokens','--timeout':'timeout','--output-json':'json','--output-csv':'csv','--csv':'csv','--output-md':'md','--output-html':'html','--html':'html','--report':'report','--render':'report' };
+  const numeric = new Set(['limit','runs','concurrency','maxTokens','timeout']);
+  for (let i=0;i<argv.length;i+=1) {
+    const a = argv[i];
+    if (a === '-h' || a === '--help') opts.help = true;
+    else if (a === '-v' || a === '--version') opts.version = true;
+    else if (a === '--no-report') opts.noReport = true;
+    else if (a === '--update') opts.update = true;
+    else if (a === '--uninstall') opts.uninstall = true;
+    else if (a === '--open') opts.open = true;
+    else if (aliases[a]) {
+      const key = aliases[a];
+      const value = argv[++i];
+      if (value == null) throw new Error(`Missing value for ${a}`);
+      opts[key] = numeric.has(key) ? Number(value) : value;
+    } else if (a.startsWith('-')) throw new Error(`Unknown option: ${a}`);
+    else pos.push(a);
+  }
+  opts.endpoint ||= pos[0] ?? '';
+  opts.key ||= pos[1] ?? '';
+  opts.filter ||= pos[2] ?? '';
+  if (/^(update|upgrade)$/i.test(opts.endpoint)) { opts.update = true; opts.endpoint = ''; }
+  if (/^(uninstall|remove)$/i.test(opts.endpoint)) { opts.uninstall = true; opts.endpoint = ''; }
+  if (/\.json$/i.test(opts.endpoint) && fs.existsSync(opts.endpoint)) { opts.report = opts.endpoint; opts.endpoint = ''; }
+  return opts;
+}
+
+function runNpm(args, label) {
+  console.log(`\n${C.cyan}→ Running: npm ${args.join(' ')}${C.reset}\n`);
+  const result = spawnSync('npm', args, { stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf8', shell: process.platform === 'win32' });
+
+  if (result.error) {
+    console.error(`${C.red}✖ Could not run npm: ${result.error.message}${C.reset}`);
+    console.error(`${C.yellow}  Make sure Node.js/npm is installed and on your PATH.${C.reset}`);
+    process.exit(1);
+  }
+
+  if (result.stdout) process.stdout.write(result.stdout);
+  if (result.stderr) process.stderr.write(result.stderr);
+
+  if (result.status !== 0) {
+    console.error(`\n${C.red}✖ ${label} failed (npm exit code ${result.status}).${C.reset}`);
+    if (/E404/.test(result.stderr ?? '')) {
+      console.error(`${C.yellow}  Package not found on the npm registry (check your connection or the package name).${C.reset}`);
+      console.error(`${C.yellow}  For now, test from a local copy: npm install -g .  (run inside the package folder)${C.reset}`);
+    } else if (/EACCES|EPERM/.test(result.stderr ?? '')) {
+      console.error(`${C.yellow}  Permission denied. Try running your terminal as Administrator (Windows) or with sudo (macOS/Linux).${C.reset}`);
+    }
+    process.exit(result.status ?? 1);
+  }
+
+  console.log(`\n${C.green}✔ ${label} completed.${C.reset}`);
+  process.exit(0);
+}
+
+async function wizard() {
+  const rl = createInterface({ input, output });
+  try {
+    console.log(`\n${C.cyan}⚡ Interactive Setup Wizard${C.reset}`);
+    const endpoint = (await rl.question('1. API Endpoint URL: ')).trim();
+    if (/^(update|upgrade)$/i.test(endpoint)) runNpm(['install','-g','@mijanlab/llmtest@latest'], 'Update');
+    if (/^(uninstall|remove)$/i.test(endpoint)) runNpm(['uninstall','-g','@mijanlab/llmtest'], 'Uninstall');
+    const key = (await rl.question('2. API Key [Enter if none]: ')).trim();
+    const filter = (await rl.question('3. Model Filter [Enter for all]: ')).trim();
+    const c = (await rl.question('4. Concurrency [default 3]: ')).trim();
+    return { endpoint, key, filter, concurrency: c ? Math.max(1, Number(c) || 3) : 3 };
+  } finally { rl.close(); }
+}
+
+async function runPool(items, concurrency, worker, onDone) {
+  let index = 0;
+  const runners = Array.from({ length: Math.max(1, Math.min(concurrency, items.length || 1)) }, async () => {
+    while (true) {
+      const current = index++;
+      if (current >= items.length) return;
+      const value = await worker(items[current]);
+      onDone(value, items[current]);
+    }
+  });
+  await Promise.all(runners);
+}
+
+function maybeOpen(file) {
+  if (!file) return;
+  const command = process.platform === 'darwin' ? ['open',[file]] : process.platform === 'win32' ? ['cmd',['/c','start','',file]] : ['xdg-open',[file]];
+  spawnSync(command[0], command[1], { stdio:'ignore', detached:true });
+}
+
+async function main() {
+  console.log(banner);
+  let args;
+  try { args = parseArgs(process.argv.slice(2)); } catch (e) { console.error(`${C.red}✖ ${e.message}${C.reset}`); help(); process.exitCode=2; return; }
+  if (args.help) { help(); return; }
+  if (args.version) { console.log(`llmtest ${VERSION}`); return; }
+  if (args.update) runNpm(['install','-g','@mijanlab/llmtest@latest'], 'Update');
+  if (args.uninstall) runNpm(['uninstall','-g','@mijanlab/llmtest'], 'Uninstall');
+
+  if (args.report) {
+    const data = JSON.parse(fs.readFileSync(args.report, 'utf8'));
+    const results = data.results ?? [];
+    printTable(results);
+    printSummary(results, 0);
+    if (!args.noReport) {
+      const ts = timestamp();
+      const created = exportReports(results, data.endpoint ?? 'Unknown Endpoint', {
+        html: args.html || `benchmark_report_${ts}.html`,
+        md: args.md || `benchmark_report_${ts}.md`,
+        csv: args.csv || `benchmark_report_${ts}.csv`
+      });
+      created.forEach((file) => console.log(` ✔ ${pathToFileURL(file).href}`));
+    }
+    return;
+  }
+
+  if (!args.endpoint) Object.assign(args, await wizard());
+  if (!args.endpoint) { console.error(`${C.red}✖ No endpoint specified.${C.reset}`); return; }
+  const { modelsUrl, chatUrl } = normalizeUrls(args.endpoint);
+  const masked = args.key ? (args.key.length > 12 ? `${args.key.slice(0,6)}...${args.key.slice(-4)}` : '••••••••') : '(none)';
+  console.log(`${C.bold}Connecting to endpoint:${C.reset} ${C.cyan}${args.endpoint}${C.reset}`);
+  console.log(` • Models URL : ${modelsUrl}`);
+  console.log(` • Chat URL   : ${chatUrl}`);
+  console.log(` • API Key    : ${masked}\n`);
+
+  let models = await fetchAvailableModels(modelsUrl, args.key, args.timeout);
+  if (!models.length) { console.error(`${C.red}✖ No models discovered. Check endpoint URL and API key.${C.reset}`); process.exitCode=1; return; }
+  console.log(` ${C.green}✔ Discovered ${models.length} total models.${C.reset}`);
+  if (args.filter) {
+    models = args.filter.toLowerCase() === 'free' ? models.filter((m) => m.is_free || m.id.includes(':free')) : models.filter((m) => m.id.toLowerCase().includes(args.filter.toLowerCase()));
+    console.log(` ${C.cyan}✔ Filtered to ${models.length} models.${C.reset}`);
+  }
+  if (args.limit > 0) models = models.slice(0, args.limit);
+  if (!models.length) { console.error(`${C.yellow}⚠ No models match the filter.${C.reset}`); return; }
+
+  console.log(`\n${C.bold}🚀 Starting benchmark (${models.length} models | Concurrency: ${args.concurrency} | Runs: ${args.runs})...${C.reset}\n`);
+  const results = [];
+  const started = performance.now();
+  await runPool(models, Math.max(1,args.concurrency), (m) => benchmarkModel({ chatUrl, modelId:m.id, apiKey:args.key, prompt:args.prompt, systemPrompt:args.system, maxTokens:args.maxTokens, timeout:args.timeout, runs:Math.max(1,args.runs) }), (r) => {
+    results.push(r);
+    const ttft = r.avg_ttft == null ? '—' : r.avg_ttft < 1 ? `${Math.round(r.avg_ttft*1000)}ms` : `${r.avg_ttft.toFixed(2)}s`;
+    const tps = r.avg_tps == null ? '—' : `${r.avg_tps.toFixed(1)} tok/s`;
+    console.log(` [${String(results.length).padStart(2)}/${models.length}] ${statusLabel(r.status).padEnd(10)} ${r.model.slice(0,29).padEnd(29)} │ TTFT: ${ttft.padEnd(7)} │ Speed: ${tps}`);
+  });
+  const rank = { PASS: 0, PARTIAL: 1, SKIPPED: 2, FAIL: 3 };
+  results.sort((a,b) => ((rank[a.status] ?? 9) - (rank[b.status] ?? 9)) || ((a.avg_ttft ?? 999) - (b.avg_ttft ?? 999)));
+  const elapsed = (performance.now() - started) / 1000;
+  printTable(results);
+  printSummary(results, elapsed);
+  if (!args.noReport) {
+    const ts = timestamp();
+    args.json ||= `benchmark_report_${ts}.json`;
+    args.csv ||= `benchmark_report_${ts}.csv`;
+    args.md ||= `benchmark_report_${ts}.md`;
+    args.html ||= `benchmark_report_${ts}.html`;
+    const created = exportReports(results, args.endpoint, { json:args.json, csv:args.csv, md:args.md, html:args.html });
+    console.log(`\n${C.bold}📁 Exported Reports:${C.reset}`);
+    created.forEach((file) => console.log(` ✔ ${pathToFileURL(file).href}`));
+    if (args.open) maybeOpen(args.html);
+  }
+  console.log(`\n 💡 Next time: ${C.cyan}llmtest ${args.endpoint} <your_api_key>${args.filter ? ` ${args.filter}` : ''}${C.reset}\n`);
+}
+
+main().catch((error) => { console.error(`${C.red}✖ ${error.stack ?? error.message}${C.reset}`); process.exitCode = 1; });