fm-bench 0.3.0 → 0.3.1

package/README.md

@@ -95,6 +95,7 @@ Useful flags:
 - `--capture-output`: include raw model output in JSON reports.
 - `--json`, `--csv`, `--format table|json|csv`: choose output format.
 - `--ascii`: use plain ASCII table borders.
+- `--color`, `--no-color`: force or disable semantic ANSI colors. Colors are automatic on TTYs.
 - `--compact`: force the narrow terminal layout.
 - `--width <n>`: render as if the terminal has `n` columns.
 - `--out <file>`: save a report.
@@ -143,6 +144,18 @@ Measured runs stream by default so `fm-bench` can capture TTFT. Use `--no-stream
 
 Terminal output is responsive. Wide terminals show full scoreboard and detail tables, medium terminals show a tighter operating-point table, and narrow terminals switch to compact model cards. Use `--width` to preview a layout and `--ascii` for log systems that do not render Unicode borders well.
 
+## Terminal Colors
+
+Table output uses semantic ANSI color on interactive terminals:
+
+- green: passing, steadier, or better than the current comparison set.
+- yellow: marginal, partial, or near a budget.
+- red: failing a budget, unstable, or slower/lower than peers.
+
+Success rate, goodput, repeatability, and CV use fixed benchmark thresholds. Throughput columns use relative ranking within the current run because “good” depends on the machine, model, prompt mix, and concurrency. TTFT, E2E, and TPOT use SLO thresholds when you pass `--slo-ttft-ms`, `--slo-e2e-ms`, or `--slo-tpot-ms`; otherwise they use lower-is-better relative ranking across the models and operating points in the report.
+
+Use `--color` to force ANSI colors in captured logs, or `--no-color` for plain output. `NO_COLOR=1` disables automatic color and `FORCE_COLOR=1` enables it.
+
 See [docs/methodology.md](docs/methodology.md) for the benchmark methodology and source references.
 
 ## Requirements

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fm-bench",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "Dynamic benchmark CLI for Apple's fm command on macOS 27+.",
   "type": "module",
   "bin": {

package/src/cli.js

@@ -84,6 +84,7 @@ export function parseArgs(argv) {
     failFast: false,
     verbose: false,
     ascii: false,
+    color: 'auto',
     compact: false,
     width: null
   };
@@ -193,6 +194,12 @@ export function parseArgs(argv) {
       case '--ascii':
         options.ascii = true;
         break;
+      case '--color':
+        options.color = 'always';
+        break;
+      case '--no-color':
+        options.color = 'never';
+        break;
       case '--compact':
         options.compact = true;
         break;
@@ -291,11 +298,20 @@ function parsePositiveIntList(value, option) {
 function renderOptions(parsed) {
   return {
     ascii: parsed.ascii,
+    color: resolveColor(parsed.color),
     compact: parsed.compact,
     width: parsed.width
   };
 }
 
+function resolveColor(value) {
+  if (value === 'always') return true;
+  if (value === 'never') return false;
+  if (process.env.NO_COLOR) return false;
+  if (process.env.FORCE_COLOR && process.env.FORCE_COLOR !== '0') return true;
+  return Boolean(process.stdout.isTTY);
+}
+
 function helpText() {
   return `fm-bench ${packageJson.version}
 
@@ -336,6 +352,8 @@ Output:
       --json                Alias for --format json
       --csv                 Alias for --format csv
       --ascii               Use plain ASCII tables instead of Unicode
+      --color               Force ANSI colors in table output
+      --no-color            Disable ANSI colors in table output
       --compact             Force compact terminal layout
       --width <n>           Render for a specific terminal width
   -o, --out <file>          Save JSON or CSV report based on file extension

package/src/table.js

@@ -1,9 +1,17 @@
+import { stripAnsi } from './ansi.js';
+
 export function renderTable(headers, rows, options = {}) {
   const ascii = Boolean(options.ascii);
   const maxCellWidth = options.maxCellWidth || 60;
-  const stringRows = rows.map((row) => row.map((cell) => truncate(formatCell(cell), maxCellWidth)));
+  const normalizedRows = rows.map((row) => row.map((cell) => {
+    const normalized = normalizeCell(cell);
+    return {
+      ...normalized,
+      text: truncate(normalized.text, maxCellWidth)
+    };
+  }));
   const widths = headers.map((header, index) => {
-    const values = [truncate(header, maxCellWidth), ...stringRows.map((row) => row[index] ?? '')];
+    const values = [truncate(header, maxCellWidth), ...normalizedRows.map((row) => row[index]?.text ?? '')];
     return Math.max(...values.map(visibleLength));
   });
   const style = ascii ? ASCII_TABLE : UNICODE_TABLE;
@@ -11,8 +19,8 @@ export function renderTable(headers, rows, options = {}) {
   const top = rule(style.topLeft, style.topJoin, style.topRight, style.horizontal, widths);
   const middle = rule(style.midLeft, style.midJoin, style.midRight, style.horizontal, widths);
   const bottom = rule(style.bottomLeft, style.bottomJoin, style.bottomRight, style.horizontal, widths);
-  const headerLine = rowLine(headers.map((header) => truncate(header, maxCellWidth)), widths, style, true);
-  const bodyLines = stringRows.map((row) => rowLine(row, widths, style));
+  const headerLine = rowLine(headers.map((header) => truncate(header, maxCellWidth)), widths, style, true, options);
+  const bodyLines = normalizedRows.map((row) => rowLine(row, widths, style, false, options));
 
   return [top, headerLine, middle, ...bodyLines, bottom].join('\n');
 }
@@ -41,11 +49,11 @@ export function renderBenchmarkReport(payload, options = {}) {
   lines.push('');
 
   if (mode === 'compact') {
-    lines.push(renderCompactSummary(payload.summary, { ...options, width }));
+    lines.push(renderCompactSummary(payload.summary, { ...options, width, slo: payload.options.slo }));
   } else {
-    lines.push(renderSummaryTable(payload.summary, { ...options, mode, width }));
+    lines.push(renderSummaryTable(payload.summary, { ...options, mode, width, slo: payload.options.slo }));
     lines.push('');
-    lines.push(renderDetailTable(payload.summary, { ...options, mode, width }));
+    lines.push(renderDetailTable(payload.summary, { ...options, mode, width, slo: payload.options.slo }));
   }
 
   lines.push('');
@@ -74,24 +82,26 @@ export function formatPercent(value, digits = 0) {
 export function renderSummaryTable(summary, options = {}) {
   const mode = options.mode || 'wide';
   const hasGoodput = summary.some((item) => item.goodputRate != null);
+  const ranks = rankSummary(summary);
   const rows = summary.map((item) => {
     const status = item.available ? (item.failures > 0 ? 'partial' : 'ok') : 'skipped';
+    const tones = metricTones(item, ranks, options.slo);
     const base = [
-      formatConcurrency(item.concurrency),
-      item.model,
-      status,
-      item.attempted ? `${item.successes}/${item.attempted}` : '-',
-      formatPercent(item.successRate),
-      formatPercent(item.goodputRate),
-      formatMs(item.ttft.p50),
-      formatMs(item.ttft.p95),
-      formatMs(item.latency.p50),
-      formatMs(item.latency.p95),
-      formatNumber(item.tokensPerSecond.avg),
-      formatNumber(item.outputTokenThroughput),
-      formatNumber(item.rps),
-      formatPercent(item.latency.cv),
-      item.available ? '' : compactReason(item.skippedReason)
+      cell(formatConcurrency(item.concurrency)),
+      cell(item.model, item.available ? null : 'muted'),
+      cell(status, statusTone(status)),
+      cell(item.attempted ? `${item.successes}/${item.attempted}` : '-', item.failures > 0 ? 'yellow' : item.available ? 'green' : 'muted'),
+      cell(formatPercent(item.successRate), percentTone(item.successRate, 0.95, 1)),
+      cell(formatPercent(item.goodputRate), percentTone(item.goodputRate, 0.8, 1)),
+      cell(formatMs(item.ttft.p50), tones.ttft),
+      cell(formatMs(item.ttft.p95), tones.ttftP95),
+      cell(formatMs(item.latency.p50), tones.e2e),
+      cell(formatMs(item.latency.p95), tones.e2eP95),
+      cell(formatNumber(item.tokensPerSecond.avg), tones.userTps),
+      cell(formatNumber(item.outputTokenThroughput), tones.systemTps),
+      cell(formatNumber(item.rps), tones.rps),
+      cell(formatPercent(item.latency.cv), cvTone(item.latency.cv)),
+      cell(item.available ? '' : compactReason(item.skippedReason), item.available ? null : 'yellow')
     ];
 
     if (mode === 'medium') {
@@ -113,8 +123,8 @@ export function renderSummaryTable(summary, options = {}) {
       base[7],
       base[8],
       base[9],
-      formatMs(item.tpot.p50),
-      formatMs(item.tpot.p95),
+      cell(formatMs(item.tpot.p50), tones.tpot),
+      cell(formatMs(item.tpot.p95), tones.tpotP95),
       base[10],
       base[11],
       base[12],
@@ -135,17 +145,19 @@ export function renderSummaryTable(summary, options = {}) {
 
 export function renderDetailTable(summary, options = {}) {
   const mode = options.mode || 'wide';
+  const ranks = rankSummary(summary);
   const rows = summary.map((item) => {
+    const tones = metricTones(item, ranks, options.slo);
     const base = [
-      formatConcurrency(item.concurrency),
-      item.model,
-      formatNumber(item.promptTokens.avg, 0),
-      formatNumber(item.outputTokens.avg, 0),
-      formatNumber(item.decodeTokensPerSecond.avg),
-      formatMs(item.latency.p99),
-      formatRangeMs(item.latency.ci95Low, item.latency.ci95High),
-      formatPercent(item.repeatability),
-      item.description || '-'
+      cell(formatConcurrency(item.concurrency)),
+      cell(item.model, item.available ? null : 'muted'),
+      cell(formatNumber(item.promptTokens.avg, 0)),
+      cell(formatNumber(item.outputTokens.avg, 0)),
+      cell(formatNumber(item.decodeTokensPerSecond.avg), tones.decodeTps),
+      cell(formatMs(item.latency.p99), tones.e2eP99),
+      cell(formatRangeMs(item.latency.ci95Low, item.latency.ci95High), cvTone(item.latency.cv)),
+      cell(formatPercent(item.repeatability), percentTone(item.repeatability, 0.5, 0.9)),
+      cell(item.description || '-', 'muted')
     ];
 
     if (mode === 'medium') {
@@ -176,19 +188,21 @@ export function renderCompactSummary(summary, options = {}) {
   const width = options.width || 80;
   const separator = options.ascii ? '-' : '─';
   const lines = [];
+  const ranks = rankSummary(summary);
 
   for (const item of summary) {
     const status = item.available ? (item.failures > 0 ? 'partial' : 'ok') : 'skipped';
     const title = `${item.model} c${formatConcurrency(item.concurrency)} ${status} ${item.attempted ? `${item.successes}/${item.attempted}` : '-'}`;
-    lines.push(truncate(title, width));
+    lines.push(toneText(truncate(title, width), statusTone(status), options));
 
     if (item.available) {
-      lines.push(truncate(`  TTFT ${formatMs(item.ttft.p50)} p95 ${formatMs(item.ttft.p95)} | E2E ${formatMs(item.latency.p50)} p95 ${formatMs(item.latency.p95)} p99 ${formatMs(item.latency.p99)}`, width));
       const goodput = item.goodputRate == null ? '' : ` | good ${formatPercent(item.goodputRate)}`;
-      lines.push(truncate(`  user ${formatNumber(item.tokensPerSecond.avg)} tok/s | system ${formatNumber(item.outputTokenThroughput)} tok/s | RPS ${formatNumber(item.rps)} | CV ${formatPercent(item.latency.cv)}${goodput}`, width));
-      lines.push(truncate(`  in/out ${formatNumber(item.promptTokens.avg, 0)}/${formatNumber(item.outputTokens.avg, 0)} tok avg | TPOT ${formatMs(item.tpot.p50)} | repeat ${formatPercent(item.repeatability)}`, width));
+      const tones = metricTones(item, ranks, options.slo);
+      lines.push(toneText(truncate(`  TTFT ${formatMs(item.ttft.p50)} p95 ${formatMs(item.ttft.p95)} | E2E ${formatMs(item.latency.p50)} p95 ${formatMs(item.latency.p95)} p99 ${formatMs(item.latency.p99)}`, width), worstTone(tones.ttft, tones.e2e, tones.e2eP95), options));
+      lines.push(toneText(truncate(`  user ${formatNumber(item.tokensPerSecond.avg)} tok/s | system ${formatNumber(item.outputTokenThroughput)} tok/s | RPS ${formatNumber(item.rps)} | CV ${formatPercent(item.latency.cv)}${goodput}`, width), worstTone(tones.userTps, tones.systemTps, cvTone(item.latency.cv), percentTone(item.goodputRate, 0.8, 1)), options));
+      lines.push(toneText(truncate(`  in/out ${formatNumber(item.promptTokens.avg, 0)}/${formatNumber(item.outputTokens.avg, 0)} tok avg | TPOT ${formatMs(item.tpot.p50)} | repeat ${formatPercent(item.repeatability)}`, width), worstTone(tones.tpot, percentTone(item.repeatability, 0.5, 0.9)), options));
     } else {
-      lines.push(truncate(`  ${compactReason(item.skippedReason)}`, width));
+      lines.push(toneText(truncate(`  ${compactReason(item.skippedReason)}`, width), 'yellow', options));
     }
     lines.push(separator.repeat(Math.min(width, 72)));
   }
@@ -201,12 +215,15 @@ export function renderModelsTable(models, options = {}) {
   const width = terminalWidth(options);
   const compact = options.compact || width < 88;
   if (compact) {
-    return models.map((model) => `${model.name} ${model.available ? 'yes' : 'no'} ${compactReason(model.reason || model.description || '-')}`).join('\n');
+    return models.map((model) => {
+      const status = model.available ? 'yes' : 'no';
+      return `${model.name} ${toneText(status, model.available ? 'green' : 'yellow', options)} ${compactReason(model.reason || model.description || '-')}`;
+    }).join('\n');
   }
 
   return renderTable(['model', 'available', 'description', 'quota'], models.map((model) => [
-    model.name,
-    model.available ? 'yes' : 'no',
+    cell(model.name),
+    cell(model.available ? 'yes' : 'no', model.available ? 'green' : 'yellow'),
     model.description || '-',
     compactReason(model.quota || model.reason || '-')
   ]), options);
@@ -270,13 +287,32 @@ function rule(left, join, right, horizontal, widths) {
   return `${left}${widths.map((width) => horizontal.repeat(width + 2)).join(join)}${right}`;
 }
 
-function rowLine(row, widths, style, header = false) {
+function rowLine(row, widths, style, header = false, options = {}) {
   return `${style.vertical}${row.map((cell, index) => {
-    const value = header ? String(cell).toUpperCase() : formatCell(cell);
-    return ` ${pad(value, widths[index], !header && isNumericCell(value))} `;
+    const normalized = header ? { text: String(cell).toUpperCase(), tone: 'header' } : normalizeCell(cell);
+    const value = normalized.text;
+    const padded = pad(value, widths[index], !header && isNumericCell(value));
+    return ` ${toneText(padded, normalized.tone, options)} `;
   }).join(style.vertical)}${style.vertical}`;
 }
 
+function cell(text, tone = null) {
+  return { text, tone };
+}
+
+function normalizeCell(value) {
+  if (value && typeof value === 'object' && Object.hasOwn(value, 'text')) {
+    return {
+      text: formatCell(value.text),
+      tone: value.tone || null
+    };
+  }
+  return {
+    text: formatCell(value),
+    tone: null
+  };
+}
+
 function formatCell(value) {
   if (value == null) return '';
   return String(value);
@@ -293,7 +329,7 @@ function isNumericCell(value) {
 }
 
 function visibleLength(value) {
-  return String(value).length;
+  return stripAnsi(value).length;
 }
 
 function compactReason(value) {
@@ -308,3 +344,112 @@ function truncate(value, width) {
   if (width <= 1) return '…';
   return `${text.slice(0, width - 1)}…`;
 }
+
+const TONES = {
+  header: ['\x1b[1m', '\x1b[0m'],
+  green: ['\x1b[32m', '\x1b[0m'],
+  yellow: ['\x1b[33m', '\x1b[0m'],
+  red: ['\x1b[31m', '\x1b[0m'],
+  muted: ['\x1b[2m', '\x1b[0m']
+};
+
+function toneText(text, tone, options = {}) {
+  if (!options.color || !tone || !TONES[tone]) return text;
+  const [open, close] = TONES[tone];
+  return `${open}${text}${close}`;
+}
+
+function statusTone(status) {
+  if (status === 'ok') return 'green';
+  if (status === 'partial') return 'yellow';
+  if (status === 'skipped') return 'yellow';
+  return 'red';
+}
+
+function percentTone(value, yellowAt, greenAt) {
+  if (value == null || !Number.isFinite(value)) return null;
+  if (value >= greenAt) return 'green';
+  if (value >= yellowAt) return 'yellow';
+  return 'red';
+}
+
+function cvTone(value) {
+  if (value == null || !Number.isFinite(value)) return null;
+  if (value <= 0.1) return 'green';
+  if (value <= 0.25) return 'yellow';
+  return 'red';
+}
+
+function thresholdTone(value, threshold) {
+  if (value == null || !Number.isFinite(value) || !Number.isFinite(threshold)) return null;
+  if (value <= threshold * 0.8) return 'green';
+  if (value <= threshold) return 'yellow';
+  return 'red';
+}
+
+function rankSummary(summary) {
+  return {
+    ttft: collectMetric(summary, (item) => item.ttft?.p50),
+    ttftP95: collectMetric(summary, (item) => item.ttft?.p95),
+    e2e: collectMetric(summary, (item) => item.latency?.p50),
+    e2eP95: collectMetric(summary, (item) => item.latency?.p95),
+    e2eP99: collectMetric(summary, (item) => item.latency?.p99),
+    tpot: collectMetric(summary, (item) => item.tpot?.p50),
+    tpotP95: collectMetric(summary, (item) => item.tpot?.p95),
+    userTps: collectMetric(summary, (item) => item.tokensPerSecond?.avg),
+    systemTps: collectMetric(summary, (item) => item.outputTokenThroughput),
+    rps: collectMetric(summary, (item) => item.rps),
+    decodeTps: collectMetric(summary, (item) => item.decodeTokensPerSecond?.avg)
+  };
+}
+
+function collectMetric(summary, accessor) {
+  return summary.map(accessor).filter((value) => Number.isFinite(value));
+}
+
+function rankTone(value, values) {
+  if (value == null || !Number.isFinite(value) || values.length === 0) return null;
+  if (values.length === 1) return 'green';
+  const min = Math.min(...values);
+  const max = Math.max(...values);
+  if (max === min) return 'green';
+  const position = (value - min) / (max - min);
+  if (position >= 0.67) return 'green';
+  if (position >= 0.34) return 'yellow';
+  return 'red';
+}
+
+function rankToneLower(value, values) {
+  if (value == null || !Number.isFinite(value) || values.length === 0) return null;
+  if (values.length === 1) return 'green';
+  const min = Math.min(...values);
+  const max = Math.max(...values);
+  if (max === min) return 'green';
+  const position = (max - value) / (max - min);
+  if (position >= 0.67) return 'green';
+  if (position >= 0.34) return 'yellow';
+  return 'red';
+}
+
+function metricTones(item, ranks, slo = {}) {
+  return {
+    ttft: thresholdTone(item.ttft?.p50, slo.ttftMs) || rankToneLower(item.ttft?.p50, ranks.ttft),
+    ttftP95: thresholdTone(item.ttft?.p95, slo.ttftMs) || rankToneLower(item.ttft?.p95, ranks.ttftP95),
+    e2e: thresholdTone(item.latency?.p50, slo.e2eMs) || rankToneLower(item.latency?.p50, ranks.e2e),
+    e2eP95: thresholdTone(item.latency?.p95, slo.e2eMs) || rankToneLower(item.latency?.p95, ranks.e2eP95),
+    e2eP99: thresholdTone(item.latency?.p99, slo.e2eMs) || rankToneLower(item.latency?.p99, ranks.e2eP99),
+    tpot: thresholdTone(item.tpot?.p50, slo.tpotMs) || rankToneLower(item.tpot?.p50, ranks.tpot),
+    tpotP95: thresholdTone(item.tpot?.p95, slo.tpotMs) || rankToneLower(item.tpot?.p95, ranks.tpotP95),
+    userTps: rankTone(item.tokensPerSecond?.avg, ranks.userTps),
+    systemTps: rankTone(item.outputTokenThroughput, ranks.systemTps),
+    rps: rankTone(item.rps, ranks.rps),
+    decodeTps: rankTone(item.decodeTokensPerSecond?.avg, ranks.decodeTps)
+  };
+}
+
+function worstTone(...tones) {
+  if (tones.includes('red')) return 'red';
+  if (tones.includes('yellow')) return 'yellow';
+  if (tones.includes('green')) return 'green';
+  return null;
+}