fm-bench 0.4.0 → 0.4.2

package/README.md

@@ -51,6 +51,7 @@ prompts 5 | runs 3 | concurrency 1,2 | stream on | measured 30 | failed 0 | skip
 ```sh
 fm-bench [run] [options]
 fm-bench models [options]
+fm-bench legend [options]
 fm-bench doctor [options]
 ```
 
@@ -58,6 +59,8 @@ fm-bench doctor [options]
 
 `models` lists discovered models, availability, descriptions, and quota output.
 
+`legend` explains every terminal table column, compact-card field, model-list column, and color rule. It does not run `fm`.
+
 `doctor` checks Node, macOS, `fm`, and model availability.
 
 ## Benchmark Options
@@ -72,6 +75,8 @@ fm-bench --models system --runs 5 --slo-ttft-ms 750 --slo-e2e-ms 4000
 fm-bench --prompt "Reply with exactly: ok" --runs 5
 fm-bench --prompt-file prompts.json --format json --out reports/bench.json
 fm-bench --format csv --out reports/bench.csv
+fm-bench legend
+fm-bench legend --json
 ```
 
 Useful flags:
@@ -146,6 +151,16 @@ Measured runs stream by default so `fm-bench` can capture TTFT and streaming smo
 
 Terminal output is responsive. Wide terminals show full scoreboard and detail tables, medium terminals show a tighter operating-point table, and narrow terminals switch to compact model cards. Use `--width` to preview a layout and `--ascii` for log systems that do not render Unicode borders well.
 
+## Table Legend
+
+Benchmark output stays focused on results and does not print the metric legend footer. Use `fm-bench legend` when you want definitions for every table column and color rule:
+
+```sh
+fm-bench legend
+fm-bench legend --json
+fm-bench legend --csv
+```
+
 ## Live Progress
 
 Interactive terminal runs show a single-line status indicator on stderr while prompts are loaded, models are inspected, tokens are counted, warmups run, and benchmark jobs complete. The final report still prints to stdout, so `--json`, `--csv`, and `--out` remain automation-friendly.
@@ -160,7 +175,7 @@ Table output uses semantic ANSI color on interactive terminals:
 - yellow: marginal, partial, or near a budget.
 - red: failing a budget, unstable, or slower/lower than peers.
 
-Success rate, goodput, repeatability, and CV use fixed benchmark thresholds. Throughput columns use relative ranking within the current run because “good” depends on the machine, model, prompt mix, and concurrency. TTFT, E2E, and TPOT use SLO thresholds when you pass `--slo-ttft-ms`, `--slo-e2e-ms`, or `--slo-tpot-ms`; otherwise they use lower-is-better relative ranking across the models and operating points in the report.
+Success rate, goodput, repeatability, and CV use fixed benchmark thresholds. CV is green at `<=10%`, yellow at `<=25%`, and red above `25%` because higher CV means less steady latency. Throughput columns use relative ranking within the current run because “good” depends on the machine, model, prompt mix, and concurrency. TTFT, E2E, and TPOT use SLO thresholds when you pass `--slo-ttft-ms`, `--slo-e2e-ms`, or `--slo-tpot-ms`; otherwise they use lower-is-better relative ranking across the models and operating points in the report.
 
 Use `--color` to force ANSI colors in captured logs, or `--no-color` for plain output. `NO_COLOR=1` disables automatic color and `FORCE_COLOR=1` enables it.
 

package/docs/methodology.md

@@ -16,6 +16,8 @@ The metric set follows common LLM inference benchmark practice:
 
 ## Metrics
 
+For a column-by-column terminal reference, run `fm-bench legend`.
+
 - `TTFT`: time from starting `fm respond` to the first streamed stdout chunk. This is a practical terminal-side proxy for time to first token.
 - `E2E latency`: time from starting `fm respond` until the process exits and the full response is captured.
 - `generation_ms`: `E2E - TTFT`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fm-bench",
-  "version": "0.4.0",
+  "version": "0.4.2",
   "description": "Dynamic benchmark CLI for Apple's fm command on macOS 27+.",
   "type": "module",
   "bin": {

package/src/cli.js

@@ -4,7 +4,7 @@ import { inspectModels, runBenchmark } from './bench.js';
 import { runProcess } from './process.js';
 import { createProgress } from './progress.js';
 import { flattenResults, toCsv, writeReport } from './report.js';
-import { renderBenchmarkReport, renderModelsTable } from './table.js';
+import { legendEntries, renderBenchmarkReport, renderLegend, renderModelsTable } from './table.js';
 
 const require = createRequire(import.meta.url);
 const packageJson = require('../package.json');
@@ -22,6 +22,17 @@ export async function runCli(argv = process.argv.slice(2)) {
     return;
   }
 
+  if (parsed.command === 'legend') {
+    if (parsed.format === 'json') {
+      console.log(JSON.stringify(legendEntries(), null, 2));
+    } else if (parsed.format === 'csv') {
+      console.log(toCsv(legendEntries()));
+    } else {
+      console.log(renderLegend(renderOptions(parsed)));
+    }
+    return;
+  }
+
   if (parsed.command === 'doctor') {
     await runDoctor(parsed);
     return;
@@ -105,10 +116,14 @@ export function parseArgs(argv) {
   };
 
   const args = [...argv];
-  if (args[0] && !args[0].startsWith('-') && ['run', 'models', 'doctor', 'help'].includes(args[0])) {
+  if (args[0] && !args[0].startsWith('-') && ['run', 'models', 'doctor', 'legend', 'metrics', 'help'].includes(args[0])) {
     options.command = args.shift();
   }
 
+  if (options.command === 'metrics') {
+    options.command = 'legend';
+  }
+
   if (options.command === 'help') {
     options.help = true;
     return options;
@@ -359,8 +374,15 @@ Dynamic benchmark CLI for Apple's fm command on macOS 27+.
 Usage:
   fm-bench [run] [options]
   fm-bench models [options]
+  fm-bench legend [options]
   fm-bench doctor [options]
 
+Commands:
+  run                  Benchmark discovered or selected fm models
+  models               List discovered models and availability
+  legend               Explain every terminal table column and color rule
+  doctor               Check Node, macOS, fm, and model availability
+
 Run options:
   -m, --models <list>       Models to benchmark, comma-separated or repeated
   -r, --runs <n>            Runs per prompt/model (default: 1)
@@ -412,6 +434,7 @@ Examples:
   fm-bench --models system,pcc --runs 3 --profile stress
   fm-bench --profile client --sweep-concurrency 1,2 --request-rate 0.5
   fm-bench --prompt "Reply with exactly: ok" --json --out bench.json
+  fm-bench legend
   fm-bench models
   fm-bench doctor
 `;

package/src/table.js

@@ -56,12 +56,71 @@ export function renderBenchmarkReport(payload, options = {}) {
     lines.push(renderDetailTable(payload.summary, { ...options, mode, width, slo: payload.options.slo }));
   }
 
-  lines.push('');
-  lines.push(compactLegend(width));
-
   return lines.join('\n');
 }
 
+export function legendEntries() {
+  return [
+    entry('summary', 'C', 'Concurrency operating point for this row.', 'Higher C means more parallel fm respond processes.'),
+    entry('summary', 'MODEL', 'fm model name, such as system or pcc.', ''),
+    entry('summary', 'STATUS', 'Run status for this model and operating point.', 'ok = all measured jobs passed; partial = at least one failed; skipped = unavailable.'),
+    entry('summary', 'OK / OK/RUNS', 'Successful measured runs over attempted measured runs.', 'Green when no failures; yellow when partial.'),
+    entry('summary', 'SUCC / SUCCESS', 'Success rate: successful runs divided by attempted runs.', 'Green 100%, yellow >=95%, red <95%.'),
+    entry('summary', 'GOOD', 'Goodput rate: successful runs that also met every configured SLO.', 'Only appears when SLO flags are set. Green 100%, yellow >=80%, red <80%.'),
+    entry('summary', 'GOOD RPS', 'SLO-passing requests per second during this measured window.', 'Zero is shown when SLOs are set and no request meets them.'),
+    entry('summary', 'TTFT', 'Time to first streamed output chunk, p50.', 'Lower is better. Uses SLO threshold when set; otherwise relative ranking.'),
+    entry('summary', 'TTFT P95', '95th percentile time to first streamed output chunk.', 'Lower is better.'),
+    entry('summary', 'E2E', 'End-to-end latency, p50, from starting fm respond until full response exits.', 'Lower is better.'),
+    entry('summary', 'E2E P95', '95th percentile end-to-end latency.', 'Lower is better; this is usually the main interactive tail-latency signal.'),
+    entry('summary', 'TPOT', 'Time per output token after the first output token, p50.', 'Lower is better. Requires streaming and token counts.'),
+    entry('summary', 'TPOT P95', '95th percentile time per output token after first token.', 'Lower is better.'),
+    entry('summary', 'USER/S / USER T/S', 'Per-request output tokens per second.', 'Higher is better; relative ranking.'),
+    entry('summary', 'SYS/S / SYS T/S', 'Aggregate successful output-token throughput for the model row.', 'Higher is better; relative ranking.'),
+    entry('summary', 'RPS', 'Successful requests per second over the model row measured window.', 'Higher is better; relative ranking.'),
+    entry('summary', 'CV', 'Coefficient of variation for E2E latency: sample stddev divided by mean.', 'Lower is steadier. Green <=10%, yellow <=25%, red >25%.'),
+    entry('summary', 'NOTE', 'Short unavailable, skipped, or error note.', ''),
+    entry('detail', 'IN AVG / IN TOK AVG', 'Average prompt/input token count from fm token-count.', ''),
+    entry('detail', 'OUT AVG / OUT TOK AVG', 'Average output token count from fm token-count.', ''),
+    entry('detail', 'PREFILL/S / PREFILL TOK/S', 'Prompt tokens divided by TTFT seconds.', 'Higher is better; estimates prompt-processing speed for streaming runs.'),
+    entry('detail', 'DECODE/S / DECODE TOK/S', 'Output tokens after the first token divided by generation seconds.', 'Higher is better; requires streaming and token counts.'),
+    entry('detail', '2ND CHUNK', 'Delay between the first and second streamed stdout chunks, p50.', 'Lower is smoother startup. Chunk-based, not raw token telemetry.'),
+    entry('detail', 'CHUNK P95', '95th percentile gap between consecutive streamed stdout chunks.', 'Lower is smoother streaming.'),
+    entry('detail', 'E2E P99', '99th percentile end-to-end latency.', 'Lower is better; useful for worst-case UX.'),
+    entry('detail', 'E2E 95% CI', '95% confidence interval around mean E2E latency.', 'Narrower usually means a steadier estimate. Treat small samples carefully.'),
+    entry('detail', 'REPEAT', 'Share of repeated runs for a prompt that produced the most common normalized output hash.', 'Green 90%+, yellow 50%+, red below 50%. Blank when there are not repeated comparable outputs.'),
+    entry('detail', 'DESCRIPTION', 'Model description discovered from fm help.', ''),
+    entry('models', 'AVAILABLE', 'Whether fm available reports the model as usable on this machine right now.', ''),
+    entry('models', 'QUOTA', 'Raw fm quota-usage output or unavailable reason.', 'Mostly relevant for Private Cloud Compute.'),
+    entry('compact', 'GOOD / CV / TPOT / CHUNK', 'Compact output combines the same summary and detail metrics into model cards.', 'Same definitions and color rules as table columns.'),
+    entry('colors', 'GREEN', 'Passing, steadier, faster, or better within this benchmark context.', ''),
+    entry('colors', 'YELLOW', 'Marginal, partial, near a budget, or middle-ranked within this benchmark context.', ''),
+    entry('colors', 'RED', 'Failed budget, unstable, slower, lower, or worse within this benchmark context.', ''),
+    entry('colors', 'MUTED', 'Unavailable model, skipped value, or descriptive text.', '')
+  ];
+}
+
+export function renderLegend(options = {}) {
+  const width = terminalWidth(options);
+  const rows = legendEntries().map((item) => [
+    item.table,
+    item.column,
+    item.definition,
+    item.rule || '-'
+  ]);
+
+  if (options.compact || width < 88) {
+    return legendEntries().map((item) => {
+      const suffix = item.rule ? ` Rule: ${item.rule}` : '';
+      return truncate(`${item.table.toUpperCase()} ${item.column}: ${item.definition}${suffix}`, width);
+    }).join('\n');
+  }
+
+  return renderTable(['table', 'column', 'definition', 'rule'], rows, {
+    ...options,
+    maxCellWidth: Math.max(20, Math.floor((width - 48) / 2))
+  });
+}
+
 export function formatMs(value) {
   if (value == null || !Number.isFinite(value)) return '-';
   if (value >= 1000) return `${(value / 1000).toFixed(2)}s`;
@@ -250,11 +309,6 @@ function terminalWidth(options = {}) {
   return options.width || process.stdout.columns || 120;
 }
 
-function compactLegend(width) {
-  const text = 'TTFT = first streamed output. E2E = full response. TPOT = post-first-token decode cadence. CV = lower is steadier.';
-  return truncate(text, width);
-}
-
 function formatSlo(slo = {}) {
   const parts = [];
   if (slo.ttftMs) parts.push(`TTFT<=${formatMs(slo.ttftMs)}`);
@@ -263,6 +317,15 @@ function formatSlo(slo = {}) {
   return parts.length ? `SLO ${parts.join(',')}` : '';
 }
 
+function entry(table, column, definition, rule) {
+  return {
+    table,
+    column,
+    definition,
+    rule
+  };
+}
+
 const ASCII_TABLE = {
   topLeft: '+',
   topJoin: '+',