fm-bench 0.4.1 → 0.4.2

package/README.md

@@ -51,6 +51,7 @@ prompts 5 | runs 3 | concurrency 1,2 | stream on | measured 30 | failed 0 | skip
 ```sh
 fm-bench [run] [options]
 fm-bench models [options]
+fm-bench legend [options]
 fm-bench doctor [options]
 ```
 
@@ -58,6 +59,8 @@ fm-bench doctor [options]
 
 `models` lists discovered models, availability, descriptions, and quota output.
 
+`legend` explains every terminal table column, compact-card field, model-list column, and color rule. It does not run `fm`.
+
 `doctor` checks Node, macOS, `fm`, and model availability.
 
 ## Benchmark Options
@@ -72,6 +75,8 @@ fm-bench --models system --runs 5 --slo-ttft-ms 750 --slo-e2e-ms 4000
 fm-bench --prompt "Reply with exactly: ok" --runs 5
 fm-bench --prompt-file prompts.json --format json --out reports/bench.json
 fm-bench --format csv --out reports/bench.csv
+fm-bench legend
+fm-bench legend --json
 ```
 
 Useful flags:
@@ -146,6 +151,16 @@ Measured runs stream by default so `fm-bench` can capture TTFT and streaming smo
 
 Terminal output is responsive. Wide terminals show full scoreboard and detail tables, medium terminals show a tighter operating-point table, and narrow terminals switch to compact model cards. Use `--width` to preview a layout and `--ascii` for log systems that do not render Unicode borders well.
 
+## Table Legend
+
+Benchmark output stays focused on results and does not print the metric legend footer. Use `fm-bench legend` when you want definitions for every table column and color rule:
+
+```sh
+fm-bench legend
+fm-bench legend --json
+fm-bench legend --csv
+```
+
 ## Live Progress
 
 Interactive terminal runs show a single-line status indicator on stderr while prompts are loaded, models are inspected, tokens are counted, warmups run, and benchmark jobs complete. The final report still prints to stdout, so `--json`, `--csv`, and `--out` remain automation-friendly.

package/docs/methodology.md

@@ -16,6 +16,8 @@ The metric set follows common LLM inference benchmark practice:
 
 ## Metrics
 
+For a column-by-column terminal reference, run `fm-bench legend`.
+
 - `TTFT`: time from starting `fm respond` to the first streamed stdout chunk. This is a practical terminal-side proxy for time to first token.
 - `E2E latency`: time from starting `fm respond` until the process exits and the full response is captured.
 - `generation_ms`: `E2E - TTFT`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fm-bench",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "description": "Dynamic benchmark CLI for Apple's fm command on macOS 27+.",
   "type": "module",
   "bin": {

package/src/cli.js

@@ -4,7 +4,7 @@ import { inspectModels, runBenchmark } from './bench.js';
 import { runProcess } from './process.js';
 import { createProgress } from './progress.js';
 import { flattenResults, toCsv, writeReport } from './report.js';
-import { renderBenchmarkReport, renderModelsTable } from './table.js';
+import { legendEntries, renderBenchmarkReport, renderLegend, renderModelsTable } from './table.js';
 
 const require = createRequire(import.meta.url);
 const packageJson = require('../package.json');
@@ -22,6 +22,17 @@ export async function runCli(argv = process.argv.slice(2)) {
     return;
   }
 
+  if (parsed.command === 'legend') {
+    if (parsed.format === 'json') {
+      console.log(JSON.stringify(legendEntries(), null, 2));
+    } else if (parsed.format === 'csv') {
+      console.log(toCsv(legendEntries()));
+    } else {
+      console.log(renderLegend(renderOptions(parsed)));
+    }
+    return;
+  }
+
   if (parsed.command === 'doctor') {
     await runDoctor(parsed);
     return;
@@ -105,10 +116,14 @@ export function parseArgs(argv) {
   };
 
   const args = [...argv];
-  if (args[0] && !args[0].startsWith('-') && ['run', 'models', 'doctor', 'help'].includes(args[0])) {
+  if (args[0] && !args[0].startsWith('-') && ['run', 'models', 'doctor', 'legend', 'metrics', 'help'].includes(args[0])) {
     options.command = args.shift();
   }
 
+  if (options.command === 'metrics') {
+    options.command = 'legend';
+  }
+
   if (options.command === 'help') {
     options.help = true;
     return options;
@@ -359,8 +374,15 @@ Dynamic benchmark CLI for Apple's fm command on macOS 27+.
 Usage:
   fm-bench [run] [options]
   fm-bench models [options]
+  fm-bench legend [options]
   fm-bench doctor [options]
 
+Commands:
+  run                  Benchmark discovered or selected fm models
+  models               List discovered models and availability
+  legend               Explain every terminal table column and color rule
+  doctor               Check Node, macOS, fm, and model availability
+
 Run options:
   -m, --models <list>       Models to benchmark, comma-separated or repeated
   -r, --runs <n>            Runs per prompt/model (default: 1)
@@ -412,6 +434,7 @@ Examples:
   fm-bench --models system,pcc --runs 3 --profile stress
   fm-bench --profile client --sweep-concurrency 1,2 --request-rate 0.5
   fm-bench --prompt "Reply with exactly: ok" --json --out bench.json
+  fm-bench legend
   fm-bench models
   fm-bench doctor
 `;

package/src/table.js

@@ -56,12 +56,71 @@ export function renderBenchmarkReport(payload, options = {}) {
     lines.push(renderDetailTable(payload.summary, { ...options, mode, width, slo: payload.options.slo }));
   }
 
-  lines.push('');
-  lines.push(compactLegend(width));
-
   return lines.join('\n');
 }
 
+export function legendEntries() {
+  return [
+    entry('summary', 'C', 'Concurrency operating point for this row.', 'Higher C means more parallel fm respond processes.'),
+    entry('summary', 'MODEL', 'fm model name, such as system or pcc.', ''),
+    entry('summary', 'STATUS', 'Run status for this model and operating point.', 'ok = all measured jobs passed; partial = at least one failed; skipped = unavailable.'),
+    entry('summary', 'OK / OK/RUNS', 'Successful measured runs over attempted measured runs.', 'Green when no failures; yellow when partial.'),
+    entry('summary', 'SUCC / SUCCESS', 'Success rate: successful runs divided by attempted runs.', 'Green 100%, yellow >=95%, red <95%.'),
+    entry('summary', 'GOOD', 'Goodput rate: successful runs that also met every configured SLO.', 'Only appears when SLO flags are set. Green 100%, yellow >=80%, red <80%.'),
+    entry('summary', 'GOOD RPS', 'SLO-passing requests per second during this measured window.', 'Zero is shown when SLOs are set and no request meets them.'),
+    entry('summary', 'TTFT', 'Time to first streamed output chunk, p50.', 'Lower is better. Uses SLO threshold when set; otherwise relative ranking.'),
+    entry('summary', 'TTFT P95', '95th percentile time to first streamed output chunk.', 'Lower is better.'),
+    entry('summary', 'E2E', 'End-to-end latency, p50, from starting fm respond until full response exits.', 'Lower is better.'),
+    entry('summary', 'E2E P95', '95th percentile end-to-end latency.', 'Lower is better; this is usually the main interactive tail-latency signal.'),
+    entry('summary', 'TPOT', 'Time per output token after the first output token, p50.', 'Lower is better. Requires streaming and token counts.'),
+    entry('summary', 'TPOT P95', '95th percentile time per output token after first token.', 'Lower is better.'),
+    entry('summary', 'USER/S / USER T/S', 'Per-request output tokens per second.', 'Higher is better; relative ranking.'),
+    entry('summary', 'SYS/S / SYS T/S', 'Aggregate successful output-token throughput for the model row.', 'Higher is better; relative ranking.'),
+    entry('summary', 'RPS', 'Successful requests per second over the model row measured window.', 'Higher is better; relative ranking.'),
+    entry('summary', 'CV', 'Coefficient of variation for E2E latency: sample stddev divided by mean.', 'Lower is steadier. Green <=10%, yellow <=25%, red >25%.'),
+    entry('summary', 'NOTE', 'Short unavailable, skipped, or error note.', ''),
+    entry('detail', 'IN AVG / IN TOK AVG', 'Average prompt/input token count from fm token-count.', ''),
+    entry('detail', 'OUT AVG / OUT TOK AVG', 'Average output token count from fm token-count.', ''),
+    entry('detail', 'PREFILL/S / PREFILL TOK/S', 'Prompt tokens divided by TTFT seconds.', 'Higher is better; estimates prompt-processing speed for streaming runs.'),
+    entry('detail', 'DECODE/S / DECODE TOK/S', 'Output tokens after the first token divided by generation seconds.', 'Higher is better; requires streaming and token counts.'),
+    entry('detail', '2ND CHUNK', 'Delay between the first and second streamed stdout chunks, p50.', 'Lower is smoother startup. Chunk-based, not raw token telemetry.'),
+    entry('detail', 'CHUNK P95', '95th percentile gap between consecutive streamed stdout chunks.', 'Lower is smoother streaming.'),
+    entry('detail', 'E2E P99', '99th percentile end-to-end latency.', 'Lower is better; useful for worst-case UX.'),
+    entry('detail', 'E2E 95% CI', '95% confidence interval around mean E2E latency.', 'Narrower usually means a steadier estimate. Treat small samples carefully.'),
+    entry('detail', 'REPEAT', 'Share of repeated runs for a prompt that produced the most common normalized output hash.', 'Green 90%+, yellow 50%+, red below 50%. Blank when there are not repeated comparable outputs.'),
+    entry('detail', 'DESCRIPTION', 'Model description discovered from fm help.', ''),
+    entry('models', 'AVAILABLE', 'Whether fm available reports the model as usable on this machine right now.', ''),
+    entry('models', 'QUOTA', 'Raw fm quota-usage output or unavailable reason.', 'Mostly relevant for Private Cloud Compute.'),
+    entry('compact', 'GOOD / CV / TPOT / CHUNK', 'Compact output combines the same summary and detail metrics into model cards.', 'Same definitions and color rules as table columns.'),
+    entry('colors', 'GREEN', 'Passing, steadier, faster, or better within this benchmark context.', ''),
+    entry('colors', 'YELLOW', 'Marginal, partial, near a budget, or middle-ranked within this benchmark context.', ''),
+    entry('colors', 'RED', 'Failed budget, unstable, slower, lower, or worse within this benchmark context.', ''),
+    entry('colors', 'MUTED', 'Unavailable model, skipped value, or descriptive text.', '')
+  ];
+}
+
+export function renderLegend(options = {}) {
+  const width = terminalWidth(options);
+  const rows = legendEntries().map((item) => [
+    item.table,
+    item.column,
+    item.definition,
+    item.rule || '-'
+  ]);
+
+  if (options.compact || width < 88) {
+    return legendEntries().map((item) => {
+      const suffix = item.rule ? ` Rule: ${item.rule}` : '';
+      return truncate(`${item.table.toUpperCase()} ${item.column}: ${item.definition}${suffix}`, width);
+    }).join('\n');
+  }
+
+  return renderTable(['table', 'column', 'definition', 'rule'], rows, {
+    ...options,
+    maxCellWidth: Math.max(20, Math.floor((width - 48) / 2))
+  });
+}
+
 export function formatMs(value) {
   if (value == null || !Number.isFinite(value)) return '-';
   if (value >= 1000) return `${(value / 1000).toFixed(2)}s`;
@@ -250,13 +309,6 @@ function terminalWidth(options = {}) {
   return options.width || process.stdout.columns || 120;
 }
 
-function compactLegend(width) {
-  return [
-    'TTFT = first streamed output. E2E = full response. TPOT = post-first-token decode cadence.',
-    'CV = latency variation; lower is steadier: green <=10%, yellow <=25%, red >25%.'
-  ].map((line) => truncate(line, width)).join('\n');
-}
-
 function formatSlo(slo = {}) {
   const parts = [];
   if (slo.ttftMs) parts.push(`TTFT<=${formatMs(slo.ttftMs)}`);
@@ -265,6 +317,15 @@ function formatSlo(slo = {}) {
   return parts.length ? `SLO ${parts.join(',')}` : '';
 }
 
+function entry(table, column, definition, rule) {
+  return {
+    table,
+    column,
+    definition,
+    rule
+  };
+}
+
 const ASCII_TABLE = {
   topLeft: '+',
   topJoin: '+',