fm-bench 0.6.2 → 0.7.0

package/src/capabilities.js

@@ -0,0 +1,196 @@
+// Capability detection for the installed `fm` CLI.
+//
+// fm-bench talks to whatever `fm` build is on the machine, and Apple has
+// changed subcommand names and flags between releases (for example
+// `count-tokens` replaces the older `token-count`). Everything downstream
+// reads this detection result instead of hardcoding subcommand names, so a
+// compatible `fm` build is supported without a code change and an
+// incompatible one is reported instead of producing wrong numbers.
+
+import crypto from 'node:crypto';
+import { stripAnsi } from './ansi.js';
+import { parseModelsFromHelp } from './fm-help.js';
+import { runProcess } from './process.js';
+
+const SECTION_HEADER = /^\s*[A-Z][A-Z0-9 /-]+\s*$/;
+
+/**
+ * Parse the COMMANDS section of `fm --help`.
+ * @param {string} helpText
+ * @returns {string[]}
+ */
+export function parseCommandsFromHelp(helpText = '') {
+  const lines = stripAnsi(helpText).split(/\r?\n/);
+  const commands = [];
+  let inCommands = false;
+
+  for (const line of lines) {
+    if (/^\s*COMMANDS\s*$/.test(line)) {
+      inCommands = true;
+      continue;
+    }
+    if (inCommands && SECTION_HEADER.test(line)) {
+      inCommands = false;
+    }
+    if (!inCommands) continue;
+
+    const match = line.match(/^\s{2,}([a-z][a-z0-9-]*)\s{2,}\S/);
+    if (match) commands.push(match[1]);
+  }
+
+  return commands;
+}
+
+/**
+ * True when a long flag appears in `fm` help text. Apple prints boolean flags
+ * with a negatable form (`--[no-]stream`), so accept that shape too.
+ * @param {string} helpText
+ * @param {string} flag e.g. "--use-case"
+ */
+export function hasFlagInHelp(helpText, flag) {
+  const clean = stripAnsi(helpText);
+  if (new RegExp(`${escapeRegExp(flag)}\\b`).test(clean)) return true;
+  if (flag.startsWith('--no-')) {
+    return clean.includes(`[no-]${flag.slice('--no-'.length)}`);
+  }
+  return false;
+}
+
+export function helpDigest(helpText = '') {
+  const text = stripAnsi(helpText).trim();
+  if (!text) return null;
+  return crypto.createHash('sha256').update(text).digest('hex').slice(0, 16);
+}
+
+/**
+ * Pick the subcommand this `fm` build exposes for token counting.
+ * @returns {string|null}
+ */
+export function resolveTokenCountCommand(commands = []) {
+  if (commands.includes('count-tokens')) return 'count-tokens';
+  if (commands.includes('token-count')) return 'token-count';
+  return null;
+}
+
+function buildFeatures(helpText, respondHelpText, commands) {
+  const tokenCountCommand = resolveTokenCountCommand(commands);
+  const respondHelp = respondHelpText || '';
+  return {
+    tokenCounting: tokenCountCommand != null,
+    tokenCountCommand,
+    quota: commands.includes('quota-usage'),
+    streaming: hasFlagInHelp(respondHelp, '--no-stream') || hasFlagInHelp(respondHelp, '--stream'),
+    modelSelection: hasFlagInHelp(respondHelp, '--model'),
+    instructions: hasFlagInHelp(respondHelp, '--instructions'),
+    greedy: hasFlagInHelp(respondHelp, '--greedy'),
+    useCase: hasFlagInHelp(respondHelp, '--use-case'),
+    guardrails: hasFlagInHelp(respondHelp, '--guardrails'),
+    images: hasFlagInHelp(respondHelp, '--image'),
+    tools: hasFlagInHelp(respondHelp, '--tool'),
+    structuredOutput: hasFlagInHelp(respondHelp, '--schema'),
+    server: commands.includes('serve')
+  };
+}
+
+function capabilityWarnings(commands, features, models) {
+  const warnings = [];
+  if (!features.tokenCounting) {
+    warnings.push('this fm build exposes no token-counting command, so token counts and token throughput are unavailable');
+  }
+  if (!features.quota) {
+    warnings.push('this fm build exposes no quota command, so quota is not reported');
+  }
+  if (!features.streaming) {
+    warnings.push('this fm build does not document a streaming flag, so TTFT cannot be measured');
+  }
+  if (models.length === 0) {
+    warnings.push('could not discover any models from fm help output');
+  }
+  if (commands.length === 0) {
+    warnings.push('could not read the command list from fm --help');
+  }
+  return warnings;
+}
+
+/**
+ * Probe the installed `fm` binary once and describe what it can do.
+ *
+ * @param {string} fmBin
+ * @param {{ timeoutMs?: number, env?: NodeJS.ProcessEnv, help?: { text: string } }} [options]
+ */
+export async function detectFmCapabilities(fmBin, options = {}) {
+  const timeoutMs = options.timeoutMs ?? 10_000;
+  const env = options.env ?? process.env;
+  let helpText = options.help?.text ?? null;
+
+  if (helpText == null) {
+    const help = await runProcess(fmBin, ['--help'], { timeoutMs, env });
+    if (help.error) {
+      return {
+        ok: false,
+        bin: fmBin,
+        error: `Unable to execute ${fmBin}: ${help.stderr || help.error.message}`,
+        commands: [],
+        models: [],
+        features: buildFeatures('', '', []),
+        digest: null,
+        help: '',
+        warnings: [`cannot execute ${fmBin}`]
+      };
+    }
+    helpText = `${help.stdout}${help.stderr}`;
+  }
+
+  const cleanHelp = stripAnsi(helpText);
+  const commands = parseCommandsFromHelp(cleanHelp);
+
+  let respondHelpText = '';
+  if (commands.includes('respond')) {
+    const respondHelp = await runProcess(fmBin, ['respond', '--help'], { timeoutMs, env });
+    if (!respondHelp.error) respondHelpText = `${respondHelp.stdout}${respondHelp.stderr}`;
+  }
+
+  const features = buildFeatures(cleanHelp, respondHelpText, commands);
+  let models = parseModelsFromHelp(cleanHelp);
+
+  if (models.length === 0 && commands.includes('available')) {
+    models = await discoverModelsFromAvailability(fmBin, { ...options, env });
+  }
+
+  return {
+    ok: commands.length > 0,
+    bin: fmBin,
+    commands,
+    models,
+    features,
+    digest: helpDigest(cleanHelp),
+    help: cleanHelp,
+    warnings: capabilityWarnings(commands, features, models)
+  };
+}
+
+/**
+ * Fallback discovery: ask `fm available` without a model filter and read the
+ * model names it reports. Used when `fm --help` has no MODELS section.
+ */
+async function discoverModelsFromAvailability(fmBin, options = {}) {
+  const result = await runProcess(fmBin, ['available'], {
+    timeoutMs: options.timeoutMs ?? 15_000,
+    env: options.env ?? process.env
+  });
+  if (result.error) return [];
+  const text = stripAnsi(`${result.stdout}${result.stderr}`);
+  const models = new Map();
+  for (const line of text.split(/\r?\n/)) {
+    const match = line.match(/^\s*([A-Za-z0-9._-]+)(?:\s+model)?\s+(?:is\s+)?(available|unavailable|not available)\b/i);
+    if (!match) continue;
+    const name = match[1].toLowerCase();
+    if (['the', 'no', 'model', 'system'].includes(name) && name !== 'system') continue;
+    models.set(name, { name, description: '' });
+  }
+  return [...models.values()];
+}
+
+function escapeRegExp(value) {
+  return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}

package/src/cli.js

@@ -3,18 +3,34 @@ import { createRequire } from 'node:module';
 import { inspectModels, runBenchmark } from './bench.js';
 import { diffReports, renderCompareReport } from './compare.js';
 import { renderHtmlReport } from './export.js';
+import { formatCapabilitySummary } from './metrics.js';
 import { validateReport } from './schema.js';
 import { loadHistory, renderHistoryReport } from './history.js';
+import { detectMacosVersion, evaluateMacosSupport, formatMacosRequirementError, MIN_SUPPORTED_MACOS, parseMacosVersion } from './macos.js';
 import { runProcess } from './process.js';
 import { createProgress } from './progress.js';
 import { flattenResults, toCsv, writeReport } from './report.js';
 import { parseBatteryOutput, parseThermalOutput } from './system.js';
-import { legendEntries, renderBenchmarkReport, renderLatencyHistogram, renderLegend, renderModelsTable } from './table.js';
+import { legendEntries, renderBenchmarkReport, renderLatencyHistogram, renderLegend, renderModelsReport } from './table.js';
 
 const require = createRequire(import.meta.url);
 const packageJson = require('../package.json');
 
-export async function runCli(argv = process.argv.slice(2)) {
+/** Usage / environment error: bad flags, missing arguments, unsupported host. */
+function usageError(message) {
+  const error = new Error(message);
+  error.exitCode = 2;
+  return error;
+}
+
+/** Operational failure: invalid report data, failed benchmark gate. */
+function operationalError(message) {
+  const error = new Error(message);
+  error.exitCode = 1;
+  return error;
+}
+
+export async function runCli(argv = process.argv.slice(2), env = {}) {
   const parsed = parseArgs(argv);
 
   if (parsed.help) {
@@ -27,6 +43,8 @@ export async function runCli(argv = process.argv.slice(2)) {
     return;
   }
 
+  await assertSupportedMacos(parsed, env);
+
   if (parsed.command === 'legend') {
     if (parsed.format === 'json') {
       console.log(JSON.stringify(legendEntries(), null, 2));
@@ -66,9 +84,14 @@ export async function runCli(argv = process.argv.slice(2)) {
   if (parsed.command === 'models') {
     const inspection = await inspectModels(parsed);
     if (parsed.format === 'json') {
+      // Shape stays a plain array so existing automation keeps working;
+      // full capability detail lives in `doctor --json` and in report payloads.
       console.log(JSON.stringify(inspection.models, null, 2));
     } else {
-      console.log(renderModelsTable(inspection.models, renderOptions(parsed)));
+      console.log(renderModelsReport(inspection.models, {
+        ...renderOptions(parsed),
+        capabilities: inspection.capabilities
+      }));
     }
     return;
   }
@@ -145,14 +168,35 @@ export async function runCli(argv = process.argv.slice(2)) {
     if (!ciResult.passed) {
       const reasons = ciResult.reasons.join('; ');
       console.error(`fm-bench ci: FAIL — ${reasons}`);
-      const error = new Error(`CI checks failed: ${reasons}`);
-      error.exitCode = 1;
-      throw error;
+      throw operationalError(`CI checks failed: ${reasons}`);
     }
     console.error(`fm-bench ci: PASS`);
   }
 }
 
+// Offline report tools stay usable anywhere. `doctor` is also allowed through so
+// it can print the detected version and the latest supported macOS when the host
+// is too old. Only commands that launch `fm` benchmarks are hard-gated.
+const SKIP_MACOS_GATE = new Set(['compare', 'history', 'validate', 'export', 'legend', 'doctor']);
+
+async function assertSupportedMacos(parsed, env = {}) {
+  if (SKIP_MACOS_GATE.has(parsed.command)) return;
+
+  // The gate guards the default fm discovery path, because Apple only ships the
+  // CLI from macOS 27. An explicitly configured binary is honoured on any host;
+  // the capability probe below still fails with exit 2 if it is unusable.
+  if (parsed.fmBin || env.FM_BIN || process.env.FM_BIN) return;
+
+  const evaluation = evaluateMacosSupport(
+    env.platform ?? process.platform,
+    parseMacosVersion(await detectMacosVersion(env))
+  );
+  if (evaluation.supported) return;
+
+  const error = usageError(formatMacosRequirementError(evaluation));
+  throw error;
+}
+
 function evaluateCi(payload) {
   const reasons = [];
   const totalFailed = payload.summary.reduce((sum, item) => sum + item.failures, 0);
@@ -293,7 +337,7 @@ export function parseArgs(argv) {
       case '--profile':
         options.profile = requireValue(arg, args);
         if (!['quick', 'standard', 'interactive', 'throughput', 'client', 'stress', 'reasoning', 'coding', 'creative'].includes(options.profile)) {
-          throw new Error(`--profile must be one of: quick, standard, interactive, throughput, client, stress, reasoning, coding, creative`);
+          throw usageError('--profile must be one of: quick, standard, interactive, throughput, client, stress, reasoning, coding, creative');
         }
         break;
       case '-i':
@@ -330,7 +374,7 @@ export function parseArgs(argv) {
       case '--format':
         options.format = requireValue(arg, args);
         if (!['table', 'json', 'csv'].includes(options.format)) {
-          throw new Error('--format must be one of: table, json, csv');
+          throw usageError('--format must be one of: table, json, csv');
         }
         break;
       case '--ascii':
@@ -403,7 +447,7 @@ export function parseArgs(argv) {
         break;
       default:
         if (arg.startsWith('-')) {
-          throw new Error(`Unknown option: ${arg}`);
+          throw usageError(`Unknown option: ${arg}`);
         }
         if (options.command === 'compare') {
           options.compareFiles.push(arg);
@@ -442,30 +486,27 @@ async function runHistory(options, renderOpts) {
 async function runCompare(options, renderOpts) {
   const files = options.compareFiles;
   if (files.length < 2) {
-    throw new Error('compare requires two JSON report files: fm-bench compare before.json after.json');
+    throw usageError('compare requires two JSON report files: fm-bench compare before.json after.json');
   }
   if (files.length > 2) {
-    throw new Error('compare accepts exactly two JSON report files');
+    throw usageError('compare accepts exactly two JSON report files');
   }
 
   const [beforePath, afterPath] = files;
-  const [beforeText, afterText] = await Promise.all([
-    fs.readFile(beforePath, 'utf8'),
-    fs.readFile(afterPath, 'utf8')
-  ]);
-
-  let before, after;
+  let beforeText;
+  let afterText;
   try {
-    before = JSON.parse(beforeText);
-  } catch {
-    throw new Error(`Cannot parse ${beforePath} as JSON`);
-  }
-  try {
-    after = JSON.parse(afterText);
-  } catch {
-    throw new Error(`Cannot parse ${afterPath} as JSON`);
+    [beforeText, afterText] = await Promise.all([
+      fs.readFile(beforePath, 'utf8'),
+      fs.readFile(afterPath, 'utf8')
+    ]);
+  } catch (error) {
+    throw operationalError(`Cannot read report: ${error.message}`);
   }
 
+  const before = parseReportJson(beforeText, beforePath);
+  const after = parseReportJson(afterText, afterPath);
+
   const diff = diffReports(before, after);
 
   if (options.format === 'json') {
@@ -480,64 +521,77 @@ async function runCompare(options, renderOpts) {
   }
 
   if (options.strictCompare && diff.compatibility && !diff.compatibility.suiteMatch) {
-    const error = new Error('compare: benchmark suites differ (--strict)');
-    error.exitCode = 2;
-    throw error;
+    throw usageError('compare: benchmark suites differ (--strict)');
+  }
+}
+
+function parseReportJson(text, filePath) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    throw operationalError(`Cannot parse ${filePath} as JSON`);
   }
 }
 
 async function runValidate(options) {
   const files = options.validateFiles;
   if (files.length === 0) {
-    throw new Error('validate requires at least one JSON report: fm-bench validate report.json');
+    throw usageError('validate requires at least one JSON report: fm-bench validate report.json');
   }
 
-  let failed = 0;
+  const results = [];
   for (const filePath of files) {
     let parsed;
     try {
-      const text = await fs.readFile(filePath, 'utf8');
-      parsed = JSON.parse(text);
-    } catch {
-      console.error(`invalid  ${filePath}  (cannot read or parse JSON)`);
-      failed += 1;
+      parsed = JSON.parse(await fs.readFile(filePath, 'utf8'));
+    } catch (error) {
+      results.push({ file: filePath, ok: false, errors: [error.code === 'ENOENT'
+        ? 'file not found'
+        : 'cannot read or parse JSON'] });
       continue;
     }
     const result = validateReport(parsed);
-    if (result.ok) {
-      const id = result.report.reportId ?? '—';
-      const schema = result.report.schemaVersion ?? 'legacy';
-      console.log(`ok       ${filePath}  schema=${schema} id=${id}`);
-    } else {
-      console.error(`invalid  ${filePath}  ${result.errors.join('; ')}`);
-      failed += 1;
+    results.push(result.ok
+      ? { file: filePath, ok: true, schema: result.report.schemaVersion ?? 'legacy', id: result.report.reportId ?? null }
+      : { file: filePath, ok: false, errors: result.errors });
+  }
+
+  if (options.format === 'json') {
+    console.log(JSON.stringify({ ok: results.every((item) => item.ok), files: results }, null, 2));
+  } else {
+    for (const item of results) {
+      if (item.ok) {
+        console.log(`ok       ${item.file}  schema=${item.schema} id=${item.id ?? '—'}`);
+      } else {
+        console.error(`invalid  ${item.file}  ${item.errors.join('; ')}`);
+      }
     }
   }
 
+  const failed = results.filter((item) => !item.ok).length;
   if (failed > 0) {
-    const error = new Error(`${failed} report(s) failed validation`);
-    error.exitCode = 1;
-    throw error;
+    throw operationalError(`${failed} report(s) failed validation`);
   }
 }
 
 async function runExport(options) {
   const files = options.validateFiles;
   if (files.length === 0) {
-    throw new Error('export requires a JSON report: fm-bench export report.json [-o out.html]');
+    throw usageError('export requires a JSON report: fm-bench export report.json [-o out.html]');
   }
 
   const filePath = files[0];
-  const text = await fs.readFile(filePath, 'utf8');
   let report;
   try {
-    report = JSON.parse(text);
-  } catch {
-    throw new Error(`Cannot parse ${filePath} as JSON`);
+    report = JSON.parse(await fs.readFile(filePath, 'utf8'));
+  } catch (error) {
+    throw error.code === 'ENOENT'
+      ? operationalError(`Cannot read ${filePath}: file not found`)
+      : operationalError(`Cannot parse ${filePath} as JSON`);
   }
   const validation = validateReport(report);
   if (!validation.ok) {
-    throw new Error(`Not a valid fm-bench report: ${validation.errors.join('; ')}`);
+    throw operationalError(`Not a valid fm-bench report: ${validation.errors.join('; ')}`);
   }
 
   const html = renderHtmlReport(validation.report);
@@ -550,15 +604,19 @@ async function runExport(options) {
 }
 
 async function runDoctor(options) {
+  const json = options.format === 'json';
   const checks = [];
   checks.push(['node', process.version, true]);
   checks.push(['platform', `${process.platform}/${process.arch}`, process.platform === 'darwin']);
 
-  const swVers = await runProcess('sw_vers', [], { timeoutMs: 5_000 });
-  const macOS = swVers.stdout || swVers.stderr;
-  const versionMatch = macOS.match(/ProductVersion:\s*([0-9.]+)/);
-  const major = versionMatch ? Number.parseInt(versionMatch[1].split('.')[0], 10) : null;
-  checks.push(['macOS', versionMatch?.[1] || 'unknown', major == null || major >= 27]);
+  const macOS = await detectMacosVersion();
+  const parsedVersion = parseMacosVersion(macOS);
+  const support = evaluateMacosSupport(process.platform, parsedVersion);
+  checks.push(['macOS', parsedVersion?.version || 'unknown', support.supported]);
+  if (!support.supported) {
+    checks.push(['macOS support', support.reason, false]);
+    checks.push(['latest supported', support.latestSupported, false]);
+  }
 
   const hwModel = await runProcess('sysctl', ['-n', 'hw.model'], { timeoutMs: 3_000 });
   const hwModelStr = (hwModel.stdout || '').trim();
@@ -595,41 +653,83 @@ async function runDoctor(options) {
     checks.push(['battery', `${pct} (${source})`, ok]);
   }
 
-  const inspection = await inspectModels(options);
-  checks.push(['fm', inspection.fmBin, inspection.models.length > 0]);
-  for (const model of inspection.models) {
-    checks.push([`model:${model.name}`, model.available ? 'available' : model.reason || 'unavailable', model.available]);
+  let models = [];
+  let capabilities = null;
+  try {
+    const inspection = await inspectModels(options);
+    models = inspection.models;
+    capabilities = inspection.capabilities;
+    checks.push(['fm', inspection.fmBin, capabilities.ok]);
+    if (capabilities.digest) checks.push(['fm help digest', capabilities.digest, true]);
+    checks.push(['fm commands', capabilities.commands.join(', ') || 'none found', capabilities.commands.length > 0]);
+    checks.push(['fm token counting', capabilities.features.tokenCounting ? `yes (${capabilities.features.tokenCountCommand})` : 'no', capabilities.features.tokenCounting]);
+    checks.push(['fm streaming', capabilities.features.streaming ? 'yes' : 'no', capabilities.features.streaming]);
+    checks.push(['fm quota', capabilities.features.quota ? 'yes' : 'no (not exposed by this build)', true]);
+    for (const model of inspection.models) {
+      checks.push([`model:${model.name}`, model.available ? 'available' : model.reason || 'unavailable', model.available]);
+    }
+  } catch (error) {
+    checks.push(['fm', error.message || String(error), false]);
   }
 
-  const lines = checks.map(([name, detail, ok]) => `${ok ? 'ok  ' : 'warn'} ${name.padEnd(16)} ${String(detail).replace(/\s+/g, ' ').trim()}`);
-  console.log(lines.join('\n'));
+  const payload = {
+    checks: checks.map(([name, detail, ok]) => ({ name, detail: String(detail), ok })),
+    models,
+    capabilities: capabilities ? describeCapabilities(capabilities) : null
+  };
+
+  if (json) {
+    console.log(JSON.stringify(payload, null, 2));
+  } else {
+    const lines = checks.map(([name, detail, ok]) => `${ok ? 'ok  ' : 'warn'} ${name.padEnd(16)} ${String(detail).replace(/\s+/g, ' ').trim()}`);
+    console.log(lines.join('\n'));
+    if (capabilities) {
+      console.log('');
+      console.log(`fm capabilities: ${formatCapabilitySummary(capabilities)}`);
+      for (const warning of capabilities.warnings) {
+        console.log(`limit: ${warning}`);
+      }
+    }
+  }
 
   if (options.out) {
-    await fs.writeFile(options.out, `${JSON.stringify({ checks, models: inspection.models }, null, 2)}\n`, 'utf8');
+    await fs.writeFile(options.out, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
   }
 }
 
+function describeCapabilities(capabilities) {
+  return {
+    bin: capabilities.bin,
+    ok: capabilities.ok,
+    digest: capabilities.digest,
+    commands: capabilities.commands,
+    models: capabilities.models,
+    features: capabilities.features,
+    warnings: capabilities.warnings
+  };
+}
+
 function requireValue(option, args) {
   const value = args.shift();
-  if (value == null || value === '') throw new Error(`${option} requires a value`);
+  if (value == null || value === '') throw usageError(`${option} requires a value`);
   return value;
 }
 
 function parsePositiveInt(value, option) {
   const parsed = Number.parseInt(value, 10);
-  if (!Number.isInteger(parsed) || parsed < 1) throw new Error(`${option} must be a positive integer`);
+  if (!Number.isInteger(parsed) || parsed < 1) throw usageError(`${option} must be a positive integer`);
   return parsed;
 }
 
 function parsePositiveNumber(value, option) {
   const parsed = Number.parseFloat(value);
-  if (!Number.isFinite(parsed) || parsed <= 0) throw new Error(`${option} must be a positive number`);
+  if (!Number.isFinite(parsed) || parsed <= 0) throw usageError(`${option} must be a positive number`);
   return parsed;
 }
 
 function parseNonNegativeInt(value, option) {
   const parsed = Number.parseInt(value, 10);
-  if (!Number.isInteger(parsed) || parsed < 0) throw new Error(`${option} must be a non-negative integer`);
+  if (!Number.isInteger(parsed) || parsed < 0) throw usageError(`${option} must be a non-negative integer`);
   return parsed;
 }
 
@@ -639,7 +739,7 @@ function parsePositiveIntList(value, option) {
     .map((item) => item.trim())
     .filter(Boolean)
     .map((item) => parsePositiveInt(item, option));
-  if (parsed.length === 0) throw new Error(`${option} requires at least one positive integer`);
+  if (parsed.length === 0) throw usageError(`${option} requires at least one positive integer`);
   return parsed;
 }
 
@@ -669,27 +769,28 @@ function resolveProgress(parsed) {
 function helpText() {
   return `fm-bench ${packageJson.version}
 
-Dynamic benchmark CLI for Apple's fm command on macOS 27+.
+Dynamic benchmark CLI for Apple's fm command on macOS ${MIN_SUPPORTED_MACOS}+.
 
 Usage:
   fm-bench [run] [options]
   fm-bench models [options]
   fm-bench compare <before.json> <after.json> [options]
   fm-bench history [dir] [options]
-  fm-bench validate <report.json> [more...]
+  fm-bench validate <report.json> [more...] [options]
   fm-bench export <report.json> [-o report.html]
   fm-bench legend [options]
   fm-bench doctor [options]
 
 Commands:
   run                  Benchmark discovered or selected fm models
-  models               List discovered models and availability
+  models               List discovered models, availability, and fm capabilities
   compare              Compare two saved JSON reports and show metric deltas
   history              Show a trend table from all fm-bench JSON reports in a directory
   validate             Verify report JSON structure (schema v1)
   export               Render a shareable standalone HTML report from JSON
-  legend               Explain every terminal table column and color rule
-  doctor               Check Node, macOS, fm, and model availability
+  legend               Explain every terminal table column, color rule, and metric source
+  doctor               Check Node, macOS, fm capabilities, and model availability
+  metrics              Alias for legend
 
 Run options:
   -m, --models <list>       Models to benchmark, comma-separated or repeated
@@ -744,15 +845,32 @@ Compare:
 
 Environment:
       --fm-bin <path>       fm binary to execute (default: FM_BIN or fm)
+      --                    Treat the rest of the line as the prompt
   -h, --help                Show this help
       --version             Print version
 
+Machine-readable output:
+  --json and --csv write only data to stdout; progress and diagnostics go to stderr.
+  "validate --json" prints { ok, files }; "doctor --json" prints the full check list.
+
+Exit codes:
+  0   success
+  1   operational failure (failed runs with --ci, invalid reports, fm errors)
+  2   usage or environment error (bad flags, missing arguments, unsupported macOS, fm not found)
+
+Capability detection:
+  fm-bench probes "fm --help" and "fm respond --help" once per run. Metrics the
+  installed fm cannot supply are reported as unavailable instead of being
+  guessed, and unsupported models are refused before any benchmark starts.
+
 Examples:
   fm-bench
-  fm-bench --models system,pcc --runs 3 --profile stress
+  fm-bench --models system --runs 3 --profile stress
   fm-bench --profile client --sweep-concurrency 1,2 --request-rate 0.5
   fm-bench --prompt "Reply with exactly: ok" --json --out bench.json
   fm-bench --profile reasoning --runs 5 --retry 2
+  fm-bench models
+  fm-bench doctor --json
   fm-bench compare before.json after.json
   fm-bench compare before.json after.json --json
   fm-bench compare before.json after.json --strict
@@ -762,7 +880,5 @@ Examples:
   fm-bench history ./reports
   fm-bench history ./reports --json
   fm-bench legend
-  fm-bench models
-  fm-bench doctor
 `;
 }