fm-bench 0.7.2 → 0.8.0

package/src/bench.js

@@ -1,6 +1,15 @@
 import crypto from 'node:crypto';
 import { detectFmCapabilities } from './capabilities.js';
-import { checkModelAvailability, collectEnvironment, countTokens, fmBinaryFromOptions, getQuotaUsage, respond } from './fm.js';
+import {
+  calibrateTokenCounter,
+  checkModelAvailability,
+  collectEnvironment,
+  countTokens,
+  fmBinaryFromOptions,
+  getQuotaUsage,
+  listModelStatus,
+  respond
+} from './fm.js';
 import { metricAvailability } from './metrics.js';
 import { loadPrompts } from './prompts.js';
 import { finalizeReportPayload } from './schema.js';
@@ -21,18 +30,22 @@ export async function inspectModels(options = {}) {
       ?? { name, description: 'Requested model not reported by this fm build' })
     : discovered.models;
 
+  const modelList = models.length > 0 ? await listModelStatus(discovered.fmBin, { ...options, capabilities }) : null;
   const inspected = [];
   for (const model of models) {
     const availability = await checkModelAvailability(discovered.fmBin, model.name, {
       ...options,
-      capabilities
-    });
-    const quota = await getQuotaUsage(discovered.fmBin, model.name, {
-      ...options,
-      capabilities
+      capabilities,
+      modelList
     });
+    // A model the build does not have has no quota; asking would only put fm's
+    // text for some other model where the "not supported" reason belongs.
+    const quota = availability.unsupported
+      ? { supported: false, raw: '', reason: '' }
+      : await getQuotaUsage(discovered.fmBin, model.name, { ...options, capabilities });
     inspected.push({
       ...model,
+      identity: availability.identity || '',
       available: availability.available,
       unsupported: Boolean(availability.unsupported),
       reason: availability.available ? '' : (availability.reason || availability.raw || 'unavailable'),
@@ -73,7 +86,7 @@ export async function runBenchmark(options = {}) {
     : inspection.models;
   const runnableModels = modelStatuses.filter((model) => model.available);
   if (runnableModels.length === 0) {
-    throw noRunnableModelsError(inspection.models, options);
+    throw noRunnableModelsError(inspection.models, capabilities.models, options);
   }
   const environment = await collectEnvironment(inspection.fmBin, { ...options, capabilities });
   const metrics = metricAvailability(capabilities, {
@@ -84,6 +97,16 @@ export async function runBenchmark(options = {}) {
   const concurrencies = normalizeConcurrencySweep(options);
   const totalRuns = concurrencies.length * runnableModels.length * prompts.length * options.runs;
 
+  const tokenCounter = {
+    command: capabilities.features.tokenCountCommand,
+    overhead: 0,
+    calibrated: false
+  };
+  if (metrics.outputTokens.available) {
+    notify(options, { type: 'phase', phase: 'tokens', message: 'calibrating token counter' });
+    Object.assign(tokenCounter, await calibrateTokenCounter(inspection.fmBin, { ...options, capabilities }));
+  }
+
   notify(options, {
     type: 'tokens:start',
     total: prompts.length,
@@ -123,6 +146,7 @@ export async function runBenchmark(options = {}) {
       modelStatuses,
       promptTokenCounts,
       tokenCounting: metrics.outputTokens.available,
+      tokenOverhead: tokenCounter.overhead,
       options,
       concurrency,
       scenarioIndex: scenarioIndex + 1,
@@ -171,6 +195,7 @@ export async function runBenchmark(options = {}) {
       warnings: capabilities.warnings
     },
     metrics,
+    tokenCounter,
     prompts: prompts.map((prompt) => ({
       id: prompt.id,
       prompt: prompt.prompt,
@@ -199,6 +224,7 @@ async function runScenario(context) {
     modelStatuses,
     promptTokenCounts,
     tokenCounting,
+    tokenOverhead,
     options,
     concurrency,
     scenarioIndex,
@@ -263,6 +289,7 @@ async function runScenario(context) {
       job,
       promptTokenCounts,
       tokenCounting,
+      tokenOverhead,
       options,
       benchmarkStartedAt
     });
@@ -289,7 +316,7 @@ async function runScenario(context) {
 }
 
 async function runSingleBenchmark(context) {
-  const { fmBin, capabilities, job, promptTokenCounts, tokenCounting, options, benchmarkStartedAt } = context;
+  const { fmBin, capabilities, job, promptTokenCounts, tokenCounting, tokenOverhead = 0, options, benchmarkStartedAt } = context;
   const maxAttempts = 1 + Math.max(0, options.retry ?? 0);
   const startOffsetMs = Number(process.hrtime.bigint() - benchmarkStartedAt) / 1e6;
   let response;
@@ -309,24 +336,34 @@ async function runSingleBenchmark(context) {
 
   const ok = response.ok;
   const seconds = response.durationMs / 1000;
-  const chunks = response.stdoutChunks ?? 0;
 
-  // A single stdout chunk carries the whole answer, so the streamed portion is
-  // not separable: report generation time and TPOT as unavailable rather than
-  // as a near-zero decode phase.
+  // When one delivery carries the whole answer, the streamed portion is not
+  // separable: report generation time and TPOT as unavailable rather than as
+  // a near-zero decode phase. Otherwise the generation window runs from the
+  // first to the last delivery, which excludes process teardown.
   const firstTokenMs = ok ? response.firstOutputMs : null;
-  const generationMs = ok && firstTokenMs != null && chunks > 1
-    ? Math.max(0, response.durationMs - firstTokenMs)
+  const deliveryTimes = response.deliveryTimesMs ?? [];
+  const generationMs = ok && firstTokenMs != null && deliveryTimes.length > 1
+    ? Math.max(0, deliveryTimes[deliveryTimes.length - 1] - firstTokenMs)
     : null;
 
-  const outputTokens = ok && tokenCounting
-    ? await countTokens(fmBin, response.output, { ...options, capabilities })
-    : { ok: false, count: null };
-  const countedOutputTokens = outputTokens.ok ? outputTokens.count : null;
+  const countOptions = { ...options, capabilities };
+  const countedOutputTokens = ok && tokenCounting
+    ? await countOutputTokens(fmBin, response.output, tokenOverhead, countOptions)
+    : null;
+  // fm streams coarse deltas: the first delivery carries about 20 tokens on
+  // macOS 27.2. Decode cadence is therefore measured over the tokens that
+  // arrived after it, not over "all tokens but one".
+  const firstChunkTokens = generationMs != null && countedOutputTokens != null
+    ? await countOutputTokens(fmBin, response.firstChunkText ?? '', tokenOverhead, countOptions)
+    : null;
   // Two decode tokens is the minimum for an inter-token interval that is not
   // simply the inverse of a single chunk gap.
-  const decodeTokenCount = countedOutputTokens != null && countedOutputTokens > 2
-    ? countedOutputTokens - 1
+  const tokensAfterFirstChunk = firstChunkTokens != null
+    ? countedOutputTokens - Math.min(firstChunkTokens, countedOutputTokens)
+    : null;
+  const decodeTokenCount = tokensAfterFirstChunk != null && tokensAfterFirstChunk >= 2
+    ? tokensAfterFirstChunk
     : null;
   const hasDecodeCadence = generationMs != null && generationMs > 0 && decodeTokenCount != null;
   const tpotMs = hasDecodeCadence ? generationMs / decodeTokenCount : null;
@@ -338,7 +375,7 @@ async function runSingleBenchmark(context) {
   const prefillTokensPerSecond = promptTokens != null && firstTokenMs != null && firstTokenMs > 0
     ? promptTokens / (firstTokenMs / 1000)
     : null;
-  const chunkGapsMs = ok ? chunkGaps(response.stdoutChunkTimesMs) : [];
+  const chunkGapsMs = ok ? chunkGaps(deliveryTimes) : [];
   const secondChunkMs = chunkGapsMs.length > 0 ? chunkGapsMs[0] : null;
 
   return {
@@ -354,6 +391,8 @@ async function runSingleBenchmark(context) {
     tpotMs,
     promptTokens,
     outputTokens: countedOutputTokens,
+    firstChunkTokens,
+    decodeTokens: hasDecodeCadence ? decodeTokenCount : null,
     chars,
     words,
     tokensPerSecond: countedOutputTokens != null && seconds > 0 ? countedOutputTokens / seconds : null,
@@ -379,6 +418,17 @@ async function runSingleBenchmark(context) {
   };
 }
 
+/**
+ * Output token count with the token counter's framing overhead removed.
+ * Empty text is zero tokens; `fm count-tokens` rejects an empty prompt.
+ * @returns {Promise<number|null>}
+ */
+async function countOutputTokens(fmBin, text, overhead, options) {
+  if (!String(text).trim()) return 0;
+  const counted = await countTokens(fmBin, text, options);
+  return counted.ok ? Math.max(0, counted.count - overhead) : null;
+}
+
 async function runLimited(items, concurrency, worker, options = {}) {
   let nextIndex = 0;
   // fail-fast stops admitting new work; calls already in flight are allowed to
@@ -404,9 +454,9 @@ async function runLimited(items, concurrency, worker, options = {}) {
 
 // A benchmark with nothing to run is a configuration error, not an empty
 // report: say which models were asked for and which ones the build supports.
-function noRunnableModelsError(models, options) {
+function noRunnableModelsError(models, discoveredModels, options) {
   const requested = normalizeModelSelection(options.models);
-  const supported = models.filter((model) => !model.unsupported).map((model) => model.name);
+  const supported = discoveredModels.map((model) => model.name);
   const lines = ['No benchmark was run: none of the requested models are usable right now.'];
   if (requested.length > 0) lines.push(`  requested: ${requested.join(', ')}`);
   if (supported.length > 0) lines.push(`  models reported by this fm build: ${supported.join(', ')}`);

package/src/capabilities.js

@@ -9,7 +9,7 @@
 
 import crypto from 'node:crypto';
 import { stripAnsi } from './ansi.js';
-import { parseAvailabilityList, parseModelsFromHelp } from './fm-help.js';
+import { parseAvailabilityList, parseModelList, parseModelsFromHelp } from './fm-help.js';
 import { runProcess } from './process.js';
 
 const SECTION_HEADER = /^\s*[A-Z][A-Z0-9 /-]+\s*$/;
@@ -72,12 +72,26 @@ export function resolveTokenCountCommand(commands = []) {
   return null;
 }
 
+/**
+ * Pick the subcommand this `fm` build uses to report model availability.
+ * macOS 27.2 renamed `available` to `models`; the old name still works there
+ * but prints a deprecation warning, so prefer the new one.
+ * @returns {'models'|'available'|null}
+ */
+export function resolveModelListCommand(commands = []) {
+  if (commands.includes('models')) return 'models';
+  if (commands.includes('available')) return 'available';
+  return null;
+}
+
 function buildFeatures(helpText, respondHelpText, commands) {
   const tokenCountCommand = resolveTokenCountCommand(commands);
   const respondHelp = respondHelpText || '';
   return {
     tokenCounting: tokenCountCommand != null,
     tokenCountCommand,
+    modelListCommand: resolveModelListCommand(commands),
+    license: commands.includes('license'),
     quota: commands.includes('quota-usage'),
     streaming: hasFlagInHelp(respondHelp, '--no-stream') || hasFlagInHelp(respondHelp, '--stream'),
     modelSelection: hasFlagInHelp(respondHelp, '--model'),
@@ -153,8 +167,8 @@ export async function detectFmCapabilities(fmBin, options = {}) {
   const features = buildFeatures(cleanHelp, respondHelpText, commands);
   let models = parseModelsFromHelp(cleanHelp);
 
-  if (models.length === 0 && commands.includes('available')) {
-    models = await discoverModelsFromAvailability(fmBin, { ...options, env });
+  if (models.length === 0 && features.modelListCommand) {
+    models = await discoverModelsFromList(fmBin, features.modelListCommand, { ...options, env });
   }
 
   return {
@@ -170,17 +184,20 @@ export async function detectFmCapabilities(fmBin, options = {}) {
 }
 
 /**
- * Fallback discovery: ask `fm available` without a model filter and read the
- * model names it reports. Used when `fm --help` has no MODELS section.
+ * Fallback discovery: ask `fm models` (or legacy `fm available`) without a
+ * model filter and read the model names it reports. Used when `fm --help` has
+ * no MODELS section.
  */
-async function discoverModelsFromAvailability(fmBin, options = {}) {
-  const result = await runProcess(fmBin, ['available'], {
+async function discoverModelsFromList(fmBin, command, options = {}) {
+  const result = await runProcess(fmBin, [command], {
     timeoutMs: options.timeoutMs ?? 15_000,
     env: options.env ?? process.env
   });
   if (result.error) return [];
-  return parseAvailabilityList(`${result.stdout}${result.stderr}`)
-    .map((model) => ({ name: model.name, description: '' }));
+  const output = `${result.stdout}${result.stderr}`;
+  const listed = parseModelList(output);
+  const models = listed.length > 0 ? listed : parseAvailabilityList(output);
+  return models.map((model) => ({ name: model.name, description: '' }));
 }
 
 function escapeRegExp(value) {

package/src/cli.js

@@ -3,6 +3,7 @@ import { createRequire } from 'node:module';
 import { inspectModels, runBenchmark } from './bench.js';
 import { diffReports, renderCompareReport } from './compare.js';
 import { renderHtmlReport } from './export.js';
+import { getLicenseStatus } from './fm.js';
 import { formatCapabilitySummary } from './metrics.js';
 import { validateReport } from './schema.js';
 import { loadHistory, renderHistoryReport } from './history.js';
@@ -30,6 +31,53 @@ function operationalError(message) {
   return error;
 }
 
+// One unmeasured call per model absorbs the cold model load, which otherwise
+// dominates the first measured run (several hundred ms on Apple silicon).
+const DEFAULT_WARMUP = 1;
+
+const COMMANDS = ['run', 'models', 'doctor', 'legend', 'metrics', 'compare', 'history', 'validate', 'export', 'help'];
+
+const KNOWN_OPTIONS = [
+  '--help', '--version', '--model', '--models', '--runs', '--warmup', '--concurrency', '--sweep-concurrency',
+  '--request-rate', '--ramp-up-ms', '--timeout', '--timeout-ms', '--slo-ttft-ms', '--slo-e2e-ms', '--slo-tpot-ms',
+  '--prompt', '--prompt-file', '--profile', '--instructions', '--fm-bin', '--use-case', '--guardrails',
+  '--greedy', '--no-greedy', '--stream', '--no-stream', '--json', '--csv', '--format', '--ascii', '--color',
+  '--no-color', '--progress', '--no-progress', '--compact', '--width', '--histogram', '--export-html', '--strict',
+  '--out', '--output-dir', '--capture-output', '--available-only', '--fail-fast', '--retry', '--ci', '--tag',
+  '--note', '--verbose'
+];
+
+/** Closest candidate within a small edit distance, or null. */
+function closestMatch(input, candidates) {
+  let best = null;
+  let bestDistance = Infinity;
+  for (const candidate of candidates) {
+    const distance = editDistance(input, candidate);
+    if (distance < bestDistance) {
+      best = candidate;
+      bestDistance = distance;
+    }
+  }
+  const limit = input.replace(/^-+/, '').length <= 6 ? 1 : 2;
+  return bestDistance > 0 && bestDistance <= limit ? best : null;
+}
+
+/** Optimal string alignment distance: an adjacent swap counts as one edit. */
+function editDistance(a, b) {
+  const rows = Array.from({ length: a.length + 1 }, (_, i) => [i, ...Array(b.length).fill(0)]);
+  for (let j = 1; j <= b.length; j += 1) rows[0][j] = j;
+  for (let i = 1; i <= a.length; i += 1) {
+    for (let j = 1; j <= b.length; j += 1) {
+      const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+      rows[i][j] = Math.min(rows[i - 1][j] + 1, rows[i][j - 1] + 1, rows[i - 1][j - 1] + cost);
+      if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1]) {
+        rows[i][j] = Math.min(rows[i][j], rows[i - 2][j - 2] + 1);
+      }
+    }
+  }
+  return rows[a.length][b.length];
+}
+
 export async function runCli(argv = process.argv.slice(2), env = {}) {
   const parsed = parseArgs(argv);
 
@@ -230,7 +278,7 @@ export function parseArgs(argv) {
     models: [],
     prompts: [],
     runs: 1,
-    warmup: 0,
+    warmup: DEFAULT_WARMUP,
     concurrency: 1,
     sweepConcurrency: [],
     requestRate: null,
@@ -263,8 +311,15 @@ export function parseArgs(argv) {
   };
 
   const args = [...argv];
-  if (args[0] && !args[0].startsWith('-') && ['run', 'models', 'doctor', 'legend', 'metrics', 'compare', 'history', 'validate', 'export', 'help'].includes(args[0])) {
+  if (args[0] && !args[0].startsWith('-') && COMMANDS.includes(args[0])) {
     options.command = args.shift();
+  } else if (args[0] && /^[a-z]{3,}$/.test(args[0])) {
+    // A bare word is a prompt, but one that is a near miss for a command is
+    // almost always a typo; benchmarking "modles" as a prompt helps nobody.
+    const suggestion = closestMatch(args[0], COMMANDS);
+    if (suggestion) {
+      throw usageError(`Unknown command "${args[0]}". Did you mean "${suggestion}"? To benchmark it as a prompt, use: fm-bench -- ${args[0]}`);
+    }
   }
 
   if (options.command === 'metrics') {
@@ -447,7 +502,8 @@ export function parseArgs(argv) {
         break;
       default:
         if (arg.startsWith('-')) {
-          throw usageError(`Unknown option: ${arg}`);
+          const suggestion = arg.startsWith('--') ? closestMatch(arg, KNOWN_OPTIONS) : null;
+          throw usageError(`Unknown option: ${arg}${suggestion ? `. Did you mean ${suggestion}?` : ''} (see fm-bench --help)`);
         }
         if (options.command === 'compare') {
           options.compareFiles.push(arg);
@@ -665,8 +721,17 @@ async function runDoctor(options) {
     checks.push(['fm token counting', capabilities.features.tokenCounting ? `yes (${capabilities.features.tokenCountCommand})` : 'no', capabilities.features.tokenCounting]);
     checks.push(['fm streaming', capabilities.features.streaming ? 'yes' : 'no', capabilities.features.streaming]);
     checks.push(['fm quota', capabilities.features.quota ? 'yes' : 'no (not exposed by this build)', true]);
+    if (capabilities.features.license) {
+      const license = await getLicenseStatus(inspection.fmBin, { capabilities });
+      checks.push(['fm license', license.agreed === false
+        ? `${license.detail || 'not agreed'} — run "fm license" to review and accept the terms`
+        : license.detail, license.agreed !== false]);
+    }
     for (const model of inspection.models) {
-      checks.push([`model:${model.name}`, model.available ? 'available' : model.reason || 'unavailable', model.available]);
+      const detail = model.available
+        ? `available${model.identity ? ` (${model.identity})` : ''}`
+        : model.reason || 'unavailable';
+      checks.push([`model:${model.name}`, detail, model.available]);
     }
   } catch (error) {
     checks.push(['fm', error.message || String(error), false]);
@@ -681,7 +746,8 @@ async function runDoctor(options) {
   if (json) {
     console.log(JSON.stringify(payload, null, 2));
   } else {
-    const lines = checks.map(([name, detail, ok]) => `${ok ? 'ok  ' : 'warn'} ${name.padEnd(16)} ${String(detail).replace(/\s+/g, ' ').trim()}`);
+    const nameWidth = Math.max(...checks.map(([name]) => name.length));
+    const lines = checks.map(([name, detail, ok]) => `${ok ? 'ok  ' : 'warn'} ${name.padEnd(nameWidth)}  ${String(detail).replace(/\s+/g, ' ').trim()}`);
     console.log(lines.join('\n'));
     if (capabilities) {
       console.log('');
@@ -795,7 +861,7 @@ Commands:
 Run options:
   -m, --models <list>       Models to benchmark, comma-separated or repeated
   -r, --runs <n>            Runs per prompt/model (default: 1)
-      --warmup <n>          Warmup runs per model before measurement
+      --warmup <n>          Unmeasured warmup runs per model (default: 1; 0 measures cold start)
   -c, --concurrency <n>     Parallel fm processes (default: 1)
       --sweep-concurrency <list>
                             Run separate operating points, e.g. 1,2,4
@@ -859,10 +925,16 @@ Exit codes:
   2   usage or environment error (bad flags, missing arguments, unsupported macOS, fm not found)
 
 Capability detection:
-  fm-bench probes "fm --help" and "fm respond --help" once per run. Metrics the
-  installed fm cannot supply are reported as unavailable instead of being
+  fm-bench probes "fm --help" and "fm respond --help" once per run and reads
+  model status from "fm models" (or "fm available" on older builds). Metrics
+  the installed fm cannot supply are reported as unavailable instead of being
   guessed, and unsupported models are refused before any benchmark starts.
 
+Private Cloud Compute:
+  fm reports the pcc model as "not available in this context" outside the
+  Terminal app (for example in editor terminals). Run fm-bench from Terminal
+  to benchmark pcc; elsewhere it is listed as skipped with that reason.
+
 Examples:
   fm-bench
   fm-bench --models system --runs 3 --profile stress

package/src/compare.js

@@ -77,10 +77,19 @@ function buildDiffRow(key, before, after) {
     goodputRate: diffPercent(before?.goodputRate, after?.goodputRate),
     tokensPerSecond: diffNumber(before?.tokensPerSecond?.avg, after?.tokensPerSecond?.avg, false),
     rps: diffNumber(before?.rps, after?.rps, false),
-    cv: diffPercent(before?.latency?.cv, after?.latency?.cv)
+    cv: diffPercent(...comparableCv(before, after))
   };
 }
 
+// Compare the run-to-run CV only when both reports carry it; otherwise fall
+// back to the suite-wide latency CV on both sides so the definitions match.
+function comparableCv(before, after) {
+  const bothStable = before && after && Object.hasOwn(before, 'stabilityCv') && Object.hasOwn(after, 'stabilityCv');
+  return bothStable
+    ? [before.stabilityCv, after.stabilityCv]
+    : [before?.latency?.cv, after?.latency?.cv];
+}
+
 function diffMs(before, after) {
   return {
     before: before ?? null,

package/src/export.js

@@ -1,4 +1,5 @@
 import { environmentFingerprint } from './schema.js';
+import { stabilityCv } from './table.js';
 
 /**
  * Self-contained HTML report for sharing (paste, email, GitHub gist, static host).
@@ -33,12 +34,13 @@ export function renderHtmlReport(report) {
     ['macOS', fp.macOSProductVersion ?? '—'],
     ['Build', fp.macOSBuildVersion ?? '—'],
     ['Node', fp.node ?? '—'],
-    ['fm CLI digest', fp.fmHelpDigest ?? '—']
+    ['fm CLI digest', fp.fmHelpDigest ?? '—'],
+    ['Models', Object.entries(fp.modelIdentities).map(([name, identity]) => `${name} = ${identity}`).join(', ') || '—']
   ];
 
   const summaryRows = summary.map((row) => {
     const ttft = /** @type {{ p50?: number, p95?: number }} */ (row.ttft ?? {});
-    const lat = /** @type {{ p50?: number, p95?: number, cv?: number }} */ (row.latency ?? {});
+    const lat = /** @type {{ p50?: number, p95?: number }} */ (row.latency ?? {});
     const tps = /** @type {{ avg?: number }} */ (row.tokensPerSecond ?? {});
     return `<tr>
       <td>${esc(row.model)}</td>
@@ -51,7 +53,7 @@ export function renderHtmlReport(report) {
       <td>${fmtMs(lat.p50)}</td>
       <td>${fmtMs(lat.p95)}</td>
       <td>${fmtNum(tps.avg)}</td>
-      <td>${fmtPct(lat.cv)}</td>
+      <td>${fmtPct(stabilityCv(row))}</td>
     </tr>`;
   }).join('\n');
 
@@ -114,7 +116,7 @@ export function renderHtmlReport(report) {
   </section>
 
   <footer>
-    Generated by <a href="https://github.com/devinoldenburg/fm-bench">fm-bench</a>.
+    Generated by <a href="https://github.com/dvnold/fm-bench">fm-bench</a>.
     Compare two reports: <code>fm-bench compare before.json after.json</code>.
     Validate: <code>fm-bench validate report.json</code>.
   </footer>

package/src/fm-help.js

@@ -60,6 +60,47 @@ export function parseModelsFromHelp(helpText = '') {
   return [...models.values()];
 }
 
+/**
+ * Read `fm models` output (macOS 27.2+), one model per line:
+ *
+ *   Apple Foundation Models
+ *     ✓ system (AFM 3 Core Advanced)
+ *     ✗ pcc    (Private Cloud Compute is not available in this context. ...)
+ *
+ * The parenthesised text is the model identity for an available model and
+ * the reason for an unavailable one.
+ * @param {string} output
+ * @returns {{ name: string, available: boolean, identity: string, reason: string }[]}
+ */
+export function parseModelList(output = '') {
+  const models = new Map();
+  for (const line of withoutWarnings(output).split(/\r?\n/)) {
+    const match = line.match(/^\s*([✓✔✗✘×])\s+([A-Za-z0-9][A-Za-z0-9._:/-]*)\s*(?:\((.*)\))?\s*$/);
+    if (!match) continue;
+    const available = match[1] === '✓' || match[1] === '✔';
+    const detail = (match[3] ?? '').trim();
+    models.set(match[2], {
+      name: match[2],
+      available,
+      identity: available ? detail : '',
+      reason: available ? '' : detail
+    });
+  }
+  return [...models.values()];
+}
+
+/**
+ * Drop `warning:` lines (for example the `fm available` rename notice) so
+ * they are never mistaken for a model status or an error cause.
+ * @param {string} text
+ */
+export function withoutWarnings(text = '') {
+  return stripAnsi(text)
+    .split(/\r?\n/)
+    .filter((line) => !/^\s*warning:/i.test(line))
+    .join('\n');
+}
+
 /**
  * Read model names and availability out of `fm available` (no model filter).
  * @param {string} output
@@ -67,7 +108,7 @@ export function parseModelsFromHelp(helpText = '') {
  */
 export function parseAvailabilityList(output = '') {
   const models = new Map();
-  for (const line of stripAnsi(output).split(/\r?\n/)) {
+  for (const line of withoutWarnings(output).split(/\r?\n/)) {
     const match = line.match(/^\s*([A-Za-z][A-Za-z0-9._-]*)(?:\s+model)?\s+(?:is\s+)?(available|unavailable|not available)\b/i);
     if (!match) continue;
     const name = match[1].toLowerCase();
@@ -88,7 +129,17 @@ export function parseAvailabilityList(output = '') {
  * @param {number|null} code process exit code
  */
 export function parseAvailabilityOutput(model, output = '', code = null) {
-  const clean = stripAnsi(output).trim();
+  const clean = withoutWarnings(output).trim();
+  const listed = parseModelList(clean).find((entry) => entry.name === model);
+  if (listed) {
+    return {
+      model,
+      available: listed.available,
+      identity: listed.identity,
+      raw: clean,
+      reason: listed.reason
+    };
+  }
   const lower = clean.toLowerCase();
   const modelLower = String(model).toLowerCase();
   const hasError = /\berror:|\bunavailable\b|\bnot available\b|\bnot supported\b|\bis invalid for\b/.test(lower);
@@ -106,14 +157,18 @@ export function parseAvailabilityOutput(model, output = '', code = null) {
 
 /**
  * Collapse `fm` diagnostics into one actionable line. `fm` writes multi-line
- * usage blocks for argument errors; benchmark output only needs the cause.
+ * usage blocks for argument errors and `warning:` notices; benchmark output
+ * only needs the line that states the cause (including any hint on it, such
+ * as "Please use the Terminal app.").
  * @param {string} text
  */
 export function firstLine(text = '') {
-  const clean = stripAnsi(text).replace(/\s+/g, ' ').trim();
-  if (!clean) return '';
-  const sentences = clean.split(/(?<=\.)\s+(?=[A-Z])/);
-  const head = sentences.find((part) => /error|invalid|unavailable|not supported|failed/i.test(part)) || sentences[0];
+  const lines = withoutWarnings(text)
+    .split(/\r?\n/)
+    .map((line) => line.replace(/\s+/g, ' ').trim())
+    .filter((line) => line && !/^(usage|help|see)\b/i.test(line));
+  if (lines.length === 0) return '';
+  const head = lines.find((line) => /error|invalid|unavailable|not available|not supported|failed/i.test(line)) || lines[0];
   return head.replace(/^Error:\s*/i, '').trim();
 }