fm-bench 0.7.1 → 0.8.0

package/src/fm.js

@@ -1,7 +1,7 @@
 import os from 'node:os';
 import { stripAnsi } from './ansi.js';
 import { detectFmCapabilities } from './capabilities.js';
-import { firstLine, isUnsupportedModelError, parseAvailabilityOutput } from './fm-help.js';
+import { firstLine, isUnsupportedModelError, parseAvailabilityOutput, parseModelList, withoutWarnings } from './fm-help.js';
 import { runProcess } from './process.js';
 import { parseBatteryOutput, parseThermalOutput } from './system.js';
 
@@ -25,12 +25,33 @@ export async function getFmHelp(fmBin, timeoutMs = 10_000) {
   };
 }
 
+/**
+ * Ask `fm models` once for every model's status. Returns `null` when the
+ * build has no `models` command (older builds use per-model `fm available`).
+ * @returns {Promise<{ entries: Map<string, { name: string, available: boolean, identity: string, reason: string }>, error: string } | null>}
+ */
+export async function listModelStatus(fmBin, options = {}) {
+  if (options.capabilities?.features?.modelListCommand !== 'models') return null;
+  const result = await runProcess(fmBin, ['models'], {
+    timeoutMs: options.timeoutMs ?? 15_000
+  });
+  const output = `${result.stdout}${result.stderr}`;
+  const entries = new Map(parseModelList(output).map((entry) => [entry.name, entry]));
+  const error = result.error
+    ? (result.stderr || result.error.message)
+    : (entries.size === 0 ? firstLine(output) || `fm models exited with code ${result.code}` : '');
+  return { entries, error };
+}
+
 /**
  * Check one model against the detected `fm` build.
  *
  * Models the build does not expose are reported as unsupported without
  * spawning `fm` at all, so a raw argument-error blob from `fm` can never end
  * up in a report or in `fm-bench models` output.
+ *
+ * Pass `options.modelList` (from `listModelStatus`) to reuse one `fm models`
+ * call across several models.
  */
 export async function checkModelAvailability(fmBin, model, options = {}) {
   const capabilities = options.capabilities;
@@ -40,11 +61,33 @@ export async function checkModelAvailability(fmBin, model, options = {}) {
       model,
       available: false,
       unsupported: true,
+      identity: '',
       raw: '',
       reason: `not supported by this fm build (supported: ${known.join(', ')})`
     };
   }
 
+  const listCommand = capabilities ? capabilities.features?.modelListCommand : 'available';
+  if (listCommand === 'models') {
+    const list = options.modelList ?? await listModelStatus(fmBin, options);
+    const entry = list?.entries.get(model);
+    if (entry) {
+      return { model, available: entry.available, identity: entry.identity, raw: '', reason: entry.reason };
+    }
+    return {
+      model,
+      available: false,
+      identity: '',
+      raw: '',
+      reason: list?.error || 'not reported by fm models'
+    };
+  }
+  if (listCommand == null) {
+    // No availability command at all: the first fm respond call is the only
+    // way to find out, and a failure there is recorded per run.
+    return { model, available: true, identity: '', raw: '', reason: '' };
+  }
+
   const result = await runProcess(fmBin, ['available', '--model', model], {
     timeoutMs: options.timeoutMs ?? 15_000
   });
@@ -61,7 +104,27 @@ export async function checkModelAvailability(fmBin, model, options = {}) {
     const supported = known.length > 0 ? ` (supported: ${known.join(', ')})` : '';
     parsed.reason = `not supported by this fm build${supported}`;
   }
-  return parsed;
+  return { identity: '', ...parsed };
+}
+
+/**
+ * Whether the fm Legal Notice & Terms have been accepted. `fm respond` cannot
+ * run until they are, so `doctor` reports it explicitly.
+ * @returns {Promise<{ supported: boolean, agreed: boolean|null, detail: string }>}
+ */
+export async function getLicenseStatus(fmBin, options = {}) {
+  if (options.capabilities && !options.capabilities.features?.license) {
+    return { supported: false, agreed: null, detail: 'this fm build has no license command' };
+  }
+  const result = await runProcess(fmBin, ['license', '--status'], {
+    timeoutMs: options.timeoutMs ?? 10_000
+  });
+  const output = withoutWarnings(`${result.stdout}${result.stderr}`).replace(/\s+/g, ' ').trim();
+  if (result.error) {
+    return { supported: true, agreed: null, detail: result.stderr || result.error.message };
+  }
+  const agreed = result.code === 0 && /\bagreed\b/i.test(output) && !/\bnot\b|\bnever\b/i.test(output);
+  return { supported: true, agreed, detail: output || `fm license --status exited with code ${result.code}` };
 }
 
 /**
@@ -133,6 +196,61 @@ export async function countTokens(fmBin, text, options = {}) {
   };
 }
 
+/**
+ * Measure the constant framing overhead `fm count-tokens` adds to every count.
+ *
+ * On macOS 27.2 `count-tokens` reports 2 for "a", 3 for "a a", and 5 for
+ * "a a a a": one token per word plus one framing token. Model output must not
+ * carry that extra token, so the overhead is derived from three counts whose
+ * per-word step must agree; anything inconsistent leaves counts uncorrected.
+ *
+ * @returns {Promise<{ overhead: number, calibrated: boolean }>}
+ */
+export async function calibrateTokenCounter(fmBin, options = {}) {
+  const counts = [];
+  for (const text of ['a', 'a a', 'a a a']) {
+    const counted = await countTokens(fmBin, text, options);
+    if (!counted.ok) return { overhead: 0, calibrated: false };
+    counts.push(counted.count);
+  }
+  const step = counts[1] - counts[0];
+  const overhead = counts[0] - step;
+  const consistent = step >= 1 && counts[2] - counts[1] === step && overhead >= 0 && overhead <= 16;
+  return consistent ? { overhead, calibrated: true } : { overhead: 0, calibrated: false };
+}
+
+// Stdout chunks closer together than this are one write burst from fm, not
+// separate streaming steps. On macOS 27.2 a short answer's tail arrives as
+// several writes well under 1 ms apart, while real deltas are 20 ms or more
+// apart; counting the burst as decode steps reported tens of thousands of
+// tokens per second.
+export const DELIVERY_COALESCE_MS = 5;
+
+/**
+ * Group stdout chunk arrivals into deliveries: runs of chunks that each
+ * arrived less than `coalesceMs` after the previous one.
+ * @param {number[]} timesMs chunk arrival times
+ * @param {number[]} lengths chunk lengths in characters
+ * @returns {{ atMs: number, endChars: number }[]} delivery start time and the
+ *   stdout length once the delivery is complete
+ */
+export function groupDeliveries(timesMs = [], lengths = [], coalesceMs = DELIVERY_COALESCE_MS) {
+  const deliveries = [];
+  let chars = 0;
+  let previousAtMs = null;
+  for (const [index, atMs] of timesMs.entries()) {
+    chars += lengths[index] ?? 0;
+    const current = deliveries.at(-1);
+    if (current && atMs - previousAtMs < coalesceMs) {
+      current.endChars = chars;
+    } else {
+      deliveries.push({ atMs, endChars: chars });
+    }
+    previousAtMs = atMs;
+  }
+  return deliveries;
+}
+
 export async function respond(fmBin, model, prompt, options = {}) {
   const features = options.capabilities?.features;
   const streamControl = features ? features.streaming : true;
@@ -155,6 +273,7 @@ export async function respond(fmBin, model, prompt, options = {}) {
   const output = stripAnsi(result.stdout).trim();
   const errorText = stripAnsi(result.stderr).trim();
   const failed = result.code !== 0 || result.timedOut;
+  const deliveries = streamed ? groupDeliveries(result.stdoutChunkTimesMs, result.stdoutChunkLengths) : [];
 
   return {
     ok: !failed,
@@ -167,9 +286,11 @@ export async function respond(fmBin, model, prompt, options = {}) {
     timedOut: result.timedOut,
     durationMs: result.durationMs,
     firstOutputMs: streamed ? result.firstStdoutMs : null,
+    firstChunkText: deliveries.length > 0 ? stripAnsi(result.stdout.slice(0, deliveries[0].endChars)) : null,
     streamed,
     stdoutChunks: result.stdoutChunks,
     stdoutChunkTimesMs: streamed ? result.stdoutChunkTimesMs : [],
+    deliveryTimesMs: deliveries.map((delivery) => delivery.atMs),
     error: failed
       ? (result.timedOut
         ? `timed out after ${options.timeoutMs ?? 60_000}ms`

package/src/metrics.js

@@ -31,7 +31,7 @@ const DEFINITIONS = [
     key: 'generationMs',
     label: 'generation time',
     kind: 'derived',
-    source: 'E2E latency minus TTFT',
+    source: 'last streamed stdout delivery minus the first (chunks under 5 ms apart are one delivery)',
     requires: ['streaming'],
     reason: 'requires at least two streamed output chunks to separate prefill from decode'
   },
@@ -39,9 +39,17 @@ const DEFINITIONS = [
     key: 'tpot',
     label: 'TPOT',
     kind: 'derived',
-    source: '(E2E - TTFT) / (output tokens - 1)',
+    source: 'generation time / output tokens that arrived after the first chunk',
     requires: ['streaming', 'tokenCounting'],
-    reason: 'requires streaming, a token-counting fm command, and at least three output tokens'
+    reason: 'requires streaming, a token-counting fm command, and at least two tokens after the first chunk'
+  },
+  {
+    key: 'firstChunkTokens',
+    label: 'first-chunk tokens',
+    kind: 'measured',
+    source: 'fm count-tokens on the first streamed stdout delivery, minus the counter framing overhead',
+    requires: ['streaming', 'tokenCounting'],
+    reason: 'requires streaming plus a token-counting fm command'
   },
   {
     key: 'promptTokens',
@@ -55,7 +63,7 @@ const DEFINITIONS = [
     key: 'outputTokens',
     label: 'output tokens',
     kind: 'measured',
-    source: 'fm count-tokens on the captured output',
+    source: 'fm count-tokens on the captured output, minus the calibrated counter framing overhead',
     requires: ['tokenCounting'],
     reason: 'this fm build exposes no token-counting command'
   },
@@ -71,9 +79,9 @@ const DEFINITIONS = [
     key: 'decodeTokensPerSecond',
     label: 'decode tokens/s',
     kind: 'derived',
-    source: '(output tokens - 1) / generation seconds',
+    source: 'output tokens after the first chunk / generation seconds',
     requires: ['streaming', 'tokenCounting'],
-    reason: 'requires streaming, a token-counting fm command, and at least three output tokens'
+    reason: 'requires streaming, a token-counting fm command, and at least two tokens after the first chunk'
   },
   {
     key: 'prefillTokensPerSecond',
@@ -102,7 +110,7 @@ const DEFINITIONS = [
     key: 'chunkGaps',
     label: 'chunk gaps and second-chunk delay',
     kind: 'proxy',
-    source: 'gaps between consecutive streamed stdout chunks',
+    source: 'gaps between consecutive streamed stdout deliveries (chunks under 5 ms apart are one delivery)',
     requires: ['streaming'],
     reason: 'requires an fm build whose output can be streamed'
   },

package/src/process.js

@@ -48,6 +48,7 @@ export function runProcess(command, args = [], options = {}) {
     let stdoutChunks = 0;
     let stderrChunks = 0;
     const stdoutChunkTimesMs = [];
+    const stdoutChunkLengths = [];
     let firstStdoutMs = null;
     let firstStderrMs = null;
     let timedOut = false;
@@ -71,7 +72,10 @@ export function runProcess(command, args = [], options = {}) {
       if (firstStdoutMs == null && chunk.length > 0) {
         firstStdoutMs = chunkAtMs;
       }
-      if (chunk.length > 0) stdoutChunkTimesMs.push(chunkAtMs);
+      if (chunk.length > 0) {
+        stdoutChunkTimesMs.push(chunkAtMs);
+        stdoutChunkLengths.push(chunk.length);
+      }
       stdout += chunk;
     });
     child.stderr.on('data', (chunk) => {
@@ -101,6 +105,7 @@ export function runProcess(command, args = [], options = {}) {
         stdoutChunks,
         stderrChunks,
         stdoutChunkTimesMs,
+        stdoutChunkLengths,
         firstStdoutMs,
         firstStderrMs,
         error,
@@ -120,6 +125,7 @@ export function runProcess(command, args = [], options = {}) {
         stdoutChunks,
         stderrChunks,
         stdoutChunkTimesMs,
+        stdoutChunkLengths,
         firstStdoutMs,
         firstStderrMs,
         error: null,

package/src/report.js

@@ -1,5 +1,6 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { renderHtmlReport } from './export.js';
 
 export function toCsv(rows) {
   if (rows.length === 0) return '';
@@ -37,7 +38,8 @@ export function flattenResults(results) {
     chunk_gap_max_ms: round(result.chunkGapMaxMs),
     output_hash: result.outputHash || '',
     good: result.good == null ? '' : result.good,
-    error: result.error || ''
+    error: result.error || '',
+    first_chunk_tokens: result.firstChunkTokens ?? ''
   }));
 }
 
@@ -48,7 +50,6 @@ export async function writeReport(filePath, payload, format) {
   if (format === 'csv') {
     content = toCsv(flattenResults(payload.results));
   } else if (format === 'html') {
-    const { renderHtmlReport } = await import('./export.js');
     content = renderHtmlReport(payload);
   } else {
     content = `${JSON.stringify(payload, null, 2)}\n`;

package/src/schema.js

@@ -62,11 +62,30 @@ export function environmentFingerprint(report) {
     macOSBuildVersion: build,
     fmBin: env.fmBin ?? null,
     fmHelpDigest: env.fmHelpDigest ?? null,
+    modelIdentities: modelIdentities(report),
     thermal: env.thermal ?? null,
     power: env.power ?? null
   };
 }
 
+/**
+ * Model name → identity reported by `fm models` (for example
+ * `{ system: 'AFM 3 Core Advanced' }`). Empty for reports from builds or
+ * fm-bench versions that do not expose it.
+ * @param {Record<string, unknown>} report
+ * @returns {Record<string, string>}
+ */
+export function modelIdentities(report) {
+  const identities = {};
+  const models = Array.isArray(report.models) ? report.models : [];
+  for (const model of models) {
+    if (model && typeof model.name === 'string' && typeof model.identity === 'string' && model.identity) {
+      identities[model.name] = model.identity;
+    }
+  }
+  return identities;
+}
+
 /**
  * @param {Record<string, unknown>} before
  * @param {Record<string, unknown>} after
@@ -99,6 +118,12 @@ export function compareCompatibility(before, after) {
   if (bFp.macOSBuildVersion && aFp.macOSBuildVersion && bFp.macOSBuildVersion !== aFp.macOSBuildVersion) {
     warnings.push(`macOS build differs (${bFp.macOSBuildVersion} vs ${aFp.macOSBuildVersion})`);
   }
+  for (const [name, identity] of Object.entries(bFp.modelIdentities)) {
+    const other = aFp.modelIdentities[name];
+    if (other && other !== identity) {
+      warnings.push(`model ${name} differs (${identity} vs ${other})`);
+    }
+  }
 
   return {
     compatible: errors.length === 0,

package/src/stats.js

@@ -84,6 +84,7 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
         model: status.name,
         concurrency,
         description: status.description,
+        identity: status.identity || '',
         available: status.available,
         unsupported: Boolean(status.unsupported),
         skippedReason: status.available ? '' : status.reason || 'Unavailable',
@@ -99,6 +100,7 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
         model: result.model,
         concurrency: result.concurrency,
         description: '',
+        identity: '',
         available: true,
         unsupported: false,
         skippedReason: '',
@@ -119,12 +121,19 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
     const tpot = summarizeNumbers(successes.map((result) => result.tpotMs).filter((value) => value != null));
     const promptTokens = summarizeNumbers(successes.map((result) => result.promptTokens).filter((value) => value != null));
     const outputTokens = summarizeNumbers(successes.map((result) => result.outputTokens).filter((value) => value != null));
+    const firstChunkTokens = summarizeNumbers(successes.map((result) => result.firstChunkTokens).filter((value) => value != null));
     const charsPerSecond = summarizeNumbers(successes.map((result) => result.charsPerSecond).filter((value) => value != null));
     const tokensPerSecond = summarizeNumbers(successes.map((result) => result.tokensPerSecond).filter((value) => value != null));
     const decodeTokensPerSecond = summarizeNumbers(successes.map((result) => result.decodeTokensPerSecond).filter((value) => value != null));
     const prefillTokensPerSecond = summarizeNumbers(successes.map((result) => result.prefillTokensPerSecond).filter((value) => value != null));
     const secondChunk = summarizeNumbers(successes.map((result) => result.secondChunkMs).filter((value) => value != null));
     const chunkGap = summarizeNumbers(successes.flatMap((result) => result.chunkGapsMs || []));
+    const decodeRuns = successes.filter((result) => result.decodeTokens != null && result.generationMs > 0);
+    const decodeMs = decodeRuns.reduce((sum, result) => sum + result.generationMs, 0);
+    // Token-weighted: a 3-token tail cannot outweigh a 200-token generation.
+    const decodeThroughput = decodeMs > 0
+      ? decodeRuns.reduce((sum, result) => sum + result.decodeTokens, 0) / (decodeMs / 1000)
+      : null;
     const windowMs = modelWindowMs(successes);
     const rps = successes.length > 0 && windowMs > 0 ? successes.length / (windowMs / 1000) : null;
     const goodputRps = goodMeasured.length > 0 && windowMs > 0 ? goodResults.length / (windowMs / 1000) : null;
@@ -137,6 +146,7 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
       model: entry.model,
       concurrency: entry.concurrency,
       description: entry.description,
+      identity: entry.identity,
       available: entry.available,
       unsupported: entry.unsupported,
       skippedReason: entry.skippedReason,
@@ -147,6 +157,7 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
       failures: failures.length,
       successRate: entry.results.length > 0 ? successes.length / entry.results.length : null,
       goodputRate: goodMeasured.length > 0 ? goodResults.length / goodMeasured.length : null,
+      stabilityCv: summarizeStability(successes),
       rps,
       goodputRps,
       outputTokenThroughput,
@@ -158,9 +169,11 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
       tpot,
       promptTokens,
       outputTokens,
+      firstChunkTokens,
       charsPerSecond,
       tokensPerSecond,
       decodeTokensPerSecond,
+      decodeThroughput,
       prefillTokensPerSecond,
       secondChunk,
       chunkGap
@@ -222,6 +235,29 @@ function modelWindowMs(results) {
   return Math.max(...ends) - Math.min(...starts);
 }
 
+/**
+ * Run-to-run latency stability: the E2E coefficient of variation of each
+ * prompt across its repeated runs, averaged over prompts.
+ *
+ * The pooled `latency.cv` mixes every prompt, so a suite with a one-line answer and a
+ * long generation reports high "variation" even when every prompt is
+ * perfectly steady. Grouping by prompt isolates repeat noise. `null` until at
+ * least one prompt has two successful runs.
+ */
+function summarizeStability(results) {
+  const byPrompt = new Map();
+  for (const result of results) {
+    if (!Number.isFinite(result.durationMs)) continue;
+    if (!byPrompt.has(result.promptId)) byPrompt.set(result.promptId, []);
+    byPrompt.get(result.promptId).push(result.durationMs);
+  }
+  const cvs = [...byPrompt.values()]
+    .map((durations) => summarizeNumbers(durations).cv)
+    .filter((cv) => cv != null);
+  if (cvs.length === 0) return null;
+  return cvs.reduce((sum, cv) => sum + cv, 0) / cvs.length;
+}
+
 function summarizeRepeatability(results) {
   const byPrompt = new Map();
   for (const result of results) {

package/src/table.js

@@ -55,6 +55,12 @@ export function renderBenchmarkReport(payload, options = {}) {
   const note = payload.options?.note ?? null;
   lines.push(...wrapText(title, width));
   lines.push(...wrapText(meta, width));
+  const identities = (payload.models ?? [])
+    .filter((model) => model.available && model.identity)
+    .map((model) => `${model.name} = ${model.identity}`);
+  if (identities.length > 0) {
+    lines.push(...wrapText(`models: ${identities.join(', ')}`, width));
+  }
   if (tags.length > 0) {
     for (const line of wrapText(`tags: ${tags.join(', ')}`, width)) lines.push(line);
   }
@@ -88,21 +94,22 @@ export function legendEntries() {
     entry('summary', 'SUCC / SUCCESS', 'Success rate: successful runs divided by attempted runs.', 'Green 100%, yellow >=95%, red <95%.', 'measured'),
     entry('summary', 'GOOD', 'Goodput rate: successful runs that also met every configured SLO.', 'Only appears when SLO flags are set. Runs whose SLO metric is unmeasurable count as not good.', 'derived'),
     entry('summary', 'GOOD RPS', 'SLO-passing requests per second during this measured window.', 'Zero is shown when SLOs are set and no request meets them.', 'derived'),
-    entry('summary', 'TTFT', 'Time from starting fm respond to the first streamed stdout chunk, p50.', 'Proxy for time to first token: measured at chunk granularity, not per token. Lower is better.', 'proxy'),
+    entry('summary', 'TTFT', 'Time from starting fm respond to the first streamed stdout chunk, p50.', 'Proxy for time to first token: measured at chunk granularity, and the first chunk can already hold many tokens (see 1ST CHUNK). Lower is better.', 'proxy'),
     entry('summary', 'TTFT P95', '95th percentile time to the first streamed stdout chunk.', 'Lower is better.', 'proxy'),
     entry('summary', 'E2E', 'End-to-end latency, p50, from starting fm respond until full response exits.', 'Lower is better. This is a direct wall-clock measurement.', 'measured'),
     entry('summary', 'E2E P95', '95th percentile end-to-end latency.', 'Lower is better; this is usually the main interactive tail-latency signal.', 'measured'),
-    entry('summary', 'TPOT', 'Time per output token after the first output token, p50.', 'Derived from fm token counts and stream timings. Lower is better.', 'derived'),
-    entry('summary', 'TPOT P95', '95th percentile time per output token after first token.', 'Lower is better.', 'derived'),
+    entry('summary', 'TPOT', 'Time per output token for tokens that arrived after the first streamed chunk, p50.', 'Derived from fm token counts and stream timings. Lower is better.', 'derived'),
+    entry('summary', 'TPOT P95', '95th percentile time per output token after the first streamed chunk.', 'Lower is better.', 'derived'),
     entry('summary', 'USER/S / USER T/S', 'Per-request output tokens per second.', 'Derived from fm token counts. Higher is better.', 'derived'),
     entry('summary', 'SYS/S / SYS T/S', 'Aggregate successful output-token throughput for the model row.', 'Higher is better.', 'derived'),
     entry('summary', 'RPS', 'Successful requests per second over the model row measured window.', 'Measured from process timings. Higher is better.', 'measured'),
-    entry('summary', 'CV', 'Coefficient of variation for E2E latency: sample stddev divided by mean.', 'Lower is steadier. Green <=10%, yellow <=25%, red >25%.', 'derived'),
+    entry('summary', 'CV', 'Run-to-run E2E variation: coefficient of variation of each prompt across its repeated runs, averaged over prompts.', 'Lower is steadier. Green <=10%, yellow <=25%, red >25%. Blank until a prompt has two successful runs (use --runs 2 or more).', 'derived'),
     entry('summary', 'NOTE', 'Short unavailable, skipped, or error note.', '', 'measured'),
     entry('detail', 'IN AVG / IN TOK AVG', 'Average prompt/input token count from fm count-tokens.', 'Blank when the fm build cannot count tokens.', 'measured'),
-    entry('detail', 'OUT AVG / OUT TOK AVG', 'Average output token count from fm count-tokens.', 'Blank when the fm build cannot count tokens.', 'measured'),
+    entry('detail', 'OUT AVG / OUT TOK AVG', 'Average output token count from fm count-tokens, minus its constant framing token.', 'Blank when the fm build cannot count tokens.', 'measured'),
+    entry('detail', '1ST CHUNK', 'Average output tokens carried by the first streamed stdout chunk.', 'Shows what TTFT really measures: fm streams coarse deltas (about 20 tokens in the first chunk on macOS 27.2). Wide layout only.', 'measured'),
     entry('detail', 'PREFILL/S / PREFILL TOK/S', 'Prompt tokens divided by TTFT seconds.', 'Proxy: prefill is inferred from time to first chunk, not observed directly. Higher is better.', 'proxy'),
-    entry('detail', 'DECODE/S / DECODE TOK/S', 'Output tokens after the first token divided by generation seconds.', 'Derived; requires streaming and token counts. Higher is better.', 'derived'),
+    entry('detail', 'DECODE/S / DECODE TOK/S', 'Output tokens that arrived after the first streamed delivery, summed over runs, divided by the summed generation time.', 'Derived and token-weighted, so long generations dominate short tails; requires streaming and token counts. Higher is better.', 'derived'),
     entry('detail', '2ND CHUNK', 'Delay between the first and second streamed stdout chunks, p50.', 'Lower is smoother startup. Chunk-based, not raw token telemetry.', 'proxy'),
     entry('detail', 'CHUNK P95', '95th percentile gap between consecutive streamed stdout chunks.', 'Proxy for decode smoothness at chunk granularity. Lower is smoother.', 'proxy'),
     entry('detail', 'E2E P99', '99th percentile end-to-end latency.', 'Lower is better; useful for worst-case UX.', 'measured'),
@@ -110,7 +117,8 @@ export function legendEntries() {
     entry('detail', 'REPEAT', 'Share of repeated runs for a prompt that produced the most common normalized output hash.', 'Green 90%+, yellow 50%+, red below 50%. Blank when there are not repeated comparable outputs.', 'derived'),
     entry('detail', 'ATTEMPTS', 'Total fm invocations including retries.', 'Shown in reports; a retried run is still one measured result.', 'measured'),
     entry('detail', 'DESCRIPTION', 'Model description discovered from fm help.', '', 'measured'),
-    entry('models', 'AVAILABLE', 'Whether fm available reports the model as usable on this machine right now.', '', 'measured'),
+    entry('models', 'AVAILABLE', 'Whether fm models (fm available on older builds) reports the model as usable on this machine right now.', '', 'measured'),
+    entry('models', 'IDENTITY', 'Model identity reported by fm models, for example AFM 3 Core Advanced.', 'Recorded in reports; compare warns when it changes between two runs.', 'measured'),
     entry('models', 'QUOTA', 'Quota information when the fm build exposes a quota command.', 'Column is omitted entirely when the installed fm has no quota command.', 'measured'),
     entry('metrics', 'SOURCE', 'How a metric is obtained: measured, proxy, derived, or controlled.', 'measured = observed directly; proxy = observed at coarser granularity; derived = computed from measured values.', 'measured'),
     entry('compact', 'GOOD / CV / TPOT / CHUNK', 'Compact output combines the same summary and detail metrics into model cards.', 'Same definitions and color rules as table columns.', 'derived'),
@@ -220,7 +228,7 @@ export function renderSummaryTable(summary, options = {}) {
       cell(formatNumber(item.tokensPerSecond.avg), tones.userTps),
       cell(formatNumber(item.outputTokenThroughput), tones.systemTps),
       cell(formatNumber(item.rps), tones.rps),
-      cell(formatPercent(item.latency.cv), cvTone(item.latency.cv)),
+      cell(formatPercent(stabilityCv(item)), cvTone(stabilityCv(item))),
       cell(item.available ? '' : cleanReason(item.skippedReason), item.available ? null : 'yellow')
     ];
 
@@ -274,20 +282,21 @@ export function renderDetailTable(summary, options = {}) {
       cell(formatNumber(item.promptTokens.avg, 0)),
       cell(formatNumber(item.outputTokens.avg, 0)),
       cell(formatNumber(item.prefillTokensPerSecond?.avg), tones.prefillTps),
-      cell(formatNumber(item.decodeTokensPerSecond?.avg), tones.decodeTps),
+      cell(formatNumber(decodeRate(item)), tones.decodeTps),
       cell(formatMs(item.secondChunk?.p50), tones.secondChunk),
       cell(formatMs(item.chunkGap?.p95), tones.chunkGapP95),
       cell(formatMs(item.latency.p99), tones.e2eP99),
-      cell(formatRangeMs(item.latency.ci95Low, item.latency.ci95High), cvTone(item.latency.cv)),
+      cell(formatRangeMs(item.latency.ci95Low, item.latency.ci95High), cvTone(stabilityCv(item))),
       cell(formatPercent(item.repeatability), percentTone(item.repeatability, 0.5, 0.9)),
-      cell(item.description || '-', 'muted')
+      cell(item.description || '-', 'muted'),
+      cell(formatNumber(item.firstChunkTokens?.avg, 0))
     ];
 
     if (mode === 'medium') {
       return [base[0], base[1], base[2], base[3], base[4], base[5], base[7], base[8], base[10]];
     }
 
-    return base;
+    return [base[0], base[1], base[2], base[3], base[12], ...base.slice(4, 12)];
   });
 
   const headers = mode === 'medium'
@@ -297,6 +306,7 @@ export function renderDetailTable(summary, options = {}) {
       'model',
       'in tok avg',
       'out tok avg',
+      '1st chunk',
       'prefill tok/s',
       'decode tok/s',
       '2nd chunk',
@@ -325,7 +335,7 @@ export function renderCompactSummary(summary, options = {}) {
       const goodput = item.goodputRate == null ? '' : ` | good ${formatPercent(item.goodputRate)}`;
       const tones = metricTones(item, ranks, options.slo);
       lines.push(toneText(truncate(`  TTFT ${formatMs(item.ttft.p50)} p95 ${formatMs(item.ttft.p95)} | E2E ${formatMs(item.latency.p50)} p95 ${formatMs(item.latency.p95)} p99 ${formatMs(item.latency.p99)}`, width), worstTone(tones.ttft, tones.e2e, tones.e2eP95), options));
-      lines.push(toneText(truncate(`  user ${formatNumber(item.tokensPerSecond.avg)} tok/s | system ${formatNumber(item.outputTokenThroughput)} tok/s | RPS ${formatNumber(item.rps)} | CV ${formatPercent(item.latency.cv)}${goodput}`, width), worstTone(tones.userTps, tones.systemTps, cvTone(item.latency.cv), percentTone(item.goodputRate, 0.8, 1)), options));
+      lines.push(toneText(truncate(`  user ${formatNumber(item.tokensPerSecond.avg)} tok/s | system ${formatNumber(item.outputTokenThroughput)} tok/s | RPS ${formatNumber(item.rps)} | CV ${formatPercent(stabilityCv(item))}${goodput}`, width), worstTone(tones.userTps, tones.systemTps, cvTone(stabilityCv(item)), percentTone(item.goodputRate, 0.8, 1)), options));
       lines.push(toneText(truncate(`  prefill ${formatNumber(item.prefillTokensPerSecond?.avg)} tok/s | TPOT ${formatMs(item.tpot.p50)} | chunk p95 ${formatMs(item.chunkGap?.p95)}`, width), worstTone(tones.prefillTps, tones.tpot, tones.chunkGapP95), options));
       lines.push(toneText(truncate(`  in/out ${formatNumber(item.promptTokens.avg, 0)}/${formatNumber(item.outputTokens.avg, 0)} tok avg | repeat ${formatPercent(item.repeatability)}`, width), percentTone(item.repeatability, 0.5, 0.9), options));
     } else {
@@ -344,29 +354,37 @@ export function renderModelsTable(models, options = {}) {
   if (compact) {
     return models.map((model) => {
       const status = model.available ? 'yes' : 'no';
-      return `${model.name} ${toneText(status, model.available ? 'green' : 'yellow', options)} ${compactReason(model.reason || model.description || '-')}`;
+      const detail = model.available ? (model.identity || model.description) : model.reason;
+      return `${model.name} ${toneText(status, model.available ? 'green' : 'yellow', options)} ${compactReason(detail || model.description || '-')}`;
     }).join('\n');
   }
 
   const quotaSupported = options.capabilities
     ? Boolean(options.capabilities.features?.quota)
     : models.some((model) => model.quotaSupported === true || (model.quota != null && model.quota !== ''));
+  const hasIdentity = models.some((model) => model.identity);
+  const base = (model) => {
+    const cells = [
+      cell(model.name),
+      cell(model.available ? 'yes' : 'no', model.available ? 'green' : 'yellow')
+    ];
+    if (hasIdentity) cells.push(model.identity || '-');
+    cells.push(model.description || '-');
+    return cells;
+  };
+  const leading = hasIdentity ? ['model', 'available', 'identity', 'description'] : ['model', 'available', 'description'];
 
   if (!quotaSupported) {
-    return renderTable(['model', 'available', 'description', 'notes'], models.map((model) => [
-      cell(model.name),
-      cell(model.available ? 'yes' : 'no', model.available ? 'green' : 'yellow'),
-      model.description || '-',
+    return renderTable([...leading, 'notes'], models.map((model) => [
+      ...base(model),
       cleanReason(model.reason || '-')
-    ]), { ...options, wrapColumns: ['description', 'notes'] });
+    ]), { ...options, wrapColumns: ['identity', 'description', 'notes'] });
   }
 
-  return renderTable(['model', 'available', 'description', 'quota'], models.map((model) => [
-    cell(model.name),
-    cell(model.available ? 'yes' : 'no', model.available ? 'green' : 'yellow'),
-    model.description || '-',
+  return renderTable([...leading, 'quota'], models.map((model) => [
+    ...base(model),
     cleanReason(model.quota || model.reason || '-')
-  ]), { ...options, wrapColumns: ['description', 'quota'] });
+  ]), { ...options, wrapColumns: ['identity', 'description', 'quota'] });
 }
 
 /**
@@ -390,6 +408,22 @@ export function renderModelsReport(models, options = {}) {
   return lines.join('\n');
 }
 
+/**
+ * Run-to-run CV for a summary row. Reports written before `stabilityCv`
+ * existed only have the suite-wide `latency.cv`.
+ */
+export function stabilityCv(item) {
+  return Object.hasOwn(item, 'stabilityCv') ? item.stabilityCv : item.latency?.cv ?? null;
+}
+
+/**
+ * Token-weighted decode rate for a summary row. Reports written before
+ * `decodeThroughput` existed only have the mean of per-run rates.
+ */
+export function decodeRate(item) {
+  return Object.hasOwn(item, 'decodeThroughput') ? item.decodeThroughput : item.decodeTokensPerSecond?.avg ?? null;
+}
+
 function formatRangeMs(low, high) {
   if (low == null || high == null || !Number.isFinite(low) || !Number.isFinite(high)) return '-';
   return `${formatMs(Math.max(0, low))}..${formatMs(Math.max(0, high))}`;
@@ -739,7 +773,7 @@ function rankSummary(summary) {
     totalTps: collectMetric(summary, (item) => item.totalTokenThroughput),
     rps: collectMetric(summary, (item) => item.rps),
     goodputRps: collectMetric(summary, (item) => item.goodputRps),
-    decodeTps: collectMetric(summary, (item) => item.decodeTokensPerSecond?.avg),
+    decodeTps: collectMetric(summary, decodeRate),
     prefillTps: collectMetric(summary, (item) => item.prefillTokensPerSecond?.avg)
   };
 }
@@ -788,7 +822,7 @@ function metricTones(item, ranks, slo = {}) {
     totalTps: rankTone(item.totalTokenThroughput, ranks.totalTps),
     rps: rankTone(item.rps, ranks.rps),
     goodputRps: goodputRpsTone(item, ranks.goodputRps),
-    decodeTps: rankTone(item.decodeTokensPerSecond?.avg, ranks.decodeTps),
+    decodeTps: rankTone(decodeRate(item), ranks.decodeTps),
     prefillTps: rankTone(item.prefillTokensPerSecond?.avg, ranks.prefillTps)
   };
 }