fm-bench 0.6.2 → 0.7.0

package/src/metrics.js

@@ -0,0 +1,212 @@
+// Metric provenance catalog.
+//
+// fm-bench measures `fm` from the outside, so not every named metric can be
+// observed directly. Each metric declares how it is obtained:
+//
+//   measured   — observed directly (process timings, exit codes, token counts)
+//   proxy      — observed at a coarser granularity than the ideal metric
+//   derived    — computed from other measured values
+//   controlled — an input setting, not a measurement
+//
+// A metric whose requirement is missing from the detected `fm` build is
+// reported as unavailable with a reason instead of a blank or invented value.
+
+const DEFINITIONS = [
+  {
+    key: 'ttft',
+    label: 'TTFT',
+    kind: 'proxy',
+    source: 'arrival time of the first streamed stdout chunk',
+    requires: ['streaming'],
+    reason: 'requires an fm build whose output can be streamed'
+  },
+  {
+    key: 'e2eLatency',
+    label: 'E2E latency',
+    kind: 'measured',
+    source: 'wall clock from spawning fm until it exits',
+    requires: []
+  },
+  {
+    key: 'generationMs',
+    label: 'generation time',
+    kind: 'derived',
+    source: 'E2E latency minus TTFT',
+    requires: ['streaming'],
+    reason: 'requires at least two streamed output chunks to separate prefill from decode'
+  },
+  {
+    key: 'tpot',
+    label: 'TPOT',
+    kind: 'derived',
+    source: '(E2E - TTFT) / (output tokens - 1)',
+    requires: ['streaming', 'tokenCounting'],
+    reason: 'requires streaming, a token-counting fm command, and at least three output tokens'
+  },
+  {
+    key: 'promptTokens',
+    label: 'prompt tokens',
+    kind: 'measured',
+    source: 'fm count-tokens on the prompt',
+    requires: ['tokenCounting'],
+    reason: 'this fm build exposes no token-counting command'
+  },
+  {
+    key: 'outputTokens',
+    label: 'output tokens',
+    kind: 'measured',
+    source: 'fm count-tokens on the captured output',
+    requires: ['tokenCounting'],
+    reason: 'this fm build exposes no token-counting command'
+  },
+  {
+    key: 'tokensPerSecond',
+    label: 'per-request output tokens/s',
+    kind: 'derived',
+    source: 'output tokens / E2E seconds',
+    requires: ['tokenCounting'],
+    reason: 'requires a token-counting fm command'
+  },
+  {
+    key: 'decodeTokensPerSecond',
+    label: 'decode tokens/s',
+    kind: 'derived',
+    source: '(output tokens - 1) / generation seconds',
+    requires: ['streaming', 'tokenCounting'],
+    reason: 'requires streaming, a token-counting fm command, and at least three output tokens'
+  },
+  {
+    key: 'prefillTokensPerSecond',
+    label: 'prefill tokens/s',
+    kind: 'proxy',
+    source: 'prompt tokens / TTFT seconds',
+    requires: ['streaming', 'tokenCounting'],
+    reason: 'requires streaming plus a token-counting fm command'
+  },
+  {
+    key: 'outputTokenThroughput',
+    label: 'aggregate output token throughput',
+    kind: 'derived',
+    source: 'successful output tokens / measured wall-clock window',
+    requires: ['tokenCounting'],
+    reason: 'requires a token-counting fm command'
+  },
+  {
+    key: 'rps',
+    label: 'request throughput',
+    kind: 'measured',
+    source: 'successful requests / measured wall-clock window',
+    requires: []
+  },
+  {
+    key: 'chunkGaps',
+    label: 'chunk gaps and second-chunk delay',
+    kind: 'proxy',
+    source: 'gaps between consecutive streamed stdout chunks',
+    requires: ['streaming'],
+    reason: 'requires an fm build whose output can be streamed'
+  },
+  {
+    key: 'percentiles',
+    label: 'percentiles',
+    kind: 'derived',
+    source: 'percentile interpolation over successful samples',
+    requires: []
+  },
+  {
+    key: 'successRate',
+    label: 'success rate',
+    kind: 'measured',
+    source: 'successful runs / attempted runs',
+    requires: []
+  },
+  {
+    key: 'goodput',
+    label: 'goodput',
+    kind: 'derived',
+    source: 'successful runs meeting every configured SLO / runs with an SLO verdict',
+    requires: ['slo'],
+    reason: 'set --slo-ttft-ms, --slo-e2e-ms, or --slo-tpot-ms to enable goodput'
+  },
+  {
+    key: 'repeatability',
+    label: 'repeatability',
+    kind: 'derived',
+    source: 'most common normalized output hash share across repeated runs',
+    requires: []
+  },
+  {
+    key: 'variability',
+    label: 'CV and 95% confidence interval',
+    kind: 'derived',
+    source: 'sample standard deviation over successful samples',
+    requires: [],
+    reason: 'needs at least two successful samples'
+  },
+  {
+    key: 'quota',
+    label: 'quota',
+    kind: 'measured',
+    source: 'fm quota-usage',
+    requires: ['quota'],
+    reason: 'this fm build exposes no quota command'
+  }
+];
+
+function requirementsMet(requires = [], context = {}) {
+  return requires.every((name) => {
+    if (name === 'streaming') return Boolean(context.streaming);
+    if (name === 'tokenCounting') return Boolean(context.tokenCounting);
+    if (name === 'quota') return Boolean(context.quota);
+    if (name === 'slo') return Boolean(context.slo);
+    return true;
+  });
+}
+
+/**
+ * Describe which metrics the detected `fm` build can support for a run.
+ *
+ * @param {{ features?: Record<string, any> }} [capabilities]
+ * @param {{ stream?: boolean, slo?: boolean }} [options]
+ */
+export function metricAvailability(capabilities = {}, options = {}) {
+  const features = capabilities.features ?? {};
+  const context = {
+    streaming: options.stream !== false && features.streaming !== false,
+    tokenCounting: features.tokenCounting !== false,
+    quota: features.quota === true,
+    slo: options.slo === true
+  };
+
+  const metrics = {};
+  for (const definition of DEFINITIONS) {
+    const available = requirementsMet(definition.requires, context);
+    metrics[definition.key] = {
+      label: definition.label,
+      kind: definition.kind,
+      source: definition.source,
+      available,
+      unavailableReason: available ? '' : (definition.reason ?? 'unavailable in this environment')
+    };
+  }
+  return metrics;
+}
+
+export function metricDefinition(key) {
+  const definition = DEFINITIONS.find((item) => item.key === key);
+  return definition ? { ...definition, requires: [...definition.requires] } : null;
+}
+
+export function metricDefinitions() {
+  return DEFINITIONS.map((definition) => ({ ...definition, requires: [...definition.requires] }));
+}
+
+/** Short human line for CLI output, e.g. "token counting: yes, quota: no". */
+export function formatCapabilitySummary(capabilities = {}) {
+  const features = capabilities.features ?? {};
+  return [
+    `token counting ${features.tokenCounting ? 'yes' : 'no'}`,
+    `streaming ${features.streaming ? 'yes' : 'no'}`,
+    `quota ${features.quota ? 'yes' : 'no'}`
+  ].join(', ');
+}

package/src/process.js

@@ -1,5 +1,31 @@
 import { spawn } from 'node:child_process';
 
+// Children currently alive. The CLI registers signal handlers that terminate
+// these so Ctrl+C cannot leave `fm` processes behind.
+const activeChildren = new Set();
+
+export function activeChildCount() {
+  return activeChildren.size;
+}
+
+/**
+ * Terminate every running child process. Returns how many were signalled.
+ * @param {NodeJS.Signals} signal
+ */
+export function killActiveChildren(signal = 'SIGTERM') {
+  let killed = 0;
+  for (const child of activeChildren) {
+    if (child.exitCode != null || child.signalCode != null) continue;
+    try {
+      child.kill(signal);
+      killed += 1;
+    } catch {
+      // Process already gone.
+    }
+  }
+  return killed;
+}
+
 export function runProcess(command, args = [], options = {}) {
   const {
     input,
@@ -15,6 +41,7 @@ export function runProcess(command, args = [], options = {}) {
       env,
       stdio: ['pipe', 'pipe', 'pipe']
     });
+    activeChildren.add(child);
 
     let stdout = '';
     let stderr = '';
@@ -55,11 +82,16 @@ export function runProcess(command, args = [], options = {}) {
       stderr += chunk;
     });
 
-    child.on('error', (error) => {
+    const finish = (result) => {
+      if (settled) return;
       settled = true;
+      activeChildren.delete(child);
       if (timer) clearTimeout(timer);
-      const endedAt = process.hrtime.bigint();
-      resolve({
+      resolve(result);
+    };
+
+    child.on('error', (error) => {
+      finish({
         command,
         args,
         code: null,
@@ -73,15 +105,12 @@ export function runProcess(command, args = [], options = {}) {
         firstStderrMs,
         error,
         timedOut,
-        durationMs: Number(endedAt - startedAt) / 1e6
+        durationMs: Number(process.hrtime.bigint() - startedAt) / 1e6
       });
     });
 
     child.on('close', (code, signal) => {
-      settled = true;
-      if (timer) clearTimeout(timer);
-      const endedAt = process.hrtime.bigint();
-      resolve({
+      finish({
         command,
         args,
         code,
@@ -93,8 +122,9 @@ export function runProcess(command, args = [], options = {}) {
         stdoutChunkTimesMs,
         firstStdoutMs,
         firstStderrMs,
+        error: null,
         timedOut,
-        durationMs: Number(endedAt - startedAt) / 1e6
+        durationMs: Number(process.hrtime.bigint() - startedAt) / 1e6
       });
     });
 

package/src/prompts.js

@@ -195,16 +195,28 @@ export async function loadPrompts(options = {}) {
 
 async function loadPromptFile(filePath) {
   const absolutePath = path.resolve(filePath);
-  const content = await fs.readFile(absolutePath, 'utf8');
+  let content;
+  try {
+    content = await fs.readFile(absolutePath, 'utf8');
+  } catch (error) {
+    throw new Error(error.code === 'ENOENT'
+      ? `Prompt file not found: ${absolutePath}`
+      : `Cannot read prompt file ${absolutePath}: ${error.message}`);
+  }
   const trimmed = content.trim();
 
   if (!trimmed) return [];
 
   if (absolutePath.endsWith('.json')) {
-    const parsed = JSON.parse(trimmed);
+    let parsed;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch (error) {
+      throw new Error(`Cannot parse ${absolutePath} as JSON: ${error.message}`);
+    }
     const items = Array.isArray(parsed) ? parsed : parsed.prompts;
     if (!Array.isArray(items)) {
-      throw new Error('Prompt JSON must be an array or an object with a prompts array');
+      throw new Error(`${absolutePath} must be a JSON array of prompts or an object with a prompts array`);
     }
     return items.map((item, index) => normalizePromptItem(item, index));
   }
@@ -212,7 +224,13 @@ async function loadPromptFile(filePath) {
   if (absolutePath.endsWith('.jsonl')) {
     return trimmed.split(/\r?\n/)
       .filter(Boolean)
-      .map((line, index) => normalizePromptItem(JSON.parse(line), index));
+      .map((line, index) => {
+        try {
+          return normalizePromptItem(JSON.parse(line), index);
+        } catch (error) {
+          throw new Error(`Cannot parse line ${index + 1} of ${absolutePath} as JSON: ${error.message}`);
+        }
+      });
   }
 
   return trimmed.split(/\n\s*\n/g).map((prompt, index) => ({

package/src/report.js

@@ -16,6 +16,7 @@ export function flattenResults(results) {
     concurrency: result.concurrency ?? '',
     prompt_id: result.promptId,
     run: result.run,
+    attempts: result.attempts ?? 1,
     ok: result.ok,
     duration_ms: round(result.durationMs),
     ttft_ms: round(result.firstTokenMs),
@@ -58,9 +59,19 @@ export async function writeReport(filePath, payload, format) {
 
 function csvEscape(value) {
   const text = String(value ?? '');
-  if (/[",\n\r]/.test(text)) {
-    return `"${text.replaceAll('"', '""')}"`;
+  const safe = guardFormula(text);
+  if (/[",\n\r]/.test(safe)) {
+    return `"${safe.replaceAll('"', '""')}"`;
   }
+  return safe;
+}
+
+// Spreadsheet programs execute cells that start with =, +, @, or a non-numeric
+// leading -. Prompt text and captured output are untrusted input, so prefix
+// those cells with a single quote to keep them literal text.
+function guardFormula(text) {
+  if (/^[=+@\t\r]/.test(text)) return `'${text}`;
+  if (text.startsWith('-') && !/^-?\d+(\.\d+)?$/.test(text)) return `'${text}`;
   return text;
 }
 

package/src/schema.js

@@ -126,10 +126,26 @@ export function validateReport(value) {
   for (const key of REQUIRED_TOP_LEVEL) {
     if (!(key in report)) errors.push(`missing required field: ${key}`);
   }
-  if (!Array.isArray(report.summary)) errors.push('summary must be an array');
+  if (!Array.isArray(report.summary)) {
+    errors.push('summary must be an array');
+  } else {
+    for (const [index, item] of report.summary.entries()) {
+      if (!item || typeof item !== 'object') {
+        errors.push(`summary[${index}] must be an object`);
+        continue;
+      }
+      const row = /** @type {Record<string, unknown>} */ (item);
+      if (typeof row.model !== 'string' || row.model === '') {
+        errors.push(`summary[${index}].model must be a non-empty string`);
+      }
+    }
+  }
   if (report.schemaVersion != null && report.schemaVersion !== REPORT_SCHEMA_VERSION) {
     errors.push(`unsupported schemaVersion: ${report.schemaVersion} (expected ${REPORT_SCHEMA_VERSION})`);
   }
+  if (report.metrics != null && (typeof report.metrics !== 'object' || Array.isArray(report.metrics))) {
+    errors.push('metrics must be an object when present');
+  }
   if (errors.length > 0) return { ok: false, errors };
   return { ok: true, report };
 }

package/src/stats.js

@@ -1,3 +1,12 @@
+/**
+ * Sample statistics for one metric.
+ *
+ * Spread metrics (`stddev`, `cv`, `ci95*`) need at least two samples. With a
+ * single sample they are `null` rather than `0`, because a "0% variation" or a
+ * zero-width confidence interval is invented precision, not a measurement.
+ *
+ * @param {number[]} values
+ */
 export function summarizeNumbers(values) {
   const clean = values.filter((value) => Number.isFinite(value)).sort((a, b) => a - b);
   if (clean.length === 0) {
@@ -20,11 +29,12 @@ export function summarizeNumbers(values) {
 
   const total = clean.reduce((sum, value) => sum + value, 0);
   const avg = total / clean.length;
-  const variance = clean.length > 1
+  const hasSpread = clean.length > 1;
+  const variance = hasSpread
     ? clean.reduce((sum, value) => sum + (value - avg) ** 2, 0) / (clean.length - 1)
-    : 0;
-  const stddev = Math.sqrt(variance);
-  const margin = clean.length > 1 ? tCritical95(clean.length) * (stddev / Math.sqrt(clean.length)) : 0;
+    : null;
+  const stddev = variance == null ? null : Math.sqrt(variance);
+  const margin = hasSpread ? tCritical95(clean.length) * (stddev / Math.sqrt(clean.length)) : null;
   return {
     count: clean.length,
     min: clean[0],
@@ -32,9 +42,9 @@ export function summarizeNumbers(values) {
     avg,
     sum: total,
     stddev,
-    cv: avg !== 0 ? stddev / Math.abs(avg) : null,
-    ci95Low: avg - margin,
-    ci95High: avg + margin,
+    cv: hasSpread && avg !== 0 ? stddev / Math.abs(avg) : null,
+    ci95Low: hasSpread ? avg - margin : null,
+    ci95High: hasSpread ? avg + margin : null,
     p50: percentile(clean, 50),
     p90: percentile(clean, 90),
     p95: percentile(clean, 95),
@@ -42,16 +52,25 @@ export function summarizeNumbers(values) {
   };
 }
 
-export function percentile(sortedValues, percentileValue) {
-  if (sortedValues.length === 0) return null;
-  if (sortedValues.length === 1) return sortedValues[0];
+/**
+ * Percentile with linear interpolation between closest ranks (the same method
+ * as Excel's PERCENTILE.INC / NumPy's default). Accepts unsorted input so
+ * callers cannot silently get a wrong answer from an unsorted array.
+ *
+ * @param {number[]} values
+ * @param {number} percentileValue 0-100
+ */
+export function percentile(values, percentileValue) {
+  const sorted = values.filter((value) => Number.isFinite(value)).sort((a, b) => a - b);
+  if (sorted.length === 0) return null;
+  if (sorted.length === 1) return sorted[0];
 
-  const rank = (percentileValue / 100) * (sortedValues.length - 1);
+  const rank = (percentileValue / 100) * (sorted.length - 1);
   const low = Math.floor(rank);
   const high = Math.ceil(rank);
-  if (low === high) return sortedValues[low];
+  if (low === high) return sorted[low];
   const weight = rank - low;
-  return sortedValues[low] * (1 - weight) + sortedValues[high] * weight;
+  return sorted[low] * (1 - weight) + sorted[high] * weight;
 }
 
 export function summarizeByModel(results, modelStatuses = [], options = {}) {
@@ -66,6 +85,7 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
         concurrency,
         description: status.description,
         available: status.available,
+        unsupported: Boolean(status.unsupported),
         skippedReason: status.available ? '' : status.reason || 'Unavailable',
         results: []
       });
@@ -80,6 +100,7 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
         concurrency: result.concurrency,
         description: '',
         available: true,
+        unsupported: false,
         skippedReason: '',
         results: []
       });
@@ -98,7 +119,7 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
     const tpot = summarizeNumbers(successes.map((result) => result.tpotMs).filter((value) => value != null));
     const promptTokens = summarizeNumbers(successes.map((result) => result.promptTokens).filter((value) => value != null));
     const outputTokens = summarizeNumbers(successes.map((result) => result.outputTokens).filter((value) => value != null));
-    const charsPerSecond = summarizeNumbers(successes.map((result) => result.charsPerSecond));
+    const charsPerSecond = summarizeNumbers(successes.map((result) => result.charsPerSecond).filter((value) => value != null));
     const tokensPerSecond = summarizeNumbers(successes.map((result) => result.tokensPerSecond).filter((value) => value != null));
     const decodeTokensPerSecond = summarizeNumbers(successes.map((result) => result.decodeTokensPerSecond).filter((value) => value != null));
     const prefillTokensPerSecond = summarizeNumbers(successes.map((result) => result.prefillTokensPerSecond).filter((value) => value != null));
@@ -110,14 +131,18 @@ export function summarizeByModel(results, modelStatuses = [], options = {}) {
     const outputTokenThroughput = outputTokens.sum > 0 && windowMs > 0 ? outputTokens.sum / (windowMs / 1000) : null;
     const totalTokens = promptTokens.sum + outputTokens.sum;
     const totalTokenThroughput = totalTokens > 0 && windowMs > 0 ? totalTokens / (windowMs / 1000) : null;
+    const attempts = entry.results.reduce((sum, result) => sum + (result.attempts ?? 1), 0);
 
     return {
       model: entry.model,
       concurrency: entry.concurrency,
       description: entry.description,
       available: entry.available,
+      unsupported: entry.unsupported,
       skippedReason: entry.skippedReason,
       attempted: entry.results.length,
+      attempts,
+      retried: entry.results.length > 0 ? Math.max(0, attempts - entry.results.length) : 0,
       successes: successes.length,
       failures: failures.length,
       successRate: entry.results.length > 0 ? successes.length / entry.results.length : null,
@@ -147,6 +172,10 @@ function summaryKey(model, concurrency) {
   return `${model}::${concurrency ?? 'default'}`;
 }
 
+// Two-sided 95% t critical values. Exact table entries up to 30 degrees of
+// freedom, then the standard 2.0 / 1.96 approximations for larger samples.
+// fm-bench uses this for a mean confidence interval, which is context for
+// small samples rather than a hypothesis test.
 function tCritical95(n) {
   const df = Math.max(1, n - 1);
   const table = {