fm-bench 0.6.3 → 0.7.1

package/src/fm.js

@@ -1,60 +1,16 @@
-import crypto from 'node:crypto';
 import os from 'node:os';
 import { stripAnsi } from './ansi.js';
+import { detectFmCapabilities } from './capabilities.js';
+import { firstLine, isUnsupportedModelError, parseAvailabilityOutput } from './fm-help.js';
 import { runProcess } from './process.js';
 import { parseBatteryOutput, parseThermalOutput } from './system.js';
 
-const DEFAULT_MODELS = [
-  { name: 'system', description: 'On-device Apple Foundation Model' },
-  { name: 'pcc', description: 'Apple Foundation Model on Private Cloud Compute' }
-];
+export { parseModelsFromHelp, parseAvailabilityOutput } from './fm-help.js';
 
 export function fmBinaryFromOptions(options = {}) {
   return options.fmBin || process.env.FM_BIN || 'fm';
 }
 
-export function parseModelsFromHelp(helpText) {
-  const clean = stripAnsi(helpText);
-  const lines = clean.split(/\r?\n/);
-  const models = new Map();
-  let inModels = false;
-
-  for (const line of lines) {
-    if (/^\s*MODELS\s*$/.test(line)) {
-      inModels = true;
-      continue;
-    }
-
-    if (inModels && /^\s*[A-Z][A-Z -]+\s*$/.test(line) && !/^\s*MODELS\s*$/.test(line)) {
-      inModels = false;
-    }
-
-    if (inModels) {
-      const match = line.match(/^\s*([A-Za-z0-9._:-]+)\s{2,}(.+?)\s*$/);
-      if (match) {
-        models.set(match[1], {
-          name: match[1],
-          description: match[2].replace(/\s*\(default\)\s*$/, '').trim()
-        });
-      }
-    }
-
-    const optionMatch = /--model\b/.test(line)
-      ? line.match(/\bmodel\b.*?\(([^)]+)\)/i)
-      : null;
-    if (optionMatch) {
-      for (const raw of optionMatch[1].split(',')) {
-        const name = raw.trim();
-        if (/^[A-Za-z0-9._:-]+$/.test(name) && !models.has(name)) {
-          models.set(name, { name, description: '' });
-        }
-      }
-    }
-  }
-
-  return [...models.values()];
-}
-
 export async function getFmHelp(fmBin, timeoutMs = 10_000) {
   const result = await runProcess(fmBin, ['--help'], { timeoutMs });
   if (result.error) {
@@ -69,40 +25,26 @@ export async function getFmHelp(fmBin, timeoutMs = 10_000) {
   };
 }
 
-export async function discoverModels(options = {}) {
-  const fmBin = fmBinaryFromOptions(options);
-  const help = await getFmHelp(fmBin, options.timeoutMs ?? 10_000);
-  let models = parseModelsFromHelp(help.text);
-
-  if (models.length === 0 && /Apple Foundation Models CLI/i.test(stripAnsi(help.text))) {
-    models = DEFAULT_MODELS;
+/**
+ * Check one model against the detected `fm` build.
+ *
+ * Models the build does not expose are reported as unsupported without
+ * spawning `fm` at all, so a raw argument-error blob from `fm` can never end
+ * up in a report or in `fm-bench models` output.
+ */
+export async function checkModelAvailability(fmBin, model, options = {}) {
+  const capabilities = options.capabilities;
+  const known = capabilities?.models?.map((entry) => entry.name) ?? [];
+  if (capabilities && known.length > 0 && !known.includes(model)) {
+    return {
+      model,
+      available: false,
+      unsupported: true,
+      raw: '',
+      reason: `not supported by this fm build (supported: ${known.join(', ')})`
+    };
   }
 
-  return {
-    fmBin,
-    models,
-    help: stripAnsi(help.text)
-  };
-}
-
-export function parseAvailabilityOutput(model, output, code) {
-  const clean = stripAnsi(output).trim();
-  const lower = clean.toLowerCase();
-  const modelLower = model.toLowerCase();
-  const hasError = /\berror:|\bunavailable\b|\bnot available\b|\bnot supported\b/.test(lower);
-  const hasAvailable = new RegExp(`\\b${escapeRegExp(modelLower)}\\b[\\s\\S]{0,80}\\bavailable\\b|\\bavailable\\b[\\s\\S]{0,80}\\b${escapeRegExp(modelLower)}\\b`).test(lower)
-    || lower.includes(`${modelLower} model available`)
-    || lower.includes(`${titleCase(modelLower)} model available`.toLowerCase());
-
-  return {
-    model,
-    available: code === 0 && hasAvailable && !hasError,
-    raw: clean,
-    reason: hasError ? clean : ''
-  };
-}
-
-export async function checkModelAvailability(fmBin, model, options = {}) {
   const result = await runProcess(fmBin, ['available', '--model', model], {
     timeoutMs: options.timeoutMs ?? 15_000
   });
@@ -111,53 +53,99 @@ export async function checkModelAvailability(fmBin, model, options = {}) {
   if (result.error) {
     parsed.available = false;
     parsed.reason = result.stderr || result.error.message;
+    return parsed;
+  }
+  if (isUnsupportedModelError(output)) {
+    parsed.available = false;
+    parsed.unsupported = true;
+    const supported = known.length > 0 ? ` (supported: ${known.join(', ')})` : '';
+    parsed.reason = `not supported by this fm build${supported}`;
   }
   return parsed;
 }
 
+/**
+ * Query quota information when the build exposes it.
+ * @returns {Promise<{ model: string, supported: boolean, ok: boolean, raw: string, reason: string }>}
+ */
 export async function getQuotaUsage(fmBin, model, options = {}) {
+  const features = options.capabilities?.features;
+  if (features && !features.quota) {
+    return {
+      model,
+      supported: false,
+      ok: false,
+      raw: '',
+      reason: 'unavailable: this fm build exposes no quota command'
+    };
+  }
+
   const result = await runProcess(fmBin, ['quota-usage', '--model', model], {
     timeoutMs: options.timeoutMs ?? 15_000
   });
   const output = stripAnsi(`${result.stdout}${result.stderr}`).trim();
   return {
     model,
+    supported: true,
     ok: result.code === 0,
     raw: output,
-    unavailable: /\bunavailable\b|\bnot available\b|\berror:/i.test(output)
+    reason: result.code === 0 ? '' : firstLine(output)
   };
 }
 
+/**
+ * Count tokens with whichever token-counting command this `fm` build exposes.
+ * Returns `ok: false` with a reason when the build cannot count tokens; it
+ * never invents a count.
+ */
 export async function countTokens(fmBin, text, options = {}) {
-  const result = await runProcess(fmBin, ['token-count', '--quiet'], {
+  const command = options.capabilities?.features?.tokenCountCommand ?? 'count-tokens';
+  const supported = options.capabilities?.features?.tokenCounting ?? true;
+  if (!supported || !command) {
+    return {
+      ok: false,
+      count: null,
+      unsupported: true,
+      raw: '',
+      reason: 'token counting is unavailable in this fm build'
+    };
+  }
+
+  const result = await runProcess(fmBin, [command, '--quiet'], {
     input: text,
     timeoutMs: options.timeoutMs ?? 15_000
   });
   const output = stripAnsi(`${result.stdout}${result.stderr}`).trim();
-  const match = output.match(/-?\d+/);
-  if (result.code !== 0 || !match) {
+  const match = output.match(/\d+/);
+  if (result.error || result.code !== 0 || !match) {
     return {
       ok: false,
       count: null,
-      raw: output
+      raw: output,
+      reason: result.error?.message || firstLine(output) || `fm ${command} exited with code ${result.code}`
     };
   }
   return {
     ok: true,
     count: Number.parseInt(match[0], 10),
-    raw: output
+    raw: output,
+    reason: ''
   };
 }
 
 export async function respond(fmBin, model, prompt, options = {}) {
-  const args = ['respond', '--model', model];
-  const streamed = options.stream !== false;
-
-  if (!streamed) args.push('--no-stream');
-  if (options.greedy) args.push('--greedy');
-  if (options.instructions) args.push('--instructions', options.instructions);
-  if (options.useCase) args.push('--use-case', options.useCase);
-  if (options.guardrails) args.push('--guardrails', options.guardrails);
+  const features = options.capabilities?.features;
+  const streamControl = features ? features.streaming : true;
+  const modelSelection = features ? features.modelSelection : true;
+  const streamed = streamControl && options.stream !== false;
+
+  const args = ['respond'];
+  if (modelSelection) args.push('--model', model);
+  if (streamControl && !streamed) args.push('--no-stream');
+  if (options.greedy && (features?.greedy ?? true)) args.push('--greedy');
+  if (options.instructions && (features?.instructions ?? true)) args.push('--instructions', options.instructions);
+  if (options.useCase && (features?.useCase ?? true)) args.push('--use-case', options.useCase);
+  if (options.guardrails && (features?.guardrails ?? true)) args.push('--guardrails', options.guardrails);
 
   const result = await runProcess(fmBin, args, {
     input: prompt,
@@ -166,8 +154,10 @@ export async function respond(fmBin, model, prompt, options = {}) {
 
   const output = stripAnsi(result.stdout).trim();
   const errorText = stripAnsi(result.stderr).trim();
+  const failed = result.code !== 0 || result.timedOut;
+
   return {
-    ok: result.code === 0 && !result.timedOut,
+    ok: !failed,
     model,
     prompt,
     output,
@@ -179,11 +169,17 @@ export async function respond(fmBin, model, prompt, options = {}) {
     firstOutputMs: streamed ? result.firstStdoutMs : null,
     streamed,
     stdoutChunks: result.stdoutChunks,
-    stdoutChunkTimesMs: streamed ? result.stdoutChunkTimesMs : []
+    stdoutChunkTimesMs: streamed ? result.stdoutChunkTimesMs : [],
+    error: failed
+      ? (result.timedOut
+        ? `timed out after ${options.timeoutMs ?? 60_000}ms`
+        : firstLine(errorText) || `fm exited with code ${result.code ?? result.signal}`)
+      : ''
   };
 }
 
-export async function collectEnvironment(fmBin) {
+export async function collectEnvironment(fmBin, options = {}) {
+  const capabilities = options.capabilities;
   const swVers = await runProcess('sw_vers', [], { timeoutMs: 5_000 });
   const macOS = stripAnsi(swVers.stdout).trim() || null;
 
@@ -196,15 +192,15 @@ export async function collectEnvironment(fmBin) {
   const batteryResult = await runProcess('pmset', ['-g', 'batt'], { timeoutMs: 5_000 });
   const battery = parseBatteryOutput(`${batteryResult.stdout || ''}${batteryResult.stderr || ''}`);
 
-  let fmHelpDigest = null;
-  try {
-    const help = await getFmHelp(fmBin, 10_000);
-    const text = stripAnsi(help.text).trim();
-    if (text) {
-      fmHelpDigest = crypto.createHash('sha256').update(text).digest('hex').slice(0, 16);
+  let fmHelpDigest = capabilities?.digest ?? null;
+  if (fmHelpDigest == null) {
+    try {
+      const help = await getFmHelp(fmBin, 10_000);
+      const detected = await detectFmCapabilities(fmBin, { help: { text: help.text } });
+      fmHelpDigest = detected.digest;
+    } catch {
+      fmHelpDigest = null;
     }
-  } catch {
-    fmHelpDigest = null;
   }
 
   const memRaw = (memBytes.stdout || '').trim();
@@ -237,11 +233,3 @@ export async function collectEnvironment(fmBin) {
       : null
   };
 }
-
-function escapeRegExp(value) {
-  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-}
-
-function titleCase(value) {
-  return value.slice(0, 1).toUpperCase() + value.slice(1);
-}

package/src/history.js

@@ -1,5 +1,6 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { stripAnsi } from './ansi.js';
 import { formatMs, formatNumber, formatPercent } from './table.js';
 
 export async function loadHistory(dir) {
@@ -100,7 +101,7 @@ export function renderHistoryReport(reports, options = {}) {
 }
 
 function fit(text, width) {
-  const str = String(text ?? '');
+  const str = stripAnsi(String(text ?? '')).replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, '');
   if (str.length <= width) return str + ' '.repeat(width - str.length);
   return `${str.slice(0, width - 1)}…`;
 }

package/src/macos.js

@@ -57,7 +57,8 @@ export function evaluateMacosSupport(platform, parsed) {
 export function formatMacosRequirementError({ reason, latestSupported }) {
   return [
     `unsupported macOS: ${reason}`,
-    `Latest supported: ${latestSupported} (fm is not available on older macOS releases).`
+    `Latest supported: ${latestSupported} (fm is not available on older macOS releases).`,
+    'Pass --fm-bin <path> (or set FM_BIN) to benchmark an fm binary you provide on this host.'
   ].join('\n');
 }
 

package/src/metrics.js

@@ -0,0 +1,203 @@
+// Metric provenance catalog.
+//
+// fm-bench measures `fm` from the outside, so not every named metric can be
+// observed directly. Each metric declares how it is obtained:
+//
+//   measured   — observed directly (process timings, exit codes, token counts)
+//   proxy      — observed at a coarser granularity than the ideal metric
+//   derived    — computed from other measured values
+//   controlled — an input setting, not a measurement
+//
+// A metric whose requirement is missing from the detected `fm` build is
+// reported as unavailable with a reason instead of a blank or invented value.
+
+const DEFINITIONS = [
+  {
+    key: 'ttft',
+    label: 'TTFT',
+    kind: 'proxy',
+    source: 'arrival time of the first streamed stdout chunk',
+    requires: ['streaming'],
+    reason: 'requires an fm build whose output can be streamed'
+  },
+  {
+    key: 'e2eLatency',
+    label: 'E2E latency',
+    kind: 'measured',
+    source: 'wall clock from spawning fm until it exits',
+    requires: []
+  },
+  {
+    key: 'generationMs',
+    label: 'generation time',
+    kind: 'derived',
+    source: 'E2E latency minus TTFT',
+    requires: ['streaming'],
+    reason: 'requires at least two streamed output chunks to separate prefill from decode'
+  },
+  {
+    key: 'tpot',
+    label: 'TPOT',
+    kind: 'derived',
+    source: '(E2E - TTFT) / (output tokens - 1)',
+    requires: ['streaming', 'tokenCounting'],
+    reason: 'requires streaming, a token-counting fm command, and at least three output tokens'
+  },
+  {
+    key: 'promptTokens',
+    label: 'prompt tokens',
+    kind: 'measured',
+    source: 'fm count-tokens on the prompt',
+    requires: ['tokenCounting'],
+    reason: 'this fm build exposes no token-counting command'
+  },
+  {
+    key: 'outputTokens',
+    label: 'output tokens',
+    kind: 'measured',
+    source: 'fm count-tokens on the captured output',
+    requires: ['tokenCounting'],
+    reason: 'this fm build exposes no token-counting command'
+  },
+  {
+    key: 'tokensPerSecond',
+    label: 'per-request output tokens/s',
+    kind: 'derived',
+    source: 'output tokens / E2E seconds',
+    requires: ['tokenCounting'],
+    reason: 'requires a token-counting fm command'
+  },
+  {
+    key: 'decodeTokensPerSecond',
+    label: 'decode tokens/s',
+    kind: 'derived',
+    source: '(output tokens - 1) / generation seconds',
+    requires: ['streaming', 'tokenCounting'],
+    reason: 'requires streaming, a token-counting fm command, and at least three output tokens'
+  },
+  {
+    key: 'prefillTokensPerSecond',
+    label: 'prefill tokens/s',
+    kind: 'proxy',
+    source: 'prompt tokens / TTFT seconds',
+    requires: ['streaming', 'tokenCounting'],
+    reason: 'requires streaming plus a token-counting fm command'
+  },
+  {
+    key: 'outputTokenThroughput',
+    label: 'aggregate output token throughput',
+    kind: 'derived',
+    source: 'successful output tokens / measured wall-clock window',
+    requires: ['tokenCounting'],
+    reason: 'requires a token-counting fm command'
+  },
+  {
+    key: 'rps',
+    label: 'request throughput',
+    kind: 'measured',
+    source: 'successful requests / measured wall-clock window',
+    requires: []
+  },
+  {
+    key: 'chunkGaps',
+    label: 'chunk gaps and second-chunk delay',
+    kind: 'proxy',
+    source: 'gaps between consecutive streamed stdout chunks',
+    requires: ['streaming'],
+    reason: 'requires an fm build whose output can be streamed'
+  },
+  {
+    key: 'percentiles',
+    label: 'percentiles',
+    kind: 'derived',
+    source: 'percentile interpolation over successful samples',
+    requires: []
+  },
+  {
+    key: 'successRate',
+    label: 'success rate',
+    kind: 'measured',
+    source: 'successful runs / attempted runs',
+    requires: []
+  },
+  {
+    key: 'goodput',
+    label: 'goodput',
+    kind: 'derived',
+    source: 'successful runs meeting every configured SLO / runs with an SLO verdict',
+    requires: ['slo'],
+    reason: 'set --slo-ttft-ms, --slo-e2e-ms, or --slo-tpot-ms to enable goodput'
+  },
+  {
+    key: 'repeatability',
+    label: 'repeatability',
+    kind: 'derived',
+    source: 'most common normalized output hash share across repeated runs',
+    requires: []
+  },
+  {
+    key: 'variability',
+    label: 'CV and 95% confidence interval',
+    kind: 'derived',
+    source: 'sample standard deviation over successful samples',
+    requires: [],
+    reason: 'needs at least two successful samples'
+  },
+  {
+    key: 'quota',
+    label: 'quota',
+    kind: 'measured',
+    source: 'fm quota-usage',
+    requires: ['quota'],
+    reason: 'this fm build exposes no quota command'
+  }
+];
+
+function requirementsMet(requires = [], context = {}) {
+  return requires.every((name) => {
+    if (name === 'streaming') return Boolean(context.streaming);
+    if (name === 'tokenCounting') return Boolean(context.tokenCounting);
+    if (name === 'quota') return Boolean(context.quota);
+    if (name === 'slo') return Boolean(context.slo);
+    return true;
+  });
+}
+
+/**
+ * Describe which metrics the detected `fm` build can support for a run.
+ *
+ * @param {{ features?: Record<string, any> }} [capabilities]
+ * @param {{ stream?: boolean, slo?: boolean }} [options]
+ */
+export function metricAvailability(capabilities = {}, options = {}) {
+  const features = capabilities.features ?? {};
+  const context = {
+    streaming: options.stream !== false && features.streaming !== false,
+    tokenCounting: features.tokenCounting !== false,
+    quota: features.quota === true,
+    slo: options.slo === true
+  };
+
+  const metrics = {};
+  for (const definition of DEFINITIONS) {
+    const available = requirementsMet(definition.requires, context);
+    metrics[definition.key] = {
+      label: definition.label,
+      kind: definition.kind,
+      source: definition.source,
+      available,
+      unavailableReason: available ? '' : (definition.reason ?? 'unavailable in this environment')
+    };
+  }
+  return metrics;
+}
+
+/** Short human line for CLI output, e.g. "token counting: yes, quota: no". */
+export function formatCapabilitySummary(capabilities = {}) {
+  const features = capabilities.features ?? {};
+  return [
+    `token counting ${features.tokenCounting ? 'yes' : 'no'}`,
+    `streaming ${features.streaming ? 'yes' : 'no'}`,
+    `quota ${features.quota ? 'yes' : 'no'}`
+  ].join(', ');
+}

package/src/process.js

@@ -1,5 +1,31 @@
 import { spawn } from 'node:child_process';
 
+// Children currently alive. The CLI registers signal handlers that terminate
+// these so Ctrl+C cannot leave `fm` processes behind.
+const activeChildren = new Set();
+
+export function activeChildCount() {
+  return activeChildren.size;
+}
+
+/**
+ * Terminate every running child process. Returns how many were signalled.
+ * @param {NodeJS.Signals} signal
+ */
+export function killActiveChildren(signal = 'SIGTERM') {
+  let killed = 0;
+  for (const child of activeChildren) {
+    if (child.exitCode != null || child.signalCode != null) continue;
+    try {
+      child.kill(signal);
+      killed += 1;
+    } catch {
+      // Process already gone.
+    }
+  }
+  return killed;
+}
+
 export function runProcess(command, args = [], options = {}) {
   const {
     input,
@@ -15,6 +41,7 @@ export function runProcess(command, args = [], options = {}) {
       env,
       stdio: ['pipe', 'pipe', 'pipe']
     });
+    activeChildren.add(child);
 
     let stdout = '';
     let stderr = '';
@@ -55,11 +82,16 @@ export function runProcess(command, args = [], options = {}) {
       stderr += chunk;
     });
 
-    child.on('error', (error) => {
+    const finish = (result) => {
+      if (settled) return;
       settled = true;
+      activeChildren.delete(child);
       if (timer) clearTimeout(timer);
-      const endedAt = process.hrtime.bigint();
-      resolve({
+      resolve(result);
+    };
+
+    child.on('error', (error) => {
+      finish({
         command,
         args,
         code: null,
@@ -73,15 +105,12 @@ export function runProcess(command, args = [], options = {}) {
         firstStderrMs,
         error,
         timedOut,
-        durationMs: Number(endedAt - startedAt) / 1e6
+        durationMs: Number(process.hrtime.bigint() - startedAt) / 1e6
       });
     });
 
     child.on('close', (code, signal) => {
-      settled = true;
-      if (timer) clearTimeout(timer);
-      const endedAt = process.hrtime.bigint();
-      resolve({
+      finish({
         command,
         args,
         code,
@@ -93,8 +122,9 @@ export function runProcess(command, args = [], options = {}) {
         stdoutChunkTimesMs,
         firstStdoutMs,
         firstStderrMs,
+        error: null,
         timedOut,
-        durationMs: Number(endedAt - startedAt) / 1e6
+        durationMs: Number(process.hrtime.bigint() - startedAt) / 1e6
       });
     });
 

package/src/prompts.js

@@ -195,16 +195,28 @@ export async function loadPrompts(options = {}) {
 
 async function loadPromptFile(filePath) {
   const absolutePath = path.resolve(filePath);
-  const content = await fs.readFile(absolutePath, 'utf8');
+  let content;
+  try {
+    content = await fs.readFile(absolutePath, 'utf8');
+  } catch (error) {
+    throw new Error(error.code === 'ENOENT'
+      ? `Prompt file not found: ${absolutePath}`
+      : `Cannot read prompt file ${absolutePath}: ${error.message}`);
+  }
   const trimmed = content.trim();
 
   if (!trimmed) return [];
 
   if (absolutePath.endsWith('.json')) {
-    const parsed = JSON.parse(trimmed);
+    let parsed;
+    try {
+      parsed = JSON.parse(trimmed);
+    } catch (error) {
+      throw new Error(`Cannot parse ${absolutePath} as JSON: ${error.message}`);
+    }
     const items = Array.isArray(parsed) ? parsed : parsed.prompts;
     if (!Array.isArray(items)) {
-      throw new Error('Prompt JSON must be an array or an object with a prompts array');
+      throw new Error(`${absolutePath} must be a JSON array of prompts or an object with a prompts array`);
     }
     return items.map((item, index) => normalizePromptItem(item, index));
   }
@@ -212,7 +224,13 @@ async function loadPromptFile(filePath) {
   if (absolutePath.endsWith('.jsonl')) {
     return trimmed.split(/\r?\n/)
       .filter(Boolean)
-      .map((line, index) => normalizePromptItem(JSON.parse(line), index));
+      .map((line, index) => {
+        try {
+          return normalizePromptItem(JSON.parse(line), index);
+        } catch (error) {
+          throw new Error(`Cannot parse line ${index + 1} of ${absolutePath} as JSON: ${error.message}`);
+        }
+      });
   }
 
   return trimmed.split(/\n\s*\n/g).map((prompt, index) => ({