fm-bench 0.6.2 → 0.7.0

package/src/compare.js

@@ -1,3 +1,4 @@
+import { stripAnsi } from './ansi.js';
 import { compareCompatibility } from './schema.js';
 import { formatMs, formatNumber, formatPercent } from './table.js';
 
@@ -266,12 +267,19 @@ function applyTone(text, tone) {
 }
 
 function pad(text, width) {
-  const len = String(text ?? '').length;
-  return String(text ?? '') + ' '.repeat(Math.max(0, width - len));
+  const str = safeCell(text);
+  const len = str.length;
+  return str + ' '.repeat(Math.max(0, width - len));
 }
 
 function fitCell(text, width) {
-  const str = String(text ?? '');
+  const str = safeCell(text);
   if (str.length <= width) return str + ' '.repeat(width - str.length);
   return `${str.slice(0, width - 1)}…`;
 }
+
+// Values can originate from report files that fm-bench did not write, so strip
+// ANSI escapes and control characters before they reach the terminal.
+function safeCell(text) {
+  return stripAnsi(String(text ?? '')).replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, '');
+}

package/src/fm-help.js

@@ -0,0 +1,131 @@
+// Pure parsers for `fm` help and status text.
+//
+// These functions take raw text and return plain data so they can be unit
+// tested without spawning a process, and so the capability layer, the model
+// discovery path, and the availability path agree on how `fm` output is read.
+
+import { stripAnsi } from './ansi.js';
+
+const SECTION_HEADER = /^\s*[A-Z][A-Z0-9 /-]+\s*$/;
+
+/**
+ * Extract models from the MODELS section of `fm --help`, falling back to a
+ * `--model` option list such as `Model to use (system, pcc)`.
+ * @param {string} helpText
+ * @returns {{ name: string, description: string }[]}
+ */
+export function parseModelsFromHelp(helpText = '') {
+  const lines = stripAnsi(helpText).split(/\r?\n/);
+  const models = new Map();
+  let inModels = false;
+
+  for (const line of lines) {
+    if (/^\s*MODELS\s*$/.test(line)) {
+      inModels = true;
+      continue;
+    }
+
+    if (inModels && SECTION_HEADER.test(line)) {
+      inModels = false;
+    }
+
+    if (inModels) {
+      const match = line.match(/^\s*([A-Za-z0-9._:-]+)\s{2,}(.+?)\s*$/);
+      if (match) {
+        const name = match[1];
+        const description = match[2].replace(/\s*\(default\)\s*$/, '').trim();
+        const existing = models.get(name);
+        // The MODELS section is authoritative over a bare option list.
+        if (existing) {
+          if (!existing.description) existing.description = description;
+        } else {
+          models.set(name, { name, description });
+        }
+      }
+    }
+
+    if (/(^|\s)--model\b/.test(line)) {
+      const optionMatch = line.match(/\(([^)]*)\)/);
+      if (optionMatch) {
+        for (const raw of optionMatch[1].split(',')) {
+          const name = raw.trim();
+          if (/^[A-Za-z0-9._:-]+$/.test(name) && !models.has(name)) {
+            models.set(name, { name, description: '' });
+          }
+        }
+      }
+    }
+  }
+
+  return [...models.values()];
+}
+
+/**
+ * Read model names and availability out of `fm available` (no model filter).
+ * @param {string} output
+ * @returns {{ name: string, available: boolean }[]}
+ */
+export function parseAvailabilityList(output = '') {
+  const models = new Map();
+  for (const line of stripAnsi(output).split(/\r?\n/)) {
+    const match = line.match(/^\s*([A-Za-z][A-Za-z0-9._-]*)(?:\s+model)?\s+(?:is\s+)?(available|unavailable|not available)\b/i);
+    if (!match) continue;
+    const name = match[1].toLowerCase();
+    models.set(name, { name, available: /^available$/i.test(match[2]) });
+  }
+  return [...models.values()];
+}
+
+/**
+ * Interpret `fm available --model <model>` output.
+ *
+ * `fm` reports the model plus the word "available" on success. Anything with
+ * an explicit error/unavailable marker, or a non-zero exit code, counts as
+ * unavailable so a partially supported host is never reported as working.
+ *
+ * @param {string} model
+ * @param {string} output combined stdout+stderr
+ * @param {number|null} code process exit code
+ */
+export function parseAvailabilityOutput(model, output = '', code = null) {
+  const clean = stripAnsi(output).trim();
+  const lower = clean.toLowerCase();
+  const modelLower = String(model).toLowerCase();
+  const hasError = /\berror:|\bunavailable\b|\bnot available\b|\bnot supported\b|\bis invalid for\b/.test(lower);
+  const modelPattern = escapeRegExp(modelLower);
+  const hasAvailable = new RegExp(`\\b${modelPattern}\\b[\\s\\S]{0,80}\\bavailable\\b`).test(lower)
+    || new RegExp(`\\bavailable\\b[\\s\\S]{0,80}\\b${modelPattern}\\b`).test(lower);
+
+  return {
+    model,
+    available: code === 0 && hasAvailable && !hasError,
+    raw: clean,
+    reason: hasError ? firstLine(clean) : ''
+  };
+}
+
+/**
+ * Collapse `fm` diagnostics into one actionable line. `fm` writes multi-line
+ * usage blocks for argument errors; benchmark output only needs the cause.
+ * @param {string} text
+ */
+export function firstLine(text = '') {
+  const clean = stripAnsi(text).replace(/\s+/g, ' ').trim();
+  if (!clean) return '';
+  const sentences = clean.split(/(?<=\.)\s+(?=[A-Z])/);
+  const head = sentences.find((part) => /error|invalid|unavailable|not supported|failed/i.test(part)) || sentences[0];
+  return head.replace(/^Error:\s*/i, '').trim();
+}
+
+/**
+ * True when an `fm` diagnostic means "this build does not have that model"
+ * rather than "the model exists but is currently unusable".
+ * @param {string} text
+ */
+export function isUnsupportedModelError(text = '') {
+  return /is invalid for '--model|unknown model|no such model|unrecognized model/i.test(stripAnsi(text));
+}
+
+function escapeRegExp(value) {
+  return String(value).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}

package/src/fm.js

@@ -1,60 +1,16 @@
-import crypto from 'node:crypto';
 import os from 'node:os';
 import { stripAnsi } from './ansi.js';
+import { detectFmCapabilities } from './capabilities.js';
+import { firstLine, isUnsupportedModelError, parseAvailabilityOutput } from './fm-help.js';
 import { runProcess } from './process.js';
 import { parseBatteryOutput, parseThermalOutput } from './system.js';
 
-const DEFAULT_MODELS = [
-  { name: 'system', description: 'On-device Apple Foundation Model' },
-  { name: 'pcc', description: 'Apple Foundation Model on Private Cloud Compute' }
-];
+export { parseModelsFromHelp, parseAvailabilityOutput } from './fm-help.js';
 
 export function fmBinaryFromOptions(options = {}) {
   return options.fmBin || process.env.FM_BIN || 'fm';
 }
 
-export function parseModelsFromHelp(helpText) {
-  const clean = stripAnsi(helpText);
-  const lines = clean.split(/\r?\n/);
-  const models = new Map();
-  let inModels = false;
-
-  for (const line of lines) {
-    if (/^\s*MODELS\s*$/.test(line)) {
-      inModels = true;
-      continue;
-    }
-
-    if (inModels && /^\s*[A-Z][A-Z -]+\s*$/.test(line) && !/^\s*MODELS\s*$/.test(line)) {
-      inModels = false;
-    }
-
-    if (inModels) {
-      const match = line.match(/^\s*([A-Za-z0-9._:-]+)\s{2,}(.+?)\s*$/);
-      if (match) {
-        models.set(match[1], {
-          name: match[1],
-          description: match[2].replace(/\s*\(default\)\s*$/, '').trim()
-        });
-      }
-    }
-
-    const optionMatch = /--model\b/.test(line)
-      ? line.match(/\bmodel\b.*?\(([^)]+)\)/i)
-      : null;
-    if (optionMatch) {
-      for (const raw of optionMatch[1].split(',')) {
-        const name = raw.trim();
-        if (/^[A-Za-z0-9._:-]+$/.test(name) && !models.has(name)) {
-          models.set(name, { name, description: '' });
-        }
-      }
-    }
-  }
-
-  return [...models.values()];
-}
-
 export async function getFmHelp(fmBin, timeoutMs = 10_000) {
   const result = await runProcess(fmBin, ['--help'], { timeoutMs });
   if (result.error) {
@@ -69,40 +25,42 @@ export async function getFmHelp(fmBin, timeoutMs = 10_000) {
   };
 }
 
+/**
+ * Discover models, preferring an already-detected capability probe so a run
+ * does not spawn `fm --help` more than once.
+ * @param {Record<string, any>} options
+ */
 export async function discoverModels(options = {}) {
   const fmBin = fmBinaryFromOptions(options);
-  const help = await getFmHelp(fmBin, options.timeoutMs ?? 10_000);
-  let models = parseModelsFromHelp(help.text);
-
-  if (models.length === 0 && /Apple Foundation Models CLI/i.test(stripAnsi(help.text))) {
-    models = DEFAULT_MODELS;
-  }
-
+  const capabilities = options.capabilities ?? await detectFmCapabilities(fmBin, options);
   return {
     fmBin,
-    models,
-    help: stripAnsi(help.text)
-  };
-}
-
-export function parseAvailabilityOutput(model, output, code) {
-  const clean = stripAnsi(output).trim();
-  const lower = clean.toLowerCase();
-  const modelLower = model.toLowerCase();
-  const hasError = /\berror:|\bunavailable\b|\bnot available\b|\bnot supported\b/.test(lower);
-  const hasAvailable = new RegExp(`\\b${escapeRegExp(modelLower)}\\b[\\s\\S]{0,80}\\bavailable\\b|\\bavailable\\b[\\s\\S]{0,80}\\b${escapeRegExp(modelLower)}\\b`).test(lower)
-    || lower.includes(`${modelLower} model available`)
-    || lower.includes(`${titleCase(modelLower)} model available`.toLowerCase());
-
-  return {
-    model,
-    available: code === 0 && hasAvailable && !hasError,
-    raw: clean,
-    reason: hasError ? clean : ''
+    models: capabilities.models,
+    help: capabilities.help,
+    capabilities
   };
 }
 
+/**
+ * Check one model against the detected `fm` build.
+ *
+ * Models the build does not expose are reported as unsupported without
+ * spawning `fm` at all, so a raw argument-error blob from `fm` can never end
+ * up in a report or in `fm-bench models` output.
+ */
 export async function checkModelAvailability(fmBin, model, options = {}) {
+  const capabilities = options.capabilities;
+  const known = capabilities?.models?.map((entry) => entry.name) ?? [];
+  if (capabilities && known.length > 0 && !known.includes(model)) {
+    return {
+      model,
+      available: false,
+      unsupported: true,
+      raw: '',
+      reason: `not supported by this fm build (supported: ${known.join(', ')})`
+    };
+  }
+
   const result = await runProcess(fmBin, ['available', '--model', model], {
     timeoutMs: options.timeoutMs ?? 15_000
   });
@@ -111,53 +69,99 @@ export async function checkModelAvailability(fmBin, model, options = {}) {
   if (result.error) {
     parsed.available = false;
     parsed.reason = result.stderr || result.error.message;
+    return parsed;
+  }
+  if (isUnsupportedModelError(output)) {
+    parsed.available = false;
+    parsed.unsupported = true;
+    const supported = known.length > 0 ? ` (supported: ${known.join(', ')})` : '';
+    parsed.reason = `not supported by this fm build${supported}`;
   }
   return parsed;
 }
 
+/**
+ * Query quota information when the build exposes it.
+ * @returns {Promise<{ model: string, supported: boolean, ok: boolean, raw: string, reason: string }>}
+ */
 export async function getQuotaUsage(fmBin, model, options = {}) {
+  const features = options.capabilities?.features;
+  if (features && !features.quota) {
+    return {
+      model,
+      supported: false,
+      ok: false,
+      raw: '',
+      reason: 'unavailable: this fm build exposes no quota command'
+    };
+  }
+
   const result = await runProcess(fmBin, ['quota-usage', '--model', model], {
     timeoutMs: options.timeoutMs ?? 15_000
   });
   const output = stripAnsi(`${result.stdout}${result.stderr}`).trim();
   return {
     model,
+    supported: true,
     ok: result.code === 0,
     raw: output,
-    unavailable: /\bunavailable\b|\bnot available\b|\berror:/i.test(output)
+    reason: result.code === 0 ? '' : firstLine(output)
   };
 }
 
+/**
+ * Count tokens with whichever token-counting command this `fm` build exposes.
+ * Returns `ok: false` with a reason when the build cannot count tokens; it
+ * never invents a count.
+ */
 export async function countTokens(fmBin, text, options = {}) {
-  const result = await runProcess(fmBin, ['token-count', '--quiet'], {
+  const command = options.capabilities?.features?.tokenCountCommand ?? 'count-tokens';
+  const supported = options.capabilities?.features?.tokenCounting ?? true;
+  if (!supported || !command) {
+    return {
+      ok: false,
+      count: null,
+      unsupported: true,
+      raw: '',
+      reason: 'token counting is unavailable in this fm build'
+    };
+  }
+
+  const result = await runProcess(fmBin, [command, '--quiet'], {
     input: text,
     timeoutMs: options.timeoutMs ?? 15_000
   });
   const output = stripAnsi(`${result.stdout}${result.stderr}`).trim();
-  const match = output.match(/-?\d+/);
-  if (result.code !== 0 || !match) {
+  const match = output.match(/\d+/);
+  if (result.error || result.code !== 0 || !match) {
     return {
       ok: false,
       count: null,
-      raw: output
+      raw: output,
+      reason: result.error?.message || firstLine(output) || `fm ${command} exited with code ${result.code}`
     };
   }
   return {
     ok: true,
     count: Number.parseInt(match[0], 10),
-    raw: output
+    raw: output,
+    reason: ''
   };
 }
 
 export async function respond(fmBin, model, prompt, options = {}) {
-  const args = ['respond', '--model', model];
-  const streamed = options.stream !== false;
-
-  if (!streamed) args.push('--no-stream');
-  if (options.greedy) args.push('--greedy');
-  if (options.instructions) args.push('--instructions', options.instructions);
-  if (options.useCase) args.push('--use-case', options.useCase);
-  if (options.guardrails) args.push('--guardrails', options.guardrails);
+  const features = options.capabilities?.features;
+  const streamControl = features ? features.streaming : true;
+  const modelSelection = features ? features.modelSelection : true;
+  const streamed = streamControl && options.stream !== false;
+
+  const args = ['respond'];
+  if (modelSelection) args.push('--model', model);
+  if (streamControl && !streamed) args.push('--no-stream');
+  if (options.greedy && (features?.greedy ?? true)) args.push('--greedy');
+  if (options.instructions && (features?.instructions ?? true)) args.push('--instructions', options.instructions);
+  if (options.useCase && (features?.useCase ?? true)) args.push('--use-case', options.useCase);
+  if (options.guardrails && (features?.guardrails ?? true)) args.push('--guardrails', options.guardrails);
 
   const result = await runProcess(fmBin, args, {
     input: prompt,
@@ -166,8 +170,10 @@ export async function respond(fmBin, model, prompt, options = {}) {
 
   const output = stripAnsi(result.stdout).trim();
   const errorText = stripAnsi(result.stderr).trim();
+  const failed = result.code !== 0 || result.timedOut;
+
   return {
-    ok: result.code === 0 && !result.timedOut,
+    ok: !failed,
     model,
     prompt,
     output,
@@ -179,11 +185,17 @@ export async function respond(fmBin, model, prompt, options = {}) {
     firstOutputMs: streamed ? result.firstStdoutMs : null,
     streamed,
     stdoutChunks: result.stdoutChunks,
-    stdoutChunkTimesMs: streamed ? result.stdoutChunkTimesMs : []
+    stdoutChunkTimesMs: streamed ? result.stdoutChunkTimesMs : [],
+    error: failed
+      ? (result.timedOut
+        ? `timed out after ${options.timeoutMs ?? 60_000}ms`
+        : firstLine(errorText) || `fm exited with code ${result.code ?? result.signal}`)
+      : ''
   };
 }
 
-export async function collectEnvironment(fmBin) {
+export async function collectEnvironment(fmBin, options = {}) {
+  const capabilities = options.capabilities;
   const swVers = await runProcess('sw_vers', [], { timeoutMs: 5_000 });
   const macOS = stripAnsi(swVers.stdout).trim() || null;
 
@@ -196,15 +208,15 @@ export async function collectEnvironment(fmBin) {
   const batteryResult = await runProcess('pmset', ['-g', 'batt'], { timeoutMs: 5_000 });
   const battery = parseBatteryOutput(`${batteryResult.stdout || ''}${batteryResult.stderr || ''}`);
 
-  let fmHelpDigest = null;
-  try {
-    const help = await getFmHelp(fmBin, 10_000);
-    const text = stripAnsi(help.text).trim();
-    if (text) {
-      fmHelpDigest = crypto.createHash('sha256').update(text).digest('hex').slice(0, 16);
+  let fmHelpDigest = capabilities?.digest ?? null;
+  if (fmHelpDigest == null) {
+    try {
+      const help = await getFmHelp(fmBin, 10_000);
+      const detected = await detectFmCapabilities(fmBin, { help: { text: help.text } });
+      fmHelpDigest = detected.digest;
+    } catch {
+      fmHelpDigest = null;
     }
-  } catch {
-    fmHelpDigest = null;
   }
 
   const memRaw = (memBytes.stdout || '').trim();
@@ -237,11 +249,3 @@ export async function collectEnvironment(fmBin) {
       : null
   };
 }
-
-function escapeRegExp(value) {
-  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-}
-
-function titleCase(value) {
-  return value.slice(0, 1).toUpperCase() + value.slice(1);
-}

package/src/history.js

@@ -1,5 +1,6 @@
 import fs from 'node:fs/promises';
 import path from 'node:path';
+import { stripAnsi } from './ansi.js';
 import { formatMs, formatNumber, formatPercent } from './table.js';
 
 export async function loadHistory(dir) {
@@ -100,7 +101,7 @@ export function renderHistoryReport(reports, options = {}) {
 }
 
 function fit(text, width) {
-  const str = String(text ?? '');
+  const str = stripAnsi(String(text ?? '')).replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/g, '');
   if (str.length <= width) return str + ' '.repeat(width - str.length);
   return `${str.slice(0, width - 1)}…`;
 }

package/src/macos.js

@@ -0,0 +1,72 @@
+import { runProcess } from './process.js';
+
+// macOS releases that ship Apple's `fm` CLI. Older macOS versions cannot run
+// fm-bench at all because the benchmarked binary does not exist there.
+export const SUPPORTED_MACOS_MAJOR = 27;
+
+export const MIN_SUPPORTED_MACOS = `${SUPPORTED_MACOS_MAJOR}.0`;
+
+export const MACOS_REQUIREMENT_MESSAGE =
+  `fm-bench requires macOS ${SUPPORTED_MACOS_MAJOR} or newer (Apple's fm CLI is preinstalled there).`;
+
+export function parseMacosVersion(text = '') {
+  const match = String(text).match(/ProductVersion:\s*(\d+)(?:\.(\d+))?(?:\.(\d+))?/);
+  if (!match) return null;
+  return {
+    major: Number.parseInt(match[1], 10),
+    minor: match[2] != null ? Number.parseInt(match[2], 10) : 0,
+    patch: match[3] != null ? Number.parseInt(match[3], 10) : 0,
+    version: [match[1], match[2], match[3]].filter((part) => part != null).join('.')
+  };
+}
+
+export function isMacosSupported(parsed) {
+  return parsed != null && Number.isInteger(parsed.major) && parsed.major >= SUPPORTED_MACOS_MAJOR;
+}
+
+export function evaluateMacosSupport(platform, parsed) {
+  if (platform !== 'darwin') {
+    return {
+      supported: false,
+      reason: `fm-bench only runs on macOS with Apple's fm CLI (detected platform: ${platform}).`,
+      latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer`
+    };
+  }
+  if (parsed == null) {
+    return {
+      supported: true,
+      warnOnly: true,
+      reason: 'could not determine the macOS version',
+      latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer`
+    };
+  }
+  if (isMacosSupported(parsed)) {
+    return {
+      supported: true,
+      reason: null,
+      latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer`
+    };
+  }
+  return {
+    supported: false,
+    reason: `detected macOS ${parsed.version}, but ${MACOS_REQUIREMENT_MESSAGE}`,
+    latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer`
+  };
+}
+
+export function formatMacosRequirementError({ reason, latestSupported }) {
+  return [
+    `unsupported macOS: ${reason}`,
+    `Latest supported: ${latestSupported} (fm is not available on older macOS releases).`,
+    'Pass --fm-bin <path> (or set FM_BIN) to benchmark an fm binary you provide on this host.'
+  ].join('\n');
+}
+
+export async function detectMacosVersion(options = {}) {
+  const timeoutMs = options.timeoutMs ?? 5_000;
+  if (typeof options.swVers === 'function') {
+    return options.swVers();
+  }
+  const result = await runProcess('sw_vers', [], { timeoutMs });
+  return result.stdout || result.stderr;
+}