fm-bench 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Devin Oldenburg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,153 @@
+# fm-bench
+
+`fm-bench` is a dynamic benchmark CLI for Apple's `fm` command on macOS 27 and newer.
+
+It discovers the models reported by `fm --help`, checks availability with `fm available`, runs repeatable prompt suites through `fm respond`, counts tokens with `fm token-count`, and prints a terminal table with latency and throughput stats.
+
+Apple introduced the preinstalled `fm` command for macOS 27 as part of the Foundation Models tooling. `fm-bench` intentionally shells out to the system `fm` binary instead of linking private APIs, so it can adapt as Apple adds models or changes availability.
+
+## Install
+
+```sh
+npm install -g fm-bench
+```
+
+You can also install directly from GitHub:
+
+```sh
+npm install -g --install-links git+https://github.com/devinoldenburg/fm-bench.git
+```
+
+For local development from this repository:
+
+```sh
+npm install
+npm link
+fm-bench doctor
+```
+
+## Quick Start
+
+```sh
+fm-bench
+```
+
+Example output:
+
+```text
++--------+---------+------+----+------+-------+-------+-------+-------+--------+---------+------+
+| model  | status  | runs | ok | fail | p50   | p95   | avg   | tok/s | char/s | out tok | note |
++--------+---------+------+----+------+-------+-------+-------+-------+--------+---------+------+
+| system | ok      |    3 |  3 |    - | 1.21s | 1.85s | 1.34s |  18.7 |    83  |      24 |      |
+| pcc    | skipped |    - |  - |    - | -     | -     | -     | -     | -      | -       | PCC inference is not available... |
++--------+---------+------+----+------+-------+-------+-------+-------+--------+---------+------+
+```
+
+## Commands
+
+```sh
+fm-bench [run] [options]
+fm-bench models [options]
+fm-bench doctor [options]
+```
+
+`run` is the default command. It benchmarks all models discovered from `fm` and skips models that are currently unavailable.
+
+`models` lists discovered models, availability, descriptions, and quota output.
+
+`doctor` checks Node, macOS, `fm`, and model availability.
+
+## Benchmark Options
+
+```sh
+fm-bench --models system,pcc --runs 3 --profile stress
+fm-bench --prompt "Reply with exactly: ok" --runs 5
+fm-bench --prompt-file prompts.json --format json --out reports/bench.json
+fm-bench --format csv --out reports/bench.csv
+```
+
+Useful flags:
+
+- `--models <list>`: comma-separated or repeated model names.
+- `--runs <n>`: measured runs per prompt/model.
+- `--warmup <n>`: warmup runs per model before measurement.
+- `--concurrency <n>`: parallel `fm` processes.
+- `--timeout-ms <n>`: timeout per `fm` call.
+- `--profile quick|standard|stress`: built-in prompt suite.
+- `--prompt <text>`: custom prompt, repeatable.
+- `--prompt-file <file>`: JSON, JSONL, or blank-line separated text prompts.
+- `--instructions <text>`: passed to `fm respond`.
+- `--available-only`: hide unavailable discovered models.
+- `--capture-output`: include raw model output in JSON reports.
+- `--json`, `--csv`, `--format table|json|csv`: choose output format.
+- `--out <file>`: save a report.
+
+## Prompt Files
+
+JSON array:
+
+```json
+[
+  { "id": "tiny", "prompt": "Reply with exactly: ok" },
+  { "id": "json", "prompt": "Convert alpha, beta, gamma into JSON." }
+]
+```
+
+JSONL:
+
+```jsonl
+{"id":"tiny","prompt":"Reply with exactly: ok"}
+{"id":"latency","prompt":"Explain p95 latency in one sentence."}
+```
+
+Plain text files are split on blank lines.
+
+## Metrics
+
+`fm-bench` reports:
+
+- `p50`, `p95`, and average wall-clock latency.
+- average output tokens per second.
+- average characters per second.
+- average output tokens.
+- success and failure counts.
+- unavailable model notes.
+
+Token counts come from `fm token-count --quiet`. If `fm` cannot count a response, the token fields are left blank while character throughput is still reported.
+
+## Requirements
+
+- macOS 27 or newer for Apple's `fm` CLI.
+- Node.js 20 or newer.
+- Apple Intelligence and model availability configured for the machine.
+
+Private Cloud Compute (`pcc`) availability depends on Apple's current eligibility and context. If `fm available --model pcc` reports unavailable, `fm-bench` will show it as skipped.
+
+## Development
+
+```sh
+npm install
+npm test
+npm run lint
+npm pack
+```
+
+The package has no runtime npm dependencies.
+
+## Releases
+
+Releases are tag-driven:
+
+```sh
+npm run release:patch
+npm run release:minor
+npm run release:major
+```
+
+Pushing a `v*.*.*` tag runs the GitHub Release workflow, publishes to npm using the repository `NPM_TOKEN` secret, and creates a GitHub Release with generated notes.
+
+Maintainers can also run the **Version** workflow manually from GitHub Actions to bump the version and push the tag.
+
+## License
+
+MIT

package/bin/fm-bench.js

@@ -0,0 +1,8 @@
+#!/usr/bin/env node
+import { runCli } from '../src/cli.js';
+
+runCli(process.argv.slice(2)).catch((error) => {
+  const message = error?.message || String(error);
+  console.error(`fm-bench: ${message}`);
+  process.exitCode = typeof error?.exitCode === 'number' ? error.exitCode : 1;
+});

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "fm-bench",
+  "version": "0.1.0",
+  "description": "Dynamic benchmark CLI for Apple's fm command on macOS 27+.",
+  "type": "module",
+  "bin": {
+    "fm-bench": "./bin/fm-bench.js"
+  },
+  "files": [
+    "bin",
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "test": "node --test",
+    "lint": "node --check bin/fm-bench.js && find src test -name '*.js' -print0 | xargs -0 -n1 node --check",
+    "prepack": "npm test && npm run lint",
+    "release:patch": "npm version patch && git push --follow-tags",
+    "release:minor": "npm version minor && git push --follow-tags",
+    "release:major": "npm version major && git push --follow-tags",
+    "publish:dry-run": "npm publish --dry-run --access public"
+  },
+  "keywords": [
+    "apple",
+    "foundation-models",
+    "fm",
+    "benchmark",
+    "macos",
+    "cli"
+  ],
+  "author": "Devin Oldenburg",
+  "license": "MIT",
+  "engines": {
+    "node": ">=20"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/devinoldenburg/fm-bench.git"
+  },
+  "bugs": {
+    "url": "https://github.com/devinoldenburg/fm-bench/issues"
+  },
+  "homepage": "https://github.com/devinoldenburg/fm-bench#readme",
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  }
+}

package/src/ansi.js

@@ -0,0 +1,5 @@
+const ANSI_PATTERN = /\u001b\[[0-?]*[ -/]*[@-~]/g;
+
+export function stripAnsi(value) {
+  return String(value ?? '').replace(ANSI_PATTERN, '');
+}

package/src/bench.js

@@ -0,0 +1,161 @@
+import { checkModelAvailability, collectEnvironment, countTokens, discoverModels, getQuotaUsage, respond } from './fm.js';
+import { loadPrompts } from './prompts.js';
+import { summarizeByModel } from './stats.js';
+
+export async function inspectModels(options = {}) {
+  const discovered = await discoverModels(options);
+  const requested = normalizeModelSelection(options.models);
+  const models = requested.length > 0
+    ? discovered.models.filter((model) => requested.includes(model.name))
+    : discovered.models;
+
+  const missing = requested.filter((name) => !models.some((model) => model.name === name));
+  for (const name of missing) {
+    models.push({ name, description: 'Requested model not reported by fm --help' });
+  }
+
+  const inspected = [];
+  for (const model of models) {
+    const availability = await checkModelAvailability(discovered.fmBin, model.name, options);
+    const quota = await getQuotaUsage(discovered.fmBin, model.name, options);
+    inspected.push({
+      ...model,
+      available: availability.available,
+      reason: availability.reason || availability.raw,
+      quota: quota.raw
+    });
+  }
+
+  return {
+    fmBin: discovered.fmBin,
+    models: inspected,
+    help: discovered.help
+  };
+}
+
+export async function runBenchmark(options = {}) {
+  const startedAt = new Date().toISOString();
+  const prompts = await loadPrompts(options);
+  const inspection = await inspectModels(options);
+  const modelStatuses = options.availableOnly
+    ? inspection.models.filter((model) => model.available)
+    : inspection.models;
+  const runnableModels = modelStatuses.filter((model) => model.available);
+  const environment = await collectEnvironment(inspection.fmBin);
+  const promptTokenCounts = new Map();
+
+  for (const prompt of prompts) {
+    const counted = await countTokens(inspection.fmBin, prompt.prompt, options);
+    promptTokenCounts.set(prompt.id, counted.ok ? counted.count : null);
+  }
+
+  for (let warmupIndex = 0; warmupIndex < options.warmup; warmupIndex += 1) {
+    for (const model of runnableModels) {
+      await respond(inspection.fmBin, model.name, prompts[0].prompt, {
+        ...options,
+        stream: false
+      });
+    }
+  }
+
+  const jobs = [];
+  for (const model of runnableModels) {
+    for (const prompt of prompts) {
+      for (let run = 1; run <= options.runs; run += 1) {
+        jobs.push({ model, prompt, run });
+      }
+    }
+  }
+
+  const results = [];
+  await runLimited(jobs, Math.max(1, options.concurrency), async (job) => {
+    const result = await runSingleBenchmark(inspection.fmBin, job, promptTokenCounts, options);
+    results.push(result);
+    if (!result.ok && options.failFast) {
+      const error = new Error(result.error || `Benchmark failed for ${job.model.name}`);
+      error.exitCode = 1;
+      throw error;
+    }
+  });
+
+  const summary = summarizeByModel(results, modelStatuses);
+  return {
+    tool: 'fm-bench',
+    version: options.version,
+    startedAt,
+    finishedAt: new Date().toISOString(),
+    options: publicOptions(options),
+    environment,
+    prompts: prompts.map((prompt) => ({
+      id: prompt.id,
+      prompt: prompt.prompt,
+      promptTokens: promptTokenCounts.get(prompt.id)
+    })),
+    models: modelStatuses,
+    summary,
+    results
+  };
+}
+
+async function runSingleBenchmark(fmBin, job, promptTokenCounts, options) {
+  const response = await respond(fmBin, job.model.name, job.prompt.prompt, {
+    ...options,
+    stream: false
+  });
+  const outputTokens = response.ok
+    ? await countTokens(fmBin, response.output, options)
+    : { ok: false, count: null };
+  const seconds = response.durationMs / 1000;
+  const chars = response.output.length;
+  const words = response.output.trim() ? response.output.trim().split(/\s+/).length : 0;
+
+  return {
+    model: job.model.name,
+    promptId: job.prompt.id,
+    run: job.run,
+    ok: response.ok,
+    durationMs: response.durationMs,
+    promptTokens: promptTokenCounts.get(job.prompt.id),
+    outputTokens: outputTokens.ok ? outputTokens.count : null,
+    chars,
+    words,
+    tokensPerSecond: outputTokens.ok && seconds > 0 ? outputTokens.count / seconds : null,
+    charsPerSecond: seconds > 0 ? chars / seconds : 0,
+    output: options.captureOutput ? response.output : undefined,
+    error: response.ok ? '' : response.stderr || `fm exited with code ${response.code ?? response.signal}`
+  };
+}
+
+async function runLimited(items, concurrency, worker) {
+  let nextIndex = 0;
+  const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
+    while (nextIndex < items.length) {
+      const index = nextIndex;
+      nextIndex += 1;
+      await worker(items[index]);
+    }
+  });
+  await Promise.all(workers);
+}
+
+function normalizeModelSelection(models) {
+  if (!models) return [];
+  const values = Array.isArray(models) ? models : [models];
+  return values.flatMap((value) => String(value).split(','))
+    .map((value) => value.trim())
+    .filter(Boolean);
+}
+
+function publicOptions(options) {
+  return {
+    models: normalizeModelSelection(options.models),
+    runs: options.runs,
+    warmup: options.warmup,
+    concurrency: options.concurrency,
+    timeoutMs: options.timeoutMs,
+    profile: options.profile,
+    promptCount: options.promptCount,
+    greedy: options.greedy,
+    instructions: options.instructions ? '[set]' : ''
+  };
+}

package/src/cli.js

@@ -0,0 +1,290 @@
+import fs from 'node:fs/promises';
+import { createRequire } from 'node:module';
+import { inspectModels, runBenchmark } from './bench.js';
+import { runProcess } from './process.js';
+import { flattenResults, toCsv, writeReport } from './report.js';
+import { renderModelsTable, renderSummaryTable } from './table.js';
+
+const require = createRequire(import.meta.url);
+const packageJson = require('../package.json');
+
+export async function runCli(argv = process.argv.slice(2)) {
+  const parsed = parseArgs(argv);
+
+  if (parsed.help) {
+    console.log(helpText());
+    return;
+  }
+
+  if (parsed.versionOnly) {
+    console.log(packageJson.version);
+    return;
+  }
+
+  if (parsed.command === 'doctor') {
+    await runDoctor(parsed);
+    return;
+  }
+
+  if (parsed.command === 'models') {
+    const inspection = await inspectModels(parsed);
+    if (parsed.format === 'json') {
+      console.log(JSON.stringify(inspection.models, null, 2));
+    } else {
+      console.log(renderModelsTable(inspection.models));
+    }
+    return;
+  }
+
+  const payload = await runBenchmark({
+    ...parsed,
+    version: packageJson.version
+  });
+
+  if (parsed.format === 'json') {
+    console.log(JSON.stringify(payload, null, 2));
+  } else if (parsed.format === 'csv') {
+    console.log(toCsv(flattenResults(payload.results)));
+  } else {
+    console.log(renderSummaryTable(payload.summary));
+    if (parsed.verbose) {
+      console.log();
+      console.log(toCsv(flattenResults(payload.results)));
+    }
+  }
+
+  if (parsed.out) {
+    const reportFormat = parsed.out.endsWith('.csv') ? 'csv' : 'json';
+    const written = await writeReport(parsed.out, payload, reportFormat);
+    if (parsed.format !== 'json') {
+      console.error(`Saved ${reportFormat.toUpperCase()} report to ${written}`);
+    }
+  }
+}
+
+export function parseArgs(argv) {
+  const options = {
+    command: 'run',
+    models: [],
+    prompts: [],
+    runs: 1,
+    warmup: 0,
+    concurrency: 1,
+    timeoutMs: 60_000,
+    profile: 'standard',
+    greedy: true,
+    format: 'table',
+    captureOutput: false,
+    availableOnly: false,
+    failFast: false,
+    verbose: false
+  };
+
+  const args = [...argv];
+  if (args[0] && !args[0].startsWith('-') && ['run', 'models', 'doctor', 'help'].includes(args[0])) {
+    options.command = args.shift();
+  }
+
+  if (options.command === 'help') {
+    options.help = true;
+    return options;
+  }
+
+  while (args.length > 0) {
+    const arg = args.shift();
+    switch (arg) {
+      case '-h':
+      case '--help':
+        options.help = true;
+        break;
+      case '--version':
+        options.versionOnly = true;
+        break;
+      case '-m':
+      case '--model':
+      case '--models':
+        options.models.push(requireValue(arg, args));
+        break;
+      case '-r':
+      case '--runs':
+        options.runs = parsePositiveInt(requireValue(arg, args), arg);
+        break;
+      case '--warmup':
+        options.warmup = parseNonNegativeInt(requireValue(arg, args), arg);
+        break;
+      case '-c':
+      case '--concurrency':
+        options.concurrency = parsePositiveInt(requireValue(arg, args), arg);
+        break;
+      case '--timeout':
+      case '--timeout-ms':
+        options.timeoutMs = parsePositiveInt(requireValue(arg, args), arg);
+        break;
+      case '-p':
+      case '--prompt':
+        options.prompts.push(requireValue(arg, args));
+        break;
+      case '--prompt-file':
+        options.promptFile = requireValue(arg, args);
+        break;
+      case '--profile':
+        options.profile = requireValue(arg, args);
+        break;
+      case '-i':
+      case '--instructions':
+        options.instructions = requireValue(arg, args);
+        break;
+      case '--fm-bin':
+        options.fmBin = requireValue(arg, args);
+        break;
+      case '--use-case':
+        options.useCase = requireValue(arg, args);
+        break;
+      case '--guardrails':
+        options.guardrails = requireValue(arg, args);
+        break;
+      case '--greedy':
+        options.greedy = true;
+        break;
+      case '--no-greedy':
+        options.greedy = false;
+        break;
+      case '--json':
+        options.format = 'json';
+        break;
+      case '--csv':
+        options.format = 'csv';
+        break;
+      case '--format':
+        options.format = requireValue(arg, args);
+        if (!['table', 'json', 'csv'].includes(options.format)) {
+          throw new Error('--format must be one of: table, json, csv');
+        }
+        break;
+      case '-o':
+      case '--out':
+        options.out = requireValue(arg, args);
+        break;
+      case '--capture-output':
+        options.captureOutput = true;
+        break;
+      case '--available-only':
+        options.availableOnly = true;
+        break;
+      case '--fail-fast':
+        options.failFast = true;
+        break;
+      case '-v':
+      case '--verbose':
+        options.verbose = true;
+        break;
+      case '--':
+        if (args.length > 0) {
+          options.prompts.push(args.join(' '));
+          args.length = 0;
+        }
+        break;
+      default:
+        if (arg.startsWith('-')) {
+          throw new Error(`Unknown option: ${arg}`);
+        }
+        options.prompts.push([arg, ...args].join(' '));
+        args.length = 0;
+        break;
+    }
+  }
+
+  return options;
+}
+
+async function runDoctor(options) {
+  const checks = [];
+  checks.push(['node', process.version, true]);
+  checks.push(['platform', `${process.platform}/${process.arch}`, process.platform === 'darwin']);
+
+  const swVers = await runProcess('sw_vers', [], { timeoutMs: 5_000 });
+  const macOS = swVers.stdout || swVers.stderr;
+  const versionMatch = macOS.match(/ProductVersion:\s*([0-9.]+)/);
+  const major = versionMatch ? Number.parseInt(versionMatch[1].split('.')[0], 10) : null;
+  checks.push(['macOS', versionMatch?.[1] || 'unknown', major == null || major >= 27]);
+
+  const inspection = await inspectModels(options);
+  checks.push(['fm', inspection.fmBin, inspection.models.length > 0]);
+  for (const model of inspection.models) {
+    checks.push([`model:${model.name}`, model.available ? 'available' : model.reason || 'unavailable', model.available]);
+  }
+
+  const lines = checks.map(([name, detail, ok]) => `${ok ? 'ok  ' : 'warn'} ${name.padEnd(14)} ${String(detail).replace(/\s+/g, ' ').trim()}`);
+  console.log(lines.join('\n'));
+
+  if (options.out) {
+    await fs.writeFile(options.out, `${JSON.stringify({ checks, models: inspection.models }, null, 2)}\n`, 'utf8');
+  }
+}
+
+function requireValue(option, args) {
+  const value = args.shift();
+  if (value == null || value === '') throw new Error(`${option} requires a value`);
+  return value;
+}
+
+function parsePositiveInt(value, option) {
+  const parsed = Number.parseInt(value, 10);
+  if (!Number.isInteger(parsed) || parsed < 1) throw new Error(`${option} must be a positive integer`);
+  return parsed;
+}
+
+function parseNonNegativeInt(value, option) {
+  const parsed = Number.parseInt(value, 10);
+  if (!Number.isInteger(parsed) || parsed < 0) throw new Error(`${option} must be a non-negative integer`);
+  return parsed;
+}
+
+function helpText() {
+  return `fm-bench ${packageJson.version}
+
+Dynamic benchmark CLI for Apple's fm command on macOS 27+.
+
+Usage:
+  fm-bench [run] [options]
+  fm-bench models [options]
+  fm-bench doctor [options]
+
+Run options:
+  -m, --models <list>       Models to benchmark, comma-separated or repeated
+  -r, --runs <n>            Runs per prompt/model (default: 1)
+      --warmup <n>          Warmup runs per model before measurement
+  -c, --concurrency <n>     Parallel fm processes (default: 1)
+      --timeout-ms <n>      Timeout per fm call in ms (default: 60000)
+  -p, --prompt <text>       Prompt to benchmark; repeatable
+      --prompt-file <file>  .json, .jsonl, or blank-line separated text prompts
+      --profile <name>      quick, standard, or stress (default: standard)
+  -i, --instructions <text> Instructions passed to fm respond
+      --use-case <case>     Pass a system model use case through to fm
+      --guardrails <level>  Pass a system model guardrail level through to fm
+      --greedy              Use greedy sampling (default)
+      --no-greedy           Do not request greedy sampling
+      --available-only      Hide unavailable discovered models
+      --capture-output      Include raw model output in JSON reports
+      --fail-fast           Stop after the first failed measured run
+
+Output:
+      --format <type>       table, json, or csv (default: table)
+      --json                Alias for --format json
+      --csv                 Alias for --format csv
+  -o, --out <file>          Save JSON or CSV report based on file extension
+  -v, --verbose             Include per-run CSV after the summary table
+
+Environment:
+      --fm-bin <path>       fm binary to execute (default: FM_BIN or fm)
+  -h, --help                Show this help
+      --version             Print version
+
+Examples:
+  fm-bench
+  fm-bench --models system,pcc --runs 3 --profile stress
+  fm-bench --prompt "Reply with exactly: ok" --json --out bench.json
+  fm-bench models
+  fm-bench doctor
+`;
+}

package/src/fm.js

@@ -0,0 +1,197 @@
+import os from 'node:os';
+import { stripAnsi } from './ansi.js';
+import { runProcess } from './process.js';
+
+const DEFAULT_MODELS = [
+  { name: 'system', description: 'On-device Apple Foundation Model' },
+  { name: 'pcc', description: 'Apple Foundation Model on Private Cloud Compute' }
+];
+
+export function fmBinaryFromOptions(options = {}) {
+  return options.fmBin || process.env.FM_BIN || 'fm';
+}
+
+export function parseModelsFromHelp(helpText) {
+  const clean = stripAnsi(helpText);
+  const lines = clean.split(/\r?\n/);
+  const models = new Map();
+  let inModels = false;
+
+  for (const line of lines) {
+    if (/^\s*MODELS\s*$/.test(line)) {
+      inModels = true;
+      continue;
+    }
+
+    if (inModels && /^\s*[A-Z][A-Z -]+\s*$/.test(line) && !/^\s*MODELS\s*$/.test(line)) {
+      inModels = false;
+    }
+
+    if (inModels) {
+      const match = line.match(/^\s*([A-Za-z0-9._:-]+)\s{2,}(.+?)\s*$/);
+      if (match) {
+        models.set(match[1], {
+          name: match[1],
+          description: match[2].replace(/\s*\(default\)\s*$/, '').trim()
+        });
+      }
+    }
+
+    const optionMatch = /--model\b/.test(line)
+      ? line.match(/\bmodel\b.*?\(([^)]+)\)/i)
+      : null;
+    if (optionMatch) {
+      for (const raw of optionMatch[1].split(',')) {
+        const name = raw.trim();
+        if (/^[A-Za-z0-9._:-]+$/.test(name) && !models.has(name)) {
+          models.set(name, { name, description: '' });
+        }
+      }
+    }
+  }
+
+  return [...models.values()];
+}
+
+export async function getFmHelp(fmBin, timeoutMs = 10_000) {
+  const result = await runProcess(fmBin, ['--help'], { timeoutMs });
+  if (result.error) {
+    const error = new Error(`Unable to execute ${fmBin}: ${result.stderr || result.error.message}`);
+    error.exitCode = 2;
+    throw error;
+  }
+  return {
+    ok: result.code === 0,
+    text: `${result.stdout}${result.stderr}`,
+    result
+  };
+}
+
+export async function discoverModels(options = {}) {
+  const fmBin = fmBinaryFromOptions(options);
+  const help = await getFmHelp(fmBin, options.timeoutMs ?? 10_000);
+  let models = parseModelsFromHelp(help.text);
+
+  if (models.length === 0 && /Apple Foundation Models CLI/i.test(stripAnsi(help.text))) {
+    models = DEFAULT_MODELS;
+  }
+
+  return {
+    fmBin,
+    models,
+    help: stripAnsi(help.text)
+  };
+}
+
+export function parseAvailabilityOutput(model, output, code) {
+  const clean = stripAnsi(output).trim();
+  const lower = clean.toLowerCase();
+  const modelLower = model.toLowerCase();
+  const hasError = /\berror:|\bunavailable\b|\bnot available\b|\bnot supported\b/.test(lower);
+  const hasAvailable = new RegExp(`\\b${escapeRegExp(modelLower)}\\b[\\s\\S]{0,80}\\bavailable\\b|\\bavailable\\b[\\s\\S]{0,80}\\b${escapeRegExp(modelLower)}\\b`).test(lower)
+    || lower.includes(`${modelLower} model available`)
+    || lower.includes(`${titleCase(modelLower)} model available`.toLowerCase());
+
+  return {
+    model,
+    available: code === 0 && hasAvailable && !hasError,
+    raw: clean,
+    reason: hasError ? clean : ''
+  };
+}
+
+export async function checkModelAvailability(fmBin, model, options = {}) {
+  const result = await runProcess(fmBin, ['available', '--model', model], {
+    timeoutMs: options.timeoutMs ?? 15_000
+  });
+  const output = `${result.stdout}${result.stderr}`;
+  const parsed = parseAvailabilityOutput(model, output, result.code);
+  if (result.error) {
+    parsed.available = false;
+    parsed.reason = result.stderr || result.error.message;
+  }
+  return parsed;
+}
+
+export async function getQuotaUsage(fmBin, model, options = {}) {
+  const result = await runProcess(fmBin, ['quota-usage', '--model', model], {
+    timeoutMs: options.timeoutMs ?? 15_000
+  });
+  const output = stripAnsi(`${result.stdout}${result.stderr}`).trim();
+  return {
+    model,
+    ok: result.code === 0,
+    raw: output,
+    unavailable: /\bunavailable\b|\bnot available\b|\berror:/i.test(output)
+  };
+}
+
+export async function countTokens(fmBin, text, options = {}) {
+  const result = await runProcess(fmBin, ['token-count', '--quiet'], {
+    input: text,
+    timeoutMs: options.timeoutMs ?? 15_000
+  });
+  const output = stripAnsi(`${result.stdout}${result.stderr}`).trim();
+  const match = output.match(/-?\d+/);
+  if (result.code !== 0 || !match) {
+    return {
+      ok: false,
+      count: null,
+      raw: output
+    };
+  }
+  return {
+    ok: true,
+    count: Number.parseInt(match[0], 10),
+    raw: output
+  };
+}
+
+export async function respond(fmBin, model, prompt, options = {}) {
+  const args = ['respond', '--model', model];
+
+  if (options.stream === false) args.push('--no-stream');
+  if (options.greedy) args.push('--greedy');
+  if (options.instructions) args.push('--instructions', options.instructions);
+  if (options.useCase) args.push('--use-case', options.useCase);
+  if (options.guardrails) args.push('--guardrails', options.guardrails);
+
+  const result = await runProcess(fmBin, args, {
+    input: prompt,
+    timeoutMs: options.timeoutMs ?? 60_000
+  });
+
+  const output = stripAnsi(result.stdout).trim();
+  const errorText = stripAnsi(result.stderr).trim();
+  return {
+    ok: result.code === 0 && !result.timedOut,
+    model,
+    prompt,
+    output,
+    stderr: errorText,
+    code: result.code,
+    signal: result.signal,
+    timedOut: result.timedOut,
+    durationMs: result.durationMs
+  };
+}
+
+export async function collectEnvironment(fmBin) {
+  const swVers = await runProcess('sw_vers', [], { timeoutMs: 5_000 });
+  return {
+    platform: process.platform,
+    arch: process.arch,
+    node: process.version,
+    host: os.hostname(),
+    fmBin,
+    macOS: stripAnsi(swVers.stdout).trim() || null
+  };
+}
+
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function titleCase(value) {
+  return value.slice(0, 1).toUpperCase() + value.slice(1);
+}

package/src/process.js

@@ -0,0 +1,82 @@
+import { spawn } from 'node:child_process';
+
+export function runProcess(command, args = [], options = {}) {
+  const {
+    input,
+    timeoutMs = 30_000,
+    env = process.env,
+    cwd = process.cwd()
+  } = options;
+
+  return new Promise((resolve) => {
+    const startedAt = process.hrtime.bigint();
+    const child = spawn(command, args, {
+      cwd,
+      env,
+      stdio: ['pipe', 'pipe', 'pipe']
+    });
+
+    let stdout = '';
+    let stderr = '';
+    let timedOut = false;
+    let settled = false;
+
+    const timer = timeoutMs > 0
+      ? setTimeout(() => {
+          timedOut = true;
+          child.kill('SIGTERM');
+          setTimeout(() => {
+            if (!settled) child.kill('SIGKILL');
+          }, 1_000).unref();
+        }, timeoutMs)
+      : null;
+
+    child.stdout.setEncoding('utf8');
+    child.stderr.setEncoding('utf8');
+    child.stdout.on('data', (chunk) => {
+      stdout += chunk;
+    });
+    child.stderr.on('data', (chunk) => {
+      stderr += chunk;
+    });
+
+    child.on('error', (error) => {
+      settled = true;
+      if (timer) clearTimeout(timer);
+      const endedAt = process.hrtime.bigint();
+      resolve({
+        command,
+        args,
+        code: null,
+        signal: null,
+        stdout,
+        stderr: stderr || error.message,
+        error,
+        timedOut,
+        durationMs: Number(endedAt - startedAt) / 1e6
+      });
+    });
+
+    child.on('close', (code, signal) => {
+      settled = true;
+      if (timer) clearTimeout(timer);
+      const endedAt = process.hrtime.bigint();
+      resolve({
+        command,
+        args,
+        code,
+        signal,
+        stdout,
+        stderr,
+        timedOut,
+        durationMs: Number(endedAt - startedAt) / 1e6
+      });
+    });
+
+    if (input != null) {
+      child.stdin.end(input);
+    } else {
+      child.stdin.end();
+    }
+  });
+}

package/src/prompts.js

@@ -0,0 +1,120 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+const PROFILES = {
+  quick: [
+    {
+      id: 'echo-ok',
+      prompt: 'Reply with exactly: ok'
+    }
+  ],
+  standard: [
+    {
+      id: 'echo-ok',
+      prompt: 'Reply with exactly: ok'
+    },
+    {
+      id: 'explain-latency',
+      prompt: 'In one concise paragraph, explain why AI model benchmarks should report latency percentiles.'
+    },
+    {
+      id: 'json-transform',
+      prompt: 'Convert this list into a valid compact JSON array of strings: alpha, beta, gamma.'
+    }
+  ],
+  stress: [
+    {
+      id: 'echo-ok',
+      prompt: 'Reply with exactly: ok'
+    },
+    {
+      id: 'explain-latency',
+      prompt: 'In one concise paragraph, explain why AI model benchmarks should report latency percentiles.'
+    },
+    {
+      id: 'json-transform',
+      prompt: 'Convert this list into a valid compact JSON array of strings: alpha, beta, gamma.'
+    },
+    {
+      id: 'reasoning',
+      prompt: 'A build starts at 09:12, takes 17 minutes, waits 8 minutes for review, then takes another 11 minutes. What time does it finish? Show only the answer and one short explanation.'
+    },
+    {
+      id: 'summarize',
+      prompt: 'Summarize this in two bullets: local model benchmarks should measure first-token latency, total latency, throughput, failures, and the exact prompt suite so results can be compared later.'
+    }
+  ]
+};
+
+export function getPromptProfile(name = 'standard') {
+  if (!PROFILES[name]) {
+    throw new Error(`Unknown prompt profile "${name}". Use one of: ${Object.keys(PROFILES).join(', ')}`);
+  }
+  return PROFILES[name].map((prompt) => ({ ...prompt }));
+}
+
+export async function loadPrompts(options = {}) {
+  const prompts = [];
+
+  if (options.promptFile) {
+    prompts.push(...await loadPromptFile(options.promptFile));
+  }
+
+  for (const prompt of options.prompts || []) {
+    prompts.push({
+      id: `custom-${prompts.length + 1}`,
+      prompt
+    });
+  }
+
+  if (prompts.length === 0) {
+    prompts.push(...getPromptProfile(options.profile || 'standard'));
+  }
+
+  return prompts.map((entry, index) => ({
+    id: entry.id || `prompt-${index + 1}`,
+    prompt: String(entry.prompt ?? entry.text ?? '').trim()
+  })).filter((entry) => entry.prompt.length > 0);
+}
+
+async function loadPromptFile(filePath) {
+  const absolutePath = path.resolve(filePath);
+  const content = await fs.readFile(absolutePath, 'utf8');
+  const trimmed = content.trim();
+
+  if (!trimmed) return [];
+
+  if (absolutePath.endsWith('.json')) {
+    const parsed = JSON.parse(trimmed);
+    const items = Array.isArray(parsed) ? parsed : parsed.prompts;
+    if (!Array.isArray(items)) {
+      throw new Error('Prompt JSON must be an array or an object with a prompts array');
+    }
+    return items.map((item, index) => normalizePromptItem(item, index));
+  }
+
+  if (absolutePath.endsWith('.jsonl')) {
+    return trimmed.split(/\r?\n/)
+      .filter(Boolean)
+      .map((line, index) => normalizePromptItem(JSON.parse(line), index));
+  }
+
+  return trimmed.split(/\n\s*\n/g).map((prompt, index) => ({
+    id: `file-${index + 1}`,
+    prompt: prompt.trim()
+  }));
+}
+
+function normalizePromptItem(item, index) {
+  if (typeof item === 'string') {
+    return {
+      id: `file-${index + 1}`,
+      prompt: item
+    };
+  }
+
+  return {
+    id: item.id || item.name || `file-${index + 1}`,
+    prompt: item.prompt || item.text || ''
+  };
+}

package/src/report.js

@@ -0,0 +1,51 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+export function toCsv(rows) {
+  if (rows.length === 0) return '';
+  const headers = Object.keys(rows[0]);
+  return [
+    headers.join(','),
+    ...rows.map((row) => headers.map((header) => csvEscape(row[header])).join(','))
+  ].join('\n');
+}
+
+export function flattenResults(results) {
+  return results.map((result) => ({
+    model: result.model,
+    prompt_id: result.promptId,
+    run: result.run,
+    ok: result.ok,
+    duration_ms: round(result.durationMs),
+    prompt_tokens: result.promptTokens ?? '',
+    output_tokens: result.outputTokens ?? '',
+    chars: result.chars,
+    words: result.words,
+    tokens_per_second: result.tokensPerSecond == null ? '' : round(result.tokensPerSecond),
+    chars_per_second: round(result.charsPerSecond),
+    error: result.error || ''
+  }));
+}
+
+export async function writeReport(filePath, payload, format) {
+  const target = path.resolve(filePath);
+  await fs.mkdir(path.dirname(target), { recursive: true });
+  const content = format === 'csv'
+    ? toCsv(flattenResults(payload.results))
+    : `${JSON.stringify(payload, null, 2)}\n`;
+  await fs.writeFile(target, content, 'utf8');
+  return target;
+}
+
+function csvEscape(value) {
+  const text = String(value ?? '');
+  if (/[",\n\r]/.test(text)) {
+    return `"${text.replaceAll('"', '""')}"`;
+  }
+  return text;
+}
+
+function round(value) {
+  if (!Number.isFinite(value)) return '';
+  return Math.round(value * 100) / 100;
+}

package/src/stats.js

@@ -0,0 +1,85 @@
+export function summarizeNumbers(values) {
+  const clean = values.filter((value) => Number.isFinite(value)).sort((a, b) => a - b);
+  if (clean.length === 0) {
+    return {
+      count: 0,
+      min: null,
+      max: null,
+      avg: null,
+      p50: null,
+      p95: null
+    };
+  }
+
+  const total = clean.reduce((sum, value) => sum + value, 0);
+  return {
+    count: clean.length,
+    min: clean[0],
+    max: clean[clean.length - 1],
+    avg: total / clean.length,
+    p50: percentile(clean, 50),
+    p95: percentile(clean, 95)
+  };
+}
+
+export function percentile(sortedValues, percentileValue) {
+  if (sortedValues.length === 0) return null;
+  if (sortedValues.length === 1) return sortedValues[0];
+
+  const rank = (percentileValue / 100) * (sortedValues.length - 1);
+  const low = Math.floor(rank);
+  const high = Math.ceil(rank);
+  if (low === high) return sortedValues[low];
+  const weight = rank - low;
+  return sortedValues[low] * (1 - weight) + sortedValues[high] * weight;
+}
+
+export function summarizeByModel(results, modelStatuses = []) {
+  const byModel = new Map();
+
+  for (const status of modelStatuses) {
+    byModel.set(status.name, {
+      model: status.name,
+      description: status.description,
+      available: status.available,
+      skippedReason: status.available ? '' : status.reason || 'Unavailable',
+      results: []
+    });
+  }
+
+  for (const result of results) {
+    if (!byModel.has(result.model)) {
+      byModel.set(result.model, {
+        model: result.model,
+        description: '',
+        available: true,
+        skippedReason: '',
+        results: []
+      });
+    }
+    byModel.get(result.model).results.push(result);
+  }
+
+  return [...byModel.values()].map((entry) => {
+    const successes = entry.results.filter((result) => result.ok);
+    const failures = entry.results.filter((result) => !result.ok);
+    const latency = summarizeNumbers(successes.map((result) => result.durationMs));
+    const outputTokens = summarizeNumbers(successes.map((result) => result.outputTokens).filter((value) => value != null));
+    const charsPerSecond = summarizeNumbers(successes.map((result) => result.charsPerSecond));
+    const tokensPerSecond = summarizeNumbers(successes.map((result) => result.tokensPerSecond).filter((value) => value != null));
+
+    return {
+      model: entry.model,
+      description: entry.description,
+      available: entry.available,
+      skippedReason: entry.skippedReason,
+      attempted: entry.results.length,
+      successes: successes.length,
+      failures: failures.length,
+      latency,
+      outputTokens,
+      charsPerSecond,
+      tokensPerSecond
+    };
+  });
+}

package/src/table.js

@@ -0,0 +1,94 @@
+export function renderTable(headers, rows) {
+  const stringRows = rows.map((row) => row.map(formatCell));
+  const widths = headers.map((header, index) => {
+    const values = [header, ...stringRows.map((row) => row[index] ?? '')];
+    return Math.max(...values.map(visibleLength));
+  });
+
+  const separator = `+-${widths.map((width) => '-'.repeat(width)).join('-+-')}-+`;
+  const headerLine = `| ${headers.map((header, index) => pad(header, widths[index])).join(' | ')} |`;
+  const bodyLines = stringRows.map((row) => `| ${row.map((cell, index) => pad(cell, widths[index], isNumericCell(cell))).join(' | ')} |`);
+
+  return [separator, headerLine, separator, ...bodyLines, separator].join('\n');
+}
+
+export function formatMs(value) {
+  if (value == null) return '-';
+  if (value >= 1000) return `${(value / 1000).toFixed(2)}s`;
+  return `${Math.round(value)}ms`;
+}
+
+export function formatNumber(value, digits = 1) {
+  if (value == null || !Number.isFinite(value)) return '-';
+  if (Math.abs(value) >= 100) return Math.round(value).toLocaleString('en-US');
+  return value.toFixed(digits);
+}
+
+export function renderSummaryTable(summary) {
+  const rows = summary.map((item) => {
+    const status = item.available ? (item.failures > 0 ? 'partial' : 'ok') : 'skipped';
+    return [
+      item.model,
+      status,
+      item.attempted || '-',
+      item.successes || '-',
+      item.failures || '-',
+      formatMs(item.latency.p50),
+      formatMs(item.latency.p95),
+      formatMs(item.latency.avg),
+      formatNumber(item.tokensPerSecond.avg),
+      formatNumber(item.charsPerSecond.avg, 0),
+      formatNumber(item.outputTokens.avg, 0),
+      item.available ? '' : compactReason(item.skippedReason)
+    ];
+  });
+
+  return renderTable([
+    'model',
+    'status',
+    'runs',
+    'ok',
+    'fail',
+    'p50',
+    'p95',
+    'avg',
+    'tok/s',
+    'char/s',
+    'out tok',
+    'note'
+  ], rows);
+}
+
+export function renderModelsTable(models) {
+  return renderTable(['model', 'available', 'description', 'quota'], models.map((model) => [
+    model.name,
+    model.available ? 'yes' : 'no',
+    model.description || '-',
+    compactReason(model.quota || model.reason || '-')
+  ]));
+}
+
+function formatCell(value) {
+  if (value == null) return '';
+  return String(value);
+}
+
+function pad(value, width, left = false) {
+  const length = visibleLength(value);
+  const padding = ' '.repeat(Math.max(0, width - length));
+  return left ? `${padding}${value}` : `${value}${padding}`;
+}
+
+function isNumericCell(value) {
+  return /^-?$|^[\d,.]+(?:ms|s)?$/.test(value);
+}
+
+function visibleLength(value) {
+  return String(value).length;
+}
+
+function compactReason(value) {
+  const clean = String(value || '').replace(/\s+/g, ' ').trim();
+  if (clean.length <= 58) return clean;
+  return `${clean.slice(0, 55)}...`;
+}