fm-bench 0.6.2 → 0.6.3

package/README.md

@@ -30,7 +30,7 @@ fm-bench
 One command discovers your models, runs the standard prompt suite, and prints a full benchmark report:
 
 ```text
-fm-bench 0.6.2 | darwin/arm64 | fm
+fm-bench 0.6.3 | darwin/arm64 | fm
 prompts 3 | runs 5 | concurrency 1 | stream on | measured 15 | failed 0 | skipped 0 | elapsed 38.20s
 
 ┌───┬────────┬────────┬─────────┬──────┬──────┬──────────┬──────┬──────────┬─────┬─────┐
@@ -273,10 +273,12 @@ A live single-line progress indicator runs on stderr during interactive sessions
 
 ## Requirements
 
-- macOS 27 or newer (Apple's `fm` CLI is preinstalled).
+- macOS 27.0 or newer (Apple's `fm` CLI is preinstalled there).
 - Node.js 20 or newer.
 - Apple Intelligence enabled on the device.
 
+Benchmark commands refuse to start on older macOS versions and report the detected version plus the latest supported macOS — see [docs/supported-platforms.md](docs/supported-platforms.md).
+
 `pcc` (Private Cloud Compute) availability depends on Apple's current eligibility. `fm-bench` shows it as skipped if `fm available --model pcc` reports unavailable.
 
 See [docs/methodology.md](docs/methodology.md) for benchmark methodology and metric references.

package/docs/releasing.md

@@ -17,7 +17,7 @@
 ## Option B — Local
 
 ```sh
-npm ci && npm test && npm run lint
+npm ci && npm test && npm run lint && npm run check:pack
 npm version patch   # or minor / major
 git push --follow-tags
 ```
@@ -37,6 +37,7 @@ gh release edit v0.6.0 --notes-file /tmp/notes.md --repo devinoldenburg/fm-bench
 
 ```sh
 npm run publish:dry-run
+npm run check:pack
 ```
 
 CI runs the same dry-run on every push to `main` and on pull requests.

package/docs/supported-platforms.md

@@ -0,0 +1,26 @@
+# Supported platforms
+
+`fm-bench` benchmarks Apple's `fm` command, so support follows the platforms where Apple ships that CLI.
+
+## Requirements
+
+- **macOS 27.0 or newer** — Apple's `fm` CLI is preinstalled starting with macOS 27. Older macOS releases do not ship `fm`, so `fm-bench` cannot run there.
+- **Node.js 20 or newer**.
+- **Apple Intelligence enabled** on the device.
+
+`pcc` (Private Cloud Compute) availability additionally depends on Apple's current eligibility. `fm-bench` reports it as skipped when `fm available --model pcc` reports unavailable.
+
+## Version enforcement
+
+Commands that launch `fm` benchmarks — the default `run` command and `models` — check the macOS version first and refuse to start on anything older than macOS 27:
+
+```text
+fm-bench: unsupported macOS: detected macOS 26.1, but fm-bench requires macOS 27 or newer (Apple's fm CLI is preinstalled there).
+Latest supported: macOS 27.0 or newer (fm is not available on older macOS releases).
+```
+
+The process exits with code `2`. This is deliberate: running on an unsupported macOS could never produce a valid benchmark, so the CLI fails fast with the exact version it found and the latest supported macOS version.
+
+Commands that only read local report files — `compare`, `history`, `validate`, `export`, and `legend` — are not gated and work anywhere Node.js runs.
+
+`fm-bench doctor` still runs on unsupported hosts so you can diagnose the environment: it prints the detected macOS version, an explicit `macOS support` line, and the latest supported version.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fm-bench",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "Dynamic benchmark CLI for Apple's fm command on macOS 27+.",
   "type": "module",
   "bin": {
@@ -16,11 +16,12 @@
   "scripts": {
     "test": "node --test",
     "lint": "node --check bin/fm-bench.js && find src test -name '*.js' -print0 | xargs -0 -n1 node --check",
-    "prepack": "npm test && npm run lint",
+    "prepack": "npm test && npm run lint && npm run check:pack",
     "release:patch": "npm version patch && git push --follow-tags",
     "release:minor": "npm version minor && git push --follow-tags",
     "release:major": "npm version major && git push --follow-tags",
-    "publish:dry-run": "npm publish --dry-run --access public"
+    "publish:dry-run": "npm publish --dry-run --access public",
+    "check:pack": "node scripts/check-package.mjs"
   },
   "keywords": [
     "apple",
@@ -46,5 +47,8 @@
   "publishConfig": {
     "access": "public",
     "registry": "https://registry.npmjs.org/"
-  }
+  },
+  "os": [
+    "darwin"
+  ]
 }

package/src/cli.js

@@ -5,6 +5,7 @@ import { diffReports, renderCompareReport } from './compare.js';
 import { renderHtmlReport } from './export.js';
 import { validateReport } from './schema.js';
 import { loadHistory, renderHistoryReport } from './history.js';
+import { detectMacosVersion, evaluateMacosSupport, formatMacosRequirementError, MIN_SUPPORTED_MACOS, parseMacosVersion } from './macos.js';
 import { runProcess } from './process.js';
 import { createProgress } from './progress.js';
 import { flattenResults, toCsv, writeReport } from './report.js';
@@ -14,7 +15,7 @@ import { legendEntries, renderBenchmarkReport, renderLatencyHistogram, renderLeg
 const require = createRequire(import.meta.url);
 const packageJson = require('../package.json');
 
-export async function runCli(argv = process.argv.slice(2)) {
+export async function runCli(argv = process.argv.slice(2), env = {}) {
   const parsed = parseArgs(argv);
 
   if (parsed.help) {
@@ -27,6 +28,8 @@ export async function runCli(argv = process.argv.slice(2)) {
     return;
   }
 
+  await assertSupportedMacos(parsed, env);
+
   if (parsed.command === 'legend') {
     if (parsed.format === 'json') {
       console.log(JSON.stringify(legendEntries(), null, 2));
@@ -153,6 +156,25 @@ export async function runCli(argv = process.argv.slice(2)) {
   }
 }
 
+// Offline report tools stay usable anywhere. `doctor` is also allowed through so
+// it can print the detected version and the latest supported macOS when the host
+// is too old. Only commands that launch `fm` benchmarks are hard-gated.
+const SKIP_MACOS_GATE = new Set(['compare', 'history', 'validate', 'export', 'legend', 'doctor']);
+
+async function assertSupportedMacos(parsed, env = {}) {
+  if (SKIP_MACOS_GATE.has(parsed.command)) return;
+
+  const evaluation = evaluateMacosSupport(
+    env.platform ?? process.platform,
+    parseMacosVersion(await detectMacosVersion(env))
+  );
+  if (evaluation.supported) return;
+
+  const error = new Error(formatMacosRequirementError(evaluation));
+  error.exitCode = 2;
+  throw error;
+}
+
 function evaluateCi(payload) {
   const reasons = [];
   const totalFailed = payload.summary.reduce((sum, item) => sum + item.failures, 0);
@@ -554,11 +576,14 @@ async function runDoctor(options) {
   checks.push(['node', process.version, true]);
   checks.push(['platform', `${process.platform}/${process.arch}`, process.platform === 'darwin']);
 
-  const swVers = await runProcess('sw_vers', [], { timeoutMs: 5_000 });
-  const macOS = swVers.stdout || swVers.stderr;
-  const versionMatch = macOS.match(/ProductVersion:\s*([0-9.]+)/);
-  const major = versionMatch ? Number.parseInt(versionMatch[1].split('.')[0], 10) : null;
-  checks.push(['macOS', versionMatch?.[1] || 'unknown', major == null || major >= 27]);
+  const macOS = await detectMacosVersion();
+  const parsedVersion = parseMacosVersion(macOS);
+  const support = evaluateMacosSupport(process.platform, parsedVersion);
+  checks.push(['macOS', parsedVersion?.version || 'unknown', support.supported]);
+  if (!support.supported) {
+    checks.push(['macOS support', support.reason, false]);
+    checks.push(['latest supported', support.latestSupported, false]);
+  }
 
   const hwModel = await runProcess('sysctl', ['-n', 'hw.model'], { timeoutMs: 3_000 });
   const hwModelStr = (hwModel.stdout || '').trim();
@@ -595,17 +620,23 @@ async function runDoctor(options) {
     checks.push(['battery', `${pct} (${source})`, ok]);
   }
 
-  const inspection = await inspectModels(options);
-  checks.push(['fm', inspection.fmBin, inspection.models.length > 0]);
-  for (const model of inspection.models) {
-    checks.push([`model:${model.name}`, model.available ? 'available' : model.reason || 'unavailable', model.available]);
+  let models = [];
+  try {
+    const inspection = await inspectModels(options);
+    models = inspection.models;
+    checks.push(['fm', inspection.fmBin, inspection.models.length > 0]);
+    for (const model of inspection.models) {
+      checks.push([`model:${model.name}`, model.available ? 'available' : model.reason || 'unavailable', model.available]);
+    }
+  } catch (error) {
+    checks.push(['fm', error.message || String(error), false]);
   }
 
   const lines = checks.map(([name, detail, ok]) => `${ok ? 'ok  ' : 'warn'} ${name.padEnd(16)} ${String(detail).replace(/\s+/g, ' ').trim()}`);
   console.log(lines.join('\n'));
 
   if (options.out) {
-    await fs.writeFile(options.out, `${JSON.stringify({ checks, models: inspection.models }, null, 2)}\n`, 'utf8');
+    await fs.writeFile(options.out, `${JSON.stringify({ checks, models }, null, 2)}\n`, 'utf8');
   }
 }
 
@@ -669,7 +700,7 @@ function resolveProgress(parsed) {
 function helpText() {
   return `fm-bench ${packageJson.version}
 
-Dynamic benchmark CLI for Apple's fm command on macOS 27+.
+Dynamic benchmark CLI for Apple's fm command on macOS ${MIN_SUPPORTED_MACOS}+.
 
 Usage:
   fm-bench [run] [options]

package/src/macos.js

@@ -0,0 +1,71 @@
+import { runProcess } from './process.js';
+
+// macOS releases that ship Apple's `fm` CLI. Older macOS versions cannot run
+// fm-bench at all because the benchmarked binary does not exist there.
+export const SUPPORTED_MACOS_MAJOR = 27;
+
+export const MIN_SUPPORTED_MACOS = `${SUPPORTED_MACOS_MAJOR}.0`;
+
+export const MACOS_REQUIREMENT_MESSAGE =
+  `fm-bench requires macOS ${SUPPORTED_MACOS_MAJOR} or newer (Apple's fm CLI is preinstalled there).`;
+
+export function parseMacosVersion(text = '') {
+  const match = String(text).match(/ProductVersion:\s*(\d+)(?:\.(\d+))?(?:\.(\d+))?/);
+  if (!match) return null;
+  return {
+    major: Number.parseInt(match[1], 10),
+    minor: match[2] != null ? Number.parseInt(match[2], 10) : 0,
+    patch: match[3] != null ? Number.parseInt(match[3], 10) : 0,
+    version: [match[1], match[2], match[3]].filter((part) => part != null).join('.')
+  };
+}
+
+export function isMacosSupported(parsed) {
+  return parsed != null && Number.isInteger(parsed.major) && parsed.major >= SUPPORTED_MACOS_MAJOR;
+}
+
+export function evaluateMacosSupport(platform, parsed) {
+  if (platform !== 'darwin') {
+    return {
+      supported: false,
+      reason: `fm-bench only runs on macOS with Apple's fm CLI (detected platform: ${platform}).`,
+      latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer`
+    };
+  }
+  if (parsed == null) {
+    return {
+      supported: true,
+      warnOnly: true,
+      reason: 'could not determine the macOS version',
+      latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer`
+    };
+  }
+  if (isMacosSupported(parsed)) {
+    return {
+      supported: true,
+      reason: null,
+      latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer`
+    };
+  }
+  return {
+    supported: false,
+    reason: `detected macOS ${parsed.version}, but ${MACOS_REQUIREMENT_MESSAGE}`,
+    latestSupported: `macOS ${MIN_SUPPORTED_MACOS} or newer`
+  };
+}
+
+export function formatMacosRequirementError({ reason, latestSupported }) {
+  return [
+    `unsupported macOS: ${reason}`,
+    `Latest supported: ${latestSupported} (fm is not available on older macOS releases).`
+  ].join('\n');
+}
+
+export async function detectMacosVersion(options = {}) {
+  const timeoutMs = options.timeoutMs ?? 5_000;
+  if (typeof options.swVers === 'function') {
+    return options.swVers();
+  }
+  const result = await runProcess('sw_vers', [], { timeoutMs });
+  return result.stdout || result.stderr;
+}