fm-bench 0.7.0 → 0.7.2

package/bin/fm-bench.js

@@ -4,18 +4,29 @@ import { killActiveChildren } from '../src/process.js';
 
 // Terminate any in-flight `fm` child process before exiting, then use the
 // conventional 128 + signal exit code. A second signal exits immediately.
-let interrupting = false;
+let signalsReceived = 0;
+let interrupted = false;
 for (const [signal, exitCode] of [['SIGINT', 130], ['SIGTERM', 143]]) {
   process.on(signal, () => {
+    signalsReceived += 1;
+    interrupted = true;
+    // Carry the status on the process too, so the code survives even if the
+    // exit below is preempted.
+    process.exitCode = exitCode;
     const killed = killActiveChildren(signal);
-    if (interrupting) process.exit(exitCode);
-    interrupting = true;
-    const graceMs = killed > 0 ? 500 : 0;
-    setTimeout(() => process.exit(exitCode), graceMs).unref();
+    if (signalsReceived > 1 || killed === 0) process.exit(exitCode);
+    // Deliberately not unref'd: this timer performs the exit after giving the
+    // killed children a moment to be reaped.
+    setTimeout(() => process.exit(exitCode), 500);
   });
 }
 
 runCli(process.argv.slice(2)).catch((error) => {
+  if (interrupted) {
+    // The user asked to stop; a follow-up benchmark error is noise.
+    process.exitCode = typeof error?.exitCode === 'number' ? error.exitCode : 1;
+    return;
+  }
   const message = error?.message || String(error);
   console.error(`fm-bench: ${message}`);
   process.exitCode = typeof error?.exitCode === 'number' ? error.exitCode : 1;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fm-bench",
-  "version": "0.7.0",
+  "version": "0.7.2",
   "description": "Dynamic benchmark CLI for Apple's fm command on macOS 27+.",
   "type": "module",
   "bin": {
@@ -21,14 +21,20 @@
     "release:patch": "npm version patch && git push --follow-tags",
     "release:minor": "npm version minor && git push --follow-tags",
     "release:major": "npm version major && git push --follow-tags",
-    "publish:dry-run": "npm publish --dry-run --access public",
+    "publish:dry-run": "node scripts/publish-dry-run.mjs",
     "check:pack": "node scripts/check-package.mjs"
   },
   "keywords": [
     "apple",
+    "apple-intelligence",
+    "apple-silicon",
     "foundation-models",
     "fm",
+    "llm",
     "benchmark",
+    "benchmarking",
+    "latency",
+    "throughput",
     "macos",
     "cli"
   ],

package/src/bench.js

@@ -1,13 +1,13 @@
 import crypto from 'node:crypto';
 import { detectFmCapabilities } from './capabilities.js';
-import { checkModelAvailability, collectEnvironment, countTokens, getQuotaUsage, respond } from './fm.js';
+import { checkModelAvailability, collectEnvironment, countTokens, fmBinaryFromOptions, getQuotaUsage, respond } from './fm.js';
 import { metricAvailability } from './metrics.js';
 import { loadPrompts } from './prompts.js';
 import { finalizeReportPayload } from './schema.js';
 import { summarizeByModel } from './stats.js';
 
 export async function inspectModels(options = {}) {
-  const fmBin = options.fmBin || process.env.FM_BIN || 'fm';
+  const fmBin = fmBinaryFromOptions(options);
   const capabilities = options.capabilities ?? await detectFmCapabilities(fmBin, options);
   const discovered = {
     fmBin,
@@ -52,7 +52,7 @@ export async function inspectModels(options = {}) {
 
 export async function runBenchmark(options = {}) {
   const startedAt = new Date().toISOString();
-  const fmBin = options.fmBin || process.env.FM_BIN || 'fm';
+  const fmBin = fmBinaryFromOptions(options);
 
   notify(options, { type: 'phase', phase: 'capabilities', message: 'probing fm capabilities' });
   const capabilities = options.capabilities ?? await detectFmCapabilities(fmBin, options);
@@ -381,13 +381,22 @@ async function runSingleBenchmark(context) {
 
 async function runLimited(items, concurrency, worker, options = {}) {
   let nextIndex = 0;
+  // fail-fast stops admitting new work; calls already in flight are allowed to
+  // finish so no `fm` process is abandoned mid-run.
+  let cancelled = false;
   const waitForSlot = createPacer(options.requestRate, options.rampUpMs);
   const workers = Array.from({ length: Math.min(concurrency, items.length) }, async () => {
-    while (nextIndex < items.length) {
+    while (!cancelled && nextIndex < items.length) {
       const index = nextIndex;
       nextIndex += 1;
       await waitForSlot(index);
-      await worker(items[index]);
+      if (cancelled) break;
+      try {
+        await worker(items[index]);
+      } catch (error) {
+        cancelled = true;
+        throw error;
+      }
     }
   });
   await Promise.all(workers);

package/src/capabilities.js

@@ -9,7 +9,7 @@
 
 import crypto from 'node:crypto';
 import { stripAnsi } from './ansi.js';
-import { parseModelsFromHelp } from './fm-help.js';
+import { parseAvailabilityList, parseModelsFromHelp } from './fm-help.js';
 import { runProcess } from './process.js';
 
 const SECTION_HEADER = /^\s*[A-Z][A-Z0-9 /-]+\s*$/;
@@ -179,16 +179,8 @@ async function discoverModelsFromAvailability(fmBin, options = {}) {
     env: options.env ?? process.env
   });
   if (result.error) return [];
-  const text = stripAnsi(`${result.stdout}${result.stderr}`);
-  const models = new Map();
-  for (const line of text.split(/\r?\n/)) {
-    const match = line.match(/^\s*([A-Za-z0-9._-]+)(?:\s+model)?\s+(?:is\s+)?(available|unavailable|not available)\b/i);
-    if (!match) continue;
-    const name = match[1].toLowerCase();
-    if (['the', 'no', 'model', 'system'].includes(name) && name !== 'system') continue;
-    models.set(name, { name, description: '' });
-  }
-  return [...models.values()];
+  return parseAvailabilityList(`${result.stdout}${result.stderr}`)
+    .map((model) => ({ name: model.name, description: '' }));
 }
 
 function escapeRegExp(value) {

package/src/fm.js

@@ -25,22 +25,6 @@ export async function getFmHelp(fmBin, timeoutMs = 10_000) {
   };
 }
 
-/**
- * Discover models, preferring an already-detected capability probe so a run
- * does not spawn `fm --help` more than once.
- * @param {Record<string, any>} options
- */
-export async function discoverModels(options = {}) {
-  const fmBin = fmBinaryFromOptions(options);
-  const capabilities = options.capabilities ?? await detectFmCapabilities(fmBin, options);
-  return {
-    fmBin,
-    models: capabilities.models,
-    help: capabilities.help,
-    capabilities
-  };
-}
-
 /**
  * Check one model against the detected `fm` build.
  *

package/src/metrics.js

@@ -192,15 +192,6 @@ export function metricAvailability(capabilities = {}, options = {}) {
   return metrics;
 }
 
-export function metricDefinition(key) {
-  const definition = DEFINITIONS.find((item) => item.key === key);
-  return definition ? { ...definition, requires: [...definition.requires] } : null;
-}
-
-export function metricDefinitions() {
-  return DEFINITIONS.map((definition) => ({ ...definition, requires: [...definition.requires] }));
-}
-
 /** Short human line for CLI output, e.g. "token counting: yes, quota: no". */
 export function formatCapabilitySummary(capabilities = {}) {
   const features = capabilities.features ?? {};