fm-bench 0.6.0 → 0.6.2

package/README.md

@@ -30,7 +30,7 @@ fm-bench
 One command discovers your models, runs the standard prompt suite, and prints a full benchmark report:
 
 ```text
-fm-bench 0.6.0 | darwin/arm64 | fm
+fm-bench 0.6.2 | darwin/arm64 | fm
 prompts 3 | runs 5 | concurrency 1 | stream on | measured 15 | failed 0 | skipped 0 | elapsed 38.20s
 
 ┌───┬────────┬────────┬─────────┬──────┬──────┬──────────┬──────┬──────────┬─────┬─────┐

package/docs/releasing.md

@@ -1,6 +1,6 @@
 # Releasing
 
-`fm-bench` uses semver tags (`v*.*.*`). Pushing a tag runs the **Release** workflow: test, lint, npm publish (with provenance), and GitHub release notes.
+`fm-bench` uses semver tags (`v*.*.*`). Pushing a tag runs the **Release** workflow: test, lint, npm publish (with provenance), and a GitHub release whose body is taken from the matching section in **`CHANGELOG.md`** (plus a compare link).
 
 ## Prerequisites
 
@@ -24,7 +24,14 @@ git push --follow-tags
 
 ## Re-run Release without republishing
 
-If npm already has the version but GitHub release failed (or vice versa), use **Actions → Release → Run workflow** and enter the existing tag (for example `v0.5.3`). The workflow skips npm publish when that version is already on the registry and skips creating a duplicate GitHub release.
+If npm already has the version but GitHub release failed (or vice versa), use **Actions → Release → Run workflow** and enter the existing tag (for example `v0.5.3`). The workflow skips npm publish when that version is already on the registry. If the GitHub release already exists, it **updates the release notes** from `CHANGELOG.md`.
+
+Refresh notes locally without re-publishing:
+
+```sh
+node scripts/changelog-release-notes.mjs 0.6.0 > /tmp/notes.md
+gh release edit v0.6.0 --notes-file /tmp/notes.md --repo devinoldenburg/fm-bench
+```
 
 ## Dry run
 

package/docs/report-format.md

@@ -12,7 +12,7 @@ Every measured run can be saved as JSON. Reports from fm-bench **0.6.0+** includ
 | `reportId` | Random hex id for citing a single run |
 | `startedAt` / `finishedAt` | ISO-8601 timestamps |
 | `options` | Public run configuration (profile, runs, concurrency, SLOs, tags, note) |
-| `environment` | Host fingerprint: platform, arch, Node, hardware model, CPU, memory, macOS version, `fm` help digest, thermal/power snapshot |
+| `environment` | Host fingerprint: platform, arch, Node, hardware model, CPU, memory, macOS version/build, `fm` help digest, thermal/power snapshot |
 | `suite` | Derived suite key + fingerprint for apples-to-apples comparison |
 | `prompts` | Prompt ids, text, and token counts |
 | `models` | Discovered models and availability |
@@ -38,6 +38,7 @@ For fair comparison, match:
 - Same `--profile` (or same `--prompt-file`)
 - Same `--runs` and `--warmup`
 - Same concurrency operating points (`--concurrency` or `--sweep-concurrency`)
+- Same or intentionally changed macOS build (especially for beta-to-beta comparisons)
 - Similar power/thermal state (see `environment.power` and `environment.thermal`)
 
 ```sh
@@ -45,8 +46,8 @@ fm-bench compare before.json after.json
 fm-bench compare before.json after.json --strict   # exit 2 if suites differ
 ```
 
-`compare` warns when hardware, macOS, or suite configuration differ. Use `--tag` and `--note` so `fm-bench history` stays readable.
+`compare` warns when hardware, macOS product version, macOS build, or suite configuration differ. Use `--tag` and `--note` so `fm-bench history` stays readable.
 
 ## Legacy reports
 
-Reports from fm-bench before 0.6.0 remain valid JSON. They lack `schemaVersion`, `reportId`, `suite`, and enriched `environment`. `validate` still checks required fields; `compare` works on `summary` as before.
+Reports from fm-bench before 0.6.0 remain valid JSON. They lack `schemaVersion`, `reportId`, `suite`, and enriched `environment`. `validate` still checks required fields; `compare` works on `summary` as before.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "fm-bench",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "description": "Dynamic benchmark CLI for Apple's fm command on macOS 27+.",
   "type": "module",
   "bin": {

package/src/compare.js

@@ -25,6 +25,12 @@ export function diffReports(before, after) {
 
 function reportMeta(report) {
   const tags = report.options?.tags;
+  const macOS = report.suite?.fingerprint?.macOSProductVersion
+    ?? report.environment?.macOS?.match(/ProductVersion:\s*([^\n]+)/)?.[1]?.trim()
+    ?? null;
+  const macOSBuild = report.suite?.fingerprint?.macOSBuildVersion
+    ?? report.environment?.macOS?.match(/BuildVersion:\s*([^\n]+)/)?.[1]?.trim()
+    ?? null;
   return {
     version: report.version ?? '?',
     schemaVersion: report.schemaVersion ?? null,
@@ -37,9 +43,8 @@ function reportMeta(report) {
     tags: Array.isArray(tags) ? tags : [],
     note: report.options?.note ?? null,
     hwModel: report.environment?.hwModel ?? report.suite?.fingerprint?.hwModel ?? null,
-    macOS: report.suite?.fingerprint?.macOSProductVersion
-      ?? report.environment?.macOS?.match(/ProductVersion:\s*([^\n]+)/)?.[1]?.trim()
-      ?? null
+    macOS,
+    macOSBuild
   };
 }
 
@@ -134,8 +139,8 @@ export function renderCompareReport(diff, options = {}) {
   const afterLabel = `${diff.after.version}  ${diff.after.startedAt ? diff.after.startedAt.slice(0, 19).replace('T', ' ') : '?'}`;
 
   lines.push(`fm-bench compare`);
-  lines.push(`  before: ${beforeLabel}${diff.before.hwModel ? ` | ${diff.before.hwModel}` : ''}`);
-  lines.push(`  after:  ${afterLabel}${diff.after.hwModel ? ` | ${diff.after.hwModel}` : ''}`);
+  lines.push(`  before: ${beforeLabel}${diff.before.hwModel ? ` | ${diff.before.hwModel}` : ''}${formatMacOSMeta(diff.before)}`);
+  lines.push(`  after:  ${afterLabel}${diff.after.hwModel ? ` | ${diff.after.hwModel}` : ''}${formatMacOSMeta(diff.after)}`);
   if (diff.before.profile || diff.after.profile) {
     lines.push(`  suite:  profile=${diff.before.profile ?? '?'} runs=${diff.before.runs ?? '?'} (before) → profile=${diff.after.profile ?? '?'} runs=${diff.after.runs ?? '?'}`);
   }
@@ -221,6 +226,13 @@ export function renderCompareReport(diff, options = {}) {
   return lines.join('\n');
 }
 
+function formatMacOSMeta(meta) {
+  if (!meta.macOS && !meta.macOSBuild) return '';
+  const label = meta.macOS ? `macOS ${meta.macOS}` : 'macOS';
+  const build = meta.macOSBuild ? ` (${meta.macOSBuild})` : '';
+  return ` | ${label}${build}`;
+}
+
 function fmtDiffMs(diff, which, color) {
   const value = diff[which];
   if (value == null) return '-';

package/src/schema.js

@@ -96,6 +96,9 @@ export function compareCompatibility(before, after) {
   if (bFp.macOSProductVersion && aFp.macOSProductVersion && bFp.macOSProductVersion !== aFp.macOSProductVersion) {
     warnings.push(`macOS version differs (${bFp.macOSProductVersion} vs ${aFp.macOSProductVersion})`);
   }
+  if (bFp.macOSBuildVersion && aFp.macOSBuildVersion && bFp.macOSBuildVersion !== aFp.macOSBuildVersion) {
+    warnings.push(`macOS build differs (${bFp.macOSBuildVersion} vs ${aFp.macOSBuildVersion})`);
+  }
 
   return {
     compatible: errors.length === 0,
@@ -155,4 +158,4 @@ function cryptoRandomId() {
   const bytes = new Uint8Array(8);
   globalThis.crypto.getRandomValues(bytes);
   return [...bytes].map((b) => b.toString(16).padStart(2, '0')).join('');
-}
+}