@linzumi/cli 1.0.6 → 1.0.8

package/scripts/golden-scale-test.mjs

@@ -0,0 +1,738 @@
+#!/usr/bin/env node
+// golden-scale-test.mjs
+//
+// SCALE TEST: find the empirical "agents-per-host" number for the forkable-golden
+// model on THIS Mac (Apple Silicon, HVF). Cold-boot ONE forkable golden, provision
+// a warm base (toolchain + small project + npm install), then ramp concurrent forks
+// N = 1,2,4,8,16,24,32,48,64 off that single golden and measure at each N:
+//   - fork wall time + per-fork latency distribution (p50/p95)
+//   - concurrent light-job completion time
+//   - RAM: sum RSS of `smol-vmm _boot-vm` procs (ps) + host free/used (vm_stat)
+//     BEFORE and AFTER the fork wave -> COW marginal-RSS-per-fork
+//   - EAGAIN (os error 35) retries triggered (instrumented helper)
+//   - native crash / connection-closed / instability (caught + recorded)
+// Tear down the N forks before the next stage (delete()); keep the golden.
+//
+// Fork pattern reused from /tmp/smol-rt-132/fork-proof.mjs:
+//   Machine.create({ forkable:true, network:true }) golden -> golden.fork(name)
+//   The golden is exec-DEAD after forking, so we ONLY fork it (never exec post-fork);
+//   all exec jobs run IN the forks. EAGAIN/os-error-35 retry+backoff on every
+//   exec AND fork.
+//
+// HARD HYGIENE: exclusive HVF. Synchronous. ALWAYS delete() in finally. No lingering
+// background processes. NEVER pkill golden-*. No git. Confirm 0 leaked smol-vmm at exit.
+
+import { createRequire } from 'node:module';
+import { join } from 'node:path';
+import { execSync } from 'node:child_process';
+
+const rtDir = '/tmp/smol-rt-132';
+const require = createRequire(join(rtDir, 'package.json'));
+const { Machine } = require('smolmachines');
+
+// ----- config ---------------------------------------------------------------
+const STAGES = [1, 2, 4, 8, 16, 24, 32, 48, 64];
+const PROJECT = '/root/project';
+const FORK_CPUS = 2;
+const FORK_MEM_MB = 2048;
+const MIN_FREE_HEADROOM_GB = 8; // stop ramping if host free RAM < this after a wave
+const FORK_STAGGER_MS = 60; // pace fork launches to respect control-plane EAGAIN
+const PAGE_BYTES = 16384; // Apple Silicon vm_stat page size (verified below)
+// CPU-saturating per-fork job: a fixed-size busy loop. Each fork runs the SAME
+// amount of CPU work, so as N exceeds available cores the per-job wall time grows
+// (jobs queue behind cores). The point where per-job time departs from the N<=cores
+// baseline is the empirical CPU ceiling -- the real "active agents per host".
+const CPU_JOB = `node -e "const t=Date.now();let x=0;for(let i=0;i<8e8;i++){x+=Math.sqrt(i)%7;}console.log(Date.now()-t,x|0)"`;
+
+const t = () => Date.now();
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const log = (...a) => console.log('[scale]', ...a);
+const GB = (bytes) => (bytes / 1024 / 1024 / 1024).toFixed(2);
+const MB = (bytes) => (bytes / 1024 / 1024).toFixed(0);
+
+// ----- EAGAIN-aware retry helpers (instrumented) ----------------------------
+let EAGAIN_RETRIES = 0; // global; snapshotted per stage
+const isEagain = (e) =>
+  /os error 35|temporarily unavailable|EAGAIN|resource temporarily/i.test(
+    String(e?.stack || e)
+  );
+const isInstability = (e) =>
+  /connection closed|connection reset|ECONNRESET|EPIPE|broken pipe|hangup|panic|signal|abort/i.test(
+    String(e?.stack || e)
+  );
+
+async function withRetry(label, fn, { tries = 8, base = 150 } = {}) {
+  let lastErr;
+  for (let attempt = 0; attempt < tries; attempt++) {
+    try {
+      return await fn();
+    } catch (e) {
+      lastErr = e;
+      if (isEagain(e)) {
+        EAGAIN_RETRIES++;
+        await sleep(base * (attempt + 1));
+        continue;
+      }
+      throw e; // non-EAGAIN (incl. instability) bubbles up to be classified
+    }
+  }
+  throw lastErr;
+}
+
+async function sh(m, label, cmd, timeout = 240) {
+  return withRetry(label, async () => {
+    const r = await m.exec(['/bin/sh', '-c', cmd], { timeout });
+    return {
+      code: r.exitCode,
+      out: (r.stdout || '').trim(),
+      err: (r.stderr || '').trim(),
+    };
+  });
+}
+
+// ----- host measurement -----------------------------------------------------
+function sysctlStr(key) {
+  try {
+    return execSync(`sysctl -n ${key}`, { encoding: 'utf8' }).trim();
+  } catch {
+    return '?';
+  }
+}
+
+// Free RAM (bytes) that's actually reclaimable: free + inactive + speculative.
+// Apple's vm_stat counts inactive/speculative as available under pressure.
+function hostMem() {
+  let pageBytes = PAGE_BYTES;
+  let free = 0,
+    inactive = 0,
+    speculative = 0,
+    wired = 0,
+    active = 0,
+    compressed = 0;
+  try {
+    const out = execSync('vm_stat', { encoding: 'utf8' });
+    const m = out.match(/page size of (\d+) bytes/);
+    if (m) pageBytes = Number(m[1]);
+    const grab = (re) => {
+      const mm = out.match(re);
+      return mm ? Number(mm[1]) : 0;
+    };
+    free = grab(/Pages free:\s+(\d+)/);
+    active = grab(/Pages active:\s+(\d+)/);
+    inactive = grab(/Pages inactive:\s+(\d+)/);
+    speculative = grab(/Pages speculative:\s+(\d+)/);
+    wired = grab(/Pages wired down:\s+(\d+)/);
+    compressed = grab(/Pages occupied by compressor:\s+(\d+)/);
+  } catch {}
+  const avail = (free + inactive + speculative) * pageBytes;
+  const used = (active + wired + compressed) * pageBytes;
+  return {
+    availBytes: avail,
+    usedBytes: used,
+    freePagesBytes: free * pageBytes,
+  };
+}
+
+// Sum RSS (bytes) of all `smol-vmm _boot-vm` processes, and count them.
+function vmmRss() {
+  try {
+    // ps rss is in KB. Match the boot-vm worker that backs each live VM.
+    const out = execSync(
+      "ps -axo rss,command | grep 'smol-vmm' | grep '_boot-vm' | grep -v grep",
+      { encoding: 'utf8' }
+    );
+    const lines = out.split('\n').filter((l) => l.trim());
+    let sum = 0;
+    for (const l of lines) {
+      const kb = Number(l.trim().split(/\s+/)[0]);
+      if (Number.isFinite(kb)) sum += kb * 1024;
+    }
+    return { count: lines.length, rssBytes: sum };
+  } catch {
+    return { count: 0, rssBytes: 0 };
+  }
+}
+
+function countLeakedVmm() {
+  try {
+    const out = execSync(
+      "ps -axo pid,command | grep 'smol-vmm' | grep -v grep",
+      {
+        encoding: 'utf8',
+      }
+    );
+    return out.split('\n').filter((l) => l.trim()).length;
+  } catch {
+    return 0;
+  }
+}
+
+function percentile(arr, p) {
+  if (!arr.length) return 0;
+  const s = [...arr].sort((a, b) => a - b);
+  const idx = Math.min(s.length - 1, Math.floor((p / 100) * s.length));
+  return s[idx];
+}
+
+// ----- one-time wedged-runtime recovery -------------------------------------
+async function maybeUnwedge() {
+  // If the runtime is wedged, a first exec returns connection-closed. Probe once
+  // with a throwaway non-forkable micro-VM; if it fails, kill stray boot-vms and
+  // clear the vms cache ONCE, then let it re-extract.
+  log('probe: booting a throwaway micro-VM to check runtime health ...');
+  let probe;
+  try {
+    probe = await Machine.create({
+      name: `probe-${process.pid}`,
+      resources: { cpus: 1, memoryMb: 512 },
+    });
+    const r = await probe.exec(['/bin/sh', '-c', 'echo healthy'], {
+      timeout: 60,
+    });
+    log(`probe exec -> "${(r.stdout || '').trim()}" (exit ${r.exitCode})`);
+    await probe.delete().catch(() => {});
+    return true;
+  } catch (e) {
+    log(
+      `probe FAILED (${String(e).slice(0, 120)}). Attempting one-time unwedge ...`
+    );
+    try {
+      if (probe) await probe.delete().catch(() => {});
+    } catch {}
+    try {
+      execSync(
+        "ps -axo pid,command | grep smol-vmm | grep _boot-vm | grep -v grep | awk '{print $1}' | xargs -r kill -9",
+        {
+          encoding: 'utf8',
+        }
+      );
+    } catch {}
+    try {
+      execSync('rm -rf ~/Library/Caches/smolvm/vms/*', {
+        encoding: 'utf8',
+        shell: '/bin/zsh',
+      });
+    } catch {}
+    log(
+      'unwedge done (killed stray boot-vms + cleared vms cache). Re-probing ...'
+    );
+    await sleep(2000);
+    try {
+      probe = await Machine.create({
+        name: `probe2-${process.pid}`,
+        resources: { cpus: 1, memoryMb: 512 },
+      });
+      const r = await probe.exec(['/bin/sh', '-c', 'echo healthy'], {
+        timeout: 90,
+      });
+      log(`re-probe exec -> "${(r.stdout || '').trim()}" (exit ${r.exitCode})`);
+      await probe.delete().catch(() => {});
+      return true;
+    } catch (e2) {
+      log(`re-probe STILL failing: ${String(e2).slice(0, 160)}`);
+      try {
+        if (probe) await probe.delete().catch(() => {});
+      } catch {}
+      return false;
+    }
+  }
+}
+
+// ----------------------------------------------------------------------------
+const created = new Set(); // track everything for finally cleanup
+let golden;
+const rows = [];
+
+const HOST = {
+  ncpu: sysctlStr('hw.ncpu'),
+  physicalcpu: sysctlStr('hw.physicalcpu'),
+  memsize: Number(sysctlStr('hw.memsize')) || 0,
+  hv_support: sysctlStr('kern.hv_support'),
+  brand: sysctlStr('machdep.cpu.brand_string'),
+};
+
+async function main() {
+  console.log('\n==================== HOST FACTS ====================');
+  console.log(`cpu.brand_string : ${HOST.brand}`);
+  console.log(
+    `hw.ncpu          : ${HOST.ncpu}  (physical ${HOST.physicalcpu})`
+  );
+  console.log(
+    `hw.memsize       : ${HOST.memsize} bytes (${GB(HOST.memsize)} GB)`
+  );
+  console.log(`kern.hv_support  : ${HOST.hv_support}`);
+  console.log(`smolmachines     : 1.3.2  (runtime ${rtDir})`);
+  console.log(
+    `fork resources   : cpus=${FORK_CPUS} memoryMb=${FORK_MEM_MB} per fork`
+  );
+  console.log('====================================================\n');
+
+  const healthy = await maybeUnwedge();
+  if (!healthy) {
+    console.error(
+      '[scale] runtime did not become healthy after one unwedge; aborting.'
+    );
+    process.exitCode = 1;
+    return;
+  }
+
+  // 1) Cold-boot ONE forkable golden + provision warm base ------------------
+  log('cold-booting ONE forkable golden (network:true) ...');
+  const tBoot = t();
+  golden = await withRetry('golden create', () =>
+    Machine.create({
+      name: `golden-${process.pid}`,
+      resources: { cpus: FORK_CPUS, memoryMb: FORK_MEM_MB, network: true },
+      forkable: true,
+    })
+  );
+  created.add(golden);
+  const goldenBootMs = t() - tBoot;
+  log(`golden booted forkable in ${goldenBootMs}ms`);
+
+  log('provisioning warm base (toolchain + small project + npm install) ...');
+  const tProv = t();
+  await sh(
+    golden,
+    'apk',
+    'apk add --no-cache ca-certificates nodejs npm git >/dev/null 2>&1 && update-ca-certificates >/dev/null 2>&1 || true',
+    600
+  );
+  await sh(
+    golden,
+    'project',
+    [
+      `mkdir -p ${PROJECT} && cd ${PROJECT}`,
+      'git init -q && git config user.email a@b.c && git config user.name base',
+      'printf "# scale base\\n" > README.md',
+      'npm init -y >/dev/null 2>&1',
+      'npm install --no-fund --no-audit is-odd >/dev/null 2>&1 || true',
+      'git add -A && git commit -q -m "pristine base"',
+    ].join(' && '),
+    600
+  );
+  const baseCommit = (
+    await sh(
+      golden,
+      'base commit',
+      `cd ${PROJECT} && git rev-parse --short HEAD`
+    )
+  ).out;
+  const baseNode = (await sh(golden, 'base node', 'node --version')).out;
+  const provMs = t() - tProv;
+  log(
+    `golden provisioned in ${provMs}ms (commit=${baseCommit}, node=${baseNode})`
+  );
+
+  // Baseline RAM with golden up, no forks.
+  const goldenVmm = vmmRss();
+  const goldenMem = hostMem();
+  log(
+    `baseline: vmm procs=${goldenVmm.count} sumRSS=${MB(goldenVmm.rssBytes)}MB  host avail=${GB(
+      goldenMem.availBytes
+    )}GB`
+  );
+
+  // NOTE: golden is exec-DEAD after the first fork. We never exec it again.
+  let goldenForked = false;
+
+  // 2) Ramp the stages -----------------------------------------------------
+  for (const N of STAGES) {
+    log(`\n---------- STAGE N=${N} ----------`);
+    const eagainBefore = EAGAIN_RETRIES;
+    const memBefore = hostMem();
+    const vmmBefore = vmmRss();
+    let status = 'ok';
+    let note = '';
+    const forks = [];
+    const forkLatencies = [];
+    let forkWall = 0;
+    let workMs = 0;
+
+    try {
+      // ---- fork wave (paced, EAGAIN-retried, concurrent-ish) ----
+      const tFork = t();
+      const forkPromises = [];
+      for (let i = 1; i <= N; i++) {
+        // small stagger so we don't slam the control plane (EAGAIN source)
+        await sleep(FORK_STAGGER_MS);
+        const name = `f-${N}-${i}-${process.pid}`;
+        forkPromises.push(
+          (async () => {
+            const tf = t();
+            const child = await withRetry(`fork ${name}`, () =>
+              golden.fork(name)
+            );
+            const lat = t() - tf;
+            goldenForked = true;
+            created.add(child);
+            forks.push(child);
+            forkLatencies.push(lat);
+            return child;
+          })()
+        );
+      }
+      await Promise.all(forkPromises);
+      forkWall = t() - tFork;
+      log(
+        `forked ${forks.length}/${N} in ${forkWall}ms  (p50=${percentile(
+          forkLatencies,
+          50
+        )}ms p95=${percentile(forkLatencies, 95)}ms)`
+      );
+
+      // ---- RAM after fork wave (settle a beat for COW pages to fault in) ----
+      await sleep(800);
+      const vmmAfter = vmmRss();
+      const memAfter = hostMem();
+
+      // ---- job in each fork, concurrently ----
+      // Each fork: light I/O (write+commit) + a CPU-saturating busy loop. The CPU
+      // loop is what exposes the core ceiling: per-job in-VM ms is captured so we
+      // can see queueing once N > cores.
+      const tWork = t();
+      const perJobMs = [];
+      const workResults = await Promise.allSettled(
+        forks.map((child, idx) =>
+          (async () => {
+            await sh(
+              child,
+              `io${idx}`,
+              [
+                `cd ${PROJECT}`,
+                `printf "fork ${idx} active\\n" > fork-${idx}.txt`,
+                `git add -A && git commit -q -m "fork ${idx}"`,
+              ].join(' && '),
+              180
+            );
+            const r = await sh(child, `cpu${idx}`, CPU_JOB, 300);
+            const ms = Number((r.out || '').split(/\s+/)[0]);
+            if (Number.isFinite(ms)) perJobMs.push(ms);
+          })()
+        )
+      );
+      workMs = t() - tWork;
+      const jobP50 = percentile(perJobMs, 50);
+      const jobP95 = percentile(perJobMs, 95);
+      const workOk = workResults.filter((r) => r.status === 'fulfilled').length;
+      const workFail = workResults.length - workOk;
+      if (workFail > 0) {
+        status = 'degraded';
+        note += `${workFail}/${forks.length} jobs failed; `;
+        const firstErr = workResults.find((r) => r.status === 'rejected');
+        if (firstErr && isInstability(firstErr.reason)) {
+          status = 'crash';
+          note += `instability: ${String(firstErr.reason).slice(0, 80)}; `;
+        }
+      }
+      log(`concurrent work: ${workOk}/${forks.length} ok in ${workMs}ms`);
+
+      // ---- COW / marginal RSS accounting ----
+      const deltaVmmRss = vmmAfter.rssBytes - vmmBefore.rssBytes;
+      const newVmms = vmmAfter.count - vmmBefore.count;
+      const marginalPerFork = newVmms > 0 ? deltaVmmRss / newVmms : 0;
+      const hostUsedDelta = memBefore.availBytes - memAfter.availBytes; // host RAM consumed by the wave
+      const marginalHostPerFork =
+        forks.length > 0 ? hostUsedDelta / forks.length : 0;
+
+      const eagainThisStage = EAGAIN_RETRIES - eagainBefore;
+
+      rows.push({
+        N,
+        forked: forks.length,
+        forkWall,
+        p50: percentile(forkLatencies, 50),
+        p95: percentile(forkLatencies, 95),
+        marginalRssMB: Number(MB(marginalPerFork)),
+        marginalHostMB: Number(MB(marginalHostPerFork)),
+        totalVmmRssGB: Number(GB(vmmAfter.rssBytes)),
+        vmmCount: vmmAfter.count,
+        hostAvailGB: Number(GB(memAfter.availBytes)),
+        eagain: eagainThisStage,
+        workMs,
+        jobP50,
+        jobP95,
+        workOk,
+        status,
+        note: note.trim(),
+      });
+
+      log(
+        `CPU job in-VM ms: p50=${jobP50} p95=${jobP95} (baseline ~single-fork time => >cores means queueing)`
+      );
+      log(
+        `RAM: vmm sumRSS ${MB(vmmBefore.rssBytes)}->${MB(vmmAfter.rssBytes)}MB (+${MB(
+          deltaVmmRss
+        )}MB for ${newVmms} new vmm) -> marginal ${MB(marginalPerFork)}MB/fork (proc) / ${MB(
+          marginalHostPerFork
+        )}MB/fork (host)`
+      );
+      log(
+        `host avail ${GB(memBefore.availBytes)}->${GB(memAfter.availBytes)}GB  EAGAIN(stage)=${eagainThisStage}  status=${status}`
+      );
+
+      // ---- tear down THIS stage's forks (keep golden) ----
+      log(`tearing down ${forks.length} forks ...`);
+      for (const child of forks) {
+        try {
+          await withRetry('delete', () => child.delete(), {
+            tries: 5,
+            base: 120,
+          });
+        } catch {
+          try {
+            await child.stop();
+          } catch {}
+        }
+        created.delete(child);
+      }
+      await sleep(1500); // let teardown settle before next stage measures baseline
+
+      // ---- stop conditions ----
+      if (status === 'crash') {
+        log(`STOP: native/connection instability at N=${N}.`);
+        break;
+      }
+      if (memAfter.availBytes / 1024 / 1024 / 1024 < MIN_FREE_HEADROOM_GB) {
+        log(
+          `STOP: host free RAM (${GB(memAfter.availBytes)}GB) dropped below ${MIN_FREE_HEADROOM_GB}GB headroom.`
+        );
+        break;
+      }
+    } catch (e) {
+      // A stage-level throw = fork wave or measurement failed hard.
+      const crash = isInstability(e);
+      status = crash ? 'crash' : 'degraded';
+      note += String(e?.stack || e).slice(0, 140);
+      const memAfter = hostMem();
+      rows.push({
+        N,
+        forked: forks.length,
+        forkWall,
+        p50: percentile(forkLatencies, 50),
+        p95: percentile(forkLatencies, 95),
+        marginalRssMB: NaN,
+        marginalHostMB: NaN,
+        totalVmmRssGB: Number(GB(vmmRss().rssBytes)),
+        vmmCount: vmmRss().count,
+        hostAvailGB: Number(GB(memAfter.availBytes)),
+        eagain: EAGAIN_RETRIES - eagainBefore,
+        workMs,
+        jobP50: 0,
+        jobP95: 0,
+        workOk: 0,
+        status,
+        note: note.trim(),
+      });
+      log(`STAGE N=${N} FAILED (${status}): ${note.slice(0, 140)}`);
+      // best-effort teardown of whatever forked
+      for (const child of forks) {
+        try {
+          await child.delete();
+        } catch {
+          try {
+            await child.stop();
+          } catch {}
+        }
+        created.delete(child);
+      }
+      break; // stop ramping past the failure ceiling
+    }
+  }
+
+  printReport(goldenBootMs, provMs, baseNode);
+}
+
+function printReport(goldenBootMs, provMs, baseNode) {
+  const pad = (s, n) => String(s).padStart(n);
+  console.log('\n\n==================== SCALE TABLE ====================');
+  console.log(
+    [
+      pad('N', 3),
+      pad('forkWall', 9),
+      pad('p50', 6),
+      pad('p95', 6),
+      pad('mRSS/fk', 8),
+      pad('totVmmGB', 9),
+      pad('hostFreeGB', 11),
+      pad('EAGAIN', 7),
+      pad('cpuJobp50', 10),
+      pad('cpuJobp95', 10),
+      pad('allMs', 7),
+      pad('status', 9),
+    ].join(' | ')
+  );
+  console.log('-'.repeat(110));
+  for (const r of rows) {
+    console.log(
+      [
+        pad(r.N, 3),
+        pad(`${r.forkWall}ms`, 9),
+        pad(`${r.p50}`, 6),
+        pad(`${r.p95}`, 6),
+        pad(`${r.marginalRssMB}MB`, 8),
+        pad(`${r.totalVmmRssGB}`, 9),
+        pad(`${r.hostAvailGB}`, 11),
+        pad(`${r.eagain}`, 7),
+        pad(`${r.jobP50}ms`, 10),
+        pad(`${r.jobP95}ms`, 10),
+        pad(`${r.workMs}`, 7),
+        pad(`${r.status}`, 9),
+      ].join(' | ')
+    );
+    if (r.note) console.log(`     note(N=${r.N}): ${r.note}`);
+  }
+  console.log('====================================================');
+
+  // ---- SUMMARY: binding constraint analysis ----
+  const okRows = rows.filter((r) => r.status === 'ok');
+  const maxStable = okRows.length ? Math.max(...okRows.map((r) => r.N)) : 0;
+  const crashRow = rows.find((r) => r.status === 'crash');
+  const degradedRow = rows.find((r) => r.status === 'degraded');
+
+  // COW marginal: average of valid marginal-RSS figures across stages.
+  const validMarg = rows
+    .map((r) => r.marginalRssMB)
+    .filter((x) => Number.isFinite(x) && x > 0);
+  const avgMarginalMB = validMarg.length
+    ? validMarg.reduce((a, b) => a + b, 0) / validMarg.length
+    : NaN;
+  const validHostMarg = rows
+    .map((r) => r.marginalHostMB)
+    .filter((x) => Number.isFinite(x) && x > 0);
+  const avgHostMarginalMB = validHostMarg.length
+    ? validHostMarg.reduce((a, b) => a + b, 0) / validHostMarg.length
+    : NaN;
+
+  const cores = Number(HOST.ncpu) || 0;
+  const totalRamGB = HOST.memsize / 1024 / 1024 / 1024;
+  // RAM-bound cap = (totalRAM - reserve(golden+host headroom)) / marginal-per-fork
+  const reserveGB = 16; // golden + macOS + headroom
+  const ramCap =
+    Number.isFinite(avgHostMarginalMB) && avgHostMarginalMB > 0
+      ? Math.floor(((totalRamGB - reserveGB) * 1024) / avgHostMarginalMB)
+      : NaN;
+  // CPU-bound cap for CPU-active forks ~= cores / FORK_CPUS (oversubscribe up to ~cores)
+  const cpuCapActive = Math.floor(cores / FORK_CPUS);
+  const cpuCapBurst = cores;
+
+  // Empirical CPU ceiling: per-job CPU time vs the N=1 baseline. When N exceeds
+  // available cores, jobs queue and per-job wall time inflates. Find the largest N
+  // where per-job p50 stayed within ~1.5x of baseline (the "no meaningful queueing"
+  // throughput frontier).
+  const baseJob = rows.length ? rows[0].jobP50 || 0 : 0;
+  let cpuFrontier = 0;
+  let cpuKneeN = 0;
+  for (const r of rows) {
+    if (!baseJob) break;
+    if ((r.jobP50 || 0) <= baseJob * 1.5) cpuFrontier = r.N;
+    else if (!cpuKneeN) cpuKneeN = r.N;
+  }
+
+  // Determine binding constraint.
+  let binding = 'undetermined';
+  if (crashRow)
+    binding = `native/control-plane stability (broke at N=${crashRow.N})`;
+  else if (degradedRow && degradedRow.eagain > 0)
+    binding = 'control-plane EAGAIN saturation';
+  else if (cpuKneeN && cpuKneeN <= maxStable)
+    binding = `CPU cores (per-job time knees up at N=${cpuKneeN}; throughput frontier N~${cpuFrontier} vs ${cores} cores)`;
+  else if (Number.isFinite(ramCap) && ramCap <= maxStable)
+    binding = 'RAM (memory headroom / marginal-RSS-per-fork)';
+  else
+    binding =
+      'none reached (control plane + CPU + RAM all had headroom through max N tested)';
+
+  console.log('\n==================== SUMMARY ====================');
+  console.log(
+    `host                       : ${HOST.brand}, ${cores} cores, ${GB(HOST.memsize)}GB RAM, HVF=${HOST.hv_support}`
+  );
+  console.log(
+    `golden cold-boot           : ${goldenBootMs}ms;  provision warm base: ${provMs}ms;  node in fork: ${baseNode}`
+  );
+  console.log(
+    `max STABLE concurrent N    : ${maxStable}${crashRow ? ` (broke/degraded at N=${crashRow.N})` : ''}`
+  );
+  console.log(
+    `COW marginal RAM / fork    : ~${Number.isFinite(avgMarginalMB) ? avgMarginalMB.toFixed(0) : '?'}MB (vmm proc RSS) / ~${
+      Number.isFinite(avgHostMarginalMB) ? avgHostMarginalMB.toFixed(0) : '?'
+    }MB (host) vs ${FORK_MEM_MB}MB full VM  => COW sharing ${
+      Number.isFinite(avgHostMarginalMB) && avgHostMarginalMB < FORK_MEM_MB
+        ? `${(100 - (avgHostMarginalMB / FORK_MEM_MB) * 100).toFixed(0)}% saved`
+        : 'NOT observed'
+    }`
+  );
+  console.log(`EAGAIN total (all stages)  : ${EAGAIN_RETRIES}`);
+  console.log(
+    `CPU per-job p50 (baseline ${baseJob}ms): ${rows.map((r) => `N${r.N}=${r.jobP50}ms`).join('  ')}`
+  );
+  console.log(
+    `CPU throughput frontier    : N~${cpuFrontier} within 1.5x baseline (knee at N=${cpuKneeN || 'none<=maxN'})`
+  );
+  console.log(`BINDING CONSTRAINT         : ${binding}`);
+  console.log(
+    `  - CPU-bound cap (active) : ~${cpuCapActive} forks (cores/${FORK_CPUS}); burst up to ~${cpuCapBurst}`
+  );
+  console.log(
+    `  - RAM-bound cap          : ~${Number.isFinite(ramCap) ? ramCap : '?'} forks ((${totalRamGB.toFixed(
+      0
+    )}GB - ${reserveGB}GB reserve) / ${Number.isFinite(avgHostMarginalMB) ? avgHostMarginalMB.toFixed(0) : '?'}MB)`
+  );
+  // Recommended cap for CPU-ACTIVE forks = empirical CPU throughput frontier
+  // (where per-job time stays ~baseline), bounded by stability + RAM.
+  const candidates = [
+    cpuFrontier || cpuCapBurst,
+    maxStable,
+    Number.isFinite(ramCap) ? ramCap : Infinity,
+  ].filter((x) => Number.isFinite(x) && x > 0);
+  const recommended = candidates.length ? Math.min(...candidates) : maxStable;
+  console.log(
+    `RECOMMENDED concurrent-ACTIVE-fork cap (THIS Mac): ~${recommended}`
+  );
+  console.log(
+    `  scaling rule: CPU-bound active work -> cap ~= cores (${cores}); long-lived idle forks -> cap = (RAM_headroom_GB*1024)/marginalMB (~${
+      Number.isFinite(ramCap) ? ramCap : '?'
+    }).`
+  );
+  console.log(
+    `smol 1.3.2 stability       : ${crashRow ? `BROKE at N=${crashRow.N} (${crashRow.note})` : `STABLE through N=${maxStable}`}`
+  );
+  console.log('================================================');
+}
+
+// ----------------------------------------------------------------------------
+main()
+  .catch((e) => {
+    console.error('[scale] FATAL:', e?.stack || String(e));
+    process.exitCode = 1;
+  })
+  .finally(async () => {
+    log('\nfinal cleanup: deleting all tracked machines (forks + golden) ...');
+    for (const m of [...created]) {
+      try {
+        await m.delete();
+      } catch {
+        try {
+          await m.stop();
+        } catch {}
+      }
+    }
+    // give the runtime a moment to reap boot-vm workers
+    await sleep(2500);
+    const leaked = countLeakedVmm();
+    console.log(
+      `[scale] leaked smol-vmm processes at exit: ${leaked} (expect 0)`
+    );
+    if (leaked > 0) {
+      console.log(
+        '[scale] WARNING: leaked vmm detected; listing (not killing golden-* per hygiene):'
+      );
+      try {
+        console.log(
+          execSync('ps -axo pid,command | grep smol-vmm | grep -v grep', {
+            encoding: 'utf8',
+          })
+        );
+      } catch {}
+    }
+  });