@codeyam-editor/codeyam-editor 0.1.5 → 0.1.7

package/npm/fleet-memory.js

@@ -20,38 +20,167 @@ const MEMORY_DEFAULTS = Object.freeze({
   drainCheckIntervalMs: 5000,
   snapshotIntervalMs: 30_000,
   snapshotMaxAgeMs: 5 * 60 * 1000,
+  // Fraction of the cgroup's own memory.high at which the drain engages. A
+  // FRACTION, not an absolute MB, because the absolute number silently
+  // desynchronizes from the unit file: codeyam-dashboard.service was raised
+  // from MemoryHigh=1.5G to 2.5G on 2026-07-28, while the operator host kept
+  // throttling at 1536MB — a hardcoded threshold cannot track that, a fraction
+  // of the value the kernel actually reports does. 0.80 of the 1536MB limit in
+  // force during the 2026-07-30 wedge is 1229MB, comfortably ahead of the
+  // 1309MB (85%) geometry at which the event loop had already stalled.
+  drainHighFraction: 0.80,
+  // Fraction of memory.high at which the drain is no longer enough and the
+  // process restarts itself to actually reclaim. Sits in a deliberately narrow
+  // band ABOVE the drain and BELOW the measured stall: 0.83 of 1536MB is
+  // 1275MB, ~34MB before the 1309MB geometry that wedged the dashboard. The
+  // band is tight because the evidence leaves little room — reclaim was already
+  // continuous at 85%.
+  restartHighFraction: 0.83,
+  // Crash-loop guard for the self-restart remedy. Counted over a rolling hour
+  // and persisted across restarts by the caller (an in-memory counter would
+  // reset on every restart and so could never limit anything).
+  maxRestartsPerHour: 3,
+  restartWindowMs: 60 * 60 * 1000,
 });
 
 /** Should the dashboard refuse to admit new heavy jobs?
  *
- *  Returns true when current RSS exceeds the soft drain threshold. A non-
- *  positive or non-finite threshold disables the drain (returns false) — a
- *  misconfigured env var degrades to "no drain" instead of silently rejecting
- *  every heavy job. Bytes on both sides so the caller doesn't have to pick a
- *  unit; server.js passes MB→bytes once at module init.
+ *  Decides from the CGROUP total when a cgroup reading is available, and only
+ *  falls back to this process's RSS when it is not.
  *
- *  Pure; no I/O. */
-function decideMemoryDrain({ rssBytes, drainThresholdBytes } = {}) {
-  const rss = Number(rssBytes);
+ *  Why the cgroup and not RSS: RSS is the wrong layer. The kernel throttles on
+ *  the cgroup's memory.current against memory.high, and the cgroup runs
+ *  100-230MB ABOVE this process's RSS (109MB fresh, 226MB wedged, measured
+ *  2026-07-30). Sampling RSS therefore under-reads the condition by design —
+ *  readCgroupMemory() in server.js has said so in a comment since improve37,
+ *  but the lesson was only ever applied to operatorFaultVerdict, never carried
+ *  back here. During the 2026-07-30 wedge RSS read 1083MB against a 1500MB
+ *  threshold while the cgroup sat at 1309MB of a 1536MB limit, already emitting
+ *  42546 reclaim events: the drain could not fire before the stall, and no
+ *  [memory] line was ever logged across the whole leak-to-wedge cycle. No RSS
+ *  number fixes that, because the RSS-to-cgroup gap is itself variable.
+ *
+ *  Contracts preserved exactly from the RSS-only version:
+ *    • strict greater-than at the threshold, so a reading pinned at the limit
+ *      cannot oscillate the drain on and off each tick;
+ *    • a non-positive or non-finite drainThresholdBytes disables the drain
+ *      entirely and returns false. That is the documented operator escape
+ *      hatch (MEMORY_DRAIN_THRESHOLD_MB=0) and it is checked FIRST, so it
+ *      disables the cgroup path too, not just the RSS one;
+ *    • any unusable input degrades to "no drain", never to "always drain" — a
+ *      dev laptop with no cgroup must not self-throttle.
+ *
+ *  memoryHigh of null/0 means an unlimited cgroup (memory.high == 'max') or no
+ *  cgroup at all (macOS, cgroup v1, a container without cgroup v2). Either way
+ *  it contributes no pressure signal and the RSS fallback decides.
+ *
+ *  Pure; no I/O — the caller supplies the numbers, as it always has. */
+function decideMemoryDrain({
+  rssBytes,
+  drainThresholdBytes,
+  memoryCurrent,
+  memoryHigh,
+  drainHighFraction,
+} = {}) {
   const cap = Number(drainThresholdBytes);
   if (!Number.isFinite(cap) || cap <= 0) return false;
+
+  // Preferred path: the quantity the kernel actually limits.
+  const current = Number(memoryCurrent);
+  const high = Number(memoryHigh);
+  const asked = Number(drainHighFraction);
+  const fraction = (Number.isFinite(asked) && asked > 0)
+    ? asked
+    : MEMORY_DEFAULTS.drainHighFraction;
+  if (Number.isFinite(current) && current >= 0 && Number.isFinite(high) && high > 0) {
+    return current > high * fraction;
+  }
+
+  // Documented fallback: no cgroup reading available, so keep the pre-existing
+  // RSS behavior rather than losing the drain entirely off-cgroup.
+  const rss = Number(rssBytes);
   if (!Number.isFinite(rss) || rss < 0) return false;
   return rss > cap;
 }
 
+/** Should the dashboard restart itself to reclaim memory?
+ *
+ *  The drain is an admission brake: it refuses NEW heavy jobs. On an idle-but-
+ *  leaking process it changes nothing — during the 2026-07-30 wedge no job had
+ *  run since Jul 28, so even a correctly-firing drain would not have helped.
+ *  Something has to actually reclaim, and a restart is cheap here: systemd's
+ *  Restart=always brings the process straight back and flushStateSnapshot has
+ *  already persisted {jobs, vms, orphans} within the last 30s for
+ *  restoreFromSnapshot to rehydrate.
+ *
+ *  Returns { restart, reason } rather than a bare boolean so the caller can log
+ *  WHY it declined — silence was the original failure mode, where the
+ *  operator's first signal was an apparently-dead fleet.
+ *
+ *  Refuses, in order:
+ *    • the drain escape hatch is set (drainThresholdBytes <= 0) — disabling the
+ *      drain disables the whole memory remedy, including this;
+ *    • no usable cgroup reading — an off-cgroup dev laptop never self-restarts;
+ *    • below the high-water fraction;
+ *    • a job is in flight — a restart mid-provision would strand it;
+ *    • the rolling-window restart budget is spent, which is the crash-loop guard.
+ *
+ *  restartsInWindow is supplied by the caller because it must be PERSISTED: an
+ *  in-memory counter resets on the very restart it is meant to be limiting.
+ *
+ *  Pure; no I/O. */
+function decideMemoryRestart({
+  memoryCurrent,
+  memoryHigh,
+  restartHighFraction,
+  drainThresholdBytes,
+  jobInFlight,
+  restartsInWindow,
+  maxRestartsPerWindow,
+} = {}) {
+  const cap = Number(drainThresholdBytes);
+  if (!Number.isFinite(cap) || cap <= 0) return { restart: false, reason: 'memory-remedy-disabled' };
+
+  const current = Number(memoryCurrent);
+  const high = Number(memoryHigh);
+  if (!Number.isFinite(current) || current < 0) return { restart: false, reason: 'no-cgroup-reading' };
+  if (!Number.isFinite(high) || high <= 0) return { restart: false, reason: 'no-cgroup-limit' };
+
+  const asked = Number(restartHighFraction);
+  const fraction = (Number.isFinite(asked) && asked > 0)
+    ? asked
+    : MEMORY_DEFAULTS.restartHighFraction;
+  if (!(current > high * fraction)) return { restart: false, reason: 'below-high-water' };
+
+  if (jobInFlight) return { restart: false, reason: 'job-in-flight' };
+
+  const askedMax = Number(maxRestartsPerWindow);
+  const max = Number.isFinite(askedMax) ? askedMax : MEMORY_DEFAULTS.maxRestartsPerHour;
+  if (max <= 0) return { restart: false, reason: 'self-restart-disabled' };
+  const used = Number(restartsInWindow);
+  if (Number.isFinite(used) && used >= max) return { restart: false, reason: 'rate-limited' };
+
+  return { restart: true, reason: 'over-high-water' };
+}
+
 /** Build the serialized state snapshot envelope.
  *
  *  Keeps only the fields the dashboard needs to recover after a launchd
  *  respawn: in-flight jobs, last VM poll merge, operator-visible orphans.
  *  Times come from the caller so the same input yields the same output
  *  (snapshot equality testable). Pure; no I/O. */
-function buildStateSnapshot({ jobs, vms, orphans, nowIso }) {
+function buildStateSnapshot({ jobs, vms, orphans, orphanedIngress, nowIso }) {
   return {
     version: 1,
     timestamp: nowIso,
     jobs: jobs || {},
     vms: vms || {},
     orphans: orphans || {},
+    // Fleet-level orphaned-LB-ingress attention items keyed by destroyed VM
+    // number (vm--destroy-fully-tears-down-lb-ingress). Persisted so a genuine
+    // orphan survives a dashboard restart — the VM is gone, so it can't be
+    // re-derived from live poll state the way jobs/vms are.
+    orphanedIngress: orphanedIngress || {},
   };
 }
 
@@ -145,12 +274,163 @@ function restoreFromSnapshot({ snapshot, maxAgeMs, nowIso, sidecarKeys } = {}) {
     jobs,
     vms: (snapshot.vms && typeof snapshot.vms === 'object') ? snapshot.vms : {},
     orphans: (snapshot.orphans && typeof snapshot.orphans === 'object') ? snapshot.orphans : {},
+    orphanedIngress: (snapshot.orphanedIngress && typeof snapshot.orphanedIngress === 'object')
+      ? snapshot.orphanedIngress : {},
   };
 }
 
+/** Render the three memory layers as one line.
+ *
+ *  Deliberately reports RSS, the cgroup total, AND the gap between them
+ *  together. Reading any one of those alone is how the drain sat on the wrong
+ *  layer for so long: RSS said 1083MB and looked healthy against a 1500MB
+ *  threshold while the cgroup was at 1309MB of 1536MB and already in continuous
+ *  reclaim. The gap is not noise, it is the diagnostic.
+ *
+ *  Degrades to 'cgroup=unavailable' rather than printing zeros off-cgroup, so a
+ *  laptop log cannot be misread as a cgroup at 0MB.
+ *
+ *  Pure; no I/O. */
+function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
+  const mb = (b) => (Number(b) / 1024 / 1024).toFixed(0);
+  const current = Number(memoryCurrent);
+  const high = Number(memoryHigh);
+  const parts = [`RSS=${Number.isFinite(Number(rssBytes)) ? mb(rssBytes) : '?'}MB`];
+  if (Number.isFinite(current) && current > 0) {
+    parts.push(`cgroup=${mb(current)}MB`);
+    if (Number.isFinite(high) && high > 0) {
+      parts.push(`high=${mb(high)}MB`);
+      parts.push(`used=${((current / high) * 100).toFixed(0)}%`);
+      if (Number.isFinite(Number(rssBytes))) parts.push(`gap=${mb(current - Number(rssBytes))}MB`);
+    } else {
+      parts.push('high=unlimited');
+    }
+  } else {
+    parts.push('cgroup=unavailable');
+  }
+  return parts.join(' ');
+}
+
+/** Is any job doing real work right now?
+ *
+ *  Gates the self-restart: reclaiming memory by exiting is cheap when the fleet
+ *  is quiet and expensive mid-provision, where it would strand the job.
+ *
+ *  `recovering` counts as in flight even though nothing is executing — the
+ *  startup reap has not yet resolved it against gcloud, and restarting again
+ *  would only defer it a second time. `queued` does NOT count: restoreFromSnapshot
+ *  drops queued jobs on restore anyway, and a queued job has done no work worth
+ *  protecting. The dashboard's own self-update counts, being a heavy job by any
+ *  other name.
+ *
+ *  Pure; no I/O — the caller passes state.jobs / state.dashboardUpdate. */
+function hasJobInFlight({ jobs, dashboardUpdate } = {}) {
+  const busy = (j) => !!j && (j.state === 'running' || j.state === 'recovering');
+  const anyJob = Object.values((jobs && typeof jobs === 'object') ? jobs : {}).some(busy);
+  return anyJob || !!(dashboardUpdate && dashboardUpdate.state === 'running');
+}
+
+/** Should this escalation level be announced to the journal?
+ *
+ *  Announce on a genuine level TRANSITION only. The check runs every 5s, so
+ *  announcing unconditionally would bury the journal, while announcing nothing
+ *  is what let the original leak run to a wedge with no warning — no [memory]
+ *  line was emitted even once across the whole leak-to-wedge cycle.
+ *
+ *  The quiet levels are the steady states that carry no news: healthy operation
+ *  (below-high-water) and the structural "this host has no cgroup / the remedy
+ *  is switched off" conditions, which would otherwise announce once per boot on
+ *  every dev laptop for no reason.
+ *
+ *  Pure; no I/O. */
+const QUIET_MEMORY_LEVELS = Object.freeze([
+  'below-high-water',
+  'no-cgroup-reading',
+  'no-cgroup-limit',
+  'memory-remedy-disabled',
+]);
+
+function shouldAnnounceMemoryLevel({ level, lastLevel } = {}) {
+  if (!level) return false;
+  if (level === lastLevel) return false;
+  return !QUIET_MEMORY_LEVELS.includes(level);
+}
+
+/** Resolve the directory holding THIS process's memory.current / memory.high,
+ *  or null when no cgroup-derived reading is available.
+ *
+ *  The cgroup-v2 hierarchy ROOT exposes no memory.current and no memory.high —
+ *  only a leaf cgroup does. Reading `/sys/fs/cgroup/memory.current` directly is
+ *  therefore ENOENT on every systemd host, and that is exactly what made the
+ *  2026-07-30 memory remedy inert: both reads failed, every cgroup number
+ *  degraded to null, decideMemoryRestart answered 'no-cgroup-limit' on every
+ *  tick forever, and the drain silently fell back to the RSS threshold the
+ *  incident had already proven cannot fire before the event loop stalls.
+ *
+ *  Takes the CONTENTS of /proc/self/cgroup rather than reading it, so the path
+ *  derivation — the part that was wrong — is unit-testable on a macOS dev box
+ *  with no cgroup filesystem at all. That untestability is why a root-vs-leaf
+ *  path error shipped green.
+ *
+ *  Returns null for every shape that has no leaf memory file to read:
+ *    • the unified line `0::/` — the process IS in the root cgroup, which
+ *      genuinely has no memory files; this must NOT look like a valid path;
+ *    • cgroup v1 (`N:controller:/path` lines, no `0::` entry);
+ *    • empty, unreadable, or malformed input (a non-Linux host has no
+ *      /proc/self/cgroup at all, and the caller passes the null through).
+ *
+ *  Pure; no I/O. */
+function resolveCgroupMemoryDir({ procSelfCgroup, root } = {}) {
+  if (typeof procSelfCgroup !== 'string') return null;
+  const base = (typeof root === 'string' && root)
+    ? root.replace(/\/+$/, '')
+    : '/sys/fs/cgroup';
+  for (const line of procSelfCgroup.split('\n')) {
+    const trimmed = line.trim();
+    // cgroup v2 unified hierarchy is the single `0::<path>` entry. v1 lines
+    // (`N:controller:/path`) never match, which is the intended null path.
+    if (!trimmed.startsWith('0::')) continue;
+    const rel = trimmed.slice(3);
+    if (!rel.startsWith('/')) return null;
+    const clean = rel.replace(/\/+$/, '');
+    if (!clean) return null;
+    return base + clean;
+  }
+  return null;
+}
+
+/** Keep only the restart timestamps still inside the rolling window.
+ *
+ *  Split out from decideMemoryRestart so the "how many restarts have I already
+ *  spent" bookkeeping is testable on its own, and so the caller can prune the
+ *  persisted file with the same rule the decision uses. Non-parseable entries
+ *  are dropped rather than throwing — a hand-edited or partially-written file
+ *  degrades to "fewer restarts counted", which fails toward allowing a restart
+ *  that reclaims memory rather than wedging.
+ *
+ *  Pure; no I/O. */
+function pruneRestartWindow({ timestamps, nowIso, windowMs } = {}) {
+  const now = Date.parse(nowIso);
+  const span = Number(windowMs);
+  if (!Number.isFinite(now)) return [];
+  const cap = (Number.isFinite(span) && span > 0) ? span : MEMORY_DEFAULTS.restartWindowMs;
+  return (Array.isArray(timestamps) ? timestamps : [])
+    .filter((t) => {
+      const ms = Date.parse(t);
+      return Number.isFinite(ms) && now - ms <= cap && ms <= now;
+    });
+}
+
 module.exports = {
   MEMORY_DEFAULTS,
+  QUIET_MEMORY_LEVELS,
   decideMemoryDrain,
+  decideMemoryRestart,
+  formatMemoryText,
+  hasJobInFlight,
+  shouldAnnounceMemoryLevel,
+  resolveCgroupMemoryDir,
+  pruneRestartWindow,
   buildStateSnapshot,
   decodeStateSnapshot,
   restoreFromSnapshot,

package/npm/fleet-plans.js

@@ -1,7 +1,16 @@
 'use strict';
 
 /**
- * Indexes the currently-reachable VMs by their live sessionId.
+ * Indexes the reachable VMs by every session id that identifies them: their
+ * live `sessionId` (from /api/session-info) AND their workflow-lock
+ * `holderSessionId` (from /api/step's editor-step.json). A claim maps to a VM
+ * through this index, so including the holder is what lets a claim attribute to
+ * the executing VM even after an editor restart drops the live sessionId (the
+ * VM reports sessionId:null but editor-step.json still names the holder).
+ *
+ * A live `sessionId` always wins a key collision over a `holderSessionId`: the
+ * live registration is the authoritative current identity, so it is indexed in
+ * a first pass and the holder pass only fills in ids not already claimed.
  * @param {Array|Object} vms The merged VM states on the dashboard.
  * @returns {Object} A map of sessionId -> VM number (n).
  */
@@ -14,6 +23,39 @@ function indexVmsBySessionId(vms) {
       map[vm.sessionId] = vm.n;
     }
   }
+  // Second pass: holderSessionId fills in only ids a live sessionId didn't
+  // already claim, so a live registration never loses to a stale holder.
+  for (const vm of list) {
+    if (vm && vm.holderSessionId && vm.n !== undefined && map[vm.holderSessionId] === undefined) {
+      map[vm.holderSessionId] = vm.n;
+    }
+  }
+  return map;
+}
+
+/**
+ * Indexes the reachable VMs by their fleet machine name, so a claim that only
+ * carries `claimedBy.machine` (and no resolvable sessionId) can still attribute
+ * to a VM. A VM's machine name is `<instancePrefix><n>` (the GCE instance name
+ * the editor stamps into CODEYAM_MACHINE), so the map is derived, not probed.
+ *
+ * This is the fallback for backend/HTTP-"Run" claims: the editor stamps
+ * `claimedBy.sessionId` only for a PTY-attached `plan-select`, so a
+ * server-driven autonomous claim has an empty sessionId and never resolves via
+ * {@link indexVmsBySessionId}. Its machine name always resolves.
+ * @param {Array|Object} vms The merged VM states on the dashboard.
+ * @param {string} instancePrefix The fleet instance-name prefix (INSTANCE_PREFIX).
+ * @returns {Object} A map of machine name -> VM number (n).
+ */
+function indexVmsByMachine(vms, instancePrefix) {
+  const map = {};
+  if (!vms || !instancePrefix) return map;
+  const list = Array.isArray(vms) ? vms : Object.values(vms);
+  for (const vm of list) {
+    if (vm && vm.n !== undefined) {
+      map[`${instancePrefix}${vm.n}`] = vm.n;
+    }
+  }
   return map;
 }
 
@@ -36,9 +78,11 @@ function unsatisfiedDeps(dependsOn, completedSlugs) {
  * @param {Object[]} params.claims Array of PlanClaim objects from /api/plan-claims.
  * @param {Set|string[]} params.completedSlugs Completed plan slugs.
  * @param {Object} params.vmsBySessionId Map of sessionId -> VM number.
+ * @param {Object} [params.vmsByMachine] Map of machine name -> VM number, the
+ *   fallback for backend claims whose sessionId is empty.
  * @returns {Object[]} Annotated plan rows.
  */
-function resolvePlanRows({ plans, claims, completedSlugs, vmsBySessionId }) {
+function resolvePlanRows({ plans, claims, completedSlugs, vmsBySessionId, vmsByMachine }) {
   const list = Array.isArray(plans) ? plans : [];
   const claimsList = Array.isArray(claims) ? claims : [];
 
@@ -60,8 +104,14 @@ function resolvePlanRows({ plans, claims, completedSlugs, vmsBySessionId }) {
     if (claim) {
       const claimedBy = claim.claimedBy || claim.claimed_by || {};
       const sessionId = claimedBy.sessionId || claimedBy.session_id || null;
-      const vm = (sessionId && vmsBySessionId && vmsBySessionId[sessionId] !== undefined) ? vmsBySessionId[sessionId] : null;
-      
+      // Prefer the authoritative live sessionId match; fall back to the claim's
+      // machine name (backend claims have an empty sessionId, so this is the
+      // only key that resolves them to a VM).
+      let vm = (sessionId && vmsBySessionId && vmsBySessionId[sessionId] !== undefined) ? vmsBySessionId[sessionId] : null;
+      if (vm === null && claimedBy.machine && vmsByMachine && vmsByMachine[claimedBy.machine] !== undefined) {
+        vm = vmsByMachine[claimedBy.machine];
+      }
+
       claimInfo = {
         vm,
         machine: claimedBy.machine || null,
@@ -218,8 +268,52 @@ function parsePlansSnapshotStdout(stdout) {
   };
 }
 
+/**
+ * Unions the `/api/plan-claims` responses of several VMs in a group into one
+ * claim list, deduped by `(slug, uuid)`.
+ *
+ * Sourcing a group's claims from a single representative VM is fragile: that VM
+ * may lag, be unreachable, or — the shape this fixes — be the very VM holding
+ * the claim, whose own display filter is the one place the claim can be
+ * suppressed. Every *other* VM in the group carries the same ref as a plain
+ * `remote` claim, so unioning surfaces it regardless.
+ *
+ * When the same `(slug, uuid)` arrives from several VMs, the most-informed copy
+ * wins: a same-machine (`local`) view beats a `remote` one (only the holder
+ * knows its own session liveness), and among equals a live claim beats a
+ * not-live one. `fromVm` is stamped from the VM that supplied the winner so the
+ * panel's provenance stays meaningful.
+ * @param {Array<{vm: number, claims: Object[]}>} claimLists Per-VM claim responses.
+ * @returns {Object[]} The deduped union, each claim carrying its winning `fromVm`.
+ */
+function unionClaims(claimLists) {
+  const lists = Array.isArray(claimLists) ? claimLists : [];
+  // Rank a claim by how authoritative its view is. Higher wins a tie on key.
+  const rank = claim => {
+    const local = claim && claim.source === 'local' ? 2 : 0;
+    const alive = claim && claim.sessionAlive === true ? 1 : 0;
+    return local + alive;
+  };
+  const byKey = new Map();
+  for (const entry of lists) {
+    if (!entry || !Array.isArray(entry.claims)) continue;
+    for (const claim of entry.claims) {
+      if (!claim || !claim.slug) continue;
+      const key = `${claim.slug}${claim.uuid || ''}`;
+      const candidate = { ...claim, fromVm: entry.vm };
+      const held = byKey.get(key);
+      if (!held || rank(candidate) > rank(held)) {
+        byKey.set(key, candidate);
+      }
+    }
+  }
+  return Array.from(byKey.values());
+}
+
 module.exports = {
   indexVmsBySessionId,
+  indexVmsByMachine,
+  unionClaims,
   unsatisfiedDeps,
   resolvePlanRows,
   planSourceForProject,

package/npm/fleet-pool.js

@@ -22,8 +22,11 @@
 //
 // On-disk shape (tolerant: a missing or malformed file is the empty pool):
 //   { "vms": { "<n>": { status, branch, freshAt, claimedAt } } }
-//     status:    "pool"      idle, claim-ready
-//                "assigned"  claimed by an operator, rebranded to `branch`
+//     status:    "pool"          idle, claim-ready (authenticated + push-ready)
+//                "pending-auth"   reachable + LB-wired but awaiting operator auth;
+//                                 counts toward the provisioning target (so the
+//                                 keeper converges) but is NOT claimable
+//                "assigned"       claimed by an operator, rebranded to `branch`
 //     branch:    the branch the VM is on (base branch while pooled)
 //     freshAt:   ISO timestamp of the last successful auth/health refresh —
 //                drives evict-before-claim of a member whose token lapsed
@@ -45,6 +48,20 @@ const POOL_STATE_FILE = process.env.POOL_STATE_FILE
 
 const POOL = 'pool';
 const ASSIGNED = 'assigned';
+// A VM that passed every readiness item EXCEPT provider auth (reachable,
+// on-branch, broker up, editor+preview 200, git-push-ready). Provisioning has
+// converged — the VM exists and is LB-wired — so it counts toward the refill
+// target and the keeper stops churning; but it is never claimable, so the claim
+// path still hands out only fully-authenticated VMs (the 2026-06-29 guarantee).
+const PENDING_AUTH = 'pending-auth';
+
+// Canonical status coercion: an unrecognized status is not a pool member.
+function coerceStatus(status) {
+  if (status === ASSIGNED) return ASSIGNED;
+  if (status === PENDING_AUTH) return PENDING_AUTH;
+  if (status === POOL) return POOL;
+  return null;
+}
 
 // ── Pure state helpers ──────────────────────────────────────────────────────
 
@@ -61,7 +78,7 @@ function normalizeState(raw) {
     const n = Number(key);
     if (!Number.isInteger(n) || n <= 0) continue;
     const e = vms[key] || {};
-    const status = e.status === ASSIGNED ? ASSIGNED : e.status === POOL ? POOL : null;
+    const status = coerceStatus(e.status);
     if (!status) continue;
     out.vms[n] = {
       status,
@@ -91,6 +108,13 @@ function assignedMembers(state) {
   return membersByStatus(state, ASSIGNED);
 }
 
+// VMs provisioned + LB-wired but awaiting operator auth. They count toward the
+// refill target (provisioning has converged) yet are excluded from poolMembers,
+// so selectForClaim never hands one out.
+function pendingAuthMembers(state) {
+  return membersByStatus(state, PENDING_AUTH);
+}
+
 // True when a pool entry's auth/health is older than `ttlMs` (its token may
 // have lapsed). An entry with no `freshAt` is treated as stale — "freshness
 // not proven" fails closed, mirroring fleet-ready's auth gate. Only meaningful
@@ -109,12 +133,16 @@ function staleMembers(state, nowMs, ttlMs) {
   return poolMembers(state).filter((n) => isStale(state.vms[n], nowMs, ttlMs));
 }
 
-// How many NEW pool VMs the keeper must provision to reach `target` idle
-// members. Assigned VMs do not count toward the pool — once claimed they're
-// gone from the warm set until released — so a claim naturally triggers a
-// refill. Never negative (an over-target pool is shrunk by policy, not here).
+// How many NEW VMs the keeper must provision to reach `target`. `have` counts
+// BOTH claim-ready pool members AND pending-auth members: provisioning has
+// converged once N reachable+wired VMs exist, regardless of auth, so the deficit
+// closes and the keeper stops churning (the 2026-07-09 endless-refill fix). Only
+// the provisioning denominator widens — poolMembers (claim eligibility) is
+// untouched, so a claim still selects only fully-authenticated VMs. Assigned VMs
+// do not count (claimed → gone from the warm set until released). Never negative
+// (an over-target pool is shrunk by policy, not here).
 function refillCount(state, target) {
-  const have = poolMembers(state).length;
+  const have = poolMembers(state).length + pendingAuthMembers(state).length;
   return Math.max(0, target - have);
 }
 
@@ -141,16 +169,18 @@ function selectForClaim(state, count, branch, nowIso) {
 }
 
 // Register (or overwrite) a VM in the pool state. The keeper calls this after
-// provisioning a fresh READY VM (status=pool) and the claim path can re-add an
-// assigned VM. `freshAt` defaults to now for a pool member so a just-verified
-// VM isn't immediately stale.
+// provisioning: status=pool for a fully-READY VM, status=pending-auth for one
+// that is reachable+wired but still awaiting operator auth; the claim path can
+// re-add an assigned VM. `freshAt` is stamped to now for a pool OR pending-auth
+// member (both were just verified live) so neither is immediately stale; an
+// unknown status coerces to pool.
 function applyAdd(state, n, status, branch, nowIso) {
   const next = normalizeState(state);
-  const st = status === ASSIGNED ? ASSIGNED : POOL;
+  const st = coerceStatus(status) || POOL;
   next.vms[n] = {
     status: st,
     branch: branch || null,
-    freshAt: st === POOL ? nowIso : (next.vms[n] && next.vms[n].freshAt) || null,
+    freshAt: st === ASSIGNED ? (next.vms[n] && next.vms[n].freshAt) || null : nowIso,
     claimedAt: st === ASSIGNED ? nowIso : null,
   };
   return next;
@@ -188,6 +218,18 @@ function applyMarkFresh(state, n, nowIso) {
   return next;
 }
 
+// Promote a pending-auth VM to a claim-ready pool member — the dashboard calls
+// this the instant an operator completes that VM's OAuth, so it becomes
+// claimable with NO re-provision. `freshAt` resets to now (it was just proven
+// authenticated). A no-op unless the VM is currently pending-auth, so it is safe
+// to call for any authenticated VM (an already-pooled or absent VM is untouched).
+function applyPromote(state, n, nowIso) {
+  const next = normalizeState(state);
+  if (!next.vms[n] || next.vms[n].status !== PENDING_AUTH) return next;
+  next.vms[n] = { ...next.vms[n], status: POOL, freshAt: nowIso, claimedAt: null };
+  return next;
+}
+
 // ── Atomic file layer ───────────────────────────────────────────────────────
 
 // Synchronous millisecond sleep without a busy CPU spin — used by withLock's
@@ -260,9 +302,11 @@ module.exports = {
   POOL_STATE_FILE,
   POOL,
   ASSIGNED,
+  PENDING_AUTH,
   normalizeState,
   poolMembers,
   assignedMembers,
+  pendingAuthMembers,
   isStale,
   staleMembers,
   refillCount,
@@ -271,6 +315,7 @@ module.exports = {
   applyRelease,
   applyEvict,
   applyMarkFresh,
+  applyPromote,
   withLock,
   readState,
   writeState,
@@ -312,6 +357,9 @@ function main(argv) {
     case 'assigned-members':
       process.stdout.write(`${assignedMembers(readState()).join(' ')}\n`);
       return 0;
+    case 'pending-auth-members':
+      process.stdout.write(`${pendingAuthMembers(readState()).join(' ')}\n`);
+      return 0;
     case 'refill-count': {
       const target = Number(flags.target);
       if (!Number.isInteger(target) || target < 0) {
@@ -357,8 +405,10 @@ function main(argv) {
         process.stderr.write('fleet-pool: add needs --vm <pos int>\n');
         return 2;
       }
-      const status = flags.status === ASSIGNED ? ASSIGNED : POOL;
-      withLock(() => writeState(applyAdd(readState(), vm, status, flags.branch, now)));
+      // applyAdd coerces the status (pool | pending-auth | assigned; unknown →
+      // pool), so pass the raw flag through — this is how the keeper adds a
+      // reachable-but-unauthed VM with `--status pending-auth`.
+      withLock(() => writeState(applyAdd(readState(), vm, flags.status, flags.branch, now)));
       return 0;
     }
     case 'release': {
@@ -388,9 +438,18 @@ function main(argv) {
       withLock(() => writeState(applyMarkFresh(readState(), vm, now)));
       return 0;
     }
+    case 'promote': {
+      const vm = Number(flags.vm);
+      if (!Number.isInteger(vm) || vm <= 0) {
+        process.stderr.write('fleet-pool: promote needs --vm <pos int>\n');
+        return 2;
+      }
+      withLock(() => writeState(applyPromote(readState(), vm, now)));
+      return 0;
+    }
     default:
       process.stderr.write(
-        'fleet-pool: usage: <list|pool-members|assigned-members|refill-count|stale|claim|add|release|evict|mark-fresh> [--flags]\n',
+        'fleet-pool: usage: <list|pool-members|assigned-members|pending-auth-members|refill-count|stale|claim|add|release|evict|mark-fresh|promote> [--flags]\n',
       );
       return 2;
   }