agent-relay-orchestrator 0.129.11 → 0.129.13

package/src/host-admission.ts

@@ -0,0 +1,426 @@
+import os from "node:os";
+import { readFileSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import type { QueuedSpawn } from "agent-relay-sdk";
+
+// #893 — memory/load-aware spawn admission. The #930/#880 spawnSlots cap bounds how many
+// spawns can be MID-REGISTRATION at once (a port-collision fix); it does nothing about the
+// STEADY-STATE count of already-registered running agents. The 2026-06-30 incident hung the
+// relay by accumulating 24 running agents until the host swapped to 96% and load average hit
+// 43 — no gate anywhere rejected or queued spawns past what the host could actually hold. This
+// is the per-host resource check that closes that gap: a cheap, local memory/`loadavg()`
+// read the orchestrator (which already has host access) can do before every spawn, no relay/
+// protocol change required.
+//
+// #1516 — the gate must reflect *usable* memory, not raw `os.freemem()`. On darwin macOS keeps
+// raw "free" low by holding pages as reclaimable file cache, so raw free under-reports true
+// headroom. During the 2026-07-18 macmini incident raw free dipped to 146–217MB and the gate
+// rejected a spawn at 185MB while GBs were idle-and-reclaimable and memory_pressure reported
+// "normal". On linux we derive MemAvailable from /proc/meminfo and gate on that.
+//
+// #1516-r3 — on darwin, STOP deriving a usable-bytes number by summing `vm_stat` page classes.
+// Two prior rounds (r1: "Pages inactive"; r2: "File-backed pages") both over-ADMITTED (OOM risk):
+// every "reclaimable" page class secretly contains dirty content (dirty anonymous pages must swap;
+// XNU counts the FULL external pageable population — including dirty/writeback external pages — in
+// the file-backed figure). No page-class sum is a sound "immediately usable" number.
+//
+// Instead we gate darwin on the KERNEL'S OWN memory-pressure signal, which already accounts for
+// dirty/reclaimable/compressed memory internally:
+//   * `sysctl -n kern.memorystatus_vm_pressure_level` → 1 = NORMAL, 2 = WARN, 4 = CRITICAL (the
+//     dispatch_source_memorypressure flag values; XNU raises WARN/CRITICAL precisely when it can no
+//     longer reclaim enough — including the dirty-writeback case both prior rounds mis-admitted).
+//   * NORMAL  → the kernel reports headroom → report usable at physical RAM so the memory gate
+//     passes (this is what fixes macmini's false-low flicker: normal pressure despite low raw-free).
+//   * WARN / CRITICAL / any non-normal level → HOLD: no reclaimable boost, fall back to raw
+//     `os.freemem()` so a genuinely tight host still blocks. Never harder-closed than raw-free.
+//   * usable is sanity-capped at os.totalmem() and floored at a SINGLE os.freemem() sample threaded
+//     through the whole computation, so `freeMemMb <= usableMemMb <= totalMemMb` holds deterministically.
+//   * The sysctl probe uses a short timeout and a short-TTL cache (keyed on a MONOTONIC clock, so a
+//     wall-clock rollback can't keep a stale high probe alive) so a synchronous execFileSync can't
+//     serialize into multi-second event-loop stalls across admission polls.
+// The probe stays best-effort: any failure (sysctl missing, timeout, unparseable, unknown platform)
+// falls back to os.freemem() (fail-open) — the gate is never harder-closed than raw-free today.
+
+const DEFAULT_MIN_FREE_MEM_MB = 512;
+const DEFAULT_MAX_LOAD_PER_CPU = 4;
+const DEFAULT_ADMISSION_MAX_WAIT_MS = 60_000;
+const DEFAULT_ADMISSION_POLL_MS = 2_000;
+
+// #1516-r2 — execFileSync(sysctl) synchronously blocks the event loop. Keep the per-probe timeout
+// short and cache the result briefly so admission polling (every ~2s) doesn't spawn the probe on
+// every iteration and can't serialize failing probes into multi-second stalls.
+const PROBE_TIMEOUT_MS = 400;
+const PROBE_CACHE_TTL_MS = 3_000;
+
+// #1516-r3 — darwin `kern.memorystatus_vm_pressure_level` values. These are the XNU
+// dispatch_source_memorypressure flag values (NOTE_MEMORYSTATUS_PRESSURE_{NORMAL,WARN,CRITICAL}).
+export const VM_PRESSURE_NORMAL = 1;
+export const VM_PRESSURE_WARN = 2;
+export const VM_PRESSURE_CRITICAL = 4;
+
+// #1516-r3 — the byte value the darwin NORMAL-pressure branch reports: "the kernel says there is
+// headroom, impose no memory-class ceiling." readUsableMemMb clamps this down to the SINGLE
+// os.totalmem() sample threaded through checkHostHeadroom, so it resolves to exactly physical RAM
+// without the probe taking a second totalmem sample (preserves the free<=usable<=total invariant).
+const PRESSURE_HEADROOM_SENTINEL_BYTES = Number.MAX_SAFE_INTEGER;
+
+// #1513 — stable marker prefixed onto the rejection reason when a spawn is held back for the full
+// admission wait ceiling and then fails. Lets the spawner (and the failed-command notification)
+// distinguish a genuinely-can't-be-admitted queue timeout from other spawn failures.
+export const HOST_HEADROOM_TIMEOUT_REASON = "host-headroom-timeout";
+
+export interface HostHeadroomResult {
+  ok: boolean;
+  /** Present when ok is false — a human-readable reason suitable for a failed-command message. */
+  reason?: string;
+  /**
+   * #1516 — usable memory the gate actually decides on. On darwin (#1516-r3) this is driven by the
+   * kernel's own VM pressure level: NORMAL reports physical RAM (headroom), WARN/CRITICAL falls back
+   * to raw free (hold); on linux it is MemAvailable. Floored at raw `os.freemem()` and capped at
+   * `os.totalmem()`, so `freeMemMb <= usableMemMb <= totalMemMb` always holds.
+   */
+  usableMemMb: number;
+  /** Raw `os.freemem()`, retained for observability alongside the usable figure the gate uses. */
+  freeMemMb: number;
+  totalMemMb: number;
+  loadAvgPerCpu: number;
+}
+
+export function resolveMinFreeMemMb(env: NodeJS.ProcessEnv = process.env): number {
+  const raw = env.AGENT_RELAY_SPAWN_MIN_FREE_MEM_MB;
+  if (raw === undefined || raw.trim() === "") return DEFAULT_MIN_FREE_MEM_MB;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_MIN_FREE_MEM_MB;
+  return parsed;
+}
+
+export function resolveMaxLoadPerCpu(env: NodeJS.ProcessEnv = process.env): number {
+  const raw = env.AGENT_RELAY_SPAWN_MAX_LOAD_PER_CPU;
+  if (raw === undefined || raw.trim() === "") return DEFAULT_MAX_LOAD_PER_CPU;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_MAX_LOAD_PER_CPU;
+  return parsed;
+}
+
+export function resolveAdmissionMaxWaitMs(env: NodeJS.ProcessEnv = process.env): number {
+  const raw = env.AGENT_RELAY_SPAWN_ADMISSION_MAX_WAIT_MS;
+  if (raw === undefined || raw.trim() === "") return DEFAULT_ADMISSION_MAX_WAIT_MS;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed < 0) return DEFAULT_ADMISSION_MAX_WAIT_MS;
+  return parsed;
+}
+
+export function resolveAdmissionPollMs(env: NodeJS.ProcessEnv = process.env): number {
+  const raw = env.AGENT_RELAY_SPAWN_ADMISSION_POLL_MS;
+  if (raw === undefined || raw.trim() === "") return DEFAULT_ADMISSION_POLL_MS;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_ADMISSION_POLL_MS;
+  return parsed;
+}
+
+export function isHostAdmissionDisabled(env: NodeJS.ProcessEnv = process.env): boolean {
+  const raw = env.AGENT_RELAY_SPAWN_ADMISSION_DISABLED;
+  return raw === "1" || raw?.toLowerCase() === "true";
+}
+
+export interface WaitForHostHeadroomOptions {
+  disabled: boolean;
+  maxWaitMs: number;
+  pollMs: number;
+  check: () => HostHeadroomResult;
+  /** Called once per failed poll with a human-readable "still queued" message. */
+  log?: (message: string) => void;
+  sleep?: (ms: number) => Promise<void>;
+  // #1513 — called on each poll while the spawn is held back for headroom, with the current headroom
+  // snapshot (its reason/freeMem update as the host recovers). Lets the caller surface a "queued"
+  // dashboard row and notify the spawner that the spawn is queued rather than lost.
+  onQueued?: (result: HostHeadroomResult) => void;
+  // #1513 — called exactly once when a wait that ever queued ends (admitted OR timed out), so the
+  // caller can clear the queued row it published via onQueued. Not called for an immediate admit.
+  onSettled?: () => void;
+}
+
+/**
+ * Poll host headroom until it's admissible or `maxWaitMs` elapses. A spawn racing an agent
+ * that's about to exit gets admitted once headroom frees; only past the deadline does this
+ * throw — a clear, actionable rejection rather than silently oversubscribing the host.
+ *
+ * #1513 — while it holds a spawn back it drives onQueued/onSettled so the queued state is
+ * observable (dashboard row + spawner notification) instead of a silent wait, and the timeout
+ * rejection is tagged with HOST_HEADROOM_TIMEOUT_REASON so a can't-ever-admit queue is unambiguous.
+ */
+export async function waitForHostHeadroom(opts: WaitForHostHeadroomOptions): Promise<void> {
+  if (opts.disabled) return;
+  const sleep = opts.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  const deadline = Date.now() + opts.maxWaitMs;
+  let queued = false;
+  try {
+    for (;;) {
+      const result = opts.check();
+      if (result.ok) return;
+      if (Date.now() >= deadline) {
+        throw new Error(`spawn rejected: ${HOST_HEADROOM_TIMEOUT_REASON} — insufficient host headroom after waiting ${opts.maxWaitMs}ms (${result.reason})`);
+      }
+      queued = true;
+      opts.onQueued?.(result);
+      opts.log?.(`spawn queued: waiting for host headroom (${result.reason})`);
+      await sleep(Math.min(opts.pollMs, Math.max(0, deadline - Date.now())));
+    }
+  } finally {
+    // Only fire onSettled when we actually published a queued row (queued===true); an immediate
+    // admit never surfaced one. Runs on both the admit (return) and timeout (throw) paths.
+    if (queued) opts.onSettled?.();
+  }
+}
+
+/**
+ * #1516-r3/r4/r5 — parse `sysctl -n kern.memorystatus_vm_pressure_level` into the kernel's VM
+ * pressure level (1 = NORMAL, 2 = WARN, 4 = CRITICAL). `-n` prints just the integer, e.g. "1\n";
+ * this also tolerates the `name: value` form (`sysctl` without `-n`), but ONLY when `name` is the
+ * exact literal key `kern.memorystatus_vm_pressure_level` — e.g. "kern.memorystatus_vm_pressure_level: 1".
+ * Returns null on anything else — including empty/whitespace-only output, a leading sign or decimal,
+ * extra tokens/words, embedded diagnostics, multiple numbers, a `name: value` form with any other
+ * name, or an integer that isn't a known pressure level — so the caller falls back to
+ * os.freemem() (HOLD, never a false NORMAL).
+ *
+ * #1516-r4 — this MUST be strict. The r3 parser took the LAST `/\d+/` token in the output, which
+ * stripped signs and picked digits out of garbage: `"-1\n"` -> 1, `"warning: -1\n"` -> 1,
+ * `"2 warning 1\n"` -> 1. Any of those misread as NORMAL inflates usable memory to physical RAM
+ * under real memory pressure (over-open/OOM) — ambiguous/unexpected probe output must always HOLD.
+ *
+ * #1516-r5 — the r4 `name: value` branch still accepted ANY identifier before the colon
+ * (`/^[A-Za-z0-9_.]+:\s*(\d+)$/`), so diagnostic/garbage output like `"warning: 1"` or `"foo: 1"`
+ * parsed as pressure level 1 (NORMAL) even though no real sysctl emits those keys. The name must
+ * match the literal sysctl key exactly — anything else is unrecognized output and must HOLD.
+ *
+ * We gate on this signal — not a sum of `vm_stat` page classes — because every "reclaimable" page
+ * class (inactive, file-backed) secretly holds dirty content that isn't immediately usable, so any
+ * such sum over-admits (the r1 and r2 OOM bugs). The kernel's pressure level already accounts for
+ * dirty/reclaimable/compressed memory internally and raises WARN/CRITICAL when it genuinely can't
+ * reclaim — including the dirty-writeback-to-slow-storage case both prior rounds mis-admitted.
+ */
+export function parseVmPressureLevel(output: string): number | null {
+  if (!output) return null;
+  const trimmed = output.trim();
+  if (trimmed === "") return null;
+
+  let valueStr: string;
+  if (/^\d+$/.test(trimmed)) {
+    valueStr = trimmed;
+  } else {
+    const match = trimmed.match(/^kern\.memorystatus_vm_pressure_level:\s*(\d+)$/);
+    if (!match) return null;
+    valueStr = match[1]!;
+  }
+
+  const level = Number(valueStr);
+  if (!Number.isSafeInteger(level)) return null;
+  if (level !== VM_PRESSURE_NORMAL && level !== VM_PRESSURE_WARN && level !== VM_PRESSURE_CRITICAL) {
+    return null;
+  }
+  return level;
+}
+
+/**
+ * #1516 — parse `/proc/meminfo` on linux for `MemAvailable` (kernel's own estimate of memory
+ * available for new work without swapping, which already discounts page-cache that can't be
+ * reclaimed). Returns bytes, or null if the field is absent (caller falls back to os.freemem()).
+ */
+export function parseMemAvailableBytes(meminfo: string): number | null {
+  if (!meminfo) return null;
+  const m = meminfo.match(/^MemAvailable:\s*(\d+)\s*kB/im);
+  if (!m) return null;
+  const kb = Number(m[1]);
+  if (!Number.isSafeInteger(kb) || kb < 0) return null;
+  const bytes = kb * 1024;
+  if (!Number.isSafeInteger(bytes) || bytes < 0) return null;
+  return bytes;
+}
+
+/**
+ * #1516-r5 — the darwin raw-sysctl-output → usable-bytes mapping, factored out of
+ * {@link computeUsableMemBytes} so tests can drive this EXACT production mapping (parse, then
+ * NORMAL→sentinel/else→hold) over injected sysctl text, instead of hand-reimplementing it.
+ */
+export function darwinUsableMemBytesFromSysctlOutput(out: string): number | null {
+  const level = parseVmPressureLevel(out);
+  if (level === null) return null; // unparseable → fail OPEN (raw free)
+  if (level === VM_PRESSURE_NORMAL) {
+    // Kernel reports headroom → no memory-class ceiling; readUsableMemMb clamps this to the
+    // threaded os.totalmem() sample and floors at raw free, so the memory gate passes. This is
+    // what fixes macmini's false-low flicker (normal pressure despite a low raw-free reading).
+    return PRESSURE_HEADROOM_SENTINEL_BYTES;
+  }
+  // WARN / CRITICAL / any unexpected non-normal level → HOLD: no reclaimable boost. Falling back
+  // to raw free keeps a genuinely tight host blocked while never being harder-closed than
+  // raw-free, and is safe against the dirty-writeback OOM path (the kernel is why we're here).
+  return null;
+}
+
+/**
+ * The raw, un-cached probe: run the platform-specific headroom read (darwin: kernel VM pressure
+ * level → headroom-sentinel/hold; linux: /proc/meminfo MemAvailable). Never throws — any failure
+ * (binary missing, timeout, permissions, unexpected format, unknown platform) → null (fail-open).
+ */
+function computeUsableMemBytes(platform: NodeJS.Platform): number | null {
+  try {
+    if (platform === "darwin") {
+      // #1516-r3 — gate on the kernel's own memory-pressure signal, not a page-class sum.
+      const out = execFileSync("sysctl", ["-n", "kern.memorystatus_vm_pressure_level"], {
+        encoding: "utf8",
+        timeout: PROBE_TIMEOUT_MS,
+        // Capture stdout; discard stderr so a failing/absent sysctl fails silently (fail-open) rather
+        // than spamming the orchestrator's stderr on every re-probe.
+        stdio: ["ignore", "pipe", "ignore"],
+      });
+      return darwinUsableMemBytesFromSysctlOutput(out);
+    }
+    if (platform === "linux") {
+      const meminfo = readFileSync("/proc/meminfo", "utf8");
+      return parseMemAvailableBytes(meminfo);
+    }
+  } catch {
+    // Probe failed — fail OPEN (caller floors at os.freemem()).
+    return null;
+  }
+  return null;
+}
+
+interface ProbeCacheEntry {
+  atMs: number;
+  value: number | null;
+}
+const probeCache = new Map<string, ProbeCacheEntry>();
+
+export interface ProbeUsableMemOptions {
+  /**
+   * Injectable clock for tests. Defaults to a MONOTONIC source (`performance.now`) — #1516-r2/r3:
+   * the cache TTL must not be measured on the wall clock, or a backwards clock step (NTP/DST) makes
+   * the age negative and keeps a stale high probe alive past the TTL.
+   */
+  now?: () => number;
+  /** Injectable raw probe for tests (defaults to the real platform read). */
+  compute?: (platform: NodeJS.Platform) => number | null;
+}
+
+// #1516-r3 — monotonic default clock for the probe cache TTL. `performance.now()` never runs
+// backwards, so a wall-clock rollback can't extend the cache the way Date.now() could.
+const monotonicNowMs = (): number => performance.now();
+
+/** Clear the probe cache — for tests, so cached results don't leak across cases. */
+export function resetUsableMemProbeCache(): void {
+  probeCache.clear();
+}
+
+/**
+ * #1516 — best-effort probe of *usable* host memory in bytes. Prefers genuinely-reclaimable
+ * platform signals (darwin vm_stat, linux MemAvailable); on any probe failure, or an unrecognized
+ * platform, returns null so the caller falls back to raw `os.freemem()`. Never throws.
+ *
+ * #1516-r2 — the result is cached per-platform for {@link PROBE_CACHE_TTL_MS} so a synchronous
+ * execFileSync(sysctl) isn't spawned on every admission poll and can't serialize into long
+ * event-loop stalls. #1516-r3 — the TTL is measured on a MONOTONIC clock, and a negative age (a
+ * clock that ran backwards) is treated as expired, so a stale high probe can't survive a rollback.
+ */
+export function probeUsableMemBytes(
+  platform: NodeJS.Platform = process.platform,
+  opts: ProbeUsableMemOptions = {},
+): number | null {
+  const now = opts.now ?? monotonicNowMs;
+  const compute = opts.compute ?? computeUsableMemBytes;
+  const t = now();
+  const cached = probeCache.get(platform);
+  if (cached) {
+    const age = t - cached.atMs;
+    if (age >= 0 && age < PROBE_CACHE_TTL_MS) {
+      return cached.value;
+    }
+  }
+  const value = compute(platform);
+  probeCache.set(platform, { atMs: t, value });
+  return value;
+}
+
+/**
+ * #1516 — usable memory in MB the admission gate decides on. Takes the platform probe when
+ * available (darwin: pressure-level → physical RAM on NORMAL / null on WARN+CRITICAL; linux:
+ * MemAvailable), floors it at raw `os.freemem()` so the gate is NEVER stricter than the historical
+ * raw-free behavior even if the probe under-reports or holds, and caps it at physical RAM so a
+ * headroom signal can't inflate the figure past `os.totalmem()`.
+ *
+ * #1516-r2 — `rawFreeBytes` and `totalBytes` are threaded in from a SINGLE sample by the caller,
+ * so `usableMemMb >= freeMemMb` holds deterministically even while free memory churns.
+ */
+export function readUsableMemMb(
+  platform: NodeJS.Platform = process.platform,
+  rawFreeBytes: number = os.freemem(),
+  totalBytes: number = os.totalmem(),
+): number {
+  const probed = probeUsableMemBytes(platform);
+  let usableBytes = probed === null ? rawFreeBytes : Math.max(probed, rawFreeBytes);
+  // Sanity cap: usable can never exceed physical RAM (guards a plausible-but-wrong probe).
+  if (Number.isSafeInteger(totalBytes) && totalBytes > 0) {
+    usableBytes = Math.min(usableBytes, totalBytes);
+  }
+  return usableBytes / (1024 * 1024);
+}
+
+// #1513 — build the dashboard/notification descriptor for a spawn the headroom gate is holding back.
+// `spawnedBy` (the routing target) and `spawnRequestId` (idempotency) come off the spawn command
+// params; the reason/freeMem come from the live headroom poll so the row updates as the host recovers.
+export function queuedSpawnFromCommand(
+  params: Record<string, any>,
+  result: HostHeadroomResult,
+  queuedAt: number,
+  minFreeMemMb: number,
+  maxWaitMs: number,
+): QueuedSpawn {
+  return {
+    provider: typeof params.provider === "string" ? params.provider : "unknown",
+    ...(typeof params.spawnRequestId === "string" ? { spawnRequestId: params.spawnRequestId } : {}),
+    ...(typeof params.label === "string" ? { label: params.label } : {}),
+    ...(typeof params.cwd === "string" ? { cwd: params.cwd } : {}),
+    ...(typeof params.spawnedBy === "string" ? { spawnedBy: params.spawnedBy } : {}),
+    reason: result.reason ?? "waiting for host memory headroom",
+    freeMemMb: Math.round(result.freeMemMb),
+    minFreeMemMb,
+    queuedAt,
+    maxWaitMs,
+  };
+}
+
+/** Read current host memory/load headroom and decide whether it can admit another spawn. */
+export function checkHostHeadroom(opts: { minFreeMemMb?: number; maxLoadPerCpu?: number } = {}): HostHeadroomResult {
+  // #1516-r2 — sample raw free / total ONCE and thread them through so the usable figure is
+  // computed against the same free reading it is floored at (usableMemMb >= freeMemMb always).
+  const rawFreeBytes = os.freemem();
+  const totalBytes = os.totalmem();
+  const freeMemMb = rawFreeBytes / (1024 * 1024);
+  const usableMemMb = readUsableMemMb(process.platform, rawFreeBytes, totalBytes);
+  const totalMemMb = totalBytes / (1024 * 1024);
+  const cpuCount = Math.max(1, os.cpus().length);
+  const loadAvgPerCpu = os.loadavg()[0]! / cpuCount;
+  const minFreeMemMb = opts.minFreeMemMb ?? resolveMinFreeMemMb();
+  const maxLoadPerCpu = opts.maxLoadPerCpu ?? resolveMaxLoadPerCpu();
+  if (usableMemMb < minFreeMemMb) {
+    return {
+      ok: false,
+      reason: `usable memory ${usableMemMb.toFixed(0)}MB is below the ${minFreeMemMb}MB minimum required to admit another agent`,
+      usableMemMb,
+      freeMemMb,
+      totalMemMb,
+      loadAvgPerCpu,
+    };
+  }
+  if (loadAvgPerCpu > maxLoadPerCpu) {
+    return {
+      ok: false,
+      reason: `load average ${loadAvgPerCpu.toFixed(2)} per CPU exceeds the ${maxLoadPerCpu} max allowed to admit another agent`,
+      usableMemMb,
+      freeMemMb,
+      totalMemMb,
+      loadAvgPerCpu,
+    };
+  }
+  return { ok: true, usableMemMb, freeMemMb, totalMemMb, loadAvgPerCpu };
+}

package/src/index.ts

@@ -1,8 +1,10 @@
 #!/usr/bin/env bun
 import { errMessage } from "agent-relay-sdk";
+import { fireAndForget, guardedInterval, installUnhandledRejectionGuard } from "./async-guard";
 import { loadConfig, initConfigFile } from "./config";
 import { createRelayClient } from "./relay";
 import type { ManagedSessionExitDiagnostics } from "./relay";
+import { createRegistrationDriver } from "./registration";
 import { createControlHandler } from "./control";
 import { diagnoseSessionExit, hydrateTerminalGuests, managedSessionLivenessDetailed, reapTerminalGuests, refreshManagedAgentReport } from "./spawn";
 import { startApiServer } from "./api";
@@ -13,6 +15,8 @@ import { startOrchestratorMaintenanceScheduler } from "./maintenance";
 import { OrchestratorQuotaPoller, resolveLiveCodexResetConsumeTransport, type CodexResetFiredEvent } from "./quota-poller";
 import { SharedCallmuxSupervisor } from "./shared-callmux";
 import { createCommandPoller } from "./command-poller";
+import { defaultLocalHealthProbe, localReadinessUrl, resolveRuntimePath, selfHealLingeringUpgradeGuard } from "./self-upgrade";
+import { detectSelfSupervision } from "./self-supervision";
 
 const args = process.argv.slice(2);
 
@@ -50,10 +54,24 @@ Config file: ~/.agent-relay/orchestrator.json
   process.exit(0);
 }
 
+// #1676 — last-resort backstop, installed before anything can schedule work. An unhandled
+// rejection in the orchestrator is NOT fatal: every realistic source is a recoverable I/O
+// transient against the relay or a peer host, and the process that supervises other hosts'
+// agents makes the fleet less recoverable by dying. `uncaughtException` deliberately stays
+// at the runtime default (fatal) — see the SDK function for the full reasoning. If this
+// handler ever fires, a guarded site was missed; it logs at the volume a crash would have.
+installUnhandledRejectionGuard();
+
 const config = loadConfig();
 const probeCache = new ProviderProbeCache(config);
 const relay = createRelayClient(config, probeCache);
 const control = createControlHandler(config, relay);
+// #1478 — on a relay RECONNECT (e.g. after a relay restart), re-report the live managed-agent roster
+// so the relay's orchestrator↔agent control association is rebuilt for still-running agents. Without
+// this, register() re-establishes messaging but never re-sends the roster, and the relay never
+// reconstructs it — so relay_shutdown_agent stays -32004 and the dashboard web-terminal button stays
+// missing for reconnecting agents until the next spawn/exit change happens to trigger a re-report.
+relay.setOnReconnected(() => relay.updateManagedAgents(control.getManagedAgents()));
 // #1425 finding 1 — the live banked-reset consume transport is wired ONLY for a genuine production
 // run (explicit AGENT_RELAY_CODEX_RESET_CONSUME_LIVE=1 opt-in) and NEVER for the dev-service profile
 // (AGENT_RELAY_DEV_PROFILE=1) / a test / CI. In every other case the resolver returns undefined, so
@@ -69,14 +87,16 @@ const quotaPoller = new OrchestratorQuotaPoller(config, relay, {
   ...(liveCodexResetConsume ? { codexResetCreditConsume: liveCodexResetConsume } : {}),
   onResetFired: (event: CodexResetFiredEvent) => {
     if (event.trigger !== "auto") return;
-    void relay.reportCodexResetFired(event).catch((err) =>
-      console.error(`[orchestrator] codex reset-fired event report failed: ${errMessage(err)}`));
+    void fireAndForget("codex reset-fired event report", relay.reportCodexResetFired(event));
   },
 });
 const sharedCallmux = new SharedCallmuxSupervisor(config);
 
+// #1676 — registration retry: exponential backoff + a surfaced degraded state, replacing
+// the fixed 5s `for(;;)` spin that was the second half of the crash loop.
+const registration = createRegistrationDriver({ register: () => relay.register() });
+
 const POLL_INTERVAL_MS = 3_000;
-const REGISTER_RETRY_MS = 5_000;
 const GUEST_REAP_INTERVAL_MS = 60_000;
 let commandPoller: ReturnType<typeof createCommandPoller> | null = null;
 let healthCheckTimer: Timer | null = null;
@@ -91,20 +111,42 @@ async function startup(): Promise<void> {
   console.error(`[orchestrator]   env keys: ${Object.keys(config.env).length}`);
 
   // Start API server before registration so we can advertise the URL
-  apiServer = startApiServer(config, probeCache, relay, quotaPoller);
+  apiServer = startApiServer(config, probeCache, relay, quotaPoller, registration);
   console.error(`[orchestrator]   apiUrl: ${apiServer.url}`);
 
+  relay.setApiUrl(apiServer.url);
+
+  // #1509 r7 §13 — COSMETIC self-heal against a lingering OWN-unit self-upgrade restart guard,
+  // kicked off BEFORE registration and NOT awaited. Under the r7 one-shot model (RunAtLoad,
+  // KeepAlive=false) a guard that fails to boot itself out sits loaded-but-idle at PID `-`: it can
+  // NEITHER loop (KeepAlive=false ⇒ never relaunches) NOR wedge a later dispatch (the dispatch
+  // precondition blocks only on a RUNNING guard, and this reap is not on the critical path), so
+  // this once-at-startup unawaited call can no longer leave a wedge — it merely tidies the launchd
+  // listing. It also cleans up any PRE-r7 KeepAlive guard left over across the upgrade transition;
+  // for those it gates on a LOCAL /api/health signal (Relay-independent) and reaps only when proven
+  // healthy. It removes each stale generation by its exact unique `.s<seq>.<uuid>` label; a NEW
+  // upgrade's guard is RUNNING under a different uuid-unique label, so it is never mistaken for a
+  // stale one nor removed by a concurrent reap. Scoped to `${selfUnit}.upgrade-guard`.
+  const selfHealSupervision = detectSelfSupervision();
+  void fireAndForget("Upgrade-guard self-heal", selfHealLingeringUpgradeGuard(selfHealSupervision.selfUnit, {
+    probe: defaultLocalHealthProbe(localReadinessUrl(config.apiPort)),
+    runtimePath: resolveRuntimePath(selfHealSupervision.runtimePrefix),
+  }).then((reapedGuards) => {
+    if (reapedGuards.length > 0) {
+      console.error(`[orchestrator] Reaped ${reapedGuards.length} lingering own-unit self-upgrade guard job(s): ${reapedGuards.join(", ")}`);
+    }
+  }));
+
   // Register with relay. The server and orchestrator are often restarted
   // together, so startup must tolerate the server not listening yet.
-  relay.setApiUrl(apiServer.url);
-  await registerUntilConnected();
+  await registration.registerUntilConnected();
   relay.startHeartbeatLoop();
 
   // Recover existing tmux sessions
   await recoverManagedAgents(config, control, relay);
 
   // Host-local maintenance must run where the agent processes and tmux sockets live.
-  startOrchestratorMaintenanceScheduler();
+  startOrchestratorMaintenanceScheduler(config);
   quotaPoller.start();
   try {
     sharedCallmux.start();
@@ -125,18 +167,18 @@ async function startup(): Promise<void> {
   // Start polling for command requests
   startPolling();
 
-  // Periodic health check — remove dead sessions
-  healthCheckTimer = setInterval(healthCheck, 60_000);
+  // Periodic health check — remove dead sessions.
+  // #1676 — this was `setInterval(healthCheck, 60_000)`, and it is the most likely site
+  // of the incident's UNCAUGHT AbortError: healthCheck awaits relay.updateManagedAgents(),
+  // whose fetch is timeout-aborted, and an async timer callback's rejection has no handler
+  // — in Bun that exits the process. guardedInterval consumes it and logs.
+  healthCheckTimer = guardedInterval("Managed-agent health check", 60_000, healthCheck);
 
   // Periodic guest-terminal reaper — enforces guest TTL without requiring a new
   // guest creation to trigger cleanup (#144).
-  guestReaperTimer = setInterval(() => {
-    try {
-      reapTerminalGuests(config);
-    } catch (err) {
-      console.error(`[orchestrator] Guest reap error: ${err}`);
-    }
-  }, GUEST_REAP_INTERVAL_MS);
+  guestReaperTimer = guardedInterval("Guest terminal reap", GUEST_REAP_INTERVAL_MS, () => {
+    reapTerminalGuests(config);
+  });
 
   console.error("[orchestrator] Ready. Polling for command requests...");
 }
@@ -146,19 +188,6 @@ function startPolling(): void {
   commandPoller.start();
 }
 
-async function registerUntilConnected(): Promise<void> {
-  for (;;) {
-    try {
-      await relay.register();
-      return;
-    } catch (err) {
-      console.error(`[orchestrator] Register failed: ${err}`);
-      console.error(`[orchestrator] Retrying registration in ${Math.round(REGISTER_RETRY_MS / 1000)}s...`);
-      await new Promise((resolve) => setTimeout(resolve, REGISTER_RETRY_MS));
-    }
-  }
-}
-
 async function healthCheck(): Promise<void> {
   const agents = control.getManagedAgents();
   let changed = false;