@openparachute/hub 0.5.14-rc.8 → 0.6.0

package/src/commands/lifecycle.ts

@@ -1,5 +1,11 @@
-import { existsSync, openSync } from "node:fs";
+import { existsSync, openSync, readFileSync } from "node:fs";
+import { Socket } from "node:net";
 import { join } from "node:path";
+import {
+  MissingDependencyError,
+  ensureExecutable,
+  rethrowIfMissing,
+} from "@openparachute/depcheck";
 import { CONFIG_DIR, SERVICES_MANIFEST_PATH } from "../config.ts";
 import { readEnvFileValues } from "../env-file.ts";
 import { readExposeState } from "../expose-state.ts";
@@ -12,8 +18,10 @@ import {
   readHubPort,
   stopHub,
 } from "../hub-control.ts";
+import { hubDbPath, openHubDb } from "../hub-db.ts";
 import { HUB_ORIGIN_ENV, deriveHubOrigin } from "../hub-origin.ts";
 import { ModuleManifestError, readModuleManifest } from "../module-manifest.ts";
+import { type OperatorIssuerHealStatus, selfHealOperatorTokenIssuer } from "../operator-token.ts";
 import {
   type AliveFn,
   clearPid,
@@ -32,7 +40,13 @@ import {
   knownServices,
   shortNameForManifest,
 } from "../service-spec.ts";
-import { type ServiceEntry, readManifest } from "../services-manifest.ts";
+import {
+  type ServiceEntry,
+  clearStartError,
+  readManifest,
+  recordStartError,
+} from "../services-manifest.ts";
+import { persistVaultHubOrigin, selfHealVaultHubOrigin } from "../vault-hub-origin-env.ts";
 
 /**
  * Tiny seam over `Bun.spawn` for lifecycle tests. The real spawner opens the
@@ -83,6 +97,44 @@ export const defaultSpawner: Spawner = {
 export type KillFn = (pid: number, signal: NodeJS.Signals | number) => void;
 export type SleepFn = (ms: number) => Promise<void>;
 
+/**
+ * "Is something listening on this TCP port on loopback?" seam. Pairs with the
+ * spawn-then-die settle (hub#194) to catch the *other* silent-start failure
+ * shape (hub#487): a service that lives long enough to clear the liveness
+ * check but never binds its port because the port is already held (EADDRINUSE
+ * from an orphan). The recorded pid stays alive (vault's process supervisor
+ * retries / lingers) so `alive(pid)` says "running" while `parachute status`
+ * shows it inactive because nothing answers on the port.
+ *
+ * Tests inject a deterministic stub; production uses `defaultPortListening`.
+ */
+export type PortListeningFn = (port: number) => Promise<boolean>;
+
+/**
+ * Connect-probe: open a TCP socket to 127.0.0.1:<port> and see if it's
+ * accepted. A successful connect means *something* is listening; we close
+ * immediately. Connection refused / timeout means nothing is bound yet.
+ * `node:net` rather than `Bun.connect` because the latter has no clean
+ * "connection refused → false" without a custom socket handler, and the net
+ * Socket's `error`/`connect` events map directly onto the boolean we want.
+ */
+export const defaultPortListening: PortListeningFn = (port) =>
+  new Promise((resolve) => {
+    const socket = new Socket();
+    let settled = false;
+    const done = (listening: boolean) => {
+      if (settled) return;
+      settled = true;
+      socket.destroy();
+      resolve(listening);
+    };
+    socket.setTimeout(1000);
+    socket.once("connect", () => done(true));
+    socket.once("timeout", () => done(false));
+    socket.once("error", () => done(false));
+    socket.connect(port, "127.0.0.1");
+  });
+
 /**
  * Group-aware liveness: returns true if the process group (pgid == pid)
  * still has any member. Pairs with `defaultSpawner`'s `detached: true` —
@@ -129,6 +181,35 @@ export const defaultKill: KillFn = (pid, signal) => {
 
 export const defaultSleep: SleepFn = (ms) => new Promise((r) => setTimeout(r, ms));
 
+/**
+ * Read the trailing `n` lines of a logfile, best-effort. Used to surface the
+ * real boot error when a start fails — operators shouldn't have to manually
+ * `tail` the log to learn *why* the daemon died. Returns [] on any read
+ * error (missing file, permissions) so the caller falls back to the generic
+ * "tail the log" hint without throwing.
+ */
+function readLogTail(logFile: string, n: number): string[] {
+  try {
+    const content = readFileSync(logFile, "utf8");
+    const trimmed = content.replace(/\n$/, "");
+    if (trimmed === "") return [];
+    return trimmed.split("\n").slice(-n);
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Heuristic EADDRINUSE detector over a logfile tail. cloudflared, Bun, and
+ * Node all surface port collisions with recognizable phrases; we match the
+ * common ones rather than parse a structured error (there isn't one across
+ * runtimes). False positives are harmless — the worst case is we *also* print
+ * the port-in-use remedy on an unrelated failure, which is still actionable.
+ */
+function detectAddrInUse(logTail: readonly string[]): boolean {
+  return logTail.some((line) => /EADDRINUSE|address already in use|port .* in use/i.test(line));
+}
+
 export interface LifecycleOpts {
   spawner?: Spawner;
   kill?: KillFn;
@@ -160,6 +241,30 @@ export interface LifecycleOpts {
    * settle.
    */
   startSettleMs?: number;
+  /**
+   * Probe whether the service's port is listening, post-spawn. Pairs with the
+   * settle (hub#194) to catch the EADDRINUSE-orphan shape (hub#487): the
+   * process survives the liveness window (vault lingers / retries) but never
+   * binds because the port is already held, so `start` would otherwise report
+   * "✓ started" while `status` shows it inactive. Tests inject a stub;
+   * production uses `defaultPortListening` (a loopback TCP connect probe).
+   */
+  portListening?: PortListeningFn;
+  /**
+   * How long `start` polls for the service to bind its port after the
+   * liveness settle passes. Default 4000ms in production — long enough to
+   * cover vault/scribe cold-boot (DB open, route registration) without making
+   * a healthy start feel laggy. Polled at `startReadyPollMs` intervals; the
+   * first time the port answers we declare success. If the window elapses
+   * with the process still alive but the port silent, we print a non-fatal
+   * warning (the daemon may still be coming up) rather than failing — only a
+   * *dead* process is a hard failure. Defaulting policy mirrors
+   * `startSettleMs`: 0 (skipped) unless `portListening` is injected or the
+   * production path (no spawner override) is active.
+   */
+  startReadyMs?: number;
+  /** Poll interval while waiting for the port to come up. Default 200ms. */
+  startReadyPollMs?: number;
   /**
    * Override the hub origin passed to services as PARACHUTE_HUB_ORIGIN. If
    * unset, `start` derives it from `expose-state.json` (when exposed) or
@@ -175,9 +280,36 @@ export interface LifecycleOpts {
    * `ensureHubRunning` and `lifecycle.stop("hub")` dispatches to
    * `stopHub`. Tests inject stubs to avoid spawning real bun processes.
    */
+  /**
+   * PATH-resolution seam for the start preflight (`@openparachute/depcheck`
+   * `ensureExecutable`). Production uses the real `Bun.which`; a missing
+   * startCmd binary then surfaces the friendly missing-dependency UX +
+   * persists it to services.json.
+   *
+   * Defaulting policy mirrors `startSettleMs`: when a stub `spawner` is
+   * injected (the test path) `which` defaults to a permissive resolver
+   * (`() => "<stub>"`) so existing stub-spawner tests don't trip the preflight
+   * against binaries that aren't on the test host's PATH (`parachute-vault`,
+   * `notes-serve`). Production (no spawner override) gets the real `Bun.which`.
+   * Tests that want to exercise the missing-binary branch inject `which`
+   * explicitly (e.g. `which: () => null`).
+   */
+  which?: (cmd: string) => string | null;
   hub?: {
     ensureRunning?: (opts: EnsureHubOpts) => Promise<EnsureHubResult>;
     stop?: (opts: StopHubOpts) => Promise<boolean>;
+    /**
+     * Self-heal the operator token's stale `iss` after `start hub` (hub#481).
+     * Production opens hub.db at `<configDir>/hub.db` and delegates to
+     * `selfHealOperatorTokenIssuer`. Tests inject a stub to assert the call
+     * happens — or to make it throw and prove a self-heal failure never fails
+     * `start hub`.
+     */
+    selfHealOperatorToken?: (args: {
+      issuer: string;
+      configDir: string;
+      log: (line: string) => void;
+    }) => Promise<OperatorIssuerHealStatus>;
   };
 }
 
@@ -193,9 +325,42 @@ interface Resolved {
   killWaitMs: number;
   pollIntervalMs: number;
   startSettleMs: number;
+  portListening: PortListeningFn;
+  startReadyMs: number;
+  startReadyPollMs: number;
+  which: (cmd: string) => string | null;
   hubOrigin: string | undefined;
   ensureHub: (opts: EnsureHubOpts) => Promise<EnsureHubResult>;
   stopHubFn: (opts: StopHubOpts) => Promise<boolean>;
+  selfHealOperatorTokenFn: (args: {
+    issuer: string;
+    configDir: string;
+    log: (line: string) => void;
+  }) => Promise<OperatorIssuerHealStatus>;
+}
+
+/**
+ * Production self-heal: open hub.db at `<configDir>/hub.db`, run
+ * `selfHealOperatorTokenIssuer`, and close the db. Derives the db path the
+ * same way the rest of the repo does (`hubDbPath(configDir)`); `openHubDb`
+ * runs migrations + WAL on open, matching `commands/auth.ts`. Tests override
+ * this whole seam, so the db-open only happens on the production path.
+ */
+async function defaultSelfHealOperatorToken(args: {
+  issuer: string;
+  configDir: string;
+  log: (line: string) => void;
+}): Promise<OperatorIssuerHealStatus> {
+  const db = openHubDb(hubDbPath(args.configDir));
+  try {
+    return await selfHealOperatorTokenIssuer(db, {
+      issuer: args.issuer,
+      configDir: args.configDir,
+      log: args.log,
+    });
+  } finally {
+    db.close();
+  }
 }
 
 function resolve(opts: LifecycleOpts): Resolved {
@@ -219,9 +384,26 @@ function resolve(opts: LifecycleOpts): Resolved {
     // override `alive`, which re-enables the default 250ms.
     startSettleMs:
       opts.startSettleMs ?? (opts.spawner === undefined || opts.alive !== undefined ? 250 : 0),
+    portListening: opts.portListening ?? defaultPortListening,
+    // Same defaulting policy as startSettleMs: production (no spawner
+    // override) gets the real 4s readiness window; tests that inject a stub
+    // spawner get 0 (skipped) unless they explicitly opt in via
+    // `portListening` or `startReadyMs`, so existing stub-spawner tests don't
+    // start probing a fake port.
+    startReadyMs:
+      opts.startReadyMs ??
+      (opts.spawner === undefined || opts.portListening !== undefined ? 4000 : 0),
+    startReadyPollMs: opts.startReadyPollMs ?? 200,
+    // Same defaulting policy as startSettleMs/startReadyMs: production (no
+    // spawner override) preflights with the real Bun.which; stub-spawner tests
+    // get a permissive resolver so the preflight doesn't trip against binaries
+    // that aren't on the test host's PATH. Explicit `which` always wins.
+    which:
+      opts.which ?? (opts.spawner === undefined ? Bun.which : () => "/stub/bin/preflight-skipped"),
     hubOrigin: resolveHubOrigin(opts.hubOrigin, configDir),
     ensureHub: opts.hub?.ensureRunning ?? ensureHubRunning,
     stopHubFn: opts.hub?.stop ?? stopHub,
+    selfHealOperatorTokenFn: opts.hub?.selfHealOperatorToken ?? defaultSelfHealOperatorToken,
   };
 }
 
@@ -452,42 +634,185 @@ export async function start(svc: string | undefined, opts: LifecycleOpts = {}):
     if (entry.installDir) spawnerOpts.cwd = entry.installDir;
     const passOpts =
       spawnerOpts.env !== undefined || spawnerOpts.cwd !== undefined ? spawnerOpts : undefined;
+
+    // Pre-flight the startCmd binary (`@openparachute/depcheck`) so a missing
+    // executable surfaces the friendly install UX inline AND is persisted onto
+    // the services.json row, so a *later* `parachute status` (a separate
+    // invocation that only reads the manifest) + the SPA modules pane show
+    // "vault: failed to start — parachute-vault not installed" with install
+    // info, rather than a bare "failed"/orphan-timeout. The binary is `cmd[0]`
+    // (e.g. `parachute-vault` for an npm install, `bun` for a bun-linked one).
+    const startBinary = cmd[0];
+    if (startBinary) {
+      try {
+        ensureExecutable(startBinary, { which: r.which });
+      } catch (err) {
+        if (err instanceof MissingDependencyError) {
+          failures++;
+          r.log(`✗ ${short} failed to start:`);
+          for (const line of err.message.split("\n")) r.log(`  ${line}`);
+          recordStartError(entry.name, err.toWire(), r.manifestPath);
+          continue;
+        }
+        throw err;
+      }
+    }
+
     let pid: number;
     try {
       pid = r.spawner.spawn(cmd, logFile, passOpts);
     } catch (err) {
+      // Belt-and-suspenders: a missing binary that slipped past the pre-flight
+      // (race) still becomes a MissingDependencyError via rethrowIfMissing.
+      if (startBinary) {
+        try {
+          rethrowIfMissing(err, startBinary);
+        } catch (missing) {
+          if (missing instanceof MissingDependencyError) {
+            failures++;
+            r.log(`✗ ${short} failed to start:`);
+            for (const line of missing.message.split("\n")) r.log(`  ${line}`);
+            recordStartError(entry.name, missing.toWire(), r.manifestPath);
+            continue;
+          }
+        }
+      }
       failures++;
       const msg = err instanceof Error ? err.message : String(err);
       r.log(`✗ ${short} failed to start: ${msg}`);
       continue;
     }
+    // A successful spawn clears any stale start-error recorded from a prior
+    // missing-dependency failure so `parachute status` doesn't keep showing it.
+    clearStartError(entry.name, r.manifestPath);
     writePid(short, pid, r.configDir);
 
-    // Settle-poll for spawn-then-immediately-die (hub#194). A spawn returning
-    // a pid only proves the kernel forked the process; the child may exit
-    // microseconds later if its main code path throws before listening
-    // (e.g. notes-serve's Bun.resolveSync failing for bun-linked installs).
-    // Without this poll, we'd report success and the operator would chase
-    // a phantom 502.
+    // Boot-readiness gating (hub#194 + hub#487). A spawn returning a pid only
+    // proves the kernel forked the process — it says nothing about whether the
+    // service survived its boot or bound its port. Two silent-start shapes:
+    //
+    //   (1) spawn-then-immediately-die (hub#194): the child throws before
+    //       listening (notes-serve's Bun.resolveSync failing for bun-linked
+    //       installs) and exits microseconds later. Caught by the settle below.
+    //
+    //   (2) alive-but-never-bound (hub#487): the port is already held by an
+    //       orphan, the child hits EADDRINUSE, but its process *lingers* (or a
+    //       supervisor retries) long enough to clear the liveness check. `start`
+    //       would report "✓ started" while `parachute status` shows it inactive
+    //       because nothing answers on the port. Aaron hit exactly this with an
+    //       orphan holding vault's 1940 on a fresh EC2 box. Caught by the
+    //       port-readiness poll below.
+    //
+    // On any failure we surface the tail of the logfile so the operator sees
+    // the real boot error inline, and we specifically call out EADDRINUSE with
+    // the `lsof -ti:<port>` remedy.
+    const reportStartFailure = (reason: string): void => {
+      clearPid(short, r.configDir);
+      failures++;
+      const tail = readLogTail(logFile, 20);
+      if (detectAddrInUse(tail)) {
+        r.log(
+          `✗ ${short} failed to start: port ${entry.port} is already in use. Stop the existing process first — find it with \`lsof -ti:${entry.port}\` (then \`kill <pid>\`), or run \`parachute restart ${short}\`.`,
+        );
+      } else {
+        r.log(`✗ ${short} failed to start: ${reason}`);
+      }
+      if (tail.length > 0) {
+        r.log(`  ── last ${tail.length} log line(s) (${logFile}) ──`);
+        for (const line of tail) r.log(`  │ ${line}`);
+      } else {
+        r.log(`  Tail the log for details: tail -50 ${logFile}`);
+      }
+    };
+
     if (r.startSettleMs > 0) {
       await r.sleep(r.startSettleMs);
       if (!r.alive(pid)) {
-        clearPid(short, r.configDir);
-        failures++;
+        reportStartFailure(
+          `spawned pid ${pid} but the process exited within ${r.startSettleMs}ms.`,
+        );
+        continue;
+      }
+    }
+
+    // Port-readiness poll (hub#487). The process is alive; now confirm it
+    // actually bound its port before claiming success. Poll up to
+    // `startReadyMs`, re-checking liveness each iteration so a *later* death
+    // (e.g. a slow EADDRINUSE crash) is still reported as a failure. A process
+    // that stays alive but never binds within the window gets a non-fatal
+    // warning rather than a hard failure — some daemons legitimately do slow
+    // boot work, and we'd rather not flip a healthy-but-slow start to red.
+    if (r.startReadyMs > 0) {
+      const deadline = r.now() + r.startReadyMs;
+      let listening = false;
+      let died = false;
+      while (r.now() < deadline) {
+        if (!r.alive(pid)) {
+          died = true;
+          break;
+        }
+        if (await r.portListening(entry.port)) {
+          listening = true;
+          break;
+        }
+        await r.sleep(r.startReadyPollMs);
+      }
+      if (died) {
+        reportStartFailure(`spawned pid ${pid} but the process exited during startup.`);
+        continue;
+      }
+      if (!listening) {
+        // Last-chance liveness check — the loop may have exited on the
+        // deadline right as the process died.
+        if (!r.alive(pid)) {
+          reportStartFailure(`spawned pid ${pid} but the process exited during startup.`);
+          continue;
+        }
         r.log(
-          `✗ ${short} failed to start: spawned pid ${pid} but the process exited within ${r.startSettleMs}ms.`,
+          `⚠ ${short} started (pid ${pid}) but port ${entry.port} isn't accepting connections yet after ${r.startReadyMs}ms.`,
         );
-        r.log(`  Tail the log for details: tail -50 ${logFile}`);
+        r.log(
+          `  It may still be coming up — check \`parachute status\` and \`parachute logs ${short}\`.`,
+        );
+        if (r.hubOrigin) r.log(`  ${HUB_ORIGIN_ENV}=${r.hubOrigin}`);
+        if (short === "vault") persistVaultHubOriginForStart(r);
         continue;
       }
     }
 
     r.log(`✓ ${short} started (pid ${pid}); logs: ${logFile}`);
     if (r.hubOrigin) r.log(`  ${HUB_ORIGIN_ENV}=${r.hubOrigin}`);
+    if (short === "vault") persistVaultHubOriginForStart(r);
   }
   return failures === 0 ? 0 : 1;
 }
 
+/**
+ * Durable-persist vault's `PARACHUTE_HUB_ORIGIN` on a vault `start`. Two cases,
+ * in order:
+ *
+ *  1. The resolved spawn origin (`r.hubOrigin`) is a real public origin — write
+ *     it. This is the long-standing happy path: an exposure is live, the
+ *     launchd / systemd daemon (which boots vault out-of-band and never sees
+ *     this spawn env) needs it in `.env` to validate hub-minted JWTs' `iss`.
+ *     `persistVaultHubOrigin` skips loopback / unchanged values itself.
+ *
+ *  2. Self-heal: even when `r.hubOrigin` resolved to loopback or undefined
+ *     (e.g. the hub.port file outran the expose-state read, or this is a bare
+ *     `restart vault` on a deploy whose `.env` was never written), consult
+ *     `expose-state.json` directly. If it advertises a public origin and
+ *     vault's persisted value is unset / loopback, write the public origin.
+ *     This is what lets an EXISTING broken Cloudflare deploy self-correct on
+ *     the next `parachute restart vault`, not only fresh exposes.
+ *
+ * Case 1 covers the override / freshly-resolved path; case 2 catches the gap
+ * the Cloudflare 401 P0 fell through. See `vault-hub-origin-env.ts`.
+ */
+function persistVaultHubOriginForStart(r: Resolved): void {
+  if (r.hubOrigin) persistVaultHubOrigin(r.configDir, r.hubOrigin, r.log);
+  selfHealVaultHubOrigin(r.configDir, r.log, join(r.configDir, "expose-state.json"));
+}
+
 export async function stop(svc: string | undefined, opts: LifecycleOpts = {}): Promise<number> {
   const r = resolve(opts);
   if (svc === HUB_SVC) return stopHubSvc(r);
@@ -567,6 +892,12 @@ async function startHubSvc(r: Resolved): Promise<number> {
     } else {
       r.log(`hub already running (pid ${result.pid}) on port ${result.port}.`);
     }
+    // Self-heal a stale operator-token issuer (hub#481). Runs whether the hub
+    // was freshly started OR already running — a token stamped at loopback
+    // before exposure must heal even when the hub is already up. The loopback /
+    // provenance guards live inside `selfHealOperatorTokenIssuer`, so the only
+    // gate here is "is there a real issuer to heal toward?".
+    await selfHealOperatorTokenOnStart(r);
     return 0;
   } catch (err) {
     r.log(`✗ hub failed to start: ${err instanceof Error ? err.message : String(err)}`);
@@ -574,6 +905,36 @@ async function startHubSvc(r: Resolved): Promise<number> {
   }
 }
 
+/**
+ * Re-issue the operator token under the hub's current origin when its `iss`
+ * went stale after an init-at-loopback → expose transition (hub#481). Mirrors
+ * `persistVaultHubOriginForStart`'s quiet style: emit a single line only when
+ * a rotation actually happens; stay silent for fresh / absent / skipped.
+ *
+ * The ENTIRE self-heal is wrapped here so it can NEVER block or fail
+ * `start hub` — a db-open error, a corrupt token, anything — degrades to a
+ * brief warning and `start hub` still returns 0.
+ */
+async function selfHealOperatorTokenOnStart(r: Resolved): Promise<void> {
+  if (!r.hubOrigin) return;
+  try {
+    const status = await r.selfHealOperatorTokenFn({
+      issuer: r.hubOrigin,
+      configDir: r.configDir,
+      log: r.log,
+    });
+    if (status.kind === "rotated") {
+      r.log(`  refreshed operator.token issuer → ${r.hubOrigin} (was stale after exposure)`);
+    }
+  } catch (err) {
+    r.log(
+      `  note: operator.token issuer self-heal skipped (${
+        err instanceof Error ? err.message : String(err)
+      })`,
+    );
+  }
+}
+
 /**
  * Stop the internal hub. `stopHub` returns false when nothing was running
  * (no pidfile, or stale pidfile cleared) — that's a clean no-op for the
@@ -659,11 +1020,19 @@ export async function logs(svc: string, opts: LogsOpts = {}): Promise<number> {
       spawn(cmd) {
         // Inherit env so `tail` sees PATH, etc. Bun.spawn defaults to empty
         // env — see api-modules-ops.ts:defaultRun.
-        const proc = Bun.spawn([...cmd], {
-          stdio: ["ignore", "inherit", "inherit"],
-          env: process.env,
-        });
-        return proc.pid;
+        try {
+          const proc = Bun.spawn([...cmd], {
+            stdio: ["ignore", "inherit", "inherit"],
+            env: process.env,
+          });
+          return proc.pid;
+        } catch (err) {
+          // A missing `tail` (minimal container without coreutils) surfaces
+          // the friendly install UX instead of a raw spawn throw. The CLI
+          // top-level catch in cli.ts renders the MissingDependencyError.
+          rethrowIfMissing(err, "tail");
+          throw err;
+        }
       },
     };
     spawner.spawn(["tail", "-n", String(lines), "-f", path], path);

package/src/commands/status.ts

@@ -146,6 +146,14 @@ interface StatusRow {
    * stale-after-rebuild row without comparing columns by eye.
    */
   staleNote?: string;
+  /**
+   * Persisted last-start failure (`lastStartError`, written by the lifecycle
+   * start preflight when a startCmd binary is missing). Surfaced on a
+   * continuation line so a *later* `parachute status` explains why the row
+   * isn't active — "failed to start: <binary> not installed" — rather than
+   * just showing it inactive. Cleared on the next successful start.
+   */
+  startErrorNote?: string;
 }
 
 /**
@@ -264,6 +272,17 @@ export async function status(opts: StatusOpts = {}): Promise<number> {
         ? `STALE: services.json cached ${entry.version}; live package.json ${source.livePackageVersion}`
         : undefined;
 
+      // Persisted last-start failure (lifecycle preflight wrote a missing-
+      // dependency wire). Surface a one-line summary; the full install recipe
+      // lives in services.json + the admin SPA card. Keeps `parachute status`
+      // scannable while still telling the operator "this is why it's down."
+      const startErrorNote =
+        entry.lastStartError !== undefined
+          ? entry.lastStartError.binary !== undefined
+            ? `failed to start: ${entry.lastStartError.binary} not installed — run \`parachute status\` detail or see /admin/modules for install steps`
+            : `failed to start: ${entry.lastStartError.error_description.split("\n")[0]}`
+          : undefined;
+
       // Only skip probe when we know the process is dead (PID file was
       // present but kill(pid, 0) failed). "unknown" status (no PID file)
       // still probes — externally-managed services should report health.
@@ -287,6 +306,7 @@ export async function status(opts: StatusOpts = {}): Promise<number> {
           skipped: true,
           driftWarning,
           staleNote,
+          startErrorNote,
         };
       }
 
@@ -324,6 +344,7 @@ export async function status(opts: StatusOpts = {}): Promise<number> {
         skipped: false,
         driftWarning,
         staleNote,
+        startErrorNote,
       };
     }),
   );
@@ -378,6 +399,7 @@ export async function status(opts: StatusOpts = {}): Promise<number> {
     }
     if (row.driftWarning) print(`  ! ${row.driftWarning}`);
     if (row.staleNote) print(`  ! ${row.staleNote}`);
+    if (row.startErrorNote) print(`  ! ${row.startErrorNote}`);
   }
 
   /**

package/src/commands/upgrade.ts

@@ -72,25 +72,69 @@ export interface UpgradeRunner {
   ): Promise<{ code: number; stdout: string }>;
 }
 
+/**
+ * Exit code we synthesize when a binary can't be spawned at all. 127 is the
+ * POSIX shell convention for "command not found" — it lets every git call
+ * degrade to a normal non-zero result instead of crashing the whole command.
+ */
+const SPAWN_NOT_FOUND_CODE = 127;
+
+/**
+ * True when an error thrown by `Bun.spawn` means "the executable doesn't
+ * exist on this host" (ENOENT). On a minimal server with no `git` installed —
+ * a legitimate, common shape for a published-npm install on the canonical
+ * install path — `Bun.spawn(["git", ...])` throws *synchronously* with this
+ * shape. We catch it so `parachute upgrade` degrades to the npm path rather
+ * than dying with an uncaught `Executable not found in $PATH: "git"`.
+ */
+function isSpawnNotFound(err: unknown): boolean {
+  if (typeof err !== "object" || err === null) return false;
+  const code = (err as { code?: unknown }).code;
+  const message = (err as { message?: unknown }).message;
+  return (
+    code === "ENOENT" ||
+    (typeof message === "string" && message.includes("Executable not found in $PATH"))
+  );
+}
+
 export const defaultRunner: UpgradeRunner = {
   async run(cmd, opts) {
     // Inherit env so `bun add -g` etc. see TMPDIR, BUN_INSTALL, PATH, HOME.
     // Bun.spawn defaults to empty env — see api-modules-ops.ts:defaultRun.
-    const proc = Bun.spawn([...cmd], {
-      cwd: opts?.cwd,
-      stdio: ["inherit", "inherit", "inherit"],
-      env: process.env,
-    });
+    let proc: Bun.Subprocess;
+    try {
+      proc = Bun.spawn([...cmd], {
+        cwd: opts?.cwd,
+        stdio: ["inherit", "inherit", "inherit"],
+        env: process.env,
+      });
+    } catch (err) {
+      // Binary not on this host (e.g. no `git` on a minimal server). Degrade
+      // to a non-zero exit rather than letting the throw crash the command.
+      if (isSpawnNotFound(err)) return SPAWN_NOT_FOUND_CODE;
+      throw err;
+    }
     return await proc.exited;
   },
   async capture(cmd, opts) {
     // Inherit env — same rationale as `run` above.
-    const proc = Bun.spawn([...cmd], {
-      cwd: opts?.cwd,
-      stdout: "pipe",
-      stderr: "pipe",
-      env: process.env,
-    });
+    let proc: Bun.Subprocess<"ignore", "pipe", "pipe">;
+    try {
+      proc = Bun.spawn([...cmd], {
+        cwd: opts?.cwd,
+        stdout: "pipe",
+        stderr: "pipe",
+        env: process.env,
+      });
+    } catch (err) {
+      // See `run` above: ENOENT (binary-not-found) becomes a captured
+      // non-zero result so every git call degrades to "command failed".
+      if (isSpawnNotFound(err)) {
+        const bin = cmd[0] ?? "command";
+        return { code: SPAWN_NOT_FOUND_CODE, stdout: `${bin}: not found on this host\n` };
+      }
+      throw err;
+    }
     const [stdout, stderr] = await Promise.all([
       new Response(proc.stdout).text(),
       new Response(proc.stderr).text(),