@monster-orange/den 0.2.3 → 0.2.5

package/src/shared/environment-manager/health.ts

@@ -9,8 +9,13 @@ import { hasRuntime, repoKind } from "../repo-kind.js";
 import * as https from "https";
 import * as tls from "tls";
 import { readEdgeConfig } from "../edge/config.js";
-import { composeHealthCmd } from "../stacks/manifest-commands.js";
+import {
+  composeHealth,
+  composeHealthTarget,
+  type ComposeHealthTarget,
+} from "../stacks/compose-health.js";
 import * as fs from "fs";
+import * as os from "os";
 import * as path from "path";
 import * as dns from "dns";
 import { spawn } from "child_process";
@@ -38,7 +43,15 @@ import type { Environment } from "../types/index.js";
 
 import { product } from "../product.js";
 export interface PingResult {
-  ok: boolean;
+  /**
+   * true: the check answered healthy. false: it COMPLETED and failed. null:
+   * it did not complete or could not tell (no answer within its budget, its
+   * process could not start, or it reported its state unknown). Only false
+   * is down.
+   */
+  ok: boolean | null;
+  /** With ok null: the check gave no answer within its budget. */
+  pending?: boolean;
   status?: number;
   error?: string;
 }
@@ -50,7 +63,12 @@ export interface ServiceHealthResult {
   repoPath: string;
   mode: "dev-managed" | "managed";
   branch: string | null;
-  health: "ok" | "error" | "unknown";
+  /**
+   * "pending": a check ran but gave no verdict (no answer in time, or it
+   * could not tell). Unlike "unknown" (no check configured) it is not a
+   * reason to proceed; unlike "error" it is not down.
+   */
+  health: "ok" | "error" | "unknown" | "pending";
   healthDetail?: string;
   /** Where the service runs for the env checked (R13). */
   runs?: Placement["runs"];
@@ -175,6 +193,36 @@ export function __setExtraCaForTests(pems: string[] | null): void {
   _extraCa = pems;
 }
 
+/**
+ * One TLS context with the public roots plus extraCa(), built once. Passing
+ * `ca` per request made Node parse ~150 root certificates on every HTTPS
+ * health check, synchronously: tens of ms per check, per poll.
+ */
+/** Build the HTTPS checks' trust context now (web server startup). */
+export function prewarmHealthTls(): void {
+  trustContext();
+}
+
+let _trust: {
+  ca: string[];
+  context: tls.SecureContext;
+  agent: https.Agent;
+} | null = null;
+function trustContext(): { context: tls.SecureContext; agent: https.Agent } {
+  const extra = extraCa();
+  if (_trust?.ca !== extra) {
+    _trust?.agent.destroy();
+    _trust = {
+      ca: extra,
+      context: tls.createSecureContext({
+        ca: [...tls.rootCertificates, ...extra],
+      }),
+      agent: new https.Agent({ keepAlive: false }),
+    };
+  }
+  return _trust;
+}
+
 /**
  * HTTPS check. Connects to the c-ares-resolved IP (see above) but keeps the
  * hostname for SNI and certificate verification; rewriting the URL to the IP,
@@ -201,7 +249,12 @@ async function pingHttps(parsed: URL, timeoutMs: number): Promise<PingResult> {
       {
         method: "GET",
         servername: isIpLiteral(host) ? undefined : host,
-        ca: [...tls.rootCertificates, ...extraCa()],
+        // tls.connect's option, passed through by the agent (not in the
+        // https.RequestOptions type).
+        ...({ secureContext: trustContext().context } as https.RequestOptions),
+        // Its own agent: a socket kept alive under one trust set must never
+        // be reused for a check under another.
+        agent: trustContext().agent,
         lookup: (_h, opts, cb) => {
           // Node may ask for all addresses (autoSelectFamily).
           if ((opts as { all?: boolean }).all)
@@ -374,40 +427,185 @@ export async function pingShell(
   cmd: string,
   extraEnv: Record<string, string> = {},
   cwd?: string,
+  opts: {
+    /** The answer budget (default CLI_SPAWN_TIMEOUT_MS). */
+    timeoutMs?: number;
+  } = {},
+): Promise<PingResult> {
+  // Bounded concurrency: a health cycle starts every check at once, and a
+  // distribution's shell checks are processes. Queued checks start their
+  // budget when they start, not while they wait.
+  const release = await shellSlots.acquire();
+  try {
+    return await runShellCheck(cmd, extraEnv, cwd, opts);
+  } finally {
+    release();
+  }
+}
+
+/** A counting semaphore. */
+class Slots {
+  private free: number;
+  private waiting: (() => void)[] = [];
+  constructor(readonly size: number) {
+    this.free = size;
+  }
+  async acquire(): Promise<() => void> {
+    if (this.free > 0) this.free--;
+    else await new Promise<void>((r) => this.waiting.push(r));
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      const next = this.waiting.shift();
+      if (next) next();
+      else this.free++;
+    };
+  }
+}
+
+/** Shell checks running at once: max(2, cpus/2). */
+export const SHELL_CHECK_CONCURRENCY = Math.max(
+  2,
+  Math.floor(os.cpus().length / 2),
+);
+const shellSlots = new Slots(SHELL_CHECK_CONCURRENCY);
+
+function runShellCheck(
+  cmd: string,
+  extraEnv: Record<string, string>,
+  cwd: string | undefined,
+  opts: { timeoutMs?: number },
 ): Promise<PingResult> {
+  const budget = opts.timeoutMs ?? CLI_SPAWN_TIMEOUT_MS;
   return new Promise((resolve) => {
     let settled = false;
     const settle = (r: PingResult): void => {
       if (settled) return;
       settled = true;
+      clearTimeout(timer);
       clearTimeout(guard);
       resolve(r);
     };
-    const proc = spawn(cmd, [], {
-      shell: true,
-      timeout: CLI_SPAWN_TIMEOUT_MS,
-      env: { ...process.env, ...extraEnv },
-      cwd,
-      // Output is never read: piped stdio would block a chatty script once the
-      // 64KB buffer fills, and 'close' would wait on grandchildren holding the
-      // pipes open. Nothing consumes it — don't create it.
-      stdio: "ignore",
+    // A check that has not answered is not a failed check: past the budget
+    // it is pending (ok null), never down. Under load a CLI-based check
+    // (`<cli> stack health`, a full tsx start) can need longer than the
+    // budget on a cold poll, and calling that "down" painted healthy stacks
+    // red until the next poll.
+    const noAnswer = (): PingResult => ({
+      ok: null,
+      pending: true,
+      error: `no answer within ${Math.round(budget / 1000)}s`,
     });
+    let proc: ReturnType<typeof spawn>;
+    try {
+      proc = spawn(cmd, [], {
+        shell: true,
+        env: { ...process.env, ...extraEnv },
+        cwd,
+        // Output is never read: piped stdio would block a chatty script once
+        // the 64KB buffer fills, and 'close' would wait on grandchildren
+        // holding the pipes open. Nothing consumes it — don't create it.
+        stdio: "ignore",
+      });
+    } catch (err) {
+      resolve({ ok: null, error: (err as Error).message });
+      return;
+    }
+    const timer = setTimeout(() => {
+      settle(noAnswer());
+      proc.kill("SIGTERM");
+    }, budget);
     // 'exit', NOT 'close': close waits for stdio to drain, which a killed
     // shell's surviving grandchildren can hold open forever — one wedged
     // health check then hangs every /api/health poll for good. The guard
-    // timer backstops even a spawn timeout that never fires.
-    const guard = setTimeout(
-      () => settle({ ok: false, error: "health check timed out" }),
-      CLI_SPAWN_TIMEOUT_MS + 1_000,
-    );
-    proc.on("exit", (code) =>
-      settle(code === 0 ? { ok: true } : { ok: false, error: `exit ${code}` }),
-    );
-    proc.on("error", (err) => settle({ ok: false, error: err.message }));
+    // timer backstops even a kill that never lands.
+    const guard = setTimeout(() => settle(noAnswer()), budget + 1_000);
+    proc.on("exit", (code, signal) => {
+      if (code === 0) return settle({ ok: true });
+      // Killed (by the budget above, or from outside): it never answered.
+      if (code === null)
+        return settle({ ...noAnswer(), error: `killed (${signal})` });
+      settle({ ok: false, error: `exit ${code}` });
+    });
+    // The check could not be started (e.g. EAGAIN under load): no verdict.
+    proc.on("error", (err) => settle({ ok: null, error: err.message }));
   });
 }
 
+// ── Timeouts: pending first, down when repeated ─────────────────────────────
+// A check that gets no answer within its budget (an HTTP timeout, a shell
+// check past CLI_SPAWN_TIMEOUT_MS) has not completed: one such cycle is
+// pending, not down. A cold first probe on a loaded machine (cold docker, a
+// tsx-started `stack health`, cold TLS) can miss the budget while the service
+// is fine, and reporting that as down painted healthy repos red until the
+// next poll. Two consecutive timeouts are down: a hung service still shows.
+// The web health row counts cycles (TimeoutStreaks); a one-shot caller
+// (`<cli> status`, checkServiceHealth) re-probes once at once instead, so
+// both say the same thing about the same service.
+
+/** Whether a ping got no answer within its budget. */
+export function timedOut(r: PingResult): boolean {
+  return r.pending === true || (r.ok === false && r.error === "timeout");
+}
+
+/** A timeout's verdict: pending on its first cycle, down on a repeat. */
+function timeoutVerdict(r: PingResult, repeated: boolean): PingResult {
+  const error = r.error ?? "timeout";
+  return repeated
+    ? { ok: false, error: `${error} (twice in a row)` }
+    : { ok: null, pending: true, error };
+}
+
+/** Consecutive timed-out cycles per check, for a caller that polls. */
+export class TimeoutStreaks {
+  private streak = new Map<string, number>();
+  /** `r` as this cycle's verdict for check `key`. */
+  judge(key: string, r: PingResult): PingResult {
+    if (!timedOut(r)) {
+      this.streak.delete(key);
+      return r;
+    }
+    const n = (this.streak.get(key) ?? 0) + 1;
+    this.streak.set(key, n);
+    return timeoutVerdict(r, n >= 2);
+  }
+}
+
+/** Probe once; on a timeout, probe again at once: down only if it times out twice. */
+export async function pingTwiceOnTimeout(
+  probe: () => Promise<PingResult>,
+): Promise<PingResult> {
+  const first = await probe();
+  if (!timedOut(first)) return first;
+  const second = await probe();
+  return timedOut(second) ? timeoutVerdict(second, true) : second;
+}
+
+/**
+ * Run a shell health check. Den's own compose check (the `<cli> stack
+ * health` command it generates, `compose.cmd`) is answered in-process from
+ * the shared docker listing: no CLI process per check. Any other command
+ * (a distribution's own) runs as a shell command.
+ */
+export async function pingShellCheck(
+  cmd: string,
+  extraEnv: Record<string, string>,
+  cwd: string | undefined,
+  compose: ComposeHealthTarget | null,
+): Promise<PingResult> {
+  if (!compose || cmd !== compose.cmd) return pingShell(cmd, extraEnv, cwd);
+  const v = await composeHealth(compose);
+  if (v.ok === true) return { ok: true };
+  if (v.ok === false) return { ok: false, error: v.detail };
+  return { ok: null, error: v.detail, ...(v.pending ? { pending: true } : {}) };
+}
+
+/** A ping result as a ServiceHealthResult health state. */
+export function healthState(r: PingResult): ServiceHealthResult["health"] {
+  return r.ok === true ? "ok" : r.ok === false ? "error" : "pending";
+}
+
 // ── Per-service health check ──────────────────────────────────────────────────
 
 /**
@@ -445,9 +643,9 @@ export async function checkServiceHealth(
     };
   }
 
-  const composeHealth =
+  const compose =
     env === "local"
-      ? composeHealthCmd(
+      ? composeHealthTarget(
           manifest,
           resolved.subsystem,
           resolved.repoName,
@@ -458,7 +656,7 @@ export async function checkServiceHealth(
     manifest,
     resolved.subsystem,
     env,
-    composeHealth,
+    compose?.cmd ?? null,
   );
   const placed: Pick<ServiceHealthResult, "runs" | "target"> = {
     runs: plan.placement.runs,
@@ -509,21 +707,22 @@ export async function checkServiceHealth(
     };
   }
 
-  const result =
+  const result = await pingTwiceOnTimeout(() =>
     healthCfg.type === "shell"
-      ? await pingShell(healthCfg.cmd, {}, resolved.repoPath)
-      : await pingHttp(
+      ? pingShellCheck(healthCfg.cmd, {}, resolved.repoPath, compose)
+      : pingHttp(
           healthCfg.url,
           env === "local" && plan.placement.runs !== "remote"
             ? HEALTH_CHECK_TIMEOUT_MS
             : HOSTED_HEALTH_CHECK_TIMEOUT_MS,
           plan.expect,
-        );
+        ),
+  );
 
   return {
     ...base,
     ...placed,
-    health: result.ok ? "ok" : "error",
+    health: healthState(result),
     healthDetail:
       result.error ??
       (!result.ok && result.status !== undefined

package/src/shared/environment-manager/opctl.ts

@@ -10,7 +10,7 @@
  *     repo's dev/up op, captured at spawn time and/or parsed on demand.
  */
 
-import { execFileSync, execSync } from "child_process";
+import { execFile, execFileSync, execSync } from "child_process";
 import * as fs from "fs";
 import * as path from "path";
 import { parse as parseYaml } from "yaml";
@@ -333,3 +333,85 @@ export function diffSpawnedContainers(
   for (const c of after) if (!had.has(c.base)) fresh.add(c.base);
   return [...fresh];
 }
+
+// ── Async variants (the web server) ──────────────────────────────────────────
+// The sync calls above block the event loop for the whole `docker ps` /
+// `opctl container ls` (hundreds of ms on a loaded machine). The server uses
+// these instead; concurrent callers share one in-flight call.
+
+function execOut(cmd: string, args: string[]): Promise<string | null> {
+  return new Promise((resolve) => {
+    execFile(
+      cmd,
+      args,
+      { encoding: "utf-8", timeout: 5_000, maxBuffer: 16 * 1024 * 1024 },
+      (err, stdout) => resolve(err ? null : stdout),
+    );
+  });
+}
+
+function shared<T>(run: () => Promise<T>): () => Promise<T> {
+  let inFlight: Promise<T> | null = null;
+  return () =>
+    (inFlight ??= run().finally(() => {
+      inFlight = null;
+    }));
+}
+
+/** Whether `opctl` is on PATH, without spawning a shell. Cached. */
+export async function isOpctlAvailableAsync(): Promise<boolean> {
+  if (opctlAvailableCache !== null) return opctlAvailableCache;
+  for (const dir of (process.env.PATH ?? "").split(path.delimiter)) {
+    if (!dir) continue;
+    try {
+      await fs.promises.access(path.join(dir, "opctl"), fs.constants.X_OK);
+      return (opctlAvailableCache = true);
+    } catch {
+      /* next */
+    }
+  }
+  return (opctlAvailableCache = false);
+}
+
+/** runningDockerNames, async; null when docker is unavailable. */
+export const runningDockerNamesAsync = shared(
+  async (): Promise<string[] | null> => {
+    const out = await execOut("docker", ["ps", "--format", "{{.Names}}"]);
+    return out === null
+      ? null
+      : out
+          .split("\n")
+          .map((l) => l.trim())
+          .filter(Boolean);
+  },
+);
+
+/** listRunningOpctlContainers, async. */
+export const listRunningOpctlContainersAsync = shared(
+  async (): Promise<{ name: string; base: string }[]> => {
+    const byBase = new Map<string, { name: string; base: string }>();
+    const [opctlOut, dockerNames] = await Promise.all([
+      isOpctlAvailableAsync().then((ok) =>
+        ok ? execOut("opctl", ["container", "ls"]) : null,
+      ),
+      runningDockerNamesAsync(),
+    ]);
+    if (opctlOut !== null) {
+      const lines = opctlOut.split("\n");
+      const firstToken = lines[0]?.trim().split(/\s+/)[0];
+      const body =
+        firstToken === "NAME" || firstToken === "ID" ? lines.slice(1) : lines;
+      for (const line of body) {
+        const name = line.trim().split(/\s+/)[0];
+        if (!name) continue;
+        const base = name.replace(/-[0-9a-f]{8}$/i, "");
+        byBase.set(base, { name, base });
+      }
+    }
+    for (const name of dockerNames ?? []) {
+      const m = name.match(DOCKER_OPCTL_NAME);
+      if (m && !byBase.has(m[1])) byBase.set(m[1], { name, base: m[1] });
+    }
+    return [...byBase.values()];
+  },
+);

package/src/shared/environment-manager/service-runner.ts

@@ -44,6 +44,8 @@ import {
   inspectOpctlContainer,
   isOpctlAvailable,
   listRunningOpctlContainers,
+  listRunningOpctlContainersAsync,
+  isOpctlAvailableAsync,
 } from "./opctl.js";
 import {
   getActiveWorkspace,
@@ -357,6 +359,24 @@ export function activeDaemonOwners(
   return (record?.workspaces ?? []).filter((w) => active.has(w));
 }
 
+/** Whether workspace `name`'s current config lists `daemon`'s service. */
+function workspaceContainsDaemon(
+  name: string,
+  daemon: Pick<DaemonRecord, "address" | "repoName">,
+): boolean {
+  const services = getWorkspaceByName(name)?.config.services ?? {};
+  return daemon.address in services || daemon.repoName in services;
+}
+
+/** Replace a running daemon's owners (re-derived from membership). */
+function setDaemonOwners(address: string, workspaces: string[]): void {
+  loadDaemonsFromFile();
+  const existing = _daemonRegistry.get(address);
+  if (!existing) return;
+  _daemonRegistry.set(address, { ...existing, workspaces });
+  saveDaemonsFile(getDaemonsFile());
+}
+
 /** Outcome of releasing one daemon during a workspace deactivation. */
 export interface DaemonRelease {
   address: string;
@@ -382,15 +402,19 @@ export async function releaseWorkspaceDaemons(
   const owned = listDaemons().filter((d) =>
     (d.workspaces ?? []).includes(workspace),
   );
-  const active = new Set(getActiveWorkspaces());
+  const active = getActiveWorkspaces();
   const releases: DaemonRelease[] = [];
 
   for (const daemon of owned) {
-    const remaining = activeDaemonOwners(daemon, active).filter(
-      (w) => w !== workspace,
+    // Owners re-derived from the CURRENT membership, not the ones recorded at
+    // start: a repo moved into another workspace while its daemon ran belongs
+    // to that workspace now, and one moved out no longer belongs to this one.
+    // Stopped only when no active workspace still contains it.
+    const remaining = active.filter(
+      (w) => w !== workspace && workspaceContainsDaemon(w, daemon),
     );
     if (remaining.length > 0) {
-      detachDaemonOwner(daemon.address, workspace);
+      setDaemonOwners(daemon.address, remaining);
       onLine(
         `detached "${workspace}" from ${daemon.address} — still running for [${remaining.join(", ")}]`,
       );
@@ -639,6 +663,27 @@ export function pruneDeadDaemons(): string[] {
   const running = isOpctlAvailable()
     ? new Set(listRunningOpctlContainers().map((c) => c.base))
     : null;
+  return pruneAgainst(daemons, running);
+}
+
+/**
+ * pruneDeadDaemons without blocking: the container listing runs as async
+ * child processes (the web server's startup, which used to hold the event
+ * loop for the `docker ps`).
+ */
+export async function pruneDeadDaemonsAsync(): Promise<string[]> {
+  const daemons = listDaemons();
+  if (daemons.length === 0) return [];
+  const running = (await isOpctlAvailableAsync())
+    ? new Set((await listRunningOpctlContainersAsync()).map((c) => c.base))
+    : null;
+  return pruneAgainst(daemons, running);
+}
+
+function pruneAgainst(
+  daemons: DaemonRecord[],
+  running: Set<string> | null,
+): string[] {
   const pruned: string[] = [];
   for (const d of daemons) {
     try {

package/src/shared/git/origin-url.ts

@@ -0,0 +1,77 @@
+/**
+ * A checkout's `origin` URL, read from its git config file instead of by
+ * running `git`. Callers on the web server's request path (manifest
+ * resolution for every repo without its own manifest) used to spawn
+ * `git config --get remote.origin.url` synchronously, one repo after another:
+ * ~40-100ms each, which held the event loop for a second on a cold server.
+ *
+ * Same answer as `git -C <dir> config --get remote.origin.url` for ordinary
+ * checkouts and linked worktrees: the repo is found by walking up from `dir`
+ * to the first `.git` (a directory, or a `gitdir:` file), and a worktree's
+ * config is its common dir's.
+ */
+import * as fs from "fs";
+import * as path from "path";
+
+/** The git directory for `start` (walking up), or null outside a repo. */
+export function findGitDir(start: string): string | null {
+  let dir = path.resolve(start);
+  for (;;) {
+    const dotGit = path.join(dir, ".git");
+    try {
+      const st = fs.statSync(dotGit);
+      if (st.isDirectory()) return dotGit;
+      if (st.isFile()) {
+        const m = /^gitdir:\s*(.+?)\s*$/m.exec(
+          fs.readFileSync(dotGit, "utf-8"),
+        );
+        if (m) return path.resolve(dir, m[1]);
+      }
+    } catch {
+      /* no .git here: keep walking */
+    }
+    const parent = path.dirname(dir);
+    if (parent === dir) return null;
+    dir = parent;
+  }
+}
+
+/** The `url` of `[remote "origin"]` in a git config file's text, or null. */
+export function originUrlFromConfig(text: string): string | null {
+  let inOrigin = false;
+  for (const raw of text.split(/\r?\n/)) {
+    const line = raw.trim();
+    if (!line || line.startsWith("#") || line.startsWith(";")) continue;
+    if (line.startsWith("[")) {
+      inOrigin = /^\[\s*remote\s+"origin"\s*\]$/i.test(line);
+      continue;
+    }
+    if (!inOrigin) continue;
+    const m = /^url\s*=\s*(.*)$/i.exec(line);
+    if (m) {
+      const value = m[1].replace(/\s+[#;].*$/, "").trim();
+      return value.replace(/^"(.*)"$/, "$1") || null;
+    }
+  }
+  return null;
+}
+
+/** `dir`'s origin URL, or null (not a checkout, or no origin). Never spawns. */
+export function readOriginUrl(dir: string): string | null {
+  const gitDir = findGitDir(dir);
+  if (!gitDir) return null;
+  let common = gitDir;
+  try {
+    const rel = fs.readFileSync(path.join(gitDir, "commondir"), "utf-8").trim();
+    if (rel) common = path.resolve(gitDir, rel);
+  } catch {
+    /* not a linked worktree */
+  }
+  try {
+    return originUrlFromConfig(
+      fs.readFileSync(path.join(common, "config"), "utf-8"),
+    );
+  } catch {
+    return null;
+  }
+}