@monster-orange/den 0.2.4 → 0.2.6

package/src/shared/environment-manager/health.ts

@@ -9,8 +9,16 @@ import { hasRuntime, repoKind } from "../repo-kind.js";
 import * as https from "https";
 import * as tls from "tls";
 import { readEdgeConfig } from "../edge/config.js";
-import { composeHealthCmd } from "../stacks/manifest-commands.js";
+import {
+  composeHealth,
+  composeHealthTarget,
+  containerHealth,
+  type ComposeHealthTarget,
+  type ComposeVerdict,
+  type ContainerHealthTarget,
+} from "../stacks/compose-health.js";
 import * as fs from "fs";
+import * as os from "os";
 import * as path from "path";
 import * as dns from "dns";
 import { spawn } from "child_process";
@@ -33,12 +41,24 @@ import {
   listWorkspaceServices,
   readManifestAnyBranch,
 } from "./workspaces.js";
-import type { ResolvedService, ServiceCatalog } from "../types/index.js";
+import type {
+  ResolvedService,
+  ServiceCatalog,
+  ServiceHealthConfig,
+} from "../types/index.js";
 import type { Environment } from "../types/index.js";
 
 import { product } from "../product.js";
 export interface PingResult {
-  ok: boolean;
+  /**
+   * true: the check answered healthy. false: it COMPLETED and failed. null:
+   * it did not complete or could not tell (no answer within its budget, its
+   * process could not start, or it reported its state unknown). Only false
+   * is down.
+   */
+  ok: boolean | null;
+  /** With ok null: the check gave no answer within its budget. */
+  pending?: boolean;
   status?: number;
   error?: string;
 }
@@ -50,7 +70,12 @@ export interface ServiceHealthResult {
   repoPath: string;
   mode: "dev-managed" | "managed";
   branch: string | null;
-  health: "ok" | "error" | "unknown";
+  /**
+   * "pending": a check ran but gave no verdict (no answer in time, or it
+   * could not tell). Unlike "unknown" (no check configured) it is not a
+   * reason to proceed; unlike "error" it is not down.
+   */
+  health: "ok" | "error" | "unknown" | "pending";
   healthDetail?: string;
   /** Where the service runs for the env checked (R13). */
   runs?: Placement["runs"];
@@ -409,40 +434,212 @@ export async function pingShell(
   cmd: string,
   extraEnv: Record<string, string> = {},
   cwd?: string,
+  opts: {
+    /** The answer budget (default CLI_SPAWN_TIMEOUT_MS). */
+    timeoutMs?: number;
+  } = {},
+): Promise<PingResult> {
+  // Bounded concurrency: a health cycle starts every check at once, and a
+  // distribution's shell checks are processes. Queued checks start their
+  // budget when they start, not while they wait.
+  const release = await shellSlots.acquire();
+  try {
+    return await runShellCheck(cmd, extraEnv, cwd, opts);
+  } finally {
+    release();
+  }
+}
+
+/** A counting semaphore. */
+class Slots {
+  private free: number;
+  private waiting: (() => void)[] = [];
+  constructor(readonly size: number) {
+    this.free = size;
+  }
+  async acquire(): Promise<() => void> {
+    if (this.free > 0) this.free--;
+    else await new Promise<void>((r) => this.waiting.push(r));
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      const next = this.waiting.shift();
+      if (next) next();
+      else this.free++;
+    };
+  }
+}
+
+/** Shell checks running at once: max(2, cpus/2). */
+export const SHELL_CHECK_CONCURRENCY = Math.max(
+  2,
+  Math.floor(os.cpus().length / 2),
+);
+const shellSlots = new Slots(SHELL_CHECK_CONCURRENCY);
+
+function runShellCheck(
+  cmd: string,
+  extraEnv: Record<string, string>,
+  cwd: string | undefined,
+  opts: { timeoutMs?: number },
 ): Promise<PingResult> {
+  const budget = opts.timeoutMs ?? CLI_SPAWN_TIMEOUT_MS;
   return new Promise((resolve) => {
     let settled = false;
     const settle = (r: PingResult): void => {
       if (settled) return;
       settled = true;
+      clearTimeout(timer);
       clearTimeout(guard);
       resolve(r);
     };
-    const proc = spawn(cmd, [], {
-      shell: true,
-      timeout: CLI_SPAWN_TIMEOUT_MS,
-      env: { ...process.env, ...extraEnv },
-      cwd,
-      // Output is never read: piped stdio would block a chatty script once the
-      // 64KB buffer fills, and 'close' would wait on grandchildren holding the
-      // pipes open. Nothing consumes it — don't create it.
-      stdio: "ignore",
+    // A check that has not answered is not a failed check: past the budget
+    // it is pending (ok null), never down. Under load a CLI-based check
+    // (`<cli> stack health`, a full tsx start) can need longer than the
+    // budget on a cold poll, and calling that "down" painted healthy stacks
+    // red until the next poll.
+    const noAnswer = (): PingResult => ({
+      ok: null,
+      pending: true,
+      error: `no answer within ${Math.round(budget / 1000)}s`,
     });
+    let proc: ReturnType<typeof spawn>;
+    try {
+      proc = spawn(cmd, [], {
+        shell: true,
+        env: { ...process.env, ...extraEnv },
+        cwd,
+        // Output is never read: piped stdio would block a chatty script once
+        // the 64KB buffer fills, and 'close' would wait on grandchildren
+        // holding the pipes open. Nothing consumes it — don't create it.
+        stdio: "ignore",
+      });
+    } catch (err) {
+      resolve({ ok: null, error: (err as Error).message });
+      return;
+    }
+    const timer = setTimeout(() => {
+      settle(noAnswer());
+      proc.kill("SIGTERM");
+    }, budget);
     // 'exit', NOT 'close': close waits for stdio to drain, which a killed
     // shell's surviving grandchildren can hold open forever — one wedged
     // health check then hangs every /api/health poll for good. The guard
-    // timer backstops even a spawn timeout that never fires.
-    const guard = setTimeout(
-      () => settle({ ok: false, error: "health check timed out" }),
-      CLI_SPAWN_TIMEOUT_MS + 1_000,
-    );
-    proc.on("exit", (code) =>
-      settle(code === 0 ? { ok: true } : { ok: false, error: `exit ${code}` }),
-    );
-    proc.on("error", (err) => settle({ ok: false, error: err.message }));
+    // timer backstops even a kill that never lands.
+    const guard = setTimeout(() => settle(noAnswer()), budget + 1_000);
+    proc.on("exit", (code, signal) => {
+      if (code === 0) return settle({ ok: true });
+      // Killed (by the budget above, or from outside): it never answered.
+      if (code === null)
+        return settle({ ...noAnswer(), error: `killed (${signal})` });
+      settle({ ok: false, error: `exit ${code}` });
+    });
+    // The check could not be started (e.g. EAGAIN under load): no verdict.
+    proc.on("error", (err) => settle({ ok: null, error: err.message }));
   });
 }
 
+// ── Timeouts: pending first, down when repeated ─────────────────────────────
+// A check that gets no answer within its budget (an HTTP timeout, a shell
+// check past CLI_SPAWN_TIMEOUT_MS) has not completed: one such cycle is
+// pending, not down. A cold first probe on a loaded machine (cold docker, a
+// tsx-started `stack health`, cold TLS) can miss the budget while the service
+// is fine, and reporting that as down painted healthy repos red until the
+// next poll. Two consecutive timeouts are down: a hung service still shows.
+// The web health row counts cycles (TimeoutStreaks); a one-shot caller
+// (`<cli> status`, checkServiceHealth) re-probes once at once instead, so
+// both say the same thing about the same service.
+
+/** Whether a ping got no answer within its budget. */
+export function timedOut(r: PingResult): boolean {
+  return r.pending === true || (r.ok === false && r.error === "timeout");
+}
+
+/** A timeout's verdict: pending on its first cycle, down on a repeat. */
+function timeoutVerdict(r: PingResult, repeated: boolean): PingResult {
+  const error = r.error ?? "timeout";
+  return repeated
+    ? { ok: false, error: `${error} (twice in a row)` }
+    : { ok: null, pending: true, error };
+}
+
+/** Consecutive timed-out cycles per check, for a caller that polls. */
+export class TimeoutStreaks {
+  private streak = new Map<string, number>();
+  /** `r` as this cycle's verdict for check `key`. */
+  judge(key: string, r: PingResult): PingResult {
+    if (!timedOut(r)) {
+      this.streak.delete(key);
+      return r;
+    }
+    const n = (this.streak.get(key) ?? 0) + 1;
+    this.streak.set(key, n);
+    return timeoutVerdict(r, n >= 2);
+  }
+}
+
+/** Probe once; on a timeout, probe again at once: down only if it times out twice. */
+export async function pingTwiceOnTimeout(
+  probe: () => Promise<PingResult>,
+): Promise<PingResult> {
+  const first = await probe();
+  if (!timedOut(first)) return first;
+  const second = await probe();
+  return timedOut(second) ? timeoutVerdict(second, true) : second;
+}
+
+/**
+ * Run a shell health check. Den's own compose check (the `<cli> stack
+ * health` command it generates, `compose.cmd`) is answered in-process from
+ * the shared docker listing: no CLI process per check. Any other command
+ * (a distribution's own) runs as a shell command.
+ */
+export async function pingShellCheck(
+  cmd: string,
+  extraEnv: Record<string, string>,
+  cwd: string | undefined,
+  compose: ComposeHealthTarget | null,
+): Promise<PingResult> {
+  if (!compose || cmd !== compose.cmd) return pingShell(cmd, extraEnv, cwd);
+  return verdictPing(await composeHealth(compose));
+}
+
+/**
+ * A compose or container verdict as a ping. One conversion for both, so a
+ * docker that could not be asked reads the same: `ok: null`, pending (and so
+ * down on a repeat) only when the listing ran past its budget.
+ */
+function verdictPing(v: ComposeVerdict): PingResult {
+  if (v.ok === true) return { ok: true };
+  if (v.ok === false) return { ok: false, error: v.detail };
+  return { ok: null, error: v.detail, ...(v.pending ? { pending: true } : {}) };
+}
+
+/**
+ * Run a `type: container` check: answered in-process from the shared docker
+ * listing (the one Den's compose checks use), never a docker call of its own.
+ */
+export async function pingContainerCheck(
+  target: ContainerHealthTarget,
+): Promise<PingResult> {
+  return verdictPing(await containerHealth(target));
+}
+
+/** The container a `type: container` health config names. */
+export function containerTarget(
+  h: Extract<ServiceHealthConfig, { type: "container" }>,
+): ContainerHealthTarget {
+  return h.container_name !== undefined
+    ? { container_name: h.container_name }
+    : { project: h.project, service: h.service };
+}
+
+/** A ping result as a ServiceHealthResult health state. */
+export function healthState(r: PingResult): ServiceHealthResult["health"] {
+  return r.ok === true ? "ok" : r.ok === false ? "error" : "pending";
+}
+
 // ── Per-service health check ──────────────────────────────────────────────────
 
 /**
@@ -480,9 +677,9 @@ export async function checkServiceHealth(
     };
   }
 
-  const composeHealth =
+  const compose =
     env === "local"
-      ? composeHealthCmd(
+      ? composeHealthTarget(
           manifest,
           resolved.subsystem,
           resolved.repoName,
@@ -493,7 +690,7 @@ export async function checkServiceHealth(
     manifest,
     resolved.subsystem,
     env,
-    composeHealth,
+    compose?.cmd ?? null,
   );
   const placed: Pick<ServiceHealthResult, "runs" | "target"> = {
     runs: plan.placement.runs,
@@ -544,21 +741,24 @@ export async function checkServiceHealth(
     };
   }
 
-  const result =
+  const result = await pingTwiceOnTimeout(() =>
     healthCfg.type === "shell"
-      ? await pingShell(healthCfg.cmd, {}, resolved.repoPath)
-      : await pingHttp(
-          healthCfg.url,
-          env === "local" && plan.placement.runs !== "remote"
-            ? HEALTH_CHECK_TIMEOUT_MS
-            : HOSTED_HEALTH_CHECK_TIMEOUT_MS,
-          plan.expect,
-        );
+      ? pingShellCheck(healthCfg.cmd, {}, resolved.repoPath, compose)
+      : healthCfg.type === "container"
+        ? pingContainerCheck(containerTarget(healthCfg))
+        : pingHttp(
+            healthCfg.url,
+            env === "local" && plan.placement.runs !== "remote"
+              ? HEALTH_CHECK_TIMEOUT_MS
+              : HOSTED_HEALTH_CHECK_TIMEOUT_MS,
+            plan.expect,
+          ),
+  );
 
   return {
     ...base,
     ...placed,
-    health: result.ok ? "ok" : "error",
+    health: healthState(result),
     healthDetail:
       result.error ??
       (!result.ok && result.status !== undefined

package/src/shared/manifest/placement.ts

@@ -52,7 +52,7 @@ export type IsProduction = (env: string) => boolean;
 const LOCAL = "local";
 
 function isHttp(h: ServiceHealthConfig | undefined): h is { url: string } {
-  return !!h && h.type !== "shell" && typeof h.url === "string";
+  return !!h && "url" in h && typeof h.url === "string";
 }
 
 function hasCommands(

package/src/shared/manifest/schema.ts

@@ -32,7 +32,54 @@ const commandDef = z.union([
 //   { url: "http://..." }                         — implicit HTTP
 //   { type: "http", url: "http://..." }           — explicit HTTP
 //   { type: "shell", cmd: "pg_isready ..." }      — shell
+//   { type: "container", project, service }       — a compose container
+//   { type: "container", container_name }         — a container by name
+const CONTAINER_HEALTH_KEYS = new Set([
+  "type",
+  "project",
+  "service",
+  "container_name",
+  "label",
+]);
+export const CONTAINER_HEALTH_SHAPE =
+  'container checks need "project" and "service" (compose labels), or "container_name"';
+const containerHealth = z
+  .object({
+    type: z.literal("container"),
+    project: z.string().min(1).optional(),
+    service: z.string().min(1).optional(),
+    container_name: z.string().min(1).optional(),
+    label: z.string().optional(),
+  })
+  .passthrough()
+  .superRefine((h, ctx) => {
+    const unknown = Object.keys(h).filter((k) => !CONTAINER_HEALTH_KEYS.has(k));
+    if (unknown.length)
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `unknown key${unknown.length > 1 ? "s" : ""} ${unknown.map((k) => `"${k}"`).join(", ")} in a container check: ${CONTAINER_HEALTH_SHAPE}`,
+      });
+    const byLabels = h.project !== undefined || h.service !== undefined;
+    const byName = h.container_name !== undefined;
+    if (byLabels && byName)
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `a container check matches by "project"/"service" or by "container_name", not both`,
+      });
+    else if (byLabels && (h.project === undefined || h.service === undefined))
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `a container check by compose labels needs both "project" and "service"`,
+      });
+    else if (!byLabels && !byName)
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: CONTAINER_HEALTH_SHAPE,
+      });
+  });
+
 const healthConfig = z.union([
+  containerHealth,
   // Shell health check — must have cmd
   z.object({
     type: z.literal("shell"),

package/src/shared/manifest/service-links.ts

@@ -47,7 +47,7 @@ export function openUrlFor(
   const declared = httpUrl(svc.urls?.[env]);
   if (declared) return declared;
   const health = svc.health?.[env];
-  if (!health || ("type" in health && health.type === "shell")) return null;
+  if (!health || !("url" in health)) return null;
   return httpOrigin((health as { url?: string }).url);
 }
 

package/src/shared/manifest/validate-manifest.ts

@@ -57,7 +57,7 @@ function suggestionFor(issue: z.ZodIssue): string | undefined {
 
   // health with missing url/cmd
   if (pathStr.includes("health")) {
-    return 'HTTP checks need "url"; shell checks need "type: shell" and "cmd"';
+    return 'HTTP checks need "url"; shell checks need "type: shell" and "cmd"; container checks need "type: container" and "project" + "service", or "container_name"';
   }
 
   return undefined;

package/src/shared/plural.ts

@@ -0,0 +1,13 @@
+/**
+ * Counts in output: "1 repo", "3 repos", never "1 repos" or "repo(s)".
+ */
+
+/** `word` in the form `n` takes: "repo" for 1, "repos" (or `many`) otherwise. */
+export function noun(n: number, word: string, many = `${word}s`): string {
+  return n === 1 ? word : many;
+}
+
+/** `n` and its noun: plural(1, "repo") → "1 repo", plural(2, "repo") → "2 repos". */
+export function plural(n: number, word: string, many?: string): string {
+  return `${n} ${noun(n, word, many)}`;
+}

package/src/shared/service-url.ts

@@ -24,11 +24,8 @@ function healthUrlForConfig(healthConfig?: ServiceHealthConfig): string | null {
     return null;
   }
 
-  if ("type" in healthConfig && healthConfig.type === "shell") {
-    return null;
-  }
-
-  return healthConfig.url;
+  // Shell and container checks have no URL.
+  return "url" in healthConfig ? healthConfig.url : null;
 }
 
 function normalizeServiceBaseUrl(