@monster-orange/den 0.2.3 → 0.2.5

package/src/ui/server/index.ts

@@ -10,7 +10,10 @@
  *   GET /api/run?pillar=X&...    — SSE stream: runs tests, emits output line-by-line
  */
 
-import { composeHealthCmd } from "../../shared/stacks/manifest-commands.js";
+import {
+  composeHealthTarget,
+  type ComposeHealthTarget,
+} from "../../shared/stacks/compose-health.js";
 import { healthCheckFields } from "./health-row.js";
 import { hasRuntime, repoKind } from "../../shared/repo-kind.js";
 import * as http from "http";
@@ -39,7 +42,6 @@ import type {
 import { getRunDonePayload } from "./perf-report.js";
 import { readCatalog } from "../../shared/environment-manager/catalog.js";
 import {
-  readWorkspacesFile,
   getActiveWorkspace,
   getActiveWorkspaces,
   getFocusedWorkspaceName,
@@ -49,10 +51,7 @@ import {
   resolveDepAddress,
   parseAddress,
   readManifestAnyBranch,
-  noAutostartRepos,
 } from "../../shared/environment-manager/workspaces.js";
-import { deriveGraph } from "../../shared/environment-manager/derivation/derive-graph.js";
-import { orderForDown } from "../../shared/environment-manager/derivation/derived-up.js";
 import { findServiceContainer } from "../../shared/environment-manager/service-container.js";
 import {
   resolveHistorySource,
@@ -64,7 +63,9 @@ import {
 } from "./log-history.js";
 import {
   pingHttp,
-  pingShell,
+  pingShellCheck,
+  TimeoutStreaks,
+  prewarmHealthTls,
 } from "../../shared/environment-manager/health.js";
 import {
   runServiceCommand,
@@ -74,7 +75,7 @@ import {
   unregisterDaemon,
   checkServiceHealth,
   loadDaemonsFromFile,
-  pruneDeadDaemons,
+  pruneDeadDaemonsAsync,
   isInteractiveServiceCommand,
   rewriteLegacyOpctlKillCommandForContainer,
   shouldDetachServiceCommand,
@@ -142,6 +143,10 @@ import {
 import { handleReadonlyRoutes } from "./readonly-routes.js";
 import { serveUiAsset } from "./ui-assets.js";
 import { graphPrefs, mergeGraphPrefs } from "./graph-prefs.js";
+import { singleFlight } from "./single-flight.js";
+import { graphOrder } from "./graph-order.js";
+import { clearReadCache, enableReadCache } from "../../shared/read-cache.js";
+import { buildServicesList, type ServicesList } from "./services-list.js";
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -166,7 +171,10 @@ interface HealthResult {
   repoPath: string;
   mode: "dev-managed" | "managed";
   isEntrypoint: boolean;
+  /** true up; false a completed check failed (down); null not known. */
   ok: boolean | null;
+  /** With ok null: a check ran but has not answered (pending, not down). */
+  pending?: boolean;
   status?: number;
   error?: string;
   /** Where the app opens for this env (open-in-new-tab), or null. */
@@ -208,7 +216,32 @@ function viewerRewrite(
 }
 
 const HEALTH_CACHE_TTL_MS = 8_000;
+/** Consecutive timed-out health cycles per check (see TimeoutStreaks). */
+const healthTimeouts = new TimeoutStreaks();
 const healthCache = new Map<string, { results: HealthResult[]; at: number }>();
+/**
+ * The full-env health run in flight per cache key. Concurrent polls and
+ * streams join it (a stream replays what already landed, then follows), so
+ * each cycle runs every check once: a second concurrent run doubled the
+ * shell checks' processes and pushed them past their budget under load.
+ */
+const healthInFlight = new Map<
+  string,
+  {
+    run: Promise<HealthResult[]>;
+    landed: HealthResult[];
+    listeners: Set<(r: HealthResult) => void>;
+  }
+>();
+/** /api/services builds, shared by concurrent polls and reused briefly. */
+/** The startup sweep of stale daemon records (see below the routes). */
+let startupPrune: Promise<void> = Promise.resolve();
+const SERVICES_CACHE_TTL_MS = 2_000;
+// Manifests, the edge config and the derived graph order are reused across
+// a request and the polls after it (cleared by any mutating request).
+const READ_CACHE_TTL_MS = 5_000;
+enableReadCache(READ_CACHE_TTL_MS);
+const servicesCache = singleFlight<ServicesList>(SERVICES_CACHE_TTL_MS);
 /** A workspace-scoped run covers a different service set than the default one. */
 function healthCacheKey(env: Environment, workspace: string | null): string {
   return `${env}|${workspace ?? "*"}`;
@@ -534,6 +567,15 @@ async function handleRequest(
 ): Promise<void> {
   const url = new URL(req.url ?? "/", `http://localhost:${PORT}`);
   const pathname = url.pathname;
+  // A mutating request may change what /api/services would list (exec and
+  // sync stream over GET, for EventSource).
+  if (
+    (req.method !== "GET" && req.method !== "HEAD") ||
+    /\/(exec|sync)$/.test(pathname)
+  ) {
+    servicesCache.clear();
+    clearReadCache();
+  }
 
   // ── Request logging ────────────────────────────────────────────────────────
   // /api/health fires on the UI's 5s polling interval forever — it dominates
@@ -897,6 +939,8 @@ async function handleRequest(
       org: string;
       /** Codes that count as up for a remote target.url probe. */
       expect?: Expect;
+      /** pingShell options (Den's own `stack health` can report unknown). */
+      compose: ComposeHealthTarget | null;
       meta: Pick<HealthResult, "runs" | "target">;
     }> = [];
     let edgeDomain: string | null = null;
@@ -929,6 +973,7 @@ async function handleRequest(
           openUrl: null,
           links: [],
           org: orgOf(resolved.repoName, catalog),
+          compose: null,
           meta: { runs: "none", target: null },
         });
         continue;
@@ -936,9 +981,9 @@ async function handleRequest(
       const svcCfg = manifest.services?.[resolved.subsystem];
       // A compose-run repo with no declared check gets the stack's own health
       // (container state + Docker healthchecks), as `den status` does.
-      const composeHealth =
+      const compose =
         env === "local"
-          ? composeHealthCmd(
+          ? composeHealthTarget(
               manifest,
               resolved.subsystem,
               resolved.repoName,
@@ -951,7 +996,7 @@ async function handleRequest(
         manifest,
         resolved.subsystem,
         env,
-        composeHealth,
+        compose?.cmd ?? null,
       );
       const healthCfg = plan.check;
       const remote = plan.placement.runs === "remote";
@@ -1010,6 +1055,7 @@ async function handleRequest(
         links: linksFor(manifest, resolved.subsystem, env),
         org: orgOf(resolved.repoName, catalog),
         ...(plan.expect ? { expect: plan.expect } : {}),
+        compose,
         meta: { runs: plan.placement.runs, target: plan.placement.target },
       });
     }
@@ -1019,9 +1065,7 @@ async function handleRequest(
     // direction as the graph (and `den status`). orderForDown is the reverse
     // of the deps-first startup order.
     const rank = new Map(
-      orderForDown(
-        deriveGraph({ scope: "all", workspace: workspaces[0]?.name }),
-      ).map((repo, i) => [repo, i]),
+      graphOrder(workspaces[0]?.name).map((repo, i) => [repo, i]),
     );
     const sortedEntries = [...entries].sort((a, b) => {
       const ra = rank.get(a.repo) ?? Number.MAX_SAFE_INTEGER;
@@ -1041,12 +1085,18 @@ async function handleRequest(
       conventional,
       meta,
       expect,
+      compose,
     }) => {
       if (checkType !== "none") {
         const started = Date.now();
         const ping =
           checkType === "shell"
-            ? await pingShell(target, awsProfileEnv(env), repoPath)
+            ? await pingShellCheck(
+                target,
+                awsProfileEnv(env),
+                repoPath,
+                compose,
+              )
             : await pingHttp(
                 target,
                 env === "local" && meta.runs !== "remote"
@@ -1064,7 +1114,11 @@ async function handleRequest(
           isEntrypoint,
           ...meta,
           ms: Date.now() - started,
-          ...ping,
+          // A first timeout is pending; a repeat on the next cycle is down.
+          ...healthTimeouts.judge(
+            `${env}|${repo}|${service}|${checkType}|${target}`,
+            ping,
+          ),
         };
       }
       // No declared health check: try the conventional probe when eligible.
@@ -1117,6 +1171,37 @@ async function handleRequest(
     return results;
   }
 
+  /** Join the full-env health run in flight for (env, workspace), or start one. */
+  function sharedHealthRun(env: Environment, workspace: string | null) {
+    const key = healthCacheKey(env, workspace);
+    const inFlight = healthInFlight.get(key);
+    if (inFlight) return inFlight;
+    const landed: HealthResult[] = [];
+    const listeners = new Set<(r: HealthResult) => void>();
+    const run = computeHealth(
+      env,
+      null,
+      (r) => {
+        landed.push(r);
+        for (const l of listeners)
+          try {
+            l(r);
+          } catch {
+            /* a gone client must not stop the run */
+          }
+      },
+      workspace,
+    )
+      .then((results) => {
+        healthCache.set(key, { results, at: Date.now() });
+        return results;
+      })
+      .finally(() => healthInFlight.delete(key));
+    const shared = { run, landed, listeners };
+    healthInFlight.set(key, shared);
+    return shared;
+  }
+
   // Full-env health, cached. Every page navigation used to re-run all 30 checks
   // from scratch — and the slowest are shell probes that shell out to the aws
   // CLI (~0.45s each) — so the dots took ~0.6s to resolve on EVERY switch. A
@@ -1143,9 +1228,10 @@ async function handleRequest(
     if (hit && Date.now() - hit.at < HEALTH_CACHE_TTL_MS) {
       return sendJson(res, hit.results.map(forViewer));
     }
-    const results = await computeHealth(env, null, undefined, wsParam);
-    healthCache.set(cacheKey, { results, at: Date.now() });
-    return sendJson(res, results.map(forViewer));
+    return sendJson(
+      res,
+      (await sharedHealthRun(env, wsParam).run).map(forViewer),
+    );
   }
 
   // Same checks, streamed: each result is sent the moment it lands so the fast
@@ -1177,13 +1263,16 @@ async function handleRequest(
     }
 
     try {
-      const results = await computeHealth(
-        env,
-        null,
-        (r) => send({ type: "result", result: forViewer(r) }),
-        wsParam,
-      );
-      healthCache.set(cacheKey, { results, at: Date.now() });
+      const shared = sharedHealthRun(env, wsParam);
+      const forward = (r: HealthResult): void =>
+        send({ type: "result", result: forViewer(r) });
+      for (const r of shared.landed) forward(r);
+      shared.listeners.add(forward);
+      try {
+        await shared.run;
+      } finally {
+        shared.listeners.delete(forward);
+      }
       send({ type: "done", cached: false });
     } catch (e) {
       send({ type: "error", error: (e as Error).message });
@@ -1196,169 +1285,15 @@ async function handleRequest(
     const env = (url.searchParams.get("env") ?? "local") as Environment;
     // Optional: caller may request services for a specific (non-active) workspace.
     const wsOverride = url.searchParams.get("workspace") ?? null;
-
-    // Conventional lifecycle locally: `down` is opctl-conventional (stops
-    // running containers with no declared command, no-ops gracefully), and
-    // `restart` is orchestrated by the exec runner as down → verify → up — so
-    // every service with an `up` can offer both even when the manifest doesn't
-    // declare them.
-    const withDown = (cmds: string[]): string[] => {
-      if (env !== "local") return cmds;
-      const out = [...cmds];
-      if (!out.includes("down")) out.push("down");
-      if (!out.includes("restart") && out.includes("up")) out.push("restart");
-      return out;
-    };
-
-    // Legacy: discovered repos from .den/ symlinks / registered-repos.json
-    const legacyRepos = discoverRepos();
-    const legacyEntries = legacyRepos.flatMap((repo) => {
-      const manifestLayers = loadManifest(repo.repoPath).layers;
-      return Object.entries(repo.manifest.services ?? {}).map(
-        ([svcName, svc]) => ({
-          repo: repo.repoName,
-          service: svcName,
-          address: `${repo.repoName}/${svcName}`,
-          repoPath: repo.repoPath,
-          commands: withDown(Object.keys(svc.commands?.[env] ?? {})),
-          logs: svc.logs?.[env] ?? null,
-          mode: "dev-managed" as const,
-          branch: null as string | null,
-          fromCatalog: false,
-          isEntrypoint: repo.manifest.entrypoint === svcName,
-          autostart: true, // legacy discovery has no workspace entry to toggle
-          deps: (svc.depends_on ?? []).map(depService),
-          cloned: true, // legacy discovery only ever finds on-disk repos
-          manifestLayers,
-          ...servicePlacement(repo.manifest, svcName, env),
-        }),
-      );
-    });
-
-    // Catalog entries (workspace-aware)
-    const catalog = readCatalog();
-    const wsFile = wsOverride ? readWorkspacesFile() : null;
-    const ws = wsOverride
-      ? wsFile?.workspaces[wsOverride]
-        ? { name: wsOverride, config: wsFile!.workspaces[wsOverride] }
-        : null
-      : getActiveWorkspace();
-    const getCurrentBranch = (repoPath: string): string | null => {
-      try {
-        return execSync("git rev-parse --abbrev-ref HEAD", {
-          cwd: repoPath,
-          encoding: "utf-8",
-          stdio: ["ignore", "pipe", "ignore"],
-          timeout: GIT_REMOTE_TIMEOUT_MS,
-        }).trim();
-      } catch {
-        return null;
-      }
-    };
-
-    const noAutostart = ws
-      ? noAutostartRepos(ws.config, catalog)
-      : new Set<string>();
-    const daemonOwners = new Map(
-      listDaemons().map((d) => [d.address, d.workspaces ?? []] as const),
-    );
-    // Modules and libraries have no runtime: no card, health or commands.
-    // They show as contributors on the Applications tab instead.
-    const catalogEntries = ws
-      ? listWorkspaceServices(ws.config, catalog, ws.name)
-          .filter((r) => hasRuntime(repoKind(r.repoPath)))
-          .flatMap((resolved) => {
-            // The manifest and the layers it was built from: repo or held,
-            // plus override when manifests/<repo>.override.yml applied.
-            const { manifest, layers: manifestLayers } = loadManifest(
-              resolved.repoPath,
-            );
-            const svc = manifest?.services?.[resolved.subsystem];
-            if (!manifest || !svc) {
-              // Workspace membership is config-driven: a managed entry
-              // whose clone isn't on disk yet (or a checkout without a usable
-              // manifest) must still render everywhere — flagged `cloned:false`
-              // so the UI can offer Sync/clone instead of dropping the system.
-              // `branch` comes from the workspace entry; never probe git on a
-              // path that may not exist.
-              return [
-                {
-                  repo: resolved.repoName,
-                  service: resolved.subsystem,
-                  address: resolved.address,
-                  repoPath: resolved.repoPath,
-                  commands: [] as string[],
-                  logs: null,
-                  mode: resolved.mode,
-                  branch: resolved.branch,
-                  fromCatalog: true,
-                  isEntrypoint: false,
-                  autostart: !noAutostart.has(resolved.repoName),
-                  owners: daemonOwners.get(resolved.address) ?? [],
-                  deps: [] as string[],
-                  cloned: fs.existsSync(resolved.repoPath),
-                  manifestLayers,
-                  ...servicePlacement(null, resolved.subsystem, env),
-                },
-              ];
-            }
-            return [
-              {
-                repo: resolved.repoName,
-                service: resolved.subsystem,
-                address: resolved.address,
-                repoPath: resolved.repoPath,
-                commands: withDown(Object.keys(svc.commands?.[env] ?? {})),
-                logs: svc.logs?.[env] ?? null,
-                mode: resolved.mode,
-                branch: resolved.branch ?? getCurrentBranch(resolved.repoPath),
-                fromCatalog: true,
-                isEntrypoint: manifest.entrypoint === resolved.subsystem,
-                autostart: !noAutostart.has(resolved.repoName),
-                owners: daemonOwners.get(resolved.address) ?? [],
-                deps: (svc.depends_on ?? []).map((d) =>
-                  resolveDepAddress(depService(d), catalog, ws.config, ws.name),
-                ),
-                cloned: true,
-                manifestLayers,
-                ...servicePlacement(manifest, resolved.subsystem, env),
-              },
-            ];
-          })
-      : [];
-
-    // Merge: catalog entries take precedence over legacy for same repo.
-    // When a workspace filter is active, also exclude legacy entries not in
-    // the workspace (legacy discovery is not workspace-scoped).
-    const catalogRepos = new Set(Object.keys(catalog));
-    const wsServiceKeys = ws ? new Set(Object.keys(ws.config.services)) : null;
-    const merged = [
-      ...legacyEntries.filter((e) => {
-        if (catalogRepos.has(e.repo)) return false;
-        if (wsServiceKeys && !wsServiceKeys.has(e.repo)) return false;
-        return true;
-      }),
-      ...catalogEntries,
-    ];
-
-    // Attach the dependency-graph position so consumers can present systems
-    // top-down (entry repo first, infra-stack last) — the same order the
-    // health row and `den status` use. Added as a FIELD rather than by
-    // re-sorting the array, because four pages consume this payload and some
-    // rely on the current order.
-    const graphRank = new Map(
-      orderForDown(deriveGraph({ scope: "all", workspace: ws?.name })).map(
-        (repo, i) => [repo, i],
-      ),
-    );
-    const infraRepo = readDenConfig()?.graph?.infra?.repo ?? null;
+    // Every open tab polls this: concurrent requests share one build, and a
+    // settled one is reused briefly (any mutating request clears it).
     return sendJson(
       res,
-      merged.map((e) => ({
-        ...e,
-        graphRank: graphRank.get(e.repo) ?? Number.MAX_SAFE_INTEGER,
-        infra: infraRepo !== null && e.repo === infraRepo,
-      })),
+      await servicesCache.get(
+        `${env}|${wsOverride ?? ""}`,
+        () => buildServicesList(env, wsOverride),
+        url.searchParams.get("refresh") === "1",
+      ),
     );
   }
 
@@ -1836,6 +1771,7 @@ async function handleRequest(
 
   // ── API: running daemons list ──────────────────────────────────────────────
   if (pathname === "/api/service/daemons") {
+    await startupPrune;
     const daemons = listDaemons().map((d) => ({
       address: d.address,
       repoName: d.repoName,
@@ -3206,15 +3142,20 @@ server.on("error", (err: NodeJS.ErrnoException) => {
 // service stopped outside den — otherwise reopen as tabs in the web UI and
 // clutter the shutdown prompt. Container liveness (not health) is the signal: a
 // service's shared-infra health check stays green even when the service is down.
-// Runs synchronously at startup so /api/service/daemons never hands the UI a
-// stale entry on its first load.
+// The container listing runs as async child processes (a sync `docker ps`
+// here held the event loop at startup); /api/service/daemons waits for it,
+// so the UI never gets a stale entry on its first load.
 loadDaemonsFromFile();
-const prunedDaemons = pruneDeadDaemons();
-if (prunedDaemons.length > 0) {
-  console.error(
-    `Cleared ${prunedDaemons.length} stale daemon(s): ${prunedDaemons.join(", ")}`,
-  );
-}
+startupPrune = pruneDeadDaemonsAsync()
+  .then((pruned) => {
+    if (pruned.length > 0)
+      console.error(
+        `Cleared ${pruned.length} stale daemon(s): ${pruned.join(", ")}`,
+      );
+  })
+  .catch((e: unknown) => {
+    console.error(`[server] stale-daemon sweep failed: ${String(e)}`);
+  });
 
 // Background recovery: after a short delay (long enough for services to be
 // responding), health-check each loaded daemon. Removal requires strong
@@ -3272,6 +3213,16 @@ process.on("unhandledRejection", (reason) => {
   console.error("[server] unhandled rejection (continuing):", reason);
 });
 
+// Build, before serving, what the first health poll and /api/services would
+// otherwise build inside a request: the dependency-graph order and the HTTPS
+// checks' trust context (both synchronous).
+try {
+  for (const ws of getActiveWorkspaces()) graphOrder(ws);
+  prewarmHealthTls();
+} catch (e) {
+  serverLog(`  ⚠ cache warm-up failed: ${(e as Error).message}`);
+}
+
 // Bind loopback only — this UI runs privileged local actions (sudo-backed
 // service commands), so it must not be reachable from the LAN. Opt out with
 // DEN_BIND_HOST=0.0.0.0 for deliberate exposure.
@@ -3375,7 +3326,20 @@ async function stopDaemons(): Promise<void> {
 
 async function handleShutdown(signal: string): Promise<void> {
   clearSudoPassword();
-  const daemons = listDaemons();
+  // The registry lives in the state dir, which is resolved from the
+  // distribution root on every call: if that can no longer be found (den.yml
+  // moved or removed while the server ran), still exit. Never product() here:
+  // it resolves the same root.
+  let daemons: ReturnType<typeof listDaemons>;
+  try {
+    daemons = listDaemons();
+  } catch (e) {
+    console.error(
+      `\n[server] ${signal}: cannot read the running-services registry ` +
+        `(${(e as Error).message}); exiting and leaving any services running.`,
+    );
+    process.exit(0);
+  }
   if (daemons.length === 0) {
     process.exit(0);
   }
@@ -3419,16 +3383,29 @@ async function handleShutdown(signal: string): Promise<void> {
   process.exit(0);
 }
 
+/**
+ * Run the shutdown for `signal`. A failure still ends the process: the guard
+ * below ignores every later signal, so a shutdown that rejected would leave
+ * the server running and deaf to SIGTERM (a service manager then waits out
+ * its timeout and SIGKILLs it).
+ */
+function shutdownOn(signal: string): void {
+  handleShutdown(signal).catch((e: unknown) => {
+    console.error(`[server] ${signal}: shutdown failed (exiting):`, e);
+    process.exit(1);
+  });
+}
+
 let _shuttingDown = false;
 process.on("SIGINT", () => {
   if (_shuttingDown) return;
   _shuttingDown = true;
   console.log(""); // newline after ^C
-  void handleShutdown("SIGINT");
+  shutdownOn("SIGINT");
 });
 
 process.on("SIGTERM", () => {
   if (_shuttingDown) return;
   _shuttingDown = true;
-  void handleShutdown("SIGTERM");
+  shutdownOn("SIGTERM");
 });

package/src/ui/server/public/changelog-view.js

@@ -0,0 +1,16 @@
+/**
+ * What the version pop-up shows: the newest release only, or every
+ * version once the reader asks for them, plus the one button that toggles
+ * between the two (null when there is nothing earlier to show).
+ */
+export function changelogView(entries, showAll) {
+  const list = Array.isArray(entries) ? entries : [];
+  const earlier = list.length - 1;
+  if (earlier < 1) return { shown: list.slice(0, 1), toggle: null };
+  return {
+    shown: showAll ? list : list.slice(0, 1),
+    toggle: showAll
+      ? { label: "Show latest only", expanded: true }
+      : { label: `Show all versions (${earlier} earlier)`, expanded: false },
+  };
+}