@monster-orange/den 0.2.2 → 0.2.4

package/CHANGELOG.md

@@ -4,6 +4,157 @@ All notable changes to Den are documented here. Each release lists the
 config and manifest changes a distribution must make to move to it under
 **Migrations**.
 
+## [0.2.4] - 2026-11-04
+
+### Fixed
+
+- **`GET /api/services` no longer blocks the web server.** It looked up
+  each workspace checkout's branch with a synchronous `git rev-parse`, one
+  after another (~40ms each), so with a dozen repos every request held the
+  event loop for 0.5-1.5s and everything else waited behind it: the UI,
+  the MCP server's reads, and `/api/server-info`, which `<cli> mcp connect`
+  probes with a 1.5s timeout. The lookups now run as async child processes
+  in parallel; concurrent requests (one per open tab) share one build, and
+  a settled build is reused for 2s (`?refresh=1` forces a new one; any
+  mutating request clears it). Concurrent `/api/health` polls also share
+  one run instead of each starting its own.
+- **`<cli> mcp connect` / `mcp clients` tell a slow web server from a
+  stopped one.** A server that accepts the connection but does not answer
+  within 1.5s is reported as "not answering within 1.5s", not "not
+  running". When the org config sets `web.service` (the web UI runs as a
+  launchd service), the advice is `<cli> service status` instead of
+  `<cli> web up`.
+- **Web server shutdown no longer leaks an unhandled rejection, or hangs.**
+  On SIGTERM/SIGINT the server reads the running-services registry, whose
+  path resolves the distribution root again; if that walk fails (the dist's
+  den.yml moved or removed while the server ran), the async shutdown
+  rejected unhandled ("unhandled rejection (continuing): Error: cannot
+  locate the distribution root: walked up from
+  <dist>/node_modules/@monster-orange/den/src/ui/server") and, its signal
+  guard already set, the process never exited, so a service manager had to
+  wait out its timeout and kill it. The shutdown now logs the failure and
+  exits.
+- **Deactivating a workspace no longer stops a service that moved to
+  another workspace.** A daemon's owners were the workspaces recorded when
+  it started, so a repo moved from `default` into `ws-b` while its daemon
+  ran was still "owned" only by `default`, and `workspace deactivate
+  default` stopped it. Release now re-derives the owners from the current
+  workspaces.yml membership: a daemon is stopped only when no active
+  workspace still lists its repo, and otherwise its recorded owners become
+  the active workspaces that do.
+- **`<cli> workspace --help` lists every subcommand**: activate,
+  deactivate and focus were missing (and `workspace focus <name>` now
+  exists in the CLI, as it did in the web UI). Deactivate's help and output
+  say that it releases only the services Den tracks: compose stacks of the
+  workspace's repos keep running, and the output names the running ones
+  with `<cli> down <repo>` for each (from one `docker ps`; a generic line
+  when docker can't be asked).
+- **The first `/api/health/stream` (and `/api/health`) cycle no longer
+  blocks the web server for a second.** Resolving the manifest of a repo
+  without its own (a held manifest under `manifests/`) looked up the
+  checkout's origin with a synchronous `git config --get
+  remote.origin.url`, one repo after another (~40-100ms each), so a
+  server's first health run held the event loop for 1-1.5s. The origin is
+  now read from the checkout's git config file (worktrees and
+  subdirectories included), never by spawning git. Parsed manifests are
+  also reused while the file is unchanged: one request read the same
+  manifests several times, and parsing them held the loop ~100-200ms per
+  health run.
+- **`GET /api/workspaces/available-repos` no longer blocks the web
+  server.** It ran `git remote get-url origin` (and, for managed clones,
+  `git rev-parse`) synchronously per checkout; the lookups are now async
+  and in parallel, and concurrent requests share one walk (reused for 2s;
+  `?refresh=1` forces a new one, any mutating request clears it).
+- **`workspace deactivate` names the compose stacks a repo runs itself.**
+  It only knew Den stacks (a manifest's `compose:` section, project
+  `<stack>-<workspace>`), so a distribution whose repos start their own
+  compose projects (`make up`) got no line at all. A running project now
+  counts when Compose started it from inside one of the workspace's
+  checkouts (its `com.docker.compose.project.working_dir` label), as well
+  as when it is a Den stack's project.
+- **`<cli> workspace <sub> --help`** prints that subcommand's help, not the
+  whole `workspace` help.
+- **No synchronous directory walks or spawns on the web server's hot
+  request paths** (`/api/workspaces/available-repos`, the `/api/health`
+  stream). On a loaded machine (load average ~50) the dev.1 fixes still
+  left stalls of 1-5s; a CPU profile showed why:
+  - available-repos recursed into every directory without a manifest file
+    of its own, so in a distribution whose repos use held manifests it
+    walked each repo's whole tree (to depth 4) with `readdirSync` /
+    `existsSync`: seconds per call. It now never descends into a git
+    checkout or `node_modules`, and the walk is `fs.promises`.
+  - The distribution root (and so `product()`) was re-resolved on every
+    call, a den.yml `existsSync` per `*_ROOT` variable each time, hundreds
+    of times per request. It is now resolved once per set of inputs (the
+    env vars and entry script it reads); the engine root likewise.
+  - Manifests, the edge config and the derived graph order are reused
+    across a request and the polls after it (5s; any mutating request
+    clears them), and the graph order is recomputed after a request, never
+    inside one.
+  - HTTPS health checks built a TLS context from ~150 root certificates on
+    every check; one context is built once, at startup.
+  - The startup sweep of stale daemon records ran `docker ps` / `opctl
+    container ls` synchronously; it is async now (concurrent callers share
+    one call), and `/api/service/daemons` waits for it.
+  A new test runs the server with a spy on every synchronous fs and
+  child_process call and fails if either route makes a sync spawn or
+  directory walk, or any sync call outside a short allowlist of single
+  config-file reads.
+- **The web UI service no longer runs at background priority.**
+  `service install` wrote `ProcessType: Background` into the LaunchAgent,
+  so macOS ran the UI server on efficiency cores with throttled I/O: on a
+  busy machine its first requests stalled for 5-11s. It is now
+  `Interactive`, and `service status` shows launchd's spawn type.
+- **`workspace deactivate` never suggests stopping a stack another active
+  workspace still holds.** Such a stack is listed as "in use by another
+  workspace: <project> (still in <ws>)" instead of with a `down <repo>`
+  command, and a repo with a stack still in use gets no `down` suggestion.
+
+### Migrations
+
+- Re-run `<cli> service install` to pick up the new LaunchAgent priority
+  (`<cli> service status` should show `spawn type = interactive`).
+
+## [0.2.3] - 2026-11-04
+
+### Added
+
+- **`secret_env`: secrets resolved by Den into a stack's or a service's env**
+  (R16), so a consumer no longer wraps `<provider> run -- <cli> restart
+  <repo>` by hand, and a plain restart no longer drops a key. In a manifest,
+  `compose.secret_env` (for the docker compose process: the compose file's
+  `${VAR}` or pass-through `environment:` entries) and
+  `services.<svc>.secret_env` (for that service's own up/restart command)
+  map an env var to a fully qualified ref, `<vault>/<item>/<field>`. For
+  Doppler that is `<project>/<config>/<key>`; for 1Password vault / item /
+  field; for `env` the field is the env var to read. Existing refs (the
+  engine's named secrets, test users) are unchanged. Resolved through the
+  configured `secrets.provider` at `up` and `restart` (also the in-place
+  restart of one compose service), all or nothing, before anything stops or
+  starts. A missing key fails that target, exit 1, naming the key, ref and
+  provider. Values go only into the spawned process's env: never into the
+  compose override, logs, activity records, MCP output or messages. A dry
+  run (`-n`) never asks the provider; it lists the key names it would
+  inject. A CLI-tabs start of a service with `secret_env` runs as a daemon,
+  so the values stay out of the tab's shell. `validate` rejects a ref that
+  is not exactly three non-empty parts, and a key that is not an env var
+  name. See "Secrets in a target's env" in `docs/distributions.md`.
+
+### Fixed
+
+- **`<cli> mcp connect` for Claude Code failed whenever the server has env**
+  ("Invalid environment variable format: <name>"): `claude mcp add`'s `-e`
+  takes several values, so the server name after the env pairs was read as
+  one more. The name now comes right after `add`.
+- **The up/restart secrets check said the backend was "not ready" when it
+  was signed in with no default scope.** `secret_env` refs name their own
+  scope, so they still resolve; it now says "signed in, no default scope
+  set" instead of warning.
+
+### Migrations
+
+- None required.
+
 ## [0.2.2] - 2026-11-04
 
 ### Added

package/docs/distributions.md

@@ -213,6 +213,57 @@ ones (`⇄ hybrid`).
   production_envs: [prod] # default: [prod, production]
   ```
 
+## Secrets in a target's env
+
+`secret_env` puts secrets from the configured secrets provider
+(`secrets.provider`: `doppler`, `1password` or `env`) into the env of what
+Den starts, instead of wrapping `<provider> run -- <cli> restart <repo>` by
+hand:
+
+```yaml
+# .acme.yml
+compose:
+  secret_env: # the docker compose process's env, at stack up / restart
+    OPENAI_API_KEY: shop/dev_personal/OPENAI_API_KEY
+services:
+  worker:
+    secret_env: # the env of this service's own up / restart command
+      STRIPE_KEY: Engineering/Stripe Test/secret key
+```
+
+Each value is a fully qualified ref, `<vault>/<item>/<field>`:
+
+| Provider    | `a/b/c` means                                                              |
+| ----------- | -------------------------------------------------------------------------- |
+| `doppler`   | `--project a --config b`, key `c` (`secrets.doppler` is not consulted)     |
+| `1password` | vault `a`, item `b`, field `c`                                             |
+| `env`       | the env var `c` of the process running Den (`secrets.keys` is not applied) |
+
+A `compose.secret_env` key reaches a container only through the compose
+file: `${OPENAI_API_KEY}` in it, or a pass-through `environment:
+[OPENAI_API_KEY]`.
+
+- **When.** `up` and `restart`, including `restart <repo>/<service>` of one
+  compose service (`stack up`, `stack restart`, `stack restart --service`),
+  from the CLI, the web UI and MCP. Every key is resolved before anything
+  stops or starts.
+- **Missing.** A key the provider cannot resolve (unset, empty, no access)
+  fails that target, exit 1, naming the key, the ref and the provider. The
+  rest of the set is not used: nothing starts with a partial set, and a
+  restart leaves the old instance running.
+- **Where values go.** Only into the spawned process's env (the docker
+  compose process, or the service's command). Never into the generated
+  compose override, logs, activity records, MCP output, dry runs or
+  messages. A CLI-tabs (`cli_tabs`) start of a service with `secret_env`
+  runs as a daemon, since a tab would put them in its shell history.
+- **Dry run.** `-n`/`--dry-run` never asks the provider. It lists the key
+  names it would inject: `shop: would inject secret_env compose:
+OPENAI_API_KEY (names only; not resolved in a dry run)`.
+- **Validation.** `<cli> validate` rejects a ref that is not exactly three
+  non-empty parts and a key that is not an env var name
+  (`[A-Za-z_][A-Za-z0-9_]*`). A `stacks:` entry in the org config takes
+  `secret_env` too.
+
 ## den.yml: the product profile
 
 Every runtime name derives from `product.name`. Set only what you want to

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@monster-orange/den",
-  "version": "0.2.2",
+  "version": "0.2.4",
   "description": "Den: a local multi-repo dev orchestrator, HTTPS edge and E2E harness",
   "type": "module",
   "bin": {

package/scripts/setup-secrets.ts

@@ -72,6 +72,17 @@ async function main(): Promise<number> {
     const r = await provider.preflight();
     if (r.ok) return 0;
     if (gate && !cfg.required) {
+      // Signed in but no default scope: secret_env refs name their own
+      // (<project>/<config>/<key>), so only the engine's own secrets lack one.
+      if (await provider.isReady()) {
+        console.error(
+          chalk.gray(
+            `   ${label}: signed in, no default scope set (secret_env refs still resolve;` +
+              ` test logins and the saved sudo password need one: \`make ${product().cli}-setup-secrets\`).`,
+          ),
+        );
+        return 0;
+      }
       console.error(
         chalk.yellow(
           `⚠️  ${label} is not ready — continuing.\n` +

package/service/install.sh

@@ -65,7 +65,9 @@ data = {
     "RunAtLoad": True,
     "KeepAlive": True,
     "ThrottleInterval": 10,
-    "ProcessType": "Background",
+    # Interactive, not Background: macOS gives Background agents efficiency
+    # cores and throttled I/O, and the UI stalled for seconds under load.
+    "ProcessType": "Interactive",
     "StandardOutPath": f"{logs}/web.log",
     "StandardErrorPath": f"{logs}/web.log",
 }
@@ -115,7 +117,7 @@ case "${1:-install}" in
   status)
     # launchctl's own "Could not find service" is noise when it isn't loaded.
     if out="$(launchctl print "$DOMAIN/$LABEL" 2>/dev/null)"; then
-      printf '%s\n' "$out" | grep -E '^\s*(state|pid|last exit code|runs)'
+      printf '%s\n' "$out" | grep -E '^\s*(state|pid|last exit code|runs|spawn type)'
     else
       echo "$LABEL is not loaded"
     fi

package/src/shared/edge/config.ts

@@ -31,6 +31,7 @@ import * as yaml from "yaml";
 import { brandSlug, defaultBrand } from "../brand.js";
 import { DEFAULT_PORT } from "../constants.js";
 import { getDistRoot } from "../paths.js";
+import { cachedRead } from "../read-cache.js";
 import { readManifestStacks } from "../stacks/config.js";
 import { getActiveWorkspaces } from "../environment-manager/workspaces.js";
 import { assertDomain } from "./render.js";
@@ -154,6 +155,21 @@ function manifestStackNames(): string[] {
 }
 
 export function readEdgeConfig(root: string = getDistRoot()): EdgeConfig {
+  // The web server reads this on every request (open links, health rows);
+  // reused for the read cache's TTL there, read fresh everywhere else. An
+  // error is cached as such: most instances have no edge block at all.
+  const r = cachedRead(`edge-config:${root}`, () => {
+    try {
+      return { config: computeEdgeConfig(root) };
+    } catch (e) {
+      return { error: (e as Error).message };
+    }
+  });
+  if ("error" in r) throw new Error(r.error);
+  return r.config;
+}
+
+function computeEdgeConfig(root: string): EdgeConfig {
   const { team: t, user: u } = layers(root);
   const team = (t.edge ?? {}) as RawEdge;
   const user = (u.edge ?? {}) as RawEdge;

package/src/shared/environment-manager/health.ts

@@ -175,6 +175,36 @@ export function __setExtraCaForTests(pems: string[] | null): void {
   _extraCa = pems;
 }
 
+/**
+ * One TLS context with the public roots plus extraCa(), built once. Passing
+ * `ca` per request made Node parse ~150 root certificates on every HTTPS
+ * health check, synchronously: tens of ms per check, per poll.
+ */
+/** Build the HTTPS checks' trust context now (web server startup). */
+export function prewarmHealthTls(): void {
+  trustContext();
+}
+
+let _trust: {
+  ca: string[];
+  context: tls.SecureContext;
+  agent: https.Agent;
+} | null = null;
+function trustContext(): { context: tls.SecureContext; agent: https.Agent } {
+  const extra = extraCa();
+  if (_trust?.ca !== extra) {
+    _trust?.agent.destroy();
+    _trust = {
+      ca: extra,
+      context: tls.createSecureContext({
+        ca: [...tls.rootCertificates, ...extra],
+      }),
+      agent: new https.Agent({ keepAlive: false }),
+    };
+  }
+  return _trust;
+}
+
 /**
  * HTTPS check. Connects to the c-ares-resolved IP (see above) but keeps the
  * hostname for SNI and certificate verification; rewriting the URL to the IP,
@@ -201,7 +231,12 @@ async function pingHttps(parsed: URL, timeoutMs: number): Promise<PingResult> {
       {
         method: "GET",
         servername: isIpLiteral(host) ? undefined : host,
-        ca: [...tls.rootCertificates, ...extraCa()],
+        // tls.connect's option, passed through by the agent (not in the
+        // https.RequestOptions type).
+        ...({ secureContext: trustContext().context } as https.RequestOptions),
+        // Its own agent: a socket kept alive under one trust set must never
+        // be reused for a check under another.
+        agent: trustContext().agent,
         lookup: (_h, opts, cb) => {
           // Node may ask for all addresses (autoSelectFamily).
           if ((opts as { all?: boolean }).all)

package/src/shared/environment-manager/opctl.ts

@@ -10,7 +10,7 @@
  *     repo's dev/up op, captured at spawn time and/or parsed on demand.
  */
 
-import { execFileSync, execSync } from "child_process";
+import { execFile, execFileSync, execSync } from "child_process";
 import * as fs from "fs";
 import * as path from "path";
 import { parse as parseYaml } from "yaml";
@@ -333,3 +333,85 @@ export function diffSpawnedContainers(
   for (const c of after) if (!had.has(c.base)) fresh.add(c.base);
   return [...fresh];
 }
+
+// ── Async variants (the web server) ──────────────────────────────────────────
+// The sync calls above block the event loop for the whole `docker ps` /
+// `opctl container ls` (hundreds of ms on a loaded machine). The server uses
+// these instead; concurrent callers share one in-flight call.
+
+function execOut(cmd: string, args: string[]): Promise<string | null> {
+  return new Promise((resolve) => {
+    execFile(
+      cmd,
+      args,
+      { encoding: "utf-8", timeout: 5_000, maxBuffer: 16 * 1024 * 1024 },
+      (err, stdout) => resolve(err ? null : stdout),
+    );
+  });
+}
+
+function shared<T>(run: () => Promise<T>): () => Promise<T> {
+  let inFlight: Promise<T> | null = null;
+  return () =>
+    (inFlight ??= run().finally(() => {
+      inFlight = null;
+    }));
+}
+
+/** Whether `opctl` is on PATH, without spawning a shell. Cached. */
+export async function isOpctlAvailableAsync(): Promise<boolean> {
+  if (opctlAvailableCache !== null) return opctlAvailableCache;
+  for (const dir of (process.env.PATH ?? "").split(path.delimiter)) {
+    if (!dir) continue;
+    try {
+      await fs.promises.access(path.join(dir, "opctl"), fs.constants.X_OK);
+      return (opctlAvailableCache = true);
+    } catch {
+      /* next */
+    }
+  }
+  return (opctlAvailableCache = false);
+}
+
+/** runningDockerNames, async; null when docker is unavailable. */
+export const runningDockerNamesAsync = shared(
+  async (): Promise<string[] | null> => {
+    const out = await execOut("docker", ["ps", "--format", "{{.Names}}"]);
+    return out === null
+      ? null
+      : out
+          .split("\n")
+          .map((l) => l.trim())
+          .filter(Boolean);
+  },
+);
+
+/** listRunningOpctlContainers, async. */
+export const listRunningOpctlContainersAsync = shared(
+  async (): Promise<{ name: string; base: string }[]> => {
+    const byBase = new Map<string, { name: string; base: string }>();
+    const [opctlOut, dockerNames] = await Promise.all([
+      isOpctlAvailableAsync().then((ok) =>
+        ok ? execOut("opctl", ["container", "ls"]) : null,
+      ),
+      runningDockerNamesAsync(),
+    ]);
+    if (opctlOut !== null) {
+      const lines = opctlOut.split("\n");
+      const firstToken = lines[0]?.trim().split(/\s+/)[0];
+      const body =
+        firstToken === "NAME" || firstToken === "ID" ? lines.slice(1) : lines;
+      for (const line of body) {
+        const name = line.trim().split(/\s+/)[0];
+        if (!name) continue;
+        const base = name.replace(/-[0-9a-f]{8}$/i, "");
+        byBase.set(base, { name, base });
+      }
+    }
+    for (const name of dockerNames ?? []) {
+      const m = name.match(DOCKER_OPCTL_NAME);
+      if (m && !byBase.has(m[1])) byBase.set(m[1], { name, base: m[1] });
+    }
+    return [...byBase.values()];
+  },
+);