@monster-orange/den 0.0.0-stage → 0.1.0

package/src/shared/process/log-rotation.ts

@@ -0,0 +1,147 @@
+/**
+ * Size-capped rotation for service logs.
+ *
+ * Service logs are written by DETACHED child processes that inherited an
+ * O_APPEND file descriptor (see spawnDaemon): den opens the file, hands the
+ * fd to the child, then closes its own copy. Nothing can make that child
+ * reopen — the command is arbitrary shell with no SIGHUP contract — and it
+ * outlives den. A chatty service therefore grows the file forever; a
+ * localstack tail was measured at 8.6 GB.
+ *
+ * That single fact rules out the usual strategies, and is why no logging
+ * library helps here (they all rotate a stream THEY own):
+ *
+ *  - RENAME (`mv x.log x.log.1`) is silently fatal. The child's fd follows the
+ *    inode, so it keeps writing into the renamed file while `x.log` stays
+ *    empty — and every reader tails by name, so live logs stop forever with no
+ *    error anywhere.
+ *  - TRUNCATE-HEAD (keep the last N bytes at offset 0) makes `tail -F` notice
+ *    the file shrank and restart from the beginning, re-flooding every viewer
+ *    with the whole retained tail. It also shifts every byte offset, so the
+ *    history pager silently renders the wrong page.
+ *
+ * What works is copy-then-truncate-to-ZERO: copy the tail into `<name>.log.1`,
+ * then truncate the live file in place. The inode never moves, so the child's
+ * fd stays valid and O_APPEND puts its next write at the new EOF. Readers cope:
+ * `tail -F` restarts at offset 0 and finds one banner line rather than a flood,
+ * and a stale history cursor clamps to a tiny file and ends paging cleanly.
+ *
+ * INVARIANT: the child's fd must stay O_APPEND. With plain O_WRONLY the kernel
+ * keeps the old offset and truncation produces a multi-gigabyte sparse hole of
+ * NULs. spawn-streaming opens with "a"; a test pins it.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import { MAX_LOG_BYTES, ROTATED_KEEP_BYTES } from "../constants.js";
+
+import { product } from "../product.js";
+/** Where a rotated generation is kept — one only, overwritten each time. */
+export function rotatedPathFor(logPath: string): string {
+  return `${logPath}.1`;
+}
+
+/**
+ * Rotate `logPath` when it exceeds `maxBytes`. Returns true when it rotated.
+ *
+ * Best-effort by design: a log that cannot be rotated must never break the
+ * service that writes to it, so every failure degrades to "left as-is".
+ */
+export function rotateIfOversized(
+  logPath: string,
+  maxBytes: number = MAX_LOG_BYTES,
+  keepBytes: number = ROTATED_KEEP_BYTES,
+): boolean {
+  let size: number;
+  try {
+    const st = fs.statSync(logPath);
+    if (!st.isFile()) return false;
+    size = st.size;
+  } catch {
+    return false; // no log yet
+  }
+  if (size <= maxBytes) return false;
+
+  try {
+    copyTail(logPath, rotatedPathFor(logPath), Math.min(keepBytes, size));
+    // Truncate to ZERO, not to a retained tail — see the module docstring.
+    fs.truncateSync(logPath, 0);
+    fs.appendFileSync(
+      logPath,
+      `\n─── ${product().cli} rotated ${new Date().toISOString()} — earlier output in ${path.basename(rotatedPathFor(logPath))} ───\n`,
+    );
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Copy the last `bytes` of `from` into `to`, starting at the first line break
+ * so the rotated file doesn't open mid-line. Streams in chunks — the whole
+ * point is that these files are too big to hold in memory.
+ */
+function copyTail(from: string, to: string, bytes: number): void {
+  const src = fs.openSync(from, "r");
+  try {
+    const size = fs.fstatSync(src).size;
+    let start = Math.max(0, size - bytes);
+    if (start > 0) {
+      // Snap forward to just past the next newline.
+      const probe = Buffer.alloc(Math.min(64 * 1024, size - start));
+      const read = fs.readSync(src, probe, 0, probe.length, start);
+      const nl = probe.subarray(0, read).indexOf(0x0a);
+      if (nl !== -1) start += nl + 1;
+    }
+    const dst = fs.openSync(to, "w");
+    try {
+      const buf = Buffer.alloc(1024 * 1024);
+      let pos = start;
+      for (;;) {
+        const read = fs.readSync(src, buf, 0, buf.length, pos);
+        if (read <= 0) break;
+        fs.writeSync(dst, buf, 0, read);
+        pos += read;
+      }
+    } finally {
+      fs.closeSync(dst);
+    }
+  } finally {
+    fs.closeSync(src);
+  }
+}
+
+/**
+ * Rotate every oversized `*.log` under a log root (`<root>/<repo>/<name>.log`).
+ * Returns the paths rotated. Never throws — a sweep failure must not take the
+ * server down.
+ */
+export function sweepLogs(
+  logRoot: string,
+  maxBytes: number = MAX_LOG_BYTES,
+): string[] {
+  const rotated: string[] = [];
+  let dirs: fs.Dirent[];
+  try {
+    dirs = fs.readdirSync(logRoot, { withFileTypes: true });
+  } catch {
+    return rotated; // no logs yet
+  }
+  for (const dir of dirs) {
+    if (!dir.isDirectory()) continue;
+    const sub = path.join(logRoot, dir.name);
+    let files: string[];
+    try {
+      files = fs.readdirSync(sub);
+    } catch {
+      continue;
+    }
+    for (const file of files) {
+      // `.log.1` is the rotated generation — capping it again would discard it.
+      if (!file.endsWith(".log")) continue;
+      const full = path.join(sub, file);
+      if (rotateIfOversized(full, maxBytes)) rotated.push(full);
+    }
+  }
+  return rotated;
+}

package/src/shared/process/spawn-streaming.ts

@@ -0,0 +1,229 @@
+/**
+ * Shared streaming process spawner
+ *
+ * Extracted from src/ui/cli/test-runner.ts so both the CLI test runner
+ * and the environment manager can spawn processes without importing from
+ * the UI layer.
+ */
+
+import { spawn } from "child_process";
+import fs from "fs";
+import path from "path";
+import { rotateIfOversized } from "./log-rotation.js";
+import { getStateDir } from "../paths.js";
+
+import { product } from "../product.js";
+// ── Log path ──────────────────────────────────────────────────────────────────
+
+/**
+ * Canonical path for a service's daemon log file.
+ * ~/.den/logs/<repoName>/<subsystem>.log
+ */
+export function getServiceLogPath(repoName: string, subsystem: string): string {
+  const dir = path.join(getServiceLogRoot(), repoName);
+  fs.mkdirSync(dir, { recursive: true });
+  return path.join(dir, `${subsystem}.log`);
+}
+
+/** Root of the per-service log tree: `~/.den/logs/<repo>/<service>.log`. */
+export function getServiceLogRoot(): string {
+  return path.join(getStateDir(), "logs");
+}
+
+// ── ANSI strip ────────────────────────────────────────────────────────────────
+
+function stripAnsi(data: Buffer, onLine: (line: string) => void): void {
+  // eslint-disable-next-line no-control-regex
+  const text = data.toString().replace(/\x1b\[[0-9;]*[a-zA-Z]/g, "");
+  for (const line of text.split("\n")) {
+    const visible = line.split("\r").filter(Boolean).pop() ?? "";
+    if (visible.trim()) onLine(visible);
+  }
+}
+
+// ── Daemon spawner ────────────────────────────────────────────────────────────
+
+/**
+ * Spawn a long-running service as a true daemon, fully decoupled from den's
+ * process space.
+ *
+ * Why this matters
+ * ────────────────
+ * `spawnStreaming` with `stdio: pipe` keeps a file-descriptor link between
+ * den and the service. When den exits (or the SSE response closes and
+ * Node GC's the stream), the pipe read-end closes → the child gets SIGPIPE on
+ * its next write → it dies. `proc.unref()` only removes the process from
+ * Node's event-loop ref-count; it does NOT sever the FD connection.
+ *
+ * This function fixes that by:
+ *   1. Writing a start-marker to `logPath` then opening it with `O_APPEND`.
+ *   2. Spawning the service with `stdio: ['ignore', logFd, logFd]` — no pipe,
+ *      no connection to den's FD table.
+ *   3. Immediately closing the parent's copy of the log fd.
+ *   4. Starting a `tail -F` subprocess to stream output back to the caller.
+ *
+ * The returned process is the `tail` subprocess — kill it when you want to
+ * stop streaming without stopping the service.  `onClose` fires when tail
+ * exits (i.e. when the caller kills it or the SSE detach window expires).
+ */
+export function spawnDaemon(
+  command: string,
+  env: NodeJS.ProcessEnv,
+  logPath: string,
+  onLine: (line: string) => void,
+  onClose: (exitCode: number) => void,
+  cwd?: string,
+  onServiceExit?: (exitCode: number) => void,
+): ReturnType<typeof spawn> {
+  fs.mkdirSync(path.dirname(logPath), { recursive: true });
+
+  // Cap the log before the next run appends to it. Cheap (one statSync) and it
+  // covers restart-heavy services; the periodic sweep in the server catches
+  // long-lived ones that never respawn.
+  rotateIfOversized(logPath);
+
+  // Stamp the log so the tail starts reading from a clean boundary.
+  const ts = new Date().toISOString();
+  fs.appendFileSync(logPath, `\n\n─── ${product().cli} start ${ts} ───\n`);
+
+  // Open the log fd in the parent so we can hand it to the child.
+  // MUST stay "a" (O_APPEND): rotation truncates this file in place, and only
+  // O_APPEND recomputes the offset to EOF on every write. With plain O_WRONLY
+  // the child would keep its old offset and write a multi-GB sparse hole of
+  // NULs. See log-rotation.ts.
+  const logFd = fs.openSync(logPath, "a");
+
+  const svcProc = spawn(command, [], {
+    env,
+    cwd,
+    shell: true,
+    detached: true, // own process group → immune to Ctrl+C on den
+    stdio: ["ignore", logFd, logFd], // NO pipe to the CLI — survives its shutdown
+  });
+  // Notify caller when the service process itself exits (e.g. `make up` finishes
+  // after starting Docker containers).  The service may still be "running" even
+  // after this fires — the process just handed off to the container runtime.
+  if (onServiceExit) {
+    svcProc.on("exit", (code) => onServiceExit(code ?? 0));
+  }
+  // Without an 'error' listener a spawn failure (e.g. cwd missing, ENOENT)
+  // emits an unhandled 'error' that crashes den. Surface it instead.
+  svcProc.on("error", (err) => {
+    onLine(`Error: ${err.message}`);
+    onServiceExit?.(1);
+  });
+  svcProc.unref(); // den's event loop can exit independently
+  fs.closeSync(logFd); // parent closes its copy; child keeps its own fd
+
+  // Tail the log for streaming back to the caller.
+  // -F follows by name (handles log rotation). -n 0 shows only new lines written
+  // after tail started (we already stamped the marker above).
+  const tailProc = spawn("tail", ["-F", "-n", "0", logPath], {
+    stdio: ["ignore", "pipe", "ignore"],
+  });
+
+  tailProc.stdout?.on("data", (data: Buffer) => stripAnsi(data, onLine));
+  tailProc.on("close", () => onClose(0));
+  tailProc.on("error", () => onClose(0));
+
+  return tailProc;
+}
+
+// ── Streaming spawner ─────────────────────────────────────────────────────────
+
+/**
+ * Spawn a process and stream stdout/stderr line-by-line via callbacks.
+ * ANSI colour codes are stripped so browser/terminal output renders cleanly.
+ *
+ * When `detached` is true AND `logPath` is provided the process is spawned as
+ * a true daemon (see `spawnDaemon`).  The returned proc is then the `tail`
+ * subprocess used for streaming — kill it to stop streaming without stopping
+ * the service.
+ *
+ * @param command  Executable name or shell string (when args is empty, shell mode is used)
+ * @param args     Argument array — pass [] to invoke command as a raw shell string
+ * @param env      Process environment
+ * @param onLine   Called for each non-empty output line
+ * @param onClose  Called with exit code when the process terminates
+ * @param cwd      Working directory (defaults to process.cwd())
+ * @param detached Put the child in its own process group (default false)
+ * @param logPath  If provided together with detached=true, run as a daemon writing to this path
+ */
+export function spawnStreaming(
+  command: string,
+  args: string[],
+  env: NodeJS.ProcessEnv,
+  onLine: (line: string) => void,
+  onClose: (exitCode: number) => void,
+  cwd?: string,
+  detached?: boolean,
+  logPath?: string,
+  onServiceExit?: (exitCode: number) => void,
+): ReturnType<typeof spawn> {
+  // Daemon mode: service runs independently from den — survives restarts.
+  if (detached && logPath) {
+    return spawnDaemon(
+      command,
+      env,
+      logPath,
+      onLine,
+      onClose,
+      cwd,
+      onServiceExit,
+    );
+  }
+
+  // Standard mode: pipe stdout/stderr for streaming, optionally detached.
+  const proc = spawn(command, args, {
+    env,
+    cwd,
+    shell: args.length === 0, // shell mode when command is a raw string (service commands)
+    detached: detached ?? false,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  proc.stdout?.on("data", (data: Buffer) => stripAnsi(data, onLine));
+  proc.stderr?.on("data", (data: Buffer) => stripAnsi(data, onLine));
+
+  proc.on("close", (code) => onClose(code ?? 1));
+  proc.on("error", (err) => {
+    onLine(`Error: ${err.message}`);
+    onClose(1);
+  });
+
+  if (detached) proc.unref();
+
+  return proc;
+}
+
+// ── Interactive spawner ───────────────────────────────────────────────────────
+
+/**
+ * Spawn a process with full TTY access — stdin, stdout, and stderr are all
+ * inherited from the parent process. Use this for interactive commands that
+ * require human input (HITL), such as auth logins, license prompts, etc.
+ *
+ * No output streaming is performed — the user sees and interacts with the
+ * process directly in their terminal.
+ */
+export function spawnInteractive(
+  command: string,
+  env: NodeJS.ProcessEnv,
+  onClose: (exitCode: number) => void,
+  cwd?: string,
+): ReturnType<typeof spawn> {
+  const proc = spawn(command, [], {
+    env,
+    cwd,
+    shell: true,
+    stdio: "inherit",
+  });
+
+  proc.on("close", (code) => onClose(code ?? 1));
+  proc.on("error", (err) => {
+    void err;
+    onClose(1);
+  });
+
+  return proc;
+}

package/src/shared/product.ts

@@ -0,0 +1,92 @@
+/**
+ * The running product's names, for code that needs them at runtime.
+ *
+ * Den never hard-codes a product name: the CLI, env-var prefix, config and
+ * manifest file names and directories all come from the distribution's
+ * profile (den.yml, see profile.ts). This module is the safe accessor: it
+ * never throws, falling back to DEN_PROFILE or Den's own profile when no
+ * distribution root can be found (e.g. help text printed outside a checkout).
+ */
+
+import { getDistRoot } from "./paths.js";
+import {
+  loadProfile,
+  profileFrom,
+  readProfileFile,
+  type ProductProfile,
+} from "./profile.js";
+
+export interface Product extends ProductProfile {
+  /** Display name: the name with its first letter upper-cased ("Den"). */
+  title: string;
+  /** The YAML manifest file name (".den.yml"). */
+  manifestYml: string;
+  /** The JSON manifest file name (".den.json"). */
+  manifestJson: string;
+}
+
+function withDerived(p: ProductProfile): Product {
+  const yml = p.manifestFiles[0];
+  return {
+    ...p,
+    title: p.name.charAt(0).toUpperCase() + p.name.slice(1),
+    manifestYml: yml,
+    manifestJson: p.manifestFiles[1] ?? yml.replace(/\.ya?ml$/, "") + ".json",
+  };
+}
+
+/** The running product. Never throws. */
+export function product(): Product {
+  try {
+    return withDerived(loadProfile(getDistRoot()));
+  } catch {
+    const pinned = process.env.DEN_PROFILE?.trim();
+    try {
+      return withDerived(pinned ? readProfileFile(pinned) : profileFrom());
+    } catch {
+      return withDerived(profileFrom());
+    }
+  }
+}
+
+/** The env-var name this product uses for `key`: envKey("ROOT") → "DEN_ROOT". */
+export function envKey(key: string): string {
+  return `${product().envPrefix}_${key}`;
+}
+
+/** The value of this product's `key` env var, or undefined. */
+export function envValue(
+  key: string,
+  env: NodeJS.ProcessEnv = process.env,
+): string | undefined {
+  return env[envKey(key)];
+}
+
+/**
+ * Placeholders the web pages use for product names; serveFile fills them in.
+ * Pages are static files, so they cannot ask the profile themselves.
+ */
+export function productTokens(p: Product = product()): Record<string, string> {
+  return {
+    __DEN_CLI__: p.cli,
+    __DEN_NAME__: p.name,
+    __DEN_TITLE__: p.title,
+    __DEN_ENV_PREFIX__: p.envPrefix,
+    __DEN_CONFIG_FILE__: p.configFile,
+    __DEN_MANIFEST_YML__: p.manifestYml,
+    __DEN_MANIFEST_JSON__: p.manifestJson,
+    __DEN_SERVICES_DIR__: p.servicesDir,
+    __DEN_WORKSPACES_DIR__: p.workspacesDir,
+  };
+}
+
+/** Replace every product placeholder in `text`. */
+export function fillProductTokens(
+  text: string,
+  p: Product = product(),
+): string {
+  let out = text;
+  for (const [token, value] of Object.entries(productTokens(p)))
+    out = out.split(token).join(value);
+  return out;
+}

package/src/shared/profile.ts

@@ -0,0 +1,141 @@
+/**
+ * The product profile: who this distribution is.
+ *
+ * Den is one engine; each distribution runs it under its own name. Everything product-named comes from here: the CLI, the env-var
+ * prefix, the config and manifest file names, and the state, services and
+ * workspaces directories. The profile lives in `den.yml` at the
+ * distribution root:
+ *
+ *   product:
+ *     name: acme              # everything below derives from it by default
+ *     # cli: acme
+ *     # env_prefix: ACME
+ *     # config: config/acme.yml
+ *     # manifest: .acme.yml   (.json is also read)
+ *     # state_dir: ~/.acme
+ *     # services_dir: .acme-services
+ *     # workspaces_dir: acme-workspaces
+ *
+ * `DEN_PROFILE` may point at another profile file (tests use this). With no
+ * profile at all, Den runs under its own name.
+ */
+
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { parse as parseYaml } from "yaml";
+
+export const PROFILE_FILE = "den.yml";
+
+export interface ProductProfile {
+  /** Display and slug name ("acme"). */
+  name: string;
+  /** The command a person types. */
+  cli: string;
+  /** Upper-case env-var prefix, without the trailing "_" ("ACME"). */
+  envPrefix: string;
+  /** Org config, relative to the distribution root. */
+  configFile: string;
+  /** Manifest file names probed in a repo, first match wins. */
+  manifestFiles: string[];
+  /** Per-user state directory (absolute, `~` expanded). */
+  stateDir: string;
+  /** Managed clones, relative to the distribution root. */
+  servicesDir: string;
+  /** Workspace checkouts, relative to the distribution root. */
+  workspacesDir: string;
+}
+
+interface RawProduct {
+  name?: unknown;
+  cli?: unknown;
+  env_prefix?: unknown;
+  config?: unknown;
+  manifest?: unknown;
+  state_dir?: unknown;
+  services_dir?: unknown;
+  workspaces_dir?: unknown;
+}
+
+const SLUG = /^[a-z][a-z0-9-]*$/;
+const PREFIX = /^[A-Z][A-Z0-9_]*$/;
+
+const str = (v: unknown): string | undefined =>
+  typeof v === "string" && v.trim() ? v.trim() : undefined;
+
+const expandHome = (p: string): string =>
+  p === "~"
+    ? os.homedir()
+    : p.startsWith("~/")
+      ? path.join(os.homedir(), p.slice(2))
+      : p;
+
+/** A profile from a `product:` block; unset fields derive from the name. */
+export function profileFrom(raw: RawProduct = {}): ProductProfile {
+  const name = str(raw.name) ?? "den";
+  if (!SLUG.test(name))
+    throw new Error(
+      `den.yml: product.name "${name}" must be lower-case letters, digits and "-"`,
+    );
+  const envPrefix =
+    str(raw.env_prefix) ?? name.toUpperCase().replace(/-/g, "_");
+  if (!PREFIX.test(envPrefix))
+    throw new Error(
+      `den.yml: product.env_prefix "${envPrefix}" must be upper-case letters, digits and "_"`,
+    );
+  const manifest = str(raw.manifest) ?? `.${name}.yml`;
+  const json = manifest.replace(/\.ya?ml$/, ".json");
+  return {
+    name,
+    cli: str(raw.cli) ?? name,
+    envPrefix,
+    configFile: str(raw.config) ?? `config/${name}.yml`,
+    manifestFiles: json === manifest ? [manifest] : [manifest, json],
+    stateDir: path.resolve(expandHome(str(raw.state_dir) ?? `~/.${name}`)),
+    servicesDir: str(raw.services_dir) ?? `.${name}-services`,
+    workspacesDir: str(raw.workspaces_dir) ?? `${name}-workspaces`,
+  };
+}
+
+/** Read a profile file; a missing file is Den's own profile. */
+export function readProfileFile(file: string): ProductProfile {
+  let raw: { product?: RawProduct } | null;
+  try {
+    raw = parseYaml(fs.readFileSync(file, "utf-8")) as typeof raw;
+  } catch (e) {
+    if ((e as NodeJS.ErrnoException).code === "ENOENT") return profileFrom();
+    throw new Error(`${file}: ${(e as Error).message}`);
+  }
+  return profileFrom(raw?.product ?? {});
+}
+
+let cached: { key: string; profile: ProductProfile } | null = null;
+
+/**
+ * The profile for a distribution root: `DEN_PROFILE` if set, else
+ * `<root>/den.yml`, else Den's own. Cached per file.
+ */
+export function loadProfile(root: string): ProductProfile {
+  const file = process.env.DEN_PROFILE?.trim() || path.join(root, PROFILE_FILE);
+  if (cached?.key !== file)
+    cached = { key: file, profile: readProfileFile(file) };
+  return cached.profile;
+}
+
+export function resetProfileCache(): void {
+  cached = null;
+}
+
+/** `${PREFIX}_${key}`: the env-var name this distribution uses for `key`. */
+export function envName(profile: ProductProfile, key: string): string {
+  return `${profile.envPrefix}_${key}`;
+}
+
+/** The value of `${PREFIX}_${key}`, or undefined. */
+export function readEnv(
+  profile: ProductProfile,
+  key: string,
+  env: NodeJS.ProcessEnv = process.env,
+): string | undefined {
+  return env[envName(profile, key)];
+}

package/src/shared/repo-kind.ts

@@ -0,0 +1,32 @@
+/**
+ * Repo kinds: only a `service` has a runtime (up/down/status). A `module`
+ * (bolt-on a host invokes) or `library` (imported code) contributes to other
+ * repos instead.
+ */
+import { readManifest } from "./manifest/read-manifest.js";
+import type { DerivedGraph } from "./environment-manager/derivation/types.js";
+import type { RepoKind } from "./types/index.js";
+
+export function repoKind(repoPath: string): RepoKind {
+  return readManifest(repoPath)?.kind ?? "service";
+}
+
+export function hasRuntime(kind: RepoKind): boolean {
+  return kind === "service";
+}
+
+/**
+ * What a non-service repo contributes to: its declared `contributes_to`, plus
+ * every repo that imports it (an SDK-import edge pointing at it).
+ */
+export function contributesTo(
+  repo: string,
+  repoPath: string,
+  graph?: DerivedGraph,
+): string[] {
+  const declared = readManifest(repoPath)?.contributes_to ?? [];
+  const importers = (graph?.edges ?? [])
+    .filter((e) => e.to === repo && e.signal === "http:sdk-import")
+    .map((e) => e.from);
+  return [...new Set([...declared, ...importers])].filter((r) => r !== repo);
+}