@monster-orange/den 0.0.0 → 0.1.0

package/src/shared/environment-manager/log-tail.ts

@@ -0,0 +1,212 @@
+/**
+ * Live log tailing for a local service — the CLI-side counterpart of the web
+ * /api/service/taillog SSE endpoint. Source resolution, in order:
+ *   1. The service's manifest `logs.local` shell command (e.g. `docker logs -f`)
+ *   2. Conventional container derivation: a running `opctl_<base>_<hash>`
+ *      container → `docker logs -f` — real runtime logs with zero manifest config
+ *   3. The den daemon startup log (~/.den/logs/<repo>/<subsystem>.log) —
+ *      last resort, covers host-run processes with no container
+ * A source whose command exits within the quick-exit window (container not
+ * found, stale filter) is treated as broken and the next tier takes over.
+ */
+
+import * as fs from "fs";
+import { DAEMON_QUICK_EXIT_MS, LIVE_TAIL_LINES } from "../constants.js";
+import { readManifest } from "../manifest/read-manifest.js";
+import { findConventionalContainer } from "./opctl.js";
+import {
+  getServiceLogPath,
+  spawnStreaming,
+} from "../process/spawn-streaming.js";
+import type { LogSource, ResolvedService } from "../types/index.js";
+
+export interface TailSource {
+  kind: "manifest" | "docker" | "file";
+  /** Executable (argv mode) or a full shell command (when args is empty). */
+  cmd: string;
+  /**
+   * Argv for cmd. Empty → cmd runs through the shell (manifest `logs.local`
+   * commands are arbitrary shell by design). Den-BUILT commands always use
+   * argv mode so container/subsystem names from manifests or docker output
+   * can never be shell-interpreted (`$(…)`, backticks, quotes).
+   */
+  args: string[];
+  /** Working directory for the command (manifest commands run in the repo). */
+  cwd?: string;
+  /** Human-readable description for the stream header. */
+  label: string;
+}
+
+export interface TailSourceInputs {
+  repoPath?: string;
+  /** The subsystem's manifest `logs.local` entry, if any. */
+  logSource?: LogSource;
+  /** A running conventional opctl container name, if any. */
+  container: string | null;
+  daemonLogPath: string;
+  daemonLogExists: boolean;
+}
+
+/** Ordered tail-source candidates (best source first). Pure — see resolveTailSources. */
+export function tailSourceCandidates(
+  inputs: TailSourceInputs,
+  lines: number = LIVE_TAIL_LINES,
+): TailSource[] {
+  const candidates: TailSource[] = [];
+  if (inputs.logSource?.type === "shell" && inputs.logSource.cmd) {
+    candidates.push({
+      kind: "manifest",
+      cmd: inputs.logSource.cmd,
+      args: [],
+      cwd: inputs.repoPath,
+      label: `manifest log command: ${inputs.logSource.cmd}`,
+    });
+  }
+  if (inputs.container) {
+    candidates.push({
+      kind: "docker",
+      cmd: "docker",
+      args: ["logs", "-f", "--tail", String(lines), inputs.container],
+      label: `container logs: ${inputs.container}`,
+    });
+  }
+  if (inputs.daemonLogExists) {
+    candidates.push({
+      kind: "file",
+      cmd: "tail",
+      args: ["-F", "-n", String(lines), inputs.daemonLogPath],
+      label: `startup log: ${inputs.daemonLogPath}`,
+    });
+  }
+  return candidates;
+}
+
+/** Gather live inputs for `resolved` and build its tail-source candidates. */
+export function resolveTailSources(resolved: ResolvedService): TailSource[] {
+  const manifest = readManifest(resolved.repoPath);
+  const logSource = manifest?.services?.[resolved.subsystem]?.logs?.local;
+  const container = findConventionalContainer(
+    resolved.repoName,
+    resolved.repoPath,
+    resolved.subsystem,
+  );
+  const daemonLogPath = getServiceLogPath(
+    resolved.repoName,
+    resolved.subsystem,
+  );
+  return tailSourceCandidates({
+    repoPath: resolved.repoPath,
+    logSource,
+    container,
+    daemonLogPath,
+    daemonLogExists: fs.existsSync(daemonLogPath),
+  });
+}
+
+export interface FollowCallbacks {
+  onLine: (line: string) => void;
+  /** Stream headers / end-of-stream notices (rendered dim by the CLI). */
+  onMeta: (line: string) => void;
+}
+
+export interface FollowHandle {
+  /** Resolves with the final stream's exit code (0 after stop()). */
+  done: Promise<number>;
+  /** Stop tailing without stopping the service. */
+  stop: () => void;
+}
+
+export interface FollowDeps {
+  spawn?: typeof spawnStreaming;
+  quickExitMs?: number;
+}
+
+/**
+ * Follow the first working candidate, falling through the tiers when a
+ * command quick-exits. Lines seen before a source survives the quick-exit
+ * window are buffered (flushed on confirm, discarded on fallback) so a broken
+ * tier's error spew never reaches the caller.
+ */
+export function followTailSources(
+  candidates: TailSource[],
+  { onLine, onMeta }: FollowCallbacks,
+  deps: FollowDeps = {},
+): FollowHandle {
+  const spawn = deps.spawn ?? spawnStreaming;
+  const quickExitMs = deps.quickExitMs ?? DAEMON_QUICK_EXIT_MS;
+
+  let stopped = false;
+  let current: ReturnType<typeof spawnStreaming> | null = null;
+  let resolveDone!: (code: number) => void;
+  const done = new Promise<number>((resolve) => (resolveDone = resolve));
+
+  const tryCandidate = (index: number): void => {
+    if (index >= candidates.length) {
+      onMeta("── no live log stream available ──");
+      resolveDone(1);
+      return;
+    }
+    const source = candidates[index];
+    const buffer: string[] = [];
+    let confirmed = false;
+    let quickTimer: ReturnType<typeof setTimeout> | null = setTimeout(() => {
+      quickTimer = null;
+      confirmed = true;
+      onMeta(`── ${source.label} ──`);
+      for (const line of buffer) onLine(line);
+      buffer.length = 0;
+    }, quickExitMs);
+
+    // Detached: the tail child leads its own process group, so stop() can
+    // SIGTERM the whole tree. Shell-wrapped commands (e.g. `docker logs -f
+    // $(docker ps …)`) keep `sh` as the parent — killing only its pid would
+    // orphan a live `docker logs -f`. The child's stdio pipes keep the event
+    // loop alive; killing it stops the stream, never the service.
+    current = spawn(
+      source.cmd,
+      source.args,
+      { ...process.env },
+      (line) => (confirmed ? onLine(line) : buffer.push(line)),
+      (exitCode) => {
+        if (quickTimer !== null) {
+          clearTimeout(quickTimer);
+          quickTimer = null;
+        }
+        if (stopped) {
+          resolveDone(0);
+          return;
+        }
+        if (!confirmed) {
+          buffer.length = 0;
+          tryCandidate(index + 1);
+          return;
+        }
+        onMeta(`── log stream ended (${exitCode}) ──`);
+        resolveDone(exitCode);
+      },
+      source.cwd,
+      true,
+    );
+  };
+
+  tryCandidate(0);
+
+  return {
+    done,
+    stop: (): void => {
+      stopped = true;
+      if (!current) return;
+      try {
+        // Negative pid → the child's whole process group (it is detached).
+        if (current.pid) process.kill(-current.pid, "SIGTERM");
+        else current.kill("SIGTERM");
+      } catch {
+        try {
+          current.kill("SIGTERM");
+        } catch {
+          /* already exited */
+        }
+      }
+    },
+  };
+}

package/src/shared/environment-manager/make-targets.ts

@@ -0,0 +1,78 @@
+/**
+ * Lightweight Makefile target discovery.
+ *
+ * `den graph`/`up` annotate each node with the make targets it exposes and
+ * warn when the ones den drives (`setup`/`up`/`down`) are missing. We parse
+ * the root `Makefile` plus any `include`d `*.mk` for target definitions —
+ * deliberately simple (no variable expansion), reading a fixed set of files.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+
+export interface MakeTargets {
+  /** Sorted, unique runnable target names. */
+  targets: string[];
+  /** Makefiles parsed, relative to repo root. */
+  files: string[];
+}
+
+const TARGET_LINE = /^([A-Za-z0-9][A-Za-z0-9_.-]*)[ \t]*:(?!=)/gm;
+const INCLUDE_LINE = /^[ \t]*-?include[ \t]+(.+)$/gm;
+
+function readSafe(abs: string): string | null {
+  try {
+    return fs.readFileSync(abs, "utf-8");
+  } catch {
+    return null;
+  }
+}
+
+function collectTargets(content: string, into: Set<string>): void {
+  for (const m of content.matchAll(TARGET_LINE)) {
+    const name = m[1];
+    if (name.startsWith(".")) continue; // .PHONY, .DEFAULT_GOAL, …
+    into.add(name);
+  }
+}
+
+/** Parse the root Makefile (+ statically-includable `*.mk`) for target names. */
+export function listMakeTargets(repoPath: string): MakeTargets {
+  const rootMk = path.join(repoPath, "Makefile");
+  const rootContent = readSafe(rootMk);
+  if (rootContent === null) return { targets: [], files: [] };
+
+  const targets = new Set<string>();
+  const filesParsed: string[] = ["Makefile"];
+  collectTargets(rootContent, targets);
+
+  // Follow one level of `include`d makefiles with literal (non-variable) paths.
+  const seen = new Set<string>([rootMk]);
+  for (const inc of rootContent.matchAll(INCLUDE_LINE)) {
+    const spec = inc[1].trim();
+    if (spec.includes("$")) continue; // variable path — can't resolve statically
+    for (const rel of spec.split(/\s+/)) {
+      const abs = path.join(repoPath, rel);
+      if (seen.has(abs)) continue;
+      seen.add(abs);
+      const content = readSafe(abs);
+      if (content === null) continue;
+      filesParsed.push(rel);
+      collectTargets(content, targets);
+    }
+  }
+
+  return { targets: [...targets].sort(), files: filesParsed };
+}
+
+/** Split `required` into those present in the repo's Makefile(s) and those missing. */
+export function requireTargets(
+  repoPath: string,
+  required: string[],
+): { present: string[]; missing: string[] } {
+  const have = new Set(listMakeTargets(repoPath).targets);
+  const present: string[] = [];
+  const missing: string[] = [];
+  for (const t of required) (have.has(t) ? present : missing).push(t);
+  return { present, missing };
+}

package/src/shared/environment-manager/opctl.ts

@@ -0,0 +1,334 @@
+/**
+ * opctl runtime helpers.
+ *
+ * opctl, when installed, is a container control plane; Den leans on it for
+ * the parts that would otherwise need stale per-repo
+ * declarations:
+ *   - shutting a service down  → `opctl container down <name>` (invariant: every
+ *     service's containers can be cleanly stopped this way)
+ *   - identifying a service's containers → the `container.name` declared in the
+ *     repo's dev/up op, captured at spawn time and/or parsed on demand.
+ */
+
+import { execFileSync, execSync } from "child_process";
+import * as fs from "fs";
+import * as path from "path";
+import { parse as parseYaml } from "yaml";
+
+/** Candidate dev/up op manifests for a repo, in resolution order. */
+function devOpManifestPaths(repoPath: string): string[] {
+  return [
+    path.join(repoPath, ".opspec", "dev", "op.yml"),
+    path.join(repoPath, ".opspec", "up", "op.yml"),
+    path.join(repoPath, ".opspec", "local", "up", "op.yml"), // infra-stack
+    path.join(repoPath, "apps", "backend", ".opspec", "up", "op.yml"), // monorepo (shop)
+    path.join(repoPath, "apps", "backend", ".opspec", "dev", "op.yml"),
+  ];
+}
+
+/** Recursively collect every `container.name` string in an opctl op tree. */
+function collectContainerNames(node: unknown, out: Set<string>): void {
+  if (!node || typeof node !== "object") return;
+  if (Array.isArray(node)) {
+    for (const item of node) collectContainerNames(item, out);
+    return;
+  }
+  const obj = node as Record<string, unknown>;
+  const container = obj.container;
+  if (container && typeof container === "object" && !Array.isArray(container)) {
+    const name = (container as Record<string, unknown>).name;
+    if (typeof name === "string" && name.trim()) out.add(name.trim());
+  }
+  for (const value of Object.values(obj)) collectContainerNames(value, out);
+}
+
+/**
+ * The opctl container names a repo's dev/up op declares (e.g. ["links-api"]).
+ * These are the short `container.name` labels `opctl container down` matches on.
+ * Returns [] when no op manifest is found or it can't be parsed.
+ */
+export function devOpContainerNames(repoPath: string): string[] {
+  for (const manifestPath of devOpManifestPaths(repoPath)) {
+    try {
+      if (!fs.existsSync(manifestPath)) continue;
+      const op = parseYaml(fs.readFileSync(manifestPath, "utf-8"));
+      const names = new Set<string>();
+      collectContainerNames(op, names);
+      if (names.size > 0) return [...names];
+    } catch {
+      /* unparseable — try the next candidate */
+    }
+  }
+  return [];
+}
+
+let opctlAvailableCache: boolean | null = null;
+
+/** Whether the `opctl` CLI is on PATH (cached for the process lifetime). */
+export function isOpctlAvailable(): boolean {
+  if (opctlAvailableCache !== null) return opctlAvailableCache;
+  try {
+    execSync("command -v opctl", { stdio: "ignore" });
+    opctlAvailableCache = true;
+  } catch {
+    opctlAvailableCache = false;
+  }
+  return opctlAvailableCache;
+}
+
+/**
+ * Docker names opctl-spawned containers `opctl_<base>_<8-hex>`. Capture the
+ * base so down can find them even when `opctl container ls` is empty or
+ * opctl itself can't elevate (`sudo -E` rejected).
+ */
+const DOCKER_OPCTL_NAME = /^opctl_(.+)_([0-9a-f]{8})$/i;
+
+/**
+ * Whether a docker container name is opctl's container for exactly `base`.
+ *
+ * A bare `startsWith("opctl_<base>_")` test is ambiguous whenever one base is a
+ * prefix of another that contains an underscore — base `api` would claim
+ * `opctl_api_v2_<hash>` — and these names feed `docker rm -f`, so a wrong match
+ * tears down someone else's service. Requiring the remainder to be a single
+ * trailing segment (opctl's hash never contains `_`) makes the base comparison
+ * exact without depending on the hash's length or alphabet.
+ */
+export function isOpctlContainerFor(name: string, base: string): boolean {
+  const prefix = `opctl_${base}_`;
+  if (!name.startsWith(prefix)) return false;
+  const suffix = name.slice(prefix.length);
+  return suffix.length > 0 && !suffix.includes("_");
+}
+
+function runningDockerNames(): string[] {
+  return execSync('docker ps --format "{{.Names}}"', {
+    encoding: "utf-8",
+    stdio: ["ignore", "pipe", "ignore"],
+    timeout: 5_000,
+  })
+    .split("\n")
+    .map((l) => l.trim())
+    .filter(Boolean);
+}
+
+/**
+ * Running opctl containers discovered via the docker socket (no sudo).
+ * `name` is the full docker name; `base` is the opctl `container.name`.
+ */
+export function listRunningDockerOpctlContainers(): {
+  name: string;
+  base: string;
+}[] {
+  try {
+    const rows: { name: string; base: string }[] = [];
+    for (const name of runningDockerNames()) {
+      const m = name.match(DOCKER_OPCTL_NAME);
+      if (!m) continue;
+      rows.push({ name, base: m[1] });
+    }
+    return rows;
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Names of currently-running opctl containers.
+ *
+ * Prefers `opctl container ls` (NAME column, trailing `-<hash>` stripped to
+ * the base `container.name`), and unions in anything docker still shows as
+ * `opctl_<base>_<hash>`. Docker is the source of truth when opctl can't
+ * elevate or its ls is empty while containers keep running — exactly the
+ * failure mode that made links-api / notes-api look "already stopped"
+ * (or report a successful scripted down) while their containers stayed up.
+ */
+export function listRunningOpctlContainers(): { name: string; base: string }[] {
+  const byBase = new Map<string, { name: string; base: string }>();
+
+  if (isOpctlAvailable()) {
+    try {
+      const out = execSync("opctl container ls", {
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "ignore"],
+      });
+      const lines = out.split("\n");
+      // Skip the first line only when it is actually a header row.
+      const firstToken = lines[0]?.trim().split(/\s+/)[0];
+      const body =
+        firstToken === "NAME" || firstToken === "ID" ? lines.slice(1) : lines;
+      for (const line of body) {
+        const name = line.trim().split(/\s+/)[0];
+        if (!name) continue;
+        const base = name.replace(/-[0-9a-f]{8}$/i, "");
+        byBase.set(base, { name, base });
+      }
+    } catch {
+      /* fall through to docker */
+    }
+  }
+
+  for (const c of listRunningDockerOpctlContainers()) {
+    if (!byBase.has(c.base)) byBase.set(c.base, c);
+  }
+  return [...byBase.values()];
+}
+
+/**
+ * Resolve a service's RUNNING docker container purely by opctl naming
+ * convention — docker names opctl-spawned containers `opctl_<base>_<hash>`.
+ *
+ * Candidates for `<base>`, in order: the repo's op-manifest container name
+ * matching the subsystem, any op-manifest container name, then the repo name
+ * itself. Returns the full docker container name, or null when nothing
+ * matches (not running, not containerized, or docker unavailable).
+ */
+export function findConventionalContainer(
+  repoName: string,
+  repoPath?: string,
+  subsystem?: string,
+): string | null {
+  let running: string[];
+  try {
+    running = runningDockerNames();
+  } catch {
+    return null;
+  }
+  const declared = repoPath ? devOpContainerNames(repoPath) : [];
+  // Only candidates with NAME AFFINITY to the request — a blind scan of the
+  // declared list would hand a sidecar (e.g. shop-postgres) to an api
+  // request. The full declared list is only trusted when it's unambiguous.
+  const candidates = [
+    ...(subsystem
+      ? declared.filter(
+          (n) =>
+            n === subsystem ||
+            n === `${repoName}-${subsystem}` ||
+            n.endsWith(`-${subsystem}`),
+        )
+      : []),
+    repoName,
+    ...(declared.length === 1 ? declared : []),
+  ];
+  for (const base of candidates) {
+    const hit = running.find((n) => isOpctlContainerFor(n, base));
+    if (hit) return hit;
+  }
+  return null;
+}
+
+/** The running opctl containers for one `container.name`, and their origin. */
+export interface OpctlContainerInfo {
+  /** Full docker names (`opctl_<base>_<hash>`) — usually one, see below. */
+  names: string[];
+  /** Host paths bind-mounted into any of those containers. */
+  mountSources: string[];
+  /** Earliest container start time, or null when docker didn't report one. */
+  startedAt: Date | null;
+}
+
+/**
+ * Inspect the running containers for an opctl `container.name`.
+ *
+ * Bind-mount sources are the only trail back to the checkout a container was
+ * started from — opctl's labels record the container name, not the source repo.
+ * That distinction is what lets a caller tell "my own checkout is already
+ * running, untracked" apart from "another checkout occupies this name".
+ *
+ * A base normally has exactly one container, but a leaked container from an
+ * earlier run can coexist with the current one under a different hash. All of
+ * them are inspected and their mounts unioned: deciding adoption from whichever
+ * one `docker ps` happened to list first would make the answer depend on
+ * container age. Returns null when nothing runs or docker can't be reached.
+ */
+export function inspectOpctlContainer(base: string): OpctlContainerInfo | null {
+  try {
+    const names = runningDockerNames().filter((n) =>
+      isOpctlContainerFor(n, base),
+    );
+    if (names.length === 0) return null;
+
+    const mountSources = new Set<string>();
+    let startedAt: Date | null = null;
+    for (const name of names) {
+      const raw = execFileSync("docker", ["inspect", name], {
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "ignore"],
+        timeout: 5_000,
+      });
+      const entry = (JSON.parse(raw) as unknown[])[0] as
+        | {
+            Mounts?: { Source?: unknown }[];
+            State?: { StartedAt?: unknown };
+          }
+        | undefined;
+      for (const m of entry?.Mounts ?? []) {
+        if (typeof m.Source === "string" && m.Source)
+          mountSources.add(m.Source);
+      }
+      const startedRaw = entry?.State?.StartedAt;
+      const started =
+        typeof startedRaw === "string" ? new Date(startedRaw) : null;
+      if (started && !Number.isNaN(started.getTime())) {
+        if (!startedAt || started < startedAt) startedAt = started;
+      }
+    }
+    return { names, mountSources: [...mountSources], startedAt };
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Force-stop docker containers for opctl `container.name` bases when
+ * `opctl container down` cannot elevate (typical: sudoers grants NOPASSWD for
+ * opctl but not SETENV, so opctl's `sudo -E` is rejected).
+ *
+ * Docker names opctl containers `opctl_<base>_<hash>`; `docker rm -f` stops and
+ * removes them without needing sudo when the user can talk to the docker socket.
+ */
+export function forceStopOpctlContainers(bases: string[]): {
+  stopped: string[];
+  failed: string[];
+} {
+  const stopped: string[] = [];
+  const failed: string[] = [];
+  if (bases.length === 0) return { stopped, failed };
+
+  let running: string[];
+  try {
+    running = runningDockerNames();
+  } catch {
+    return { stopped, failed: [...bases] };
+  }
+
+  const names = running.filter((n) =>
+    bases.some((base) => isOpctlContainerFor(n, base)),
+  );
+  for (const name of names) {
+    try {
+      execFileSync("docker", ["rm", "-f", name], {
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "ignore"],
+        timeout: 30_000,
+      });
+      stopped.push(name);
+    } catch {
+      failed.push(name);
+    }
+  }
+  return { stopped, failed };
+}
+
+/**
+ * Base container names that newly appeared between two `opctl container ls`
+ * snapshots — i.e. the containers an `up` just spawned.
+ */
+export function diffSpawnedContainers(
+  before: { base: string }[],
+  after: { base: string }[],
+): string[] {
+  const had = new Set(before.map((c) => c.base));
+  const fresh = new Set<string>();
+  for (const c of after) if (!had.has(c.base)) fresh.add(c.base);
+  return [...fresh];
+}