@monster-orange/den 0.0.0 → 0.2.0

package/src/shared/stacks/manifest-commands.ts

@@ -0,0 +1,118 @@
+/**
+ * Commands den supplies for a repo whose `.den.yml` has a `compose:`
+ * section: the stack runs behind the edge with no host ports. A command the
+ * repo declares itself (services.<svc>.commands.local) always wins.
+ */
+import * as path from "path";
+import { getDistRoot } from "../paths.js";
+import { readManifest } from "../manifest/read-manifest.js";
+import type { RawManifest } from "../types/index.js";
+import { manifestStackName } from "./config.js";
+import { containerName, parseTarget, projectName } from "./render.js";
+
+import { product } from "../product.js";
+type CommandDef = string | { cmd: string; detachable?: boolean };
+
+/** Whether `subsystem` is the repo's primary service (the one the stack belongs to). */
+function isPrimary(manifest: RawManifest, subsystem: string): boolean {
+  const keys = Object.keys(manifest.services ?? {});
+  if (manifest.entrypoint) return manifest.entrypoint === subsystem;
+  return keys.length <= 1;
+}
+
+function denBin(): string {
+  return JSON.stringify(path.join(getDistRoot(), "bin", `${product().cli}`));
+}
+
+/** The synthesized stack commands, or {} when the repo isn't a compose stack. */
+export function composeCommands(
+  manifest: RawManifest,
+  subsystem: string,
+  repoName: string,
+  workspace: string,
+): Record<string, CommandDef> {
+  if (!manifest.compose || !isPrimary(manifest, subsystem)) return {};
+  // `docker compose up -d` returns once the containers are started, so these
+  // finish on their own: never daemonize them.
+  const stack = (sub: string) => ({
+    cmd: `${denBin()} stack ${sub} ${manifestStackName(manifest, repoName)} -w ${workspace}`,
+    detachable: false,
+  });
+  return {
+    up: stack("up"),
+    down: stack("down"),
+    restart: stack("restart"),
+    logs: stack("logs"),
+  };
+}
+
+/**
+ * Shell health check for a compose stack. The entry service is healthy when
+ * every container is running and healthy; any other manifest service names a
+ * compose service and is checked on its own (`--service`).
+ */
+export function composeHealthCmd(
+  manifest: RawManifest,
+  subsystem: string,
+  repoName: string,
+  workspace: string,
+): string | null {
+  if (!manifest.compose) return null;
+  const base = `${denBin()} stack health ${manifestStackName(manifest, repoName)} -w ${workspace}`;
+  return isPrimary(manifest, subsystem)
+    ? base
+    : `${base} --service ${JSON.stringify(subsystem)}`;
+}
+
+/**
+ * The containers a compose-stack service runs as, best match first ([] when
+ * the repo isn't a compose stack). Any other manifest service names a compose
+ * service; the primary one also falls back to its routes' target services.
+ * `stackWorkspace` is the Compose project suffix (see resolveStackWorkspaces).
+ */
+export function composeServiceContainers(
+  manifest: RawManifest,
+  subsystem: string,
+  repoName: string,
+  stackWorkspace: string,
+): string[] {
+  if (!manifest.compose) return [];
+  const project = projectName(
+    manifestStackName(manifest, repoName),
+    stackWorkspace,
+  );
+  const services = [subsystem];
+  if (isPrimary(manifest, subsystem))
+    for (const target of Object.values(manifest.compose.routes ?? {})) {
+      try {
+        services.push(parseTarget(target).service);
+      } catch {
+        /* a malformed route is reported by the stack commands */
+      }
+    }
+  return [...new Set(services)].map((svc) => containerName(project, svc));
+}
+
+/** Declared local commands layered over the synthesized compose ones. */
+export function effectiveLocalCommands(
+  manifest: RawManifest,
+  subsystem: string,
+  repoName: string,
+  workspace: string,
+): Record<string, CommandDef> {
+  return {
+    ...composeCommands(manifest, subsystem, repoName, workspace),
+    ...(manifest.services?.[subsystem]?.commands?.local ?? {}),
+  };
+}
+
+/** Effective local commands for a resolved service, read from its manifest. */
+export function localCommandsFor(
+  svc: { repoPath: string; repoName: string; subsystem: string },
+  workspace: string,
+): Record<string, CommandDef> {
+  const m = readManifest(svc.repoPath);
+  return m
+    ? effectiveLocalCommands(m, svc.subsystem, svc.repoName, workspace)
+    : {};
+}

package/src/shared/stacks/render.ts

@@ -0,0 +1,253 @@
+/**
+ * Pure planning for stacks: project names, container names, the compose
+ * override, the link networks and the edge routes. No I/O.
+ */
+import * as yaml from "yaml";
+import { edgeDomains, routeHost } from "../edge/render.js";
+import type { EdgeConfig } from "../edge/types.js";
+import type { ResolvedProject, StackConfig } from "./types.js";
+
+import { product } from "../product.js";
+const NAME_RE = /^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/;
+
+export function assertStackName(n: string): void {
+  if (!NAME_RE.test(n))
+    throw new Error(
+      `stack name "${n}" must be lowercase letters, digits and dashes`,
+    );
+}
+
+export function projectName(stack: string, workspace: string): string {
+  return `${stack}-${workspace}`;
+}
+
+/**
+ * Container name for a stack service. Always Compose's per-project default:
+ * the override resets any pinned `container_name`, which would otherwise
+ * collide when the same repo runs in two workspaces.
+ */
+export function containerName(project: string, service: string): string {
+  return `${project}-${service}-1`;
+}
+
+export function linkNetwork(
+  consumer: string,
+  provider: string,
+  ws: string,
+): string {
+  return `${consumer}-${ws}--${provider}`;
+}
+
+export function parseTarget(target: string): { service: string; port: number } {
+  const m = /^([a-zA-Z0-9][\w.-]*):(\d{1,5})$/.exec(target);
+  if (!m) throw new Error(`route target "${target}" must be <service>:<port>`);
+  return { service: m[1], port: Number(m[2]) };
+}
+
+/** Expand `{stack.service}` placeholders to container names. */
+export function expandEnv(
+  env: Record<string, string>,
+  ws: string,
+  known: Record<string, StackConfig>,
+): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const [k, v] of Object.entries(env))
+    out[k] = v.replace(
+      /\{([a-z0-9-]+)\.([\w.-]+)\}/g,
+      (_m, s: string, svc: string) => {
+        if (!known[s]) throw new Error(`env ${k} names unknown stack "${s}"`);
+        return containerName(projectName(s, ws), svc);
+      },
+    );
+  return out;
+}
+
+/** Providers named in `{stack.service}` placeholders → their services. */
+export function providerServices(stack: StackConfig): Record<string, string[]> {
+  const out: Record<string, string[]> = {};
+  for (const v of Object.values(stack.env))
+    for (const m of v.matchAll(/\{([a-z0-9-]+)\.([\w.-]+)\}/g))
+      (out[m[1]] ??= []).includes(m[2]) || out[m[1]].push(m[2]);
+  return out;
+}
+
+/** Start order: every stack after the stacks it uses. Throws on cycles. */
+export function startOrder(
+  names: string[],
+  all: Record<string, StackConfig>,
+): string[] {
+  const order: string[] = [];
+  const state = new Map<string, "visiting" | "done">();
+  const visit = (n: string, trail: string[]) => {
+    if (!all[n]) throw new Error(`unknown stack "${n}"`);
+    if (state.get(n) === "done") return;
+    if (state.get(n) === "visiting")
+      throw new Error(
+        `stacks use each other in a cycle: ${[...trail, n].join(" → ")}`,
+      );
+    state.set(n, "visiting");
+    for (const u of all[n].uses) visit(u, [...trail, n]);
+    state.set(n, "done");
+    order.push(n);
+  };
+  names.forEach((n) => visit(n, []));
+  return order;
+}
+
+/**
+ * Services that refer to a provider: their resolved environment names one of
+ * the provider's container names. Only these join the link network, so a
+ * consumer's postgres/redis never become resolvable from the provider.
+ */
+export function linkedServices(
+  stack: StackConfig,
+  provider: string,
+  resolved: ResolvedProject,
+  ws: string,
+): string[] {
+  const names = (providerServices(stack)[provider] ?? []).map((svc) =>
+    containerName(projectName(provider, ws), svc),
+  );
+  return Object.entries(resolved.services)
+    .filter(([, def]) =>
+      Object.values(def.environment ?? {}).some(
+        (v) => typeof v === "string" && names.some((n) => v.includes(n)),
+      ),
+    )
+    .map(([svc]) => svc)
+    .sort();
+}
+
+/**
+ * The compose override: drop every published host port and pinned container
+ * name, and put the services that refer to a provider on its link network.
+ * Written as text because `!reset` is a Compose tag, not plain YAML.
+ */
+export function renderOverride(
+  stack: StackConfig,
+  resolved: ResolvedProject,
+  ws: string,
+): string {
+  const links = stack.uses.map((p) => ({
+    net: linkNetwork(stack.name, p, ws),
+    services: linkedServices(stack, p, resolved, ws),
+  }));
+  const unknown = Object.keys(stack.serviceEnv ?? {}).filter(
+    (s) => !(s in resolved.services),
+  );
+  if (unknown.length)
+    throw new Error(
+      `stack "${stack.name}": service_env names "${unknown.join('", "')}", ` +
+        `which the compose project does not have (${Object.keys(resolved.services).sort().join(", ")})`,
+    );
+  const lines = [
+    `# Generated by \`${product().cli} stack up\`: no host ports (reach services via the`,
+    "# edge) and no pinned container names (each workspace is its own project).",
+    "services:",
+  ];
+  for (const [svc, def] of Object.entries(resolved.services).sort()) {
+    const body: string[] = [];
+    if (def.ports && def.ports.length) body.push("    ports: !reset []");
+    if (def.container_name) body.push("    container_name: !reset null");
+    const check = stack.healthchecks?.[svc];
+    if (check)
+      body.push("    healthcheck:", `      test: ${JSON.stringify(check)}`);
+    const vars = stack.serviceEnv?.[svc];
+    if (vars && Object.keys(vars).length) {
+      // Compose merges `environment` by key across files, so these win.
+      body.push("    environment:");
+      for (const [k, v] of Object.entries(vars).sort())
+        body.push(`      ${JSON.stringify(k)}: ${JSON.stringify(v)}`);
+    }
+    const mine = links
+      .filter((l) => l.services.includes(svc))
+      .map((l) => l.net);
+    if (mine.length) {
+      // Compose merges service networks by key, so listing the base ones keeps them.
+      const base = Object.keys(def.networks ?? { default: null });
+      body.push("    networks:");
+      for (const n of [...base, ...mine])
+        body.push(`      ${JSON.stringify(n)}: {}`);
+    }
+    if (body.length) lines.push(`  ${JSON.stringify(svc)}:`, ...body);
+  }
+  if (!lines.some((l) => /^ {2}"/.test(l))) lines.push("  {}");
+  const used = links.filter((l) => l.services.length).map((l) => l.net);
+  if (used.length) {
+    lines.push("networks:");
+    for (const n of used)
+      lines.push(
+        `  ${JSON.stringify(n)}:`,
+        `    name: ${JSON.stringify(n)}`,
+        "    external: true",
+      );
+  }
+  return lines.join("\n") + "\n";
+}
+
+/** Docker networks the edge must join to reach each routed service. */
+export function routedNetworks(
+  stack: StackConfig,
+  resolved: ResolvedProject,
+): string[] {
+  const nets = new Set<string>();
+  for (const target of Object.values(stack.routes)) {
+    const { service } = parseTarget(target);
+    const def = resolved.services[service];
+    if (!def)
+      throw new Error(
+        `stack ${stack.name}: route target service "${service}" not in compose`,
+      );
+    const first = Object.keys(def.networks ?? { default: null })[0];
+    nets.add(resolved.networks?.[first]?.name ?? `${resolved.name}_${first}`);
+  }
+  return [...nets].sort();
+}
+
+/** Edge dynamic config for one stack: `<route>.<stack>.<domain>` → container. */
+export function renderStackRoutes(
+  edge: EdgeConfig,
+  stack: StackConfig,
+  resolved: ResolvedProject,
+): string {
+  const routers: Record<string, unknown> = {};
+  const services: Record<string, unknown> = {};
+  for (const [route, target] of Object.entries(stack.routes).sort()) {
+    if (!NAME_RE.test(route))
+      throw new Error(
+        `route "${route}" must be lowercase letters, digits and dashes`,
+      );
+    const { service, port } = parseTarget(target);
+    const key = `stack-${stack.name}-${route}`;
+    routers[key] = {
+      entryPoints: ["websecure"],
+      rule: edgeDomains(edge)
+        .map((d) => `Host(\`${routeHost(`${route}.${stack.name}`, d)}\`)`)
+        .join(" || "),
+      service: key,
+      tls: {},
+    };
+    services[key] = {
+      loadBalancer: {
+        servers: [
+          {
+            url: `http://${containerName(resolved.name, service)}:${port}`,
+          },
+        ],
+      },
+    };
+  }
+  return (
+    `# Generated by \`${product().cli} stack up ${stack.name}\`.\n` +
+    yaml.stringify({ http: { routers, services } })
+  );
+}
+
+/**
+ * Compose-project suffix for a den workspace: its own name, except that
+ * the `default` workspace keeps the original `main` projects (and so their
+ * volumes and data).
+ */
+export function stackWorkspace(denWorkspace: string): string {
+  return denWorkspace === "default" ? "main" : denWorkspace;
+}

package/src/shared/stacks/runtime.ts

@@ -0,0 +1,334 @@
+/** Side effects for stacks: docker compose, link networks, edge routes. */
+import { randomBytes } from "crypto";
+import { spawnSync } from "child_process";
+import * as fs from "fs";
+import * as path from "path";
+import { getReposRoot } from "../den-config.js";
+import { edgeNames } from "../edge/render.js";
+import type { EdgeConfig } from "../edge/types.js";
+import {
+  containerName,
+  expandEnv,
+  linkNetwork,
+  projectName,
+  providerServices,
+  renderOverride,
+  renderStackRoutes,
+  routedNetworks,
+} from "./render.js";
+import type { ResolvedProject, StackConfig } from "./types.js";
+
+function run(
+  cmd: string,
+  args: string[],
+  opts: { cwd?: string; env?: NodeJS.ProcessEnv; inherit?: boolean } = {},
+) {
+  const r = spawnSync(cmd, args, {
+    cwd: opts.cwd,
+    encoding: "utf-8",
+    env: { ...process.env, ...opts.env },
+    stdio: opts.inherit ? "inherit" : "pipe",
+    maxBuffer: 64 * 1024 * 1024,
+  });
+  if (r.error) throw new Error(`${cmd}: ${r.error.message}`);
+  return {
+    code: r.status ?? 1,
+    out: (r.stdout ?? "").trim(),
+    err: (r.stderr ?? "").trim(),
+  };
+}
+
+function must(
+  cmd: string,
+  args: string[],
+  opts: Parameters<typeof run>[2] = {},
+): string {
+  const r = run(cmd, args, opts);
+  if (r.code !== 0)
+    throw new Error(
+      `${cmd} ${args.slice(0, 6).join(" ")}… failed:\n${r.err || r.out}`,
+    );
+  return r.out;
+}
+
+export interface StackContext {
+  edge: EdgeConfig;
+  stacks: Record<string, StackConfig>;
+  workspace: string;
+}
+
+function stackDir(ctx: StackContext, s: StackConfig): string {
+  return path.join(
+    ctx.edge.stateDir,
+    "stacks",
+    projectName(s.name, ctx.workspace),
+  );
+}
+
+export function repoDir(s: StackConfig): string {
+  const dir = path.join(
+    s.repoPath ?? path.join(getReposRoot(), s.repo),
+    s.dir ?? "",
+  );
+  if (!fs.existsSync(dir))
+    throw new Error(
+      `stack ${s.name}: ${dir} does not exist (clone ${s.repo} first)`,
+    );
+  return dir;
+}
+
+/** Generated local-only secrets, created once per stack + workspace. */
+function generatedEnv(
+  ctx: StackContext,
+  s: StackConfig,
+): Record<string, string> {
+  const file = path.join(stackDir(ctx, s), "generated.env");
+  const have: Record<string, string> = {};
+  if (fs.existsSync(file))
+    for (const line of fs.readFileSync(file, "utf-8").split("\n")) {
+      const i = line.indexOf("=");
+      if (i > 0) have[line.slice(0, i)] = line.slice(i + 1);
+    }
+  let changed = false;
+  for (const k of s.generate)
+    if (!have[k]) {
+      have[k] = randomBytes(32).toString("base64url");
+      changed = true;
+    }
+  if (changed) {
+    fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 });
+    fs.writeFileSync(
+      file,
+      Object.entries(have)
+        .map(([k, v]) => `${k}=${v}`)
+        .join("\n") + "\n",
+      { mode: 0o600 },
+    );
+  }
+  return have;
+}
+
+function composeEnv(ctx: StackContext, s: StackConfig): NodeJS.ProcessEnv {
+  return {
+    ...generatedEnv(ctx, s),
+    ...expandEnv(s.env, ctx.workspace, ctx.stacks),
+    COMPOSE_PROJECT_NAME: projectName(s.name, ctx.workspace),
+    ...(s.profiles.length ? { COMPOSE_PROFILES: s.profiles.join(",") } : {}),
+  };
+}
+
+function baseArgs(ctx: StackContext, s: StackConfig): string[] {
+  return [
+    "compose",
+    "-p",
+    projectName(s.name, ctx.workspace),
+    ...s.files.flatMap((f) => ["-f", f]),
+  ];
+}
+
+function resolve(ctx: StackContext, s: StackConfig): ResolvedProject {
+  const out = must(
+    "docker",
+    [...baseArgs(ctx, s), "config", "--format", "json"],
+    {
+      cwd: repoDir(s),
+      env: composeEnv(ctx, s),
+    },
+  );
+  return JSON.parse(out) as ResolvedProject;
+}
+
+function overrideArgs(ctx: StackContext, s: StackConfig): string[] {
+  return [
+    ...baseArgs(ctx, s),
+    "-f",
+    path.join(stackDir(ctx, s), "override.yml"),
+  ];
+}
+
+function ensureNetwork(name: string): void {
+  if (run("docker", ["network", "inspect", name]).code !== 0)
+    must("docker", ["network", "create", name]);
+}
+
+/** `docker network connect`, ignoring "already connected". Container-name alias only. */
+function connect(network: string, container: string): void {
+  const r = run("docker", ["network", "connect", network, container]);
+  if (r.code !== 0 && !/already exists|already connected/i.test(r.err))
+    throw new Error(`docker network connect ${network} ${container}: ${r.err}`);
+}
+
+function running(container: string): boolean {
+  return (
+    run("docker", ["inspect", "-f", "{{.State.Running}}", container]).out ===
+    "true"
+  );
+}
+
+export function stackUp(
+  ctx: StackContext,
+  s: StackConfig,
+  opts: { build?: boolean } = {},
+) {
+  const resolved = resolve(ctx, s);
+  // Provider side of each link: its referenced services join a private network.
+  const needs = providerServices(s);
+  for (const p of s.uses) {
+    const net = linkNetwork(s.name, p, ctx.workspace);
+    ensureNetwork(net);
+    for (const svc of needs[p] ?? []) {
+      const c = containerName(projectName(p, ctx.workspace), svc);
+      if (!running(c))
+        throw new Error(
+          `stack ${s.name} needs ${p}'s ${svc} (${c}), which is not running`,
+        );
+      connect(net, c);
+    }
+  }
+  const dir = stackDir(ctx, s);
+  fs.mkdirSync(dir, { recursive: true });
+  fs.writeFileSync(
+    path.join(dir, "override.yml"),
+    renderOverride(s, resolved, ctx.workspace),
+  );
+  must(
+    "docker",
+    [
+      ...overrideArgs(ctx, s),
+      "up",
+      "-d",
+      ...(opts.build ? ["--build"] : []),
+      "--remove-orphans",
+    ],
+    { cwd: repoDir(s), env: composeEnv(ctx, s), inherit: true },
+  );
+  // This stack as a provider: consumers already running keep their link
+  // network, but a restarted provider's new containers must rejoin it.
+  for (const c of Object.values(ctx.stacks)) {
+    if (!c.uses.includes(s.name)) continue;
+    const net = linkNetwork(c.name, s.name, ctx.workspace);
+    if (run("docker", ["network", "inspect", net]).code !== 0) continue;
+    for (const svc of providerServices(c)[s.name] ?? [])
+      connect(net, containerName(projectName(s.name, ctx.workspace), svc));
+  }
+  const edge = edgeNames(ctx.edge.org).container;
+  const nets = routedNetworks(s, resolved);
+  for (const n of nets) connect(n, edge);
+  fs.writeFileSync(
+    path.join(
+      ctx.edge.stateDir,
+      "dynamic",
+      `stack-${projectName(s.name, ctx.workspace)}.yml`,
+    ),
+    renderStackRoutes(ctx.edge, s, resolved),
+  );
+  return { project: resolved.name, networks: nets };
+}
+
+export function stackDown(ctx: StackContext, s: StackConfig): void {
+  fs.rmSync(
+    path.join(
+      ctx.edge.stateDir,
+      "dynamic",
+      `stack-${projectName(s.name, ctx.workspace)}.yml`,
+    ),
+    { force: true },
+  );
+  const edge = edgeNames(ctx.edge.org).container;
+  const dir = repoDir(s);
+  const override = path.join(stackDir(ctx, s), "override.yml");
+  const args = fs.existsSync(override)
+    ? overrideArgs(ctx, s)
+    : baseArgs(ctx, s);
+  // The edge holds a connection to the project network; let go so `down` can remove it.
+  try {
+    for (const n of routedNetworks(s, resolve(ctx, s)))
+      run("docker", ["network", "disconnect", n, edge]);
+  } catch {
+    /* compose config failed — nothing to disconnect */
+  }
+  must("docker", [...args, "down"], {
+    cwd: dir,
+    env: composeEnv(ctx, s),
+    inherit: true,
+  });
+  const needs = providerServices(s);
+  for (const p of s.uses) {
+    const net = linkNetwork(s.name, p, ctx.workspace);
+    for (const svc of needs[p] ?? [])
+      run("docker", [
+        "network",
+        "disconnect",
+        net,
+        containerName(projectName(p, ctx.workspace), svc),
+      ]);
+    run("docker", ["network", "rm", net]);
+  }
+}
+
+export function stackPs(ctx: StackContext, s: StackConfig): string {
+  return run(
+    "docker",
+    [
+      ...baseArgs(ctx, s),
+      "ps",
+      "--format",
+      "table {{.Service}}\t{{.State}}\t{{.Status}}",
+    ],
+    {
+      cwd: repoDir(s),
+      env: composeEnv(ctx, s),
+    },
+  ).out;
+}
+
+export function stackLogs(
+  ctx: StackContext,
+  s: StackConfig,
+  args: string[],
+): number {
+  return run("docker", [...baseArgs(ctx, s), "logs", ...args], {
+    cwd: repoDir(s),
+    env: composeEnv(ctx, s),
+    inherit: true,
+  }).code;
+}
+
+/**
+ * Health of a running stack: ok when it has containers, all are running, and
+ * none reports a failing or pending Docker healthcheck.
+ */
+export function stackHealth(
+  ctx: StackContext,
+  s: StackConfig,
+  service?: string,
+): { ok: boolean; detail: string } {
+  const project = projectName(s.name, ctx.workspace);
+  const r = run("docker", [
+    "ps",
+    "-a",
+    "--filter",
+    `label=com.docker.compose.project=${project}`,
+    ...(service
+      ? ["--filter", `label=com.docker.compose.service=${service}`]
+      : []),
+    "--format",
+    '{{.Label "com.docker.compose.service"}}\t{{.State}}\t{{.Status}}',
+  ]);
+  if (r.code !== 0) return { ok: false, detail: r.err || "docker ps failed" };
+  const rows = r.out.split("\n").filter(Boolean);
+  if (!rows.length)
+    return {
+      ok: false,
+      detail: `${project}${service ? `/${service}` : ""}: not running`,
+    };
+  const bad = rows.filter((row) => {
+    const [, state, status] = row.split("\t");
+    return (
+      state !== "running" || /\((unhealthy|health: starting)\)/.test(status)
+    );
+  });
+  return bad.length
+    ? { ok: false, detail: bad.join("; ") }
+    : { ok: true, detail: `${rows.length} container(s) running` };
+}