@monster-orange/den 0.2.8 → 0.4.0

package/src/shared/stacks/namespace.ts

@@ -0,0 +1,178 @@
+/**
+ * A workspace's stack namespace: the suffix its stacks' Compose projects run
+ * under (`<stack>-<namespace>`), and so the names of their containers,
+ * volumes and networks.
+ *
+ * By default it is the workspace's own name (`default`: `main`, see
+ * stackWorkspace). A workspace can set `stack_namespace` to opt out: several
+ * workspaces may then share one namespace, and a stack keeps its data when
+ * its repo moves between workspaces. First match wins:
+ *
+ *   1. `stack_namespace:` on the workspace in workspaces.yml (per machine)
+ *   2. `stack_namespace:` on the same-named blueprint workspace
+ *   3. stackWorkspace(name)
+ *
+ * Every stack code path turns a den workspace into its suffix through
+ * resolveStackWorkspaces (status.ts), which calls this; DEN_STACK_WORKSPACE
+ * still wins there, for one process.
+ */
+import {
+  checkStackNamespace,
+  readBlueprintStackNamespaces,
+  readDenConfig,
+  type DenConfig,
+} from "../den-config.js";
+import { readWorkspacesFile } from "../environment-manager/workspaces.js";
+import { cachedRead } from "../read-cache.js";
+import type { WorkspacesFile } from "../types/index.js";
+import { stackWorkspace } from "./render.js";
+
+/** Which step gave the namespace. */
+export type StackNamespaceSource = "workspace" | "blueprint" | "default";
+
+export interface StackNamespace {
+  workspace: string;
+  /** The Compose project suffix. */
+  namespace: string;
+  source: StackNamespaceSource;
+  /** What it would be with no `stack_namespace` anywhere. */
+  defaultNamespace: string;
+  /** workspaces.yml overriding a different blueprint value, as a sentence. */
+  warnings: string[];
+}
+
+export interface StackNamespaceContext {
+  file: Pick<WorkspacesFile, "workspaces">;
+  /** Blueprint `stack_namespace` by workspace name. */
+  blueprint: Record<string, string>;
+}
+
+/**
+ * workspaces.yml and the blueprint's namespaces, as they are now. A
+ * workspaces file that cannot be read sets no namespace (its other readers
+ * report it), and so does an unreadable org config; a bad
+ * `stack_namespace` value still throws.
+ */
+export function readStackNamespaceContext(): StackNamespaceContext {
+  return cachedRead("stack-namespace-context", () => {
+    let file: StackNamespaceContext["file"] = { workspaces: {} };
+    try {
+      file = readWorkspacesFile() ?? file;
+    } catch {
+      /* unreadable: no workspace sets one */
+    }
+    let config: DenConfig | null = null;
+    try {
+      config = readDenConfig();
+    } catch {
+      /* unreadable: no blueprint sets one */
+    }
+    return {
+      file,
+      blueprint: config ? readBlueprintStackNamespaces(config) : {},
+    };
+  });
+}
+
+/**
+ * Resolve one workspace's stack namespace. A malformed `stack_namespace`
+ * (workspaces.yml or blueprint) throws, naming the key: falling back to
+ * another name would run the stacks against other volumes.
+ */
+export function resolveStackNamespace(
+  name: string,
+  ctx: StackNamespaceContext = readStackNamespaceContext(),
+): StackNamespace {
+  const defaultNamespace = stackWorkspace(name);
+  const own = checkStackNamespace(
+    `workspaces.${name}.stack_namespace (config/local/workspaces.yml)`,
+    ctx.file.workspaces?.[name]?.stack_namespace,
+  );
+  const bp = ctx.blueprint[name];
+  const warnings: string[] = [];
+  if (own) {
+    if (bp && bp !== own)
+      warnings.push(
+        `workspace ${name}: stack_namespace ${own} in workspaces.yml overrides the blueprint's stack_namespace ${bp}`,
+      );
+    return {
+      workspace: name,
+      namespace: own,
+      source: "workspace",
+      defaultNamespace,
+      warnings,
+    };
+  }
+  if (bp)
+    return {
+      workspace: name,
+      namespace: bp,
+      source: "blueprint",
+      defaultNamespace,
+      warnings,
+    };
+  return {
+    workspace: name,
+    namespace: defaultNamespace,
+    source: "default",
+    defaultNamespace,
+    warnings,
+  };
+}
+
+/** One workspace's namespace as the lists show it: resolved, or the error. */
+export interface StackNamespaceRow {
+  workspace: string;
+  /** The resolved namespace; null when its config is invalid. */
+  stack_namespace: string | null;
+  source: StackNamespaceSource | null;
+  /** The suffix with no `stack_namespace` anywhere. */
+  default: string;
+  /** True when the namespace is not the default suffix. */
+  differs: boolean;
+  warnings: string[];
+  error?: string;
+}
+
+/**
+ * Every named workspace's stack namespace, for `workspace list` and
+ * `/api/workspaces`. A bad value is that row's error, never the list's.
+ */
+export function stackNamespaceRows(
+  names: string[],
+  ctx?: StackNamespaceContext,
+): StackNamespaceRow[] {
+  let c: StackNamespaceContext | null = ctx ?? null;
+  let ctxError: string | null = null;
+  if (!c)
+    try {
+      c = readStackNamespaceContext();
+    } catch (e) {
+      ctxError = (e as Error).message;
+    }
+  return names.map((workspace) => {
+    const fallback = stackWorkspace(workspace);
+    try {
+      if (!c) throw new Error(ctxError ?? "unreadable");
+      const ns = resolveStackNamespace(workspace, c);
+      return {
+        workspace,
+        stack_namespace: ns.namespace,
+        source: ns.source,
+        default: ns.defaultNamespace,
+        differs: ns.namespace !== ns.defaultNamespace,
+        warnings: ns.warnings,
+      };
+    } catch (e) {
+      return {
+        workspace,
+        stack_namespace: null,
+        source: null,
+        default: fallback,
+        differs: false,
+        warnings: [],
+        error: (e as Error).message,
+      };
+    }
+  });
+}

package/src/shared/stacks/owners.ts

@@ -0,0 +1,303 @@
+/**
+ * Who holds a stack's Compose project: the refcount a shared stack
+ * namespace needs (namespace.ts).
+ *
+ * Without `stack_namespace` each workspace's projects are its own
+ * (`<stack>-<workspace>`), so one workspace holds each. Two workspaces with
+ * the same namespace that bring up the same stack share one project, and
+ * one of them stopping it must not stop it for the other. This mirrors the
+ * owner refcount of Den's own services (a daemon's `workspaces[]`,
+ * releaseWorkspaceDaemons):
+ *
+ *  - `stack up` from workspace W records W as an owner, with the checkout
+ *    (repo path and branch) the project runs from;
+ *  - a later `up` from another workspace on the same checkout joins; from a
+ *    different checkout or branch it is refused (`identity_mismatch`), never
+ *    a silent swap of containers;
+ *  - `stack down` from W stops the project only when no other live owner
+ *    remains; otherwise it detaches W and leaves the project running.
+ *
+ * A live owner is a workspace that is still active, still resolves to this
+ * namespace and still holds the stack: re-derived on every decision, so a
+ * deactivated or moved workspace never wedges a project. A record whose
+ * project has verifiably stopped counts no one.
+ *
+ * The record is `owners.json` in the project's own state directory, next
+ * to its generated.env and override.yml. DEN_STACK_WORKSPACE (a
+ * per-command override) bypasses it: such a command neither records nor
+ * counts owners, as before.
+ */
+import { spawnSync } from "child_process";
+import * as fs from "fs";
+import * as path from "path";
+import { getReposRoot } from "../den-config.js";
+import { readCatalog } from "../environment-manager/catalog.js";
+import {
+  getActiveWorkspaces,
+  getWorkspaceByName,
+  listWorkspaceServices,
+} from "../environment-manager/workspaces.js";
+import { ServiceConflictError } from "../environment-manager/service-runner.js";
+import { readManifest } from "../manifest/read-manifest.js";
+import { manifestStackName, readStacks } from "./config.js";
+import { isPrimary } from "./manifest-commands.js";
+import { resolveStackNamespace } from "./namespace.js";
+import { projectName } from "./render.js";
+import { stackDir, type StackContext } from "./runtime.js";
+import { loadStackContext } from "./status.js";
+import type { StackConfig } from "./types.js";
+
+/** A checkout: what a shared project must run from for every owner. */
+export interface StackCheckout {
+  repoPath: string;
+  branch: string | null;
+}
+
+interface OwnersRecord extends StackCheckout {
+  project: string;
+  workspaces: string[];
+}
+
+/** Lookups the refcount needs; injectable for tests. */
+export interface StackOwnerDeps {
+  active?: () => string[];
+  /** A workspace's stack namespace; null when it cannot be resolved. */
+  namespaceOf?: (workspace: string) => string | null;
+  /** Whether a workspace still holds the stack. */
+  holds?: (workspace: string, stack: string) => boolean;
+  /** Whether the project has running containers; null: docker can't tell. */
+  running?: (project: string) => boolean | null;
+}
+
+/** Whether a context takes part in the refcount (not under the override). */
+export function stackRefcounted(
+  ctx: StackContext,
+): ctx is StackContext & { denWorkspace: string } {
+  return !!ctx.denWorkspace && ctx.namespaceSource !== "env";
+}
+
+function ownersFile(ctx: StackContext, s: StackConfig): string {
+  return path.join(stackDir(ctx, s), "owners.json");
+}
+
+function readRecord(ctx: StackContext, s: StackConfig): OwnersRecord | null {
+  try {
+    const raw = JSON.parse(
+      fs.readFileSync(ownersFile(ctx, s), "utf-8"),
+    ) as Partial<OwnersRecord>;
+    if (typeof raw.repoPath !== "string" || !Array.isArray(raw.workspaces))
+      return null;
+    return {
+      project: projectName(s.name, ctx.workspace),
+      repoPath: raw.repoPath,
+      branch: typeof raw.branch === "string" ? raw.branch : null,
+      workspaces: raw.workspaces.filter(
+        (w): w is string => typeof w === "string",
+      ),
+    };
+  } catch {
+    return null;
+  }
+}
+
+function writeRecord(
+  ctx: StackContext,
+  s: StackConfig,
+  rec: OwnersRecord,
+): void {
+  const file = ownersFile(ctx, s);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  fs.writeFileSync(file, JSON.stringify(rec, null, 2) + "\n");
+}
+
+/** Drop the record once the project is down. */
+export function forgetStackOwners(ctx: StackContext, s: StackConfig): void {
+  try {
+    fs.unlinkSync(ownersFile(ctx, s));
+  } catch {
+    /* none recorded */
+  }
+}
+
+/** The checkout `s` runs from in `workspace`: its repo path and branch. */
+export function stackCheckout(
+  workspace: string,
+  s: StackConfig,
+): StackCheckout {
+  const repoPath = s.repoPath ?? path.join(getReposRoot(), s.repo);
+  let branch: string | null = null;
+  try {
+    const ws = getWorkspaceByName(workspace);
+    if (ws)
+      branch =
+        listWorkspaceServices(ws.config, readCatalog(), ws.name).find(
+          (svc) => svc.repoPath === repoPath,
+        )?.branch ?? null;
+  } catch {
+    /* no branch to tell */
+  }
+  return { repoPath, branch };
+}
+
+const same = (a: StackCheckout, b: StackCheckout): boolean =>
+  a.repoPath === b.repoPath && (a.branch ?? null) === (b.branch ?? null);
+
+function composeRunning(project: string): boolean | null {
+  const r = spawnSync(
+    "docker",
+    ["ps", "-q", "--filter", `label=com.docker.compose.project=${project}`],
+    { encoding: "utf-8", timeout: 5_000, stdio: ["ignore", "pipe", "ignore"] },
+  );
+  if (r.error || r.status !== 0) return null;
+  return (r.stdout ?? "").trim().length > 0;
+}
+
+const defaultDeps: Required<StackOwnerDeps> = {
+  active: getActiveWorkspaces,
+  namespaceOf: (w) => {
+    try {
+      return resolveStackNamespace(w).namespace;
+    } catch {
+      return null;
+    }
+  },
+  holds: (w, stack) => {
+    try {
+      if (!getWorkspaceByName(w)) return false;
+      return stack in readStacks(undefined, { workspace: w });
+    } catch {
+      return true; // can't tell: never stop it from under them
+    }
+  },
+  running: composeRunning,
+};
+
+/** The record and the live owners other than `workspace`. */
+function coOwners(
+  ctx: StackContext,
+  s: StackConfig,
+  workspace: string,
+  deps: StackOwnerDeps,
+): { record: OwnersRecord | null; others: string[] } {
+  const d = { ...defaultDeps, ...deps };
+  const record = readRecord(ctx, s);
+  if (!record) return { record, others: [] };
+  const active = new Set(d.active());
+  const others = record.workspaces.filter(
+    (w) =>
+      w !== workspace &&
+      active.has(w) &&
+      d.namespaceOf(w) === ctx.workspace &&
+      d.holds(w, s.name),
+  );
+  // A project that has verifiably stopped is held by no one.
+  if (others.length && d.running(record.project) === false)
+    return { record, others: [] };
+  return { record, others };
+}
+
+/**
+ * Check that `workspace` may bring `s` up: no other live owner runs its
+ * project from a different checkout or branch. Returns those other owners
+ * (sharing it); throws ServiceConflictError (`identity_mismatch`) when one
+ * runs it from elsewhere.
+ */
+export function assertStackClaim(
+  ctx: StackContext,
+  s: StackConfig,
+  workspace: string,
+  deps: StackOwnerDeps = {},
+): string[] {
+  const { record, others } = coOwners(ctx, s, workspace, deps);
+  if (!record || others.length === 0) return [];
+  const requested = stackCheckout(workspace, s);
+  if (same(record, requested)) return others;
+  const project = projectName(s.name, ctx.workspace);
+  const n = (b: string | null) => b ?? "n/a";
+  throw new ServiceConflictError({
+    reason: "identity_mismatch",
+    address: s.name,
+    owners: others,
+    running: { repoPath: record.repoPath, branch: record.branch },
+    requested,
+    message:
+      `stack ${s.name}: Compose project ${project} (stack namespace "${ctx.workspace}") ` +
+      `is already running for workspace(s) [${others.join(", ")}] at ${record.repoPath} ` +
+      `(branch: ${n(record.branch)}), which differs from ${requested.repoPath} ` +
+      `(branch: ${n(requested.branch)}) for workspace "${workspace}". ` +
+      `Not swapping its containers: stop it there first, or give "${workspace}" its own stack_namespace`,
+  });
+}
+
+/**
+ * Record `workspace` as an owner of `s`'s project after it is up. Joins the
+ * record on the same checkout; a different checkout (reached only once no
+ * live owner remains, see assertStackClaim) starts a new one.
+ */
+export function recordStackOwner(
+  ctx: StackContext,
+  s: StackConfig,
+  workspace: string,
+): void {
+  const checkout = stackCheckout(workspace, s);
+  const prev = readRecord(ctx, s);
+  const kept = prev && same(prev, checkout) ? prev.workspaces : [];
+  writeRecord(ctx, s, {
+    project: projectName(s.name, ctx.workspace),
+    ...checkout,
+    workspaces: [...new Set([...kept, workspace])],
+  });
+}
+
+/**
+ * `workspace` lets go of `s`'s project. `stop: false` with `remaining`
+ * when other live owners still hold it (they stay recorded, re-derived);
+ * `stop: true` when it was the last (call forgetStackOwners once down).
+ */
+export function releaseStackOwner(
+  ctx: StackContext,
+  s: StackConfig,
+  workspace: string,
+  deps: StackOwnerDeps = {},
+): { stop: boolean; remaining: string[] } {
+  const { record, others } = coOwners(ctx, s, workspace, deps);
+  if (!record || others.length === 0) return { stop: true, remaining: [] };
+  writeRecord(ctx, s, { ...record, workspaces: others });
+  return { stop: false, remaining: others };
+}
+
+/**
+ * Record `workspace` as an owner of a compose stack it is about to skip
+ * because it already answers its health check (claimHealthyService).
+ * `attached` when it now shares the project with another workspace; null
+ * when it holds it alone (recorded all the same), when `resolved` is not a
+ * stack's primary service, or under the override;
+ * `conflict` when the running project is another checkout's (the caller
+ * then runs the real `up`, which refuses with the identity mismatch).
+ */
+export function claimHealthyStack(
+  resolved: { repoPath: string; repoName: string; subsystem: string },
+  workspace: string,
+  deps: StackOwnerDeps = {},
+): "attached" | "conflict" | null {
+  try {
+    const m = readManifest(resolved.repoPath);
+    if (!m?.compose || !isPrimary(m, resolved.subsystem)) return null;
+    const ctx = loadStackContext(workspace);
+    if (!stackRefcounted(ctx)) return null;
+    const s = ctx.stacks[manifestStackName(m, resolved.repoName)];
+    if (!s) return null;
+    let others: string[];
+    try {
+      others = assertStackClaim(ctx, s, workspace, deps);
+    } catch (e) {
+      if (e instanceof ServiceConflictError) return "conflict";
+      throw e;
+    }
+    recordStackOwner(ctx, s, workspace);
+    // Held by this workspace alone: recorded, and said as before.
+    return others.length ? "attached" : null;
+  } catch {
+    return null; // no edge, an unreadable manifest: not a stack we can tell
+  }
+}

package/src/shared/stacks/render.ts

@@ -3,7 +3,12 @@
  * override, the link networks and the edge routes. No I/O.
  */
 import * as yaml from "yaml";
-import { edgeDomains, routeHost } from "../edge/render.js";
+import {
+  edgeDomains,
+  routeEntryPoint,
+  routeHost,
+  routerTls,
+} from "../edge/render.js";
 import type { EdgeConfig } from "../edge/types.js";
 import type { ResolvedProject, StackConfig } from "./types.js";
 
@@ -220,12 +225,12 @@ export function renderStackRoutes(
     const { service, port } = parseTarget(target);
     const key = `stack-${stack.name}-${route}`;
     routers[key] = {
-      entryPoints: ["websecure"],
+      entryPoints: [routeEntryPoint(edge)],
       rule: edgeDomains(edge)
         .map((d) => `Host(\`${routeHost(`${route}.${stack.name}`, d)}\`)`)
         .join(" || "),
       service: key,
-      tls: {},
+      ...routerTls(edge),
     };
     services[key] = {
       loadBalancer: {

package/src/shared/stacks/runtime.ts

@@ -60,10 +60,16 @@ function must(
 export interface StackContext {
   edge: EdgeConfig;
   stacks: Record<string, StackConfig>;
+  /** The Compose project suffix (the stack namespace). */
   workspace: string;
+  /** The den workspace the stacks were read for (loadStackContext). */
+  denWorkspace?: string;
+  /** Where the suffix came from; `env`: DEN_STACK_WORKSPACE. */
+  namespaceSource?: "workspace" | "blueprint" | "default" | "env";
 }
 
-function stackDir(ctx: StackContext, s: StackConfig): string {
+/** The project's local state: generated.env, override.yml, owners.json. */
+export function stackDir(ctx: StackContext, s: StackConfig): string {
   return path.join(
     ctx.edge.stateDir,
     "stacks",
@@ -193,6 +199,94 @@ function connect(network: string, container: string): void {
     throw new Error(`docker network connect ${network} ${container}: ${r.err}`);
 }
 
+/**
+ * Connect the edge to each of `nets` it is not on (`on`: the networks it
+ * is on already; unknown, every one is connected, "already connected"
+ * being fine). Returns the networks it joined. stack up and the rejoin
+ * after edge up share it.
+ */
+function joinEdge(
+  edge: EdgeConfig,
+  nets: string[],
+  on: ReadonlySet<string> = new Set(),
+): string[] {
+  const container = edgeNames(edge.org).container;
+  const joined: string[] = [];
+  for (const n of nets) {
+    if (on.has(n)) continue;
+    connect(n, container);
+    joined.push(n);
+  }
+  return joined;
+}
+
+/** The dynamic file a running stack's routes live in, under this edge's state dir. */
+function stackRoutesFile(edge: EdgeConfig, project: string): string {
+  return path.join(edge.stateDir, "dynamic", `stack-${project}.yml`);
+}
+
+/**
+ * After edge up (re)creates or starts the edge, put it back on every
+ * running stack's routed networks (0.3.0-dev.40): a recreated container
+ * starts on its own network only, and each stack's routes 502 until that
+ * stack's next `stack up`. A stack counts when it has its routes file in
+ * this edge's dynamic dir and its compose project has a container up;
+ * stacks that are down, and projects with no routes file here (another
+ * state dir's, or none of Den's), are not touched. Idempotent: a network
+ * the edge is on already is left alone. One line per network joined;
+ * a stack that cannot be resolved gets a line and is skipped.
+ */
+export function rejoinEdgeNetworks(
+  contexts: StackContext[],
+  onLine: (line: string) => void = console.log,
+): string[] {
+  if (!contexts.length) return [];
+  const edge = contexts[0].edge;
+  const on = new Set(
+    run("docker", [
+      "inspect",
+      "-f",
+      "{{range $k, $v := .NetworkSettings.Networks}}{{$k}} {{end}}",
+      edgeNames(edge.org).container,
+    ]).out.split(/\s+/),
+  );
+  const joined: string[] = [];
+  const seen = new Set<string>();
+  for (const ctx of contexts) {
+    // Only this edge's state dir.
+    if (ctx.edge.stateDir !== edge.stateDir) continue;
+    for (const s of Object.values(ctx.stacks)) {
+      const project = projectName(s.name, ctx.workspace);
+      if (seen.has(project)) continue;
+      seen.add(project);
+      if (!fs.existsSync(stackRoutesFile(edge, project))) continue;
+      const up = run("docker", [
+        "ps",
+        "-q",
+        "--filter",
+        `label=com.docker.compose.project=${project}`,
+      ]);
+      if (up.code !== 0 || !up.out) continue;
+      try {
+        for (const n of joinEdge(
+          edge,
+          routedNetworks(s, resolve(ctx, s)),
+          on,
+        )) {
+          on.add(n);
+          joined.push(n);
+          onLine(`  rejoined ${n} (${project})`);
+        }
+      } catch (err) {
+        onLine(
+          `  ${project}: not rejoined (${(err as Error).message.split("\n")[0]})`,
+        );
+      }
+    }
+  }
+  return joined;
+}
+
 function running(container: string): boolean {
   return (
     run("docker", ["inspect", "-f", "{{.State.Running}}", container]).out ===
@@ -257,29 +351,19 @@ export function stackUp(
     for (const svc of providerServices(c)[s.name] ?? [])
       connect(net, containerName(projectName(s.name, ctx.workspace), svc));
   }
-  const edge = edgeNames(ctx.edge.org).container;
   const nets = routedNetworks(s, resolved);
-  for (const n of nets) connect(n, edge);
+  joinEdge(ctx.edge, nets);
   fs.writeFileSync(
-    path.join(
-      ctx.edge.stateDir,
-      "dynamic",
-      `stack-${projectName(s.name, ctx.workspace)}.yml`,
-    ),
+    stackRoutesFile(ctx.edge, projectName(s.name, ctx.workspace)),
     renderStackRoutes(ctx.edge, s, resolved),
   );
   return { project: resolved.name, networks: nets };
 }
 
 export function stackDown(ctx: StackContext, s: StackConfig): void {
-  fs.rmSync(
-    path.join(
-      ctx.edge.stateDir,
-      "dynamic",
-      `stack-${projectName(s.name, ctx.workspace)}.yml`,
-    ),
-    { force: true },
-  );
+  fs.rmSync(stackRoutesFile(ctx.edge, projectName(s.name, ctx.workspace)), {
+    force: true,
+  });
   const edge = edgeNames(ctx.edge.org).container;
   const dir = repoDir(s);
   const override = path.join(stackDir(ctx, s), "override.yml");