@monster-orange/den 0.0.0 → 0.1.0

package/src/shared/environment-manager/readiness.ts

@@ -0,0 +1,93 @@
+/**
+ * Readiness polling for `depends_on: { wait: true }` gates.
+ *
+ * Wraps `checkServiceHealth` with a poll loop. Used by the workspace `up`
+ * runner to block starting a dependent until its flagged dependencies are
+ * actually green (not just "started 5 seconds ago").
+ */
+
+import { READINESS_TIMEOUT_MS, READINESS_INTERVAL_MS } from "../constants.js";
+import { checkServiceHealth } from "./health.js";
+import type { Environment, ResolvedService } from "../types/index.js";
+
+export interface WaitForReadyOptions {
+  timeoutMs?: number;
+  intervalMs?: number;
+  /** Called with a short status string on each unsuccessful poll. */
+  onTick?: (detail: string) => void;
+  /**
+   * Polled between each readiness check. When it returns true the loop exits
+   * early with `ok: false, detail: "cancelled"`. Used by the SSE runner to
+   * stop polling when the client disconnects mid-`up`.
+   */
+  isCancelled?: () => boolean;
+  /** Override for tests. */
+  now?: () => number;
+  /** Override for tests. */
+  sleep?: (ms: number) => Promise<void>;
+}
+
+export interface WaitForReadyResult {
+  ok: boolean;
+  /** Detail from the last check — populated on timeout or fail-open. */
+  detail?: string;
+  /** True when the dep has no configured health check — we fail open and proceed. */
+  skipped?: boolean;
+  /** Time (ms) spent waiting. */
+  elapsedMs: number;
+}
+
+const defaultSleep = (ms: number): Promise<void> =>
+  new Promise((r) => setTimeout(r, ms));
+
+/**
+ * Poll the dep's health check until it returns ok or the timeout expires.
+ *
+ * Fail-open: if the dep has no `health.<env>` configured, returns immediately
+ * with `skipped: true` — we never silently hang on a misconfigured dep.
+ */
+export async function waitForReady(
+  resolved: ResolvedService,
+  env: Environment,
+  opts: WaitForReadyOptions = {},
+): Promise<WaitForReadyResult> {
+  const timeoutMs = opts.timeoutMs ?? READINESS_TIMEOUT_MS;
+  const intervalMs = opts.intervalMs ?? READINESS_INTERVAL_MS;
+  const now = opts.now ?? Date.now;
+  const sleep = opts.sleep ?? defaultSleep;
+
+  const start = now();
+  const deadline = start + timeoutMs;
+  let last = await checkServiceHealth(resolved, env);
+
+  // Fail-open: no health config → don't block.
+  if (last.health === "unknown") {
+    return {
+      ok: true,
+      skipped: true,
+      detail: last.healthDetail,
+      elapsedMs: now() - start,
+    };
+  }
+
+  while (last.health !== "ok") {
+    if (opts.isCancelled?.()) {
+      return { ok: false, detail: "cancelled", elapsedMs: now() - start };
+    }
+    if (now() >= deadline) {
+      return {
+        ok: false,
+        detail: last.healthDetail ?? "timeout",
+        elapsedMs: now() - start,
+      };
+    }
+    opts.onTick?.(last.healthDetail ?? "not ready");
+    await sleep(intervalMs);
+    if (opts.isCancelled?.()) {
+      return { ok: false, detail: "cancelled", elapsedMs: now() - start };
+    }
+    last = await checkServiceHealth(resolved, env);
+  }
+
+  return { ok: true, elapsedMs: now() - start };
+}

package/src/shared/environment-manager/register-local.ts

@@ -0,0 +1,72 @@
+/**
+ * `den register <path>`: add an existing checkout to a workspace, with no
+ * prompts.
+ *
+ * The repo is named after its manifest (`name:`), so a checkout can live
+ * anywhere on disk (Arbor lives outside the repos root) and still get a clean
+ * name. It is added dev-managed at that path: Den runs the repo's own
+ * commands there and never clones or switches its branch. One bare entry per
+ * repo: the workspace expands it to every service in the manifest.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import { readManifest } from "../manifest/read-manifest.js";
+import { getGitHubCoords } from "../smoke/git-remote.js";
+import { upsertCatalogEntry } from "./catalog.js";
+import { readWorkspacesFile, writeWorkspacesFile } from "./workspaces.js";
+
+import { product } from "../product.js";
+export interface RegisteredRepo {
+  repoName: string;
+  gh: string;
+  path: string;
+  services: string[];
+}
+
+export function registerLocalRepo(
+  repoPathArg: string,
+  workspaceName: string,
+): RegisteredRepo {
+  const repoPath = path.resolve(repoPathArg);
+  if (!fs.existsSync(repoPath))
+    throw new Error(`Path does not exist: ${repoPath}`);
+  const manifest = readManifest(repoPath);
+  if (!manifest)
+    throw new Error(
+      `No ${product().manifestYml} in ${repoPath} (and none in ${product().title}'s manifests/ for its origin repo)`,
+    );
+  const repoName =
+    typeof manifest.name === "string" && manifest.name
+      ? manifest.name
+      : path.basename(repoPath);
+  const coords = getGitHubCoords(repoPath);
+  const gh = coords ? `${coords.owner}/${coords.repo}` : "";
+
+  const file = readWorkspacesFile();
+  const ws = file.workspaces[workspaceName];
+  if (!ws) throw new Error(`Unknown workspace "${workspaceName}"`);
+
+  // Replace every existing entry for this repo, keeping its auto-start toggle.
+  let autostartOff = false;
+  for (const key of Object.keys(ws.services)) {
+    if (key === repoName || key.startsWith(`${repoName}/`)) {
+      if (ws.services[key].autostart === false) autostartOff = true;
+      delete ws.services[key];
+    }
+  }
+  ws.services[repoName] = {
+    mode: "dev-managed",
+    path: repoPath,
+    ...(autostartOff ? { autostart: false } : {}),
+  };
+
+  upsertCatalogEntry(repoName, { gh });
+  writeWorkspacesFile(file);
+  return {
+    repoName,
+    gh,
+    path: repoPath,
+    services: Object.keys(manifest.services ?? {}),
+  };
+}

package/src/shared/environment-manager/secrets-check.ts

@@ -0,0 +1,260 @@
+/**
+ * Local-secrets readiness check.
+ *
+ * Repos scaffolded by stack that declare `config.local.sidecars[].secrets`
+ * ship a COMMITTED, generated `.stack-secrets.example.yml` listing every
+ * secret they need, and expect a gitignored `.stack-secrets.yml` beside it
+ * with real values. That example file is the whole contract — den reads it
+ * generically and needs no per-repo knowledge.
+ *
+ * Why this gate exists: a sidecar's generated `up.sh` validates only that each
+ * secret is PRESENT and NON-EMPTY, so the placeholder value (`replace-me`)
+ * sails through. Starting a stack that way is not a harmless no-op — a
+ * database container bakes the password it is first started with into its data
+ * volume at initdb time, and no later edit of the secrets file changes it. The
+ * result is a stack that looks healthy (liveness probes don't authenticate)
+ * while every consumer fails to connect. Refusing to start is the only cheap
+ * moment to prevent it.
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import * as yaml from "yaml";
+import { getDistRoot } from "../paths.js";
+import { getReposRoot } from "../den-config.js";
+
+import { product } from "../product.js";
+/** stack's generated placeholder, plus the `<angle-bracket>` fill-me-in convention. */
+const PLACEHOLDER_RE = /^(replace-me|<[^>]*>)$/;
+
+const EXAMPLE_FILE = ".stack-secrets.example.yml";
+const SECRETS_FILE = ".stack-secrets.yml";
+
+export interface SecretsCheckResult {
+  ok: boolean;
+  /** Absolute path to the secrets file the engineer needs to fill in. */
+  secretsPath: string;
+  /** Absolute path to the committed example that declares the contract. */
+  examplePath: string;
+  /** True when `.stack-secrets.yml` doesn't exist at all. */
+  fileMissing: boolean;
+  /** Declared keys absent from the secrets file (names only — never values). */
+  missingKeys: string[];
+  /** Declared keys still holding a placeholder or empty value. */
+  placeholderKeys: string[];
+}
+
+function parseFlatYaml(file: string): Record<string, unknown> | null {
+  try {
+    const parsed = yaml.parse(fs.readFileSync(file, "utf-8")) as unknown;
+    if (parsed == null || typeof parsed !== "object" || Array.isArray(parsed)) {
+      return null;
+    }
+    return parsed as Record<string, unknown>;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Check a repo's local secrets against its declared contract.
+ *
+ * Returns `null` when the repo declares no contract (no example file) — i.e.
+ * most repos — so callers can treat "no contract" as "nothing to enforce".
+ */
+export function checkRepoSecrets(repoPath: string): SecretsCheckResult | null {
+  const examplePath = path.join(repoPath, EXAMPLE_FILE);
+  const secretsPath = path.join(repoPath, SECRETS_FILE);
+  if (!fs.existsSync(examplePath)) return null;
+
+  const declared = parseFlatYaml(examplePath);
+  if (!declared) return null; // unparseable contract — don't invent failures
+  const keys = Object.keys(declared);
+  if (keys.length === 0) return null;
+
+  if (!fs.existsSync(secretsPath)) {
+    return {
+      ok: false,
+      secretsPath,
+      examplePath,
+      fileMissing: true,
+      missingKeys: keys,
+      placeholderKeys: [],
+    };
+  }
+
+  const actual = parseFlatYaml(secretsPath) ?? {};
+  const missingKeys: string[] = [];
+  const placeholderKeys: string[] = [];
+  for (const key of keys) {
+    if (!(key in actual)) {
+      missingKeys.push(key);
+      continue;
+    }
+    const value = actual[key];
+    const text = value == null ? "" : String(value).trim();
+    if (text === "" || PLACEHOLDER_RE.test(text)) placeholderKeys.push(key);
+  }
+
+  return {
+    ok: missingKeys.length === 0 && placeholderKeys.length === 0,
+    secretsPath,
+    examplePath,
+    fileMissing: false,
+    missingKeys,
+    placeholderKeys,
+  };
+}
+
+/**
+ * Other checkouts of the same repo on this machine, in the order
+ * `readManifestAnyBranch` already established: other managed branch
+ * clones first, then the engineer's own sibling checkout.
+ */
+function otherCheckouts(repoName: string, excludePath: string): string[] {
+  const out: string[] = [];
+  const managedRoot = path.join(
+    getDistRoot(),
+    `${product().servicesDir}`,
+    repoName,
+  );
+  try {
+    for (const entry of fs.readdirSync(managedRoot, { withFileTypes: true })) {
+      if (entry.isDirectory()) out.push(path.join(managedRoot, entry.name));
+    }
+  } catch {
+    /* no managed clones */
+  }
+  out.push(path.join(getReposRoot(), repoName));
+  const exclude = path.resolve(excludePath);
+  return out.filter((p) => path.resolve(p) !== exclude);
+}
+
+export interface SecretsSeedResult {
+  /** Keys filled in from the donor (names only — never values). */
+  filled: string[];
+  /** Checkout the values came from. */
+  donorPath: string;
+}
+
+/**
+ * Fill a freshly cloned repo's secrets file from ANOTHER checkout of the SAME
+ * repo on this machine.
+ *
+ * A managed clone starts with stack's placeholders, which blocks `make
+ * setup` and would poison containers if started — while the engineer's own
+ * sibling checkout of that same repo usually already holds every value. Copying
+ * them is exactly what they would do by hand, so do it for them and say so.
+ *
+ * Conservative by construction: only keys DECLARED by the contract, only when
+ * unset in the target, and only from donor values that are themselves real.
+ * Never overwrites an existing value.
+ */
+export function seedSecretsFromSibling(
+  repoName: string,
+  repoPath: string,
+): SecretsSeedResult | null {
+  const check = checkRepoSecrets(repoPath);
+  if (!check || check.ok) return null;
+
+  const wanted = new Set([...check.missingKeys, ...check.placeholderKeys]);
+  for (const donorPath of otherCheckouts(repoName, repoPath)) {
+    const donorFile = path.join(donorPath, SECRETS_FILE);
+    if (!fs.existsSync(donorFile)) continue;
+    const donor = parseFlatYaml(donorFile);
+    if (!donor) continue;
+
+    const fill = new Map<string, string>();
+    for (const key of wanted) {
+      const raw = donor[key];
+      const text = raw == null ? "" : String(raw).trim();
+      if (text === "" || PLACEHOLDER_RE.test(text)) continue;
+      fill.set(key, text);
+    }
+    if (fill.size === 0) continue;
+
+    writeSecretValues(check.secretsPath, check.examplePath, fill);
+    return { filled: [...fill.keys()].sort(), donorPath };
+  }
+  return null;
+}
+
+/**
+ * Write values into a flat `KEY: VALUE` secrets file, line-wise so the
+ * generated `# Used by:` comments and key order survive. Creates the file from
+ * the example when absent.
+ */
+function writeSecretValues(
+  secretsPath: string,
+  examplePath: string,
+  values: Map<string, string>,
+): void {
+  const source = fs.existsSync(secretsPath) ? secretsPath : examplePath;
+  const lines = fs.readFileSync(source, "utf-8").split("\n");
+  const remaining = new Map(values);
+
+  const updated = lines.map((line) => {
+    const m = /^(\s*)([A-Za-z_][A-Za-z0-9_]*)(\s*:\s*)(.*)$/.exec(line);
+    if (!m) return line;
+    const key = m[2];
+    if (!remaining.has(key)) return line;
+    const value = remaining.get(key)!;
+    remaining.delete(key);
+    return `${m[1]}${key}: ${quoteIfNeeded(value)}`;
+  });
+
+  // Declared keys the file never listed (e.g. a hand-trimmed file).
+  for (const [key, value] of remaining) {
+    updated.push(`${key}: ${quoteIfNeeded(value)}`);
+  }
+
+  fs.writeFileSync(secretsPath, updated.join("\n"), { mode: 0o600 });
+  // `mode` above only applies when the file is CREATED — an existing file keeps
+  // its old permissions, so tighten explicitly. This file holds live credentials.
+  try {
+    fs.chmodSync(secretsPath, 0o600);
+  } catch {
+    /* best-effort — a read-only FS shouldn't fail the seed */
+  }
+}
+
+/**
+ * Quote values the flat-file readers would otherwise mangle. `up.sh`'s awk
+ * strips a trailing ` #…` before unquoting, so a value containing " #" cannot
+ * round-trip through that parser at all — left as-is rather than silently
+ * corrupted, since real secrets don't contain it.
+ */
+function quoteIfNeeded(value: string): string {
+  if (!/[:#'"]|^\s|\s$/.test(value)) return value;
+  return `"${value.replace(/"/g, '\\"')}"`;
+}
+
+/**
+ * Operator-facing explanation of a failed check. Key names only — never values.
+ * Leads with the absolute path because the file lives inside a managed
+ * clone, which is not somewhere an engineer would think to look.
+ */
+export function describeSecretsCheck(
+  repoName: string,
+  result: SecretsCheckResult,
+): string[] {
+  const lines: string[] = [];
+  const unset = [...result.missingKeys, ...result.placeholderKeys];
+  lines.push(
+    result.fileMissing
+      ? `${repoName}: local secrets file has not been created yet`
+      : `${repoName}: ${unset.length} local secret(s) still unset — ${unset.join(", ")}`,
+  );
+  lines.push(`  file:     ${result.secretsPath}`);
+  lines.push(`  declared: ${result.examplePath}`);
+  lines.push(
+    `  Starting now would initialize containers with placeholder credentials.`,
+  );
+  lines.push(
+    `  A database volume keeps the password it is FIRST created with, so this`,
+  );
+  lines.push(
+    `  is not self-correcting — fill the values in, then run Sync again.`,
+  );
+  return lines;
+}

package/src/shared/environment-manager/service-env.ts

@@ -0,0 +1,110 @@
+/**
+ * Shared environment for service commands.
+ *
+ * Some repos read shared locations from environment variables (e.g. a
+ * shared-infra repo's secrets file, defaulting to a sibling path like
+ * `../infra/.secrets.yml`). A sibling default breaks when the consumer is a
+ * nested managed clone, because `../infra` no longer points at that repo.
+ *
+ * Rather than patch each repo, den injects the resolved values into every
+ * service command (`up`/`down`/`restart`/`setup`). The mapping comes from the
+ * org config's `service_env` (`KEY: "{repo:<name>}/<path>"`); den has no
+ * built-in entries.
+ */
+
+import { readDenConfig } from "../den-config.js";
+import { resolveRepoPath } from "./derivation/derive-graph.js";
+
+/**
+ * Built-in shared env templates: none. Config (`service_env`) is merged over
+ * this; a key set to "" is skipped.
+ */
+export const DEFAULT_SERVICE_ENV: Record<string, string> = {};
+
+/**
+ * Expand `{repo:<name>}` placeholders to the absolute path den resolves
+ * `<name>` to (managed clone or dev sibling). Returns null when any placeholder
+ * can't be resolved, so the caller skips that variable entirely.
+ */
+export function expandServiceEnvTemplate(
+  template: string,
+  resolve: (repoName: string) => string | null = resolveRepoPath,
+): string | null {
+  const tokens = [...template.matchAll(/\{repo:([^}]+)\}/g)];
+  let out = template;
+  for (const match of tokens) {
+    const repoPath = resolve(match[1]);
+    if (!repoPath) return null;
+    out = out.split(match[0]).join(repoPath);
+  }
+  return out;
+}
+
+/**
+ * Repos that SUPPLY shared values to other repos, derived from the `{repo:X}`
+ * tokens in the (config-merged) service_env templates.
+ *
+ * These are load-bearing for every consumer: if such a repo's own local config
+ * is unusable, starting the repos that read from it produces a fleet configured
+ * against placeholder values. Config-derived, so den needs no hardcoded
+ * knowledge of which repo happens to own the shared secrets.
+ */
+export function serviceEnvProviderRepos(
+  readConfig: () =>
+    | { service_env?: Record<string, string> }
+    | null
+    | undefined = readDenConfig,
+): Set<string> {
+  const providers = new Set<string>();
+  try {
+    const templates: Record<string, string> = {
+      ...DEFAULT_SERVICE_ENV,
+      ...(readConfig()?.service_env ?? {}),
+    };
+    for (const template of Object.values(templates)) {
+      if (!template) continue; // an empty template is skipped
+      for (const match of template.matchAll(/\{repo:([^}]+)\}/g)) {
+        providers.add(match[1]);
+      }
+    }
+  } catch {
+    /* best-effort — an unreadable config just means no known providers */
+  }
+  return providers;
+}
+
+/**
+ * Build the shared environment to inject into a service command. Never overrides
+ * a variable already present (and non-empty) in `baseEnv` — an explicit value in
+ * the caller's shell wins. Variables whose template can't be resolved are
+ * omitted (e.g. the providing repo is not in this workspace).
+ */
+export function buildServiceEnv(
+  baseEnv: NodeJS.ProcessEnv = process.env,
+  resolve: (repoName: string) => string | null = resolveRepoPath,
+  readConfig: () =>
+    | { service_env?: Record<string, string> }
+    | null
+    | undefined = readDenConfig,
+): NodeJS.ProcessEnv {
+  // Best-effort: injecting shared env must never break a service command, so any
+  // failure to read config / resolve a repo path degrades to "no shared env".
+  try {
+    const templates: Record<string, string> = {
+      ...DEFAULT_SERVICE_ENV,
+      ...(readConfig()?.service_env ?? {}),
+    };
+
+    const out: NodeJS.ProcessEnv = {};
+    for (const [key, template] of Object.entries(templates)) {
+      const existing = baseEnv[key];
+      if (existing != null && existing !== "") continue; // respect caller-set value
+      if (!template) continue; // an empty template is skipped
+      const value = expandServiceEnvTemplate(template, resolve);
+      if (value !== null) out[key] = value;
+    }
+    return out;
+  } catch {
+    return {};
+  }
+}