@monster-orange/den 0.2.2 → 0.2.4

package/src/shared/secrets/doppler.ts

@@ -4,7 +4,8 @@
  *
  * Project/config come from `secrets.doppler.{project,config}` in
  * config/den.yml, else from the directory's `doppler setup` scope,
- * else from DOPPLER_PROJECT / DOPPLER_CONFIG / DOPPLER_TOKEN.
+ * else from DOPPLER_PROJECT / DOPPLER_CONFIG / DOPPLER_TOKEN. A scoped ref
+ * (a `secret_env` value, `<project>/<config>/<key>`) names its own.
  */
 import { execFile, spawn } from "child_process";
 import { promisify } from "util";
@@ -29,8 +30,12 @@ export class DopplerProvider implements SecretsProvider {
     this.timeout = opts.timeoutMs ?? 30_000;
   }
 
-  /** `--project/--config` flags when configured explicitly. */
-  scopeArgs(): string[] {
+  /**
+   * `--project/--config` flags: a scoped ref's own vault/item, else the
+   * configured ones (when set explicitly).
+   */
+  scopeArgs(ref?: SecretRef): string[] {
+    if (ref?.scoped) return ["--project", ref.vault, "--config", ref.item];
     const a: string[] = [];
     if (this.opts.project) a.push("--project", this.opts.project);
     if (this.opts.config) a.push("--config", this.opts.config);
@@ -56,7 +61,7 @@ export class DopplerProvider implements SecretsProvider {
     const key = this.keyFor(ref);
     const { stdout } = await execFileAsync(
       "doppler",
-      ["secrets", "get", key, "--plain", ...this.scopeArgs()],
+      ["secrets", "get", key, "--plain", ...this.scopeArgs(ref)],
       { timeout: this.timeout },
     );
     return String(stdout).trim();
@@ -77,7 +82,7 @@ export class DopplerProvider implements SecretsProvider {
     await new Promise<void>((resolve, reject) => {
       const child = spawn(
         "doppler",
-        ["secrets", "set", key, "--silent", ...this.scopeArgs()],
+        ["secrets", "set", key, "--silent", ...this.scopeArgs(ref)],
         { stdio: ["pipe", "ignore", "pipe"] },
       );
       let stderr = "";

package/src/shared/secrets/index.ts

@@ -26,6 +26,7 @@ export { flatKeyFor, toEnvName, refPath } from "./keys.js";
 export { DopplerProvider } from "./doppler.js";
 export { EnvProvider } from "./env.js";
 export { OnePasswordProvider } from "./onepassword.js";
+export * from "./secret-env.js";
 
 export interface SecretsConfig {
   provider?: SecretsProviderName;

package/src/shared/secrets/keys.ts

@@ -16,6 +16,7 @@ export function refPath(ref: SecretRef): string {
 
 /**
  * Resolve the flat key a key/value backend should use for `ref`:
+ *   0. a scoped ref's field, as is (it is fully qualified: no overrides);
  *   1. `ref.key` when the caller pinned one;
  *   2. the `secrets.keys` override map (`"vault/item/field": KEY`);
  *   3. the field itself when it already looks like an env var (API keys);
@@ -25,6 +26,7 @@ export function flatKeyFor(
   ref: SecretRef,
   overrides: Record<string, string> = {},
 ): string {
+  if (ref.scoped) return ref.field;
   if (ref.key) return ref.key;
   const mapped = overrides[refPath(ref)];
   if (mapped) return mapped;

package/src/shared/secrets/secret-env.ts

@@ -0,0 +1,149 @@
+/**
+ * `secret_env` (R16): environment variables whose values Den resolves from
+ * the configured secrets provider when it starts or restarts a target.
+ *
+ *   compose:
+ *     secret_env:
+ *       OPENAI_API_KEY: proj/dev_personal/OPENAI_API_KEY
+ *   services:
+ *     api:
+ *       secret_env:
+ *         SOME_KEY: vault/item/field
+ *
+ * Each value is a fully qualified `a/b/c` path, parsed into a SCOPED
+ * {@link SecretRef} (see its `scoped` field for what each provider does with
+ * it). Values only ever travel in the spawned process's env: never into the
+ * compose override, logs, activity records, MCP output, dry runs or errors.
+ * Everything here that produces text names keys and refs, never values.
+ *
+ * This module only imports types, so the manifest schema can use the parser.
+ */
+import type { SecretRef, SecretsProvider } from "./types.js";
+
+/** A `secret_env` block: env var name → `a/b/c` ref. */
+export type SecretEnvSpec = Record<string, string>;
+
+/** Env var names Den will set (POSIX-portable). */
+export const SECRET_ENV_KEY_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+/** Why `raw` is not a valid secret_env ref, or null when it is. */
+export function secretEnvRefError(raw: unknown): string | null {
+  if (typeof raw !== "string")
+    return 'must be a string "<vault>/<item>/<field>"';
+  const parts = raw.split("/");
+  if (
+    parts.length !== 3 ||
+    parts.some((p) => p.trim() === "" || p !== p.trim())
+  )
+    return `"${raw}" must be exactly three non-empty parts, "<vault>/<item>/<field>" (Doppler: <project>/<config>/<key>)`;
+  return null;
+}
+
+/** Parse an `a/b/c` ref into a scoped SecretRef. Throws when malformed. */
+export function parseSecretEnvRef(raw: string): SecretRef {
+  const err = secretEnvRefError(raw);
+  if (err) throw new Error(`secret_env ref ${err}`);
+  const [vault, item, field] = raw.split("/");
+  return { vault, item, field, scoped: true };
+}
+
+/**
+ * Problems with a whole `secret_env` block, as `KEY: message` strings ([]
+ * when it is valid). `where` prefixes each message.
+ */
+export function secretEnvSpecErrors(spec: unknown, where: string): string[] {
+  if (spec === undefined) return [];
+  if (spec === null || typeof spec !== "object" || Array.isArray(spec))
+    return [`${where} must be a map of ENV_NAME: "<vault>/<item>/<field>"`];
+  const out: string[] = [];
+  for (const [k, v] of Object.entries(spec)) {
+    if (!SECRET_ENV_KEY_RE.test(k))
+      out.push(`${where}.${k}: "${k}" is not a valid env var name`);
+    const err = secretEnvRefError(v);
+    if (err) out.push(`${where}.${k}: ref ${err}`);
+  }
+  return out;
+}
+
+/** The validated block, or throws listing every problem. */
+export function assertSecretEnvSpec(
+  spec: unknown,
+  where: string,
+): SecretEnvSpec | undefined {
+  if (spec === undefined) return undefined;
+  const errors = secretEnvSpecErrors(spec, where);
+  if (errors.length) throw new Error(errors.join("\n"));
+  return { ...(spec as SecretEnvSpec) };
+}
+
+/** One key that could not be resolved. Never carries a value. */
+export interface SecretEnvMissing {
+  key: string;
+  ref: string;
+}
+
+export type SecretEnvResolution =
+  | { ok: true; env: Record<string, string> }
+  | { ok: false; missing: SecretEnvMissing[]; message: string };
+
+/**
+ * The failure text for unresolved keys: key, ref and provider, never a value
+ * and never the provider's own error output (which could echo one).
+ */
+export function describeSecretEnvMissing(
+  target: string,
+  missing: SecretEnvMissing[],
+  provider: string,
+): string {
+  const lines = missing.map(
+    (m) => `  ${m.key} (ref ${m.ref}, provider ${provider}): not resolved`,
+  );
+  return (
+    `${target}: ${missing.length} secret_env key(s) could not be resolved; not starting it:\n` +
+    lines.join("\n")
+  );
+}
+
+/**
+ * Resolve every key of `spec` through `provider`. All or nothing: when any
+ * key is missing (unset, empty, unreadable) the result is a failure naming
+ * each missing key, and no partial env is returned.
+ */
+export async function resolveSecretEnv(
+  spec: SecretEnvSpec | undefined,
+  provider: SecretsProvider,
+  target: string,
+): Promise<SecretEnvResolution> {
+  const entries = Object.entries(spec ?? {});
+  if (!entries.length) return { ok: true, env: {} };
+  const results = await Promise.all(
+    entries.map(async ([key, raw]) => {
+      if (secretEnvRefError(raw)) return { key, raw, value: null };
+      let value: string | null = null;
+      try {
+        value = await provider.tryGet(parseSecretEnvRef(raw));
+      } catch {
+        value = null;
+      }
+      return { key, raw, value };
+    }),
+  );
+  const missing = results
+    .filter((r) => !r.value)
+    .map((r) => ({ key: r.key, ref: String(r.raw) }));
+  if (missing.length)
+    return {
+      ok: false,
+      missing,
+      message: describeSecretEnvMissing(target, missing, provider.name),
+    };
+  return {
+    ok: true,
+    env: Object.fromEntries(results.map((r) => [r.key, r.value as string])),
+  };
+}
+
+/** The keys of a block, sorted: what a dry run or status may show. */
+export function secretEnvKeys(spec: SecretEnvSpec | undefined): string[] {
+  return Object.keys(spec ?? {}).sort();
+}

package/src/shared/secrets/types.ts

@@ -22,6 +22,16 @@ export interface SecretRef {
    * `flatKeyFor`.
    */
   key?: string;
+  /**
+   * Set on a fully qualified ref (a `secret_env` value, `a/b/c`): every part
+   * names a real location and no provider config fills it in.
+   *  - Doppler: vault = `--project`, item = `--config`, field = the key
+   *    (`secrets.doppler` and `secrets.keys` are not consulted).
+   *  - 1Password: vault / item / field as for any ref.
+   *  - env: field is the env var name (`secrets.keys` is not consulted).
+   * Unscoped refs (the default) keep the provider's configured scope.
+   */
+  scoped?: true;
 }
 
 export type SecretsProviderName = "doppler" | "1password" | "env";

package/src/shared/stacks/config.ts

@@ -17,6 +17,7 @@ import {
 } from "../environment-manager/workspaces.js";
 import { readManifest } from "../manifest/read-manifest.js";
 import type { ManifestComposeConfig } from "../types/index.js";
+import { assertSecretEnvSpec } from "../secrets/secret-env.js";
 import { assertStackName, parseTarget } from "./render.js";
 import type { StackConfig } from "./types.js";
 
@@ -41,6 +42,10 @@ function toStack(
   const routes = raw.routes ?? {};
   Object.values(routes).forEach(parseTarget);
   const files = list(raw.files, `${where}.files`);
+  const secretEnv =
+    "secret_env" in raw
+      ? assertSecretEnvSpec(raw.secret_env, `${where}.secret_env`)
+      : undefined;
   return {
     name,
     repo,
@@ -69,6 +74,7 @@ function toStack(
           ),
         }
       : {}),
+    ...(secretEnv && Object.keys(secretEnv).length ? { secretEnv } : {}),
   };
 }
 

package/src/shared/stacks/runtime.ts

@@ -16,6 +16,12 @@ import {
   renderStackRoutes,
   routedNetworks,
 } from "./render.js";
+import {
+  resolveSecretEnv,
+  secretEnvKeys,
+  type SecretEnvResolution,
+} from "../secrets/secret-env.js";
+import type { SecretsProvider } from "../secrets/types.js";
 import type { ResolvedProject, StackConfig } from "./types.js";
 
 function run(
@@ -117,6 +123,35 @@ function composeEnv(ctx: StackContext, s: StackConfig): NodeJS.ProcessEnv {
   };
 }
 
+/**
+ * Resolve a stack's `secret_env` through `provider` (all or nothing). Call
+ * it before anything starts or stops, and hand the env to stackUp /
+ * stackRestartService. Nothing in the result's message carries a value.
+ */
+export function resolveStackSecretEnv(
+  s: StackConfig,
+  provider: SecretsProvider,
+): Promise<SecretEnvResolution> {
+  return resolveSecretEnv(s.secretEnv, provider, `stack ${s.name}`);
+}
+
+/**
+ * The resolved secrets for a compose process that starts containers. Fails
+ * closed: a stack that declares `secret_env` is never started without every
+ * key (that is how a plain restart used to drop one).
+ */
+function secretComposeEnv(
+  s: StackConfig,
+  secretEnv: Record<string, string> | undefined,
+): Record<string, string> {
+  const missing = secretEnvKeys(s.secretEnv).filter((k) => !secretEnv?.[k]);
+  if (missing.length)
+    throw new Error(
+      `stack ${s.name}: secret_env ${missing.join(", ")} not resolved; not starting it`,
+    );
+  return secretEnv ?? {};
+}
+
 function baseArgs(ctx: StackContext, s: StackConfig): string[] {
   return [
     "compose",
@@ -168,8 +203,15 @@ function running(container: string): boolean {
 export function stackUp(
   ctx: StackContext,
   s: StackConfig,
-  opts: { build?: boolean } = {},
+  opts: {
+    build?: boolean;
+    /** The stack's resolved `secret_env` (see resolveStackSecretEnv). */
+    secretEnv?: Record<string, string>;
+  } = {},
 ) {
+  const secrets = secretComposeEnv(s, opts.secretEnv);
+  // `compose config` runs without them: its JSON (which feeds the override)
+  // must not carry a secret.
   const resolved = resolve(ctx, s);
   // Provider side of each link: its referenced services join a private network.
   const needs = providerServices(s);
@@ -200,7 +242,11 @@ export function stackUp(
       ...(opts.build ? ["--build"] : []),
       "--remove-orphans",
     ],
-    { cwd: repoDir(s), env: composeEnv(ctx, s), inherit: true },
+    {
+      cwd: repoDir(s),
+      env: { ...composeEnv(ctx, s), ...secrets },
+      inherit: true,
+    },
   );
   // This stack as a provider: consumers already running keep their link
   // network, but a restarted provider's new containers must rejoin it.
@@ -274,14 +320,16 @@ export function stackRestartService(
   ctx: StackContext,
   s: StackConfig,
   service: string,
+  opts: { secretEnv?: Record<string, string> } = {},
 ): void {
+  const secrets = secretComposeEnv(s, opts.secretEnv);
   const override = path.join(stackDir(ctx, s), "override.yml");
   const args = fs.existsSync(override)
     ? overrideArgs(ctx, s)
     : baseArgs(ctx, s);
   must("docker", [...args, "restart", service], {
     cwd: repoDir(s),
-    env: composeEnv(ctx, s),
+    env: { ...composeEnv(ctx, s), ...secrets },
     inherit: true,
   });
 }

package/src/shared/stacks/status.ts

@@ -3,13 +3,18 @@
  * The CLI prints it; the web server serves it at /api/stacks (and through it
  * the MCP `stacks` tool).
  */
+import { spawnSync } from "child_process";
+import * as fs from "fs";
+import * as path from "path";
+import { readCatalog } from "../environment-manager/catalog.js";
 import { readEdgeConfig } from "../edge/config.js";
 import { routeUrl } from "../edge/render.js";
 import {
   getFocusedWorkspaceName,
   getWorkspaceByName,
+  listWorkspaceServices,
 } from "../environment-manager/workspaces.js";
-import { readStacks } from "./config.js";
+import { readManifestStacks, readStacks } from "./config.js";
 import { projectName, stackWorkspace } from "./render.js";
 import { stackHealth, type StackContext } from "./runtime.js";
 
@@ -97,3 +102,106 @@ export function stackStatuses(
     return status;
   });
 }
+
+/** A running Compose project started from one of a workspace's repos. */
+export interface WorkspaceStack {
+  /** The Den stack name, or the project name for a stack the repo runs. */
+  name: string;
+  repo: string;
+  project: string;
+}
+
+/** A running Compose project and the directory it was started from. */
+export interface RunningProject {
+  project: string;
+  /** `com.docker.compose.project.working_dir`; "" when unlabelled. */
+  workingDir: string;
+}
+
+/**
+ * The Compose projects with running containers, from one `docker ps`; null
+ * when docker can't be asked (not installed, daemon down, too slow).
+ */
+export function runningComposeProjects(): RunningProject[] | null {
+  const r = spawnSync(
+    "docker",
+    [
+      "ps",
+      "--format",
+      '{{.Label "com.docker.compose.project"}}\t{{.Label "com.docker.compose.project.working_dir"}}',
+    ],
+    { encoding: "utf-8", timeout: 5_000, stdio: ["ignore", "pipe", "ignore"] },
+  );
+  if (r.error || r.status !== 0) return null;
+  const seen = new Map<string, RunningProject>();
+  for (const line of (r.stdout ?? "").split("\n")) {
+    const [project = "", workingDir = ""] = line.trim().split("\t");
+    if (project && !seen.has(project))
+      seen.set(project, { project, workingDir: workingDir.trim() });
+  }
+  return [...seen.values()];
+}
+
+/** `p` and its realpath (compose may record either form). */
+function pathForms(p: string): string[] {
+  try {
+    const real = fs.realpathSync(p);
+    return real === p ? [p] : [p, real];
+  } catch {
+    return [p];
+  }
+}
+
+/**
+ * `workspace`'s running compose stacks: what `workspace deactivate` leaves
+ * up, since it releases only the daemons Den tracks. A project counts when
+ * it is a Den stack one of the workspace's repos declares (`compose:`, named
+ * by projectName), or when Compose started it from inside one of the
+ * workspace's checkouts (its working_dir label): the stacks a repo runs
+ * itself, e.g. from `make up`. Null when docker could not be asked.
+ */
+export function runningWorkspaceStacks(
+  workspace: string,
+  running: () => RunningProject[] | null = runningComposeProjects,
+): WorkspaceStack[] | null {
+  const ws = getWorkspaceByName(workspace);
+  if (!ws) return [];
+  const denStacks = new Map<string, { name: string; repo: string }>();
+  const checkouts: { repo: string; forms: string[] }[] = [];
+  try {
+    const { stackWorkspace: suffix } = resolveStackWorkspaces(workspace);
+    for (const s of Object.values(readManifestStacks(workspace)))
+      denStacks.set(projectName(s.name, suffix), {
+        name: s.name,
+        repo: s.repo,
+      });
+    const seen = new Set<string>();
+    for (const svc of listWorkspaceServices(ws.config, readCatalog(), ws.name))
+      if (svc.repoPath && !seen.has(svc.repoPath)) {
+        seen.add(svc.repoPath);
+        checkouts.push({ repo: svc.repoName, forms: pathForms(svc.repoPath) });
+      }
+  } catch {
+    return null; // an unreadable manifest or workspace name: can't tell
+  }
+  if (denStacks.size === 0 && checkouts.length === 0) return [];
+  const up = running();
+  if (!up) return null;
+  const inside = (dir: string, root: string): boolean =>
+    dir === root || dir.startsWith(`${root}${path.sep}`);
+  const out: WorkspaceStack[] = [];
+  for (const { project, workingDir } of up) {
+    const den = denStacks.get(project);
+    if (den) {
+      out.push({ ...den, project });
+      continue;
+    }
+    if (!workingDir) continue;
+    const dirs = pathForms(workingDir);
+    const owner = checkouts.find((c) =>
+      c.forms.some((root) => dirs.some((d) => inside(d, root))),
+    );
+    if (owner) out.push({ name: project, repo: owner.repo, project });
+  }
+  return out;
+}

package/src/shared/stacks/types.ts

@@ -10,6 +10,8 @@
  *       uses: [ragstack]                    # start first + private link network
  *       env:                                 # compose ${VAR} substitution
  *         RAG_PROXY_TARGET: "http://{ragstack.ragstack-server}:80"
+ *       secret_env:                          # from the secrets provider, at up/restart
+ *         OPENAI_API_KEY: proj/dev/OPENAI_API_KEY
  *
  * `{stack.service}` expands to that service's container name.
  */
@@ -36,6 +38,12 @@ export interface StackConfig {
    * `${VAR}` substitution) cannot change.
    */
   serviceEnv?: Record<string, Record<string, string>>;
+  /**
+   * `secret_env` (R16): env var → `<vault>/<item>/<field>` ref. Resolved
+   * through the secrets provider by `stack up` / `stack restart` and passed
+   * only in the docker compose process's env, never in the override file.
+   */
+  secretEnv?: Record<string, string>;
 }
 
 /** What `docker compose config --format json` tells us about a service. */

package/src/shared/types/index.ts

@@ -276,6 +276,12 @@ export interface ServiceManagementConfig {
   runs?: Partial<Record<Environment, ServiceRunsValue>>;
   /** Where a remote service runs: the env, an account label, a URL to probe. */
   target?: ServiceTargetConfig;
+  /**
+   * R16: env var → `<vault>/<item>/<field>` ref, resolved through the
+   * secrets provider when this service's up/restart command runs, and set
+   * only in that process's env (see src/shared/secrets/secret-env.ts).
+   */
+  secret_env?: Record<string, string>;
 }
 
 export type ServiceRunsValue = "local" | "remote";
@@ -572,6 +578,15 @@ export interface ManifestComposeConfig {
   profiles?: string[];
   /** service → replacement healthcheck `test` (written to den's override, not the repo). */
   healthchecks?: Record<string, string[]>;
+  /** service → env vars set on that service in the override (wins by key). */
+  service_env?: Record<string, Record<string, string | number | boolean>>;
+  /**
+   * R16: env var → `<vault>/<item>/<field>` ref, resolved through the
+   * secrets provider at stack up/restart and set only in the docker compose
+   * process's env (for the compose file's `${VAR}` / pass-through
+   * `environment:` entries). Never written to the override.
+   */
+  secret_env?: Record<string, string>;
 }
 
 export interface MigrationResult {