@monster-orange/den 0.2.1 → 0.2.3

package/src/shared/secrets/secret-env.ts

@@ -0,0 +1,149 @@
+/**
+ * `secret_env` (R16): environment variables whose values Den resolves from
+ * the configured secrets provider when it starts or restarts a target.
+ *
+ *   compose:
+ *     secret_env:
+ *       OPENAI_API_KEY: proj/dev_personal/OPENAI_API_KEY
+ *   services:
+ *     api:
+ *       secret_env:
+ *         SOME_KEY: vault/item/field
+ *
+ * Each value is a fully qualified `a/b/c` path, parsed into a SCOPED
+ * {@link SecretRef} (see its `scoped` field for what each provider does with
+ * it). Values only ever travel in the spawned process's env: never into the
+ * compose override, logs, activity records, MCP output, dry runs or errors.
+ * Everything here that produces text names keys and refs, never values.
+ *
+ * This module only imports types, so the manifest schema can use the parser.
+ */
+import type { SecretRef, SecretsProvider } from "./types.js";
+
+/** A `secret_env` block: env var name → `a/b/c` ref. */
+export type SecretEnvSpec = Record<string, string>;
+
+/** Env var names Den will set (POSIX-portable). */
+export const SECRET_ENV_KEY_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
+/** Why `raw` is not a valid secret_env ref, or null when it is. */
+export function secretEnvRefError(raw: unknown): string | null {
+  if (typeof raw !== "string")
+    return 'must be a string "<vault>/<item>/<field>"';
+  const parts = raw.split("/");
+  if (
+    parts.length !== 3 ||
+    parts.some((p) => p.trim() === "" || p !== p.trim())
+  )
+    return `"${raw}" must be exactly three non-empty parts, "<vault>/<item>/<field>" (Doppler: <project>/<config>/<key>)`;
+  return null;
+}
+
+/** Parse an `a/b/c` ref into a scoped SecretRef. Throws when malformed. */
+export function parseSecretEnvRef(raw: string): SecretRef {
+  const err = secretEnvRefError(raw);
+  if (err) throw new Error(`secret_env ref ${err}`);
+  const [vault, item, field] = raw.split("/");
+  return { vault, item, field, scoped: true };
+}
+
+/**
+ * Problems with a whole `secret_env` block, as `KEY: message` strings ([]
+ * when it is valid). `where` prefixes each message.
+ */
+export function secretEnvSpecErrors(spec: unknown, where: string): string[] {
+  if (spec === undefined) return [];
+  if (spec === null || typeof spec !== "object" || Array.isArray(spec))
+    return [`${where} must be a map of ENV_NAME: "<vault>/<item>/<field>"`];
+  const out: string[] = [];
+  for (const [k, v] of Object.entries(spec)) {
+    if (!SECRET_ENV_KEY_RE.test(k))
+      out.push(`${where}.${k}: "${k}" is not a valid env var name`);
+    const err = secretEnvRefError(v);
+    if (err) out.push(`${where}.${k}: ref ${err}`);
+  }
+  return out;
+}
+
+/** The validated block, or throws listing every problem. */
+export function assertSecretEnvSpec(
+  spec: unknown,
+  where: string,
+): SecretEnvSpec | undefined {
+  if (spec === undefined) return undefined;
+  const errors = secretEnvSpecErrors(spec, where);
+  if (errors.length) throw new Error(errors.join("\n"));
+  return { ...(spec as SecretEnvSpec) };
+}
+
+/** One key that could not be resolved. Never carries a value. */
+export interface SecretEnvMissing {
+  key: string;
+  ref: string;
+}
+
+export type SecretEnvResolution =
+  | { ok: true; env: Record<string, string> }
+  | { ok: false; missing: SecretEnvMissing[]; message: string };
+
+/**
+ * The failure text for unresolved keys: key, ref and provider, never a value
+ * and never the provider's own error output (which could echo one).
+ */
+export function describeSecretEnvMissing(
+  target: string,
+  missing: SecretEnvMissing[],
+  provider: string,
+): string {
+  const lines = missing.map(
+    (m) => `  ${m.key} (ref ${m.ref}, provider ${provider}): not resolved`,
+  );
+  return (
+    `${target}: ${missing.length} secret_env key(s) could not be resolved; not starting it:\n` +
+    lines.join("\n")
+  );
+}
+
+/**
+ * Resolve every key of `spec` through `provider`. All or nothing: when any
+ * key is missing (unset, empty, unreadable) the result is a failure naming
+ * each missing key, and no partial env is returned.
+ */
+export async function resolveSecretEnv(
+  spec: SecretEnvSpec | undefined,
+  provider: SecretsProvider,
+  target: string,
+): Promise<SecretEnvResolution> {
+  const entries = Object.entries(spec ?? {});
+  if (!entries.length) return { ok: true, env: {} };
+  const results = await Promise.all(
+    entries.map(async ([key, raw]) => {
+      if (secretEnvRefError(raw)) return { key, raw, value: null };
+      let value: string | null = null;
+      try {
+        value = await provider.tryGet(parseSecretEnvRef(raw));
+      } catch {
+        value = null;
+      }
+      return { key, raw, value };
+    }),
+  );
+  const missing = results
+    .filter((r) => !r.value)
+    .map((r) => ({ key: r.key, ref: String(r.raw) }));
+  if (missing.length)
+    return {
+      ok: false,
+      missing,
+      message: describeSecretEnvMissing(target, missing, provider.name),
+    };
+  return {
+    ok: true,
+    env: Object.fromEntries(results.map((r) => [r.key, r.value as string])),
+  };
+}
+
+/** The keys of a block, sorted: what a dry run or status may show. */
+export function secretEnvKeys(spec: SecretEnvSpec | undefined): string[] {
+  return Object.keys(spec ?? {}).sort();
+}

package/src/shared/secrets/types.ts

@@ -22,6 +22,16 @@ export interface SecretRef {
    * `flatKeyFor`.
    */
   key?: string;
+  /**
+   * Set on a fully qualified ref (a `secret_env` value, `a/b/c`): every part
+   * names a real location and no provider config fills it in.
+   *  - Doppler: vault = `--project`, item = `--config`, field = the key
+   *    (`secrets.doppler` and `secrets.keys` are not consulted).
+   *  - 1Password: vault / item / field as for any ref.
+   *  - env: field is the env var name (`secrets.keys` is not consulted).
+   * Unscoped refs (the default) keep the provider's configured scope.
+   */
+  scoped?: true;
 }
 
 export type SecretsProviderName = "doppler" | "1password" | "env";

package/src/shared/stacks/config.ts

@@ -17,6 +17,7 @@ import {
 } from "../environment-manager/workspaces.js";
 import { readManifest } from "../manifest/read-manifest.js";
 import type { ManifestComposeConfig } from "../types/index.js";
+import { assertSecretEnvSpec } from "../secrets/secret-env.js";
 import { assertStackName, parseTarget } from "./render.js";
 import type { StackConfig } from "./types.js";
 
@@ -41,6 +42,10 @@ function toStack(
   const routes = raw.routes ?? {};
   Object.values(routes).forEach(parseTarget);
   const files = list(raw.files, `${where}.files`);
+  const secretEnv =
+    "secret_env" in raw
+      ? assertSecretEnvSpec(raw.secret_env, `${where}.secret_env`)
+      : undefined;
   return {
     name,
     repo,
@@ -69,6 +74,7 @@ function toStack(
           ),
         }
       : {}),
+    ...(secretEnv && Object.keys(secretEnv).length ? { secretEnv } : {}),
   };
 }
 

package/src/shared/stacks/manifest-commands.ts

@@ -31,13 +31,23 @@ export function composeCommands(
   repoName: string,
   workspace: string,
 ): Record<string, CommandDef> {
-  if (!manifest.compose || !isPrimary(manifest, subsystem)) return {};
+  if (!manifest.compose) return {};
   // `docker compose up -d` returns once the containers are started, so these
   // finish on their own: never daemonize them.
   const stack = (sub: string) => ({
     cmd: `${denBin()} stack ${sub} ${manifestStackName(manifest, repoName)} -w ${workspace}`,
     detachable: false,
   });
+  // Any other manifest service names a compose service: it restarts on its
+  // own (`docker compose restart`, same container and env). Up and down stay
+  // the stack's, through the primary service.
+  if (!isPrimary(manifest, subsystem))
+    return {
+      restart: {
+        ...stack("restart"),
+        cmd: `${stack("restart").cmd} --service ${JSON.stringify(subsystem)}`,
+      },
+    };
   return {
     up: stack("up"),
     down: stack("down"),

package/src/shared/stacks/runtime.ts

@@ -16,6 +16,12 @@ import {
   renderStackRoutes,
   routedNetworks,
 } from "./render.js";
+import {
+  resolveSecretEnv,
+  secretEnvKeys,
+  type SecretEnvResolution,
+} from "../secrets/secret-env.js";
+import type { SecretsProvider } from "../secrets/types.js";
 import type { ResolvedProject, StackConfig } from "./types.js";
 
 function run(
@@ -117,6 +123,35 @@ function composeEnv(ctx: StackContext, s: StackConfig): NodeJS.ProcessEnv {
   };
 }
 
+/**
+ * Resolve a stack's `secret_env` through `provider` (all or nothing). Call
+ * it before anything starts or stops, and hand the env to stackUp /
+ * stackRestartService. Nothing in the result's message carries a value.
+ */
+export function resolveStackSecretEnv(
+  s: StackConfig,
+  provider: SecretsProvider,
+): Promise<SecretEnvResolution> {
+  return resolveSecretEnv(s.secretEnv, provider, `stack ${s.name}`);
+}
+
+/**
+ * The resolved secrets for a compose process that starts containers. Fails
+ * closed: a stack that declares `secret_env` is never started without every
+ * key (that is how a plain restart used to drop one).
+ */
+function secretComposeEnv(
+  s: StackConfig,
+  secretEnv: Record<string, string> | undefined,
+): Record<string, string> {
+  const missing = secretEnvKeys(s.secretEnv).filter((k) => !secretEnv?.[k]);
+  if (missing.length)
+    throw new Error(
+      `stack ${s.name}: secret_env ${missing.join(", ")} not resolved; not starting it`,
+    );
+  return secretEnv ?? {};
+}
+
 function baseArgs(ctx: StackContext, s: StackConfig): string[] {
   return [
     "compose",
@@ -168,8 +203,15 @@ function running(container: string): boolean {
 export function stackUp(
   ctx: StackContext,
   s: StackConfig,
-  opts: { build?: boolean } = {},
+  opts: {
+    build?: boolean;
+    /** The stack's resolved `secret_env` (see resolveStackSecretEnv). */
+    secretEnv?: Record<string, string>;
+  } = {},
 ) {
+  const secrets = secretComposeEnv(s, opts.secretEnv);
+  // `compose config` runs without them: its JSON (which feeds the override)
+  // must not carry a secret.
   const resolved = resolve(ctx, s);
   // Provider side of each link: its referenced services join a private network.
   const needs = providerServices(s);
@@ -200,7 +242,11 @@ export function stackUp(
       ...(opts.build ? ["--build"] : []),
       "--remove-orphans",
     ],
-    { cwd: repoDir(s), env: composeEnv(ctx, s), inherit: true },
+    {
+      cwd: repoDir(s),
+      env: { ...composeEnv(ctx, s), ...secrets },
+      inherit: true,
+    },
   );
   // This stack as a provider: consumers already running keep their link
   // network, but a restarted provider's new containers must rejoin it.
@@ -266,6 +312,28 @@ export function stackDown(ctx: StackContext, s: StackConfig): void {
   }
 }
 
+/**
+ * Restart one compose service of a running stack in place (`docker compose
+ * restart`): the same container, so the same env; nothing is recreated.
+ */
+export function stackRestartService(
+  ctx: StackContext,
+  s: StackConfig,
+  service: string,
+  opts: { secretEnv?: Record<string, string> } = {},
+): void {
+  const secrets = secretComposeEnv(s, opts.secretEnv);
+  const override = path.join(stackDir(ctx, s), "override.yml");
+  const args = fs.existsSync(override)
+    ? overrideArgs(ctx, s)
+    : baseArgs(ctx, s);
+  must("docker", [...args, "restart", service], {
+    cwd: repoDir(s),
+    env: { ...composeEnv(ctx, s), ...secrets },
+    inherit: true,
+  });
+}
+
 export function stackPs(ctx: StackContext, s: StackConfig): string {
   return run(
     "docker",

package/src/shared/stacks/types.ts

@@ -10,6 +10,8 @@
  *       uses: [ragstack]                    # start first + private link network
  *       env:                                 # compose ${VAR} substitution
  *         RAG_PROXY_TARGET: "http://{ragstack.ragstack-server}:80"
+ *       secret_env:                          # from the secrets provider, at up/restart
+ *         OPENAI_API_KEY: proj/dev/OPENAI_API_KEY
  *
  * `{stack.service}` expands to that service's container name.
  */
@@ -36,6 +38,12 @@ export interface StackConfig {
    * `${VAR}` substitution) cannot change.
    */
   serviceEnv?: Record<string, Record<string, string>>;
+  /**
+   * `secret_env` (R16): env var → `<vault>/<item>/<field>` ref. Resolved
+   * through the secrets provider by `stack up` / `stack restart` and passed
+   * only in the docker compose process's env, never in the override file.
+   */
+  secretEnv?: Record<string, string>;
 }
 
 /** What `docker compose config --format json` tells us about a service. */

package/src/shared/types/index.ts

@@ -276,6 +276,12 @@ export interface ServiceManagementConfig {
   runs?: Partial<Record<Environment, ServiceRunsValue>>;
   /** Where a remote service runs: the env, an account label, a URL to probe. */
   target?: ServiceTargetConfig;
+  /**
+   * R16: env var → `<vault>/<item>/<field>` ref, resolved through the
+   * secrets provider when this service's up/restart command runs, and set
+   * only in that process's env (see src/shared/secrets/secret-env.ts).
+   */
+  secret_env?: Record<string, string>;
 }
 
 export type ServiceRunsValue = "local" | "remote";
@@ -572,6 +578,15 @@ export interface ManifestComposeConfig {
   profiles?: string[];
   /** service → replacement healthcheck `test` (written to den's override, not the repo). */
   healthchecks?: Record<string, string[]>;
+  /** service → env vars set on that service in the override (wins by key). */
+  service_env?: Record<string, Record<string, string | number | boolean>>;
+  /**
+   * R16: env var → `<vault>/<item>/<field>` ref, resolved through the
+   * secrets provider at stack up/restart and set only in the docker compose
+   * process's env (for the compose file's `${VAR}` / pass-through
+   * `environment:` entries). Never written to the override.
+   */
+  secret_env?: Record<string, string>;
 }
 
 export interface MigrationResult {

package/src/ui/cli/flag-parser.ts

@@ -176,8 +176,9 @@ const COMMAND_FLAGS: Record<string, readonly string[]> = {
     "--force",
     "-A",
     "--app",
+    "--with-deps",
   ],
-  restart: ["-A", "--app"],
+  restart: ["-A", "--app", "-n", "--dry-run"],
   logs: [],
   setup: ["-f", "--force"],
   sync: [],