@openpalm/lib 0.9.8 → 0.10.1

package/src/control-plane/docker.ts

@@ -1,15 +1,10 @@
-/**
- * Docker integration — executes real docker compose commands.
- *
- * This module shells out to `docker compose` for lifecycle operations.
- * It reads the generated docker-compose.yml from the state directory
- * and uses it for all operations.
- *
- * Security: All commands use execFile with argument arrays to prevent
- * command injection. No user input is ever interpolated into shell strings.
- */
+/** Docker integration — executes docker compose commands via execFile (no shell). */
 import { execFile, spawn } from "node:child_process";
-import { existsSync, readFileSync } from "node:fs";
+import { existsSync } from "node:fs";
+import { parseEnvFile } from "./env.js";
+import { createLogger } from "../logger.js";
+
+const logger = createLogger("lib:docker");
 
 export type DockerResult = {
   ok: boolean;
@@ -18,29 +13,6 @@ export type DockerResult = {
   code: number;
 };
 
-/**
- * Parse a dotenv file into a key-value map.
- * Handles `KEY=value` lines; ignores comments and blank lines.
- */
-function parseEnvFile(path: string): Record<string, string> {
-  const vars: Record<string, string> = {};
-  try {
-    const content = readFileSync(path, "utf-8");
-    for (const line of content.split("\n")) {
-      let trimmed = line.trim();
-      if (!trimmed || trimmed.startsWith("#")) continue;
-      trimmed = trimmed.replace(/^export\s+/, '');
-      const eqIdx = trimmed.indexOf("=");
-      if (eqIdx > 0) {
-        vars[trimmed.slice(0, eqIdx)] = trimmed.slice(eqIdx + 1);
-      }
-    }
-  } catch {
-    // File not readable — skip
-  }
-  return vars;
-}
-
 /** Execute docker with an argument array — no shell interpolation. */
 function run(
   args: string[],
@@ -65,9 +37,9 @@ function run(
   });
 }
 
-/** Get the compose file path from state directory */
-function composeFile(stateDir: string): string {
-  return `${stateDir}/artifacts/docker-compose.yml`;
+/** Resolve the Docker Compose project name. Respects OP_PROJECT_NAME env var. */
+export function resolveComposeProjectName(): string {
+  return process.env.OP_PROJECT_NAME?.trim() || "openpalm";
 }
 
 /** Check if Docker is available */
@@ -108,142 +80,100 @@ export async function checkDockerCompose(): Promise<DockerResult> {
   });
 }
 
-/**
- * Build the `-f file1 -f file2 ...` args for docker compose.
- * Returns a flat array like ["-f", "path1", "-f", "path2"].
- */
-function composeFileArgs(stateDir: string, files?: string[]): string[] {
-  const fileList = files ?? [composeFile(stateDir)];
-  return fileList.flatMap((f) => ["-f", f]);
-}
-
-/**
- * Append `--env-file <path>` args for each existing env file.
- * Files that do not exist are silently skipped.
- */
-function pushEnvFileArgs(args: string[], envFiles?: string[]): void {
-  for (const ef of envFiles ?? []) {
+/** Build common prefix: compose -f ... --project-name ... --env-file ... */
+function buildComposeArgs(options: { files: string[]; envFiles?: string[] }): string[] {
+  const args = ["compose", ...options.files.flatMap((f) => ["-f", f]), "--project-name", resolveComposeProjectName()];
+  for (const ef of options.envFiles ?? []) {
     if (existsSync(ef)) args.push("--env-file", ef);
   }
+  return args;
+}
+
+/** Merge all env files into a single overrides object for process env. */
+function collectEnvOverrides(envFiles?: string[]): Record<string, string> {
+  const overrides: Record<string, string> = {};
+  for (const ef of envFiles ?? []) Object.assign(overrides, parseEnvFile(ef));
+  return overrides;
 }
 
 /**
- * Build the common prefix args for all docker compose commands:
- *   docker compose -f <file> ... --project-name openpalm [--env-file ...]
+ * Run `docker compose config` to validate compose file merge and variable substitution.
+ * Must be called before any lifecycle mutation (install/apply/update).
  */
-function buildComposeArgs(stateDir: string, options: { files?: string[]; envFiles?: string[] } = {}): string[] {
-  const args = ["compose", ...composeFileArgs(stateDir, options.files), "--project-name", "openpalm"];
-  pushEnvFileArgs(args, options.envFiles);
-  return args;
+export async function composePreflight(
+  options: { files: string[]; envFiles?: string[] }
+): Promise<DockerResult> {
+  const args = buildComposeArgs(options);
+  args.push("config", "--quiet");
+  return run(args, undefined, 30_000, collectEnvOverrides(options.envFiles));
+}
+
+export async function composeConfigServices(
+  options: { files: string[]; envFiles?: string[] }
+): Promise<{ ok: boolean; services: string[] }> {
+  const args = buildComposeArgs(options);
+  args.push("config", "--services");
+  const result = await run(args, undefined, 30_000, collectEnvOverrides(options.envFiles));
+  if (!result.ok) return { ok: false, services: [] };
+  const services = result.stdout.split("\n").map((s) => s.trim()).filter(Boolean);
+  return { ok: true, services };
 }
 
 /**
  * Run `docker compose up -d` with the generated compose file(s).
- * Pass `files` to merge multiple compose overlays (e.g. core + channel files).
+ * Pass `files` to merge multiple compose overlays (e.g. core + addon files).
  */
 export async function composeUp(
-  stateDir: string,
   options: {
-    files?: string[];
+    files: string[];
     profiles?: string[];
     services?: string[];
     envFiles?: string[];
     forceRecreate?: boolean;
     removeOrphans?: boolean;
-  } = {}
-): Promise<DockerResult> {
-  const primaryFile = options.files?.[0] ?? composeFile(stateDir);
-  if (!existsSync(primaryFile)) {
-    return {
-      ok: false,
-      stdout: "",
-      stderr: "Compose file not found",
-      code: 1
-    };
   }
-
-  const args = buildComposeArgs(stateDir, options);
-
-  if (options.profiles) {
-    for (const p of options.profiles) {
-      args.push("--profile", p);
-    }
+): Promise<DockerResult> {
+  if (!existsSync(options.files[0])) {
+    return { ok: false, stdout: "", stderr: "Compose file not found", code: 1 };
   }
-
+  const args = buildComposeArgs(options);
+  for (const p of options.profiles ?? []) args.push("--profile", p);
   args.push("up", "-d");
-
-  if (options.forceRecreate) {
-    args.push("--force-recreate");
-  }
-
-  if (options.removeOrphans) {
-    args.push("--remove-orphans");
-  }
-
-  if (options.services && options.services.length > 0) {
-    args.push(...options.services);
-  }
-
-  // Merge env file values into the process environment so Docker Compose
-  // resolves ${VAR} from fresh env files, not stale admin process env.
-  // Process env takes precedence over --env-file in Docker Compose,
-  // so we must override it explicitly.
-  const envOverrides: Record<string, string> = {};
-  for (const ef of options.envFiles ?? []) {
-    Object.assign(envOverrides, parseEnvFile(ef));
-  }
-
-  return run(args, stateDir, 300_000, envOverrides);
+  if (options.forceRecreate) args.push("--force-recreate");
+  if (options.removeOrphans) args.push("--remove-orphans");
+  if (options.services?.length) args.push(...options.services);
+  return run(args, undefined, 300_000, collectEnvOverrides(options.envFiles));
 }
 
 /**
  * Run `docker compose down` to stop and remove containers.
  */
 export async function composeDown(
-  stateDir: string,
   options: {
-    files?: string[];
+    files: string[];
     profiles?: string[];
     removeVolumes?: boolean;
     envFiles?: string[];
-  } = {}
-): Promise<DockerResult> {
-  const primaryFile = options.files?.[0] ?? composeFile(stateDir);
-  if (!existsSync(primaryFile)) {
-    return {
-      ok: false,
-      stdout: "",
-      stderr: "Compose file not found",
-      code: 1
-    };
   }
-
-  const args = buildComposeArgs(stateDir, options);
-
-  if (options.profiles) {
-    for (const p of options.profiles) {
-      args.push("--profile", p);
-    }
+): Promise<DockerResult> {
+  if (!existsSync(options.files[0])) {
+    return { ok: false, stdout: "", stderr: "Compose file not found", code: 1 };
   }
-
+  const args = buildComposeArgs(options);
+  for (const p of options.profiles ?? []) args.push("--profile", p);
   args.push("down");
-
-  if (options.removeVolumes) {
-    args.push("-v");
-  }
-
-  return run(args, stateDir);
+  if (options.removeVolumes) args.push("-v");
+  return run(args, undefined);
 }
 
 /**
  * Restart specific services.
  */
 export async function composeRestart(
-  stateDir: string,
   services: string[],
-  options: { files?: string[]; envFiles?: string[] } = {}
+  options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
-  const primaryFile = options.files?.[0] ?? composeFile(stateDir);
+  const primaryFile = options.files[0];
   if (!existsSync(primaryFile)) {
     return {
       ok: false,
@@ -253,79 +183,75 @@ export async function composeRestart(
     };
   }
 
-  const args = buildComposeArgs(stateDir, options);
+  const args = buildComposeArgs(options);
   args.push("restart", ...services);
 
-  return run(args, stateDir);
+  return run(args, undefined);
 }
 
 /**
  * Stop specific services.
  */
 export async function composeStop(
-  stateDir: string,
   services: string[],
-  options: { files?: string[]; envFiles?: string[] } = {}
+  options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
-  const args = buildComposeArgs(stateDir, options);
+  const args = buildComposeArgs(options);
   args.push("stop", ...services);
 
-  return run(args, stateDir);
+  return run(args, undefined);
 }
 
 /**
  * Start specific services (must already be created).
  */
 export async function composeStart(
-  stateDir: string,
   services: string[],
-  options: { files?: string[]; envFiles?: string[] } = {}
+  options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
-  const args = buildComposeArgs(stateDir, options);
+  const args = buildComposeArgs(options);
   // Use up -d for specific services to ensure they're created
   args.push("up", "-d", ...services);
 
-  return run(args, stateDir);
+  return run(args, undefined);
 }
 
 /**
  * Get the status of all containers in the project.
  */
 export async function composePs(
-  stateDir: string,
-  options: { files?: string[]; envFiles?: string[] } = {}
+  options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
-  const primaryFile = options.files?.[0] ?? composeFile(stateDir);
+  const primaryFile = options.files[0];
   if (!existsSync(primaryFile)) {
     // If no compose file, just list containers with the project label
     return run(
       [
         "ps",
         "--filter",
-        "label=com.docker.compose.project=openpalm",
+        `label=com.docker.compose.project=${resolveComposeProjectName()}`,
         "--format",
         "json"
       ],
-      stateDir
+      undefined
     );
   }
 
-  const args = buildComposeArgs(stateDir, options);
+  const args = buildComposeArgs(options);
   args.push("ps", "--format", "json");
 
-  return run(args, stateDir);
+  return run(args, undefined);
 }
 
 /**
  * Get logs for specific services or all services.
  */
 export async function composeLogs(
-  stateDir: string,
-  services?: string[],
-  tail = 100,
-  options: { files?: string[]; envFiles?: string[]; since?: string } = {}
+  services: string[] | undefined,
+  tail: number,
+  options: { files: string[]; envFiles?: string[]; since?: string }
 ): Promise<DockerResult> {
-  const args = buildComposeArgs(stateDir, options);
+  const args = buildComposeArgs(options);
   args.push("logs", "--tail", String(tail));
 
   if (options.since) {
@@ -336,67 +262,39 @@ export async function composeLogs(
     args.push(...services);
   }
 
-  return run(args, stateDir);
-}
-
-/**
- * Reload Caddy configuration by restarting the container.
- */
-export async function caddyReload(
-  stateDir: string,
-  options: { files?: string[]; envFiles?: string[] } = {}
-): Promise<DockerResult> {
-  return composeRestart(stateDir, ["caddy"], options);
+  return run(args, undefined);
 }
 
 /**
  * Pull image for a single service.
  */
 export async function composePullService(
-  stateDir: string,
   service: string,
-  options: { files?: string[]; envFiles?: string[] } = {}
+  options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
-  const args = buildComposeArgs(stateDir, options);
+  const args = buildComposeArgs(options);
   args.push("pull", service);
-
-  const envOverrides: Record<string, string> = {};
-  for (const ef of options.envFiles ?? []) {
-    Object.assign(envOverrides, parseEnvFile(ef));
-  }
-
-  return run(args, stateDir, 300_000, envOverrides);
+  return run(args, undefined, 300_000, collectEnvOverrides(options.envFiles));
 }
 
-/**
- * Pull latest images for all services.
- */
 export async function composePull(
-  stateDir: string,
-  options: { files?: string[]; envFiles?: string[] } = {}
+  options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
-  const args = buildComposeArgs(stateDir, options);
+  const args = buildComposeArgs(options);
   args.push("pull");
-
-  const envOverrides: Record<string, string> = {};
-  for (const ef of options.envFiles ?? []) {
-    Object.assign(envOverrides, parseEnvFile(ef));
-  }
-
-  return run(args, stateDir, 300_000, envOverrides);
+  return run(args, undefined, 300_000, collectEnvOverrides(options.envFiles));
 }
 
 /**
  * Get resource usage stats for all containers in the project.
  */
 export async function composeStats(
-  stateDir: string,
-  options: { files?: string[]; envFiles?: string[] } = {}
+  options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
-  const args = buildComposeArgs(stateDir, options);
+  const args = buildComposeArgs(options);
   args.push("stats", "--no-stream", "--format", "json");
 
-  return run(args, stateDir);
+  return run(args, undefined);
 }
 
 /**
@@ -421,29 +319,21 @@ export async function getDockerEvents(
  * Fire-and-forget recreation of the admin container.
  */
 export function selfRecreateAdmin(
-  stateDir: string,
-  options: { files?: string[]; envFiles?: string[] } = {}
+  options: { files: string[]; envFiles?: string[] }
 ): void {
-  const args = buildComposeArgs(stateDir, options);
+  const args = buildComposeArgs(options);
   args.push("--profile", "admin", "up", "-d", "--force-recreate", "--remove-orphans", "admin");
-
-  const envOverrides: Record<string, string> = {};
-  for (const ef of options.envFiles ?? []) {
-    Object.assign(envOverrides, parseEnvFile(ef));
-  }
-
   try {
     const child = spawn("docker", args, {
-      cwd: stateDir,
       stdio: "ignore",
       detached: true,
-      env: { ...process.env, ...envOverrides }
+      env: { ...process.env, ...collectEnvOverrides(options.envFiles) }
     });
     child.on("error", (err) => {
-      console.error("[selfRecreateAdmin] spawn error:", err.message);
+      logger.error("selfRecreateAdmin spawn error", { error: err.message });
     });
     child.unref();
   } catch (err) {
-    console.error("[selfRecreateAdmin] failed to spawn:", err instanceof Error ? err.message : err);
+    logger.error("selfRecreateAdmin failed to spawn", { error: err instanceof Error ? err.message : String(err) });
   }
 }

package/src/control-plane/env-schema-validation.test.ts

@@ -0,0 +1,118 @@
+/**
+ * Test that env schema validation uses the correct nested vault paths.
+ */
+import { describe, test, expect, beforeAll, afterAll } from "bun:test";
+import { mkdirSync, writeFileSync, rmSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import type { ControlPlaneState } from "./types.js";
+
+describe("env schema validation paths", () => {
+  let tmpDir: string;
+  let state: ControlPlaneState;
+
+  beforeAll(() => {
+    tmpDir = join(tmpdir(), `openpalm-schema-test-${Date.now()}`);
+    mkdirSync(join(tmpDir, "vault/user"), { recursive: true });
+    mkdirSync(join(tmpDir, "vault/stack"), { recursive: true });
+    mkdirSync(join(tmpDir, "data"), { recursive: true });
+    mkdirSync(join(tmpDir, "logs"), { recursive: true });
+    mkdirSync(join(tmpDir, "config"), { recursive: true });
+
+    state = {
+      adminToken: "test-token",
+      assistantToken: "test-assistant",
+      setupToken: "test-setup",
+      homeDir: tmpDir,
+      configDir: join(tmpDir, "config"),
+      vaultDir: join(tmpDir, "vault"),
+      dataDir: join(tmpDir, "data"),
+      logsDir: join(tmpDir, "logs"),
+      cacheDir: join(tmpDir, "cache"),
+      services: {},
+      artifacts: { compose: "" },
+      artifactMeta: [],
+      audit: [],
+    };
+  });
+
+  afterAll(() => {
+    if (tmpDir && existsSync(tmpDir)) {
+      rmSync(tmpDir, { recursive: true, force: true });
+    }
+  });
+
+  test("validation succeeds when no schema files exist (skip mode)", async () => {
+    const { validateProposedState } = await import("./validate.js");
+    const result = await validateProposedState(state);
+    // When schema files don't exist, validation is skipped (no errors)
+    expect(result.ok).toBe(true);
+    expect(result.errors).toEqual([]);
+  });
+
+  test("schema paths match canonical vault layout", () => {
+    const expectedUserSchema = join(tmpDir, "vault/user/user.env.schema");
+    const expectedStackSchema = join(tmpDir, "vault/stack/stack.env.schema");
+
+    writeFileSync(expectedUserSchema, "# test schema\n");
+    writeFileSync(expectedStackSchema, "# test schema\n");
+
+    expect(existsSync(expectedUserSchema)).toBe(true);
+    expect(existsSync(expectedStackSchema)).toBe(true);
+
+    // Old flat paths must NOT exist
+    expect(existsSync(join(tmpDir, "vault/user.env.schema"))).toBe(false);
+    expect(existsSync(join(tmpDir, "vault/system.env.schema"))).toBe(false);
+  });
+
+  test("validate.ts reads from nested paths, not flat paths", async () => {
+    // Write schemas at OLD flat paths — should be ignored
+    writeFileSync(join(tmpDir, "vault/user.env.schema"), "OPENAI_API_KEY\n");
+    writeFileSync(join(tmpDir, "vault/system.env.schema"), "OP_ADMIN_TOKEN\n");
+    // Write env files
+    writeFileSync(join(tmpDir, "vault/user/user.env"), "# empty\n");
+    writeFileSync(join(tmpDir, "vault/stack/stack.env"), "# empty\n");
+    // Delete nested schemas to prove flat paths are ignored
+    try { rmSync(join(tmpDir, "vault/user/user.env.schema")); } catch { /* may not exist */ }
+    try { rmSync(join(tmpDir, "vault/stack/stack.env.schema")); } catch { /* may not exist */ }
+
+    const { validateProposedState } = await import("./validate.js");
+    const result = await validateProposedState(state);
+    // Should pass because nested schemas don't exist (skipped), not because flat schemas were read
+    expect(result.ok).toBe(true);
+  });
+
+  test("validation reports warnings for missing required schema keys", async () => {
+    // Seed a schema that requires OPENAI_API_KEY
+    writeFileSync(join(tmpDir, "vault/user/user.env.schema"), "OPENAI_API_KEY=string\nOWNER_NAME=string\n");
+    // Seed an env file that is missing those keys
+    writeFileSync(join(tmpDir, "vault/user/user.env"), "# empty env\nSOME_OTHER_KEY=value\n");
+
+    const { validateProposedState } = await import("./validate.js");
+    const result = await validateProposedState(state);
+    // The validator should report warnings for missing keys (not errors — env validation is advisory)
+    expect(result.warnings.length).toBeGreaterThanOrEqual(0);
+  });
+
+  test("validation handles malformed env file gracefully", async () => {
+    writeFileSync(join(tmpDir, "vault/user/user.env.schema"), "OPENAI_API_KEY=string\n");
+    // Malformed: no = sign, just random text
+    writeFileSync(join(tmpDir, "vault/user/user.env"), "this is not a valid env file\n===\n");
+
+    const { validateProposedState } = await import("./validate.js");
+    const result = await validateProposedState(state);
+    // Should not throw — graceful handling
+    expect(typeof result.ok).toBe("boolean");
+  });
+
+  test("validation handles empty schema file gracefully", async () => {
+    writeFileSync(join(tmpDir, "vault/user/user.env.schema"), "");
+    writeFileSync(join(tmpDir, "vault/user/user.env"), "OPENAI_API_KEY=sk-test\n");
+
+    const { validateProposedState } = await import("./validate.js");
+    const result = await validateProposedState(state);
+    // Empty schema may cause varlock to report an error — that's fine,
+    // the important thing is it doesn't throw/crash
+    expect(typeof result.ok).toBe("boolean");
+  });
+});