@openpalm/lib 0.10.2 → 0.11.0-beta.2

package/src/control-plane/core-assets.ts

@@ -3,95 +3,92 @@
  *
  * Manages source-of-truth files for the ~/.openpalm/ layout:
  *   stack/              — compose runtime assets (core.compose.yml)
- *   vault/              — env schemas
  *
  * This module manages runtime-owned core files only.
  * Registry catalog refresh is handled separately in registry.ts.
- * All ensure* functions verify that the expected files exist at OP_HOME.
- * They create directories as needed but do NOT write file content — that
- * is the responsibility of `refreshCoreAssets()` (GitHub download) or
- * the CLI install command (which downloads assets before calling setup).
+ * Env validation has moved to `akm vault` + the in-house redactor — the
+ * historical `.env.schema` files (varlock format) were retired in #391.
  */
 import { mkdirSync, writeFileSync, readFileSync, existsSync, copyFileSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { resolveDataDir, resolveVaultDir, resolveOpenPalmHome, resolveBackupsDir } from "./home.js";
+import { dirname, join, resolve, sep } from "node:path";
+import { resolveStateDir, resolveOpenPalmHome, resolveBackupsDir, resolveStashDir } from "./home.js";
 import { createLogger } from "../logger.js";
 import { sha256 } from "./crypto.js";
 
 const logger = createLogger("core-assets");
 
-// ── Env Schema Files (vault/) ────────────────────────────────────────
-
-/**
- * Ensure the user env schema directory exists and return the expected
- * schema file path. The file itself may not exist yet — it is written
- * by refreshCoreAssets() or the CLI install command.
- */
-export function ensureUserEnvSchema(): string {
-  const vaultDir = resolveVaultDir();
-  const dir = `${vaultDir}/user`;
-  mkdirSync(dir, { recursive: true });
-  const path = `${dir}/user.env.schema`;
-  return path;
-}
-
-/**
- * Ensure the system env schema directory exists and return the expected
- * schema file path. The file itself may not exist yet — it is written
- * by refreshCoreAssets() or the CLI install command.
- */
-export function ensureSystemEnvSchema(): string {
-  const vaultDir = resolveVaultDir();
-  const dir = `${vaultDir}/stack`;
-  mkdirSync(dir, { recursive: true });
-  const path = `${dir}/stack.env.schema`;
-  return path;
-}
-
-// ── Memory data directory ────────────────────────────────────────────
-
-export function ensureMemoryDir(dataDir?: string): string {
-  const resolved = dataDir ?? resolveDataDir();
-  const dir = `${resolved}/memory`;
-  mkdirSync(dir, { recursive: true });
-  return dir;
-}
-
 // ── Core Compose (stack/) ─────────────────────────────────────────────
 
-function coreComposePath(): string {
-  return `${resolveOpenPalmHome()}/stack/core.compose.yml`;
-}
-
 export function ensureCoreCompose(): string {
-  const path = coreComposePath();
+  const path = `${resolveOpenPalmHome()}/config/stack/core.compose.yml`;
   mkdirSync(dirname(path), { recursive: true });
   return path;
 }
 
 export function readCoreCompose(): string {
-  const path = coreComposePath();
-  return readFileSync(path, "utf-8");
+  return readFileSync(`${resolveOpenPalmHome()}/config/stack/core.compose.yml`, "utf-8");
 }
 
 // ── OpenCode System Config ──────────────────────────────────────────
 
 export function ensureOpenCodeSystemConfig(): void {
-  const dir = `${resolveDataDir()}/assistant`;
+  const dir = `${resolveStateDir()}/assistant`;
   mkdirSync(dir, { recursive: true });
 }
 
+// ── Shared akm stash (skills / commands / agents) ────────────────────
+
+/**
+ * Seed the shared akm stash with built-in skills / commands / agents.
+ *
+ * Idempotent: **never overwrites** an existing file — user edits to a
+ * seeded asset always win, which preserves the same "config doesn't
+ * overwrite user edits" contract that governs the rest of OP_HOME.
+ *
+ * Returns the list of stash-relative paths that were actually written
+ * (empty on re-run when every seed already exists on disk).
+ *
+ * `seeds` is a map of stash-relative path → file content. Keys MUST be
+ * forward-slash relative paths that stay inside `data/stash/`; any key
+ * that escapes the stash directory after canonicalization throws,
+ * preventing a malicious caller from writing arbitrary files. Source of
+ * truth for the seeded files lives at `.openpalm/stash/` in the
+ * repo; the CLI embeds them at build time and passes the embedded
+ * record directly.
+ */
+export function seedStashAssets(seeds: Record<string, string>): string[] {
+  const stashDir = resolveStashDir();
+  const normalizedStash = resolve(stashDir);
+  const written: string[] = [];
+  for (const [relPath, content] of Object.entries(seeds)) {
+    const targetPath = join(stashDir, relPath);
+    const normalizedTarget = resolve(targetPath);
+    if (
+      normalizedTarget !== normalizedStash &&
+      !normalizedTarget.startsWith(normalizedStash + sep)
+    ) {
+      throw new Error(`Seed path escapes stash dir: ${relPath}`);
+    }
+    if (existsSync(targetPath)) continue;
+    mkdirSync(dirname(targetPath), { recursive: true });
+    writeFileSync(targetPath, content);
+    written.push(relPath);
+  }
+  return written;
+}
+
 // ── Asset Refresh (GitHub download) ──────────────────────────────────
 
 const REPO = "itlackey/openpalm";
 const VERSION = process.env.OP_ASSET_VERSION ?? "main";
 
+// Stash seeds are intentionally NOT in this list — they use seedStashAssets()
+// which never overwrites existing files (user edits win on re-install).
 const MANAGED_ASSETS: { relPath: string; githubFilename: string }[] = [
-  { relPath: "stack/core.compose.yml", githubFilename: ".openpalm/stack/core.compose.yml" },
-  { relPath: "data/assistant/opencode.jsonc", githubFilename: "core/assistant/opencode/opencode.jsonc" },
-  { relPath: "data/assistant/AGENTS.md", githubFilename: "core/assistant/opencode/AGENTS.md" },
-  { relPath: "vault/user/user.env.schema", githubFilename: ".openpalm/vault/user/user.env.schema" },
-  { relPath: "vault/stack/stack.env.schema", githubFilename: ".openpalm/vault/stack/stack.env.schema" },
+  { relPath: "config/stack/core.compose.yml",        githubFilename: ".openpalm/config/stack/core.compose.yml" },
+  { relPath: "config/assistant/opencode.jsonc",      githubFilename: ".openpalm/config/assistant/opencode.jsonc" },
+  { relPath: "config/assistant/openpalm.md",         githubFilename: ".openpalm/config/assistant/openpalm.md" },
+  { relPath: "config/assistant/system.md",           githubFilename: ".openpalm/config/assistant/system.md" },
 ];
 
 async function downloadAsset(filename: string): Promise<string> {

package/src/control-plane/docker.ts

@@ -37,9 +37,16 @@ function run(
   });
 }
 
-/** Resolve the Docker Compose project name. Respects OP_PROJECT_NAME env var. */
+/**
+ * Resolve the Docker Compose project name.
+ * Honors COMPOSE_PROJECT_NAME (Docker standard) and OP_PROJECT_NAME (legacy).
+ */
 export function resolveComposeProjectName(): string {
-  return process.env.OP_PROJECT_NAME?.trim() || "openpalm";
+  return (
+    process.env.OP_PROJECT_NAME?.trim() ||
+    process.env.COMPOSE_PROJECT_NAME?.trim() ||
+    "openpalm"
+  );
 }
 
 /** Check if Docker is available */
@@ -108,6 +115,24 @@ export async function composePreflight(
   return run(args, undefined, 30_000, collectEnvOverrides(options.envFiles));
 }
 
+/**
+ * Run compose config preflight validation before any mutation.
+ * Skipped when OP_SKIP_COMPOSE_PREFLIGHT is set (tests, CI).
+ */
+async function runPreflight(options: { files: string[]; envFiles?: string[] }): Promise<void> {
+  if (options.files.length === 0 || process.env.OP_SKIP_COMPOSE_PREFLIGHT) return;
+  const result = await composePreflight(options);
+  if (!result.ok) {
+    const project = resolveComposeProjectName();
+    const fileArgs = options.files.map((f) => `-f ${f}`).join(" ");
+    const envArgs = (options.envFiles ?? []).map((f) => `--env-file ${f}`).join(" ");
+    throw new Error(
+      `Compose preflight failed: ${result.stderr}\n` +
+      `Resolved command: docker compose ${fileArgs} --project-name ${project} ${envArgs} config --quiet`
+    );
+  }
+}
+
 export async function composeConfigServices(
   options: { files: string[]; envFiles?: string[] }
 ): Promise<{ ok: boolean; services: string[] }> {
@@ -133,6 +158,7 @@ export async function composeUp(
     removeOrphans?: boolean;
   }
 ): Promise<DockerResult> {
+  await runPreflight(options);
   if (!existsSync(options.files[0])) {
     return { ok: false, stdout: "", stderr: "Compose file not found", code: 1 };
   }
@@ -156,6 +182,7 @@ export async function composeDown(
     envFiles?: string[];
   }
 ): Promise<DockerResult> {
+  await runPreflight(options);
   if (!existsSync(options.files[0])) {
     return { ok: false, stdout: "", stderr: "Compose file not found", code: 1 };
   }
@@ -173,6 +200,7 @@ export async function composeRestart(
   services: string[],
   options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
+  await runPreflight(options);
   const primaryFile = options.files[0];
   if (!existsSync(primaryFile)) {
     return {
@@ -196,6 +224,7 @@ export async function composeStop(
   services: string[],
   options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
+  await runPreflight(options);
   const args = buildComposeArgs(options);
   args.push("stop", ...services);
 
@@ -209,6 +238,7 @@ export async function composeStart(
   services: string[],
   options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
+  await runPreflight(options);
   const args = buildComposeArgs(options);
   // Use up -d for specific services to ensure they're created
   args.push("up", "-d", ...services);
@@ -272,6 +302,7 @@ export async function composePullService(
   service: string,
   options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
+  await runPreflight(options);
   const args = buildComposeArgs(options);
   args.push("pull", service);
   return run(args, undefined, 300_000, collectEnvOverrides(options.envFiles));
@@ -280,6 +311,7 @@ export async function composePullService(
 export async function composePull(
   options: { files: string[]; envFiles?: string[] }
 ): Promise<DockerResult> {
+  await runPreflight(options);
   const args = buildComposeArgs(options);
   args.push("pull");
   return run(args, undefined, 300_000, collectEnvOverrides(options.envFiles));
@@ -315,25 +347,27 @@ export async function getDockerEvents(
   return run(args, undefined, 15_000);
 }
 
+
 /**
- * Fire-and-forget recreation of the admin container.
+ * Query Docker for a container's running state by name.
+ * Returns "running" or "stopped". Falls back to "unknown" on error.
  */
-export function selfRecreateAdmin(
-  options: { files: string[]; envFiles?: string[] }
-): void {
-  const args = buildComposeArgs(options);
-  args.push("--profile", "admin", "up", "-d", "--force-recreate", "--remove-orphans", "admin");
-  try {
-    const child = spawn("docker", args, {
-      stdio: "ignore",
-      detached: true,
-      env: { ...process.env, ...collectEnvOverrides(options.envFiles) }
-    });
-    child.on("error", (err) => {
-      logger.error("selfRecreateAdmin spawn error", { error: err.message });
-    });
-    child.unref();
-  } catch (err) {
-    logger.error("selfRecreateAdmin failed to spawn", { error: err instanceof Error ? err.message : String(err) });
-  }
+export function inspectContainerStatus(
+  containerName: string
+): Promise<"running" | "stopped" | "unknown"> {
+  return new Promise((resolve) => {
+    execFile(
+      "docker",
+      ["inspect", "--format", "{{.State.Status}}", containerName],
+      { timeout: 5000 },
+      (error, stdout) => {
+        if (error) {
+          resolve("unknown");
+          return;
+        }
+        const status = (stdout ?? "").toString().trim();
+        resolve(status === "running" ? "running" : "stopped");
+      }
+    );
+  });
 }

package/src/control-plane/env.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "bun:test";
-import { parseEnvContent, mergeEnvContent } from "./env.js";
+import { parseEnvContent, mergeEnvContent, removeEnvKey } from "./env.js";
 
 // ── Special character round-trips ────────────────────────────────────────
 // Values written by mergeEnvContent (which uses quoteEnvValue internally)
@@ -107,3 +107,27 @@ describe("mergeEnvContent updates existing keys with special char values", () =>
     expect(parsed.ADMIN_TOKEN).toBe("new#value");
   });
 });
+
+describe("removeEnvKey", () => {
+  it("removes a simple key", () => {
+    const out = removeEnvKey("FOO=1\nBAR=2\n", "FOO");
+    expect(parseEnvContent(out)).toEqual({ BAR: "2" });
+  });
+
+  it("returns content unchanged when key is absent", () => {
+    const input = "FOO=1\nBAR=2\n";
+    expect(removeEnvKey(input, "MISSING")).toBe(input);
+  });
+
+  it("handles the export prefix form", () => {
+    const out = removeEnvKey("export FOO=1\nBAR=2\n", "FOO");
+    expect(parseEnvContent(out)).toEqual({ BAR: "2" });
+  });
+
+  it("leaves comments above the deleted key intact", () => {
+    const out = removeEnvKey("# header comment\nFOO=1\nBAR=2\n", "FOO");
+    expect(out).toContain("# header comment");
+    expect(parseEnvContent(out).FOO).toBeUndefined();
+    expect(parseEnvContent(out).BAR).toBe("2");
+  });
+});

package/src/control-plane/env.ts

@@ -14,6 +14,15 @@ export function parseEnvFile(filePath: string): Record<string, string> {
   }
 }
 
+/**
+ * Resolve `${VAR}` and `${VAR:-default}` patterns in a string against the
+ * provided variable map. Unknown vars without a default expand to an empty
+ * string — mirrors compose's variable substitution semantics.
+ */
+export function expandEnvVars(input: string, vars: Record<string, string>): string {
+  return input.replace(/\$\{([^}:]+)(?::-([^}]*))?\}/g, (_, name, def) => vars[name] ?? def ?? '');
+}
+
 function quoteEnvValue(value: string): string {
   if (value.length === 0) return '';
   const needsQuoting = /[#"'\\\n\r$]/.test(value) || value !== value.trim();
@@ -25,6 +34,77 @@ function quoteEnvValue(value: string): string {
   return `"${escaped}"`;
 }
 
+/**
+ * Remove a key from .env content. Comments above the line and the
+ * surrounding blank-line structure are preserved exactly as written so
+ * round-tripping the file through this helper is non-destructive.
+ * If the key is absent the input is returned unchanged.
+ */
+export function removeEnvKey(content: string, key: string): string {
+  const lines = content.split('\n');
+  const out: string[] = [];
+  let removed = false;
+  for (const line of lines) {
+    let testLine = line.trim();
+    if (testLine.startsWith('export ')) testLine = testLine.slice(7).trimStart();
+    const eq = testLine.indexOf('=');
+    if (eq > 0 && testLine.slice(0, eq).trim() === key) {
+      removed = true;
+      continue;
+    }
+    out.push(line);
+  }
+  // If we matched, drop a trailing blank line that the deletion left behind so
+  // the file does not accumulate empty lines on repeated edits.
+  if (removed && out.length > 1 && out[out.length - 1] === '' && out[out.length - 2] === '') {
+    out.pop();
+  }
+  return out.join('\n');
+}
+
+/**
+ * Upserts a key=value pair in env file content. If the key exists, replaces the line;
+ * otherwise appends a new line.
+ */
+export function upsertEnvValue(content: string, key: string, value: string): string {
+  const escapedKey = key.replace(/[|\\{}()[\]^$+*?.-]/g, '\\$&');
+  const pattern = new RegExp(`^((?:export\\s+)?)${escapedKey}=.*$`, 'm');
+  if (pattern.test(content)) {
+    // Preserve the `export ` prefix if the original line had one
+    return content.replace(pattern, `$1${key}=${value}`);
+  }
+
+  const line = `${key}=${value}`;
+  const suffix = content.endsWith('\n') || content.length === 0 ? '' : '\n';
+  return `${content}${suffix}${line}\n`;
+}
+
+export const RELEASE_TAG_REGEX = /^v?\d+\.\d+\.\d+(?:[-+](?:[0-9A-Za-z]+(?:\.[0-9A-Za-z]+)*))?$/;
+
+/**
+ * Normalizes a repository ref to an image tag. Returns null for non-release refs.
+ * E.g. "0.9.0" → "v0.9.0", "v0.9.0" → "v0.9.0", "main" → null.
+ */
+export function resolveRequestedImageTag(repoRef: string): string | null {
+  const trimmed = repoRef.trim();
+  if (!trimmed || trimmed === 'main') return null;
+  if (!RELEASE_TAG_REGEX.test(trimmed)) return null;
+  return trimmed.startsWith('v') ? trimmed : `v${trimmed}`;
+}
+
+/**
+ * Reconciles the OP_IMAGE_TAG value in stack.env content.
+ */
+export function reconcileStackEnvImageTag(
+  content: string,
+  repoRef: string,
+  explicitImageTag?: string,
+): string {
+  const desiredImageTag = explicitImageTag || resolveRequestedImageTag(repoRef);
+  if (!desiredImageTag) return content;
+  return upsertEnvValue(content, 'OP_IMAGE_TAG', desiredImageTag);
+}
+
 export function mergeEnvContent(
   content: string,
   updates: Record<string, string>,

package/src/control-plane/home.ts

@@ -1,13 +1,14 @@
 /**
- * Home directory layout for the OpenPalm control plane (v0.10.0+).
+ * Home directory layout for the OpenPalm control plane (v0.11.0+).
  *
- * Replaces the XDG three-tier model with a single ~/.openpalm/ root:
- *   config/  — user-editable, non-secret configuration
- *   vault/   — secrets boundary (user.env, system.env)
- *   data/    — service-managed persistent data
- *   logs/    — consolidated audit/debug output
- *
- * Cache and rollback data live in ~/.cache/openpalm/ (ephemeral).
+ * Single ~/.openpalm/ root:
+ *   config/    — user-editable config + system config files (auth.json, akm/)
+ *   config/stack/ — compose runtime + stack config (stack.env, guardian.env, stack.yml, addons/)
+ *   cache/     — regenerable/semi-persistent data (akm cache, guardian cache, rollback)
+ *   state/     — persistent service data (assistant, admin, guardian, logs, backups, registry)
+ *   stash/     — akm knowledge (skills, vaults, agents)
+ *   workspace/ — shared assistant work area
+ *   config/stack/ — compose runtime assets + stack config (stack.env, guardian.env, stack.yml)
  */
 import { mkdirSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
@@ -32,28 +33,37 @@ export function resolveConfigDir(): string {
   return `${resolveOpenPalmHome()}/config`;
 }
 
-export function resolveVaultDir(): string {
-  return `${resolveOpenPalmHome()}/vault`;
+export function resolveStashDir(): string {
+  return `${resolveOpenPalmHome()}/stash`;
 }
 
-export function resolveDataDir(): string {
-  return `${resolveOpenPalmHome()}/data`;
+export function resolveWorkspaceDir(): string {
+  return `${resolveOpenPalmHome()}/workspace`;
 }
 
-export function resolveLogsDir(): string {
-  return `${resolveOpenPalmHome()}/logs`;
+export function resolveCacheDir(): string {
+  return `${resolveOpenPalmHome()}/cache`;
 }
 
-export function resolveCacheHome(): string {
-  return `${resolveHome()}/.cache/openpalm`;
+export function resolveStateDir(): string {
+  return `${resolveOpenPalmHome()}/state`;
 }
 
-export function resolveRollbackDir(): string {
-  return `${resolveCacheHome()}/rollback`;
+export function resolveStackDir(): string {
+  return `${resolveConfigDir()}/stack`;
+}
+
+// Derived from stateDir — used by registry.ts, rollback.ts, backup.ts, core-assets.ts
+export function resolveLogsDir(): string {
+  return `${resolveStateDir()}/logs`;
+}
+
+export function resolveBackupsDir(): string {
+  return `${resolveStateDir()}/backups`;
 }
 
 export function resolveRegistryDir(): string {
-  return `${resolveOpenPalmHome()}/registry`;
+  return `${resolveStateDir()}/registry`;
 }
 
 export function resolveRegistryAddonsDir(): string {
@@ -64,69 +74,56 @@ export function resolveRegistryAutomationsDir(): string {
   return `${resolveRegistryDir()}/automations`;
 }
 
-export function resolveStackDir(): string {
-  return `${resolveOpenPalmHome()}/stack`;
-}
-
-export function resolveBackupsDir(): string {
-  return `${resolveOpenPalmHome()}/backups`;
-}
-
-export function resolveWorkspaceDir(): string {
-  return `${resolveOpenPalmHome()}/data/workspace`;
+export function resolveRollbackDir(): string {
+  return `${resolveCacheDir()}/rollback`;
 }
 
 // ── Directory Setup ──────────────────────────────────────────────────
 
 /**
- * Create the full ~/.openpalm/ directory tree and cache directories.
+ * Create the full ~/.openpalm/ directory tree.
  */
 export function ensureHomeDirs(): void {
   const home = resolveOpenPalmHome();
-  const cache = resolveCacheHome();
 
   for (const dir of [
-    // config/ — user-editable, non-secret
+    // config/ — user-editable config + system config files
     `${home}/config`,
-    `${home}/config/automations`,
     `${home}/config/assistant`,
     `${home}/config/guardian`,
-
-    // vault/ — secrets boundary
-    `${home}/vault`,
-    `${home}/vault/stack`,
-    `${home}/vault/user`,
-
-    // data/ — service-managed persistent data
-    `${home}/data`,
-    `${home}/data/assistant`,
-    `${home}/data/admin`,
-    `${home}/data/memory`,
-    `${home}/data/guardian`,
-    `${home}/data/stash`,
-
-    // stack/ — compose files
-    `${home}/stack`,
-    `${home}/stack/addons`,
-
-    // registry/ — available catalog
-    `${home}/registry`,
-    `${home}/registry/addons`,
-    `${home}/registry/automations`,
-
-    // backups/ — user backups
-    `${home}/backups`,
-
-    // data/workspace/ — shared assistant workspace (compose: $OP_HOME/data/workspace:/work)
-    `${home}/data/workspace`,
-
-    // logs/ — consolidated audit/debug
-    `${home}/logs`,
-    `${home}/logs/opencode`,
-
-    // cache/ — ephemeral, regenerable
-    cache,
-    `${cache}/rollback`,
+    `${home}/config/akm`,           // AKM_CONFIG_DIR — akm setup config.json lives here
+
+    // cache/ — regenerable/semi-persistent data
+    `${home}/cache`,
+    `${home}/cache/akm`,            // akm registry index, downloaded artifacts
+    `${home}/cache/rollback`,       // rollback snapshots
+
+    // state/ — persistent service data
+    `${home}/state`,
+    `${home}/state/assistant`,      // assistant HOME bind mount
+    `${home}/state/admin`,          // admin home bind mount
+    `${home}/state/guardian`,       // guardian runtime data
+    `${home}/state/akm`,            // shared akm operational data (NOT config)
+    `${home}/state/akm/data`,
+    `${home}/state/akm/state`,
+    `${home}/state/logs`,
+    `${home}/state/logs/opencode`,
+    `${home}/state/backups`,
+    `${home}/state/registry`,
+    `${home}/state/registry/addons`,
+    `${home}/state/registry/automations`,
+
+    // stash/ — akm knowledge (skills, vaults, agents); stash/tasks/ for scheduled automations
+    `${home}/stash`,
+    `${home}/stash/vaults`,
+    `${home}/stash/tasks`,
+
+    // workspace/ — shared assistant work area
+    `${home}/workspace`,
+
+    // config/stack/ — compose runtime (addon overlays + stack config files)
+    `${home}/config/stack`,
+    `${home}/config/stack/addons`,
   ]) {
     mkdirSync(dir, { recursive: true });
   }