@openpalm/lib 0.9.9 → 0.10.2

package/src/control-plane/secret-mappings.ts

@@ -0,0 +1,185 @@
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import type { ControlPlaneState } from './types.js';
+
+export type SecretScope = 'user' | 'system';
+export type SecretKind = 'core' | 'component' | 'custom';
+
+export type SecretEntryMetadata = {
+  key: string;
+  scope: SecretScope;
+  kind: SecretKind;
+  provider: 'plaintext' | 'pass';
+  present: boolean;
+  envKey?: string;
+  updatedAt?: string;
+};
+
+export type IndexedSecretEntry = {
+  envKey: string;
+  scope: SecretScope;
+  kind: Exclude<SecretKind, 'core'>;
+  updatedAt: string;
+};
+
+type CoreSecretMapping = {
+  secretKey: string;
+  envKey: string;
+  scope: SecretScope;
+};
+
+const STATIC_CORE_MAPPINGS: CoreSecretMapping[] = [
+  // Core authentication tokens
+  { secretKey: 'openpalm/admin-token', envKey: 'OP_ADMIN_TOKEN', scope: 'system' },
+  { secretKey: 'openpalm/assistant-token', envKey: 'OP_ASSISTANT_TOKEN', scope: 'system' },
+  { secretKey: 'openpalm/memory/auth-token', envKey: 'OP_MEMORY_TOKEN', scope: 'system' },
+  { secretKey: 'openpalm/opencode/server-password', envKey: 'OP_OPENCODE_PASSWORD', scope: 'system' },
+  // LLM provider API keys
+  { secretKey: 'openpalm/openai/api-key', envKey: 'OPENAI_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/anthropic/api-key', envKey: 'ANTHROPIC_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/groq/api-key', envKey: 'GROQ_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/mistral/api-key', envKey: 'MISTRAL_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/google/api-key', envKey: 'GOOGLE_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/together/api-key', envKey: 'TOGETHER_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/deepseek/api-key', envKey: 'DEEPSEEK_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/xai/api-key', envKey: 'XAI_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/huggingface/token', envKey: 'HF_TOKEN', scope: 'user' },
+  { secretKey: 'openpalm/mcp/api-key', envKey: 'MCP_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/embedding/api-key', envKey: 'EMBEDDING_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/lmstudio/api-key', envKey: 'LMSTUDIO_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/openviking/api-key', envKey: 'OPENVIKING_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/openviking/vlm-api-key', envKey: 'VLM_API_KEY', scope: 'user' },
+  // Channel-specific credentials
+  { secretKey: 'openpalm/discord/bot-token', envKey: 'DISCORD_BOT_TOKEN', scope: 'user' },
+  { secretKey: 'openpalm/slack/bot-token', envKey: 'SLACK_BOT_TOKEN', scope: 'user' },
+  { secretKey: 'openpalm/slack/app-token', envKey: 'SLACK_APP_TOKEN', scope: 'user' },
+  { secretKey: 'openpalm/voice/stt-api-key', envKey: 'STT_API_KEY', scope: 'user' },
+  { secretKey: 'openpalm/voice/tts-api-key', envKey: 'TTS_API_KEY', scope: 'user' },
+];
+
+// 128 bits of the SHA-256 digest keeps collision risk negligible while
+// leaving enough room for the OP_SECRET_ prefix in env var names.
+const HASH_PREFIX_LENGTH = 32;
+
+type SecretIndexFile = {
+  entries: Record<string, IndexedSecretEntry>;
+};
+
+function secretIndexPath(state: ControlPlaneState): string {
+  return `${state.dataDir}/secrets/plaintext-index.json`;
+}
+
+function normalizeIndexedKey(key: string): string {
+  return key.trim().replace(/\/+/g, '/').replace(/^\/|\/$/g, '');
+}
+
+export function sanitizeSecretSegment(input: string): string {
+  return input
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')
+    .slice(0, 63);
+}
+
+export function secretKeyFromComponentField(instanceId: string, fieldName: string): string {
+  return `openpalm/component/${sanitizeSecretSegment(instanceId)}/${sanitizeSecretSegment(fieldName)}`;
+}
+
+export function classifySecretKey(key: string): SecretKind {
+  if (key.startsWith('openpalm/component/')) return 'component';
+  if (key.startsWith('openpalm/custom/')) return 'custom';
+  return 'core';
+}
+
+export function generatePlaintextEnvKey(secretKey: string): string {
+  const digest = createHash('sha256').update(secretKey).digest('hex').slice(0, HASH_PREFIX_LENGTH).toUpperCase();
+  return `OP_SECRET_${digest}`;
+}
+
+export function classifySecretScope(key: string): SecretScope {
+  if (key.startsWith('openpalm/component/')) return 'system';
+  if (key.startsWith('openpalm/custom/')) return 'user';
+  const coreMapping = STATIC_CORE_MAPPINGS.find((m) => m.secretKey === key);
+  if (coreMapping) return coreMapping.scope;
+  return 'system';
+}
+
+export function getCoreSecretMappings(systemEnv: Record<string, string>): CoreSecretMapping[] {
+  const dynamicMappings: CoreSecretMapping[] = [];
+  for (const envKey of Object.keys(systemEnv)) {
+    const match = envKey.match(/^CHANNEL_([A-Z0-9_]+)_SECRET$/);
+    if (!match?.[1]) continue;
+    dynamicMappings.push({
+      secretKey: `openpalm/channel/${match[1].toLowerCase()}/secret`,
+      envKey,
+      scope: 'system',
+    });
+  }
+  return [...STATIC_CORE_MAPPINGS, ...dynamicMappings];
+}
+
+export function findCoreSecretByKey(
+  key: string,
+  systemEnv: Record<string, string>,
+): CoreSecretMapping | null {
+  return getCoreSecretMappings(systemEnv).find((entry) => entry.secretKey === key) ?? null;
+}
+
+export function findCoreSecretByEnvKey(
+  envKey: string,
+  systemEnv: Record<string, string>,
+): CoreSecretMapping | null {
+  return getCoreSecretMappings(systemEnv).find((entry) => entry.envKey === envKey) ?? null;
+}
+
+export function readPlaintextSecretIndex(state: ControlPlaneState): SecretIndexFile {
+  const path = secretIndexPath(state);
+  if (!existsSync(path)) {
+    return { entries: {} };
+  }
+
+  try {
+    const parsed = JSON.parse(readFileSync(path, 'utf-8')) as SecretIndexFile;
+    return parsed && typeof parsed === 'object' && parsed.entries ? parsed : { entries: {} };
+  } catch {
+    return { entries: {} };
+  }
+}
+
+export function writePlaintextSecretIndex(state: ControlPlaneState, index: SecretIndexFile): void {
+  const dir = `${state.dataDir}/secrets`;
+  mkdirSync(dir, { recursive: true });
+  writeFileSync(secretIndexPath(state), JSON.stringify(index, null, 2) + '\n');
+}
+
+export function ensurePlaintextSecretEntry(
+  state: ControlPlaneState,
+  key: string,
+  scope?: SecretScope,
+): IndexedSecretEntry {
+  const normalizedKey = normalizeIndexedKey(key);
+  const index = readPlaintextSecretIndex(state);
+  const existing = index.entries[normalizedKey];
+  if (existing) {
+    return existing;
+  }
+
+  const entry: IndexedSecretEntry = {
+    envKey: generatePlaintextEnvKey(normalizedKey),
+    scope: scope ?? (normalizedKey.startsWith('openpalm/component/') ? 'system' : 'user'),
+    kind: classifySecretKey(normalizedKey) === 'component' ? 'component' : 'custom',
+    updatedAt: new Date().toISOString(),
+  };
+  index.entries[normalizedKey] = entry;
+  writePlaintextSecretIndex(state, index);
+  return entry;
+}
+
+export function removePlaintextSecretEntry(state: ControlPlaneState, key: string): void {
+  const normalizedKey = normalizeIndexedKey(key);
+  const index = readPlaintextSecretIndex(state);
+  if (!index.entries[normalizedKey]) return;
+  delete index.entries[normalizedKey];
+  writePlaintextSecretIndex(state, index);
+}

package/src/control-plane/secrets.ts

@@ -1,169 +1,243 @@
-/**
- * Secrets and connection key management for the OpenPalm control plane.
- */
-import { mkdirSync, writeFileSync, readFileSync, existsSync } from "node:fs";
+/** Secrets and capability key management. */
+import { mkdirSync, writeFileSync, readFileSync, existsSync, chmodSync, lstatSync, rmSync } from "node:fs";
 import { randomBytes } from "node:crypto";
+import { createLogger } from "../logger.js";
 import { parseEnvFile, mergeEnvContent } from './env.js';
 import type { ControlPlaneState } from "./types.js";
-import { resolveConfigHome } from "./paths.js";
+import { resolveConfigDir } from "./home.js";
 
 const OPENCODE_STARTER_CONFIG = JSON.stringify({ $schema: "https://opencode.ai/config.json" }, null, 2) + "\n";
+const logger = createLogger("secrets");
 
-// ── Connection Key Management ───────────────────────────────────────────
-
-export const ALLOWED_CONNECTION_KEYS = new Set([
-  "OPENAI_API_KEY",
-  "ANTHROPIC_API_KEY",
-  "GROQ_API_KEY",
-  "MISTRAL_API_KEY",
-  "GOOGLE_API_KEY",
-  "SYSTEM_LLM_PROVIDER",
-  "SYSTEM_LLM_BASE_URL",
-  "SYSTEM_LLM_MODEL",
-  "OPENAI_BASE_URL",
-  "EMBEDDING_MODEL",
-  "EMBEDDING_DIMS",
-  "MEMORY_USER_ID",
-  "MEMORY_AUTH_TOKEN",
-  "OWNER_NAME",
-  "OWNER_EMAIL",
-]);
-
-export const REQUIRED_LLM_PROVIDER_KEYS = [
-  "OPENAI_API_KEY",
-  "ANTHROPIC_API_KEY",
-  "GROQ_API_KEY",
-  "MISTRAL_API_KEY",
-  "GOOGLE_API_KEY"
-];
 
-/** Keys that are non-secret config — returned unmasked in connection responses. */
+/** Keys whose values are shown unmasked in the UI (not secrets). */
 export const PLAIN_CONFIG_KEYS = new Set([
-  "SYSTEM_LLM_PROVIDER",
-  "SYSTEM_LLM_BASE_URL",
-  "SYSTEM_LLM_MODEL",
   "OPENAI_BASE_URL",
-  "EMBEDDING_MODEL",
-  "EMBEDDING_DIMS",
-  "MEMORY_USER_ID",
   "OWNER_NAME",
   "OWNER_EMAIL",
 ]);
 
-// ── Secrets Management ──────────────────────────────────────────────────
 
-export function ensureSecrets(state: ControlPlaneState): void {
-  mkdirSync(state.configDir, { recursive: true });
-  const secretsPath = `${state.configDir}/secrets.env`;
-  if (existsSync(secretsPath)) {
+const VAULT_DIR_MODE = 0o700;
+const VAULT_FILE_MODE = 0o600;
+
+function enforceVaultDirMode(vaultDir: string): void {
+  mkdirSync(vaultDir, { recursive: true, mode: VAULT_DIR_MODE });
+  try {
+    chmodSync(vaultDir, VAULT_DIR_MODE);
+  } catch (error) {
+    logger.warn("failed to enforce vault directory permissions", {
+      vaultDir,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+function writeVaultFile(path: string, content: string): void {
+  writeFileSync(path, content, { mode: VAULT_FILE_MODE });
+  try {
+    chmodSync(path, VAULT_FILE_MODE);
+  } catch (error) {
+    logger.warn("failed to enforce vault file permissions", {
+      path,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+function mergeVaultEnvFile(path: string, updates: Record<string, string>, uncomment = false): void {
+  if (Object.keys(updates).length === 0) return;
+  const raw = existsSync(path) ? readFileSync(path, "utf-8") : "";
+  let merged = mergeEnvContent(raw, updates, { uncomment });
+  if (!merged.endsWith("\n")) merged += "\n";
+  writeVaultFile(path, merged);
+}
+
+function ensureSystemSecrets(state: ControlPlaneState): void {
+  const systemEnvPath = `${state.vaultDir}/stack/stack.env`;
+  const existing = existsSync(systemEnvPath) ? parseEnvFile(systemEnvPath) : {};
+  const updates: Record<string, string> = {};
+
+  if (!existing.OP_ADMIN_TOKEN && state.adminToken) {
+    updates.OP_ADMIN_TOKEN = state.adminToken;
+  }
+  if (!existing.OP_ASSISTANT_TOKEN) {
+    updates.OP_ASSISTANT_TOKEN = randomBytes(32).toString("hex");
+  }
+  if (!existing.OP_MEMORY_TOKEN) {
+    updates.OP_MEMORY_TOKEN = randomBytes(32).toString("hex");
+  }
+
+  if (!existsSync(systemEnvPath)) {
+    const header = [
+      "# OpenPalm — Stack Configuration",
+      "# All secrets and configuration live here. Advanced users may edit directly.",
+      "",
+      "# ── Authentication ──────────────────────────────────────────────────",
+      "OP_ADMIN_TOKEN=",
+      "OP_ASSISTANT_TOKEN=",
+      "",
+      "# ── Service Auth ─────────────────────────────────────────────────────",
+      "OP_MEMORY_TOKEN=",
+      "OP_OPENCODE_PASSWORD=",
+      "",
+      "# ── Provider API Keys ────────────────────────────────────────────────",
+      "OPENAI_API_KEY=",
+      "OPENAI_BASE_URL=",
+      "ANTHROPIC_API_KEY=",
+      "GROQ_API_KEY=",
+      "MISTRAL_API_KEY=",
+      "GOOGLE_API_KEY=",
+      "OPENVIKING_API_KEY=",
+      "MCP_API_KEY=",
+      "EMBEDDING_API_KEY=",
+      "LMSTUDIO_API_KEY=",
+      "",
+      "# ── Owner ────────────────────────────────────────────────────────────",
+      `OWNER_NAME=${process.env.OWNER_NAME ?? ""}`,
+      `OWNER_EMAIL=${process.env.OWNER_EMAIL ?? ""}`,
+      "",
+    ].join("\n");
+    const content = mergeEnvContent(header, updates);
+    writeVaultFile(systemEnvPath, content.endsWith("\n") ? content : content + "\n");
     return;
   }
 
-  const secretLines: string[] = [];
-  secretLines.push("# OpenPalm Secrets");
-  secretLines.push("# Edit this file to update admin token and LLM keys.");
-  secretLines.push("# System-managed secrets (database + channel HMAC) do not belong here.");
-  secretLines.push("");
-  secretLines.push("export OPENPALM_ADMIN_TOKEN=");
-  secretLines.push("export ADMIN_TOKEN=");
-  secretLines.push("");
-  secretLines.push("# LLM provider keys");
-  secretLines.push("export OPENAI_API_KEY=");
-  secretLines.push("export OPENAI_BASE_URL=");
-  secretLines.push("export ANTHROPIC_API_KEY=");
-  secretLines.push("export GROQ_API_KEY=");
-  secretLines.push("export MISTRAL_API_KEY=");
-  secretLines.push("export GOOGLE_API_KEY=");
-  secretLines.push("");
-  secretLines.push("# Memory");
-  secretLines.push(`export MEMORY_USER_ID=${process.env.MEMORY_USER_ID ?? process.env.OPENMEMORY_USER_ID ?? "default_user"}`);
-  secretLines.push("");
-  secretLines.push("# Service auth tokens (auto-generated)");
-  secretLines.push(`export MEMORY_AUTH_TOKEN=${randomBytes(32).toString("hex")}`);
-  secretLines.push("");
-  secretLines.push("# Owner");
-  secretLines.push(`export OWNER_NAME=${process.env.OWNER_NAME ?? ""}`);
-  secretLines.push(`export OWNER_EMAIL=${process.env.OWNER_EMAIL ?? ""}`);
-  writeFileSync(secretsPath, secretLines.join("\n") + "\n");
+  mergeVaultEnvFile(systemEnvPath, updates, true);
+}
+
+export function ensureSecrets(state: ControlPlaneState): void {
+  enforceVaultDirMode(state.vaultDir);
+  mkdirSync(`${state.vaultDir}/stack`, { recursive: true, mode: VAULT_DIR_MODE });
+  mkdirSync(`${state.vaultDir}/user`, { recursive: true, mode: VAULT_DIR_MODE });
+
+  // user.env is an empty placeholder — users can add custom vars here.
+  // All standard config lives in stack.env.
+  const userEnvPath = `${state.vaultDir}/user/user.env`;
+  if (!existsSync(userEnvPath)) {
+    writeVaultFile(userEnvPath, [
+      "# OpenPalm — User Extensions",
+      "# Add any custom environment variables here.",
+      "# These are loaded by compose alongside stack.env.",
+      "",
+    ].join("\n"));
+  } else {
+    try { chmodSync(userEnvPath, VAULT_FILE_MODE); } catch { /* best-effort */ }
+  }
+
+  ensureSystemSecrets(state);
+  ensureGuardianEnv(state.vaultDir);
+  ensureAuthJson(state.vaultDir);
+}
+
+/**
+ * Ensure vault/stack/guardian.env exists.
+ * Channel HMAC secrets (CHANNEL_<NAME>_SECRET) live here exclusively.
+ * This file is loaded by the guardian as an env_file and via GUARDIAN_SECRETS_PATH.
+ */
+function ensureGuardianEnv(vaultDir: string): void {
+  const guardianEnvPath = `${vaultDir}/stack/guardian.env`;
+  mkdirSync(`${vaultDir}/stack`, { recursive: true, mode: VAULT_DIR_MODE });
+  if (!existsSync(guardianEnvPath)) {
+    writeVaultFile(guardianEnvPath, [
+      "# Guardian channel HMAC secrets — managed by openpalm",
+      "# Each enabled channel gets a CHANNEL_<NAME>_SECRET entry.",
+      "",
+    ].join("\n"));
+  } else {
+    try { chmodSync(guardianEnvPath, VAULT_FILE_MODE); } catch { /* best-effort */ }
+  }
+}
+
+function ensureAuthJson(vaultDir: string): void {
+  const authJsonPath = `${vaultDir}/stack/auth.json`;
+  mkdirSync(`${vaultDir}/stack`, { recursive: true, mode: VAULT_DIR_MODE });
+
+  if (existsSync(authJsonPath)) {
+    try {
+      if (lstatSync(authJsonPath).isDirectory()) {
+        rmSync(authJsonPath, { recursive: true, force: true });
+      } else {
+        chmodSync(authJsonPath, VAULT_FILE_MODE);
+        return;
+      }
+    } catch (error) {
+      logger.warn("failed to repair auth.json path", {
+        path: authJsonPath,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+  }
+
+  writeVaultFile(authJsonPath, "{}\n");
 }
 
 export function updateSecretsEnv(
   state: ControlPlaneState,
   updates: Record<string, string>
 ): void {
-  const secretsPath = `${state.configDir}/secrets.env`;
-  if (!existsSync(secretsPath)) {
-    throw new Error("secrets.env does not exist — run setup first");
+  const stackEnvPath = `${state.vaultDir}/stack/stack.env`;
+  if (!existsSync(stackEnvPath)) {
+    throw new Error("vault/stack/stack.env does not exist — run setup first");
   }
 
-  const raw = readFileSync(secretsPath, "utf-8");
-  writeFileSync(secretsPath, mergeEnvContent(raw, updates, { uncomment: true }));
+  mergeVaultEnvFile(stackEnvPath, updates, true);
+}
+
+/** Read and parse vault/stack/stack.env. Returns {} if the file does not exist. */
+export function readStackEnv(vaultDir: string): Record<string, string> {
+  return parseEnvFile(`${vaultDir}/stack/stack.env`);
 }
 
-export function readSecretsEnvFile(configDir: string): Record<string, string> {
-  const parsed = parseEnvFile(`${configDir}/secrets.env`);
-  const result: Record<string, string> = {};
-  for (const [key, value] of Object.entries(parsed)) {
-    if (ALLOWED_CONNECTION_KEYS.has(key)) result[key] = value;
+export function updateSystemSecretsEnv(
+  state: ControlPlaneState,
+  updates: Record<string, string>
+): void {
+  const systemEnvPath = `${state.vaultDir}/stack/stack.env`;
+  enforceVaultDirMode(state.vaultDir);
+  if (!existsSync(systemEnvPath)) {
+    ensureSystemSecrets(state);
   }
-  return result;
+  mergeVaultEnvFile(systemEnvPath, updates, true);
 }
 
 export function patchSecretsEnvFile(
-  configDir: string,
+  vaultDir: string,
   patches: Record<string, string>
 ): void {
-  const allowed: Record<string, string> = {};
-  for (const [key, value] of Object.entries(patches)) {
-    if (ALLOWED_CONNECTION_KEYS.has(key)) {
-      allowed[key] = value;
-    }
-  }
-  if (Object.keys(allowed).length === 0) return;
+  if (Object.keys(patches).length === 0) return;
 
-  const secretsPath = `${configDir}/secrets.env`;
-  mkdirSync(configDir, { recursive: true });
+  const stackEnvPath = `${vaultDir}/stack/stack.env`;
+  enforceVaultDirMode(vaultDir);
+  mkdirSync(`${vaultDir}/stack`, { recursive: true, mode: VAULT_DIR_MODE });
 
   let existingContent = "";
   try {
-    if (existsSync(secretsPath)) {
-      existingContent = readFileSync(secretsPath, "utf-8");
+    if (existsSync(stackEnvPath)) {
+      existingContent = readFileSync(stackEnvPath, "utf-8");
     }
   } catch {
     // start fresh
   }
 
-  let result = mergeEnvContent(existingContent, allowed);
+  let result = mergeEnvContent(existingContent, patches);
   if (!result.endsWith("\n")) result += "\n";
-  writeFileSync(secretsPath, result);
+  writeVaultFile(stackEnvPath, result);
 }
 
-// ── Connection Value Masking ────────────────────────────────────────────
 
-export function maskConnectionValue(key: string, value: string): string {
+export function maskSecretValue(key: string, value: string): string {
   if (!value) return "";
   if (PLAIN_CONFIG_KEYS.has(key)) return value;
   if (value.length <= 4) return "****";
   return "*".repeat(value.length - 4) + value.slice(-4);
 }
 
-// ── Secrets Loading ────────────────────────────────────────────────────
-
-export function loadSecretsEnvFile(configDir?: string): Record<string, string> {
-  const base = configDir ?? resolveConfigHome();
-  const parsed = parseEnvFile(`${base}/secrets.env`);
-  const result: Record<string, string> = {};
-  for (const [key, value] of Object.entries(parsed)) {
-    if (/^[A-Z0-9_]+$/.test(key)) result[key] = value;
-  }
-  return result;
-}
-
-// ── OpenCode Config ────────────────────────────────────────────────────
 
 export function ensureOpenCodeConfig(): void {
-  const configHome = resolveConfigHome();
-  const opencodePath = `${configHome}/assistant`;
+  const configDir = resolveConfigDir();
+  const opencodePath = `${configDir}/assistant`;
   mkdirSync(opencodePath, { recursive: true });
 
   const configFile = `${opencodePath}/opencode.json`;