@openpalm/lib 0.10.1 → 0.11.0-beta.1

package/src/control-plane/validate.ts

@@ -1,61 +1,34 @@
 /**
  * Runtime configuration validation for the OpenPalm control plane.
  *
- * Proposed changes are validated against temp copies before writing
- * to live paths.
+ * Validation is a presence check on the canonical env keys we expect in
+ * the live config/stack/stack.env file. The
+ * historical schema files and external validation binary were retired in
+ * #391; everything advisory is surfaced as a non-blocking warning. The
+ * function never shells out and never reads schemas.
  */
-import { existsSync, copyFileSync, mkdirSync, rmSync } from "node:fs";
-import { join } from "node:path";
-import { mkdtempSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { execFile } from "node:child_process";
-import { promisify } from "node:util";
+import { existsSync } from "node:fs";
+import { readStackEnv } from "./secrets.js";
+import { getCoreSecretMappings } from "./secret-mappings.js";
 import type { ControlPlaneState } from "./types.js";
 
-const execFileAsync = promisify(execFile);
-
-/** Resolve the varlock binary path — honours VARLOCK_BIN for dev environments. */
-const envVarlockBin = process.env.VARLOCK_BIN;
-let VARLOCK_BIN = "varlock";
-if (envVarlockBin) {
-  if (envVarlockBin === "varlock" || envVarlockBin.startsWith("/")) {
-    VARLOCK_BIN = envVarlockBin;
-  }
-}
-
-function sanitizeVarlockMessage(msg: string): string {
-  return msg
-    .replace(/sk-[A-Za-z0-9]{20,}/g, "[REDACTED]")
-    .replace(/gsk_[A-Za-z0-9]{30,}/g, "[REDACTED]")
-    .replace(/AIza[A-Za-z0-9_\-]{35}/g, "[REDACTED]")
-    .replace(/[0-9a-f]{32,}/gi, "[REDACTED]")
-    .replace(/value '([^']*)'/g, "value '[REDACTED]'");
-}
-
-async function runVarlockLoad(
-  schemaFile: string,
-  envFile: string,
-): Promise<void> {
-  const tmpDir = mkdtempSync(join(tmpdir(), "varlock-"));
-  try {
-    copyFileSync(schemaFile, join(tmpDir, ".env.schema"));
-    copyFileSync(envFile, join(tmpDir, ".env"));
-    await execFileAsync(
-      VARLOCK_BIN,
-      ["load", "--path", `${tmpDir}/`],
-      { timeout: 10000 },
-    );
-  } finally {
-    rmSync(tmpDir, { recursive: true, force: true });
-  }
-}
+// Stack-scoped env keys that must always exist and carry a non-empty value
+// for the platform to boot. Keep this list small — anything optional
+// belongs in the warning bucket instead.
+const REQUIRED_STACK_KEYS = ["OP_UI_TOKEN", "OP_ASSISTANT_TOKEN"] as const;
 
 /**
- * Validate the current live configuration files in place.
+ * Validate the live configuration files.
  *
  * Checks:
- * 1. vault/user/user.env against vault/user/user.env.schema
- * 2. vault/stack/stack.env against vault/stack/stack.env.schema
+ * 1. config/stack/stack.env exists and carries every required key with a
+ *    non-empty value.
+ * 2. Every secret env key in getCoreSecretMappings() is present (key only
+ *    — blank values are warned about, never erred on, because operators
+ *    may opt out of providers they don't use).
+ *
+ * Errors fail the result. Warnings do not. The function never reads
+ * schema files and never spawns subprocesses.
  */
 export async function validateProposedState(state: ControlPlaneState): Promise<{
   ok: boolean;
@@ -64,44 +37,36 @@ export async function validateProposedState(state: ControlPlaneState): Promise<{
 }> {
   const errors: string[] = [];
   const warnings: string[] = [];
-  let anyFailed = false;
 
-  function collectOutput(stderr: string): void {
-    for (const line of stderr.split("\n")) {
-      const trimmed = sanitizeVarlockMessage(line.trim());
-      if (!trimmed) continue;
-      if (trimmed.includes("ERROR")) errors.push(trimmed);
-      else if (trimmed.includes("WARN")) warnings.push(trimmed);
-    }
+  const stackEnvPath = `${state.stackDir}/stack.env`;
+
+  if (!existsSync(stackEnvPath)) {
+    errors.push(`ERROR: stack env file missing at ${stackEnvPath}`);
+    return { ok: false, errors, warnings };
   }
 
-  // Validate user.env
-  const userEnvSchema = `${state.vaultDir}/user/user.env.schema`;
-  const userEnv = `${state.vaultDir}/user/user.env`;
-  if (existsSync(userEnvSchema) && existsSync(userEnv)) {
-    try {
-      await runVarlockLoad(userEnvSchema, userEnv);
-    } catch (err: unknown) {
-      anyFailed = true;
-      if (err && typeof err === "object" && "stderr" in err) {
-        collectOutput(String((err as { stderr: string }).stderr));
-      }
+  const stackEnv = readStackEnv(state.stackDir);
+  const userEnv: Record<string, string> = {};
+
+  for (const key of REQUIRED_STACK_KEYS) {
+    const value = stackEnv[key];
+    if (!value || value.trim().length === 0) {
+      errors.push(`ERROR: required key ${key} is missing or empty in config/stack/stack.env`);
     }
   }
 
-  // Validate stack.env
-  const systemEnvSchema = `${state.vaultDir}/stack/stack.env.schema`;
-  const systemEnv = `${state.vaultDir}/stack/stack.env`;
-  if (existsSync(systemEnvSchema) && existsSync(systemEnv)) {
-    try {
-      await runVarlockLoad(systemEnvSchema, systemEnv);
-    } catch (err: unknown) {
-      anyFailed = true;
-      if (err && typeof err === "object" && "stderr" in err) {
-        collectOutput(String((err as { stderr: string }).stderr));
-      }
+  // Every canonical secret should at least appear as a key somewhere in
+  // the env files so the operator sees the slot. Missing slots warn (not
+  // error) since not every provider is in use on every install.
+  for (const mapping of getCoreSecretMappings(stackEnv)) {
+    const inStack = Object.prototype.hasOwnProperty.call(stackEnv, mapping.envKey);
+    const inUser = Object.prototype.hasOwnProperty.call(userEnv, mapping.envKey);
+    if (!inStack && !inUser) {
+      warnings.push(
+        `WARN: ${mapping.envKey} (akm ${mapping.secretKey}) is not declared in config/stack/stack.env`,
+      );
     }
   }
 
-  return { ok: !anyFailed && errors.length === 0, errors, warnings };
+  return { ok: errors.length === 0, errors, warnings };
 }

package/src/index.ts

@@ -10,17 +10,22 @@
 export {
   LLM_PROVIDERS,
   EMBEDDING_DIMS,
+  lookupEmbeddingDims,
 } from "./provider-constants.js";
 
 // ── Logger ──────────────────────────────────────────────────────────────
-export { createLogger } from "./logger.js";
+export {
+  createLogger,
+  isSensitiveEnvKey,
+  redactValue,
+  redactExtra,
+} from "./logger.js";
 
 // ── Types ───────────────────────────────────────────────────────────────
 export type {
   ControlPlaneState,
   CoreServiceName,
   OptionalServiceName,
-  AccessScope,
   ChannelInfo,
   CallerType,
   ArtifactMeta,
@@ -62,25 +67,33 @@ export {
   uninstallAutomation,
 } from "./control-plane/registry.js";
 
-// ── Home Layout (v0.10.0) ───────────────────────────────────────────────
+// ── Home Layout (v0.11.0) ───────────────────────────────────────────────
 export {
   resolveOpenPalmHome,
   resolveConfigDir,
-  resolveVaultDir,
-  resolveDataDir,
+  resolveStashDir,
+  resolveWorkspaceDir,
+  resolveCacheDir,
+  resolveStateDir,
+  resolveStackDir,
   resolveLogsDir,
-  resolveCacheHome,
-  resolveRegistryDir,
-  resolveRegistryAddonsDir,
-  resolveRegistryAutomationsDir,
   ensureHomeDirs,
 } from "./control-plane/home.js";
 
+// ── Path Resolution ─────────────────────────────────────────────────────
+export * from "./control-plane/paths.js";
+
 // ── Env ─────────────────────────────────────────────────────────────────
 export {
   parseEnvContent,
   parseEnvFile,
+  expandEnvVars,
   mergeEnvContent,
+  removeEnvKey,
+  upsertEnvValue,
+  resolveRequestedImageTag,
+  reconcileStackEnvImageTag,
+  RELEASE_TAG_REGEX,
 } from "./control-plane/env.js";
 
 // ── Audit ───────────────────────────────────────────────────────────────
@@ -95,6 +108,7 @@ export {
   PLAIN_CONFIG_KEYS,
   ensureSecrets,
   updateSecretsEnv,
+  writeAuthJsonProviderKeys,
   readStackEnv,
   patchSecretsEnvFile,
   maskSecretValue,
@@ -104,6 +118,7 @@ export {
   detectSecretBackend,
   validatePassEntryName,
 } from "./control-plane/secret-backend.js";
+export type { SecretBackend } from "./control-plane/secret-backend.js";
 // ── Setup Status ────────────────────────────────────────────────────────
 export {
   isSetupComplete,
@@ -114,35 +129,24 @@ export {
   discoverChannels,
   isAllowedService,
   isValidChannel,
-  isChannelAddon,
 } from "./control-plane/channels.js";
 
-// ── Memory Config ───────────────────────────────────────────────────────
+// ── Provider Model Discovery ────────────────────────────────────────────
 export type {
-  MemoryConfig,
-} from "./control-plane/memory-config.js";
+  ProviderModelsResult,
+  ModelDiscoveryReason,
+} from "./control-plane/provider-models.js";
 export {
-  EMBED_PROVIDERS,
-  resolveApiKey,
   fetchProviderModels,
-  getDefaultConfig,
-  readMemoryConfig,
-  writeMemoryConfig,
-  ensureMemoryConfig,
-  checkVectorDimensions,
-  resetVectorStore,
-  provisionMemoryUser,
-} from "./control-plane/memory-config.js";
+} from "./control-plane/provider-models.js";
 
 // ── Core Assets ─────────────────────────────────────────────────────────
 export {
-  ensureUserEnvSchema,
-  ensureSystemEnvSchema,
-  ensureMemoryDir,
   ensureCoreCompose,
   readCoreCompose,
   ensureOpenCodeSystemConfig,
   refreshCoreAssets,
+  seedStashAssets,
 } from "./control-plane/core-assets.js";
 
 // ── Configuration Persistence ────────────────────────────────────────────
@@ -157,6 +161,7 @@ export {
   writeSystemEnv,
   readChannelSecrets,
   writeChannelSecrets,
+  ensureComposeVolumeTargets,
 } from "./control-plane/config-persistence.js";
 
 // ── Rollback ─────────────────────────────────────────────────────────────
@@ -184,7 +189,6 @@ export {
   buildManagedServices,
   normalizeCaller,
 } from "./control-plane/lifecycle.js";
-export type { UpgradeResult } from "./control-plane/lifecycle.js";
 
 // ── Docker ──────────────────────────────────────────────────────────────
 export type { DockerResult } from "./control-plane/docker.js";
@@ -204,22 +208,20 @@ export {
   composePull,
   composeStats,
   getDockerEvents,
-  selfRecreateAdmin,
+  inspectContainerStatus,
 } from "./control-plane/docker.js";
 
 // ── Scheduler ───────────────────────────────────────────────────────────
 export type {
-  ActionType,
-  AutomationAction,
   AutomationConfig,
-  ExecutionLogEntry,
+  AutomationRunResult,
 } from "./control-plane/scheduler.js";
 export {
   SCHEDULE_PRESETS,
-  resolveSchedule,
-  parseAutomationYaml,
   loadAutomations,
-  executeAction,
+  executeAutomation,
+  syncAutomations,
+  readAutomationLogs,
 } from "./control-plane/scheduler.js";
 
 // ── Model Runner (local provider detection) ─────────────────────────────
@@ -235,25 +237,17 @@ export {
 // ── Stack Spec (v2) ──────────────────────────────────────────────────────
 export type {
   StackSpec,
-  StackSpecCapabilities,
-  StackSpecEmbeddings,
-  StackSpecMemory,
-  StackSpecTts,
-  StackSpecStt,
-  StackSpecReranker,
 } from "./control-plane/stack-spec.js";
 export {
   STACK_SPEC_FILENAME,
   writeStackSpec,
   readStackSpec,
-  updateCapability,
-  parseCapabilityString,
-  formatCapabilityString,
 } from "./control-plane/stack-spec.js";
 
 // ── Spec-to-Env Derivation ──────────────────────────────────────────────
+export type { VoiceVarsConfig } from "./control-plane/spec-to-env.js";
 export {
-  writeCapabilityVars,
+  writeVoiceVars,
 } from "./control-plane/spec-to-env.js";
 
 // ── Setup ────────────────────────────────────────────────────────────────
@@ -265,3 +259,42 @@ export type {
 export {
   performSetup,
 } from "./control-plane/setup.js";
+
+// ── Install Lock (shared between performSetup and startDeploy) ───────────
+export type { InstallLockHandle } from "./control-plane/install-lock.js";
+export {
+  acquireInstallLock,
+  releaseInstallLock,
+} from "./control-plane/install-lock.js";
+
+// ── Host OpenCode Import ─────────────────────────────────────────────────
+export type {
+  HostOpenCodeStatus,
+  HostImportResult,
+} from "./control-plane/host-opencode.js";
+export {
+  detectHostOpenCode,
+  importHostOpenCode,
+} from "./control-plane/host-opencode.js";
+
+// ── AKM Vault ────────────────────────────────────────────────────────────
+export {
+  AKM_USER_VAULT_REF,
+  buildAkmEnv,
+  ensureAkmUserVault,
+  writeAkmVaultKey,
+  deleteAkmVaultKey,
+  readAkmUserVaultFile,
+  readUserVaultSync,
+} from "./control-plane/akm-vault.js";
+
+// ── UI asset seeding and resolution ─────────────────────────────────────────
+export type { UiBuildUpdateResult } from "./control-plane/ui-assets.js";
+export {
+  resolveLocalOpenpalmDir,
+  seedOpenPalmDir,
+  resolveLocalUiBuild,
+  resolveUiBuildDir,
+  seedUiBuild,
+  checkAndUpdateUiBuild,
+} from "./control-plane/ui-assets.js";

package/src/logger.test.ts

@@ -0,0 +1,228 @@
+/**
+ * Tests for the in-house log redactor introduced in #391 (replacing varlock).
+ *
+ * The contract:
+ *   - keys matching the word-bounded pattern
+ *     `(^|_)(TOKEN|SECRET|KEY|PASSWORD|HMAC)(_|$)` (case-insensitive)
+ *     have their value replaced with `'***REDACTED***'`.
+ *   - substring false positives (e.g. `MONKEY`, `PACKET_SIZE`) are NOT redacted.
+ *   - non-string sensitive values (numbers, booleans) are still redacted.
+ *   - non-secret keys are passed through unchanged.
+ *   - nested objects and arrays are walked recursively.
+ *   - createLogger() applies the same masking before writing to stdout/stderr
+ *     at every log level (debug/info/warn/error).
+ */
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import {
+  createLogger,
+  isSensitiveEnvKey,
+  redactValue,
+  redactExtra,
+} from './logger.js';
+
+describe('redactValue', () => {
+  test('masks values for sensitive key suffixes', () => {
+    expect(redactValue('OPENAI_API_KEY', 'sk-abc')).toBe('***REDACTED***');
+    expect(redactValue('SLACK_BOT_TOKEN', 'xoxb-123')).toBe('***REDACTED***');
+    expect(redactValue('CHANNEL_DISCORD_SECRET', 'hmac-bytes')).toBe('***REDACTED***');
+    expect(redactValue('OP_OPENCODE_PASSWORD', 'hunter2')).toBe('***REDACTED***');
+  });
+
+  test('matches case-insensitively', () => {
+    expect(redactValue('openai_api_key', 'sk-xyz')).toBe('***REDACTED***');
+    expect(redactValue('My_Token', 'abc')).toBe('***REDACTED***');
+  });
+
+  test('leaves non-secret values alone', () => {
+    expect(redactValue('OWNER_NAME', 'alice')).toBe('alice');
+    expect(redactValue('OP_HOME', '/openpalm')).toBe('/openpalm');
+    expect(redactValue('OP_ASSISTANT_PORT', '3800')).toBe('3800');
+  });
+});
+
+describe('isSensitiveEnvKey', () => {
+  test('returns true for token/secret/key/password/hmac suffix keys', () => {
+    expect(isSensitiveEnvKey('FOO_TOKEN')).toBe(true);
+    expect(isSensitiveEnvKey('FOO_SECRET')).toBe(true);
+    expect(isSensitiveEnvKey('FOO_KEY')).toBe(true);
+    expect(isSensitiveEnvKey('FOO_PASSWORD')).toBe(true);
+    expect(isSensitiveEnvKey('CHANNEL_FOO_HMAC')).toBe(true);
+    expect(isSensitiveEnvKey('OP_UI_TOKEN')).toBe(true);
+    expect(isSensitiveEnvKey('CHANNEL_API_KEY')).toBe(true);
+  });
+
+  test('returns true for bare or prefix forms', () => {
+    expect(isSensitiveEnvKey('TOKEN')).toBe(true);
+    expect(isSensitiveEnvKey('SECRET')).toBe(true);
+    expect(isSensitiveEnvKey('KEY')).toBe(true);
+    expect(isSensitiveEnvKey('HMAC_KEY')).toBe(true);
+    expect(isSensitiveEnvKey('PASSWORD_HASH')).toBe(true);
+  });
+
+  test('returns false for substring false positives', () => {
+    // MONKEY contains the substring KEY but no underscore boundary.
+    expect(isSensitiveEnvKey('MONKEY')).toBe(false);
+    // PACKET_SIZE: KET is not one of the words; ET_SIZE is unrelated.
+    expect(isSensitiveEnvKey('PACKET_SIZE')).toBe(false);
+    // MARKETING_KEYWORD: KEYWORD does not have a trailing underscore or EOL.
+    expect(isSensitiveEnvKey('MARKETING_KEYWORD')).toBe(false);
+    // KEYBOARD: starts with KEY but not followed by underscore or EOL.
+    expect(isSensitiveEnvKey('KEYBOARD')).toBe(false);
+  });
+
+  test('returns false for ordinary keys', () => {
+    expect(isSensitiveEnvKey('OWNER_NAME')).toBe(false);
+    expect(isSensitiveEnvKey('OP_HOME')).toBe(false);
+    expect(isSensitiveEnvKey('OP_ASSISTANT_PORT')).toBe(false);
+  });
+});
+
+describe('redactExtra', () => {
+  test('masks top-level secret string values', () => {
+    const result = redactExtra({
+      OPENAI_API_KEY: 'sk-abc',
+      OWNER_NAME: 'alice',
+    });
+    expect(result).toEqual({
+      OPENAI_API_KEY: '***REDACTED***',
+      OWNER_NAME: 'alice',
+    });
+  });
+
+  test('walks nested objects', () => {
+    const result = redactExtra({
+      env: {
+        OPENAI_API_KEY: 'sk-abc',
+        OWNER_NAME: 'alice',
+      },
+    });
+    expect(result).toEqual({
+      env: {
+        OPENAI_API_KEY: '***REDACTED***',
+        OWNER_NAME: 'alice',
+      },
+    });
+  });
+
+  test('handles arrays of objects', () => {
+    const result = redactExtra({
+      items: [
+        { OPENAI_API_KEY: 'sk-1' },
+        { OWNER_NAME: 'bob' },
+      ],
+    });
+    expect(result).toEqual({
+      items: [
+        { OPENAI_API_KEY: '***REDACTED***' },
+        { OWNER_NAME: 'bob' },
+      ],
+    });
+  });
+
+  test('returns primitive inputs unchanged', () => {
+    expect(redactExtra('plain')).toBe('plain');
+    expect(redactExtra(42)).toBe(42);
+    expect(redactExtra(null)).toBe(null);
+  });
+
+  test('redacts non-string sensitive values (numbers, booleans)', () => {
+    const result = redactExtra({
+      OP_UI_TOKEN: 12345,
+      OPENAI_API_KEY: true,
+      OWNER_NAME: 'alice',
+    });
+    expect(result).toEqual({
+      OP_UI_TOKEN: '***REDACTED***',
+      OPENAI_API_KEY: '***REDACTED***',
+      OWNER_NAME: 'alice',
+    });
+  });
+
+  test('does not redact substring false positives', () => {
+    const result = redactExtra({
+      MONKEY: 'banana',
+      PACKET_SIZE: 1500,
+      MARKETING_KEYWORD: 'free',
+      OPENAI_API_KEY: 'sk-leak',
+    });
+    expect(result).toEqual({
+      MONKEY: 'banana',
+      PACKET_SIZE: 1500,
+      MARKETING_KEYWORD: 'free',
+      OPENAI_API_KEY: '***REDACTED***',
+    });
+  });
+
+  test('redacts CHANNEL_FOO_HMAC values', () => {
+    const result = redactExtra({ CHANNEL_DISCORD_HMAC: 'hmac-bytes' });
+    expect(result).toEqual({ CHANNEL_DISCORD_HMAC: '***REDACTED***' });
+  });
+});
+
+describe('createLogger', () => {
+  const origLog = console.log;
+  const origErr = console.error;
+  let logged: string[] = [];
+
+  beforeEach(() => {
+    logged = [];
+    console.log = (...args: unknown[]) => {
+      logged.push(args.map((a) => String(a)).join(' '));
+    };
+    console.error = (...args: unknown[]) => {
+      logged.push(args.map((a) => String(a)).join(' '));
+    };
+  });
+
+  afterEach(() => {
+    console.log = origLog;
+    console.error = origErr;
+  });
+
+  test('redacts sensitive keys in the extra payload before writing the log line', () => {
+    const logger = createLogger('test');
+    logger.info('msg', { OPENAI_API_KEY: 'sk-leak', OWNER_NAME: 'alice' });
+    expect(logged.length).toBe(1);
+    expect(logged[0]).toContain('"OPENAI_API_KEY":"***REDACTED***"');
+    expect(logged[0]).toContain('"OWNER_NAME":"alice"');
+    expect(logged[0]).not.toContain('sk-leak');
+  });
+
+  test('error level still goes through redaction', () => {
+    const logger = createLogger('test');
+    logger.error('boom', { OP_UI_TOKEN: 'tok-leak' });
+    expect(logged[0]).toContain('"OP_UI_TOKEN":"***REDACTED***"');
+    expect(logged[0]).not.toContain('tok-leak');
+  });
+
+  test('warn level applies redaction', () => {
+    const logger = createLogger('test');
+    logger.warn('caution', { CHANNEL_API_KEY: 'warn-leak' });
+    expect(logged.length).toBe(1);
+    expect(logged[0]).toContain('"CHANNEL_API_KEY":"***REDACTED***"');
+    expect(logged[0]).not.toContain('warn-leak');
+  });
+
+  test('debug level applies redaction', () => {
+    const logger = createLogger('test');
+    logger.debug('detail', { CHANNEL_FOO_HMAC: 'debug-leak' });
+    expect(logged.length).toBe(1);
+    expect(logged[0]).toContain('"CHANNEL_FOO_HMAC":"***REDACTED***"');
+    expect(logged[0]).not.toContain('debug-leak');
+  });
+
+  test('substring false positives are not redacted at log time', () => {
+    const logger = createLogger('test');
+    logger.info('msg', { MONKEY: 'banana', PACKET_SIZE: 1500 });
+    expect(logged[0]).toContain('"MONKEY":"banana"');
+    expect(logged[0]).toContain('"PACKET_SIZE":1500');
+    expect(logged[0]).not.toContain('***REDACTED***');
+  });
+
+  test('non-string sensitive values are redacted at log time', () => {
+    const logger = createLogger('test');
+    logger.info('msg', { OP_UI_TOKEN: 12345 });
+    expect(logged[0]).toContain('"OP_UI_TOKEN":"***REDACTED***"');
+    expect(logged[0]).not.toContain('12345');
+  });
+});