@openpalm/lib 0.9.4

package/README.md

@@ -0,0 +1,83 @@
+# @openpalm/lib
+
+Shared control-plane library consumed by both the CLI and the admin SvelteKit app. All portable logic for managing an OpenPalm stack lives here -- paths, secrets, staging, lifecycle, Docker Compose operations, channel/connection management, scheduling, and setup orchestration.
+
+Admin is a thin UI layer that re-exports from this package. The CLI calls these functions directly.
+
+## Modules
+
+| Module | Purpose |
+|---|---|
+| `types` | Core type definitions -- `ControlPlaneState`, `CoreServiceName`, `ChannelInfo`, `CanonicalConnectionProfile`, etc. |
+| `paths` | XDG directory resolution (`resolveConfigHome`, `resolveDataHome`, `resolveStateHome`, `ensureXdgDirs`) |
+| `env` | `.env` file parsing and merging (`parseEnvContent`, `parseEnvFile`, `mergeEnvContent`) |
+| `secrets` | Secrets management -- ensure, read, patch, mask `secrets.env` |
+| `setup-status` | First-boot detection (`isSetupComplete`, `readSecretsKeys`, `detectUserId`) |
+| `setup` | Setup wizard backend (`performSetup`, `detectProviders`, `validateSetupInput`) |
+| `staging` | Artifact staging pipeline (`stageArtifacts`, `persistArtifacts`, `buildEnvFiles`) |
+| `lifecycle` | Install/update/uninstall/upgrade orchestration (`applyInstall`, `applyUpdate`, `createState`) |
+| `docker` | Docker Compose CLI wrapper (`composeUp`, `composeDown`, `composePs`, `composeLogs`, etc.) |
+| `channels` | Channel discovery, install, uninstall from registry or filesystem |
+| `connection-profiles` | CRUD for connection profiles (LLM providers, embedding, etc.) |
+| `connection-mapping` | Build OpenCode and mem0 config from connection profiles |
+| `memory-config` | Read/write memory service config, push to running memory container |
+| `scheduler` | Automation YAML parsing, Croner-based scheduler, execution log |
+| `model-runner` | Local provider detection (Ollama, Docker Model Runner, LM Studio) |
+| `core-assets` | Seed and read core infrastructure files (compose, Caddyfile, schemas) |
+| `connection-migration-flags` | Migration compatibility detection (`readConnectionMigrationFlags`, `detectConnectionCompatibilityMode`) |
+| `audit` | Append to the audit log |
+| `logger` | Structured logger factory (`createLogger`) |
+| `provider-constants` | LLM provider metadata (`LLM_PROVIDERS`, `PROVIDER_DEFAULT_URLS`, `EMBEDDING_DIMS`) |
+
+## Dependency Injection
+
+Asset loading differs between CLI and admin. Two interfaces abstract this:
+
+### CoreAssetProvider
+
+Returns the content of bundled infrastructure files (compose, Caddyfile, schemas, automations). Functions that need these files accept a `CoreAssetProvider` parameter.
+
+| Implementation | Consumer | Source |
+|---|---|---|
+| `FilesystemAssetProvider` | CLI | Reads from `DATA_HOME` on disk |
+| `ViteAssetProvider` | Admin | Reads from Vite `$assets` imports (defined in `packages/admin`, not in lib) |
+
+### RegistryProvider
+
+Returns channel and automation definitions from the registry catalog.
+
+| Implementation | Consumer | Source |
+|---|---|---|
+| `FilesystemRegistryProvider` | CLI | Reads from `registry/` directory |
+| `ViteRegistryProvider` | Admin | Reads via `import.meta.glob` (defined in `packages/admin`, not in lib) |
+
+## Usage
+
+```ts
+import {
+  createState,
+  stageArtifacts,
+  persistArtifacts,
+  applyInstall,
+  FilesystemAssetProvider,
+} from "@openpalm/lib";
+
+const assets = new FilesystemAssetProvider(dataHome);
+const state = createState(configHome, dataHome, stateHome);
+
+stageArtifacts(state, assets);
+persistArtifacts(state, assets);
+await applyInstall(state, assets, "cli");
+```
+
+Sub-path imports are also available:
+
+```ts
+import { resolveConfigHome } from "@openpalm/lib/control-plane/paths";
+import { createLogger } from "@openpalm/lib/shared/logger";
+import { LLM_PROVIDERS } from "@openpalm/lib/provider-constants";
+```
+
+## Architecture
+
+See [`docs/technical/core-principles.md`](../../docs/technical/core-principles.md) for the filesystem contract and security invariants that govern this library.

package/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@openpalm/lib",
+  "version": "0.9.4",
+  "license": "MPL-2.0",
+  "type": "module",
+  "description": "Shared control-plane library for OpenPalm — lifecycle, staging, secrets, channels, connections, scheduler",
+  "scripts": {
+    "test": "bun test"
+  },
+  "exports": {
+    ".": "./src/index.ts",
+    "./provider-constants": "./src/provider-constants.ts",
+    "./shared/logger": "./src/logger.ts",
+    "./control-plane/*": "./src/control-plane/*.ts"
+  },
+  "files": [
+    "src",
+    "assets"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/itlackey/openpalm",
+    "directory": "packages/lib"
+  },
+  "dependencies": {
+    "croner": "^9.0.0",
+    "dotenv": "^16.4.7",
+    "yaml": "^2.8.0"
+  }
+}

package/src/control-plane/audit.ts

@@ -0,0 +1,40 @@
+/**
+ * Audit logging for the OpenPalm control plane.
+ */
+import { mkdirSync, appendFileSync } from "node:fs";
+import type { ControlPlaneState, AuditEntry, CallerType } from "./types.js";
+import { MAX_AUDIT_MEMORY } from "./types.js";
+
+export function appendAudit(
+  state: ControlPlaneState,
+  actor: string,
+  action: string,
+  args: Record<string, unknown>,
+  ok: boolean,
+  requestId = "",
+  callerType: CallerType = "unknown"
+): void {
+  const entry: AuditEntry = {
+    at: new Date().toISOString(),
+    requestId,
+    actor,
+    callerType,
+    action,
+    args,
+    ok
+  };
+  state.audit.push(entry);
+  if (state.audit.length > MAX_AUDIT_MEMORY) {
+    state.audit = state.audit.slice(-MAX_AUDIT_MEMORY);
+  }
+  try {
+    const auditDir = `${state.stateDir}/audit`;
+    mkdirSync(auditDir, { recursive: true });
+    appendFileSync(
+      `${auditDir}/admin-audit.jsonl`,
+      JSON.stringify(entry) + "\n"
+    );
+  } catch {
+    // best-effort persistence
+  }
+}

package/src/control-plane/channels.ts

@@ -0,0 +1,196 @@
+/**
+ * Channel validation, discovery, install, and uninstall for the OpenPalm control plane.
+ *
+ * Install/uninstall functions accept a RegistryProvider for catalog access,
+ * decoupling from Vite-specific imports.
+ */
+import { mkdirSync, writeFileSync, existsSync, readdirSync, rmSync } from "node:fs";
+import type { ChannelInfo } from "./types.js";
+import { CORE_SERVICES } from "./types.js";
+import type { RegistryProvider } from "./registry-provider.js";
+
+// ── Channel Name Validation ───────────────────────────────────────────
+
+/** Strict channel name: lowercase alphanumeric + hyphens, 1–63 chars, must start with alnum */
+const CHANNEL_NAME_RE = /^[a-z0-9][a-z0-9-]{0,62}$/;
+
+function isValidChannelName(name: string): boolean {
+  return CHANNEL_NAME_RE.test(name);
+}
+
+// ── Channel Discovery ─────────────────────────────────────────────────
+
+/**
+ * Discover installed channels by scanning CONFIG_HOME/channels/.
+ *
+ * A channel is any .yml file in the channels directory.
+ * A .caddy file is optional — if present, the channel gets Caddy HTTP routing.
+ * If absent, the channel is only accessible on the Docker network (host + containers).
+ */
+export function discoverChannels(configDir: string): ChannelInfo[] {
+  const channelsDir = `${configDir}/channels`;
+  if (!existsSync(channelsDir)) return [];
+
+  const files = readdirSync(channelsDir);
+  const ymlFiles = files.filter((f) => f.endsWith(".yml"));
+  const caddyFiles = new Set(files.filter((f) => f.endsWith(".caddy")));
+
+  return ymlFiles
+    .map((ymlFile) => {
+      const name = ymlFile.replace(/\.yml$/, "");
+      const caddyFile = `${name}.caddy`;
+      const hasCaddy = caddyFiles.has(caddyFile);
+      return {
+        name,
+        hasRoute: hasCaddy,
+        ymlPath: `${channelsDir}/${ymlFile}`,
+        caddyPath: hasCaddy ? `${channelsDir}/${caddyFile}` : null
+      };
+    })
+    .filter((ch) => isValidChannelName(ch.name));
+}
+
+// ── Allowlist Checks ───────────────────────────────────────────────────
+
+/**
+ * Check if a service name is allowed. Core services are always allowed.
+ * Ollama is allowed when its compose overlay is staged.
+ * Channel services (channel-*) are allowed if a corresponding staged .yml exists
+ * in STATE_HOME/artifacts/channels/.
+ */
+export function isAllowedService(value: string, stateDir?: string): boolean {
+  if (!value || !value.trim() || value !== value.toLowerCase()) return false;
+  if ((CORE_SERVICES as string[]).includes(value)) return true;
+  if (value === "ollama" && stateDir) {
+    return existsSync(`${stateDir}/artifacts/ollama.yml`);
+  }
+  if (value.startsWith("channel-")) {
+    const ch = value.slice("channel-".length);
+    if (!isValidChannelName(ch)) return false;
+    if (stateDir) {
+      return existsSync(`${stateDir}/artifacts/channels/${ch}.yml`);
+    }
+  }
+  return false;
+}
+
+/**
+ * Check if a channel name is valid. Accepts any channel with a staged
+ * .yml file in STATE_HOME/artifacts/channels/.
+ */
+export function isValidChannel(value: string, stateDir?: string): boolean {
+  if (!value || !value.trim()) return false;
+  if (!isValidChannelName(value)) return false;
+  if (stateDir) {
+    return existsSync(`${stateDir}/artifacts/channels/${value}.yml`);
+  }
+  return false;
+}
+
+// ── Channel Install / Uninstall ─────────────────────────────────────────
+
+/**
+ * Install a channel from the registry catalog into CONFIG_HOME/channels/.
+ * Copies the .yml (and optional .caddy) from the registry provider.
+ * Refuses if the channel is already installed (files already exist).
+ */
+export function installChannelFromRegistry(
+  name: string,
+  configDir: string,
+  registry: RegistryProvider
+): { ok: true } | { ok: false; error: string } {
+  if (!isValidChannelName(name)) {
+    return { ok: false, error: `Invalid channel name: ${name}` };
+  }
+  const channelYml = registry.channelYml();
+  if (!(name in channelYml)) {
+    return { ok: false, error: `Channel "${name}" not found in registry` };
+  }
+  const channelsDir = `${configDir}/channels`;
+  mkdirSync(channelsDir, { recursive: true });
+
+  const ymlPath = `${channelsDir}/${name}.yml`;
+  if (existsSync(ymlPath)) {
+    return { ok: false, error: `Channel "${name}" is already installed` };
+  }
+
+  writeFileSync(ymlPath, channelYml[name]);
+  const channelCaddy = registry.channelCaddy();
+  if (name in channelCaddy) {
+    writeFileSync(`${channelsDir}/${name}.caddy`, channelCaddy[name]);
+  }
+  return { ok: true };
+}
+
+/**
+ * Uninstall a channel by removing its .yml (and .caddy) from CONFIG_HOME/channels/.
+ */
+export function uninstallChannel(
+  name: string,
+  configDir: string
+): { ok: true } | { ok: false; error: string } {
+  if (!isValidChannelName(name)) {
+    return { ok: false, error: `Invalid channel name: ${name}` };
+  }
+  const channelsDir = `${configDir}/channels`;
+  const ymlPath = `${channelsDir}/${name}.yml`;
+  if (!existsSync(ymlPath)) {
+    return { ok: false, error: `Channel "${name}" is not installed` };
+  }
+
+  rmSync(ymlPath, { force: true });
+  rmSync(`${channelsDir}/${name}.caddy`, { force: true });
+  return { ok: true };
+}
+
+// ── Automation Install / Uninstall ──────────────────────────────────────
+
+/** Strict automation name: same rules as channel names */
+const AUTOMATION_NAME_RE = /^[a-z0-9][a-z0-9-]{0,62}$/;
+
+/**
+ * Install an automation from the registry catalog into CONFIG_HOME/automations/.
+ */
+export function installAutomationFromRegistry(
+  name: string,
+  configDir: string,
+  registry: RegistryProvider
+): { ok: true } | { ok: false; error: string } {
+  if (!AUTOMATION_NAME_RE.test(name)) {
+    return { ok: false, error: `Invalid automation name: ${name}` };
+  }
+  const automationYml = registry.automationYml();
+  if (!(name in automationYml)) {
+    return { ok: false, error: `Automation "${name}" not found in registry` };
+  }
+  const automationsDir = `${configDir}/automations`;
+  mkdirSync(automationsDir, { recursive: true });
+
+  const ymlPath = `${automationsDir}/${name}.yml`;
+  if (existsSync(ymlPath)) {
+    return { ok: false, error: `Automation "${name}" is already installed` };
+  }
+
+  writeFileSync(ymlPath, automationYml[name]);
+  return { ok: true };
+}
+
+/**
+ * Uninstall an automation by removing its .yml from CONFIG_HOME/automations/.
+ */
+export function uninstallAutomation(
+  name: string,
+  configDir: string
+): { ok: true } | { ok: false; error: string } {
+  if (!AUTOMATION_NAME_RE.test(name)) {
+    return { ok: false, error: `Invalid automation name: ${name}` };
+  }
+  const automationsDir = `${configDir}/automations`;
+  const ymlPath = `${automationsDir}/${name}.yml`;
+  if (!existsSync(ymlPath)) {
+    return { ok: false, error: `Automation "${name}" is not installed` };
+  }
+
+  rmSync(ymlPath, { force: true });
+  return { ok: true };
+}

package/src/control-plane/connection-mapping.ts

@@ -0,0 +1,191 @@
+import { mkdirSync, writeFileSync, readFileSync } from 'node:fs';
+import { mem0BaseUrlConfig, mem0ProviderName } from '../provider-constants.js';
+import type { MemoryConfig } from './memory-config.js';
+import type { CanonicalConnectionProfile } from './types.js';
+
+export type OpenCodeConnectionMappingInput = {
+  provider: string;
+  baseUrl: string;
+  systemModel: string;
+  smallModel?: string;
+};
+
+export type OpenCodeConnectionMapping = {
+  provider: string;
+  model: string;
+  smallModel: string;
+  options?: {
+    baseURL?: string;
+  };
+};
+
+export type Mem0ConnectionMappingInput = {
+  llm: {
+    provider: string;
+    baseUrl: string;
+    model: string;
+    apiKeyRef: string;
+  };
+  embedder: {
+    provider: string;
+    baseUrl: string;
+    model: string;
+    apiKeyRef: string;
+  };
+  embeddingDims: number;
+  customInstructions: string;
+};
+
+export type Mem0ConnectionMapping = MemoryConfig;
+
+export function buildOpenCodeMapping(input: OpenCodeConnectionMappingInput): OpenCodeConnectionMapping {
+  const normalizedBaseUrl = input.baseUrl.trim();
+  return {
+    provider: input.provider,
+    model: input.systemModel,
+    smallModel: input.smallModel ?? input.systemModel,
+    ...(normalizedBaseUrl ? { options: { baseURL: normalizedBaseUrl } } : {}),
+  };
+}
+
+export function writeOpenCodeProviderConfig(
+  configDir: string,
+  mapping: OpenCodeConnectionMapping,
+): void {
+  const assistantDir = `${configDir}/assistant`;
+  mkdirSync(assistantDir, { recursive: true });
+
+  const configPath = `${assistantDir}/opencode.json`;
+
+  let existing: Record<string, unknown> = { $schema: 'https://opencode.ai/config.json' };
+  let raw: string | undefined;
+  try {
+    raw = readFileSync(configPath, 'utf-8');
+  } catch (err) {
+    const nodeErr = err as NodeJS.ErrnoException;
+    if (nodeErr.code !== 'ENOENT') {
+      return;
+    }
+  }
+
+  if (raw !== undefined) {
+    try {
+      existing = JSON.parse(raw) as Record<string, unknown>;
+    } catch {
+      return;
+    }
+  }
+
+  const existingProviders =
+    (existing as { providers?: Record<string, unknown> }).providers ?? {};
+  const existingProviderConfig =
+    (existingProviders as Record<string, unknown>)[mapping.provider] ?? {};
+  const existingOptions =
+    (existingProviderConfig as { options?: Record<string, unknown> }).options ?? {};
+  const updatedOptions: Record<string, unknown> = { ...existingOptions };
+  const mappingBaseUrl = mapping.options?.baseURL?.trim();
+
+  if (mappingBaseUrl) {
+    updatedOptions.baseURL = mappingBaseUrl;
+  } else {
+    delete (updatedOptions as { baseURL?: unknown }).baseURL;
+  }
+
+  const updatedProviderConfig: Record<string, unknown> = {
+    ...existingProviderConfig as Record<string, unknown>,
+    ...(Object.keys(updatedOptions).length > 0 ? { options: updatedOptions } : {}),
+  };
+
+  const updated = {
+    ...existing,
+    model: mapping.model,
+    smallModel: mapping.smallModel,
+    providers: {
+      ...existingProviders,
+      [mapping.provider]: updatedProviderConfig,
+    },
+  };
+
+  writeFileSync(configPath, JSON.stringify(updated, null, 2) + '\n');
+}
+
+export function buildMem0Mapping(input: Mem0ConnectionMappingInput): Mem0ConnectionMapping {
+  const llmConfig: Record<string, unknown> = {
+    model: input.llm.model,
+    temperature: 0.1,
+    max_tokens: 2000,
+    api_key: input.llm.apiKeyRef,
+  };
+
+  const llmBaseUrlConfig = mem0BaseUrlConfig(input.llm.provider, input.llm.baseUrl);
+  if (llmBaseUrlConfig) {
+    llmConfig[llmBaseUrlConfig.key] = llmBaseUrlConfig.value;
+  }
+
+  const embedConfig: Record<string, unknown> = {
+    model: input.embedder.model,
+    api_key: input.embedder.apiKeyRef,
+  };
+
+  const embedBaseUrlConfig = mem0BaseUrlConfig(input.embedder.provider, input.embedder.baseUrl);
+  if (embedBaseUrlConfig) {
+    embedConfig[embedBaseUrlConfig.key] = embedBaseUrlConfig.value;
+  }
+
+  return {
+    mem0: {
+      llm: {
+        provider: mem0ProviderName(input.llm.provider),
+        config: llmConfig,
+      },
+      embedder: {
+        provider: mem0ProviderName(input.embedder.provider),
+        config: embedConfig,
+      },
+      vector_store: {
+        provider: 'qdrant',
+        config: {
+          collection_name: 'memory',
+          path: '/data/qdrant',
+          embedding_model_dims: input.embeddingDims,
+        },
+      },
+    },
+    memory: {
+      custom_instructions: input.customInstructions,
+    },
+  };
+}
+
+export function resolveApiKeyRef(profile: CanonicalConnectionProfile): string {
+  if (profile.auth.mode === 'api_key' && profile.auth.apiKeySecretRef) {
+    return profile.auth.apiKeySecretRef;
+  }
+  return 'not-needed';
+}
+
+export function buildMem0MappingFromProfiles(
+  llmProfile: CanonicalConnectionProfile,
+  embedProfile: CanonicalConnectionProfile,
+  llmModel: string,
+  embedModel: string,
+  embeddingDims: number,
+  customInstructions: string,
+): Mem0ConnectionMapping {
+  return buildMem0Mapping({
+    llm: {
+      provider: llmProfile.provider,
+      baseUrl: llmProfile.baseUrl,
+      model: llmModel,
+      apiKeyRef: resolveApiKeyRef(llmProfile),
+    },
+    embedder: {
+      provider: embedProfile.provider,
+      baseUrl: embedProfile.baseUrl,
+      model: embedModel,
+      apiKeyRef: resolveApiKeyRef(embedProfile),
+    },
+    embeddingDims,
+    customInstructions,
+  });
+}

package/src/control-plane/connection-migration-flags.ts

@@ -0,0 +1,40 @@
+type ConnectionCompatibilityMode =
+  | 'legacy_patch'
+  | 'legacy_unified'
+  | 'canonical_dto';
+
+export type ConnectionMigrationFlags = {
+  enabled: boolean;
+  dualRead: boolean;
+  dualWrite: boolean;
+  preferLegacyRead: boolean;
+  annotateAudit: boolean;
+};
+
+function envFlag(name: string, fallback: boolean): boolean {
+  const raw = process.env[name];
+  if (!raw) return fallback;
+  return ['1', 'true', 'yes', 'on'].includes(raw.toLowerCase());
+}
+
+export function readConnectionMigrationFlags(): ConnectionMigrationFlags {
+  return {
+    enabled: envFlag('OPENPALM_CONNECTION_MIGRATION_ENABLED', true),
+    dualRead: envFlag('OPENPALM_CONNECTION_MIGRATION_DUAL_READ', true),
+    dualWrite: envFlag('OPENPALM_CONNECTION_MIGRATION_DUAL_WRITE', true),
+    preferLegacyRead: envFlag('OPENPALM_CONNECTION_MIGRATION_PREFER_LEGACY_READ', false),
+    annotateAudit: envFlag('OPENPALM_CONNECTION_MIGRATION_AUDIT_ANNOTATION', true),
+  };
+}
+
+export function detectConnectionCompatibilityMode(body: Record<string, unknown>): ConnectionCompatibilityMode {
+  if (Array.isArray(body.profiles) && typeof body.assignments === 'object' && body.assignments !== null) {
+    return 'canonical_dto';
+  }
+  if (typeof body.provider === 'string') {
+    return 'legacy_unified';
+  }
+  return 'legacy_patch';
+}
+
+export type { ConnectionCompatibilityMode };