@openpalm/lib 0.10.2 → 0.11.0-beta.10

package/src/logger.ts

@@ -1,8 +1,78 @@
 export type LogLevel = 'debug' | 'info' | 'warn' | 'error';
 
+/**
+ * In-house redactor. Returns `'***REDACTED***'` when `key` names something
+ * that looks like a secret (token, key, secret, password, hmac). Replaces
+ * the value-masking that varlock used to do for log output.
+ *
+ * The pattern matches the bare word at the start or end of the key, using
+ * underscore as a word boundary. This avoids substring false positives
+ * like `MONKEY` (contains `_KEY`? no, but the un-anchored pattern used
+ * to match the substring `KEY` even without an underscore) and
+ * `PACKET_SIZE` (does not actually contain `_KEY`, but the regex engine
+ * with un-anchored alternations was sloppy enough to invite future bugs).
+ *
+ * Examples:
+ *   OP_UI_LOGIN_PASSWORD → sensitive (suffix _PASSWORD)
+ *   CHANNEL_API_KEY    → sensitive (suffix _KEY)
+ *   CHANNEL_FOO_HMAC   → sensitive (suffix _HMAC)
+ *   HMAC_KEY           → sensitive (prefix HMAC_, suffix _KEY)
+ *   TOKEN              → sensitive (bare word)
+ *   MONKEY             → NOT sensitive
+ *   PACKET_SIZE        → NOT sensitive
+ *
+ * The same predicate is exported as {@link isSensitiveEnvKey} so callers
+ * that need to mask only part of a larger payload can short-circuit.
+ */
+const REDACT_PATTERN = /(?:^|_)(?:TOKEN|SECRET|KEY|PASSWORD|HMAC)(?:_|$)/i;
+
+export function isSensitiveEnvKey(key: string): boolean {
+  return REDACT_PATTERN.test(key);
+}
+
+export function redactValue(key: string, value: string): string {
+  return isSensitiveEnvKey(key) ? '***REDACTED***' : value;
+}
+
+/**
+ * Recursively walk a structured `extra` payload and mask every value whose
+ * own key (or the nearest enclosing object key) matches the sensitivity
+ * pattern. The original object is not mutated. Sensitive values of any
+ * primitive type (string, number, boolean) are replaced wholesale; nested
+ * objects under a sensitive key are still walked so that callers can mix
+ * structured payloads with redacted leaves.
+ */
+export function redactExtra<T>(extra: T): T {
+  if (extra == null || typeof extra !== 'object') return extra;
+  if (Array.isArray(extra)) {
+    return extra.map((v) => (v && typeof v === 'object' ? redactExtra(v) : v)) as unknown as T;
+  }
+  const out: Record<string, unknown> = {};
+  for (const [k, v] of Object.entries(extra as Record<string, unknown>)) {
+    if (isSensitiveEnvKey(k)) {
+      // Redact any non-null primitive (string/number/boolean) under a
+      // sensitive key. Nested objects keep being walked so a structured
+      // payload like { credentials: { ... } } still gets per-field masking.
+      if (v && typeof v === 'object') {
+        out[k] = redactExtra(v);
+      } else if (v == null) {
+        out[k] = v;
+      } else {
+        out[k] = '***REDACTED***';
+      }
+    } else if (v && typeof v === 'object') {
+      out[k] = redactExtra(v);
+    } else {
+      out[k] = v;
+    }
+  }
+  return out as T;
+}
+
 export function createLogger(service: string) {
   function log(level: LogLevel, msg: string, extra?: Record<string, unknown>): void {
-    const entry = { ts: new Date().toISOString(), level, service, msg, ...(extra ? { extra } : {}) };
+    const safeExtra = extra ? redactExtra(extra) : undefined;
+    const entry = { ts: new Date().toISOString(), level, service, msg, ...(safeExtra ? { extra: safeExtra } : {}) };
     (level === 'error' || level === 'warn' ? console.error : console.log)(JSON.stringify(entry));
   }
   return {

package/src/provider-constants.ts

@@ -40,19 +40,40 @@ export const PROVIDER_KEY_MAP: Record<string, string> = {
   huggingface: "HF_TOKEN",
 };
 
-/** Known embedding model dimensions (cloud providers). */
+/** Known embedding model dimensions. Keyed by `provider/model`. */
 export const EMBEDDING_DIMS: Record<string, number> = {
   "openai/text-embedding-3-small": 1536,
   "openai/text-embedding-3-large": 3072,
   "openai/text-embedding-ada-002": 1536,
   "ollama/nomic-embed-text": 768,
   "ollama/mxbai-embed-large": 1024,
+  "ollama/mxbai-embed-large-v1": 1024,
   "ollama/all-minilm": 384,
   "ollama/snowflake-arctic-embed": 1024,
+  "model-runner/ai/mxbai-embed-large-v1": 1024,
+  "mistral/mistral-embed": 1024,
   "google/text-embedding-004": 768,
   "huggingface/sentence-transformers/all-MiniLM-L6-v2": 384,
+  "huggingface/intfloat/multilingual-e5-large": 1024,
 };
 
+/**
+ * Look up embedding model dimensions. Tries the full key first, then strips
+ * any trailing `:tag` from the model name (Ollama-style versions).
+ * Returns 0 when no match is found.
+ */
+export function lookupEmbeddingDims(provider: string, model: string): number {
+  if (!provider || !model) return 0;
+  const key = `${provider}/${model}`;
+  if (EMBEDDING_DIMS[key]) return EMBEDDING_DIMS[key];
+  const colon = model.lastIndexOf(":");
+  if (colon > 0) {
+    const bare = `${provider}/${model.slice(0, colon)}`;
+    if (EMBEDDING_DIMS[bare]) return EMBEDDING_DIMS[bare];
+  }
+  return 0;
+}
+
 /** Provider display labels for UI. */
 export const PROVIDER_LABELS: Record<string, string> = {
   openai: "OpenAI",

package/src/control-plane/audit.ts

@@ -1,40 +0,0 @@
-/**
- * Audit logging for the OpenPalm control plane.
- */
-import { mkdirSync, appendFileSync } from "node:fs";
-import type { ControlPlaneState, AuditEntry, CallerType } from "./types.js";
-
-const MAX_AUDIT_MEMORY = 1000;
-
-export function appendAudit(
-  state: ControlPlaneState,
-  actor: string,
-  action: string,
-  args: Record<string, unknown>,
-  ok: boolean,
-  requestId = "",
-  callerType: CallerType = "unknown"
-): void {
-  const entry: AuditEntry = {
-    at: new Date().toISOString(),
-    requestId,
-    actor,
-    callerType,
-    action,
-    args,
-    ok
-  };
-  state.audit.push(entry);
-  if (state.audit.length > MAX_AUDIT_MEMORY) {
-    state.audit = state.audit.slice(-MAX_AUDIT_MEMORY);
-  }
-  try {
-    mkdirSync(state.logsDir, { recursive: true });
-    appendFileSync(
-      `${state.logsDir}/admin-audit.jsonl`,
-      JSON.stringify(entry) + "\n"
-    );
-  } catch {
-    // best-effort persistence
-  }
-}

package/src/control-plane/env-schema-validation.test.ts

@@ -1,118 +0,0 @@
-/**
- * Test that env schema validation uses the correct nested vault paths.
- */
-import { describe, test, expect, beforeAll, afterAll } from "bun:test";
-import { mkdirSync, writeFileSync, rmSync, existsSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import type { ControlPlaneState } from "./types.js";
-
-describe("env schema validation paths", () => {
-  let tmpDir: string;
-  let state: ControlPlaneState;
-
-  beforeAll(() => {
-    tmpDir = join(tmpdir(), `openpalm-schema-test-${Date.now()}`);
-    mkdirSync(join(tmpDir, "vault/user"), { recursive: true });
-    mkdirSync(join(tmpDir, "vault/stack"), { recursive: true });
-    mkdirSync(join(tmpDir, "data"), { recursive: true });
-    mkdirSync(join(tmpDir, "logs"), { recursive: true });
-    mkdirSync(join(tmpDir, "config"), { recursive: true });
-
-    state = {
-      adminToken: "test-token",
-      assistantToken: "test-assistant",
-      setupToken: "test-setup",
-      homeDir: tmpDir,
-      configDir: join(tmpDir, "config"),
-      vaultDir: join(tmpDir, "vault"),
-      dataDir: join(tmpDir, "data"),
-      logsDir: join(tmpDir, "logs"),
-      cacheDir: join(tmpDir, "cache"),
-      services: {},
-      artifacts: { compose: "" },
-      artifactMeta: [],
-      audit: [],
-    };
-  });
-
-  afterAll(() => {
-    if (tmpDir && existsSync(tmpDir)) {
-      rmSync(tmpDir, { recursive: true, force: true });
-    }
-  });
-
-  test("validation succeeds when no schema files exist (skip mode)", async () => {
-    const { validateProposedState } = await import("./validate.js");
-    const result = await validateProposedState(state);
-    // When schema files don't exist, validation is skipped (no errors)
-    expect(result.ok).toBe(true);
-    expect(result.errors).toEqual([]);
-  });
-
-  test("schema paths match canonical vault layout", () => {
-    const expectedUserSchema = join(tmpDir, "vault/user/user.env.schema");
-    const expectedStackSchema = join(tmpDir, "vault/stack/stack.env.schema");
-
-    writeFileSync(expectedUserSchema, "# test schema\n");
-    writeFileSync(expectedStackSchema, "# test schema\n");
-
-    expect(existsSync(expectedUserSchema)).toBe(true);
-    expect(existsSync(expectedStackSchema)).toBe(true);
-
-    // Old flat paths must NOT exist
-    expect(existsSync(join(tmpDir, "vault/user.env.schema"))).toBe(false);
-    expect(existsSync(join(tmpDir, "vault/system.env.schema"))).toBe(false);
-  });
-
-  test("validate.ts reads from nested paths, not flat paths", async () => {
-    // Write schemas at OLD flat paths — should be ignored
-    writeFileSync(join(tmpDir, "vault/user.env.schema"), "OPENAI_API_KEY\n");
-    writeFileSync(join(tmpDir, "vault/system.env.schema"), "OP_ADMIN_TOKEN\n");
-    // Write env files
-    writeFileSync(join(tmpDir, "vault/user/user.env"), "# empty\n");
-    writeFileSync(join(tmpDir, "vault/stack/stack.env"), "# empty\n");
-    // Delete nested schemas to prove flat paths are ignored
-    try { rmSync(join(tmpDir, "vault/user/user.env.schema")); } catch { /* may not exist */ }
-    try { rmSync(join(tmpDir, "vault/stack/stack.env.schema")); } catch { /* may not exist */ }
-
-    const { validateProposedState } = await import("./validate.js");
-    const result = await validateProposedState(state);
-    // Should pass because nested schemas don't exist (skipped), not because flat schemas were read
-    expect(result.ok).toBe(true);
-  });
-
-  test("validation reports warnings for missing required schema keys", async () => {
-    // Seed a schema that requires OPENAI_API_KEY
-    writeFileSync(join(tmpDir, "vault/user/user.env.schema"), "OPENAI_API_KEY=string\nOWNER_NAME=string\n");
-    // Seed an env file that is missing those keys
-    writeFileSync(join(tmpDir, "vault/user/user.env"), "# empty env\nSOME_OTHER_KEY=value\n");
-
-    const { validateProposedState } = await import("./validate.js");
-    const result = await validateProposedState(state);
-    // The validator should report warnings for missing keys (not errors — env validation is advisory)
-    expect(result.warnings.length).toBeGreaterThanOrEqual(0);
-  });
-
-  test("validation handles malformed env file gracefully", async () => {
-    writeFileSync(join(tmpDir, "vault/user/user.env.schema"), "OPENAI_API_KEY=string\n");
-    // Malformed: no = sign, just random text
-    writeFileSync(join(tmpDir, "vault/user/user.env"), "this is not a valid env file\n===\n");
-
-    const { validateProposedState } = await import("./validate.js");
-    const result = await validateProposedState(state);
-    // Should not throw — graceful handling
-    expect(typeof result.ok).toBe("boolean");
-  });
-
-  test("validation handles empty schema file gracefully", async () => {
-    writeFileSync(join(tmpDir, "vault/user/user.env.schema"), "");
-    writeFileSync(join(tmpDir, "vault/user/user.env"), "OPENAI_API_KEY=sk-test\n");
-
-    const { validateProposedState } = await import("./validate.js");
-    const result = await validateProposedState(state);
-    // Empty schema may cause varlock to report an error — that's fine,
-    // the important thing is it doesn't throw/crash
-    expect(typeof result.ok).toBe("boolean");
-  });
-});

package/src/control-plane/lock.test.ts

@@ -1,194 +0,0 @@
-/**
- * Tests for orchestrator lock — acquisition, contention, stale cleanup,
- * corrupt file handling, release, and idempotent release.
- */
-import { describe, it, expect, beforeEach, afterEach } from "bun:test";
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync, existsSync, readFileSync } from "node:fs";
-import { join } from "node:path";
-import { tmpdir } from "node:os";
-import {
-  acquireLock,
-  releaseLock,
-  lockPath,
-  LockAcquisitionError,
-} from "./lock.js";
-import type { LockHandle, LockInfo } from "./lock.js";
-
-let opHome: string;
-
-beforeEach(() => {
-  opHome = mkdtempSync(join(tmpdir(), "lock-test-"));
-  mkdirSync(join(opHome, "data"), { recursive: true });
-});
-
-afterEach(() => {
-  rmSync(opHome, { recursive: true, force: true });
-});
-
-// ── Acquisition ──────────────────────────────────────────────────────────
-
-describe("acquireLock", () => {
-  it("creates a lock file with correct JSON content", () => {
-    const handle = acquireLock(opHome, "install");
-    expect(existsSync(handle.path)).toBe(true);
-
-    const content = JSON.parse(readFileSync(handle.path, "utf-8"));
-    expect(content.pid).toBe(process.pid);
-    expect(content.operation).toBe("install");
-    expect(typeof content.acquiredAt).toBe("string");
-
-    releaseLock(handle);
-  });
-
-  it("returns a handle with correct info", () => {
-    const handle = acquireLock(opHome, "update");
-    expect(handle.info.pid).toBe(process.pid);
-    expect(handle.info.operation).toBe("update");
-    expect(handle.path).toBe(lockPath(opHome));
-
-    releaseLock(handle);
-  });
-
-  it("places lock at {opHome}/data/.openpalm.lock", () => {
-    const handle = acquireLock(opHome, "test");
-    expect(handle.path).toBe(join(opHome, "data", ".openpalm.lock"));
-    releaseLock(handle);
-  });
-});
-
-// ── Contention ───────────────────────────────────────────────────────────
-
-describe("contention", () => {
-  it("throws LockAcquisitionError when lock is already held by this process", () => {
-    const handle = acquireLock(opHome, "install");
-
-    try {
-      expect(() => acquireLock(opHome, "update")).toThrow(LockAcquisitionError);
-    } finally {
-      releaseLock(handle);
-    }
-  });
-
-  it("error includes holder details", () => {
-    const handle = acquireLock(opHome, "install");
-
-    try {
-      acquireLock(opHome, "update");
-      expect.unreachable("should have thrown");
-    } catch (err) {
-      expect(err).toBeInstanceOf(LockAcquisitionError);
-      const lockErr = err as LockAcquisitionError;
-      expect(lockErr.holder.pid).toBe(process.pid);
-      expect(lockErr.holder.operation).toBe("install");
-    } finally {
-      releaseLock(handle);
-    }
-  });
-});
-
-// ── Stale PID cleanup ────────────────────────────────────────────────────
-
-describe("stale PID cleanup", () => {
-  it("cleans up stale lock from a dead PID and acquires", () => {
-    // Write a lock file with a PID that does not exist
-    const stalePid = 99999999; // Very unlikely to be a real process
-    const staleInfo: LockInfo = {
-      pid: stalePid,
-      operation: "old-install",
-      acquiredAt: "2020-01-01T00:00:00.000Z",
-    };
-    writeFileSync(lockPath(opHome), JSON.stringify(staleInfo) + "\n");
-
-    // Should succeed because the PID is dead
-    const handle = acquireLock(opHome, "new-install");
-    expect(handle.info.pid).toBe(process.pid);
-    expect(handle.info.operation).toBe("new-install");
-
-    releaseLock(handle);
-  });
-});
-
-// ── Corrupt file handling ────────────────────────────────────────────────
-
-describe("corrupt lock file", () => {
-  it("recovers from a corrupt lock file", () => {
-    writeFileSync(lockPath(opHome), "not valid json{{{");
-
-    const handle = acquireLock(opHome, "install");
-    expect(handle.info.pid).toBe(process.pid);
-
-    releaseLock(handle);
-  });
-
-  it("recovers from an empty lock file", () => {
-    writeFileSync(lockPath(opHome), "");
-
-    const handle = acquireLock(opHome, "install");
-    expect(handle.info.pid).toBe(process.pid);
-
-    releaseLock(handle);
-  });
-
-  it("recovers from a lock file with missing fields", () => {
-    writeFileSync(lockPath(opHome), JSON.stringify({ pid: 1 }));
-
-    const handle = acquireLock(opHome, "install");
-    expect(handle.info.pid).toBe(process.pid);
-
-    releaseLock(handle);
-  });
-});
-
-// ── Release ──────────────────────────────────────────────────────────────
-
-describe("releaseLock", () => {
-  it("removes the lock file", () => {
-    const handle = acquireLock(opHome, "install");
-    expect(existsSync(handle.path)).toBe(true);
-
-    releaseLock(handle);
-    expect(existsSync(handle.path)).toBe(false);
-  });
-
-  it("is idempotent — second release is a no-op", () => {
-    const handle = acquireLock(opHome, "install");
-    releaseLock(handle);
-    expect(existsSync(handle.path)).toBe(false);
-
-    // Second release should not throw
-    releaseLock(handle);
-    expect(existsSync(handle.path)).toBe(false);
-  });
-
-  it("does not remove lock owned by a different PID", () => {
-    // Simulate a lock file owned by someone else
-    const otherInfo: LockInfo = {
-      pid: 99999999,
-      operation: "other",
-      acquiredAt: new Date().toISOString(),
-    };
-    writeFileSync(lockPath(opHome), JSON.stringify(otherInfo) + "\n");
-
-    // Create a handle that claims to own the lock
-    const fakeHandle: LockHandle = {
-      path: lockPath(opHome),
-      info: {
-        pid: process.pid, // Different from file content
-        operation: "mine",
-        acquiredAt: new Date().toISOString(),
-      },
-    };
-
-    releaseLock(fakeHandle);
-    // Lock file should still exist because PID doesn't match
-    expect(existsSync(lockPath(opHome))).toBe(true);
-  });
-});
-
-// ── lockPath ─────────────────────────────────────────────────────────────
-
-describe("lockPath", () => {
-  it("returns the correct path", () => {
-    expect(lockPath("/home/user/.openpalm")).toBe("/home/user/.openpalm/data/.openpalm.lock");
-  });
-});

package/src/control-plane/lock.ts

@@ -1,176 +0,0 @@
-/**
- * Orchestrator lock — prevents concurrent mutating operations.
- *
- * Uses O_CREAT | O_EXCL for atomic exclusive file creation.
- * Lock file lives at {dataDir}/.openpalm.lock containing JSON
- * with { pid, operation, acquiredAt }.
- *
- * Uses node:fs (not Bun) since lib must be Node-compatible for SvelteKit admin.
- */
-import { openSync, writeSync, closeSync, readFileSync, unlinkSync, mkdirSync, constants } from "node:fs";
-import { dirname } from "node:path";
-
-// ── Types ────────────────────────────────────────────────────────────────
-
-export type LockInfo = {
-  pid: number;
-  operation: string;
-  acquiredAt: string;
-};
-
-export type LockHandle = {
-  path: string;
-  info: LockInfo;
-};
-
-// ── Error ────────────────────────────────────────────────────────────────
-
-export class LockAcquisitionError extends Error {
-  public readonly holder: LockInfo;
-
-  constructor(holder: LockInfo) {
-    super(
-      `Cannot acquire lock: already held by PID ${holder.pid} ` +
-      `for "${holder.operation}" since ${holder.acquiredAt}`
-    );
-    this.name = "LockAcquisitionError";
-    this.holder = holder;
-  }
-}
-
-// ── Path ─────────────────────────────────────────────────────────────────
-
-export function lockPath(opHome: string): string {
-  return `${opHome}/data/.openpalm.lock`;
-}
-
-// ── Stale PID Detection ──────────────────────────────────────────────────
-
-function isProcessAlive(pid: number): boolean {
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-// ── Read existing lock info ──────────────────────────────────────────────
-
-function readLockInfo(path: string): LockInfo | null {
-  try {
-    const content = readFileSync(path, "utf-8");
-    const parsed = JSON.parse(content);
-    if (
-      typeof parsed.pid === "number" &&
-      typeof parsed.operation === "string" &&
-      typeof parsed.acquiredAt === "string"
-    ) {
-      return parsed as LockInfo;
-    }
-    return null;
-  } catch {
-    return null;
-  }
-}
-
-// ── Acquire / Release ────────────────────────────────────────────────────
-
-export function acquireLock(opHome: string, operation: string): LockHandle {
-  const path = lockPath(opHome);
-  mkdirSync(dirname(path), { recursive: true });
-  const info: LockInfo = {
-    pid: process.pid,
-    operation,
-    acquiredAt: new Date().toISOString(),
-  };
-  const content = JSON.stringify(info) + "\n";
-
-  try {
-    // Atomic exclusive create — fails if file already exists
-    const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o644);
-    try {
-      writeSync(fd, content);
-    } finally {
-      closeSync(fd);
-    }
-    return { path, info };
-  } catch (err: unknown) {
-    // File already exists — check if it's stale
-    if ((err as NodeJS.ErrnoException).code === "EEXIST") {
-      const existing = readLockInfo(path);
-
-      if (existing && !isProcessAlive(existing.pid)) {
-        // Stale lock — remove and retry once
-        try {
-          unlinkSync(path);
-        } catch {
-          // Race: another process already removed it; fall through to retry
-        }
-        // Retry acquisition
-        try {
-          const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o644);
-          try {
-            writeSync(fd, content);
-          } finally {
-            closeSync(fd);
-          }
-          return { path, info };
-        } catch (retryErr: unknown) {
-          if ((retryErr as NodeJS.ErrnoException).code === "EEXIST") {
-            // Another process won the race — read the new holder
-            const newHolder = readLockInfo(path);
-            throw new LockAcquisitionError(
-              newHolder ?? { pid: 0, operation: "unknown", acquiredAt: "unknown" }
-            );
-          }
-          throw retryErr;
-        }
-      }
-
-      // Lock is held by a live process (or corrupt file — treat as held)
-      if (existing) {
-        throw new LockAcquisitionError(existing);
-      }
-
-      // Corrupt lock file — remove and retry
-      try {
-        unlinkSync(path);
-      } catch {
-        // ignore
-      }
-      try {
-        const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o644);
-        try {
-          writeSync(fd, content);
-        } finally {
-          closeSync(fd);
-        }
-        return { path, info };
-      } catch (retryErr: unknown) {
-        if ((retryErr as NodeJS.ErrnoException).code === "EEXIST") {
-          const newHolder = readLockInfo(path);
-          throw new LockAcquisitionError(
-            newHolder ?? { pid: 0, operation: "unknown", acquiredAt: "unknown" }
-          );
-        }
-        throw retryErr;
-      }
-    }
-
-    throw err;
-  }
-}
-
-export function releaseLock(handle: LockHandle): void {
-  // Verify ownership before deleting — only remove if we still own it
-  const existing = readLockInfo(handle.path);
-  if (!existing) return; // Already gone — idempotent
-  if (existing.pid !== handle.info.pid) return; // Not ours — don't touch
-
-  try {
-    unlinkSync(handle.path);
-  } catch {
-    // Already removed — idempotent
-  }
-}