@openpalm/lib 0.10.2 → 0.11.0-beta.10

package/src/control-plane/provider-models.ts

@@ -0,0 +1,154 @@
+/**
+ * Provider model discovery and API key resolution.
+ *
+ * Used by the admin capabilities test endpoint and the CLI setup wizard
+ * to enumerate the models a configured provider exposes.
+ */
+import { readStackEnv } from "./secrets.js";
+import { PROVIDER_DEFAULT_URLS } from "../provider-constants.js";
+
+/** Static model list for Anthropic (no listing API available). */
+const ANTHROPIC_MODELS = [
+  "claude-opus-4-6",
+  "claude-sonnet-4-6",
+  "claude-opus-4-20250514",
+  "claude-sonnet-4-20250514",
+  "claude-haiku-4-5-20251001",
+  "claude-3-5-sonnet-20241022",
+  "claude-3-5-haiku-20241022",
+];
+
+
+/**
+ * Resolve an API key reference.
+ *
+ * - Empty input → empty string.
+ * - `env:NAME` form → looks up `NAME` in `process.env` first, then falls back
+ *   to `config/stack/stack.env` resolved against `stackDir`.
+ * - Anything else → returned verbatim (treated as a literal key value).
+ */
+function resolveApiKey(apiKeyRef: string, stackDir: string): string {
+  if (!apiKeyRef) return "";
+  if (!apiKeyRef.startsWith("env:")) return apiKeyRef;
+
+  const varName = apiKeyRef.slice(4);
+  if (process.env[varName]) return process.env[varName]!;
+
+  const secrets = readStackEnv(stackDir);
+  return secrets[varName] ?? "";
+}
+
+
+export type ModelDiscoveryReason =
+  | 'none'
+  | 'provider_static'
+  | 'provider_http'
+  | 'missing_base_url'
+  | 'timeout'
+  | 'network';
+
+export type ProviderModelsResult = {
+  models: string[];
+  status: 'ok' | 'recoverable_error';
+  reason: ModelDiscoveryReason;
+  error?: string;
+};
+
+const HTTP_STATUS_LABELS: Record<number, string> = {
+  401: 'Invalid or missing API key',
+  403: 'Access denied — check API key permissions',
+  404: 'Endpoint not found — verify the base URL',
+  429: 'Rate limited — try again shortly',
+  500: 'Provider internal error',
+  502: 'Provider returned a bad gateway error',
+  503: 'Provider is temporarily unavailable',
+};
+
+/**
+ * Enumerate available models for a provider. Returns an `ok` result with a
+ * sorted model list when the provider responds successfully, or a
+ * `recoverable_error` with a structured reason otherwise. Network and timeout
+ * failures are caught and mapped to a result rather than thrown.
+ */
+export async function fetchProviderModels(
+  provider: string,
+  apiKeyRef: string,
+  baseUrl: string,
+  stackDir: string
+): Promise<ProviderModelsResult> {
+  try {
+    if (provider === "anthropic") {
+      return { models: [...ANTHROPIC_MODELS], status: 'ok', reason: 'provider_static' };
+    }
+
+    const resolvedKey = resolveApiKey(apiKeyRef, stackDir);
+
+    if (provider === "ollama") {
+      const base = baseUrl?.trim() || PROVIDER_DEFAULT_URLS.ollama;
+      const url = `${base.replace(/\/+$/, "")}/api/tags`;
+      const res = await fetch(url, { signal: AbortSignal.timeout(5000) });
+      if (!res.ok) {
+        return {
+          models: [],
+          status: 'recoverable_error',
+          reason: 'provider_http',
+          error: `Ollama API returned ${res.status}: ${(HTTP_STATUS_LABELS[res.status] ?? `HTTP ${res.status}`)}`,
+        };
+      }
+      const data = (await res.json()) as { models?: { name: string }[] };
+      const models = (data.models ?? []).map((m) => m.name).sort();
+      return { models, status: 'ok', reason: 'none' };
+    }
+
+    const base = baseUrl?.trim() || PROVIDER_DEFAULT_URLS[provider] || "";
+    if (!base) {
+      return {
+        models: [],
+        status: 'recoverable_error',
+        reason: 'missing_base_url',
+        error: `No base URL configured for provider "${provider}"`,
+      };
+    }
+    const url = `${base.replace(/\/+$/, "")}/v1/models`;
+
+    const headers: Record<string, string> = {};
+    if (resolvedKey) {
+      headers["Authorization"] = `Bearer ${resolvedKey}`;
+    }
+
+    const res = await fetch(url, { headers, signal: AbortSignal.timeout(5000) });
+    if (!res.ok) {
+      let detail = '';
+      try {
+        const json = JSON.parse(await res.text()) as Record<string, unknown>;
+        const errObj = json.error as Record<string, unknown> | string | undefined;
+        detail = (typeof errObj === 'object' && errObj !== null && typeof errObj.message === 'string') ? errObj.message
+          : typeof errObj === 'string' ? errObj
+          : typeof json.message === 'string' ? json.message
+          : typeof json.detail === 'string' ? json.detail : '';
+      } catch { /* ignore parse errors */ }
+      return {
+        models: [],
+        status: 'recoverable_error',
+        reason: 'provider_http',
+        error: detail
+          ? `Provider API returned ${res.status}: ${detail}`
+          : `Provider API returned ${res.status}: ${(HTTP_STATUS_LABELS[res.status] ?? `HTTP ${res.status}`)}`,
+      };
+    }
+    const data = (await res.json()) as { data?: { id: string }[] };
+    const models = (data.data ?? []).map((m) => m.id).sort();
+    return { models, status: 'ok', reason: 'none' };
+  } catch (err) {
+    const message =
+      err instanceof Error && err.name === "TimeoutError"
+        ? "Request timed out after 5s"
+        : String(err);
+    return {
+      models: [],
+      status: 'recoverable_error',
+      reason: err instanceof Error && err.name === 'TimeoutError' ? 'timeout' : 'network',
+      error: message,
+    };
+  }
+}

package/src/control-plane/registry-components.test.ts

@@ -1,10 +1,23 @@
 /**
  * Tests for the registry component directory format.
  *
- * Validates that all components in .openpalm/registry/addons/ follow the
+ * Validates that all components in .openpalm/state/registry/addons/ follow the
  * component conventions: compose.yml with required labels, .env.schema
  * with documented variables, proper service naming, and no security
  * violations.
+ *
+ * Two component shapes are accepted:
+ *
+ *   1. Full addons — compose.yml + .env.schema. They introduce a new
+ *      service, declare env vars, and must satisfy the full structural
+ *      checklist (labels, network, healthcheck, restart policy, sensitive
+ *      fields).
+ *   2. Overlay-only addons — compose.yml only. They patch existing
+ *      services (ports, env, volumes) instead of introducing new ones,
+ *      so they have no env vars to document and no service-shaped
+ *      requirements. They still must satisfy the security invariants:
+ *      no INSTANCE_ID, no container_name, no INSTANCE_DIR, no vault
+ *      directory mounts, no docker socket.
  */
 import { describe, expect, it } from "bun:test";
 import {
@@ -18,7 +31,7 @@ import { join, resolve } from "node:path";
 
 /** Resolve path from repo root */
 const REPO_ROOT = resolve(import.meta.dir, "../../../..");
-const REGISTRY_DIR = join(REPO_ROOT, ".openpalm/registry/addons");
+const REGISTRY_DIR = join(REPO_ROOT, ".openpalm/state/registry/addons");
 
 /** List all component directories in the registry */
 function listComponentDirs(): string[] {
@@ -28,6 +41,19 @@ function listComponentDirs(): string[] {
     .map((d) => d.name);
 }
 
+/** Overlay-only addons ship compose.yml only — no .env.schema. */
+function isOverlayOnly(componentId: string): boolean {
+  return !existsSync(join(REGISTRY_DIR, componentId, ".env.schema"));
+}
+
+function listFullAddonIds(componentIds: string[]): string[] {
+  return componentIds.filter((id) => !isOverlayOnly(id));
+}
+
+function listOverlayOnlyAddonIds(componentIds: string[]): string[] {
+  return componentIds.filter(isOverlayOnly);
+}
+
 /** Read a file from a component directory */
 function readComponentFile(componentId: string, filename: string): string {
   return readFileSync(join(REGISTRY_DIR, componentId, filename), "utf-8");
@@ -118,24 +144,67 @@ describe("registry component discovery", () => {
 
 describe("registry component required files", () => {
   const componentIds = listComponentDirs();
+  const fullAddonIds = listFullAddonIds(componentIds);
 
   for (const id of componentIds) {
     it(`${id}: has compose.yml`, () => {
       expect(existsSync(join(REGISTRY_DIR, id, "compose.yml"))).toBe(true);
     });
+  }
 
-    it(`${id}: has .env.schema`, () => {
+  for (const id of fullAddonIds) {
+    it(`${id}: has .env.schema (full addon)`, () => {
       expect(existsSync(join(REGISTRY_DIR, id, ".env.schema"))).toBe(true);
     });
   }
 });
 
+// ── Overlay-only Addon Tests ─────────────────────────────────────────────
+
+describe("registry overlay-only addons", () => {
+  const componentIds = listComponentDirs();
+  const overlayIds = listOverlayOnlyAddonIds(componentIds);
+
+  it("at least one overlay-only addon (ssh) is recognized as valid", () => {
+    expect(overlayIds).toContain("ssh");
+  });
+
+  for (const id of overlayIds) {
+    describe(id, () => {
+      it("ships only compose.yml (no .env.schema, no entrypoint, no Dockerfile)", () => {
+        const dirEntries = readdirSync(join(REGISTRY_DIR, id));
+        // compose.yml is required; an optional README.md is allowed; nothing
+        // else (no .env.schema, no entrypoint*, no Dockerfile, no scripts).
+        const allowed = new Set(["compose.yml", "README.md"]);
+        for (const file of dirEntries) {
+          expect(allowed.has(file)).toBe(true);
+        }
+      });
+
+      it("compose.yml does not introduce a new service (no image: or build:)", () => {
+        // Overlay-only addons may patch existing services with new ports/env,
+        // but they MUST NOT introduce a new service that needs its own
+        // network/healthcheck/restart contract — those would belong in a
+        // full addon. Reject service definition keys that imply a new
+        // service body. A pure overlay only sets `ports:`, `environment:`,
+        // `volumes:`, etc. on already-defined services.
+        const compose = readComponentFile(id, "compose.yml");
+        expect(compose).not.toMatch(/^\s+image:\s/m);
+        expect(compose).not.toMatch(/^\s+build:\s/m);
+      });
+    });
+  }
+});
+
 // ── Compose Overlay Validation Tests ─────────────────────────────────────
 
 describe("registry compose.yml validation", () => {
   const componentIds = listComponentDirs();
+  const fullAddonIds = listFullAddonIds(componentIds);
 
-  for (const id of componentIds) {
+  // Full-addon-only assertions: anything that requires a service body
+  // (labels, network, healthcheck, restart policy) is checked here.
+  for (const id of fullAddonIds) {
     describe(id, () => {
       const compose = readComponentFile(id, "compose.yml");
 
@@ -147,18 +216,6 @@ describe("registry compose.yml validation", () => {
         expect(compose).toMatch(/openpalm\.description:/);
       });
 
-      it("uses static service name (no INSTANCE_ID)", () => {
-        expect(compose).not.toContain("${INSTANCE_ID}");
-      });
-
-      it("does not use container_name", () => {
-        expect(compose).not.toMatch(/container_name:/);
-      });
-
-      it("does not reference INSTANCE_DIR", () => {
-        expect(compose).not.toContain("${INSTANCE_DIR}");
-      });
-
       it("joins a valid stack network", () => {
         const hasValidNetwork = compose.includes("channel_lan") || compose.includes("channel_public") || compose.includes("assistant_net");
         expect(hasValidNetwork).toBe(true);
@@ -171,9 +228,28 @@ describe("registry compose.yml validation", () => {
       it("has healthcheck", () => {
         expect(compose).toMatch(/healthcheck:/);
       });
+    });
+  }
+
+  // Security/hygiene assertions apply to ALL addons (full and overlay-only).
+  for (const id of componentIds) {
+    describe(`${id} (security)`, () => {
+      const compose = readComponentFile(id, "compose.yml");
+
+      it("uses static service name (no INSTANCE_ID)", () => {
+        expect(compose).not.toContain("${INSTANCE_ID}");
+      });
+
+      it("does not use container_name", () => {
+        expect(compose).not.toMatch(/container_name:/);
+      });
+
+      it("does not reference INSTANCE_DIR", () => {
+        expect(compose).not.toContain("${INSTANCE_DIR}");
+      });
 
       it("does not mount vault directory (single-file mounts allowed)", () => {
-        // Directory-level vault mounts are a security violation — only admin gets full vault access.
+        // Directory-level vault mounts are a security violation — no container may mount the full vault.
         // Single-file mounts like vault/user/ov.conf are allowed (the source must end with a filename).
         const lines = compose.split("\n");
         for (const line of lines) {
@@ -193,8 +269,6 @@ describe("registry compose.yml validation", () => {
       });
 
       it("does not mount docker socket", () => {
-        // admin component is exempt — docker-socket-proxy IS the docker socket accessor by design
-        if (id === "admin") return;
         expect(compose).not.toContain("/var/run/docker.sock");
       });
 
@@ -209,8 +283,9 @@ describe("registry compose.yml validation", () => {
 
 describe("registry .env.schema validation", () => {
   const componentIds = listComponentDirs();
+  const fullAddonIds = listFullAddonIds(componentIds);
 
-  for (const id of componentIds) {
+  for (const id of fullAddonIds) {
     describe(id, () => {
       const schema = readComponentFile(id, ".env.schema");
       const entries = parseEnvSchema(schema);
@@ -264,11 +339,13 @@ describe("registry .env.schema validation", () => {
 
 describe("registry component sensitive fields", () => {
   const componentIds = listComponentDirs();
+  const fullAddonIds = listFullAddonIds(componentIds);
 
-  for (const id of componentIds) {
+  for (const id of fullAddonIds) {
     it(`${id}: has at least one @sensitive field (channel secret)`, () => {
-      // ollama is a local inference server — no channel secret or API key needed
-      if (id === "ollama") return;
+      // ollama and voice are local inference servers — no channel secret
+      // or upstream API key needed (LAN-only, no auth by design).
+      if (id === "ollama" || id === "voice") return;
       const schema = readComponentFile(id, ".env.schema");
       const entries = parseEnvSchema(schema);
       const sensitiveEntries = entries.filter((e) =>
@@ -283,10 +360,11 @@ describe("registry component sensitive fields", () => {
 
 describe("cross-component consistency", () => {
   const componentIds = listComponentDirs();
+  const fullAddonIds = listFullAddonIds(componentIds);
 
-  it("no duplicate openpalm.name labels across components", () => {
+  it("no duplicate openpalm.name labels across full addons", () => {
     const names = new Set<string>();
-    for (const id of componentIds) {
+    for (const id of fullAddonIds) {
       const compose = readComponentFile(id, "compose.yml");
       const nameMatch = compose.match(/openpalm\.name:\s*(.+)/);
       expect(nameMatch).not.toBeNull();
@@ -296,8 +374,8 @@ describe("cross-component consistency", () => {
     }
   });
 
-  it("all components join a valid stack network", () => {
-    for (const id of componentIds) {
+  it("all full addons join a valid stack network", () => {
+    for (const id of fullAddonIds) {
       const compose = readComponentFile(id, "compose.yml");
       const hasValidNetwork = compose.includes("channel_lan") || compose.includes("channel_public") || compose.includes("assistant_net");
       expect(hasValidNetwork).toBe(true);