@klhapp/skillmux 1.9.1 → 1.9.3

package/src/server.ts

@@ -1,10 +1,10 @@
 #!/usr/bin/env bun
-import { timingSafeEqual } from "node:crypto";
+import { createHash, timingSafeEqual } from "node:crypto";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
 import { createClients } from "./clients";
-import { loadConfig, resolveConfigPath } from "./config";
+import { isLoopbackBindHost, loadConfig, resolveConfigPath } from "./config";
 import { describeDeployment } from "./deployment";
 import { ConfigWatcher, type ReloadStatus } from "./config-watcher";
 import { RuntimeSnapshotManager } from "./snapshot";
@@ -21,6 +21,8 @@ import { SKILL_ID_PATTERN } from "./vault";
 import { MetricsRegistry } from "./metrics";
 import { ReadinessState } from "./readiness";
 import { initializeRuntime } from "./lifecycle";
+import { buildRedactor } from "./redact";
+import { insertAdminAuditRow, type AdminAuditChange } from "./db";
 import type { Clients, Config } from "./types";
 import {
   computeHash,
@@ -34,12 +36,57 @@ import {
 export const metricsRegistry = new MetricsRegistry();
 export const readinessState = new ReadinessState();
 
+// runtime-resource-hardening: positive defaults so the http transport is
+// never unbounded by omission, unlike the opt-in [egress] allowlist.
+const DEFAULT_MAX_BODY_BYTES = 1_048_576; // 1 MiB
+const DEFAULT_MAX_CONCURRENT_REQUESTS = 100;
+
+/** Pairs a fixed log prefix with a redacted error message, for console.error. */
+export function redactedErrorLog(
+  prefix: string,
+  err: unknown,
+  redact: (text: string) => string,
+): [string, string] {
+  const msg = err instanceof Error ? err.message : String(err);
+  return [prefix, redact(msg)];
+}
+
 export interface ServerHandle {
   port?: number;
+  statsPort?: number;
   reloadStatus(): ReloadStatus;
   stop(): Promise<void>;
 }
 
+/**
+ * Docker's documented deployment mode auto-switches hostname to "0.0.0.0"
+ * (config.ts's RUNNING_IN_DOCKER override) while server.auth_enabled still
+ * defaults to false and allowed_origins defaults to [] — which only blocks
+ * requests that carry an Origin header, so a plain server-to-server/curl
+ * request sails through unauthenticated. That default combination leaves MCP
+ * tools (resolve_skill/fetch_skill) and /stats (raw historical query text)
+ * open to anyone who can reach the port — the documented common case, not an
+ * edge case (SMX-91). Refuse to start rather than silently exposing it;
+ * SKILLMUX_ALLOW_INSECURE_BIND is the explicit, logged escape hatch for
+ * operators who rely on network-level isolation instead of application auth.
+ */
+export function assertSafeBindPosture(
+  hostname: string,
+  authEnabled: boolean,
+  env: Record<string, string | undefined> = process.env,
+): void {
+  if (isLoopbackBindHost(hostname) || authEnabled) return;
+  const message =
+    `refusing to bind "${hostname}" (reachable beyond this machine) with server.auth_enabled=false — ` +
+    "MCP tools (resolve_skill/fetch_skill) and /stats would be open to anyone who can reach this port. " +
+    "Set server.auth_enabled=true (with SKILLMUX_AUTH_TOKEN) or bind a loopback hostname instead. " +
+    "To start anyway — e.g. when network isolation is the intended boundary — set SKILLMUX_ALLOW_INSECURE_BIND=true.";
+  if (env.SKILLMUX_ALLOW_INSECURE_BIND !== "true") {
+    throw new Error(`skillmux: ${message}`);
+  }
+  console.error(`skillmux: WARNING — ${message}`);
+}
+
 let warnedAuthToken = false;
 function resolveAuthToken(envName: string): string {
   const value = process.env[envName];
@@ -59,11 +106,108 @@ function resolveAuthToken(envName: string): string {
   return "";
 }
 
-function safeTokenEquals(a: string, b: string): boolean {
-  const bufA = Buffer.from(a);
-  const bufB = Buffer.from(b);
-  if (bufA.length !== bufB.length) return false;
-  return timingSafeEqual(bufA, bufB);
+// SMX-94: comparing raw buffers made length itself observable — a
+// mismatched-length pair returns before ever reaching timingSafeEqual.
+// Hashing both sides to a fixed 32-byte digest first means every
+// comparison takes the same constant-time path regardless of input length.
+export function safeTokenEquals(a: string, b: string): boolean {
+  const hashA = createHash("sha256").update(a).digest();
+  const hashB = createHash("sha256").update(b).digest();
+  return timingSafeEqual(hashA, hashB);
+}
+
+/**
+ * A second, narrow HTTP listener exposing only GET /health and GET /stats —
+ * nothing from the MCP tool surface or /admin/v1/*. Lets a stdio deployment
+ * (no HTTP transport at all) still answer `skillmux report --server ...`
+ * remotely, without switching its primary transport. Reuses the exact same
+ * bind-posture guard, auth-token check, and rate limiter as the http
+ * transport's /stats route (server.ts's main Bun.serve handler) rather than
+ * inventing a separate security model for this listener.
+ */
+async function serveStatsOnly(opts: {
+  config: Config;
+  port: number;
+}): Promise<{ port: number | undefined; stop(): void }> {
+  const serverConfig = opts.config.server || {
+    auth_enabled: false,
+    auth_token_env: "SKILLMUX_AUTH_TOKEN",
+    allowed_origins: [],
+  };
+  const hostname = serverConfig.hostname ?? "127.0.0.1";
+  assertSafeBindPosture(hostname, serverConfig.auth_enabled ?? false);
+
+  const { RateLimiter } = await import("./rate-limiter");
+  const rateLimiter = new RateLimiter(
+    serverConfig.rate_limit || { enabled: false, requests_per_minute: 60 },
+  );
+
+  const bunServer = Bun.serve({
+    port: opts.port,
+    hostname,
+    async fetch(req, server) {
+      const rateLimitResult = rateLimiter.check({
+        nowMs: Date.now(),
+        auth_enabled: serverConfig.auth_enabled,
+        req,
+        server,
+      });
+      if (!rateLimitResult.allowed) {
+        return new Response("Too Many Requests", {
+          status: 429,
+          headers: rateLimitResult.headers,
+        });
+      }
+
+      const url = new URL(req.url);
+      // GET /health stays open (unauthenticated) even when auth_enabled — it
+      // carries no data, matching the http transport's /health, which returns
+      // before its own Token Auth Check for the same reason.
+      if (req.method === "GET" && (url.pathname === "/health" || url.pathname === "/health/live")) {
+        return new Response(JSON.stringify({ status: "ok" }), {
+          status: 200,
+          headers: { "Content-Type": "application/json", ...rateLimitResult.headers },
+        });
+      }
+
+      if (serverConfig.auth_enabled) {
+        const expectedToken = resolveAuthToken(serverConfig.auth_token_env);
+        if (!expectedToken) {
+          return new Response(
+            "Server authentication configured but token environment variable is empty",
+            { status: 500 },
+          );
+        }
+        const authHeader = req.headers.get("authorization") || "";
+        const token = authHeader.startsWith("Bearer ")
+          ? authHeader.slice(7)
+          : authHeader;
+        if (!token || !safeTokenEquals(token, expectedToken)) {
+          return new Response("Unauthorized", { status: 401 });
+        }
+      }
+
+      if (req.method === "GET" && url.pathname === "/stats") {
+        const since = url.searchParams.get("since") ?? "";
+        if (!SINCE_PATTERN.test(since)) {
+          return new Response(
+            JSON.stringify({
+              error: "since must be a relative window (e.g. 30d) or an absolute ISO-8601 date",
+            }),
+            { status: 400, headers: { "Content-Type": "application/json" } },
+          );
+        }
+        const { auditDb } = await getRuntime();
+        return new Response(JSON.stringify(getStats(auditDb, since)), {
+          status: 200,
+          headers: { "Content-Type": "application/json", ...rateLimitResult.headers },
+        });
+      }
+      return new Response("Not Found", { status: 404 });
+    },
+  });
+
+  return { port: bunServer.port, stop: () => bunServer.stop(true) };
 }
 
 export function createMcpServer(): McpServer {
@@ -142,12 +286,14 @@ export function createMcpServer(): McpServer {
 export async function startServer(opts?: {
   transport?: "stdio" | "http";
   port?: number;
+  statsPort?: number;
   config?: Config;
   clients?: Partial<Clients>;
   configPath?: string;
 }): Promise<ServerHandle> {
   const configPath = resolveConfigPath(opts?.configPath);
   const config = opts?.config ?? (await loadConfig(configPath));
+  const redact = buildRedactor(config);
   const initialClients = { ...createClients(config), ...opts?.clients };
   const snapshots = RuntimeSnapshotManager.create(config, initialClients);
   const inactiveReloadStatus: ReloadStatus = {
@@ -172,18 +318,18 @@ export async function startServer(opts?: {
           configure({ config: nextConfig, clients: nextClients });
         },
         onError: (error) =>
-          console.error("skillmux config reload error:", error),
+          console.error(...redactedErrorLog("skillmux config reload error:", error, redact)),
       })
     : undefined;
   const stopWatcher = await startVaultWatcher();
   const initPromise = initializeRuntime(readinessState)
     .then(() => metricsRegistry.setReadiness(readinessState.get()))
-    .catch((err) => console.error("skillmux runtime init error:", err));
+    .catch((err) => console.error(...redactedErrorLog("skillmux runtime init error:", err, redact)));
 
   // AC14: fire-and-forget so this never delays readiness or blocks a resolve;
   // not chained onto initPromise, which is awaited below for HTTP transport.
   const runAuditPrune = () =>
-    pruneAuditIfDue().catch((err) => console.error("skillmux audit prune error:", err));
+    pruneAuditIfDue().catch((err) => console.error(...redactedErrorLog("skillmux audit prune error:", err, redact)));
   runAuditPrune();
   const auditPruneInterval = setInterval(runAuditPrune, 24 * 60 * 60 * 1000);
   auditPruneInterval.unref();
@@ -191,6 +337,16 @@ export async function startServer(opts?: {
   const server = createMcpServer();
 
   const transportType = opts?.transport ?? "stdio";
+  if (opts?.statsPort !== undefined && transportType === "http") {
+    throw new Error(
+      "skillmux: --stats-port is not supported with --transport http; the http transport already serves /stats on --port",
+    );
+  }
+  const statsHandle =
+    opts?.statsPort !== undefined
+      ? await serveStatsOnly({ config, port: opts.statsPort })
+      : undefined;
+
   if (transportType === "http") {
     const { WebStandardStreamableHTTPServerTransport } =
       await import("@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js");
@@ -203,325 +359,378 @@ export async function startServer(opts?: {
     const rateLimiter = new RateLimiter(
       config.server?.rate_limit || { enabled: false, requests_per_minute: 60 },
     );
+    const { ConcurrencyLimiter, releaseOnStreamClose } = await import("./concurrency-limiter");
+    const concurrencyLimiter = new ConcurrencyLimiter(
+      config.server?.max_concurrent_requests ?? DEFAULT_MAX_CONCURRENT_REQUESTS,
+    );
 
     const port = opts?.port ?? Number(process.env.PORT || 3000);
     const hostname = config.server?.hostname ?? "127.0.0.1";
+    assertSafeBindPosture(hostname, config.server?.auth_enabled ?? false);
     const bunServer = Bun.serve({
       port,
       hostname,
+      maxRequestBodySize: config.server?.max_body_bytes ?? DEFAULT_MAX_BODY_BYTES,
       async fetch(req, server) {
-        const serverConfig = config.server || {
-          auth_enabled: false,
-          auth_token_env: "SKILLMUX_AUTH_TOKEN",
-          allowed_origins: [],
-        };
-        const origin = req.headers.get("origin") || "";
-        const allowedOrigins = serverConfig.allowed_origins;
-        const isAllowed =
-          allowedOrigins.includes("*") || allowedOrigins.includes(origin);
-        const allowOriginHeader = isAllowed
-          ? allowedOrigins.includes("*")
-            ? "*"
-            : origin
-          : "";
-
-        if (origin && !isAllowed) {
-          return new Response("CORS origin not allowed", { status: 403 });
+        // AC3: a positive bound on in-flight requests, checked before any other
+        // work — protects against connection exhaustion the same way
+        // maxRequestBodySize protects against a single oversized request.
+        if (!concurrencyLimiter.tryAcquire()) {
+          return new Response("Service Unavailable", {
+            status: 503,
+            headers: { "Retry-After": "1" },
+          });
         }
-
-        if (req.method === "OPTIONS") {
-          return new Response(null, {
-            headers: {
-              "Access-Control-Allow-Origin": allowOriginHeader,
-              "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS",
-              "Access-Control-Allow-Headers":
-                "Content-Type, Authorization, MCP-Protocol-Version",
-            },
+        try {
+          const res = await handleHttpRequest(req, server);
+          // Defer the release until the response body actually finishes —
+          // a buffered body drains almost immediately, but an open SSE
+          // stream (the MCP transport's notification channel and streaming
+          // replies) can stay open long after this async function returns,
+          // and the slot must reflect that real connection lifetime.
+          const body = releaseOnStreamClose(res.body, () => concurrencyLimiter.release());
+          return new Response(body, {
+            status: res.status,
+            statusText: res.statusText,
+            headers: res.headers,
           });
+        } catch (error) {
+          concurrencyLimiter.release();
+          throw error;
         }
 
-        // Run rate limiter check
-        const rateLimitResult = rateLimiter.check({
-          nowMs: Date.now(),
-          auth_enabled: serverConfig.auth_enabled,
-          req,
-          server,
-        });
-
-        if (!rateLimitResult.allowed) {
-          metricsRegistry.recordRateLimitExceeded();
-
-          // Count the request in requests_total under the method if possible
-          let mcpMethod = "unknown";
-          try {
-            const bodyClone = await req.clone().json();
-            if (bodyClone.method === "tools/call") {
-              mcpMethod = bodyClone.params?.name || "tools/call";
-            } else {
-              mcpMethod = bodyClone.method || "unknown";
-            }
-          } catch {
-            // Non-JSON or parsing error
+        async function handleHttpRequest(
+          req: Request,
+          server: { requestIP(request: Request): { address: string } | null },
+        ): Promise<Response> {
+          const serverConfig = config.server || {
+            auth_enabled: false,
+            auth_token_env: "SKILLMUX_AUTH_TOKEN",
+            allowed_origins: [],
+          };
+          const origin = req.headers.get("origin") || "";
+          const allowedOrigins = serverConfig.allowed_origins;
+          const isAllowed =
+            allowedOrigins.includes("*") || allowedOrigins.includes(origin);
+          const allowOriginHeader = isAllowed
+            ? allowedOrigins.includes("*")
+              ? "*"
+              : origin
+            : "";
+
+          if (origin && !isAllowed) {
+            return new Response("CORS origin not allowed", { status: 403 });
           }
-          metricsRegistry.recordRequest(mcpMethod);
 
-          const headers = new Headers(rateLimitResult.headers);
-          if (allowOriginHeader) {
-            headers.set("Access-Control-Allow-Origin", allowOriginHeader);
+          if (req.method === "OPTIONS") {
+            return new Response(null, {
+              headers: {
+                "Access-Control-Allow-Origin": allowOriginHeader,
+                "Access-Control-Allow-Methods": "GET, POST, DELETE, OPTIONS",
+                "Access-Control-Allow-Headers":
+                  "Content-Type, Authorization, MCP-Protocol-Version",
+              },
+            });
           }
-          return new Response("Too Many Requests", {
-            status: 429,
-            headers,
+
+          // Run rate limiter check
+          const rateLimitResult = rateLimiter.check({
+            nowMs: Date.now(),
+            auth_enabled: serverConfig.auth_enabled,
+            req,
+            server,
           });
-        }
 
-        const url = new URL(req.url);
-        if (req.method === "GET") {
-          if (url.pathname === "/health" || url.pathname === "/health/live") {
-            const headers = new Headers({ "Content-Type": "application/json" });
+          if (!rateLimitResult.allowed) {
+            metricsRegistry.recordRateLimitExceeded();
+
+            // Count the request in requests_total under the method if possible
+            let mcpMethod = "unknown";
+            try {
+              const bodyClone = await req.clone().json();
+              if (bodyClone.method === "tools/call") {
+                mcpMethod = bodyClone.params?.name || "tools/call";
+              } else {
+                mcpMethod = bodyClone.method || "unknown";
+              }
+            } catch {
+              // Non-JSON or parsing error
+            }
+            metricsRegistry.recordRequest(mcpMethod);
+
+            const headers = new Headers(rateLimitResult.headers);
             if (allowOriginHeader) {
               headers.set("Access-Control-Allow-Origin", allowOriginHeader);
             }
-            for (const [key, value] of Object.entries(
-              rateLimitResult.headers,
-            )) {
-              headers.set(key, value);
-            }
-            return new Response(JSON.stringify({ status: "ok" }), {
-              status: 200,
+            return new Response("Too Many Requests", {
+              status: 429,
               headers,
             });
           }
-          if (url.pathname === "/health/ready") {
-            const readiness = readinessState.get();
-            const deployment = describeDeployment(config);
+
+          const url = new URL(req.url);
+          if (req.method === "GET") {
+            if (url.pathname === "/health" || url.pathname === "/health/live") {
+              const headers = new Headers({ "Content-Type": "application/json" });
+              if (allowOriginHeader) {
+                headers.set("Access-Control-Allow-Origin", allowOriginHeader);
+              }
+              for (const [key, value] of Object.entries(
+                rateLimitResult.headers,
+              )) {
+                headers.set(key, value);
+              }
+              return new Response(JSON.stringify({ status: "ok" }), {
+                status: 200,
+                headers,
+              });
+            }
+            if (url.pathname === "/health/ready") {
+              const readiness = readinessState.get();
+              const deployment = describeDeployment(config);
+              const headers = new Headers({ "Content-Type": "application/json" });
+              if (allowOriginHeader)
+                headers.set("Access-Control-Allow-Origin", allowOriginHeader);
+              return new Response(JSON.stringify({
+                ...readiness,
+                version: deployment.version,
+                runtime: deployment.runtime,
+                image_variant: deployment.image_variant,
+              }), {
+                status: readiness.status === "ready" ? 200 : 503,
+                headers,
+              });
+            }
+            if (url.pathname === "/metrics") {
+              const headers = new Headers({
+                "Content-Type": "text/plain; version=0.0.4",
+              });
+              if (allowOriginHeader) {
+                headers.set("Access-Control-Allow-Origin", allowOriginHeader);
+              }
+              for (const [key, value] of Object.entries(
+                rateLimitResult.headers,
+              )) {
+                headers.set(key, value);
+              }
+              return new Response(metricsRegistry.render(), {
+                status: 200,
+                headers,
+              });
+            }
+          }
+
+          // Token Auth Check
+          if (serverConfig.auth_enabled) {
+            const expectedToken = resolveAuthToken(serverConfig.auth_token_env);
+            if (!expectedToken) {
+              return new Response(
+                "Server authentication configured but token environment variable is empty",
+                { status: 500 },
+              );
+            }
+            const authHeader = req.headers.get("authorization") || "";
+            const token = authHeader.startsWith("Bearer ")
+              ? authHeader.slice(7)
+              : authHeader;
+            if (!token || !safeTokenEquals(token, expectedToken)) {
+              return new Response("Unauthorized", { status: 401 });
+            }
+          }
+
+          // GET /stats — placed after the Token Auth Check above (unlike /health and /metrics,
+          // which return earlier and stay open) since audit queries carry raw user text.
+          if (req.method === "GET" && url.pathname === "/stats") {
+            const since = url.searchParams.get("since") ?? "";
+            if (!SINCE_PATTERN.test(since)) {
+              return new Response(
+                JSON.stringify({
+                  error:
+                    "since must be a relative window (e.g. 30d) or an absolute ISO-8601 date",
+                }),
+                { status: 400, headers: { "Content-Type": "application/json" } },
+              );
+            }
+            const { auditDb } = await getRuntime();
             const headers = new Headers({ "Content-Type": "application/json" });
             if (allowOriginHeader)
               headers.set("Access-Control-Allow-Origin", allowOriginHeader);
-            return new Response(JSON.stringify({
-              ...readiness,
-              version: deployment.version,
-              runtime: deployment.runtime,
-              image_variant: deployment.image_variant,
-            }), {
-              status: readiness.status === "ready" ? 200 : 503,
-              headers,
-            });
-          }
-          if (url.pathname === "/metrics") {
-            const headers = new Headers({
-              "Content-Type": "text/plain; version=0.0.4",
-            });
-            if (allowOriginHeader) {
-              headers.set("Access-Control-Allow-Origin", allowOriginHeader);
-            }
-            for (const [key, value] of Object.entries(
-              rateLimitResult.headers,
-            )) {
+            for (const [key, value] of Object.entries(rateLimitResult.headers))
               headers.set(key, value);
-            }
-            return new Response(metricsRegistry.render(), {
+            return new Response(JSON.stringify(getStats(auditDb, since)), {
               status: 200,
               headers,
             });
           }
-        }
-
-        // Token Auth Check
-        if (serverConfig.auth_enabled) {
-          const expectedToken = resolveAuthToken(serverConfig.auth_token_env);
-          if (!expectedToken) {
-            return new Response(
-              "Server authentication configured but token environment variable is empty",
-              { status: 500 },
-            );
-          }
-          const authHeader = req.headers.get("authorization") || "";
-          const token = authHeader.startsWith("Bearer ")
-            ? authHeader.slice(7)
-            : authHeader;
-          if (!token || !safeTokenEquals(token, expectedToken)) {
-            return new Response("Unauthorized", { status: 401 });
-          }
-        }
-
-        // GET /stats — placed after the Token Auth Check above (unlike /health and /metrics,
-        // which return earlier and stay open) since audit queries carry raw user text.
-        if (req.method === "GET" && url.pathname === "/stats") {
-          const since = url.searchParams.get("since") ?? "";
-          if (!SINCE_PATTERN.test(since)) {
-            return new Response(
-              JSON.stringify({
-                error:
-                  "since must be a relative window (e.g. 30d) or an absolute ISO-8601 date",
-              }),
-              { status: 400, headers: { "Content-Type": "application/json" } },
-            );
-          }
-          const { auditDb } = await getRuntime();
-          const headers = new Headers({ "Content-Type": "application/json" });
-          if (allowOriginHeader)
-            headers.set("Access-Control-Allow-Origin", allowOriginHeader);
-          for (const [key, value] of Object.entries(rateLimitResult.headers))
-            headers.set(key, value);
-          return new Response(JSON.stringify(getStats(auditDb, since)), {
-            status: 200,
-            headers,
-          });
-        }
-
-        // Admin HTTP API (/admin/v1/*)
-        if (url.pathname.startsWith("/admin/v1/")) {
-          if (!serverConfig.admin?.enabled) {
-            return new Response("Admin endpoints disabled", { status: 403 });
-          }
 
-          const adminTokenEnv =
-            serverConfig.admin.token_env || "SKILLMUX_ADMIN_TOKEN";
-          const expectedAdminToken = process.env[adminTokenEnv] || "";
-          const authHeader = req.headers.get("authorization") || "";
-          const token = authHeader.startsWith("Bearer ")
-            ? authHeader.slice(7)
-            : authHeader;
-
-          if (
-            !expectedAdminToken ||
-            !token ||
-            !safeTokenEquals(token, expectedAdminToken)
-          ) {
-            return new Response("Unauthorized", { status: 401 });
-          }
-
-          const headers = new Headers({ "Content-Type": "application/json" });
-          if (allowOriginHeader)
-            headers.set("Access-Control-Allow-Origin", allowOriginHeader);
+          // Admin HTTP API (/admin/v1/*)
+          if (url.pathname.startsWith("/admin/v1/")) {
+            if (!serverConfig.admin?.enabled) {
+              return new Response("Admin endpoints disabled", { status: 403 });
+            }
 
-          if (
-            req.method === "GET" &&
-            url.pathname === "/admin/v1/capabilities"
-          ) {
-            const isExternallyManaged =
-              process.env.SKILLMUX_CONFIG_READONLY === "true";
-            return new Response(
-              JSON.stringify({
-                config_read: true,
-                config_write: !isExternallyManaged,
-                persistence: isExternallyManaged
-                  ? "externally_managed"
-                  : "writable",
-                reloadable_keys: RELOADABLE_KEYS,
-                restart_required_keys: RESTART_REQUIRED_KEYS,
-              }),
-              { status: 200, headers },
-            );
-          }
+            const adminTokenEnv =
+              serverConfig.admin.token_env || "SKILLMUX_ADMIN_TOKEN";
+            const expectedAdminToken = process.env[adminTokenEnv] || "";
+            const authHeader = req.headers.get("authorization") || "";
+            const token = authHeader.startsWith("Bearer ")
+              ? authHeader.slice(7)
+              : authHeader;
+
+            if (
+              !expectedAdminToken ||
+              !token ||
+              !safeTokenEquals(token, expectedAdminToken)
+            ) {
+              return new Response("Unauthorized", { status: 401 });
+            }
 
-          if (req.method === "GET" && url.pathname === "/admin/v1/config") {
-            const { effective, sources } = await getEffectiveConfig(configPath);
-            const deployment = describeDeployment(config);
-            const desiredHash = computeHash(effective);
-            const snapshot = snapshots.acquire();
-            const activeRevision = computeHash(snapshot.snapshot.config);
-            snapshot.release();
-            const status =
-              configWatcher?.reloadStatus() ?? inactiveReloadStatus;
-            headers.set("ETag", `"${desiredHash}"`);
-            return new Response(
-              JSON.stringify({
-                desired: effective,
-                effective,
-                sources,
-                active_revision: activeRevision,
-                runtime: {
-                  target: "local",
-                  desired_source: configPath,
-                  desired_source_hash: desiredHash,
-                  active_revision: activeRevision,
-                  active_source_hash: activeRevision,
-                  ...status,
-                  readiness: readinessState.get(),
-                  runtime: "running",
-                  version: deployment.version,
-                  deployment_runtime: deployment.runtime,
-                  image_variant: deployment.image_variant,
-                },
-              }),
-              { status: 200, headers },
-            );
-          }
+            const headers = new Headers({ "Content-Type": "application/json" });
+            if (allowOriginHeader)
+              headers.set("Access-Control-Allow-Origin", allowOriginHeader);
 
-          if (req.method === "PATCH" && url.pathname === "/admin/v1/config") {
-            if (process.env.SKILLMUX_CONFIG_READONLY === "true") {
+            if (
+              req.method === "GET" &&
+              url.pathname === "/admin/v1/capabilities"
+            ) {
+              const isExternallyManaged =
+                process.env.SKILLMUX_CONFIG_READONLY === "true";
               return new Response(
                 JSON.stringify({
-                  error: "CONFIG_EXTERNALLY_MANAGED",
-                  message: "Configuration is externally managed",
+                  config_read: true,
+                  config_write: !isExternallyManaged,
+                  persistence: isExternallyManaged
+                    ? "externally_managed"
+                    : "writable",
+                  reloadable_keys: RELOADABLE_KEYS,
+                  restart_required_keys: RESTART_REQUIRED_KEYS,
                 }),
-                { status: 409, headers },
+                { status: 200, headers },
               );
             }
 
-            const ifMatch = req.headers.get("if-match") || "";
-            const cleanIfMatch = ifMatch.replace(/^"|"$/g, "");
-            const { effective } = await getEffectiveConfig(configPath);
-            const currentHash = computeHash(effective);
-
-            if (!ifMatch || cleanIfMatch !== currentHash) {
+            if (req.method === "GET" && url.pathname === "/admin/v1/config") {
+              const { effective, sources } = await getEffectiveConfig(configPath);
+              const deployment = describeDeployment(config);
+              const desiredHash = computeHash(effective);
+              const snapshot = snapshots.acquire();
+              const activeRevision = computeHash(snapshot.snapshot.config);
+              snapshot.release();
+              const status =
+                configWatcher?.reloadStatus() ?? inactiveReloadStatus;
+              headers.set("ETag", `"${desiredHash}"`);
               return new Response(
                 JSON.stringify({
-                  error: "CONFIG_REVISION_CONFLICT",
-                  message: "Revision conflict",
+                  desired: effective,
+                  effective,
+                  sources,
+                  active_revision: activeRevision,
+                  runtime: {
+                    target: "local",
+                    desired_source: configPath,
+                    desired_source_hash: desiredHash,
+                    active_revision: activeRevision,
+                    active_source_hash: activeRevision,
+                    ...status,
+                    readiness: readinessState.get(),
+                    runtime: "running",
+                    version: deployment.version,
+                    deployment_runtime: deployment.runtime,
+                    image_variant: deployment.image_variant,
+                  },
                 }),
-                { status: 409, headers },
+                { status: 200, headers },
               );
             }
 
-            const body = (await req.json()) as {
-              changes: Record<string, string | number | boolean>;
-            };
-            let lastResult: any = null;
-            for (const [k, v] of Object.entries(body.changes ?? {})) {
-              lastResult = await setDottedKey(k, String(v), {
-                targetName: "remote",
+            if (req.method === "PATCH" && url.pathname === "/admin/v1/config") {
+              if (process.env.SKILLMUX_CONFIG_READONLY === "true") {
+                return new Response(
+                  JSON.stringify({
+                    error: "CONFIG_EXTERNALLY_MANAGED",
+                    message: "Configuration is externally managed",
+                  }),
+                  { status: 409, headers },
+                );
+              }
+
+              const ifMatch = req.headers.get("if-match") || "";
+              const cleanIfMatch = ifMatch.replace(/^"|"$/g, "");
+              const { effective } = await getEffectiveConfig(configPath);
+              const currentHash = computeHash(effective);
+
+              if (!ifMatch || cleanIfMatch !== currentHash) {
+                return new Response(
+                  JSON.stringify({
+                    error: "CONFIG_REVISION_CONFLICT",
+                    message: "Revision conflict",
+                  }),
+                  { status: 409, headers },
+                );
+              }
+
+              const body = (await req.json()) as {
+                changes: Record<string, string | number | boolean>;
+              };
+              let lastResult: any = null;
+              const auditChanges: AdminAuditChange[] = [];
+              for (const [k, v] of Object.entries(body.changes ?? {})) {
+                lastResult = await setDottedKey(k, String(v), {
+                  targetName: "remote",
+                });
+                auditChanges.push({
+                  key: k,
+                  old_value: lastResult.prior_val,
+                  new_value: lastResult.resulting_val,
+                });
+              }
+
+              if (auditChanges.length > 0 && lastResult) {
+                const { auditDb } = await getRuntime();
+                insertAdminAuditRow(auditDb, {
+                  ts: new Date().toISOString(),
+                  changes: auditChanges,
+                  resulting_revision: lastResult.resulting_revision,
+                });
+              }
+
+              return new Response(JSON.stringify(lastResult ?? { ok: true }), {
+                status: 200,
+                headers,
               });
             }
 
-            return new Response(JSON.stringify(lastResult ?? { ok: true }), {
-              status: 200,
-              headers,
-            });
+            return new Response("Not Found", { status: 404, headers });
           }
 
-          return new Response("Not Found", { status: 404, headers });
-        }
-
-        // Record request metrics
-        let mcpMethod = "unknown";
-        try {
-          const bodyClone = await req.clone().json();
-          if (bodyClone.method === "tools/call") {
-            mcpMethod = bodyClone.params?.name || "tools/call";
-          } else {
-            mcpMethod = bodyClone.method || "unknown";
+          // Record request metrics
+          let mcpMethod = "unknown";
+          try {
+            const bodyClone = await req.clone().json();
+            if (bodyClone.method === "tools/call") {
+              mcpMethod = bodyClone.params?.name || "tools/call";
+            } else {
+              mcpMethod = bodyClone.method || "unknown";
+            }
+          } catch {
+            // Non-JSON or parsing error
           }
-        } catch {
-          // Non-JSON or parsing error
-        }
-        metricsRegistry.recordRequest(mcpMethod);
+          metricsRegistry.recordRequest(mcpMethod);
 
-        const res = await transport.handleRequest(req);
-        const headers = new Headers(res.headers);
-        if (allowOriginHeader) {
-          headers.set("Access-Control-Allow-Origin", allowOriginHeader);
-        }
-        for (const [key, value] of Object.entries(rateLimitResult.headers)) {
-          headers.set(key, value);
+          const res = await transport.handleRequest(req);
+          const headers = new Headers(res.headers);
+          if (allowOriginHeader) {
+            headers.set("Access-Control-Allow-Origin", allowOriginHeader);
+          }
+          for (const [key, value] of Object.entries(rateLimitResult.headers)) {
+            headers.set(key, value);
+          }
+          return new Response(res.body, {
+            status: res.status,
+            statusText: res.statusText,
+            headers,
+          });
         }
-        return new Response(res.body, {
-          status: res.status,
-          statusText: res.statusText,
-          headers,
-        });
       },
     });
     let stopped = false;
@@ -529,6 +738,7 @@ export async function startServer(opts?: {
     console.log(`skillmux serving over HTTP on ${hostname}:${bunServer.port}`);
     return {
       port: bunServer.port,
+      statsPort: statsHandle?.port,
       reloadStatus: () =>
         configWatcher?.reloadStatus() ?? { ...inactiveReloadStatus },
       async stop() {
@@ -538,6 +748,7 @@ export async function startServer(opts?: {
         readinessState.set({ ...readinessState.get(), status: "stopping" });
         metricsRegistry.setReadiness(readinessState.get());
         bunServer.stop(true);
+        statsHandle?.stop();
         configWatcher?.stop();
         stopWatcher();
         snapshots.dispose();
@@ -549,6 +760,7 @@ export async function startServer(opts?: {
     await server.connect(new StdioServerTransport());
     let stopped = false;
     return {
+      statsPort: statsHandle?.port,
       reloadStatus: () =>
         configWatcher?.reloadStatus() ?? { ...inactiveReloadStatus },
       async stop() {
@@ -557,6 +769,7 @@ export async function startServer(opts?: {
         clearInterval(auditPruneInterval);
         readinessState.set({ ...readinessState.get(), status: "stopping" });
         metricsRegistry.setReadiness(readinessState.get());
+        statsHandle?.stop();
         configWatcher?.stop();
         stopWatcher();
         snapshots.dispose();