@lifeaitools/clauth 1.30.14 → 1.30.16

package/cli/commands/serve.js

@@ -41,8 +41,9 @@ import {
   listTunnels,
   readSupervisorEvents,
   addTunnelRoute,
-  removeTunnelRoute,
-  runPluginAction,
+  removeTunnelRoute,
+  reconcileSurfaceHealth,
+  runPluginAction,
   runSurfaceAction,
   setPluginEnabled,
   supervisorHealth,
@@ -7819,7 +7820,7 @@ function createServer(initPassword, whitelist, port, tunnelHostnameInit = null,
 
   // If vault is already unlocked at startup (auto-unlock from boot.key skips POST /unlock),
   // mirror the tunnel-start logic that the unlock handler would have run.
-  if (initPassword) {
+  if (initPassword) {
     setImmediate(() => {
       if (tunnelHostname) {
         tunnelStatus = "starting";
@@ -7835,11 +7836,32 @@ function createServer(initPassword, whitelist, port, tunnelHostnameInit = null,
           }
         }).catch(() => { tunnelStatus = "error"; });
       }
-    });
-  }
-
-  return server;
-}
+    });
+  }
+
+  // The localhost supervisor is the only process allowed to repair clauth-owned
+  // local surfaces. Keep this loop out of the vault/staged instances and make
+  // the cadence configurable for deterministic tests.
+  if (port === getSupervisorPort() && process.env.CLAUTH_SUPERVISOR_HEALTH_RECONCILE !== "0") {
+    const configuredInterval = Number(process.env.CLAUTH_SUPERVISOR_HEALTH_INTERVAL_MS || 10000);
+    const intervalMs = Number.isFinite(configuredInterval) ? Math.max(1000, Math.min(configuredInterval, 300000)) : 10000;
+    let healthReconcileInFlight = false;
+    const runHealthReconcile = () => {
+      if (healthReconcileInFlight) return;
+      healthReconcileInFlight = true;
+      reconcileSurfaceHealth().catch((err) => {
+        try { fs.appendFileSync(LOG_FILE, `[${new Date().toISOString()}] supervisor health reconcile failed: ${err.message}\n`); } catch {}
+      }).finally(() => { healthReconcileInFlight = false; });
+    };
+    const healthTimer = setInterval(runHealthReconcile, intervalMs);
+    healthTimer.unref?.();
+    server.__supervisorHealthTimer = healthTimer;
+    server.on("close", () => clearInterval(healthTimer));
+    setImmediate(runHealthReconcile);
+  }
+
+  return server;
+}
 
 // ── Actions ──────────────────────────────────────────────────
 

package/cli/supervisor-registry.js

@@ -10,6 +10,9 @@ const DEFAULT_SUPERVISOR_PORT = 52439;
 const DESTINATIONS = new Set(["local/clauth/pm2", "vultr/clauth/pm2", "coolify/clauth/docker"]);
 const OWNERS = new Set(["clauth", "plugin", "external"]);
 const ACTIONS = new Set(["start", "stop", "restart", "reconcile", "test", "promote", "rollback"]);
+const DEFAULT_HEALTH_RECONCILE_INTERVAL_MS = 10000;
+const DEFAULT_HEALTH_TIMEOUT_MS = 2500;
+const HEALTH_RECONCILE_COOLDOWN_MS = 15000;
 
 export function getSupervisorPort() {
   return Number(process.env.CLAUTH_SUPERVISOR_PORT || DEFAULT_SUPERVISOR_PORT);
@@ -57,6 +60,100 @@ function now() {
   return new Date().toISOString();
 }
 
+function healthUrlForSurface(surface) {
+  if (!surface?.health) return null;
+  if (/^https?:\/\//i.test(surface.health)) return surface.health;
+  if (!surface.port || surface.port === "auto") return null;
+  return `http://127.0.0.1:${surface.port}${String(surface.health).startsWith("/") ? surface.health : `/${surface.health}`}`;
+}
+
+function updateSurfaceState(surfaceId, patch) {
+  const state = loadSupervisorState();
+  state.surfaces = (state.surfaces || []).map((surface) => (
+    `${surface.plugin_id}:${surface.id}` === surfaceId || surface.id === surfaceId
+      ? { ...surface, ...patch }
+      : surface
+  ));
+  saveSupervisorState(state);
+  return state.surfaces.find((surface) => `${surface.plugin_id}:${surface.id}` === surfaceId || surface.id === surfaceId) || null;
+}
+
+function appendSupervisorEvent(event) {
+  appendJsonl(file("events.jsonl"), { ts: now(), ...event });
+}
+
+async function probeSurfaceHealth(url, fetchImpl, timeoutMs) {
+  try {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      const response = await fetchImpl(url, { signal: controller.signal });
+      return response?.ok
+        ? { healthy: true, error: null }
+        : { healthy: false, error: `HTTP ${response?.status ?? "unknown"}` };
+    } finally {
+      clearTimeout(timer);
+    }
+  } catch (err) {
+    return { healthy: false, error: err?.name === "AbortError" ? "health timeout" : String(err?.message || err) };
+  }
+}
+
+/**
+ * Probe enabled local clauth-owned surfaces and repair an unavailable one via
+ * the same governed action path exposed to Dev Center. External/plugin-owned
+ * surfaces are intentionally observe-only and are never restarted here.
+ */
+export async function reconcileSurfaceHealth({ fetchImpl = globalThis.fetch, timeoutMs = DEFAULT_HEALTH_TIMEOUT_MS } = {}) {
+  const inspected = [];
+  for (const surface of listSurfaces()) {
+    if (!surface.enabled || surface.lifecycle_owner !== "clauth" || surface.destination !== "local/clauth/pm2") continue;
+    const id = `${surface.plugin_id}:${surface.id}`;
+    const url = healthUrlForSurface(surface);
+    if (!url) continue;
+    const observedAt = now();
+    const health = await probeSurfaceHealth(url, fetchImpl, timeoutMs);
+    const healthy = health.healthy;
+    const error = health.error;
+
+    if (healthy) {
+      updateSurfaceState(id, { state: "current", last_health_at: observedAt, last_health_ok: true, last_health_error: null });
+      inspected.push({ surface_id: id, state: "healthy", observed_at: observedAt });
+      continue;
+    }
+
+    const lastAttempt = Date.parse(surface.last_reconcile_at || "") || 0;
+    if (Date.now() - lastAttempt < HEALTH_RECONCILE_COOLDOWN_MS) {
+      updateSurfaceState(id, { state: "unavailable", last_health_at: observedAt, last_health_ok: false, last_health_error: error });
+      inspected.push({ surface_id: id, state: "unavailable", error, cooldown: true, observed_at: observedAt });
+      continue;
+    }
+
+    updateSurfaceState(id, {
+      state: "unavailable",
+      last_health_at: observedAt,
+      last_health_ok: false,
+      last_health_error: error,
+      last_reconcile_at: observedAt,
+    });
+    appendSupervisorEvent({ kind: "surface_health_failed", surface_id: id, error, health: url });
+    const receipt = runSurfaceAction(id, "reconcile", "supervisor-health-loop");
+    const commandCompleted = receipt?.resulting_state?.ok === true;
+    const postHealth = commandCompleted ? await probeSurfaceHealth(url, fetchImpl, timeoutMs) : { healthy: false, error: receipt?.resulting_state?.state || "reconcile_failed" };
+    const repaired = commandCompleted && postHealth.healthy;
+    updateSurfaceState(id, {
+      state: repaired ? "current" : "unavailable",
+      last_health_at: now(),
+      last_health_ok: repaired,
+      last_health_error: repaired ? null : postHealth.error,
+      last_reconcile_operation_id: receipt?.operationId || null,
+    });
+    appendSupervisorEvent({ kind: repaired ? "surface_reconciled" : "surface_reconcile_failed", surface_id: id, operation_id: receipt?.operationId || null, command_completed: commandCompleted, health_ok: postHealth.healthy, error: postHealth.error || null });
+    inspected.push({ surface_id: id, state: repaired ? "reconciled" : "reconcile_failed", error: postHealth.error || error, operation_id: receipt?.operationId || null, observed_at: observedAt });
+  }
+  return { inspected };
+}
+
 function normalizeCommand(command, field) {
   if (!command) return [];
   if (!Array.isArray(command)) throw new Error(`${field} must be a command array`);

package/cli/supervisor-registry.test.js

@@ -10,6 +10,7 @@ import {
   getClauthPm2Home,
   listPlugins,
   listSurfaces,
+  reconcileSurfaceHealth,
   runPluginAction,
   runSurfaceAction,
   removeTunnelRoute,
@@ -195,6 +196,110 @@ test("surface promote and rollback never fall through to restart commands", () =
   }
 }));
 
+test("health reconciliation marks a failed clauth surface and repairs it through reconcile", async () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "clauth-supervisor-health-"));
+  const oldDir = process.env.CLAUTH_SUPERVISOR_DIR;
+  const oldManaged = process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+  const oldUser = process.env.CLAUTH_USER_PLUGIN_ROOTS;
+  process.env.CLAUTH_SUPERVISOR_DIR = root;
+  process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = path.join(root, "managed");
+  process.env.CLAUTH_USER_PLUGIN_ROOTS = path.join(root, "user");
+  try {
+    writePlugin(root, "managed", "health-demo", baseManifest("health-demo", {
+      core: true,
+      enable_default: true,
+      surfaces: [{
+        id: "primary",
+        destination: "local/clauth/pm2",
+        lifecycle_owner: "clauth",
+        port: 39111,
+        health: "/health",
+        restart: [process.execPath, "--version"],
+      }],
+    }));
+    discoverPlugins();
+    let healthCalls = 0;
+    const result = await reconcileSurfaceHealth({
+      fetchImpl: async () => ({ ok: ++healthCalls > 1, status: 503 }),
+    });
+    assert.equal(result.inspected[0].surface_id, "health-demo:primary");
+    assert.equal(result.inspected[0].state, "reconciled");
+    const surface = listSurfaces().find((item) => item.plugin_id === "health-demo");
+    assert.equal(surface.state, "current");
+    assert.ok(surface.last_reconcile_operation_id);
+    assert.equal(healthCalls, 2);
+  } finally {
+    if (oldDir === undefined) delete process.env.CLAUTH_SUPERVISOR_DIR;
+    else process.env.CLAUTH_SUPERVISOR_DIR = oldDir;
+    if (oldManaged === undefined) delete process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+    else process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = oldManaged;
+    if (oldUser === undefined) delete process.env.CLAUTH_USER_PLUGIN_ROOTS;
+    else process.env.CLAUTH_USER_PLUGIN_ROOTS = oldUser;
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("health reconciliation does not claim current when a successful command leaves health down", async () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "clauth-supervisor-post-health-"));
+  const oldDir = process.env.CLAUTH_SUPERVISOR_DIR;
+  const oldManaged = process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+  const oldUser = process.env.CLAUTH_USER_PLUGIN_ROOTS;
+  process.env.CLAUTH_SUPERVISOR_DIR = root;
+  process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = path.join(root, "managed");
+  process.env.CLAUTH_USER_PLUGIN_ROOTS = path.join(root, "user");
+  try {
+    writePlugin(root, "managed", "post-health-demo", baseManifest("post-health-demo", {
+      core: true,
+      enable_default: true,
+      surfaces: [{ id: "primary", destination: "local/clauth/pm2", lifecycle_owner: "clauth", port: 39113, health: "/health", restart: [process.execPath, "--version"] }],
+    }));
+    discoverPlugins();
+    const result = await reconcileSurfaceHealth({ fetchImpl: async () => ({ ok: false, status: 503 }) });
+    assert.equal(result.inspected[0].state, "reconcile_failed");
+    assert.equal(listSurfaces()[0].state, "unavailable");
+    assert.equal(listSurfaces()[0].last_health_ok, false);
+  } finally {
+    if (oldDir === undefined) delete process.env.CLAUTH_SUPERVISOR_DIR;
+    else process.env.CLAUTH_SUPERVISOR_DIR = oldDir;
+    if (oldManaged === undefined) delete process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+    else process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = oldManaged;
+    if (oldUser === undefined) delete process.env.CLAUTH_USER_PLUGIN_ROOTS;
+    else process.env.CLAUTH_USER_PLUGIN_ROOTS = oldUser;
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("health reconciliation never restarts external or plugin-owned surfaces", async () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "clauth-supervisor-observe-"));
+  const oldDir = process.env.CLAUTH_SUPERVISOR_DIR;
+  const oldManaged = process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+  const oldUser = process.env.CLAUTH_USER_PLUGIN_ROOTS;
+  process.env.CLAUTH_SUPERVISOR_DIR = root;
+  process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = path.join(root, "managed");
+  process.env.CLAUTH_USER_PLUGIN_ROOTS = path.join(root, "user");
+  try {
+    writePlugin(root, "managed", "external-demo", baseManifest("external-demo", {
+      core: true,
+      enable_default: true,
+      destination: "vultr/clauth/pm2",
+      lifecycle_owner: "external",
+      surfaces: [{ id: "primary", destination: "vultr/clauth/pm2", lifecycle_owner: "external", port: 39112, health: "/health" }],
+    }));
+    discoverPlugins();
+    const result = await reconcileSurfaceHealth({ fetchImpl: async () => ({ ok: false, status: 503 }) });
+    assert.deepEqual(result.inspected, []);
+    assert.equal(listSurfaces()[0].state, "current");
+  } finally {
+    if (oldDir === undefined) delete process.env.CLAUTH_SUPERVISOR_DIR;
+    else process.env.CLAUTH_SUPERVISOR_DIR = oldDir;
+    if (oldManaged === undefined) delete process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+    else process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = oldManaged;
+    if (oldUser === undefined) delete process.env.CLAUTH_USER_PLUGIN_ROOTS;
+    else process.env.CLAUTH_USER_PLUGIN_ROOTS = oldUser;
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
 test("supervisor write-token policy is temporarily relaxed only for the localhost supervisor port", () => {
   assert.equal(supervisorRequiresWriteToken(52439, {}), false);
   assert.equal(supervisorRequiresWriteToken(52439, { CLAUTH_SUPERVISOR_REQUIRE_WRITE_TOKEN: "1" }), true);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lifeaitools/clauth",
-  "version": "1.30.14",
+  "version": "1.30.16",
   "description": "Hardware-bound credential vault for the LIFEAI infrastructure stack",
   "type": "module",
   "bin": {