@lifeaitools/clauth 2.15.5 → 2.15.6

package/cli/dashboard/panels/surfaces-panel.css

@@ -32,6 +32,15 @@
 .supervisor-pill.ok{border-color:rgba(74,222,128,.35);color:var(--green-light);background:rgba(34,197,94,.08)}
 .supervisor-pill.warn{border-color:rgba(250,204,21,.35);color:var(--gold-light);background:rgba(250,204,21,.08)}
 .supervisor-pill.bad{border-color:rgba(248,113,113,.35);color:var(--pink);background:rgba(248,113,113,.08)}
+/* NEEDS HELP -- a surface that exhausted MAX_CONSECUTIVE_RECONCILE_FAILURES
+   and clauth has stopped auto-restarting. Distinct from .bad ("down right
+   now, clauth is still retrying"): this is "clauth gave up, a human/agent
+   has to look." Pulses so it reads as an alarm among a fleet that regularly
+   has a few surfaces transiently down in .bad. */
+.supervisor-pill.critical{border-color:rgba(248,113,113,.6);color:#fff;background:rgba(220,38,38,.55);animation:supervisor-pulse 1.4s ease-in-out infinite}
+.supervisor-row.needs-help{border-color:rgba(220,38,38,.6);box-shadow:0 0 0 1px rgba(220,38,38,.35) inset}
+.supervisor-meta.critical{color:var(--pink);white-space:normal}
+@keyframes supervisor-pulse{0%,100%{opacity:1}50%{opacity:.55}}
 /* Card GRID, not a stack. flex-direction:column gave one full-width row per
    surface, so ten surfaces meant ten rows and constant scrolling to see the
    fleet. auto-fill + minmax packs as many small cards per row as the panel

package/cli/dashboard/panels/surfaces-panel.js

@@ -305,7 +305,15 @@ class ClauthSurfacesPanel extends window.ClauthPanelElement {
   renderSupervisorSurface(surface) {
     const compositeId = surface.plugin_id + ":" + surface.id;
     const ownerKind = surface.lifecycle_owner === "clauth" ? "ok" : (surface.lifecycle_owner === "plugin" ? "warn" : "");
-    const stateKind = surface.state === "current" || surface.status === "healthy" ? "ok" : (surface.state === "unavailable" ? "bad" : "warn");
+    // "needs_help" is a DIFFERENT alarm than "bad": bad means "down right now,
+    // clauth is still retrying"; needs_help means "clauth gave up after
+    // MAX_CONSECUTIVE_RECONCILE_FAILURES — a human/agent has to look." It gets
+    // its own loud, pulsing pill (see .supervisor-pill.critical) rather than
+    // reusing "bad", which would render as just another red flicker in a fleet
+    // that regularly has a few surfaces transiently down.
+    const needsHelp = surface.state === "needs_help";
+    const stateKind = needsHelp ? "critical" : (surface.state === "current" || surface.status === "healthy" ? "ok" : (surface.state === "unavailable" ? "bad" : "warn"));
+    const stateLabel = needsHelp ? "NEEDS HELP" : (surface.state || surface.status || "unknown");
     const openUrl = surface.__pseudo ? null : openUiUrlForSurface(surface);
     const isSelected = this.selectedSupervisorSurface && this.selectedSupervisorSurface.id === compositeId;
     const rowClass = "supervisor-row" + (surface.__pseudo ? " readonly" : "") + (isSelected ? " selected" : "");
@@ -339,14 +347,22 @@ class ClauthSurfacesPanel extends window.ClauthPanelElement {
       : (surface.last_health_ok === false && surface.last_health_error
         ? surface.last_health_error + " · " + (surface.health || "no health")
         : (surface.health || surface.health_url || "no health"));
-    return '<div class="' + rowClass + '" data-surface-id="' + htmlEscape(compositeId) + '"' + rowAction + '>' +
+    // The loud line: WHY it needs help and WHAT to do, spelled out on the
+    // card itself — not just a pill label a reader has to already know the
+    // meaning of. Mirrors the same wording reconcileSurfaceHealth() writes
+    // into the daemon log, so the card and the log never disagree.
+    const needsHelpLine = needsHelp
+      ? "Stopped auto-restarting after " + (surface.consecutive_reconcile_failures || "several") + " failed attempts — check the log, fix the cause, then Start/Restart to resume auto-repair."
+      : null;
+    return '<div class="' + rowClass + (needsHelp ? " needs-help" : "") + '" data-surface-id="' + htmlEscape(compositeId) + '"' + rowAction + '>' +
       '<div class="supervisor-row-top"><span class="supervisor-name">' + htmlEscape(surface.name || compositeId) + '</span>' +
       healthPill +
-      supervisorBadge(surface.lifecycle_owner || "unknown", ownerKind) + supervisorBadge(surface.state || surface.status || "unknown", stateKind) +
+      supervisorBadge(surface.lifecycle_owner || "unknown", ownerKind) + supervisorBadge(stateLabel, stateKind) +
       openBtn +
       '</div>' +
       '<div class="supervisor-meta">' + htmlEscape(surface.destination || "—") + ' · port ' + htmlEscape(surface.port || "—") + '</div>' +
       '<div class="supervisor-meta">' + htmlEscape(healthLine) + '</div>' +
+      (needsHelpLine ? '<div class="supervisor-meta critical">' + htmlEscape(needsHelpLine) + '</div>' : '') +
       '</div>';
   }
 

package/cli/supervisor-registry.js

@@ -18,6 +18,22 @@ const ACTIONS = new Set(["start", "stop", "restart", "reconcile", "test", "promo
 const DEFAULT_HEALTH_RECONCILE_INTERVAL_MS = 10000;
 const DEFAULT_HEALTH_TIMEOUT_MS = 2500;
 const HEALTH_RECONCILE_COOLDOWN_MS = 15000;
+// After this many CONSECUTIVE failed reconcile attempts (each gated by
+// HEALTH_RECONCILE_COOLDOWN_MS, so >= 5 * 15s = 75s before this fires),
+// reconcileSurfaceHealth() stops auto-restarting the surface and latches it
+// into "needs_help" instead of retrying forever. Without a cap, a surface
+// that can never come up (a missing dependency, a bad config) gets
+// relaunched every cooldown window FOREVER: the restart command itself can
+// exit 0 (pm2 accepted it) while the process crashes immediately after, so
+// this loop never sees a hard failure to stop on, and pm2's OWN crash-loop
+// backoff never engages either because clauth's periodic explicit restart
+// keeps resetting it. Root cause of the regen-media-local window-flashing
+// incident (2026-09-05): a missing `tsx` dependency meant every restart
+// "succeeded" and immediately crashed, forever, because nothing was
+// counting. A latched surface self-heals the moment its health probe
+// succeeds (handled before this cap is ever consulted) or a human/agent
+// issues an explicit non-reconcile start/stop/restart (see runSurfaceAction).
+export const MAX_CONSECUTIVE_RECONCILE_FAILURES = 5;
 const DOCUMENTATION_FIELDS = ["architecture", "operator_guide", "install", "runbook", "tool_reference", "release", "agent_context"];
 
 function hasMcpToken(value) {
@@ -182,11 +198,25 @@ export async function reconcileSurfaceHealth({ fetchImpl = globalThis.fetch, tim
     const error = health.error;
 
     if (healthy) {
-      updateSurfaceState(id, { state: "current", last_health_at: observedAt, last_health_ok: true, last_health_error: null });
+      updateSurfaceState(id, { state: "current", last_health_at: observedAt, last_health_ok: true, last_health_error: null, consecutive_reconcile_failures: 0 });
       inspected.push({ surface_id: id, state: "healthy", observed_at: observedAt });
       continue;
     }
 
+    // Latched: this surface already exhausted MAX_CONSECUTIVE_RECONCILE_FAILURES
+    // and clauth stopped auto-restarting it. Record the failed probe (so the
+    // UI's "last checked" freshness stays honest) WITHOUT calling
+    // runSurfaceAction again — that call is exactly the crash loop this latch
+    // exists to stop. The only ways out are a health probe that succeeds
+    // (the branch above, checked every tick regardless of latch state) or an
+    // explicit human/agent action (runSurfaceAction resets the counter and
+    // clears this latch on any successful non-reconcile command).
+    if (surface.state === "needs_help") {
+      updateSurfaceState(id, { last_health_at: observedAt, last_health_ok: false, last_health_error: error });
+      inspected.push({ surface_id: id, state: "needs_help", error, latched: true, observed_at: observedAt });
+      continue;
+    }
+
     const lastAttempt = Date.parse(surface.last_reconcile_at || "") || 0;
     if (Date.now() - lastAttempt < HEALTH_RECONCILE_COOLDOWN_MS) {
       updateSurfaceState(id, { state: "unavailable", last_health_at: observedAt, last_health_ok: false, last_health_error: error });
@@ -206,15 +236,52 @@ export async function reconcileSurfaceHealth({ fetchImpl = globalThis.fetch, tim
     const commandCompleted = receipt?.resulting_state?.ok === true;
     const postHealth = commandCompleted ? await probeSurfaceHealth(url, fetchImpl, timeoutMs) : { healthy: false, error: receipt?.resulting_state?.state || "reconcile_failed" };
     const repaired = commandCompleted && postHealth.healthy;
+
+    if (repaired) {
+      updateSurfaceState(id, {
+        state: "current",
+        last_health_at: now(),
+        last_health_ok: true,
+        last_health_error: null,
+        last_reconcile_operation_id: receipt?.operationId || null,
+        consecutive_reconcile_failures: 0,
+      });
+      appendSupervisorEvent({ kind: "surface_reconciled", surface_id: id, operation_id: receipt?.operationId || null, command_completed: commandCompleted, health_ok: true, error: null });
+      inspected.push({ surface_id: id, state: "reconciled", error: null, operation_id: receipt?.operationId || null, observed_at: observedAt });
+      continue;
+    }
+
+    // Not repaired: count it. MAX_CONSECUTIVE_RECONCILE_FAILURES is the
+    // circuit breaker — cross it and the surface latches to "needs_help"
+    // instead of feeding another attempt back into runSurfaceAction next tick.
+    const failures = (surface.consecutive_reconcile_failures || 0) + 1;
+    const exhausted = failures >= MAX_CONSECUTIVE_RECONCILE_FAILURES;
     updateSurfaceState(id, {
-      state: repaired ? "current" : "unavailable",
+      state: exhausted ? "needs_help" : "unavailable",
       last_health_at: now(),
-      last_health_ok: repaired,
-      last_health_error: repaired ? null : postHealth.error,
+      last_health_ok: false,
+      last_health_error: postHealth.error || error,
       last_reconcile_operation_id: receipt?.operationId || null,
+      consecutive_reconcile_failures: failures,
+    });
+    appendSupervisorEvent({
+      kind: exhausted ? "surface_needs_help" : "surface_reconcile_failed",
+      surface_id: id,
+      operation_id: receipt?.operationId || null,
+      command_completed: commandCompleted,
+      health_ok: false,
+      error: postHealth.error || error,
+      consecutive_failures: failures,
+      ...(exhausted ? { message: `${id} failed to come up after ${failures} consecutive restart attempts — clauth has stopped auto-restarting it. Check its log, fix the underlying cause, then Start/Restart it manually (dashboard or \`clauth mcp restart ${id}\`) to resume auto-repair.` } : {}),
+    });
+    inspected.push({
+      surface_id: id,
+      state: exhausted ? "needs_help" : "reconcile_failed",
+      error: postHealth.error || error,
+      operation_id: receipt?.operationId || null,
+      consecutive_failures: failures,
+      observed_at: observedAt,
     });
-    appendSupervisorEvent({ kind: repaired ? "surface_reconciled" : "surface_reconcile_failed", surface_id: id, operation_id: receipt?.operationId || null, command_completed: commandCompleted, health_ok: postHealth.healthy, error: postHealth.error || null });
-    inspected.push({ surface_id: id, state: repaired ? "reconciled" : "reconcile_failed", error: postHealth.error || error, operation_id: receipt?.operationId || null, observed_at: observedAt });
   }
   return { inspected };
 }
@@ -1245,6 +1312,22 @@ export function runSurfaceAction(id, action, actor = "localhost") {
     fallbackUsed = true;
     result.stderr = `restart exited ${restartStatus}; start fallback attempted\n${result.stderr || ""}`;
   }
+  // A deliberate, successful manual action — start/stop/restart, never
+  // "reconcile" itself — is a human or agent saying "I looked at this."
+  // Give the surface a fresh crash-loop budget instead of carrying forward a
+  // counter (or a "needs_help" latch) from before the intervention. Only
+  // reconcileSurfaceHealth's own automatic attempts count against
+  // MAX_CONSECUTIVE_RECONCILE_FAILURES, so this is the other side of that
+  // cap: the way OUT of "needs_help", not just the way IN. Reset does not
+  // claim the surface is healthy again — it has done no health probe — it
+  // only clears the latch so the next reconcile tick (or health probe) gets
+  // to judge it fresh.
+  if (action !== "reconcile" && result.status === 0) {
+    updateSurfaceState(id, {
+      consecutive_reconcile_failures: 0,
+      state: surface.state === "needs_help" ? "unavailable" : surface.state,
+    });
+  }
   return operation(action, { surface_id: id }, surface, {
     ok: result.status === 0,
     state: result.status === 0 ? "operation_completed" : "operation_failed",

package/cli/supervisor-registry.test.js

@@ -12,8 +12,10 @@ import {
   isMcpServerPlugin,
   listPlugins,
   listSurfaces,
+  MAX_CONSECUTIVE_RECONCILE_FAILURES,
   probeAllSurfaceHealth,
   surfaceOpenUrl,
+  readSupervisorEvents,
   reconcileSurfaceHealth,
   registerPlugin,
   runPluginAction,
@@ -60,6 +62,33 @@ async function withTempSupervisor(fn) {
   }
 }
 
+// Fast-forwards Date.now()/`new Date()` without real sleeping.
+// HEALTH_RECONCILE_COOLDOWN_MS is a real 15s per attempt, and proving the
+// crash-loop cap needs MAX_CONSECUTIVE_RECONCILE_FAILURES of them -- a test
+// cannot afford to actually sleep that long. Overriding ONLY Date.now() is
+// not enough: now() (supervisor-registry.js) stamps last_reconcile_at via
+// `new Date().toISOString()`, so the cooldown gate (Date.now() minus
+// Date.parse(last_reconcile_at)) would compare a mocked "now" against a REAL
+// timestamp and never agree. Subclassing keeps every other Date behavior
+// (Date.parse, `new Date(iso)`) real.
+async function withMockedClock(fn) {
+  const RealDate = global.Date;
+  let clock = RealDate.now();
+  class MockDate extends RealDate {
+    constructor(...args) {
+      if (args.length === 0) super(clock);
+      else super(...args);
+    }
+    static now() { return clock; }
+  }
+  global.Date = MockDate;
+  try {
+    return await fn({ advance: (ms) => { clock += ms; } });
+  } finally {
+    global.Date = RealDate;
+  }
+}
+
 function writePlugin(root, source, id, manifest) {
   const dir = path.join(root, source, id);
   fs.mkdirSync(dir, { recursive: true });
@@ -453,6 +482,135 @@ test("health reconciliation does not claim current when a successful command lea
   }
 });
 
+test("a crash-looping surface stops auto-restarting after MAX_CONSECUTIVE_RECONCILE_FAILURES and needs a human", async () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "clauth-supervisor-crashloop-"));
+  const oldDir = process.env.CLAUTH_SUPERVISOR_DIR;
+  const oldManaged = process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+  const oldUser = process.env.CLAUTH_USER_PLUGIN_ROOTS;
+  process.env.CLAUTH_SUPERVISOR_DIR = root;
+  process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = path.join(root, "managed");
+  process.env.CLAUTH_USER_PLUGIN_ROOTS = path.join(root, "user");
+  try {
+    // restart "succeeds" (exit 0 -- pm2 would accept it) but health NEVER
+    // recovers: exactly the regen-media-local shape -- a process that
+    // restarts cleanly and crashes immediately after, forever.
+    writePlugin(root, "managed", "crashloop-demo", baseManifest("crashloop-demo", {
+      core: true,
+      enable_default: true,
+      surfaces: [{
+        id: "primary",
+        destination: "local/clauth/pm2",
+        lifecycle_owner: "clauth",
+        port: 39121,
+        health: "/health",
+        restart: [process.execPath, "--version"],
+        start: [process.execPath, "--version"],
+      }],
+    }));
+    discoverPlugins();
+    const alwaysDown = { fetchImpl: async () => ({ ok: false, status: 503 }) };
+
+    await withMockedClock(async ({ advance }) => {
+      let last;
+      for (let i = 0; i < MAX_CONSECUTIVE_RECONCILE_FAILURES; i++) {
+        advance(20000); // past the 15s reconcile cooldown every tick
+        last = await reconcileSurfaceHealth(alwaysDown);
+      }
+      const surface = listSurfaces().find((item) => item.plugin_id === "crashloop-demo");
+      assert.equal(surface.state, "needs_help", "must latch once the cap is reached");
+      assert.equal(surface.consecutive_reconcile_failures, MAX_CONSECUTIVE_RECONCILE_FAILURES);
+      assert.equal(last.inspected[0].state, "needs_help");
+
+      // THE load-bearing assertion: one more tick past cooldown must NOT
+      // spawn another restart. Count "reconcile" operation receipts before
+      // and after -- if the count moves, the loop never actually stopped.
+      const before = readSupervisorEvents(1000).filter((e) => e.kind === "operation" && e.action === "reconcile").length;
+      advance(20000);
+      const again = await reconcileSurfaceHealth(alwaysDown);
+      const after = readSupervisorEvents(1000).filter((e) => e.kind === "operation" && e.action === "reconcile").length;
+      assert.equal(after, before, "a latched surface must not be re-restarted -- that is the crash loop this cap exists to stop");
+      assert.equal(again.inspected[0].state, "needs_help");
+      assert.equal(again.inspected[0].latched, true);
+
+      // Self-heal: health recovering on its own (independent of clauth's
+      // restart) clears the latch -- the cap is a circuit breaker, not a
+      // permanent ban.
+      advance(20000);
+      const healed = await reconcileSurfaceHealth({ fetchImpl: async () => ({ ok: true, status: 200 }) });
+      assert.equal(healed.inspected[0].state, "healthy");
+      const healedSurface = listSurfaces().find((item) => item.plugin_id === "crashloop-demo");
+      assert.equal(healedSurface.state, "current");
+      assert.equal(healedSurface.consecutive_reconcile_failures, 0);
+    });
+  } finally {
+    if (oldDir === undefined) delete process.env.CLAUTH_SUPERVISOR_DIR;
+    else process.env.CLAUTH_SUPERVISOR_DIR = oldDir;
+    if (oldManaged === undefined) delete process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+    else process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = oldManaged;
+    if (oldUser === undefined) delete process.env.CLAUTH_USER_PLUGIN_ROOTS;
+    else process.env.CLAUTH_USER_PLUGIN_ROOTS = oldUser;
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("an explicit manual restart clears a needs_help latch and gives a fresh crash-loop budget", async () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "clauth-supervisor-manual-reset-"));
+  const oldDir = process.env.CLAUTH_SUPERVISOR_DIR;
+  const oldManaged = process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+  const oldUser = process.env.CLAUTH_USER_PLUGIN_ROOTS;
+  process.env.CLAUTH_SUPERVISOR_DIR = root;
+  process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = path.join(root, "managed");
+  process.env.CLAUTH_USER_PLUGIN_ROOTS = path.join(root, "user");
+  try {
+    writePlugin(root, "managed", "manual-reset-demo", baseManifest("manual-reset-demo", {
+      core: true,
+      enable_default: true,
+      surfaces: [{
+        id: "primary",
+        destination: "local/clauth/pm2",
+        lifecycle_owner: "clauth",
+        port: 39122,
+        health: "/health",
+        restart: [process.execPath, "--version"],
+        start: [process.execPath, "--version"],
+      }],
+    }));
+    discoverPlugins();
+    const alwaysDown = { fetchImpl: async () => ({ ok: false, status: 503 }) };
+
+    await withMockedClock(async ({ advance }) => {
+      for (let i = 0; i < MAX_CONSECUTIVE_RECONCILE_FAILURES; i++) {
+        advance(20000);
+        await reconcileSurfaceHealth(alwaysDown);
+      }
+      assert.equal(listSurfaces()[0].state, "needs_help");
+
+      // A human/agent looks at it and issues an explicit restart -- through
+      // runSurfaceAction directly, never through the "reconcile" action the
+      // background loop uses.
+      const receipt = runSurfaceAction("manual-reset-demo:primary", "restart", "human");
+      assert.equal(receipt.resulting_state.ok, true);
+      const reset = listSurfaces()[0];
+      assert.equal(reset.state, "unavailable", "cleared out of needs_help, but not falsely claimed healthy without a probe");
+      assert.equal(reset.consecutive_reconcile_failures, 0);
+
+      // And the surface gets its full budget back, not zero.
+      advance(20000);
+      const tick = await reconcileSurfaceHealth(alwaysDown);
+      assert.equal(tick.inspected[0].state, "reconcile_failed");
+      assert.equal(listSurfaces()[0].consecutive_reconcile_failures, 1);
+    });
+  } finally {
+    if (oldDir === undefined) delete process.env.CLAUTH_SUPERVISOR_DIR;
+    else process.env.CLAUTH_SUPERVISOR_DIR = oldDir;
+    if (oldManaged === undefined) delete process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+    else process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = oldManaged;
+    if (oldUser === undefined) delete process.env.CLAUTH_USER_PLUGIN_ROOTS;
+    else process.env.CLAUTH_USER_PLUGIN_ROOTS = oldUser;
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
 test("health reconciliation never restarts external or plugin-owned surfaces", async () => {
   const root = fs.mkdtempSync(path.join(os.tmpdir(), "clauth-supervisor-observe-"));
   const oldDir = process.env.CLAUTH_SUPERVISOR_DIR;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lifeaitools/clauth",
-  "version": "2.15.5",
+  "version": "2.15.6",
   "description": "Hardware-bound credential vault for the LIFEAI infrastructure stack",
   "type": "module",
   "bin": {