@lifeaitools/clauth 2.15.5 → 2.15.7

package/cli/dashboard/panels/surfaces-panel.css

@@ -32,6 +32,15 @@
 .supervisor-pill.ok{border-color:rgba(74,222,128,.35);color:var(--green-light);background:rgba(34,197,94,.08)}
 .supervisor-pill.warn{border-color:rgba(250,204,21,.35);color:var(--gold-light);background:rgba(250,204,21,.08)}
 .supervisor-pill.bad{border-color:rgba(248,113,113,.35);color:var(--pink);background:rgba(248,113,113,.08)}
+/* NEEDS HELP -- a surface that exhausted MAX_CONSECUTIVE_RECONCILE_FAILURES
+   and clauth has stopped auto-restarting. Distinct from .bad ("down right
+   now, clauth is still retrying"): this is "clauth gave up, a human/agent
+   has to look." Pulses so it reads as an alarm among a fleet that regularly
+   has a few surfaces transiently down in .bad. */
+.supervisor-pill.critical{border-color:rgba(248,113,113,.6);color:#fff;background:rgba(220,38,38,.55);animation:supervisor-pulse 1.4s ease-in-out infinite}
+.supervisor-row.needs-help{border-color:rgba(220,38,38,.6);box-shadow:0 0 0 1px rgba(220,38,38,.35) inset}
+.supervisor-meta.critical{color:var(--pink);white-space:normal}
+@keyframes supervisor-pulse{0%,100%{opacity:1}50%{opacity:.55}}
 /* Card GRID, not a stack. flex-direction:column gave one full-width row per
    surface, so ten surfaces meant ten rows and constant scrolling to see the
    fleet. auto-fill + minmax packs as many small cards per row as the panel

package/cli/dashboard/panels/surfaces-panel.js

@@ -305,7 +305,15 @@ class ClauthSurfacesPanel extends window.ClauthPanelElement {
   renderSupervisorSurface(surface) {
     const compositeId = surface.plugin_id + ":" + surface.id;
     const ownerKind = surface.lifecycle_owner === "clauth" ? "ok" : (surface.lifecycle_owner === "plugin" ? "warn" : "");
-    const stateKind = surface.state === "current" || surface.status === "healthy" ? "ok" : (surface.state === "unavailable" ? "bad" : "warn");
+    // "needs_help" is a DIFFERENT alarm than "bad": bad means "down right now,
+    // clauth is still retrying"; needs_help means "clauth gave up after
+    // MAX_CONSECUTIVE_RECONCILE_FAILURES — a human/agent has to look." It gets
+    // its own loud, pulsing pill (see .supervisor-pill.critical) rather than
+    // reusing "bad", which would render as just another red flicker in a fleet
+    // that regularly has a few surfaces transiently down.
+    const needsHelp = surface.state === "needs_help";
+    const stateKind = needsHelp ? "critical" : (surface.state === "current" || surface.status === "healthy" ? "ok" : (surface.state === "unavailable" ? "bad" : "warn"));
+    const stateLabel = needsHelp ? "NEEDS HELP" : (surface.state || surface.status || "unknown");
     const openUrl = surface.__pseudo ? null : openUiUrlForSurface(surface);
     const isSelected = this.selectedSupervisorSurface && this.selectedSupervisorSurface.id === compositeId;
     const rowClass = "supervisor-row" + (surface.__pseudo ? " readonly" : "") + (isSelected ? " selected" : "");
@@ -339,14 +347,22 @@ class ClauthSurfacesPanel extends window.ClauthPanelElement {
       : (surface.last_health_ok === false && surface.last_health_error
         ? surface.last_health_error + " · " + (surface.health || "no health")
         : (surface.health || surface.health_url || "no health"));
-    return '<div class="' + rowClass + '" data-surface-id="' + htmlEscape(compositeId) + '"' + rowAction + '>' +
+    // The loud line: WHY it needs help and WHAT to do, spelled out on the
+    // card itself — not just a pill label a reader has to already know the
+    // meaning of. Mirrors the same wording reconcileSurfaceHealth() writes
+    // into the daemon log, so the card and the log never disagree.
+    const needsHelpLine = needsHelp
+      ? "Stopped auto-restarting after " + (surface.consecutive_reconcile_failures || "several") + " failed attempts — check the log, fix the cause, then Start/Restart to resume auto-repair."
+      : null;
+    return '<div class="' + rowClass + (needsHelp ? " needs-help" : "") + '" data-surface-id="' + htmlEscape(compositeId) + '"' + rowAction + '>' +
       '<div class="supervisor-row-top"><span class="supervisor-name">' + htmlEscape(surface.name || compositeId) + '</span>' +
       healthPill +
-      supervisorBadge(surface.lifecycle_owner || "unknown", ownerKind) + supervisorBadge(surface.state || surface.status || "unknown", stateKind) +
+      supervisorBadge(surface.lifecycle_owner || "unknown", ownerKind) + supervisorBadge(stateLabel, stateKind) +
       openBtn +
       '</div>' +
       '<div class="supervisor-meta">' + htmlEscape(surface.destination || "—") + ' · port ' + htmlEscape(surface.port || "—") + '</div>' +
       '<div class="supervisor-meta">' + htmlEscape(healthLine) + '</div>' +
+      (needsHelpLine ? '<div class="supervisor-meta critical">' + htmlEscape(needsHelpLine) + '</div>' : '') +
       '</div>';
   }
 

package/cli/fingerprint.js

@@ -15,7 +15,33 @@ import path from "path";
 // Cache path — avoids re-querying WMI/CIM on every daemon start.
 // This eliminates the spawnSync cmd.exe ETIMEDOUT crash that occurs
 // when PowerShell/WMI is slow on first call after boot.
-const CACHE_FILE = path.join(os.tmpdir(), "clauth-machine.cache");
+//
+// Durable as of 2026-09-06 (was os.tmpdir(), see LEGACY_CACHE_FILE below).
+// getMachineId() derives (primary, secondary) from TWO independent execSync
+// calls, each wrapped in its own try/catch that falls through silently on
+// failure (see the comments at those calls). If either call transiently
+// timed out, the resulting pairing was composed differently than a
+// successful run, producing a DIFFERENT sha256 machine hash for the exact
+// same physical machine — surfacing server-side as a false
+// machine_not_found lockout, not a security event. A cache in system temp
+// does not survive Windows Disk Cleanup or some machines' reboot behavior,
+// which re-exposed the flaky first-computation path on any later restart
+// where WMI happened to be slow. Matches the durable pattern boot.key
+// already uses (getBootKeyPath() in cli/commands/serve.js) — same
+// AppData/Roaming/clauth (win32) / ~/.config/clauth (else) directory.
+function getCacheDir() {
+  if (os.platform() === "win32") {
+    return path.join(process.env.APPDATA || path.join(os.homedir(), "AppData", "Roaming"), "clauth");
+  }
+  return path.join(os.homedir(), ".config", "clauth");
+}
+const CACHE_FILE = path.join(getCacheDir(), "machine.cache");
+// Pre-2026-09-06 location. An already-registered machine may still have a
+// valid cache here; migrate it into the durable location on first read
+// instead of recomputing via WMI — recomputing is exactly the flaky path
+// this fix exists to avoid, and the durable write means this only ever
+// happens once per machine.
+const LEGACY_CACHE_FILE = path.join(os.tmpdir(), "clauth-machine.cache");
 
 function readCache() {
   try {
@@ -23,12 +49,24 @@ function readCache() {
     // Validate: must be two non-empty lines (primary:secondary)
     const [primary, secondary] = raw.split("\n");
     if (primary && secondary) return { primary: primary.trim(), secondary: secondary.trim() };
-  } catch { /* cache miss */ }
+  } catch { /* durable cache miss */ }
+  try {
+    const raw = fs.readFileSync(LEGACY_CACHE_FILE, "utf8").trim();
+    const [primary, secondary] = raw.split("\n");
+    if (primary && secondary) {
+      const migrated = { primary: primary.trim(), secondary: secondary.trim() };
+      writeCache(migrated.primary, migrated.secondary);
+      return migrated;
+    }
+  } catch { /* no legacy cache either */ }
   return null;
 }
 
 function writeCache(primary, secondary) {
-  try { fs.writeFileSync(CACHE_FILE, `${primary}\n${secondary}`, "utf8"); } catch { /* best effort */ }
+  try {
+    fs.mkdirSync(getCacheDir(), { recursive: true });
+    fs.writeFileSync(CACHE_FILE, `${primary}\n${secondary}`, "utf8");
+  } catch { /* best effort */ }
 }
 
 function getMachineId() {
@@ -140,4 +178,8 @@ export function deriveSeedHash(machineHash, password) {
     .digest("hex");
 }
 
+// Exported for test path introspection only — not part of the public
+// CLI/API surface.
+export { CACHE_FILE, LEGACY_CACHE_FILE, getCacheDir };
+
 export default { getMachineHash, deriveToken, deriveSeedHash };

package/cli/fingerprint.test.js

@@ -4,7 +4,12 @@ import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 
-import { getMachineHash, deriveToken, deriveSeedHash } from "./fingerprint.js";
+import { getMachineHash, deriveToken, deriveSeedHash, CACHE_FILE, LEGACY_CACHE_FILE } from "./fingerprint.js";
+
+// CACHE_FILE/LEGACY_CACHE_FILE are the REAL production paths (durable
+// AppData/Roaming/clauth on win32, formerly os.tmpdir()) — read-only in this
+// suite. Every test below only ever reads mtimeMs, never asserts on or
+// writes machine-identifying content into either file.
 
 // getMachineId() normally shells out to WMI/registry/ioreg, which this harness
 // must never do (slow, platform-dependent, pollutes nothing but still real
@@ -84,12 +89,31 @@ test("deriveSeedHash changes with the password", () => {
   assert.notEqual(deriveSeedHash(hash, "one"), deriveSeedHash(hash, "two"));
 });
 
-test("machine id cache: a fresh CLAUTH_MACHINE_ID call never touches the WMI cache file", () => {
+test("machine id cache: a fresh CLAUTH_MACHINE_ID call never touches the WMI cache file (durable or legacy)", () => {
   // Positive control that the cache mechanism exists and this test can see it,
   // so "the cache file was untouched" below actually means something.
-  const cacheFile = path.join(os.tmpdir(), "clauth-machine.cache");
-  const before = fs.existsSync(cacheFile) ? fs.statSync(cacheFile).mtimeMs : null;
+  const before = {
+    durable: fs.existsSync(CACHE_FILE) ? fs.statSync(CACHE_FILE).mtimeMs : null,
+    legacy: fs.existsSync(LEGACY_CACHE_FILE) ? fs.statSync(LEGACY_CACHE_FILE).mtimeMs : null,
+  };
   withMachineId("container-fast-path", () => getMachineHash());
-  const after = fs.existsSync(cacheFile) ? fs.statSync(cacheFile).mtimeMs : null;
-  assert.equal(before, after, "the container-id fast path must not read or write the WMI cache file");
+  const after = {
+    durable: fs.existsSync(CACHE_FILE) ? fs.statSync(CACHE_FILE).mtimeMs : null,
+    legacy: fs.existsSync(LEGACY_CACHE_FILE) ? fs.statSync(LEGACY_CACHE_FILE).mtimeMs : null,
+  };
+  assert.deepEqual(before, after, "the container-id fast path must not read or write either cache file");
+});
+
+test("cache paths are durable (AppData/Roaming/clauth or ~/.config/clauth), never os.tmpdir()", () => {
+  // The whole point of this fix: CACHE_FILE must NOT live in system temp,
+  // which Windows Disk Cleanup (or equivalent) can wipe, re-exposing the
+  // flaky first-computation WMI path on the next restart.
+  assert.ok(!CACHE_FILE.startsWith(os.tmpdir()), `CACHE_FILE must be durable, got: ${CACHE_FILE}`);
+  assert.equal(path.basename(CACHE_FILE), "machine.cache");
+  const dir = path.dirname(CACHE_FILE);
+  assert.ok(dir.endsWith(path.join("clauth")), `cache dir must be a clauth-owned config directory, got: ${dir}`);
+  // LEGACY_CACHE_FILE is the pre-fix location, kept only so readCache() can
+  // migrate an already-registered machine's existing cache instead of
+  // recomputing via WMI (see fingerprint.js).
+  assert.equal(LEGACY_CACHE_FILE, path.join(os.tmpdir(), "clauth-machine.cache"));
 });

package/cli/supervisor-registry.js

@@ -18,6 +18,22 @@ const ACTIONS = new Set(["start", "stop", "restart", "reconcile", "test", "promo
 const DEFAULT_HEALTH_RECONCILE_INTERVAL_MS = 10000;
 const DEFAULT_HEALTH_TIMEOUT_MS = 2500;
 const HEALTH_RECONCILE_COOLDOWN_MS = 15000;
+// After this many CONSECUTIVE failed reconcile attempts (each gated by
+// HEALTH_RECONCILE_COOLDOWN_MS, so >= 5 * 15s = 75s before this fires),
+// reconcileSurfaceHealth() stops auto-restarting the surface and latches it
+// into "needs_help" instead of retrying forever. Without a cap, a surface
+// that can never come up (a missing dependency, a bad config) gets
+// relaunched every cooldown window FOREVER: the restart command itself can
+// exit 0 (pm2 accepted it) while the process crashes immediately after, so
+// this loop never sees a hard failure to stop on, and pm2's OWN crash-loop
+// backoff never engages either because clauth's periodic explicit restart
+// keeps resetting it. Root cause of the regen-media-local window-flashing
+// incident (2026-09-05): a missing `tsx` dependency meant every restart
+// "succeeded" and immediately crashed, forever, because nothing was
+// counting. A latched surface self-heals the moment its health probe
+// succeeds (handled before this cap is ever consulted) or a human/agent
+// issues an explicit non-reconcile start/stop/restart (see runSurfaceAction).
+export const MAX_CONSECUTIVE_RECONCILE_FAILURES = 5;
 const DOCUMENTATION_FIELDS = ["architecture", "operator_guide", "install", "runbook", "tool_reference", "release", "agent_context"];
 
 function hasMcpToken(value) {
@@ -182,11 +198,25 @@ export async function reconcileSurfaceHealth({ fetchImpl = globalThis.fetch, tim
     const error = health.error;
 
     if (healthy) {
-      updateSurfaceState(id, { state: "current", last_health_at: observedAt, last_health_ok: true, last_health_error: null });
+      updateSurfaceState(id, { state: "current", last_health_at: observedAt, last_health_ok: true, last_health_error: null, consecutive_reconcile_failures: 0 });
       inspected.push({ surface_id: id, state: "healthy", observed_at: observedAt });
       continue;
     }
 
+    // Latched: this surface already exhausted MAX_CONSECUTIVE_RECONCILE_FAILURES
+    // and clauth stopped auto-restarting it. Record the failed probe (so the
+    // UI's "last checked" freshness stays honest) WITHOUT calling
+    // runSurfaceAction again — that call is exactly the crash loop this latch
+    // exists to stop. The only ways out are a health probe that succeeds
+    // (the branch above, checked every tick regardless of latch state) or an
+    // explicit human/agent action (runSurfaceAction resets the counter and
+    // clears this latch on any successful non-reconcile command).
+    if (surface.state === "needs_help") {
+      updateSurfaceState(id, { last_health_at: observedAt, last_health_ok: false, last_health_error: error });
+      inspected.push({ surface_id: id, state: "needs_help", error, latched: true, observed_at: observedAt });
+      continue;
+    }
+
     const lastAttempt = Date.parse(surface.last_reconcile_at || "") || 0;
     if (Date.now() - lastAttempt < HEALTH_RECONCILE_COOLDOWN_MS) {
       updateSurfaceState(id, { state: "unavailable", last_health_at: observedAt, last_health_ok: false, last_health_error: error });
@@ -206,15 +236,52 @@ export async function reconcileSurfaceHealth({ fetchImpl = globalThis.fetch, tim
     const commandCompleted = receipt?.resulting_state?.ok === true;
     const postHealth = commandCompleted ? await probeSurfaceHealth(url, fetchImpl, timeoutMs) : { healthy: false, error: receipt?.resulting_state?.state || "reconcile_failed" };
     const repaired = commandCompleted && postHealth.healthy;
+
+    if (repaired) {
+      updateSurfaceState(id, {
+        state: "current",
+        last_health_at: now(),
+        last_health_ok: true,
+        last_health_error: null,
+        last_reconcile_operation_id: receipt?.operationId || null,
+        consecutive_reconcile_failures: 0,
+      });
+      appendSupervisorEvent({ kind: "surface_reconciled", surface_id: id, operation_id: receipt?.operationId || null, command_completed: commandCompleted, health_ok: true, error: null });
+      inspected.push({ surface_id: id, state: "reconciled", error: null, operation_id: receipt?.operationId || null, observed_at: observedAt });
+      continue;
+    }
+
+    // Not repaired: count it. MAX_CONSECUTIVE_RECONCILE_FAILURES is the
+    // circuit breaker — cross it and the surface latches to "needs_help"
+    // instead of feeding another attempt back into runSurfaceAction next tick.
+    const failures = (surface.consecutive_reconcile_failures || 0) + 1;
+    const exhausted = failures >= MAX_CONSECUTIVE_RECONCILE_FAILURES;
     updateSurfaceState(id, {
-      state: repaired ? "current" : "unavailable",
+      state: exhausted ? "needs_help" : "unavailable",
       last_health_at: now(),
-      last_health_ok: repaired,
-      last_health_error: repaired ? null : postHealth.error,
+      last_health_ok: false,
+      last_health_error: postHealth.error || error,
       last_reconcile_operation_id: receipt?.operationId || null,
+      consecutive_reconcile_failures: failures,
+    });
+    appendSupervisorEvent({
+      kind: exhausted ? "surface_needs_help" : "surface_reconcile_failed",
+      surface_id: id,
+      operation_id: receipt?.operationId || null,
+      command_completed: commandCompleted,
+      health_ok: false,
+      error: postHealth.error || error,
+      consecutive_failures: failures,
+      ...(exhausted ? { message: `${id} failed to come up after ${failures} consecutive restart attempts — clauth has stopped auto-restarting it. Check its log, fix the underlying cause, then Start/Restart it manually (dashboard or \`clauth mcp restart ${id}\`) to resume auto-repair.` } : {}),
+    });
+    inspected.push({
+      surface_id: id,
+      state: exhausted ? "needs_help" : "reconcile_failed",
+      error: postHealth.error || error,
+      operation_id: receipt?.operationId || null,
+      consecutive_failures: failures,
+      observed_at: observedAt,
     });
-    appendSupervisorEvent({ kind: repaired ? "surface_reconciled" : "surface_reconcile_failed", surface_id: id, operation_id: receipt?.operationId || null, command_completed: commandCompleted, health_ok: postHealth.healthy, error: postHealth.error || null });
-    inspected.push({ surface_id: id, state: repaired ? "reconciled" : "reconcile_failed", error: postHealth.error || error, operation_id: receipt?.operationId || null, observed_at: observedAt });
   }
   return { inspected };
 }
@@ -1245,6 +1312,22 @@ export function runSurfaceAction(id, action, actor = "localhost") {
     fallbackUsed = true;
     result.stderr = `restart exited ${restartStatus}; start fallback attempted\n${result.stderr || ""}`;
   }
+  // A deliberate, successful manual action — start/stop/restart, never
+  // "reconcile" itself — is a human or agent saying "I looked at this."
+  // Give the surface a fresh crash-loop budget instead of carrying forward a
+  // counter (or a "needs_help" latch) from before the intervention. Only
+  // reconcileSurfaceHealth's own automatic attempts count against
+  // MAX_CONSECUTIVE_RECONCILE_FAILURES, so this is the other side of that
+  // cap: the way OUT of "needs_help", not just the way IN. Reset does not
+  // claim the surface is healthy again — it has done no health probe — it
+  // only clears the latch so the next reconcile tick (or health probe) gets
+  // to judge it fresh.
+  if (action !== "reconcile" && result.status === 0) {
+    updateSurfaceState(id, {
+      consecutive_reconcile_failures: 0,
+      state: surface.state === "needs_help" ? "unavailable" : surface.state,
+    });
+  }
   return operation(action, { surface_id: id }, surface, {
     ok: result.status === 0,
     state: result.status === 0 ? "operation_completed" : "operation_failed",

package/cli/supervisor-registry.test.js

@@ -12,8 +12,10 @@ import {
   isMcpServerPlugin,
   listPlugins,
   listSurfaces,
+  MAX_CONSECUTIVE_RECONCILE_FAILURES,
   probeAllSurfaceHealth,
   surfaceOpenUrl,
+  readSupervisorEvents,
   reconcileSurfaceHealth,
   registerPlugin,
   runPluginAction,
@@ -60,6 +62,33 @@ async function withTempSupervisor(fn) {
   }
 }
 
+// Fast-forwards Date.now()/`new Date()` without real sleeping.
+// HEALTH_RECONCILE_COOLDOWN_MS is a real 15s per attempt, and proving the
+// crash-loop cap needs MAX_CONSECUTIVE_RECONCILE_FAILURES of them -- a test
+// cannot afford to actually sleep that long. Overriding ONLY Date.now() is
+// not enough: now() (supervisor-registry.js) stamps last_reconcile_at via
+// `new Date().toISOString()`, so the cooldown gate (Date.now() minus
+// Date.parse(last_reconcile_at)) would compare a mocked "now" against a REAL
+// timestamp and never agree. Subclassing keeps every other Date behavior
+// (Date.parse, `new Date(iso)`) real.
+async function withMockedClock(fn) {
+  const RealDate = global.Date;
+  let clock = RealDate.now();
+  class MockDate extends RealDate {
+    constructor(...args) {
+      if (args.length === 0) super(clock);
+      else super(...args);
+    }
+    static now() { return clock; }
+  }
+  global.Date = MockDate;
+  try {
+    return await fn({ advance: (ms) => { clock += ms; } });
+  } finally {
+    global.Date = RealDate;
+  }
+}
+
 function writePlugin(root, source, id, manifest) {
   const dir = path.join(root, source, id);
   fs.mkdirSync(dir, { recursive: true });
@@ -453,6 +482,135 @@ test("health reconciliation does not claim current when a successful command lea
   }
 });
 
+test("a crash-looping surface stops auto-restarting after MAX_CONSECUTIVE_RECONCILE_FAILURES and needs a human", async () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "clauth-supervisor-crashloop-"));
+  const oldDir = process.env.CLAUTH_SUPERVISOR_DIR;
+  const oldManaged = process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+  const oldUser = process.env.CLAUTH_USER_PLUGIN_ROOTS;
+  process.env.CLAUTH_SUPERVISOR_DIR = root;
+  process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = path.join(root, "managed");
+  process.env.CLAUTH_USER_PLUGIN_ROOTS = path.join(root, "user");
+  try {
+    // restart "succeeds" (exit 0 -- pm2 would accept it) but health NEVER
+    // recovers: exactly the regen-media-local shape -- a process that
+    // restarts cleanly and crashes immediately after, forever.
+    writePlugin(root, "managed", "crashloop-demo", baseManifest("crashloop-demo", {
+      core: true,
+      enable_default: true,
+      surfaces: [{
+        id: "primary",
+        destination: "local/clauth/pm2",
+        lifecycle_owner: "clauth",
+        port: 39121,
+        health: "/health",
+        restart: [process.execPath, "--version"],
+        start: [process.execPath, "--version"],
+      }],
+    }));
+    discoverPlugins();
+    const alwaysDown = { fetchImpl: async () => ({ ok: false, status: 503 }) };
+
+    await withMockedClock(async ({ advance }) => {
+      let last;
+      for (let i = 0; i < MAX_CONSECUTIVE_RECONCILE_FAILURES; i++) {
+        advance(20000); // past the 15s reconcile cooldown every tick
+        last = await reconcileSurfaceHealth(alwaysDown);
+      }
+      const surface = listSurfaces().find((item) => item.plugin_id === "crashloop-demo");
+      assert.equal(surface.state, "needs_help", "must latch once the cap is reached");
+      assert.equal(surface.consecutive_reconcile_failures, MAX_CONSECUTIVE_RECONCILE_FAILURES);
+      assert.equal(last.inspected[0].state, "needs_help");
+
+      // THE load-bearing assertion: one more tick past cooldown must NOT
+      // spawn another restart. Count "reconcile" operation receipts before
+      // and after -- if the count moves, the loop never actually stopped.
+      const before = readSupervisorEvents(1000).filter((e) => e.kind === "operation" && e.action === "reconcile").length;
+      advance(20000);
+      const again = await reconcileSurfaceHealth(alwaysDown);
+      const after = readSupervisorEvents(1000).filter((e) => e.kind === "operation" && e.action === "reconcile").length;
+      assert.equal(after, before, "a latched surface must not be re-restarted -- that is the crash loop this cap exists to stop");
+      assert.equal(again.inspected[0].state, "needs_help");
+      assert.equal(again.inspected[0].latched, true);
+
+      // Self-heal: health recovering on its own (independent of clauth's
+      // restart) clears the latch -- the cap is a circuit breaker, not a
+      // permanent ban.
+      advance(20000);
+      const healed = await reconcileSurfaceHealth({ fetchImpl: async () => ({ ok: true, status: 200 }) });
+      assert.equal(healed.inspected[0].state, "healthy");
+      const healedSurface = listSurfaces().find((item) => item.plugin_id === "crashloop-demo");
+      assert.equal(healedSurface.state, "current");
+      assert.equal(healedSurface.consecutive_reconcile_failures, 0);
+    });
+  } finally {
+    if (oldDir === undefined) delete process.env.CLAUTH_SUPERVISOR_DIR;
+    else process.env.CLAUTH_SUPERVISOR_DIR = oldDir;
+    if (oldManaged === undefined) delete process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+    else process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = oldManaged;
+    if (oldUser === undefined) delete process.env.CLAUTH_USER_PLUGIN_ROOTS;
+    else process.env.CLAUTH_USER_PLUGIN_ROOTS = oldUser;
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
+test("an explicit manual restart clears a needs_help latch and gives a fresh crash-loop budget", async () => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "clauth-supervisor-manual-reset-"));
+  const oldDir = process.env.CLAUTH_SUPERVISOR_DIR;
+  const oldManaged = process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+  const oldUser = process.env.CLAUTH_USER_PLUGIN_ROOTS;
+  process.env.CLAUTH_SUPERVISOR_DIR = root;
+  process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = path.join(root, "managed");
+  process.env.CLAUTH_USER_PLUGIN_ROOTS = path.join(root, "user");
+  try {
+    writePlugin(root, "managed", "manual-reset-demo", baseManifest("manual-reset-demo", {
+      core: true,
+      enable_default: true,
+      surfaces: [{
+        id: "primary",
+        destination: "local/clauth/pm2",
+        lifecycle_owner: "clauth",
+        port: 39122,
+        health: "/health",
+        restart: [process.execPath, "--version"],
+        start: [process.execPath, "--version"],
+      }],
+    }));
+    discoverPlugins();
+    const alwaysDown = { fetchImpl: async () => ({ ok: false, status: 503 }) };
+
+    await withMockedClock(async ({ advance }) => {
+      for (let i = 0; i < MAX_CONSECUTIVE_RECONCILE_FAILURES; i++) {
+        advance(20000);
+        await reconcileSurfaceHealth(alwaysDown);
+      }
+      assert.equal(listSurfaces()[0].state, "needs_help");
+
+      // A human/agent looks at it and issues an explicit restart -- through
+      // runSurfaceAction directly, never through the "reconcile" action the
+      // background loop uses.
+      const receipt = runSurfaceAction("manual-reset-demo:primary", "restart", "human");
+      assert.equal(receipt.resulting_state.ok, true);
+      const reset = listSurfaces()[0];
+      assert.equal(reset.state, "unavailable", "cleared out of needs_help, but not falsely claimed healthy without a probe");
+      assert.equal(reset.consecutive_reconcile_failures, 0);
+
+      // And the surface gets its full budget back, not zero.
+      advance(20000);
+      const tick = await reconcileSurfaceHealth(alwaysDown);
+      assert.equal(tick.inspected[0].state, "reconcile_failed");
+      assert.equal(listSurfaces()[0].consecutive_reconcile_failures, 1);
+    });
+  } finally {
+    if (oldDir === undefined) delete process.env.CLAUTH_SUPERVISOR_DIR;
+    else process.env.CLAUTH_SUPERVISOR_DIR = oldDir;
+    if (oldManaged === undefined) delete process.env.CLAUTH_MANAGED_PLUGIN_ROOTS;
+    else process.env.CLAUTH_MANAGED_PLUGIN_ROOTS = oldManaged;
+    if (oldUser === undefined) delete process.env.CLAUTH_USER_PLUGIN_ROOTS;
+    else process.env.CLAUTH_USER_PLUGIN_ROOTS = oldUser;
+    fs.rmSync(root, { recursive: true, force: true });
+  }
+});
+
 test("health reconciliation never restarts external or plugin-owned surfaces", async () => {
   const root = fs.mkdtempSync(path.join(os.tmpdir(), "clauth-supervisor-observe-"));
   const oldDir = process.env.CLAUTH_SUPERVISOR_DIR;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lifeaitools/clauth",
-  "version": "2.15.5",
+  "version": "2.15.7",
   "description": "Hardware-bound credential vault for the LIFEAI infrastructure stack",
   "type": "module",
   "bin": {

package/supabase/functions/auth-vault/index.ts

@@ -1,5 +1,12 @@
-// clauth — auth-vault Edge Function v2
-// Added: IP whitelist, rate limiting, machine lockout (fail_count + locked)
+// clauth — auth-vault Edge Function v3
+// IP whitelist + machine lockout (fail_count + locked) remain. Rate limiting
+// and audit logging (clauth_audit) removed 2026-09-03 -- see the comments at
+// validateHMAC and where auditLog used to be defined for why: the rate-limit
+// check was an unindexed COUNT against clauth_audit run on EVERY request,
+// pre-auth, and every rejection it produced also wrote an audit row -- a
+// feedback loop that took the whole project down under sustained load. The
+// security value it provided was already covered, tighter, by the
+// per-machine 5-failed-attempts lockout below.
 
 import { createClient } from "https://esm.sh/@supabase/supabase-js@2";
 
@@ -11,8 +18,6 @@ const ADMIN_BOOTSTRAP_TOKEN = Deno.env.get("CLAUTH_ADMIN_BOOTSTRAP_TOKEN")!;
 const ALLOWED_IPS: string[] = (Deno.env.get("CLAUTH_ALLOWED_IPS") || "")
   .split(",").map(s => s.trim()).filter(Boolean);
 
-const RATE_LIMIT_MAX    = 30;
-const RATE_LIMIT_WINDOW = 60;
 const REPLAY_WINDOW_MS  = 5 * 60 * 1000;
 const MAX_FAIL_COUNT    = 5;
 const DEFAULT_INSTALL_ID = "default";
@@ -55,20 +60,36 @@ function checkIP(ip: string): { allowed: boolean; reason?: string } {
   return { allowed: false, reason: `IP not whitelisted: ${ip}` };
 }
 
-async function checkRateLimit(sb: any, machine_hash: string): Promise<{ allowed: boolean; reason?: string }> {
-  const windowStart = new Date(Date.now() - RATE_LIMIT_WINDOW * 1000).toISOString();
-  const { count } = await sb.from("clauth_audit")
-    .select("id", { count: "exact", head: true })
-    .eq("machine_hash", machine_hash)
-    .gte("created_at", windowStart);
-  if ((count || 0) >= RATE_LIMIT_MAX) {
-    return { allowed: false, reason: `Rate limit: ${count}/${RATE_LIMIT_MAX} per ${RATE_LIMIT_WINDOW}s` };
-  }
-  return { allowed: true };
-}
-
+// rate-limiting removed 2026-09-03 -- see supervisor-registry.js's own
+// state.json lock history for the shape of this bug: checkRateLimit() ran
+// this exact query -- a COUNT against clauth_audit filtered by machine_hash
+// and created_at -- on EVERY request, BEFORE authentication (validateHMAC
+// below), against a table with no index on either column and no retention.
+// As the table grew (174,385 rows, unbounded), the scan cost grew with it;
+// under sustained traffic the queries started stacking, Postgres killed them
+// at statement_timeout, and Cloudflare killed the stacked connections at its
+// own 90s ceiling (522s) -- and every one of those rejections ALSO wrote an
+// audit row (see auditLog's removal below), so the failure fed itself. The
+// actual security protection this duplicated is already provided, tighter,
+// by the per-machine lockout in validateHMAC below (5 failed attempts locks
+// the machine -- well under the 30-per-60s this used to allow). Rate
+// limiting for a genuine runaway client belongs at Cloudflare, in front of
+// this function, not as a synchronous Postgres query on the hot path of
+// every request.
 async function validateHMAC(sb: any, body: any): Promise<{ valid: boolean; reason?: string }> {
   const now = Date.now();
+
+  // A caller that omits machine_hash entirely used to fall straight through
+  // to the clauth_machines lookup below with body.machine_hash === undefined.
+  // PostgREST's .single() then answers "0 rows" for a query that can never
+  // match, which comes back as an HTTP 406 -- 1,438 of these in 24h
+  // (2026-09-06), every one a wasted round trip for a request already known
+  // to be malformed. Reject before the DB call: same auth_failed response
+  // the caller already gets, no query, no spurious 406 in the logs.
+  if (typeof body.machine_hash !== "string" || !body.machine_hash) {
+    return { valid: false, reason: "machine_hash_missing" };
+  }
+
   if (Math.abs(now - body.timestamp) > REPLAY_WINDOW_MS) return { valid: false, reason: "timestamp_expired" };
 
   const { data: machine, error } = await sb.from("clauth_machines")
@@ -99,20 +120,23 @@ async function validateHMAC(sb: any, body: any): Promise<{ valid: boolean; reaso
   return { valid: true };
 }
 
-async function auditLog(sb: any, machine_hash: string, service_name: string, action: string, result: string, detail?: string) {
-  await sb.from("clauth_audit").insert({ machine_hash, service_name, action, result, detail });
-}
-
+// auditLog() removed 2026-09-03 alongside checkRateLimit() -- it wrote a row
+// to clauth_audit on every path through this function, including rejections
+// (rate-limited, auth-denied, IP-blocked), which is what turned "the table
+// got slow" into a feedback loop (more rejections -> more rows -> slower
+// scans -> more rejections). No retention policy ever existed for this
+// table. If durable audit logging is wanted back, it needs its own design --
+// write-only, indexed for its actual read pattern (if any), with a real
+// retention/partition policy -- not a bare insert-on-every-call with no cap.
 async function handleRetrieve(sb: any, body: any, mh: string) {
   const { service } = body;
   if (!service) return { error: "service required" };
   const { data: svc } = await sb.from("clauth_services").select("*").eq("name", service).single();
-  if (!svc)           { await auditLog(sb, mh, service, "retrieve", "fail", "service_not_found"); return { error: "service_not_found" }; }
-  if (!svc.enabled)   { await auditLog(sb, mh, service, "retrieve", "denied", "service_disabled"); return { error: "service_disabled" }; }
-  if (!svc.vault_key) { await auditLog(sb, mh, service, "retrieve", "fail", "no_key_stored"); return { error: "no_key_stored" }; }
+  if (!svc)           return { error: "service_not_found" };
+  if (!svc.enabled)   return { error: "service_disabled" };
+  if (!svc.vault_key) return { error: "no_key_stored" };
   const { data: secret } = await sb.rpc("vault_decrypt_secret", { secret_name: svc.vault_key });
   await sb.from("clauth_services").update({ last_retrieved: new Date().toISOString() }).eq("name", service);
-  await auditLog(sb, mh, service, "retrieve", "success");
   return { service, key_type: svc.key_type, value: secret };
 }
 
@@ -121,9 +145,8 @@ async function handleWrite(sb: any, body: any, mh: string) {
   if (!service || !value) return { error: "service and value required" };
   const vaultKey = `clauth.${service}`;
   const { error } = await sb.rpc("vault_upsert_secret", { secret_name: vaultKey, secret_value: typeof value === "string" ? value : JSON.stringify(value) });
-  if (error) { await auditLog(sb, mh, service, "write", "fail", error.message); return { error: error.message }; }
+  if (error) return { error: error.message };
   await sb.from("clauth_services").update({ vault_key: vaultKey, last_rotated: new Date().toISOString() }).eq("name", service);
-  await auditLog(sb, mh, service, "write", "success");
   return { success: true, service, vault_key: vaultKey };
 }
 
@@ -133,7 +156,6 @@ async function handleEnable(sb: any, body: any, mh: string) {
   q = service !== "all" ? q.eq("name", service) : q.not("vault_key", "is", null);
   const { error } = await q;
   if (error) return { error: error.message };
-  await auditLog(sb, mh, service, enabled ? "enable" : "disable", "success");
   return { success: true, service, enabled };
 }
 
@@ -144,7 +166,6 @@ async function handleAdd(sb: any, body: any, mh: string) {
   if (project) row.project = project;
   const { error } = await sb.from("clauth_services").insert(row);
   if (error) return { error: error.message };
-  await auditLog(sb, mh, name, "add", "success");
   return { success: true, name, label, key_type, project: project || null };
 }
 
@@ -157,7 +178,6 @@ async function handleUpdate(sb: any, body: any, mh: string) {
   if (description !== undefined) updates.description = description || null;
   const { error } = await sb.from("clauth_services").update(updates).eq("name", service);
   if (error) return { error: error.message };
-  await auditLog(sb, mh, service, "update", "success", `fields: ${Object.keys(updates).join(", ")}`);
   return { success: true, service, ...updates };
 }
 
@@ -166,7 +186,6 @@ async function handleRemove(sb: any, body: any, mh: string) {
   if (confirm !== `CONFIRM REMOVE ${service.toUpperCase()}`) return { error: "confirm phrase mismatch" };
   await sb.rpc("vault_delete_secret", { secret_name: `clauth.${service}` });
   await sb.from("clauth_services").delete().eq("name", service);
-  await auditLog(sb, mh, service, "remove", "success");
   return { success: true, service };
 }
 
@@ -182,7 +201,6 @@ async function handleRevoke(sb: any, body: any, mh: string) {
     await sb.rpc("vault_delete_secret", { secret_name: `clauth.${service}` });
     await sb.from("clauth_services").update({ vault_key: null, enabled: false }).eq("name", service);
   }
-  await auditLog(sb, mh, service, "revoke", "success");
   return { success: true, service };
 }
 
@@ -193,7 +211,6 @@ async function handleStatus(sb: any, body: any, mh: string) {
     .order("name");
   if (body.project) q = q.eq("project", body.project);
   const { data: services } = await q;
-  await auditLog(sb, mh, "all", "status", "success");
   return { services: services || [] };
 }
 
@@ -204,7 +221,6 @@ async function handleChangePassword(sb: any, body: any, mh: string) {
     .update({ hmac_seed_hash: new_hmac_seed_hash, fail_count: 0, locked: false })
     .eq("machine_hash", mh);
   if (error) return { error: error.message };
-  await auditLog(sb, mh, "system", "change-password", "success");
   return { success: true };
 }
 
@@ -231,12 +247,8 @@ async function handleCreateEnrollment(sb: any, body: any, mh: string) {
     created_by_machine_hash: mh,
     expires_at,
   });
-  if (error) {
-    await auditLog(sb, mh, "system", "create-enrollment", "fail", error.message);
-    return { error: error.message };
-  }
+  if (error) return { error: error.message };
 
-  await auditLog(sb, mh, "system", "create-enrollment", "success", `install_id=${install_id}`);
   return { success: true, enrollment_code: code, install_id, expires_at, label };
 }
 
@@ -274,7 +286,6 @@ async function handleRedeemEnrollment(sb: any, body: any) {
   if (consumeError) return { error: consumeError.message };
   if (!consumedRows || consumedRows.length !== 1) return { error: "enrollment_already_used" };
 
-  await auditLog(sb, machine_hash, "system", "redeem-enrollment", "success", `install_id=${install_id}`);
   return { success: true, machine_hash, install_id };
 }
 
@@ -314,21 +325,11 @@ Deno.serve(async (req: Request) => {
 
   const ipCheck = checkIP(ip);
   if (!ipCheck.allowed) {
-    await auditLog(sb, body.machine_hash || "unknown", "system", route, "blocked", ipCheck.reason);
     return Response.json({ error: "ip_blocked", reason: ipCheck.reason }, { status: 403 });
   }
 
-  if (body.machine_hash) {
-    const rateCheck = await checkRateLimit(sb, body.machine_hash);
-    if (!rateCheck.allowed) {
-      await auditLog(sb, body.machine_hash, "system", route, "rate_limited", rateCheck.reason);
-      return Response.json({ error: "rate_limited", reason: rateCheck.reason }, { status: 429 });
-    }
-  }
-
   const authResult = await validateHMAC(sb, { machine_hash: body.machine_hash, token: body.token, timestamp: body.timestamp, password: body.password });
   if (!authResult.valid) {
-    await auditLog(sb, body.machine_hash || "unknown", body.service || "unknown", route, "denied", authResult.reason);
     return Response.json({ error: "auth_failed", reason: authResult.reason }, { status: 401 });
   }