@botcord/daemon 0.2.9 → 0.2.11

package/dist/control-channel.js

@@ -9,10 +9,17 @@
 import WebSocket from "ws";
 import { buildDaemonWebSocketUrl, CONTROL_FRAME_TYPES, jcsCanonicalize, resolveHubControlPublicKey, verifyEd25519, } from "@botcord/protocol-core";
 import { log as daemonLog } from "./log.js";
-import { writeAuthExpiredFlag, } from "./user-auth.js";
+import { AuthRefreshRejectedError, writeAuthExpiredFlag, } from "./user-auth.js";
 /** Exponential backoff plan for transient disconnects. */
 const RECONNECT_BACKOFF_MS = [1000, 2000, 4000, 8000, 16000, 30000];
-const KEEPALIVE_INTERVAL_MS = 25_000;
+/**
+ * Keepalive cadence. Has to stay below the smallest idle-timeout in any
+ * intermediary on the daemon → Hub WS path. Cloudflare and AWS ALB both
+ * default to ~60s of idle without app-level data, and some tunnels strip
+ * WS-level ping/pong control frames entirely — hence we send an app-level
+ * `pong` heartbeat alongside `ws.ping()` rather than relying on it alone.
+ */
+const KEEPALIVE_INTERVAL_MS = 20_000;
 const REPLAY_DEDUPE_CAP = 256;
 /**
  * Build the canonical signing input for a control frame: RFC 8785 (JCS)
@@ -91,8 +98,18 @@ export class ControlChannel {
         });
         this.connectInflight = this.connect().catch((err) => {
             // Initial connect failure surfaces to the caller; subsequent
-            // reconnects are handled opaquely inside onClose.
-            this.scheduleReconnect(err);
+            // reconnects are handled opaquely inside onClose. A refresh-rejected
+            // error means the refresh token itself is dead — no point retrying;
+            // writeAuthExpiredFlag was already called in user-auth.refresh().
+            if (err instanceof AuthRefreshRejectedError) {
+                this.stopRequested = true;
+                daemonLog.warn("control-channel: refresh rejected; stopping (re-login required)", {
+                    status: err.status,
+                });
+            }
+            else {
+                this.scheduleReconnect(err);
+            }
             throw err;
         });
         try {
@@ -188,12 +205,24 @@ export class ControlChannel {
             const ws = this.ws;
             if (!ws || ws.readyState !== WebSocket.OPEN)
                 return;
+            // WS-level ping for normal cases.
             try {
                 ws.ping();
             }
             catch {
                 // ignore — next failed send will trigger close
             }
+            // App-level heartbeat: a `pong` daemon-initiated frame. Hub recognizes
+            // it via `_DAEMON_INITIATED_TYPES` and bumps `last_seen_at`. Critical
+            // when an intermediary (Cloudflare, AWS ALB, some k8s ingresses)
+            // drops WS-level control frames — those proxies idle-close the WS at
+            // ~60s without app-level activity, masquerading as a clean 1006 to
+            // both peers.
+            this.send({
+                id: `hb_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+                type: "pong",
+                ts: Date.now(),
+            });
         }, this.keepaliveMs);
     }
     stopKeepalive() {
@@ -223,6 +252,13 @@ export class ControlChannel {
     scheduleReconnect(err) {
         if (this.stopRequested)
             return;
+        if (err instanceof AuthRefreshRejectedError) {
+            this.stopRequested = true;
+            daemonLog.warn("control-channel: refresh rejected; halting reconnect (re-login required)", {
+                status: err.status,
+            });
+            return;
+        }
         const attempt = this.reconnectAttempts;
         this.reconnectAttempts = attempt + 1;
         const delay = this.backoff[Math.min(attempt, this.backoff.length - 1)];
@@ -254,6 +290,14 @@ export class ControlChannel {
             return;
         }
         if (!frame || typeof frame.id !== "string" || typeof frame.type !== "string") {
+            // Hub ack responses for daemon-initiated frames (runtime_snapshot push,
+            // heartbeat, etc.) carry `{id, ok}` and no `type`. They're expected,
+            // not malformed — drop silently. Anything else stays a warn.
+            if (frame &&
+                typeof frame.id === "string" &&
+                typeof frame.ok === "boolean") {
+                return;
+            }
             daemonLog.warn("control-channel: malformed frame", { frame });
             return;
         }

package/dist/doctor.js

@@ -156,6 +156,9 @@ export function renderDoctor(input) {
         const r = rows[i];
         const e = input.runtimes[i];
         lines.push(`${pad(r.runtime, widths.runtime)}  ${pad(r.name, widths.name)}  ${pad(r.status, widths.status)}  ${pad(r.version, widths.version)}  ${r.path}`);
+        if (!e.result.available && e.installHint) {
+            lines.push(`    → ${e.installHint}`);
+        }
         if (e.endpoints && e.endpoints.length > 0) {
             for (const ep of e.endpoints) {
                 const mark = ep.reachable ? "✓" : "✗";

package/dist/gateway/runtimes/hermes-agent.d.ts

@@ -1,7 +1,11 @@
 import { AcpRuntimeAdapter, type AcpPermissionRequest, type AcpPermissionResponse, type AcpUpdateCtx, type AcpUpdateParams } from "./acp-stream.js";
 import { type ProbeDeps } from "./probe.js";
 import type { RuntimeProbeResult, RuntimeRunOptions } from "../types.js";
-/** Resolve the `hermes-acp` executable on PATH. */
+/**
+ * Resolve the `hermes-acp` executable. Tries PATH first, then falls back to
+ * the upstream install.sh's private venv location (`~/.hermes/...`) before
+ * giving up. `BOTCORD_HERMES_AGENT_BIN` always wins via the adapter override.
+ */
 export declare function resolveHermesAcpCommand(deps?: ProbeDeps): string | null;
 /** Probe whether `hermes-acp` is installed and report its version. */
 export declare function probeHermesAgent(deps?: ProbeDeps): RuntimeProbeResult;

package/dist/gateway/runtimes/hermes-agent.js

@@ -3,10 +3,34 @@ import path from "node:path";
 import { agentHermesHomeDir, agentHermesWorkspaceDir, ensureAgentHermesWorkspace, } from "../../agent-workspace.js";
 import { buildCliEnv } from "../cli-resolver.js";
 import { AcpRuntimeAdapter, } from "./acp-stream.js";
-import { readCommandVersion, resolveCommandOnPath } from "./probe.js";
-/** Resolve the `hermes-acp` executable on PATH. */
+import { firstExistingPath, readCommandVersion, resolveCommandOnPath, resolveHomePath, } from "./probe.js";
+/**
+ * Known absolute locations of the `hermes-acp` entry point when it is not on
+ * PATH. The upstream `scripts/install.sh` (curl|bash installer) installs a
+ * private virtualenv under `~/.hermes/hermes-agent/venv/` and only symlinks
+ * the user-facing `hermes` command into `~/.local/bin/` — the `hermes-acp`
+ * entry point stays inside the venv. Without a fallback, daemon's PATH-only
+ * probe misses every user who installed via the README-recommended script.
+ */
+const HERMES_ACP_FALLBACK_RELATIVE_PATHS = [
+    path.join(".hermes", "hermes-agent", "venv", "bin", "hermes-acp"),
+];
+const HERMES_ACP_FALLBACK_SYSTEM_PATHS = [
+    "/opt/hermes/hermes-agent/venv/bin/hermes-acp",
+];
+/**
+ * Resolve the `hermes-acp` executable. Tries PATH first, then falls back to
+ * the upstream install.sh's private venv location (`~/.hermes/...`) before
+ * giving up. `BOTCORD_HERMES_AGENT_BIN` always wins via the adapter override.
+ */
 export function resolveHermesAcpCommand(deps = {}) {
-    return resolveCommandOnPath("hermes-acp", deps);
+    const onPath = resolveCommandOnPath("hermes-acp", deps);
+    if (onPath)
+        return onPath;
+    return firstExistingPath([
+        ...HERMES_ACP_FALLBACK_RELATIVE_PATHS.map((p) => resolveHomePath(p, deps)),
+        ...HERMES_ACP_FALLBACK_SYSTEM_PATHS,
+    ], deps);
 }
 /** Probe whether `hermes-acp` is installed and report its version. */
 export function probeHermesAgent(deps = {}) {

package/dist/gateway/runtimes/registry.d.ts

@@ -23,6 +23,11 @@ export interface RuntimeModule {
      * config loader rejects routing turns to this adapter.
      */
     supportsRun?: boolean;
+    /**
+     * Short, single-line install hint shown by `doctor` when the runtime
+     * probes as unavailable. Helps users recover without reading source.
+     */
+    installHint?: string;
 }
 /** Built-in runtime module entry for Claude Code. */
 export declare const claudeCodeModule: RuntimeModule;
@@ -58,6 +63,7 @@ export interface RuntimeProbeEntry {
     binary: string;
     supportsRun: boolean;
     result: RuntimeProbeResult;
+    installHint?: string;
 }
 /** Probe every registered runtime and report installation status. */
 export declare function detectRuntimes(): RuntimeProbeEntry[];

package/dist/gateway/runtimes/registry.js

@@ -28,6 +28,7 @@ export const hermesAgentModule = {
     envVar: "BOTCORD_HERMES_AGENT_BIN",
     probe: () => probeHermesAgent(),
     create: () => new HermesAgentAdapter(),
+    installHint: 'Install: pip install "hermes-agent[acp]"  (or set BOTCORD_HERMES_AGENT_BIN to the absolute path of hermes-acp)',
 };
 /** Built-in runtime module entry for Gemini (probe-only stub). */
 export const geminiModule = {
@@ -110,6 +111,7 @@ export function detectRuntimes() {
             binary: m.binary,
             supportsRun: m.supportsRun !== false,
             result,
+            installHint: m.installHint,
         });
     }
     return out;

package/dist/provision.d.ts

@@ -56,15 +56,19 @@ export declare function adoptDiscoveredOpenclawAgents(ctx: {
 export declare function addAgentToConfig(cfg: DaemonConfig, agentId: string): DaemonConfig | null;
 /** Inverse of {@link addAgentToConfig}. Returns `null` on no-op. */
 export declare function removeAgentFromConfig(cfg: DaemonConfig, agentId: string): DaemonConfig | null;
+/** Drop the cache (e.g. before a `doctor`-style interactive re-probe). */
+export declare function clearRuntimeProbeCache(): void;
 /**
  * Probe every registered adapter and shape the result as the wire-level
  * {@link ListRuntimesResult} — used by both the `list_runtimes` ack path and
  * the daemon-side first-connect `runtime_snapshot` push in `daemon.ts`.
  *
- * Kept pure: the only side effects are `detectRuntimes()` itself (which the
- * gateway already isolates from throwing) and reading the wall clock.
+ * Cached for {@link RUNTIME_PROBE_CACHE_TTL_MS}; pass `{ force: true }` to
+ * bypass the cache.
  */
-export declare function collectRuntimeSnapshot(): ListRuntimesResult;
+export declare function collectRuntimeSnapshot(opts?: {
+    force?: boolean;
+}): ListRuntimesResult;
 /** Maximum number of `endpoints[]` entries persisted per runtime (RFC §3.8.2). */
 export declare const RUNTIME_ENDPOINTS_CAP = 32;
 /** Injection seam for L2 + L3 endpoint probes — kept testable + side-effect-free. */

package/dist/provision.js

@@ -768,15 +768,34 @@ export function removeAgentFromConfig(cfg, agentId) {
 // ---------------------------------------------------------------------------
 // runtime-discovery snapshot (plan §8.5)
 // ---------------------------------------------------------------------------
+/**
+ * TTL for the L1 runtime-detection cache. `detectRuntimes()` shells out to
+ * each adapter binary (claude / codex / gemini / openclaw / hermes) to read
+ * `--version`, which routinely costs 1.5–2s in aggregate — long enough to
+ * push `list_runtimes` past the Hub's 10s ack budget when combined with the
+ * 3s openclaw gateway probe. Versions don't change between dashboard refresh
+ * clicks, so cache the L1 snapshot briefly and recompute on miss.
+ */
+const RUNTIME_PROBE_CACHE_TTL_MS = 30_000;
+let _runtimeProbeCache = null;
+/** Drop the cache (e.g. before a `doctor`-style interactive re-probe). */
+export function clearRuntimeProbeCache() {
+    _runtimeProbeCache = null;
+}
 /**
  * Probe every registered adapter and shape the result as the wire-level
  * {@link ListRuntimesResult} — used by both the `list_runtimes` ack path and
  * the daemon-side first-connect `runtime_snapshot` push in `daemon.ts`.
  *
- * Kept pure: the only side effects are `detectRuntimes()` itself (which the
- * gateway already isolates from throwing) and reading the wall clock.
+ * Cached for {@link RUNTIME_PROBE_CACHE_TTL_MS}; pass `{ force: true }` to
+ * bypass the cache.
  */
-export function collectRuntimeSnapshot() {
+export function collectRuntimeSnapshot(opts = {}) {
+    if (!opts.force &&
+        _runtimeProbeCache &&
+        Date.now() - _runtimeProbeCache.at < RUNTIME_PROBE_CACHE_TTL_MS) {
+        return _runtimeProbeCache.value;
+    }
     const entries = detectRuntimes();
     const runtimes = entries.map((entry) => {
         const record = {
@@ -796,7 +815,9 @@ export function collectRuntimeSnapshot() {
         // enough; filling a synthetic message would be misleading.
         return record;
     });
-    return { runtimes, probedAt: Date.now() };
+    const value = { runtimes, probedAt: Date.now() };
+    _runtimeProbeCache = { at: Date.now(), value };
+    return value;
 }
 /** Maximum number of `endpoints[]` entries persisted per runtime (RFC §3.8.2). */
 export const RUNTIME_ENDPOINTS_CAP = 32;
@@ -1024,7 +1045,7 @@ export async function collectRuntimeSnapshotAsync(opts = {}) {
     if (gateways.length === 0)
         return base;
     // Default daemon-side budget is 3s — it must stay below the Hub's
-    // `list_runtimes` ack wait (5s, see backend/hub/routers/daemon_control.py)
+    // `list_runtimes` ack wait (10s, see backend/hub/routers/daemon_control.py)
     // so a single slow gateway can't blow the whole snapshot to a 504.
     const timeoutMs = opts.timeoutMs ?? 3000;
     const capped = gateways.slice(0, RUNTIME_ENDPOINTS_CAP);

package/dist/user-auth.d.ts

@@ -40,6 +40,15 @@ export declare function writeAuthExpiredFlag(file?: string): void;
 export declare function clearAuthExpiredFlag(file?: string): void;
 /** Returns true if the stored access token is within `windowMs` of expiry. */
 export declare function isTokenNearExpiry(record: UserAuthRecord, windowMs?: number): boolean;
+/**
+ * Thrown when the Hub rejects a refresh token (401/403). Signals that the
+ * user must re-login — reconnect loops should stop instead of hammering
+ * the refresh endpoint forever with a known-bad token.
+ */
+export declare class AuthRefreshRejectedError extends Error {
+    readonly status: number;
+    constructor(status: number, message: string);
+}
 /**
  * Stateful helper that owns the in-memory copy of user-auth and knows how
  * to refresh it. Used by the control channel so reconnects always carry

package/dist/user-auth.js

@@ -144,6 +144,19 @@ export function clearAuthExpiredFlag(file = AUTH_EXPIRED_FLAG_PATH) {
 export function isTokenNearExpiry(record, windowMs = 60_000) {
     return record.expiresAt - Date.now() <= windowMs;
 }
+/**
+ * Thrown when the Hub rejects a refresh token (401/403). Signals that the
+ * user must re-login — reconnect loops should stop instead of hammering
+ * the refresh endpoint forever with a known-bad token.
+ */
+export class AuthRefreshRejectedError extends Error {
+    status;
+    constructor(status, message) {
+        super(message);
+        this.name = "AuthRefreshRejectedError";
+        this.status = status;
+    }
+}
 /**
  * Stateful helper that owns the in-memory copy of user-auth and knows how
  * to refresh it. Used by the control channel so reconnects always carry
@@ -197,13 +210,37 @@ export class UserAuthManager {
             expiresInMs: current.expiresAt - Date.now(),
         });
         this.refreshInflight = (async () => {
-            const tok = await refreshDaemonToken(current.hubUrl, current.refreshToken);
+            // Refresh tokens rotate server-side. If another local process (e.g. a
+            // second daemon racing on the same user-auth.json) refreshed in the
+            // meantime, the on-disk refreshToken now differs from our in-memory
+            // copy — using the in-memory one would 401 because the server already
+            // invalidated it. Re-read disk first and adopt any newer record.
+            let basis = current;
+            try {
+                const onDisk = loadUserAuth(this.file);
+                if (onDisk && onDisk.refreshToken !== current.refreshToken) {
+                    daemonLog.info("user-auth refresh: adopting newer on-disk token", {
+                        userId: onDisk.userId,
+                        expiresAt: onDisk.expiresAt,
+                    });
+                    this.record = onDisk;
+                    if (!isTokenNearExpiry(onDisk))
+                        return onDisk;
+                    basis = onDisk;
+                }
+            }
+            catch (err) {
+                daemonLog.debug("user-auth refresh: disk reread failed (ignored)", {
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+            const tok = await refreshDaemonToken(basis.hubUrl, basis.refreshToken);
             const next = {
-                ...current,
+                ...basis,
                 accessToken: tok.accessToken,
                 refreshToken: tok.refreshToken,
                 expiresAt: Date.now() + tok.expiresIn * 1000,
-                hubUrl: tok.hubUrl || current.hubUrl,
+                hubUrl: tok.hubUrl || basis.hubUrl,
             };
             saveUserAuth(next, this.file);
             this.record = next;
@@ -213,10 +250,22 @@ export class UserAuthManager {
             });
             return next;
         })().catch((err) => {
+            const status = typeof err.status === "number"
+                ? (err.status)
+                : null;
+            const message = err instanceof Error ? err.message : String(err);
             daemonLog.warn("user-auth refresh: failed", {
                 userId: current.userId,
-                error: err instanceof Error ? err.message : String(err),
+                status,
+                error: message,
             });
+            if (status === 401 || status === 403) {
+                // Refresh token is permanently dead — write the expired flag so
+                // `status` surfaces it and re-throw a typed error so the control
+                // channel can stop reconnect loops instead of hammering the Hub.
+                writeAuthExpiredFlag();
+                throw new AuthRefreshRejectedError(status, message);
+            }
             throw err;
         }).finally(() => {
             this.refreshInflight = null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botcord/daemon",
-  "version": "0.2.9",
+  "version": "0.2.11",
   "description": "BotCord local daemon — bridges Hub inbox push to local Claude Code / Codex / Gemini CLIs",
   "type": "module",
   "bin": {

package/src/__tests__/runtime-discovery.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it, vi } from "vitest";
+import { beforeEach, describe, expect, it, vi } from "vitest";
 
 // Hoisted mock for `../adapters/runtimes.js` so each suite can stub
 // `detectRuntimes()` independently — we want coverage of the "empty
@@ -24,7 +24,13 @@ vi.mock("../adapters/runtimes.js", async () => {
   };
 });
 
-const { collectRuntimeSnapshot, createProvisioner } = await import("../provision.js");
+const { collectRuntimeSnapshot, clearRuntimeProbeCache, createProvisioner } = await import("../provision.js");
+
+beforeEach(() => {
+  // The L1 probe is memoized for 30s in production; tests rotate the
+  // mocked runtime list between cases, so reset before each.
+  clearRuntimeProbeCache();
+});
 const { pushRuntimeSnapshot } = await import("../daemon.js");
 const { CONTROL_FRAME_TYPES } = await import("@botcord/protocol-core");
 import type { GatewayChannelConfig, GatewayRuntimeSnapshot } from "../gateway/index.js";

package/src/control-channel.ts

@@ -18,13 +18,21 @@ import {
 } from "@botcord/protocol-core";
 import { log as daemonLog } from "./log.js";
 import {
+  AuthRefreshRejectedError,
   writeAuthExpiredFlag,
   type UserAuthManager,
 } from "./user-auth.js";
 
 /** Exponential backoff plan for transient disconnects. */
 const RECONNECT_BACKOFF_MS = [1000, 2000, 4000, 8000, 16000, 30000];
-const KEEPALIVE_INTERVAL_MS = 25_000;
+/**
+ * Keepalive cadence. Has to stay below the smallest idle-timeout in any
+ * intermediary on the daemon → Hub WS path. Cloudflare and AWS ALB both
+ * default to ~60s of idle without app-level data, and some tunnels strip
+ * WS-level ping/pong control frames entirely — hence we send an app-level
+ * `pong` heartbeat alongside `ws.ping()` rather than relying on it alone.
+ */
+const KEEPALIVE_INTERVAL_MS = 20_000;
 const REPLAY_DEDUPE_CAP = 256;
 
 /**
@@ -142,8 +150,17 @@ export class ControlChannel {
     });
     this.connectInflight = this.connect().catch((err) => {
       // Initial connect failure surfaces to the caller; subsequent
-      // reconnects are handled opaquely inside onClose.
-      this.scheduleReconnect(err);
+      // reconnects are handled opaquely inside onClose. A refresh-rejected
+      // error means the refresh token itself is dead — no point retrying;
+      // writeAuthExpiredFlag was already called in user-auth.refresh().
+      if (err instanceof AuthRefreshRejectedError) {
+        this.stopRequested = true;
+        daemonLog.warn("control-channel: refresh rejected; stopping (re-login required)", {
+          status: err.status,
+        });
+      } else {
+        this.scheduleReconnect(err);
+      }
       throw err;
     });
     try {
@@ -248,11 +265,23 @@ export class ControlChannel {
     this.keepaliveTimer = setInterval(() => {
       const ws = this.ws;
       if (!ws || ws.readyState !== WebSocket.OPEN) return;
+      // WS-level ping for normal cases.
       try {
         ws.ping();
       } catch {
         // ignore — next failed send will trigger close
       }
+      // App-level heartbeat: a `pong` daemon-initiated frame. Hub recognizes
+      // it via `_DAEMON_INITIATED_TYPES` and bumps `last_seen_at`. Critical
+      // when an intermediary (Cloudflare, AWS ALB, some k8s ingresses)
+      // drops WS-level control frames — those proxies idle-close the WS at
+      // ~60s without app-level activity, masquerading as a clean 1006 to
+      // both peers.
+      this.send({
+        id: `hb_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
+        type: "pong",
+        ts: Date.now(),
+      });
     }, this.keepaliveMs);
   }
 
@@ -285,6 +314,13 @@ export class ControlChannel {
 
   private scheduleReconnect(err?: unknown): void {
     if (this.stopRequested) return;
+    if (err instanceof AuthRefreshRejectedError) {
+      this.stopRequested = true;
+      daemonLog.warn("control-channel: refresh rejected; halting reconnect (re-login required)", {
+        status: err.status,
+      });
+      return;
+    }
     const attempt = this.reconnectAttempts;
     this.reconnectAttempts = attempt + 1;
     const delay = this.backoff[Math.min(attempt, this.backoff.length - 1)];
@@ -314,6 +350,16 @@ export class ControlChannel {
       return;
     }
     if (!frame || typeof frame.id !== "string" || typeof frame.type !== "string") {
+      // Hub ack responses for daemon-initiated frames (runtime_snapshot push,
+      // heartbeat, etc.) carry `{id, ok}` and no `type`. They're expected,
+      // not malformed — drop silently. Anything else stays a warn.
+      if (
+        frame &&
+        typeof (frame as { id?: unknown }).id === "string" &&
+        typeof (frame as { ok?: unknown }).ok === "boolean"
+      ) {
+        return;
+      }
       daemonLog.warn("control-channel: malformed frame", { frame });
       return;
     }

package/src/doctor.ts

@@ -257,6 +257,9 @@ export function renderDoctor(input: DoctorInput): string {
     lines.push(
       `${pad(r.runtime, widths.runtime)}  ${pad(r.name, widths.name)}  ${pad(r.status, widths.status)}  ${pad(r.version, widths.version)}  ${r.path}`,
     );
+    if (!e.result.available && e.installHint) {
+      lines.push(`    → ${e.installHint}`);
+    }
     if (e.endpoints && e.endpoints.length > 0) {
       for (const ep of e.endpoints) {
         const mark = ep.reachable ? "✓" : "✗";

package/src/gateway/__tests__/hermes-agent-adapter.test.ts

@@ -2,6 +2,7 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest";
 import {
   chmodSync,
   existsSync,
+  mkdirSync,
   mkdtempSync,
   readFileSync,
   rmSync,
@@ -9,7 +10,10 @@ import {
 } from "node:fs";
 import os from "node:os";
 import path from "node:path";
-import { HermesAgentAdapter } from "../runtimes/hermes-agent.js";
+import {
+  HermesAgentAdapter,
+  resolveHermesAcpCommand,
+} from "../runtimes/hermes-agent.js";
 import { agentHermesWorkspaceDir } from "../../agent-workspace.js";
 
 // Spawn a tiny Node "ACP server" we control instead of the real hermes-acp.
@@ -288,6 +292,30 @@ describe("HermesAgentAdapter", () => {
     expect(res.error).toMatch(/aborted before spawn/);
   });
 
+  it("resolveHermesAcpCommand falls back to ~/.hermes venv when PATH lookup fails", () => {
+    // Upstream `scripts/install.sh` puts hermes-acp at
+    // ~/.hermes/hermes-agent/venv/bin/hermes-acp and only symlinks `hermes`
+    // into ~/.local/bin. Simulate that layout: `which hermes-acp` fails,
+    // but the venv path exists on disk.
+    const fakeHome = mkdtempSync(path.join(os.tmpdir(), "hermes-fallback-"));
+    const venvBin = path.join(fakeHome, ".hermes", "hermes-agent", "venv", "bin");
+    const target = path.join(venvBin, "hermes-acp");
+    mkdirSync(venvBin, { recursive: true });
+    writeFileSync(target, "#!/bin/sh\nexit 0\n", { mode: 0o755 });
+    chmodSync(target, 0o755);
+
+    const resolved = resolveHermesAcpCommand({
+      env: { PATH: "/nonexistent" },
+      homeDir: fakeHome,
+      execFileSyncFn: (() => {
+        throw new Error("which: not found");
+      }) as never,
+    });
+    expect(resolved).toBe(target);
+
+    rmSync(fakeHome, { recursive: true, force: true });
+  });
+
   it("surfaces non-zero exit with stderr snippet", async () => {
     const p = path.join(tmpRoot, "boom.js");
     writeFileSync(

package/src/gateway/runtimes/hermes-agent.ts

@@ -13,12 +13,45 @@ import {
   type AcpUpdateCtx,
   type AcpUpdateParams,
 } from "./acp-stream.js";
-import { readCommandVersion, resolveCommandOnPath, type ProbeDeps } from "./probe.js";
+import {
+  firstExistingPath,
+  readCommandVersion,
+  resolveCommandOnPath,
+  resolveHomePath,
+  type ProbeDeps,
+} from "./probe.js";
 import type { RuntimeProbeResult, RuntimeRunOptions, StreamBlock } from "../types.js";
 
-/** Resolve the `hermes-acp` executable on PATH. */
+/**
+ * Known absolute locations of the `hermes-acp` entry point when it is not on
+ * PATH. The upstream `scripts/install.sh` (curl|bash installer) installs a
+ * private virtualenv under `~/.hermes/hermes-agent/venv/` and only symlinks
+ * the user-facing `hermes` command into `~/.local/bin/` — the `hermes-acp`
+ * entry point stays inside the venv. Without a fallback, daemon's PATH-only
+ * probe misses every user who installed via the README-recommended script.
+ */
+const HERMES_ACP_FALLBACK_RELATIVE_PATHS = [
+  path.join(".hermes", "hermes-agent", "venv", "bin", "hermes-acp"),
+];
+const HERMES_ACP_FALLBACK_SYSTEM_PATHS = [
+  "/opt/hermes/hermes-agent/venv/bin/hermes-acp",
+];
+
+/**
+ * Resolve the `hermes-acp` executable. Tries PATH first, then falls back to
+ * the upstream install.sh's private venv location (`~/.hermes/...`) before
+ * giving up. `BOTCORD_HERMES_AGENT_BIN` always wins via the adapter override.
+ */
 export function resolveHermesAcpCommand(deps: ProbeDeps = {}): string | null {
-  return resolveCommandOnPath("hermes-acp", deps);
+  const onPath = resolveCommandOnPath("hermes-acp", deps);
+  if (onPath) return onPath;
+  return firstExistingPath(
+    [
+      ...HERMES_ACP_FALLBACK_RELATIVE_PATHS.map((p) => resolveHomePath(p, deps)),
+      ...HERMES_ACP_FALLBACK_SYSTEM_PATHS,
+    ],
+    deps,
+  );
 }
 
 /** Probe whether `hermes-acp` is installed and report its version. */

package/src/gateway/runtimes/registry.ts

@@ -29,6 +29,11 @@ export interface RuntimeModule {
    * config loader rejects routing turns to this adapter.
    */
   supportsRun?: boolean;
+  /**
+   * Short, single-line install hint shown by `doctor` when the runtime
+   * probes as unavailable. Helps users recover without reading source.
+   */
+  installHint?: string;
 }
 
 /** Built-in runtime module entry for Claude Code. */
@@ -58,6 +63,8 @@ export const hermesAgentModule: RuntimeModule = {
   envVar: "BOTCORD_HERMES_AGENT_BIN",
   probe: () => probeHermesAgent(),
   create: () => new HermesAgentAdapter(),
+  installHint:
+    'Install: pip install "hermes-agent[acp]"  (or set BOTCORD_HERMES_AGENT_BIN to the absolute path of hermes-acp)',
 };
 
 /** Built-in runtime module entry for Gemini (probe-only stub). */
@@ -143,6 +150,7 @@ export interface RuntimeProbeEntry {
   binary: string;
   supportsRun: boolean;
   result: RuntimeProbeResult;
+  installHint?: string;
 }
 
 /** Probe every registered runtime and report installation status. */
@@ -161,6 +169,7 @@ export function detectRuntimes(): RuntimeProbeEntry[] {
       binary: m.binary,
       supportsRun: m.supportsRun !== false,
       result,
+      installHint: m.installHint,
     });
   }
   return out;

package/src/provision.ts

@@ -903,15 +903,39 @@ export function removeAgentFromConfig(
 // runtime-discovery snapshot (plan §8.5)
 // ---------------------------------------------------------------------------
 
+/**
+ * TTL for the L1 runtime-detection cache. `detectRuntimes()` shells out to
+ * each adapter binary (claude / codex / gemini / openclaw / hermes) to read
+ * `--version`, which routinely costs 1.5–2s in aggregate — long enough to
+ * push `list_runtimes` past the Hub's 10s ack budget when combined with the
+ * 3s openclaw gateway probe. Versions don't change between dashboard refresh
+ * clicks, so cache the L1 snapshot briefly and recompute on miss.
+ */
+const RUNTIME_PROBE_CACHE_TTL_MS = 30_000;
+
+let _runtimeProbeCache: { at: number; value: ListRuntimesResult } | null = null;
+
+/** Drop the cache (e.g. before a `doctor`-style interactive re-probe). */
+export function clearRuntimeProbeCache(): void {
+  _runtimeProbeCache = null;
+}
+
 /**
  * Probe every registered adapter and shape the result as the wire-level
  * {@link ListRuntimesResult} — used by both the `list_runtimes` ack path and
  * the daemon-side first-connect `runtime_snapshot` push in `daemon.ts`.
  *
- * Kept pure: the only side effects are `detectRuntimes()` itself (which the
- * gateway already isolates from throwing) and reading the wall clock.
+ * Cached for {@link RUNTIME_PROBE_CACHE_TTL_MS}; pass `{ force: true }` to
+ * bypass the cache.
  */
-export function collectRuntimeSnapshot(): ListRuntimesResult {
+export function collectRuntimeSnapshot(opts: { force?: boolean } = {}): ListRuntimesResult {
+  if (
+    !opts.force &&
+    _runtimeProbeCache &&
+    Date.now() - _runtimeProbeCache.at < RUNTIME_PROBE_CACHE_TTL_MS
+  ) {
+    return _runtimeProbeCache.value;
+  }
   const entries = detectRuntimes();
   const runtimes: RuntimeProbeResult[] = entries.map((entry) => {
     const record: RuntimeProbeResult = {
@@ -929,7 +953,9 @@ export function collectRuntimeSnapshot(): ListRuntimesResult {
     // enough; filling a synthetic message would be misleading.
     return record;
   });
-  return { runtimes, probedAt: Date.now() };
+  const value: ListRuntimesResult = { runtimes, probedAt: Date.now() };
+  _runtimeProbeCache = { at: Date.now(), value };
+  return value;
 }
 
 /** Maximum number of `endpoints[]` entries persisted per runtime (RFC §3.8.2). */
@@ -1208,7 +1234,7 @@ export async function collectRuntimeSnapshotAsync(opts: {
   const gateways = opts.cfg?.openclawGateways ?? [];
   if (gateways.length === 0) return base;
   // Default daemon-side budget is 3s — it must stay below the Hub's
-  // `list_runtimes` ack wait (5s, see backend/hub/routers/daemon_control.py)
+  // `list_runtimes` ack wait (10s, see backend/hub/routers/daemon_control.py)
   // so a single slow gateway can't blow the whole snapshot to a 504.
   const timeoutMs = opts.timeoutMs ?? 3000;
   const capped = gateways.slice(0, RUNTIME_ENDPOINTS_CAP);

package/src/user-auth.ts

@@ -188,6 +188,20 @@ export function isTokenNearExpiry(record: UserAuthRecord, windowMs = 60_000): bo
   return record.expiresAt - Date.now() <= windowMs;
 }
 
+/**
+ * Thrown when the Hub rejects a refresh token (401/403). Signals that the
+ * user must re-login — reconnect loops should stop instead of hammering
+ * the refresh endpoint forever with a known-bad token.
+ */
+export class AuthRefreshRejectedError extends Error {
+  readonly status: number;
+  constructor(status: number, message: string) {
+    super(message);
+    this.name = "AuthRefreshRejectedError";
+    this.status = status;
+  }
+}
+
 /**
  * Stateful helper that owns the in-memory copy of user-auth and knows how
  * to refresh it. Used by the control channel so reconnects always carry
@@ -245,13 +259,35 @@ export class UserAuthManager {
       expiresInMs: current.expiresAt - Date.now(),
     });
     this.refreshInflight = (async () => {
-      const tok = await refreshDaemonToken(current.hubUrl, current.refreshToken);
+      // Refresh tokens rotate server-side. If another local process (e.g. a
+      // second daemon racing on the same user-auth.json) refreshed in the
+      // meantime, the on-disk refreshToken now differs from our in-memory
+      // copy — using the in-memory one would 401 because the server already
+      // invalidated it. Re-read disk first and adopt any newer record.
+      let basis = current;
+      try {
+        const onDisk = loadUserAuth(this.file);
+        if (onDisk && onDisk.refreshToken !== current.refreshToken) {
+          daemonLog.info("user-auth refresh: adopting newer on-disk token", {
+            userId: onDisk.userId,
+            expiresAt: onDisk.expiresAt,
+          });
+          this.record = onDisk;
+          if (!isTokenNearExpiry(onDisk)) return onDisk;
+          basis = onDisk;
+        }
+      } catch (err) {
+        daemonLog.debug("user-auth refresh: disk reread failed (ignored)", {
+          error: err instanceof Error ? err.message : String(err),
+        });
+      }
+      const tok = await refreshDaemonToken(basis.hubUrl, basis.refreshToken);
       const next: UserAuthRecord = {
-        ...current,
+        ...basis,
         accessToken: tok.accessToken,
         refreshToken: tok.refreshToken,
         expiresAt: Date.now() + tok.expiresIn * 1000,
-        hubUrl: tok.hubUrl || current.hubUrl,
+        hubUrl: tok.hubUrl || basis.hubUrl,
       };
       saveUserAuth(next, this.file);
       this.record = next;
@@ -261,10 +297,23 @@ export class UserAuthManager {
       });
       return next;
     })().catch((err) => {
+      const status =
+        typeof (err as { status?: unknown }).status === "number"
+          ? ((err as { status: number }).status)
+          : null;
+      const message = err instanceof Error ? err.message : String(err);
       daemonLog.warn("user-auth refresh: failed", {
         userId: current.userId,
-        error: err instanceof Error ? err.message : String(err),
+        status,
+        error: message,
       });
+      if (status === 401 || status === 403) {
+        // Refresh token is permanently dead — write the expired flag so
+        // `status` surfaces it and re-throw a typed error so the control
+        // channel can stop reconnect loops instead of hammering the Hub.
+        writeAuthExpiredFlag();
+        throw new AuthRefreshRejectedError(status, message);
+      }
       throw err;
     }).finally(() => {
       this.refreshInflight = null;