clawmatrix 0.6.2 → 0.6.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawmatrix",
-  "version": "0.6.2",
+  "version": "0.6.3",
   "description": "Decentralized mesh cluster plugin for OpenClaw — inter-gateway communication, model proxy, task handoff, and tool proxy.",
   "type": "module",
   "license": "MIT",

package/src/acp-proxy.ts

@@ -2626,6 +2626,42 @@ export class AcpProxy {
     return null;
   }
 
+  /** Reject a specific pending request by frame id (e.g. from relay_fail). */
+  rejectPending(frameId: string, reason: string) {
+    const entry = this.pending.get(frameId);
+    if (entry) {
+      clearTimeout(entry.timer);
+      this.pending.delete(frameId);
+      entry.reject(new Error(reason));
+    }
+  }
+
+  /**
+   * Fail-fast all pending requests targeting a disconnected node (or nodes
+   * that were reachable only via that node as relay).
+   * Called by ClusterRuntime on peerDisconnected.
+   */
+  onPeerDisconnected(nodeId: string) {
+    for (const [id, entry] of this.pending) {
+      const target = entry.targetNodeId;
+      if (!target) continue;
+      // Direct match: request was targeting the disconnected node
+      if (target === nodeId) {
+        clearTimeout(entry.timer);
+        this.pending.delete(id);
+        entry.reject(new Error(`Node "${target}" disconnected while request was pending`));
+        continue;
+      }
+      // Relay match: request was targeting a node reachable via the disconnected relay
+      const route = this.peerManager.router.getRoute(target);
+      if (!route || (!route.connection?.isOpen && !route.reachableVia)) {
+        clearTimeout(entry.timer);
+        this.pending.delete(id);
+        entry.reject(new Error(`Node "${target}" became unreachable (relay "${nodeId}" disconnected)`));
+      }
+    }
+  }
+
   private sendRequest(
     targetNodeId: string,
     agent: string,

package/src/cluster-service.ts

@@ -175,6 +175,9 @@ export class ClusterRuntime {
       this.refreshDiscoveredModels();
       this.healthTracker.recordPeerOffline(nodeId);
       this.healthTracker.removePeerSync(nodeId);
+      // Fail-fast any pending requests targeting this node or relayed through it
+      this.acpProxy?.onPeerDisconnected(nodeId);
+      this.handoffManager?.onPeerDisconnected(nodeId);
     });
 
     this.peerManager.on("peerCapabilitiesChanged", () => {
@@ -275,7 +278,12 @@ export class ClusterRuntime {
     // Auto-detect ACP agents if ACP is enabled but no agents are explicitly configured
     // Check both ClawMatrix and OpenClaw configs (consistent with acpProxy creation above)
     if (this.acpProxy && (!this.config.acp?.agents || this.config.acp.agents.length === 0)) {
-      AcpProxy.detectAvailableAgents(this.config.acp?.commands).then((detected) => {
+      const openclawAcp = (this.openclawConfig as Record<string, any>).acp;
+      const detectionCommands = {
+        ...(openclawAcp?.commands && typeof openclawAcp.commands === "object" ? openclawAcp.commands : {}),
+        ...(this.config.acp?.commands ?? {}),
+      };
+      AcpProxy.detectAvailableAgents(Object.keys(detectionCommands).length > 0 ? detectionCommands : undefined).then((detected) => {
         if (detected.length > 0) {
           this.logger.info(`[clawmatrix] Auto-detected ACP agents: ${detected.map((a) => a.id).join(", ")}`);
           this.peerManager.updateAcpAgents(detected);
@@ -891,6 +899,22 @@ export class ClusterRuntime {
           this.apiHandler.pushKanbanEvent(kf.payload);
         }
         break;
+      case "relay_fail": {
+        // A relay node could not forward our frame to its target.
+        // Clean up the stale relay route and reject pending requests.
+        const target = (frame.payload as any)?.target ?? "unknown";
+        debug("dispatch", `relay_fail id=${frame.id} target=${target} from=${frame.from}`);
+        // Remove the stale relay route so we don't keep sending frames into a dead path
+        if (typeof target === "string" && target !== "unknown") {
+          this.peerManager.router.removeRelayRoute(target);
+        }
+        if (frame.id) {
+          this.acpProxy?.rejectPending(frame.id, `Relay "${frame.from}" cannot reach "${target}"`);
+          this.handoffManager?.rejectPending(frame.id, `Relay "${frame.from}" cannot reach "${target}"`);
+          this.toolProxy.rejectPending(frame.id, `Relay "${frame.from}" cannot reach "${target}"`);
+        }
+        break;
+      }
     }
   }
 

package/src/handoff.ts

@@ -744,6 +744,38 @@ export class HandoffManager {
     return false;
   }
 
+  /** Reject a specific pending request by frame id (e.g. from relay_fail). */
+  rejectPending(frameId: string, reason: string) {
+    const entry = this.pending.get(frameId);
+    if (entry) {
+      clearTimeout(entry.timer);
+      this.pending.delete(frameId);
+      entry.reject(new Error(reason));
+    }
+  }
+
+  /**
+   * Fail-fast pending handoff requests targeting a disconnected node
+   * (or nodes reachable only via that relay). Called on peerDisconnected.
+   */
+  onPeerDisconnected(nodeId: string) {
+    for (const [id, entry] of this.pending) {
+      if (entry.targetNodeId === nodeId) {
+        clearTimeout(entry.timer);
+        this.pending.delete(id);
+        entry.reject(new Error(`Node "${entry.targetNodeId}" disconnected while handoff was pending`));
+        continue;
+      }
+      // Check if the target was reachable via this relay
+      const route = this.peerManager.router.getRoute(entry.targetNodeId);
+      if (!route || (!route.connection?.isOpen && !route.reachableVia)) {
+        clearTimeout(entry.timer);
+        this.pending.delete(id);
+        entry.reject(new Error(`Node "${entry.targetNodeId}" became unreachable (relay "${nodeId}" disconnected)`));
+      }
+    }
+  }
+
   destroy() {
     if (this.staleCleanupTimer) {
       clearInterval(this.staleCleanupTimer);

package/src/peer-manager.ts

@@ -50,6 +50,8 @@ const SKIP_DEDUP_EXPLICIT = new Set([
   // File transfer
   "file_transfer_chunk", "file_transfer_chunk_ack",
   "file_transfer_ack", "file_transfer_complete",
+  // Relay failure notification (shares id with the original request)
+  "relay_fail",
 ]);
 
 function skipDedup(type: string): boolean {
@@ -151,6 +153,8 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
         acpAgents = config.acp.agents;
       } else if (Array.isArray(ocAcp?.allowedAgents) && ocAcp.allowedAgents.length > 0) {
         acpAgents = ocAcp.allowedAgents.map((id: string) => ({ id, description: "" }));
+      } else if (ocAcp?.commands && typeof ocAcp.commands === "object") {
+        acpAgents = Object.keys(ocAcp.commands).map((id) => ({ id, description: "" }));
       }
     }
     this.localCapabilities = {
@@ -231,6 +235,8 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     }
     // Start route probing for peers with multiple URLs
     this.startRouteProbing();
+    // Start periodic sweep of stale relay routes
+    this.router.startRelaySweep();
   }
 
   async stop() {
@@ -1251,6 +1257,22 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
         // No remote alternative — fall through to local handling
         // (model-proxy will handle locally or send error back)
       } else {
+        // Relay failed — clean up the stale route so we don't keep trying it,
+        // and notify the sender so it can fail-fast.
+        if (frame.to) {
+          this.router.removeRelayRoute(frame.to);
+        }
+        if (frame.from && frame.id) {
+          debug("peer", `relay failed for ${frame.type} id=${frame.id} to=${frame.to}, sending relay_fail to ${frame.from}`);
+          this.sendTo(frame.from, {
+            type: "relay_fail",
+            id: frame.id,
+            from: this.config.nodeId,
+            to: frame.from,
+            timestamp: Date.now(),
+            payload: { target: frame.to, reason: "unreachable" },
+          } as AnyClusterFrame);
+        }
         return;
       }
     }
@@ -1335,21 +1357,18 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
         if (peer.nodeId === this.config.nodeId) continue;
         if (peer.nodeId === from.remoteNodeId) {
           const prev = this.router.getRoute(peer.nodeId);
-          const hadAgents = prev?.agents.length ?? 0;
-          const hadDirectPeers = prev?.directPeers.length ?? 0;
-          const hadDeviceInfo = prev?.deviceInfo?.hostname;
-          const hadAcpAgents = prev?.acpAgents?.length ?? 0;
-          const hadToolProxyEnabled = prev?.toolProxy?.enabled;
-          const hadToolProxyCatalogLen = prev?.toolProxy?.catalog?.length ?? 0;
-          const hadToolProxyAllowLen = prev?.toolProxy?.allow?.length ?? 0;
+          const prevSnapshot = prev ? JSON.stringify({
+            a: prev.agents.length, m: prev.models.length,
+            dp: prev.directPeers?.length, tp: prev.toolProxy,
+            di: prev.deviceInfo, aa: prev.acpAgents?.length,
+          }) : "";
           this.router.updatePeerCapabilities(peer.nodeId, peer);
-          if (peer.agents.length !== hadAgents || peer.models.length !== (prev?.models.length ?? 0)
-              || (peer.directPeers?.length ?? 0) !== hadDirectPeers
-              || peer.toolProxy?.enabled !== hadToolProxyEnabled
-              || (peer.toolProxy?.catalog?.length ?? 0) !== hadToolProxyCatalogLen
-              || (peer.toolProxy?.allow?.length ?? 0) !== hadToolProxyAllowLen
-              || peer.deviceInfo?.hostname !== hadDeviceInfo
-              || (peer.acpAgents?.length ?? 0) !== hadAcpAgents) {
+          const newSnapshot = JSON.stringify({
+            a: peer.agents.length, m: peer.models.length,
+            dp: peer.directPeers?.length, tp: peer.toolProxy,
+            di: peer.deviceInfo, aa: peer.acpAgents?.length,
+          });
+          if (newSnapshot !== prevSnapshot) {
             changed = true;
           }
         } else {
@@ -1357,7 +1376,14 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
           // using them as relay would create a routing loop.
           if (peer.reachableVia === this.config.nodeId) continue;
           const existing = this.router.getRoute(peer.nodeId);
-          if (!existing) changed = true;
+          if (!existing) {
+            changed = true;
+          } else {
+            // Detect changes in existing relay peer (e.g. version update after restart)
+            const prevSnap = JSON.stringify({ a: existing.agents.length, m: existing.models.length, di: existing.deviceInfo, aa: existing.acpAgents?.length });
+            const newSnap = JSON.stringify({ a: peer.agents.length, m: peer.models.length, di: peer.deviceInfo, aa: peer.acpAgents?.length });
+            if (newSnap !== prevSnap) changed = true;
+          }
           this.router.addRelayPeer(peer, from.remoteNodeId!);
         }
       }

package/src/router.ts

@@ -8,6 +8,9 @@ const MAX_SEEN_FRAMES = 10_000;
 const MAX_FAILED_REQUESTS = 5_000;
 const SEEN_FRAME_TTL = 120_000;      // 2 minutes (was ~60-120s with double-map rotation)
 const FAILED_REQUEST_TTL = 900_000;   // 15 minutes
+/** Relay routes older than this without a peer_sync refresh are considered stale. */
+const RELAY_ROUTE_MAX_AGE = 300_000;  // 5 minutes
+const RELAY_ROUTE_SWEEP_INTERVAL = 60_000; // sweep every 60s
 
 export interface RouteEntry {
   nodeId: string;
@@ -127,8 +130,51 @@ export class Router {
     }
   }
 
+  private relaySweepTimer: ReturnType<typeof setInterval> | null = null;
+
+  /** Start periodic sweep of stale relay routes. */
+  startRelaySweep() {
+    if (this.relaySweepTimer) return;
+    this.relaySweepTimer = setInterval(() => this.removeStaleRelayRoutes(), RELAY_ROUTE_SWEEP_INTERVAL);
+  }
+
+  /** Remove relay routes whose lastSeen is older than RELAY_ROUTE_MAX_AGE. */
+  removeStaleRelayRoutes(): string[] {
+    const now = Date.now();
+    const removed: string[] = [];
+    for (const [id, entry] of this.routes) {
+      if (entry.reachableVia && !entry.connection && (now - entry.lastSeen) > RELAY_ROUTE_MAX_AGE) {
+        debug("router", `removing stale relay route: ${id} (via ${entry.reachableVia}, age=${Math.round((now - entry.lastSeen) / 1000)}s)`);
+        this.unindexEntry(entry);
+        this.routes.delete(id);
+        this.syncVersion++;
+        this.removedPeers.set(id, this.syncVersion);
+        this.peerVersions.delete(id);
+        removed.push(id);
+      }
+    }
+    return removed;
+  }
+
+  /** Remove a specific relay route (e.g. on relay_fail). No-op for direct connections. */
+  removeRelayRoute(nodeId: string): boolean {
+    const entry = this.routes.get(nodeId);
+    if (!entry || entry.connection) return false; // only remove relay routes
+    debug("router", `removing relay route on failure: ${nodeId} (via ${entry.reachableVia})`);
+    this.unindexEntry(entry);
+    this.routes.delete(nodeId);
+    this.syncVersion++;
+    this.removedPeers.set(nodeId, this.syncVersion);
+    this.peerVersions.delete(nodeId);
+    return true;
+  }
+
   /** Stop periodic cleanup. Call on shutdown. */
   destroy() {
+    if (this.relaySweepTimer) {
+      clearInterval(this.relaySweepTimer);
+      this.relaySweepTimer = null;
+    }
     this.seenFrames.clear();
     this.failedRequests.clear();
     this.channels.clear();

package/src/tool-proxy.ts

@@ -130,6 +130,23 @@ export class ToolProxy {
   }
 
   // ── Incoming response ──────────────────────────────────────────
+  /** Reject a specific pending request by frame id (e.g. from relay_fail). */
+  rejectPending(frameId: string, reason: string) {
+    const pending = this.pending.get(frameId);
+    if (pending) {
+      clearTimeout(pending.timer);
+      this.pending.delete(frameId);
+      pending.reject(new Error(reason));
+      return;
+    }
+    const batch = this.pendingBatch.get(frameId);
+    if (batch) {
+      clearTimeout(batch.timer);
+      this.pendingBatch.delete(frameId);
+      batch.reject(new Error(reason));
+    }
+  }
+
   handleResponse(frame: ToolProxyResponse) {
     if (this.peerManager.router.isFailed(frame.id)) return;
     const pending = this.pending.get(frame.id);