clawmatrix 0.3.1 → 0.4.0

package/src/cluster-service.ts

@@ -105,6 +105,8 @@ export class ClusterRuntime {
   // Pre-built indexes for O(1) local agent lookup
   private agentById = new Map<string, ClawMatrixConfig["agents"][number]>();
   private agentsByTag = new Map<string, ClawMatrixConfig["agents"][number]>();
+  /** Track known relay peers to record health events on discovery/removal. */
+  private knownRelayPeers = new Set<string>();
 
   constructor(config: ClawMatrixConfig, logger: PluginLogger, openclawConfig: OpenClawConfig, openclawVersion?: string) {
     this.config = config;
@@ -155,6 +157,7 @@ export class ClusterRuntime {
 
     this.peerManager.on("peerCapabilitiesChanged", () => {
       this.refreshDiscoveredModels();
+      this.trackRelayPeerHealth();
     });
 
     // Web dashboard (must be set before peerManager.start() creates the HTTP server)
@@ -268,8 +271,33 @@ export class ClusterRuntime {
     // NOTE: intentionally do NOT stop sentinel here.
     // Sentinel must survive gateway shutdown — that's its entire purpose.
     // It will be replaced by killOldSentinel() on next gateway start.
+
+    // Wrap all async shutdown in a 4s timeout to prevent blocking gateway restart.
+    // OpenClaw's force-exit timer is 5s for stop, so we must finish before that.
+    const STOP_TIMEOUT_MS = 4000;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    await Promise.race([
+      this.stopInternal().then(() => { clearTimeout(timer); }),
+      new Promise<void>((resolve) => {
+        timer = setTimeout(() => {
+          this.logger.warn("[clawmatrix] Graceful shutdown timed out after 4s, forcing cleanup");
+          this.forceCleanup();
+          resolve();
+        }, STOP_TIMEOUT_MS);
+      }),
+    ]);
+    this.logger.info(`[clawmatrix] Node "${this.config.nodeId}" stopped`);
+  }
+
+  private async stopInternal() {
     await this.healthTracker.stop();
     await this.knowledgeSync?.stop();
+    this.syncCleanup();
+    await this.peerManager.stop();
+  }
+
+  /** Synchronous cleanup that never blocks. */
+  private syncCleanup() {
     this.webHandler?.destroy();
     this.handoffManager.destroy();
     this.acpProxy?.destroy();
@@ -277,8 +305,12 @@ export class ClusterRuntime {
     this.modelProxy.stop();
     this.fileTransferManager?.destroy();
     this.toolProxy.destroy();
-    await this.peerManager.stop();
-    this.logger.info(`[clawmatrix] Node "${this.config.nodeId}" stopped`);
+  }
+
+  /** Emergency cleanup when graceful shutdown times out. */
+  private forceCleanup() {
+    try { this.syncCleanup(); } catch { /* best effort */ }
+    try { this.peerManager.forceStop(); } catch { /* best effort */ }
   }
 
   private refreshDiscoveredModels() {
@@ -286,6 +318,32 @@ export class ClusterRuntime {
     this.modelProxy.updateDiscoveredModels(peers);
   }
 
+  /** Track relay peer health: record peer_online/peer_offline for relay peers. */
+  private trackRelayPeerHealth() {
+    const currentRelayPeers = new Set<string>();
+    for (const peer of this.peerManager.router.getAllPeers()) {
+      if (peer.reachableVia) {
+        currentRelayPeers.add(peer.nodeId);
+      }
+    }
+
+    // New relay peers discovered
+    for (const nodeId of currentRelayPeers) {
+      if (!this.knownRelayPeers.has(nodeId)) {
+        this.healthTracker.recordPeerOnline(nodeId, "relay");
+      }
+    }
+
+    // Relay peers that disappeared
+    for (const nodeId of this.knownRelayPeers) {
+      if (!currentRelayPeers.has(nodeId)) {
+        this.healthTracker.recordPeerOffline(nodeId, "relay_route_lost");
+      }
+    }
+
+    this.knownRelayPeers = currentRelayPeers;
+  }
+
   /** Fetch tool catalog from the local OpenClaw gateway and advertise to peers. */
   private async fetchToolCatalog() {
     const { spawnProcess } = await import("./compat.ts");
@@ -576,6 +634,12 @@ export class ClusterRuntime {
           this.acpProxy.handleSetConfigRequest(frame as import("./types.ts").AcpSetConfigRequest).catch((err) => {
             this.logger.error(`[clawmatrix] ACP set config error: ${err}`);
           });
+        } else {
+          const cf = frame as import("./types.ts").AcpSetConfigRequest;
+          this.peerManager.sendTo(cf.from, {
+            type: "acp_set_config_res", id: cf.id, from: this.config.nodeId, to: cf.from,
+            timestamp: Date.now(), payload: { success: false, error: "ACP not enabled on this node" },
+          } as import("./types.ts").AcpSetConfigResponse);
         }
         break;
       case "acp_set_config_res":
@@ -586,6 +650,12 @@ export class ClusterRuntime {
           this.acpProxy.handleSubscribeRequest(frame as import("./types.ts").AcpSubscribeRequest).catch((err) => {
             this.logger.error(`[clawmatrix] ACP subscribe error: ${err}`);
           });
+        } else {
+          const sf = frame as import("./types.ts").AcpSubscribeRequest;
+          this.peerManager.sendTo(sf.from, {
+            type: "acp_subscribe_res", id: sf.id, from: this.config.nodeId, to: sf.from,
+            timestamp: Date.now(), payload: { success: false, error: "ACP not enabled on this node" },
+          } as import("./types.ts").AcpSubscribeResponse);
         }
         break;
       case "acp_unsubscribe":

package/src/health-tracker.ts

@@ -257,9 +257,18 @@ export class HealthTracker {
     // Build timeline for each node (including self)
     const nodes: NodeTimeline[] = [];
 
+    // Collect nodeIds that have ever had peer_online events (actually connected)
+    const everConnected = new Set<string>();
+    for (const [, entry] of Object.entries(this.doc.nodes)) {
+      for (const ev of entry.events) {
+        if (ev.type === "peer_online" && ev.peer) everConnected.add(ev.peer);
+      }
+    }
+
     for (const [nodeId, entry] of Object.entries(this.doc.nodes)) {
-      // For self, use start/stop events to determine uptime
-      // For other nodes, use peer_online/peer_offline from the observing node
+      // Remote nodes: must have successfully connected at some point
+      if (nodeId !== this.nodeId && !everConnected.has(nodeId)) continue;
+
       const timeline = this.buildNodeTimeline(
         nodeId,
         entry,
@@ -269,7 +278,10 @@ export class HealthTracker {
         bucketCount,
         gaps,
       );
-      if (timeline) nodes.push(timeline);
+      if (!timeline) continue;
+      // Hide remote nodes that were offline for the entire requested range
+      if (nodeId !== this.nodeId && timeline.uptimeRatio === 0) continue;
+      nodes.push(timeline);
     }
 
     return {
@@ -345,19 +357,36 @@ export class HealthTracker {
     startTs: number,
     endTs: number,
   ): Array<[number, number]> {
-    const intervals: Array<[number, number]> = [];
-
     if (nodeId === this.nodeId) {
       // Self: start/stop events define uptime
-      // But we're looking at all nodes' data, so check if this nodeId
-      // has start/stop events (each node writes its own start/stop)
       return this.buildSelfIntervals(events, startTs, endTs);
     }
 
-    // For remote nodes: gather peer_online/peer_offline events from all observer nodes
-    // We have the CRDT doc with all nodes' events merged
-    // Look through ALL nodes' events for peer_online/peer_offline referencing this nodeId
-    return this.buildPeerIntervals(nodeId, startTs, endTs);
+    // For remote nodes: use BOTH self-reported start/stop intervals AND
+    // peer_online/peer_offline observations, then merge for best accuracy.
+    // Self-reported intervals are the primary signal (the node knows when
+    // it was running); peer observations supplement for relay peers or
+    // when CRDT sync hasn't propagated the remote node's own events.
+    const selfIntervals = this.buildSelfIntervals(events, startTs, endTs);
+    const peerIntervals = this.buildPeerIntervals(nodeId, startTs, endTs);
+    return this.mergeIntervals([...selfIntervals, ...peerIntervals]);
+  }
+
+  /** Merge overlapping intervals into a sorted, non-overlapping set. */
+  private mergeIntervals(intervals: Array<[number, number]>): Array<[number, number]> {
+    if (intervals.length <= 1) return intervals;
+    intervals.sort((a, b) => a[0] - b[0]);
+    const merged: Array<[number, number]> = [intervals[0]!];
+    for (let i = 1; i < intervals.length; i++) {
+      const prev = merged[merged.length - 1]!;
+      const cur = intervals[i]!;
+      if (cur[0] <= prev[1]) {
+        prev[1] = Math.max(prev[1], cur[1]);
+      } else {
+        merged.push(cur);
+      }
+    }
+    return merged;
   }
 
   private buildSelfIntervals(