clawmatrix 0.3.1 → 0.4.1

package/src/cluster-service.ts

@@ -105,6 +105,8 @@ export class ClusterRuntime {
   // Pre-built indexes for O(1) local agent lookup
   private agentById = new Map<string, ClawMatrixConfig["agents"][number]>();
   private agentsByTag = new Map<string, ClawMatrixConfig["agents"][number]>();
+  /** Track known relay peers to record health events on discovery/removal. */
+  private knownRelayPeers = new Set<string>();
 
   constructor(config: ClawMatrixConfig, logger: PluginLogger, openclawConfig: OpenClawConfig, openclawVersion?: string) {
     this.config = config;
@@ -131,6 +133,12 @@ export class ClusterRuntime {
       this.agentById.set(a.id, a);
       for (const t of a.tags) this.agentsByTag.set(t, a);
     }
+
+    // Wire up active task checker for route probing (prevents switching mid-task)
+    this.peerManager.setActiveTaskChecker((nodeId) => {
+      return this.handoffManager.hasPendingForNode(nodeId)
+        || this.modelProxy.hasPendingForNode(nodeId);
+    });
   }
 
   async start() {
@@ -142,8 +150,11 @@ export class ClusterRuntime {
     this.peerManager.on("peerConnected", (nodeId) => {
       this.logger.info(`[clawmatrix] Peer connected: ${nodeId}`);
       this.refreshDiscoveredModels();
-      this.healthTracker.recordPeerOnline(nodeId, "direct");
+      // Init sync state BEFORE recording the event — recordPeerOnline triggers
+      // broadcastSync which must use the freshly initialized syncState.
+      // Reversing this order causes syncState corruption and infinite sync loops.
       this.healthTracker.initPeerSync(nodeId);
+      this.healthTracker.recordPeerOnline(nodeId, "direct");
     });
 
     this.peerManager.on("peerDisconnected", (nodeId) => {
@@ -155,6 +166,7 @@ export class ClusterRuntime {
 
     this.peerManager.on("peerCapabilitiesChanged", () => {
       this.refreshDiscoveredModels();
+      this.trackRelayPeerHealth();
     });
 
     // Web dashboard (must be set before peerManager.start() creates the HTTP server)
@@ -268,8 +280,33 @@ export class ClusterRuntime {
     // NOTE: intentionally do NOT stop sentinel here.
     // Sentinel must survive gateway shutdown — that's its entire purpose.
     // It will be replaced by killOldSentinel() on next gateway start.
+
+    // Wrap all async shutdown in a 4s timeout to prevent blocking gateway restart.
+    // OpenClaw's force-exit timer is 5s for stop, so we must finish before that.
+    const STOP_TIMEOUT_MS = 4000;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    await Promise.race([
+      this.stopInternal().then(() => { clearTimeout(timer); }),
+      new Promise<void>((resolve) => {
+        timer = setTimeout(() => {
+          this.logger.warn("[clawmatrix] Graceful shutdown timed out after 4s, forcing cleanup");
+          this.forceCleanup();
+          resolve();
+        }, STOP_TIMEOUT_MS);
+      }),
+    ]);
+    this.logger.info(`[clawmatrix] Node "${this.config.nodeId}" stopped`);
+  }
+
+  private async stopInternal() {
     await this.healthTracker.stop();
     await this.knowledgeSync?.stop();
+    this.syncCleanup();
+    await this.peerManager.stop();
+  }
+
+  /** Synchronous cleanup that never blocks. */
+  private syncCleanup() {
     this.webHandler?.destroy();
     this.handoffManager.destroy();
     this.acpProxy?.destroy();
@@ -277,8 +314,12 @@ export class ClusterRuntime {
     this.modelProxy.stop();
     this.fileTransferManager?.destroy();
     this.toolProxy.destroy();
-    await this.peerManager.stop();
-    this.logger.info(`[clawmatrix] Node "${this.config.nodeId}" stopped`);
+  }
+
+  /** Emergency cleanup when graceful shutdown times out. */
+  private forceCleanup() {
+    try { this.syncCleanup(); } catch { /* best effort */ }
+    try { this.peerManager.forceStop(); } catch { /* best effort */ }
   }
 
   private refreshDiscoveredModels() {
@@ -286,6 +327,32 @@ export class ClusterRuntime {
     this.modelProxy.updateDiscoveredModels(peers);
   }
 
+  /** Track relay peer health: record peer_online/peer_offline for relay peers. */
+  private trackRelayPeerHealth() {
+    const currentRelayPeers = new Set<string>();
+    for (const peer of this.peerManager.router.getAllPeers()) {
+      if (peer.reachableVia) {
+        currentRelayPeers.add(peer.nodeId);
+      }
+    }
+
+    // New relay peers discovered
+    for (const nodeId of currentRelayPeers) {
+      if (!this.knownRelayPeers.has(nodeId)) {
+        this.healthTracker.recordPeerOnline(nodeId, "relay");
+      }
+    }
+
+    // Relay peers that disappeared
+    for (const nodeId of this.knownRelayPeers) {
+      if (!currentRelayPeers.has(nodeId)) {
+        this.healthTracker.recordPeerOffline(nodeId, "relay_route_lost");
+      }
+    }
+
+    this.knownRelayPeers = currentRelayPeers;
+  }
+
   /** Fetch tool catalog from the local OpenClaw gateway and advertise to peers. */
   private async fetchToolCatalog() {
     const { spawnProcess } = await import("./compat.ts");
@@ -576,6 +643,12 @@ export class ClusterRuntime {
           this.acpProxy.handleSetConfigRequest(frame as import("./types.ts").AcpSetConfigRequest).catch((err) => {
             this.logger.error(`[clawmatrix] ACP set config error: ${err}`);
           });
+        } else {
+          const cf = frame as import("./types.ts").AcpSetConfigRequest;
+          this.peerManager.sendTo(cf.from, {
+            type: "acp_set_config_res", id: cf.id, from: this.config.nodeId, to: cf.from,
+            timestamp: Date.now(), payload: { success: false, error: "ACP not enabled on this node" },
+          } as import("./types.ts").AcpSetConfigResponse);
         }
         break;
       case "acp_set_config_res":
@@ -586,6 +659,12 @@ export class ClusterRuntime {
           this.acpProxy.handleSubscribeRequest(frame as import("./types.ts").AcpSubscribeRequest).catch((err) => {
             this.logger.error(`[clawmatrix] ACP subscribe error: ${err}`);
           });
+        } else {
+          const sf = frame as import("./types.ts").AcpSubscribeRequest;
+          this.peerManager.sendTo(sf.from, {
+            type: "acp_subscribe_res", id: sf.id, from: this.config.nodeId, to: sf.from,
+            timestamp: Date.now(), payload: { success: false, error: "ACP not enabled on this node" },
+          } as import("./types.ts").AcpSubscribeResponse);
         }
         break;
       case "acp_unsubscribe":

package/src/config.ts

@@ -51,7 +51,8 @@ const ModelInfoSchema = z.object({
 
 const PeerConfigSchema = z.object({
   nodeId: z.string(),
-  url: z.string(),
+  /** Single URL or array of URLs for multi-channel connections. */
+  url: z.union([z.string(), z.array(z.string()).min(1)]),
 });
 
 const ToolProxyConfigSchema = z.object({

package/src/handoff.ts

@@ -678,6 +678,14 @@ export class HandoffManager {
   }
 
   /** Clean up on shutdown. */
+  /** Check if there are pending outbound handoffs targeting a specific node. */
+  hasPendingForNode(nodeId: string): boolean {
+    for (const p of this.pending.values()) {
+      if (p.targetNodeId === nodeId) return true;
+    }
+    return false;
+  }
+
   destroy() {
     if (this.staleCleanupTimer) {
       clearInterval(this.staleCleanupTimer);

package/src/health-tracker.ts

@@ -79,6 +79,12 @@ export class HealthTracker {
   private compactTimer: ReturnType<typeof setInterval> | null = null;
   private saveTimer: ReturnType<typeof setTimeout> | null = null;
   private dirty = false;
+  /** Debounce timer for broadcastSync (prevents rapid-fire broadcasts). */
+  private broadcastTimer: ReturnType<typeof setTimeout> | null = null;
+  /** Round counter per peer to detect non-converging sync loops. */
+  private syncRounds = new Map<string, number>();
+  private static readonly MAX_SYNC_ROUNDS = 10;
+  private static readonly BROADCAST_DEBOUNCE = 500; // ms
 
   constructor(opts: HealthTrackerOptions) {
     this.nodeId = opts.nodeId;
@@ -119,6 +125,10 @@ export class HealthTracker {
       clearInterval(this.compactTimer);
       this.compactTimer = null;
     }
+    if (this.broadcastTimer) {
+      clearTimeout(this.broadcastTimer);
+      this.broadcastTimer = null;
+    }
     if (this.saveTimer) {
       clearTimeout(this.saveTimer);
       this.saveTimer = null;
@@ -165,6 +175,16 @@ export class HealthTracker {
     const message = new Uint8Array(Buffer.from(frame.payload.data, "base64"));
     const syncKey = peerId;
 
+    // Guard against non-converging sync loops
+    const rounds = (this.syncRounds.get(peerId) ?? 0) + 1;
+    if (rounds > HealthTracker.MAX_SYNC_ROUNDS) {
+      debug(TAG, `sync with ${peerId} exceeded ${HealthTracker.MAX_SYNC_ROUNDS} rounds, resetting`);
+      this.syncStates.set(syncKey, Automerge.initSyncState());
+      this.syncRounds.delete(peerId);
+      return;
+    }
+    this.syncRounds.set(peerId, rounds);
+
     try {
       const syncState = this.syncStates.get(syncKey) ?? Automerge.initSyncState();
       const [newDoc, newSyncState] = Automerge.receiveSyncMessage(this.doc, syncState, message);
@@ -172,18 +192,19 @@ export class HealthTracker {
       this.syncStates.set(syncKey, newSyncState);
       this.scheduleSave();
 
-      // Send our response
+      // Send our response (only if there's something to send)
       this.sendSyncMessage(peerId);
     } catch (err) {
       debug(TAG, `error handling sync from ${peerId}: ${err}`);
     }
   }
 
-  /** Initiate sync with a peer (called on peer connect). */
+  /** Initialize sync state for a peer (called on peer connect).
+   *  Does NOT send a message — the subsequent recordPeerOnline → broadcastSync handles that.
+   *  Sending here would race with broadcastSync and corrupt the sync state. */
   initPeerSync(peerId: string) {
     if (peerId === this.nodeId) return;
     this.syncStates.set(peerId, Automerge.initSyncState());
-    this.sendSyncMessage(peerId);
   }
 
   /** Clean up sync state for a disconnected peer. */
@@ -196,7 +217,11 @@ export class HealthTracker {
     const [newSyncState, message] = Automerge.generateSyncMessage(this.doc, syncState);
     this.syncStates.set(peerId, newSyncState);
 
-    if (!message) return;
+    if (!message) {
+      // Sync converged — reset round counter
+      this.syncRounds.delete(peerId);
+      return;
+    }
 
     debug(TAG, `sending health sync to ${peerId} (${message.byteLength} bytes)`);
 
@@ -214,10 +239,17 @@ export class HealthTracker {
   }
 
   private broadcastSync() {
-    const peers = this.peerManager.router.getAllPeers();
-    for (const peer of peers) {
-      this.sendSyncMessage(peer.nodeId);
-    }
+    // Debounce: multiple events in quick succession → single broadcast
+    if (this.broadcastTimer) return;
+    this.broadcastTimer = setTimeout(() => {
+      this.broadcastTimer = null;
+      // Reset round counters — new broadcast starts fresh sync cycle
+      this.syncRounds.clear();
+      const peers = this.peerManager.router.getAllPeers();
+      for (const peer of peers) {
+        this.sendSyncMessage(peer.nodeId);
+      }
+    }, HealthTracker.BROADCAST_DEBOUNCE);
   }
 
   // ── Timeline aggregation ──────────────────────────────────
@@ -257,9 +289,18 @@ export class HealthTracker {
     // Build timeline for each node (including self)
     const nodes: NodeTimeline[] = [];
 
+    // Collect nodeIds that have ever had peer_online events (actually connected)
+    const everConnected = new Set<string>();
+    for (const [, entry] of Object.entries(this.doc.nodes)) {
+      for (const ev of entry.events) {
+        if (ev.type === "peer_online" && ev.peer) everConnected.add(ev.peer);
+      }
+    }
+
     for (const [nodeId, entry] of Object.entries(this.doc.nodes)) {
-      // For self, use start/stop events to determine uptime
-      // For other nodes, use peer_online/peer_offline from the observing node
+      // Remote nodes: must have successfully connected at some point
+      if (nodeId !== this.nodeId && !everConnected.has(nodeId)) continue;
+
       const timeline = this.buildNodeTimeline(
         nodeId,
         entry,
@@ -269,7 +310,10 @@ export class HealthTracker {
         bucketCount,
         gaps,
       );
-      if (timeline) nodes.push(timeline);
+      if (!timeline) continue;
+      // Hide remote nodes that were offline for the entire requested range
+      if (nodeId !== this.nodeId && timeline.uptimeRatio === 0) continue;
+      nodes.push(timeline);
     }
 
     return {
@@ -345,19 +389,36 @@ export class HealthTracker {
     startTs: number,
     endTs: number,
   ): Array<[number, number]> {
-    const intervals: Array<[number, number]> = [];
-
     if (nodeId === this.nodeId) {
       // Self: start/stop events define uptime
-      // But we're looking at all nodes' data, so check if this nodeId
-      // has start/stop events (each node writes its own start/stop)
       return this.buildSelfIntervals(events, startTs, endTs);
     }
 
-    // For remote nodes: gather peer_online/peer_offline events from all observer nodes
-    // We have the CRDT doc with all nodes' events merged
-    // Look through ALL nodes' events for peer_online/peer_offline referencing this nodeId
-    return this.buildPeerIntervals(nodeId, startTs, endTs);
+    // For remote nodes: use BOTH self-reported start/stop intervals AND
+    // peer_online/peer_offline observations, then merge for best accuracy.
+    // Self-reported intervals are the primary signal (the node knows when
+    // it was running); peer observations supplement for relay peers or
+    // when CRDT sync hasn't propagated the remote node's own events.
+    const selfIntervals = this.buildSelfIntervals(events, startTs, endTs);
+    const peerIntervals = this.buildPeerIntervals(nodeId, startTs, endTs);
+    return this.mergeIntervals([...selfIntervals, ...peerIntervals]);
+  }
+
+  /** Merge overlapping intervals into a sorted, non-overlapping set. */
+  private mergeIntervals(intervals: Array<[number, number]>): Array<[number, number]> {
+    if (intervals.length <= 1) return intervals;
+    intervals.sort((a, b) => a[0] - b[0]);
+    const merged: Array<[number, number]> = [intervals[0]!];
+    for (let i = 1; i < intervals.length; i++) {
+      const prev = merged[merged.length - 1]!;
+      const cur = intervals[i]!;
+      if (cur[0] <= prev[1]) {
+        prev[1] = Math.max(prev[1], cur[1]);
+      } else {
+        merged.push(cur);
+      }
+    }
+    return merged;
   }
 
   private buildSelfIntervals(