clawmatrix 0.2.3 → 0.2.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawmatrix",
-  "version": "0.2.3",
+  "version": "0.2.5",
   "description": "Decentralized mesh cluster plugin for OpenClaw — inter-gateway communication, model proxy, task handoff, and tool proxy.",
   "type": "module",
   "license": "MIT",

package/src/cli.ts

@@ -99,7 +99,7 @@ export const registerClusterCli = ({ program }: { program: Command }) => {
         models: Array<{ id: string }>;
         tags: string[];
         connected: boolean;
-        status: "direct" | "relay" | "unreachable";
+        status: "direct" | "relay" | "unreachable" | "sentinel-only";
         latencyMs: number;
         reachableVia: string | null;
       }>;
@@ -121,10 +121,15 @@ export const registerClusterCli = ({ program }: { program: Command }) => {
 
       for (let i = 0; i < peers.length; i++) {
         const peer = peers[i];
-        const dot = peer.status === "direct" ? green("●") : peer.status === "relay" ? yellow("●") : red("○");
+        const dot = peer.status === "direct" ? green("●")
+          : peer.status === "relay" ? yellow("●")
+          : peer.status === "sentinel-only" ? yellow("◐")
+          : red("○");
         const latency = peer.connected && peer.latencyMs > 0 ? dim(` ${peer.latencyMs}ms`) : "";
         const statusLabel = peer.status === "relay"
           ? yellow(` relay via ${peer.reachableVia}`)
+          : peer.status === "sentinel-only"
+            ? yellow(" sentinel only")
           : peer.status === "unreachable"
             ? red(" unreachable")
             : "";

package/src/cluster-service.ts

@@ -505,12 +505,14 @@ export function createClusterService(
   config: ClawMatrixConfig,
   openclawConfig: OpenClawConfig,
   openclawVersion?: string,
+  onStarted?: () => void,
 ): OpenClawPluginService {
   return {
     id: "clawmatrix",
     start(ctx: OpenClawPluginServiceContext) {
       clusterRuntime = new ClusterRuntime(config, ctx.logger, openclawConfig, openclawVersion);
       clusterRuntime.start();
+      onStarted?.();
     },
     async stop() {
       if (clusterRuntime) {

package/src/compat.ts

@@ -6,6 +6,7 @@
 
 import { spawn as cpSpawn } from "node:child_process";
 import { readFile, writeFile } from "node:fs/promises";
+import { createRequire } from "node:module";
 
 export interface SpawnResult {
   exitCode: number;
@@ -97,8 +98,8 @@ let ptyModule: {
 function loadPty() {
   if (ptyModule !== undefined) return ptyModule;
   try {
-    // eslint-disable-next-line @typescript-eslint/no-require-imports
-    ptyModule = require("node-pty");
+    const req = createRequire(import.meta.url);
+    ptyModule = req("node-pty");
   } catch {
     ptyModule = null;
   }

package/src/identity.ts

@@ -26,6 +26,7 @@
  * ECDH exchange with the peer's key.
  */
 
+import { createPrivateKey } from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
 import {
@@ -54,10 +55,12 @@ export function loadOrCreateIdentity(stateDir: string): KeyPair {
     if (fs.existsSync(filePath)) {
       const raw = fs.readFileSync(filePath, "utf-8");
       const data: IdentityData = JSON.parse(raw);
-      return keyPairFromSerialized(data.publicKey, data.privateKey);
+      const keyPair = keyPairFromSerialized(data.publicKey, data.privateKey);
+      console.error(`[clawmatrix:identity] loaded existing identity from ${filePath} (publicKey=${data.publicKey.slice(0, 12)}...)`);
+      return keyPair;
     }
-  } catch {
-    // Corrupted file — regenerate
+  } catch (err) {
+    console.error(`[clawmatrix:identity] failed to load identity from ${filePath}, regenerating: ${err}`);
   }
 
   // Generate new identity
@@ -73,13 +76,13 @@ export function loadOrCreateIdentity(stateDir: string): KeyPair {
     fs.mkdirSync(stateDir, { recursive: true });
   }
   fs.writeFileSync(filePath, JSON.stringify(data, null, 2), { mode: 0o600 });
+  console.error(`[clawmatrix:identity] generated NEW identity at ${filePath} (publicKey=${data.publicKey.slice(0, 12)}...)`);
 
   return keyPair;
 }
 
 /** Reconstruct a KeyPair from serialized base64 strings. */
 function keyPairFromSerialized(publicKeyB64: string, privateKeyB64: string): KeyPair {
-  const { createPrivateKey } = require("node:crypto");
   const publicKey = Buffer.from(publicKeyB64, "base64");
   const privateKey = Buffer.from(privateKeyB64, "base64");
 

package/src/index.ts

@@ -136,7 +136,12 @@ const plugin = {
     }
 
     // Background service: manages mesh connections, WS listener, heartbeat
-    api.registerService(createClusterService(config, api.config, api.runtime.version));
+    // onStarted callback wires up approval after the runtime is available
+    let onServiceStarted: (() => void) | undefined;
+    const serviceStartedPromise = config.peerApproval.enabled
+      ? new Promise<void>((resolve) => { onServiceStarted = resolve; })
+      : undefined;
+    api.registerService(createClusterService(config, api.config, api.runtime.version, onServiceStarted));
 
     // Model providers: register per-node providers so models are accessed as nodeId/modelId
     const baseUrl = `http://127.0.0.1:${config.proxyPort}/v1`;
@@ -425,20 +430,11 @@ const plugin = {
         }
       };
 
-      // Retry until cluster runtime is initialized (service start is async)
-      const retrySetup = (attempt = 0) => {
-        try {
-          getClusterRuntime();
-          setupApproval();
-        } catch {
-          if (attempt < 30) {
-            setTimeout(() => retrySetup(attempt + 1), 1000);
-          } else {
-            debug("approval", "setupApproval gave up after 30 attempts — cluster runtime never initialized");
-          }
-        }
-      };
-      setTimeout(() => retrySetup(), 1000);
+      // Run setupApproval once the cluster service has started
+      serviceStartedPromise!.then(() => {
+        // Small delay to ensure runtime is fully wired
+        setTimeout(setupApproval, 100);
+      });
     }
 
     // Gateway methods (queried by CLI via `openclaw gateway call`)
@@ -448,6 +444,8 @@ const plugin = {
         try {
           const runtime = getClusterRuntime();
           const peers = runtime.peerManager.router.getAllPeers();
+          const mergedPeers = mergeSentinelPeers(peers, runtime)
+            .filter((p) => (p as { nodeId: string }).nodeId !== config.nodeId);
           respond(true, {
             nodeId: config.nodeId,
             listen: config.listen ? config.listenPort : false,
@@ -455,7 +453,7 @@ const plugin = {
             agents: config.agents.map((a) => ({ id: a.id, description: a.description })),
             models: config.models.map((m) => ({ id: m.id })),
             tags: config.tags,
-            peers: mergeSentinelPeers(peers, runtime),
+            peers: mergedPeers,
           });
         } catch {
           respond(false, { error: "ClawMatrix service not running" });
@@ -736,16 +734,18 @@ function mergeSentinelPeers(
     const status = runtime.peerManager.router.getPeerStatus(p);
     const sentinel = sentinelMap.get(p.nodeId);
     const sentinelStatus = sentinel ? runtime.peerManager.router.getPeerStatus(sentinel) : undefined;
+    const sentinelOnline = sentinelStatus === "direct" || sentinelStatus === "relay";
+    const effectiveStatus = status === "unreachable" && sentinelOnline ? "sentinel-only" : status;
     result.push({
       nodeId: p.nodeId,
       agents: p.agents,
       models: p.models,
       tags: p.tags,
       connected: status !== "unreachable",
-      status,
+      status: effectiveStatus,
       reachableVia: p.reachableVia,
       latencyMs: p.latencyMs,
-      ...(sentinel ? { sentinel: sentinelStatus === "direct" || sentinelStatus === "relay" ? "online" : "offline" } : {}),
+      ...(sentinel ? { sentinel: sentinelOnline ? "online" : "offline" } : {}),
     });
   }
 
@@ -753,15 +753,16 @@ function mergeSentinelPeers(
   for (const [mainId, sentinel] of sentinelMap) {
     if (seen.has(mainId)) continue;
     const sentinelStatus = runtime.peerManager.router.getPeerStatus(sentinel);
+    const sentinelOnline = sentinelStatus === "direct" || sentinelStatus === "relay";
     result.push({
       nodeId: mainId,
       agents: [],
       models: [],
       tags: [],
       connected: false,
-      status: "unreachable",
+      status: sentinelOnline ? "sentinel-only" : "unreachable",
       latencyMs: sentinel.latencyMs,
-      sentinel: sentinelStatus === "direct" || sentinelStatus === "relay" ? "online" : "offline",
+      sentinel: sentinelOnline ? "online" : "offline",
     });
   }
 

package/src/peer-manager.ts

@@ -28,6 +28,23 @@ import type { KeyPair } from "./crypto.ts";
 const RECONNECT_BASE = 1_000;
 const RECONNECT_MAX = 60_000;
 
+/** Classify WebSocket close code into a human-readable reason. */
+function classifyCloseReason(code: number, reason: string): string {
+  if (reason) return reason;
+  switch (code) {
+    case 1006: return "unreachable (node may be down)";
+    case 1000: return "normal close";
+    case 1001: return "peer going away";
+    case 1002: return "protocol error";
+    case 1003: return "unsupported data";
+    case 1008: return "policy violation";
+    case 1011: return "server error";
+    case 4001: return "auth failed";
+    case 4003: return "auth timeout";
+    default: return `close code ${code}`;
+  }
+}
+
 /** Check if an IP is a loopback address (IPv4 127.x or IPv6 ::1). */
 function isLoopback(ip?: string): boolean {
   if (!ip) return false;
@@ -282,12 +299,26 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
 
   // ── Outbound connections (standard WebSocket) ──────────────────
   private connectToPeer(peer: PeerConfig) {
-    if (this.stopped) return;
+    if (this.stopped) {
+      debug("peer", `connectToPeer(${peer.nodeId}): skipped (stopped)`);
+      return;
+    }
+
+    const attempt = this.reconnectAttempts.get(peer.nodeId) ?? 0;
+    debug("peer", `connectToPeer(${peer.nodeId}): attempt=${attempt} url=${peer.url}`);
 
     // Use a common WS subprotocol for traffic disguise
-    const ws = new WebSocket(peer.url, ["graphql-transport-ws"]);
+    let ws: WebSocket;
+    try {
+      ws = new WebSocket(peer.url, ["graphql-transport-ws"]);
+    } catch (err) {
+      debug("peer", `connectToPeer(${peer.nodeId}): WebSocket constructor threw: ${err}`);
+      this.scheduleReconnect(peer);
+      return;
+    }
 
     ws.addEventListener("open", () => {
+      debug("peer", `connectToPeer(${peer.nodeId}): ws open`);
       const conn = new Connection(
         ws,
         "outbound",
@@ -299,6 +330,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       conn.bindWebSocket(ws);
 
       conn.on("authenticated", (caps) => {
+        debug("peer", `connectToPeer(${peer.nodeId}): authenticated`);
         this.reconnectAttempts.delete(peer.nodeId);
         this.onPeerAuthenticated(conn, caps);
       });
@@ -309,24 +341,38 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     });
 
     let reconnectScheduled = false;
+    let lastError: string | undefined;
     const tryReconnect = () => {
       if (!reconnectScheduled) {
         reconnectScheduled = true;
-        this.scheduleReconnect(peer);
+        this.scheduleReconnect(peer, lastError);
       }
     };
 
-    ws.addEventListener("error", tryReconnect);
-    ws.addEventListener("close", tryReconnect);
+    ws.addEventListener("error", (ev) => {
+      lastError = (ev as ErrorEvent).message || undefined;
+      tryReconnect();
+    });
+    ws.addEventListener("close", (ev) => {
+      if (!lastError) {
+        lastError = classifyCloseReason(ev.code, ev.reason);
+      }
+      tryReconnect();
+    });
   }
 
-  private scheduleReconnect(peer: PeerConfig) {
-    if (this.stopped) return;
+  private scheduleReconnect(peer: PeerConfig, reason?: string) {
+    if (this.stopped) {
+      debug("peer", `scheduleReconnect(${peer.nodeId}): skipped (stopped)`);
+      return;
+    }
     if (this.reconnectTimers.has(peer.nodeId)) return;
 
     const attempt = this.reconnectAttempts.get(peer.nodeId) ?? 0;
     const delay = Math.min(RECONNECT_BASE * 2 ** attempt, RECONNECT_MAX);
     this.reconnectAttempts.set(peer.nodeId, attempt + 1);
+    const tag = reason ? ` reason="${reason}"` : "";
+    debug("peer", `scheduleReconnect(${peer.nodeId}): attempt=${attempt} delay=${delay}ms${tag}`);
 
     const timer = setTimeout(() => {
       this.reconnectTimers.delete(peer.nodeId);

package/src/tools/cluster-peers.ts

@@ -14,7 +14,9 @@ export function createClusterPeersTool(): AnyAgentTool {
     async execute() {
       try {
         const runtime = getClusterRuntime();
-        const allEntries = runtime.peerManager.router.getAllPeers();
+        const localNodeId = runtime.config.nodeId;
+        const allEntries = runtime.peerManager.router.getAllPeers()
+          .filter((e) => e.nodeId !== localNodeId && e.nodeId !== `${localNodeId}:sentinel`);
 
         // Separate sentinel peers from normal peers
         const sentinelSet = new Set<string>();
@@ -34,6 +36,8 @@ export function createClusterPeersTool(): AnyAgentTool {
             const sentinelStatus = sentinelEntry
               ? runtime.peerManager.router.getPeerStatus(sentinelEntry)
               : undefined;
+            const sentinelOnline = sentinelStatus === "direct" || sentinelStatus === "relay";
+            const effectiveStatus = status === "unreachable" && sentinelOnline ? "sentinel-only" : status;
 
             return {
               nodeId: entry.nodeId,
@@ -45,11 +49,10 @@ export function createClusterPeersTool(): AnyAgentTool {
               models: entry.models.map((m) => m.id),
               tags: entry.tags,
               tools: entry.toolProxy?.enabled ? (entry.toolProxy.allow ?? []) : [],
-              status,
+              status: effectiveStatus,
               latencyMs: entry.latencyMs,
-              // Sentinel info merged into the same row
               ...(hasSentinel ? {
-                sentinel: sentinelStatus === "direct" || sentinelStatus === "relay" ? "online" : "offline",
+                sentinel: sentinelOnline ? "online" : "offline",
               } : {}),
             };
           });
@@ -61,15 +64,16 @@ export function createClusterPeersTool(): AnyAgentTool {
           if (peers.some((p) => p.nodeId === mainNodeId)) continue;
           // Main node is gone, only sentinel remains
           const sentinelStatus = runtime.peerManager.router.getPeerStatus(entry);
+          const sentinelOnline = sentinelStatus === "direct" || sentinelStatus === "relay";
           peers.push({
             nodeId: mainNodeId,
             agents: [],
             models: [],
             tags: entry.tags.filter((t) => t !== "sentinel"),
             tools: [],
-            status: "unreachable",
+            status: sentinelOnline ? "sentinel-only" : "unreachable",
             latencyMs: entry.latencyMs,
-            sentinel: sentinelStatus === "direct" || sentinelStatus === "relay" ? "online" : "offline",
+            sentinel: sentinelOnline ? "online" : "offline",
           } as (typeof peers)[number]);
         }