clawmatrix 0.2.6 → 0.2.8

package/src/cluster-service.ts

@@ -165,6 +165,18 @@ export class ClusterRuntime {
       }
     }
 
+    // Auto-detect ACP agents if ACP is enabled but no agents are explicitly configured
+    if (this.acpProxy && this.config.acp?.enabled && (!this.config.acp.agents || this.config.acp.agents.length === 0)) {
+      AcpProxy.detectAvailableAgents(this.config.acp.commands).then((detected) => {
+        if (detected.length > 0) {
+          this.logger.info(`[clawmatrix] Auto-detected ACP agents: ${detected.map((a) => a.id).join(", ")}`);
+          this.peerManager.updateAcpAgents(detected);
+        }
+      }).catch((err) => {
+        this.logger.error(`[clawmatrix] ACP agent detection failed: ${err}`);
+      });
+    }
+
     // Start subsystems
     this.peerManager.start();
     this.modelProxy.start();

package/src/compat.ts

@@ -5,7 +5,7 @@
  */
 
 import { spawn as cpSpawn } from "node:child_process";
-import { readFile, writeFile } from "node:fs/promises";
+import { open, readFile, stat, writeFile } from "node:fs/promises";
 import { createRequire } from "node:module";
 
 export interface SpawnResult {
@@ -118,12 +118,38 @@ export function spawnPty(
   const pty = loadPty();
   if (!pty) throw new Error("node-pty is not available — install it with: npm install node-pty");
 
+  // Filter out undefined values from env — node-pty's C code (posix_spawnp)
+  // cannot handle undefined entries and will fail silently.
+  const baseEnv = process.env as Record<string, string | undefined>;
+  const mergedEnv: Record<string, string> = {};
+  for (const [k, v] of Object.entries(baseEnv)) {
+    if (v !== undefined) mergedEnv[k] = v;
+  }
+  if (opts.env) {
+    for (const [k, v] of Object.entries(opts.env)) {
+      if (v !== undefined) mergedEnv[k] = v;
+    }
+  }
+
+  // Validate cwd exists to give a clear error instead of cryptic posix_spawnp failure
+  if (opts.cwd) {
+    try {
+      const fs = require("node:fs");
+      if (!fs.existsSync(opts.cwd)) {
+        throw new Error(`cwd does not exist: ${opts.cwd}`);
+      }
+    } catch (e) {
+      if (e instanceof Error && e.message.startsWith("cwd")) throw e;
+      // fs check failed, proceed anyway
+    }
+  }
+
   const proc = pty.spawn(shell, args, {
     name: "xterm-256color",
     cols: opts.cols ?? 80,
     rows: opts.rows ?? 24,
     cwd: opts.cwd,
-    env: opts.env ? { ...process.env, ...opts.env } as Record<string, string> : process.env as Record<string, string>,
+    env: mergedEnv,
   });
 
   return {
@@ -141,6 +167,41 @@ export async function readFileText(path: string): Promise<string> {
   return readFile(path, "utf-8");
 }
 
+/** Read at most `bytes` bytes from the beginning of a file as text. */
+export async function readFileHead(path: string, bytes: number): Promise<string> {
+  const fh = await open(path, "r");
+  try {
+    const buf = Buffer.alloc(bytes);
+    const { bytesRead } = await fh.read(buf, 0, bytes, 0);
+    return buf.toString("utf-8", 0, bytesRead);
+  } finally {
+    await fh.close();
+  }
+}
+
+/** Read at most `bytes` bytes from the end of a file as text. */
+export async function readFileTail(path: string, bytes: number): Promise<string> {
+  const info = await stat(path);
+  const size = info.size;
+  if (size === 0) return "";
+  const readBytes = Math.min(bytes, size);
+  const offset = size - readBytes;
+  const fh = await open(path, "r");
+  try {
+    const buf = Buffer.alloc(readBytes);
+    const { bytesRead } = await fh.read(buf, 0, readBytes, offset);
+    const text = buf.toString("utf-8", 0, bytesRead);
+    // Drop the first (possibly partial) line if we didn't read from start
+    if (offset > 0) {
+      const nl = text.indexOf("\n");
+      return nl >= 0 ? text.slice(nl + 1) : text;
+    }
+    return text;
+  } finally {
+    await fh.close();
+  }
+}
+
 /** Write text to a file (replaces Bun.write()). */
 export async function writeFileText(path: string, content: string): Promise<void> {
   await writeFile(path, content, "utf-8");

package/src/connection.ts

@@ -82,6 +82,8 @@ export class Connection extends EventEmitter<ConnectionEvents> {
   private pendingNonce: string | null = null;
   private closed = false;
   private lastPingSentAt = 0;
+  /** Timestamp of the last frame received from the remote side. */
+  private lastReceivedAt = 0;
   /** Exponential moving average of heartbeat RTT in milliseconds. */
   latencyMs = 0;
 
@@ -190,6 +192,7 @@ export class Connection extends EventEmitter<ConnectionEvents> {
   private async onRawMessage(data: unknown) {
     const str = typeof data === "string" ? data : String(data);
     if (!str.length) return;
+    this.lastReceivedAt = Date.now();
 
     let frame: AnyClusterFrame | undefined;
 
@@ -477,24 +480,54 @@ export class Connection extends EventEmitter<ConnectionEvents> {
   }
 
   // ── Heartbeat ──────────────────────────────────────────────────
+  /** Maximum silence duration before declaring the connection dead. */
+  private static readonly RECEIVE_TIMEOUT = HEARTBEAT_BASE * HEARTBEAT_TIMEOUT_COUNT + HEARTBEAT_JITTER * HEARTBEAT_TIMEOUT_COUNT;
+
   private startHeartbeat() {
+    this.lastReceivedAt = Date.now();
     const scheduleNext = () => {
       const interval = HEARTBEAT_BASE + Math.random() * HEARTBEAT_JITTER;
       this.heartbeatTimer = setTimeout(() => {
         if (this.closed) return;
+
+        // Watchdog: if no data received for a long time, the connection is dead
+        // regardless of what the heartbeat ping/pong state says.
+        const silenceMs = Date.now() - this.lastReceivedAt;
+        if (this.lastReceivedAt > 0 && silenceMs > Connection.RECEIVE_TIMEOUT) {
+          debug("heartbeat", `No data received for ${Math.round(silenceMs / 1000)}s from ${this.remoteNodeId ?? "unknown"}, closing`);
+          this.close(4002, "receive timeout");
+          return;
+        }
+
         // Increment before checking: this ping is about to be sent and
         // counts as outstanding until a pong arrives.
         this.missedPongs++;
         if (this.missedPongs >= HEARTBEAT_TIMEOUT_COUNT) {
+          debug("heartbeat", `${HEARTBEAT_TIMEOUT_COUNT} missed pongs from ${this.remoteNodeId ?? "unknown"}, closing`);
           this.close(4002, "heartbeat timeout");
           return;
         }
-        this.lastPingSentAt = Date.now();
-        this.send({
-          type: "ping",
-          from: this.nodeId,
-          timestamp: this.lastPingSentAt,
-        } as AnyClusterFrame);
+
+        // Send ping — wrapped in try-catch to prevent breaking the heartbeat chain.
+        // If send fails, the connection is dead; close it.
+        try {
+          if (this.transport.readyState !== WebSocket.OPEN) {
+            debug("heartbeat", `Transport not open (state=${this.transport.readyState}) for ${this.remoteNodeId ?? "unknown"}, closing`);
+            this.close(4002, "transport closed");
+            return;
+          }
+          this.lastPingSentAt = Date.now();
+          this.send({
+            type: "ping",
+            from: this.nodeId,
+            timestamp: this.lastPingSentAt,
+          } as AnyClusterFrame);
+        } catch (err) {
+          debug("heartbeat", `Ping send failed for ${this.remoteNodeId ?? "unknown"}: ${err}`);
+          this.close(4002, "ping send failed");
+          return;
+        }
+
         scheduleNext();
       }, interval);
     };

package/src/handoff.ts

@@ -91,6 +91,7 @@ export class HandoffManager {
     for (const [id, entry] of this.active) {
       if (entry.status === "input_required" && entry.inputRequiredAt && now - entry.inputRequiredAt > INPUT_REQUIRED_TTL) {
         this.active.delete(id);
+        this.sessionWatchers.delete(entry.sessionId);
         // Notify the requester that the handoff timed out
         this.peerManager.sendTo(entry.from, {
           type: "handoff_res",
@@ -110,6 +111,7 @@ export class HandoffManager {
       // (e.g. cancel arrived during input_required when no process was running)
       if (entry.status === "canceled") {
         this.active.delete(id);
+        this.sessionWatchers.delete(entry.sessionId);
       }
     }
     // Clean stale inputRequiredTargets (requester side)
@@ -355,6 +357,7 @@ export class HandoffManager {
 
     if (activeEntry.status === "canceled") {
       this.active.delete(id);
+      this.sessionWatchers.delete(activeEntry.sessionId);
       return;
     }
 
@@ -401,11 +404,13 @@ export class HandoffManager {
 
       if (activeEntry.status === "canceled") {
         this.active.delete(id);
+        this.sessionWatchers.delete(activeEntry.sessionId);
         return;
       }
 
       activeEntry.status = "completed";
       this.active.delete(id);
+      this.sessionWatchers.delete(activeEntry.sessionId);
 
       // Broadcast task completed
       this.taskActivity.broadcast(id, "handoff", "completed", agent, activeEntry.startedAt);
@@ -429,10 +434,12 @@ export class HandoffManager {
     } catch (err) {
       if (activeEntry.status === "canceled") {
         this.active.delete(id);
+        this.sessionWatchers.delete(activeEntry.sessionId);
         return;
       }
       activeEntry.status = "failed";
       this.active.delete(id);
+      this.sessionWatchers.delete(activeEntry.sessionId);
 
       // Broadcast task failed
       this.taskActivity.broadcast(
@@ -468,7 +475,7 @@ export class HandoffManager {
 
     const reader = body.getReader();
     const decoder = new TextDecoder();
-    let full = "";
+    const chunks: string[] = [];
     let buffer = "";
 
     try {
@@ -489,7 +496,7 @@ export class HandoffManager {
             const parsed = JSON.parse(data);
             const delta = parsed.choices?.[0]?.delta?.content;
             if (delta) {
-              full += delta;
+              chunks.push(delta);
               const streamFrame: HandoffStreamChunk = {
                 type: "handoff_stream",
                 id: handoffId,
@@ -531,7 +538,7 @@ export class HandoffManager {
     this.peerManager.sendTo(to, doneFrame);
     if (sessionId) this.sendToOtherWatchers(sessionId, to, doneFrame);
 
-    return full;
+    return chunks.join("");
   }
 
   /** Handle incoming input_required from remote (requester side). */

package/src/peer-manager.ts

@@ -28,6 +28,26 @@ import type { KeyPair } from "./crypto.ts";
 const RECONNECT_BASE = 1_000;
 const RECONNECT_MAX = 60_000;
 
+/** Frame types that bypass dedup (streams share one id across chunks; responses share id with request). */
+const SKIP_DEDUP_TYPES = new Set([
+  // Streaming
+  "model_stream", "handoff_stream", "acp_stream",
+  // Response frames (share id with their request)
+  "model_res", "tool_res", "tool_batch_res",
+  "handoff_res", "handoff_status_res", "handoff_input_required",
+  // Handoff control (reuse original handoff_req id)
+  "handoff_input", "handoff_cancel", "handoff_status",
+  // Diagnostics & approval
+  "diagnostic_exec_res", "diagnostic_status_res", "peer_approval_res",
+  // ACP responses
+  "acp_res", "acp_close_res", "acp_list_res", "acp_resume_res",
+  "acp_cancel_res", "acp_set_mode_res", "acp_get_modes_res",
+  "chat_history_res",
+  // Terminal
+  "terminal_open_res", "terminal_data", "terminal_resize",
+  "terminal_close", "terminal_close_res",
+]);
+
 /** Classify WebSocket close code into a human-readable reason. */
 function classifyCloseReason(code: number, reason: string): string {
   if (reason) return reason;
@@ -132,6 +152,16 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     this.approvalManager = new PeerApprovalManager(approvalConfig, stateDir);
   }
 
+  /** Update locally advertised ACP agents and re-broadcast to all peers. */
+  updateAcpAgents(agents: import("./types.ts").AcpAgentInfo[]) {
+    this.localCapabilities.acpAgents = agents;
+    this.router.updateLocalAcpAgents(agents);
+    // Re-sync all connected peers so they learn the updated capabilities
+    for (const conn of this.router.getDirectConnections()) {
+      this.sendPeerSync(conn);
+    }
+  }
+
   // ── Lifecycle ──────────────────────────────────────────────────
   async start() {
     await this.approvalManager.load();
@@ -354,6 +384,12 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       tryReconnect();
     });
     ws.addEventListener("close", (ev) => {
+      // Don't reconnect if this was a self-connection (peer URL points to ourselves).
+      // Without this guard, outbound detects self → closes → scheduleReconnect → loop.
+      if (ev.code === 4002 && ev.reason === "self-connection") {
+        debug("peer", `connectToPeer(${peer.nodeId}): self-connection, will not reconnect`);
+        return;
+      }
       if (!lastError) {
         lastError = classifyCloseReason(ev.code, ev.reason);
       }
@@ -387,19 +423,17 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     // Peer's persistent public key for TOFU identity binding
     const peerPublicKey = conn.remoteIdentityKey ?? undefined;
 
-    // Prevent self-connection: if outbound connection authenticated as our own nodeId
-    // (e.g. peer URL accidentally points to self), close immediately.
-    if (conn.role === "outbound" && nodeId === this.config.nodeId) {
-      debug("peer", `Self-connection detected (outbound to ${nodeId}), closing`);
+    // Prevent self-connection: close immediately if the remote side authenticated
+    // with our own nodeId.  For outbound this means the peer URL accidentally
+    // points to self; for inbound it means a remote node is (mis)using our nodeId.
+    // Exception: loopback connections with the same nodeId are local clients
+    // (Mac desktop app / iOS simulator) and are allowed through.
+    const isLocalClient = conn.role === "inbound" && nodeId === this.config.nodeId && isLoopback(ip);
+    if (nodeId === this.config.nodeId && !isLocalClient) {
+      debug("peer", `Self-connection detected (${conn.role}, nodeId=${nodeId}, ip=${ip}), closing`);
       conn.close(4002, "self-connection");
       return;
     }
-
-    // Peer approval check (inbound only — outbound peers are explicitly configured)
-    // Skip approval for same-nodeId connections from localhost (local clients
-    // like Mac desktop app / iOS simulator). An attacker would need to already
-    // be on the same machine to exploit this, which is outside our threat model.
-    const isLocalClient = nodeId === this.config.nodeId && isLoopback(ip);
     debug("approval", `onPeerAuthenticated: nodeId=${nodeId} role=${conn.role} isLocalClient=${isLocalClient} ip=${ip}`);
     if (conn.role === "inbound" && !isLocalClient) {
       // IP-level approval rate limiting (suppress noise from leaked tokens)
@@ -477,10 +511,22 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
 
   private completePeerJoin(conn: Connection, caps: NodeCapabilities) {
     const nodeId = conn.remoteNodeId!;
+
+    // If there's an existing connection for this nodeId (e.g. peer reconnected
+    // while old TCP hadn't closed yet), close it AFTER overwriting the route so
+    // the stale-close guard in onPeerDisconnected correctly skips cleanup.
+    const oldRoute = this.router.getRoute(nodeId);
+    const oldConn = oldRoute?.connection;
+
     // Same-nodeId 连接（如 Mac/iOS 桌面客户端）也正常注册路由，
     // 使得 sendTo(nodeId) 能将响应帧路由回客户端连接。
     this.router.addDirectPeer(nodeId, conn, caps);
 
+    if (oldConn && oldConn !== conn && oldConn.isOpen) {
+      debug("peer", `completePeerJoin(${nodeId}): closing replaced connection`);
+      oldConn.close(1000, "replaced by new connection");
+    }
+
     conn.on("message", (frame) => this.onFrame(frame, conn));
     conn.on("latency", (ms) => this.router.updateLatency(nodeId, ms));
     conn.on("close", () => this.onPeerDisconnected(conn));
@@ -520,6 +566,16 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     const nodeId = conn.remoteNodeId;
     if (!nodeId) return;
 
+    // Guard: only remove the peer if THIS connection is still the active one
+    // in the router. When a peer reconnects, the new connection replaces the
+    // old one in the route table. If the old connection's close event fires
+    // afterwards, it must NOT remove the new connection's route.
+    const currentRoute = this.router.getRoute(nodeId);
+    if (currentRoute?.connection && currentRoute.connection !== conn) {
+      debug("peer", `onPeerDisconnected(${nodeId}): stale connection close, current route has different connection — skipping cleanup`);
+      return;
+    }
+
     // Same-nodeId 本地客户端断开：仅清理路由，不广播 peer_leave
     if (nodeId === this.config.nodeId) {
       this.router.removePeer(nodeId);
@@ -557,32 +613,10 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       return;
     }
 
-    // Skip dedup for streaming and response frame types.
-    // Stream frames share one id across many chunks.
-    // Response frames (model_res, tool_res, handoff_res, etc.) share the same id
-    // as their request — without this exemption, a relay node that forwarded
-    // the request would mark the id as seen and then drop the returning response.
     // Skip dedup for streaming chunks (same id across many chunks) and response
     // frames (share id with their request — relay would otherwise drop the reply).
-    // handoff_input, handoff_cancel, and handoff_status reuse the original handoff_req id,
-    // so relay nodes that already forwarded the request would drop them as duplicates.
-    const skipDedup = frame.type === "model_stream" || frame.type === "handoff_stream"
-      || frame.type === "model_res" || frame.type === "tool_res"
-      || frame.type === "handoff_res" || frame.type === "handoff_status_res"
-      || frame.type === "handoff_input_required"
-      || frame.type === "handoff_input" || frame.type === "handoff_cancel"
-      || frame.type === "handoff_status"
-      || frame.type === "diagnostic_exec_res" || frame.type === "diagnostic_status_res"
-      || frame.type === "peer_approval_res"
-      || frame.type === "acp_stream" || frame.type === "acp_res"
-      || frame.type === "acp_close_res"
-      || frame.type === "acp_list_res" || frame.type === "acp_resume_res"
-      || frame.type === "acp_cancel_res"
-      || frame.type === "acp_set_mode_res" || frame.type === "acp_get_modes_res"
-      || frame.type === "terminal_open_res" || frame.type === "terminal_data"
-      || frame.type === "terminal_resize" || frame.type === "terminal_close"
-      || frame.type === "terminal_close_res";
-    if (frame.id && !skipDedup && this.router.isDuplicate(frame.id)) return;
+    // handoff_input/cancel/status reuse the original handoff_req id.
+    if (frame.id && !SKIP_DEDUP_TYPES.has(frame.type) && this.router.isDuplicate(frame.id)) return;
 
     // Handle peer approval responses locally (don't emit to cluster-service)
     if (frame.type === "peer_approval_res") {

package/src/router.ts

@@ -3,6 +3,7 @@ import type { Connection } from "./connection.ts";
 
 const DEFAULT_TTL = 3;
 const MAX_SEEN_FRAMES = 10_000;
+const MAX_FAILED_REQUESTS = 5_000;
 const ROTATE_INTERVAL = 60_000; // rotate dedup maps every 60s
 
 export interface RouteEntry {
@@ -52,6 +53,11 @@ export class Router {
     this.rotateTimer = setInterval(() => this.rotateSeenFrames(), ROTATE_INTERVAL);
   }
 
+  /** Update locally advertised ACP agents (used after auto-detection). */
+  updateLocalAcpAgents(agents: AcpAgentInfo[]) {
+    this.localAcpAgents = agents;
+  }
+
   /** Stop periodic cleanup. Call on shutdown. */
   destroy() {
     if (this.rotateTimer) {
@@ -300,6 +306,21 @@ export class Router {
    *  TTL defaults to 15 minutes — long enough for handoff timeouts. */
   markFailed(requestId: string, ttlMs = 900_000) {
     this.failedRequests.set(requestId, Date.now() + ttlMs);
+    // Evict entries when map grows too large: first expired, then FIFO
+    if (this.failedRequests.size > MAX_FAILED_REQUESTS) {
+      const now = Date.now();
+      // Pass 1: remove expired entries
+      for (const [id, expiresAt] of this.failedRequests) {
+        if (now > expiresAt) this.failedRequests.delete(id);
+      }
+      // Pass 2: if still over limit, remove oldest (insertion-order) entries
+      if (this.failedRequests.size > MAX_FAILED_REQUESTS) {
+        for (const [id] of this.failedRequests) {
+          if (this.failedRequests.size <= MAX_FAILED_REQUESTS) break;
+          this.failedRequests.delete(id);
+        }
+      }
+    }
   }
 
   isFailed(requestId: string): boolean {

package/src/terminal.ts

@@ -204,13 +204,15 @@ export class TerminalManager {
         payload: { success: true, sessionId },
       } as TerminalOpenResponse);
     } catch (err) {
+      const errMsg = err instanceof Error ? err.message : String(err);
+      debug("terminal", `PTY spawn failed: shell=${shell} cwd=${frame.payload.cwd ?? "(default)"} error=${errMsg}`);
       this.peerManager.sendTo(frame.from, {
         type: "terminal_open_res",
         id: frame.id,
         from: nodeId,
         to: frame.from,
         timestamp: Date.now(),
-        payload: { success: false, error: `Failed to spawn PTY: ${err instanceof Error ? err.message : String(err)}` },
+        payload: { success: false, error: `Failed to spawn PTY (shell=${shell}): ${errMsg}` },
       } as TerminalOpenResponse);
     }
   }