clawmatrix 0.2.8 → 0.2.11

package/src/model-proxy.ts

@@ -1090,13 +1090,28 @@ export class ModelProxy {
         ?? this.config.models.find((m) => m.id === payload.model)
       : this.config.models.find((m) => m.id === payload.model);
     if (!model) {
+      // Model not available locally — try forwarding to a remote node that has it
+      const originalTarget = frame.to;
+      const exclude = new Set([this.config.nodeId]);
+      if (originalTarget) exclude.add(originalTarget);
+      const alternatives = this.peerManager.router.findNodesForModel(payload.model, exclude);
+      for (const alt of alternatives) {
+        if (this.peerManager.sendTo(alt.nodeId, { ...frame, to: alt.nodeId })) {
+          debug("model_req", `failover: ${originalTarget ?? "local"} → ${alt.nodeId} for "${payload.model}"`);
+          return;
+        }
+      }
+      // No alternative found
+      const hint = originalTarget && originalTarget !== this.config.nodeId
+        ? `Target node "${originalTarget}" is unreachable and no alternative nodes provide model "${payload.model}"`
+        : `Model "${payload.model}" not available locally`;
       this.peerManager.sendTo(from, {
         type: "model_res",
         id,
         from: this.config.nodeId,
         to: from,
         timestamp: Date.now(),
-        payload: { success: false, error: `Model "${payload.model}" not available locally` },
+        payload: { success: false, error: hint },
       } satisfies ModelResponse);
       return;
     }

package/src/peer-manager.ts

@@ -46,6 +46,9 @@ const SKIP_DEDUP_TYPES = new Set([
   // Terminal
   "terminal_open_res", "terminal_data", "terminal_resize",
   "terminal_close", "terminal_close_res",
+  // File transfer
+  "file_transfer_chunk", "file_transfer_chunk_ack",
+  "file_transfer_ack", "file_transfer_complete",
 ]);
 
 /** Classify WebSocket close code into a human-readable reason. */
@@ -604,8 +607,11 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
 
   // ── Message handling ───────────────────────────────────────────
   private onFrame(frame: AnyClusterFrame, from: Connection) {
-    // Ignore self-echo: frames with our own nodeId that were relayed back to us
-    if (frame.from === this.config.nodeId) return;
+    // Ignore self-echo: frames with our own nodeId that were relayed back to us.
+    // Exception: frames from same-nodeId satellite connections (Mac/iOS client)
+    // are legitimate requests that must be processed or relayed.
+    const isSatellite = from.remoteNodeId === this.config.nodeId;
+    if (frame.from === this.config.nodeId && !isSatellite) return;
 
     // Validate from field: must be the direct peer or a known node (relayed)
     if (frame.from && frame.from !== from.remoteNodeId && !this.router.getRoute(frame.from)) {
@@ -674,8 +680,26 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     }
 
     if (frame.to && frame.to !== this.config.nodeId) {
-      this.router.tryRelay(frame);
-      return;
+      if (this.router.tryRelay(frame)) return;
+
+      // Relay failed — for model_req, try alternative nodes or fall through to local handling
+      if (frame.type === "model_req") {
+        const modelId = (frame as any).payload?.model;
+        if (modelId) {
+          const exclude = new Set([frame.to, this.config.nodeId]);
+          const alternatives = this.router.findNodesForModel(modelId, exclude);
+          for (const alt of alternatives) {
+            if (this.sendTo(alt.nodeId, { ...frame, to: alt.nodeId })) {
+              debug("peer", `model_req failover: ${frame.to} → ${alt.nodeId}`);
+              return;
+            }
+          }
+        }
+        // No remote alternative — fall through to local handling
+        // (model-proxy will handle locally or send error back)
+      } else {
+        return;
+      }
     }
 
     // Forward to same-nodeId satellite connection (e.g. Mac desktop app) so that

package/src/router.ts

@@ -233,6 +233,31 @@ export class Router {
     return this.routes.get(target);
   }
 
+  /** Find reachable nodes that provide a specific model, sorted by latency.
+   *  Excludes nodes in the `exclude` set. */
+  findNodesForModel(modelId: string, exclude?: Set<string>): RouteEntry[] {
+    const candidates: RouteEntry[] = [];
+    for (const entry of this.routes.values()) {
+      if (exclude?.has(entry.nodeId)) continue;
+      if (!entry.models.some((m) => m.id === modelId)) continue;
+      // Check reachability
+      if (entry.connection?.isOpen) {
+        candidates.push(entry);
+      } else if (entry.reachableVia) {
+        const relay = this.connections.get(entry.reachableVia);
+        if (relay?.isOpen) candidates.push(entry);
+      }
+    }
+    // Sort: direct first, then by latency
+    candidates.sort((a, b) => {
+      const aDirect = a.connection ? 0 : 1;
+      const bDirect = b.connection ? 0 : 1;
+      if (aDirect !== bDirect) return aDirect - bDirect;
+      return a.latencyMs - b.latencyMs;
+    });
+    return candidates;
+  }
+
   // ── Message sending and relay ──────────────────────────────────
   /** Send a frame to a specific node, relaying if necessary. Returns true if sent. */
   sendTo(targetNodeId: string, frame: ClusterFrame | AnyClusterFrame): boolean {

package/src/sentinel-manager.ts

@@ -8,7 +8,7 @@
 
 import { fork, type ChildProcess } from "node:child_process";
 import { join, dirname } from "node:path";
-import { existsSync, readFileSync, mkdirSync, openSync } from "node:fs";
+import { existsSync, readFileSync, mkdirSync, openSync, closeSync } from "node:fs";
 import { homedir, tmpdir } from "node:os";
 import type { ClawMatrixConfig } from "./config.ts";
 
@@ -42,6 +42,9 @@ export class SentinelManager {
       execArgv: this.resolveExecArgv(),
     });
 
+    // Close the log fd in the parent — the child has its own copy
+    closeSync(logFd);
+
     // Send config to sentinel via IPC (includes gateway PID for health checks)
     // If sentinel has no explicit listenPort but the gateway is a listener,
     // inherit the gateway's port for automatic takeover when gateway dies.
@@ -83,7 +86,7 @@ export class SentinelManager {
     }, 1000);
   }
 
-  stop() {
+  async stop() {
     // IPC is disconnected shortly after start, so use PID file for shutdown
     if (existsSync(this.pidFile)) {
       try {
@@ -92,13 +95,11 @@ export class SentinelManager {
           process.kill(pid, "SIGTERM");
           // Wait briefly for the process to exit so the next start()
           // doesn't race with a still-dying sentinel
-          const deadline = Date.now() + 3_000;
-          while (Date.now() < deadline) {
+          for (let i = 0; i < 60; i++) {
             try {
               process.kill(pid, 0);
-              // Still alive — brief spin
-              const waitUntil = Date.now() + 50;
-              while (Date.now() < waitUntil) { /* spin */ }
+              // Still alive — async wait
+              await new Promise((r) => setTimeout(r, 50));
             } catch {
               break; // exited
             }

package/src/sentinel.ts

@@ -569,6 +569,28 @@ process.on("exit", (code) => {
   try { process.stderr.write(`[svc ${ts}] Exit code=${code}\n`); } catch { /* ignore */ }
 });
 
+/** Connect to all configured peers (called when gateway dies). */
+function connectAllPeers() {
+  for (const peer of config.peers) {
+    if (!connections.has(peer.nodeId) && !reconnectTimers.has(peer.nodeId)) {
+      connectToPeer(peer);
+    }
+  }
+}
+
+/** Disconnect from all peers (called when gateway recovers). */
+function disconnectAllPeers() {
+  for (const [nodeId, conn] of connections) {
+    conn.close(1000, "gateway recovered");
+    connections.delete(nodeId);
+  }
+  for (const [nodeId, timer] of reconnectTimers) {
+    clearTimeout(timer);
+    reconnectTimers.delete(nodeId);
+  }
+  reconnectAttempts.clear();
+}
+
 /** Periodically check if the gateway process is still alive via kill(pid, 0). */
 function startGatewayHealthCheck() {
   if (healthCheckTimer || !gatewayPid) return;
@@ -581,11 +603,15 @@ function startGatewayHealthCheck() {
         log("Gateway process detected — back online");
         // Release the port so the gateway can reclaim it
         stopListening();
+        // Disconnect from peers — gateway handles mesh connections
+        disconnectAllPeers();
       }
     } catch {
       if (gatewayAlive) {
         gatewayAlive = false;
         log(`Gateway process (pid ${gatewayPid}) gone — entering standalone mode`);
+        // Connect to peers now that gateway is down
+        connectAllPeers();
         // Take over the gateway's listen port
         if (config.listenPort) {
           // Small delay to let the OS release the port from the dead process
@@ -608,11 +634,7 @@ function boot() {
   writePidFile();
   log(`Started (pid ${process.pid}, gateway ${gatewayPid}, nodeId ${sentinelNodeId()}, takeover port ${config.listenPort || "none"})`);
 
-  // Connect to all configured peers
-  for (const peer of config.peers) {
-    connectToPeer(peer);
-  }
-
-  // Note: we do NOT start listening here.
-  // Listening only starts when gateway dies (port takeover mode).
+  // Do NOT connect to peers on boot — gateway handles mesh connections.
+  // Sentinel only connects when gateway dies (standalone mode).
+  // Listening also only starts when gateway dies (port takeover mode).
 }

package/src/terminal.ts

@@ -109,7 +109,8 @@ export class TerminalManager {
       return;
     }
 
-    // Check allowFrom
+    // TODO(security): allowFrom 为空时默认允许所有已认证 peer 打开终端会话。
+    // 当前仅用于受信任网络。开放前需改为默认拒绝或要求显式配置。
     if (termConfig?.allowFrom && termConfig.allowFrom.length > 0) {
       if (!termConfig.allowFrom.includes(frame.from)) {
         this.peerManager.sendTo(frame.from, {

package/src/tools/cluster-diagnostic.ts

@@ -88,6 +88,8 @@ export function createClusterDiagnosticTool(): AnyAgentTool {
           };
         }
 
+        // TODO(security): exec 允许任何已认证 peer 在远程 sentinel 执行任意命令，无 allowlist 或 capability check。
+        // 当前仅用于受信任网络。开放前需添加命令白名单或 per-peer 授权。
         if (action === "exec") {
           if (!command) {
             return {

package/src/tools/cluster-transfer.ts

@@ -0,0 +1,91 @@
+import type { AnyAgentTool } from "openclaw/plugin-sdk";
+import { getClusterRuntime } from "../cluster-service.ts";
+
+export function createClusterTransferTool(): AnyAgentTool {
+  return {
+    name: "cluster_transfer",
+    label: "Cluster File Transfer",
+    description:
+      "Transfer a file between the local node and a remote cluster node. " +
+      "Supports large files (up to 100MB) with chunked transfer and SHA-256 integrity check. " +
+      "Specify source_node to pull from remote, or target_node to push to remote.",
+    parameters: {
+      type: "object",
+      properties: {
+        source_node: {
+          type: "string",
+          description: "Source nodeId (omit for local). Exactly one of source_node or target_node must be provided.",
+        },
+        source_path: {
+          type: "string",
+          description: "File path on the source node",
+        },
+        target_node: {
+          type: "string",
+          description: "Target nodeId (omit for local). Exactly one of source_node or target_node must be provided.",
+        },
+        target_path: {
+          type: "string",
+          description: "File path on the target node",
+        },
+      },
+      required: ["source_path", "target_path"],
+    },
+    async execute(_toolCallId, params) {
+      const { source_node, source_path, target_node, target_path } = params as {
+        source_node?: string;
+        source_path: string;
+        target_node?: string;
+        target_path: string;
+      };
+
+      // Validate: exactly one of source_node or target_node must be provided
+      if (source_node && target_node) {
+        return {
+          content: [{ type: "text" as const, text: "Error: Provide either source_node or target_node, not both." }],
+          details: { error: true },
+        };
+      }
+      if (!source_node && !target_node) {
+        return {
+          content: [{ type: "text" as const, text: "Error: Provide either source_node (to pull) or target_node (to push)." }],
+          details: { error: true },
+        };
+      }
+
+      try {
+        const runtime = getClusterRuntime();
+        const ftm = runtime.fileTransferManager;
+        if (!ftm) {
+          return {
+            content: [{ type: "text" as const, text: "Error: File transfer is not enabled on this node." }],
+            details: { error: true },
+          };
+        }
+
+        let result;
+        if (source_node) {
+          // Pull: remote → local
+          result = await ftm.pullFile(source_node, source_path, target_path);
+        } else {
+          // Push: local → remote
+          result = await ftm.pushFile(target_node!, source_path, target_path);
+        }
+
+        const text = result.success
+          ? `Transfer complete: ${result.bytesTransferred} bytes transferred.`
+          : `Transfer failed: ${result.error}`;
+
+        return {
+          content: [{ type: "text" as const, text }],
+          details: result,
+        };
+      } catch (err) {
+        return {
+          content: [{ type: "text" as const, text: `Transfer error: ${err instanceof Error ? err.message : String(err)}` }],
+          details: { error: true },
+        };
+      }
+    },
+  };
+}

package/src/types.ts

@@ -317,6 +317,68 @@ export interface ToolBatchResponse extends ClusterFrame {
   };
 }
 
+// ── File transfer ─────────────────────────────────────────────────
+export interface FileTransferInit extends ClusterFrame {
+  type: "file_transfer_init";
+  id: string;
+  payload: {
+    sessionId: string;
+    direction: "push" | "pull";
+    filePath: string;
+    targetPath: string;
+    fileSize: number;
+    totalChunks: number;
+    chunkSize: number;
+    checksum: string; // SHA-256 hex
+  };
+}
+
+export interface FileTransferAck extends ClusterFrame {
+  type: "file_transfer_ack";
+  id: string;
+  payload: {
+    sessionId: string;
+    accepted: boolean;
+    error?: string;
+    // Pull mode: responder includes file metadata
+    fileSize?: number;
+    totalChunks?: number;
+    checksum?: string;
+  };
+}
+
+export interface FileTransferChunk extends ClusterFrame {
+  type: "file_transfer_chunk";
+  id: string;
+  payload: {
+    sessionId: string;
+    chunkIndex: number;
+    data: string; // base64-encoded
+  };
+}
+
+export interface FileTransferChunkAck extends ClusterFrame {
+  type: "file_transfer_chunk_ack";
+  id: string;
+  payload: {
+    sessionId: string;
+    chunkIndex: number;
+    success: boolean;
+    error?: string;
+  };
+}
+
+export interface FileTransferComplete extends ClusterFrame {
+  type: "file_transfer_complete";
+  id: string;
+  payload: {
+    sessionId: string;
+    success: boolean;
+    error?: string;
+    bytesTransferred?: number;
+  };
+}
+
 // ── Device info ───────────────────────────────────────────────────
 export interface DeviceInfo {
   os: string; // e.g. "Darwin 24.6.0", "Linux 6.1.0"
@@ -411,6 +473,32 @@ export interface KnowledgeSyncFrame extends ClusterFrame {
   };
 }
 
+// ── Health sync ──────────────────────────────────────────────────
+export interface HealthSyncFrame extends ClusterFrame {
+  type: "health_sync";
+  payload: {
+    data: string; // base64-encoded Automerge sync message
+  };
+}
+
+export interface AvailabilityRequest extends ClusterFrame {
+  type: "availability_req";
+  id: string;
+  payload: {
+    range: "24h" | "7d" | "90d";
+  };
+}
+
+export interface AvailabilityResponse extends ClusterFrame {
+  type: "availability_res";
+  id: string;
+  payload: {
+    success: boolean;
+    data?: unknown;
+    error?: string;
+  };
+}
+
 // ── Diagnostic (sentinel) ────────────────────────────────────────
 export interface DiagnosticExec extends ClusterFrame {
   type: "diagnostic_exec";
@@ -831,4 +919,12 @@ export type AnyClusterFrame =
   | TerminalData
   | TerminalResize
   | TerminalCloseRequest
-  | TerminalCloseResponse;
+  | TerminalCloseResponse
+  | HealthSyncFrame
+  | AvailabilityRequest
+  | AvailabilityResponse
+  | FileTransferInit
+  | FileTransferAck
+  | FileTransferChunk
+  | FileTransferChunkAck
+  | FileTransferComplete;

package/src/web.ts

@@ -3,6 +3,7 @@ import type { PeerManager } from "./peer-manager.ts";
 import type { HandoffManager } from "./handoff.ts";
 import type { ClawMatrixConfig } from "./config.ts";
 import type { SatelliteContext, IngestedEvent } from "./types.ts";
+import type { HealthTracker } from "./health-tracker.ts";
 import { timingSafeEqual } from "./auth.ts";
 import { renderDashboard } from "./web-ui.ts";
 import { readBody } from "./http-utils.ts";
@@ -46,6 +47,7 @@ export class WebHandler {
   private ingestedEvents: IngestedEvent[] = []; // ring buffer for ingested events
   private loginAttempts = new Map<string, { count: number; resetAt: number }>(); // IP → rate limit
   private loginCleanupTimer: ReturnType<typeof setInterval> | null = null;
+  private healthTracker: HealthTracker | null = null;
   private onPeerConnected: (nodeId: string) => void;
   private onPeerDisconnected: (nodeId: string) => void;
 
@@ -91,6 +93,11 @@ export class WebHandler {
     peerManager.on("peerDisconnected", this.onPeerDisconnected);
   }
 
+  /** Set the health tracker for availability API. */
+  setHealthTracker(tracker: HealthTracker) {
+    this.healthTracker = tracker;
+  }
+
   /** Clean up timers and pending requests on shutdown. */
   destroy() {
     // Remove event listeners to prevent post-destroy callbacks
@@ -181,6 +188,11 @@ export class WebHandler {
       return;
     }
 
+    if (path === "/api/availability" && req.method === "GET") {
+      this.handleAvailability(req, res);
+      return;
+    }
+
     if (path === "/api/satellite/poll" && req.method === "GET") {
       this.handleSatellitePoll(req, res);
       return;
@@ -271,6 +283,27 @@ export class WebHandler {
     }
   }
 
+  private handleAvailability(req: IncomingMessage, res: ServerResponse) {
+    if (!this.healthTracker) {
+      res.writeHead(503, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ error: "Health tracker not available" }));
+      return;
+    }
+
+    const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`);
+    const range = (url.searchParams.get("range") ?? "24h") as "24h" | "7d" | "90d";
+
+    if (!["24h", "7d", "90d"].includes(range)) {
+      res.writeHead(400, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ error: "Invalid range. Use 24h, 7d, or 90d" }));
+      return;
+    }
+
+    const result = this.healthTracker.getAvailability(range);
+    res.writeHead(200, { "Content-Type": "application/json" });
+    res.end(JSON.stringify(result));
+  }
+
   private handleStatus(res: ServerResponse) {
     const peers = this.peerManager.router.getAllPeers();
     const localNode = {