clawmatrix 0.2.7 → 0.2.9

package/src/peer-manager.ts

@@ -28,6 +28,26 @@ import type { KeyPair } from "./crypto.ts";
 const RECONNECT_BASE = 1_000;
 const RECONNECT_MAX = 60_000;
 
+/** Frame types that bypass dedup (streams share one id across chunks; responses share id with request). */
+const SKIP_DEDUP_TYPES = new Set([
+  // Streaming
+  "model_stream", "handoff_stream", "acp_stream",
+  // Response frames (share id with their request)
+  "model_res", "tool_res", "tool_batch_res",
+  "handoff_res", "handoff_status_res", "handoff_input_required",
+  // Handoff control (reuse original handoff_req id)
+  "handoff_input", "handoff_cancel", "handoff_status",
+  // Diagnostics & approval
+  "diagnostic_exec_res", "diagnostic_status_res", "peer_approval_res",
+  // ACP responses
+  "acp_res", "acp_close_res", "acp_list_res", "acp_resume_res",
+  "acp_cancel_res", "acp_set_mode_res", "acp_get_modes_res",
+  "chat_history_res",
+  // Terminal
+  "terminal_open_res", "terminal_data", "terminal_resize",
+  "terminal_close", "terminal_close_res",
+]);
+
 /** Classify WebSocket close code into a human-readable reason. */
 function classifyCloseReason(code: number, reason: string): string {
   if (reason) return reason;
@@ -364,6 +384,12 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       tryReconnect();
     });
     ws.addEventListener("close", (ev) => {
+      // Don't reconnect if this was a self-connection (peer URL points to ourselves).
+      // Without this guard, outbound detects self → closes → scheduleReconnect → loop.
+      if (ev.code === 4002 && ev.reason === "self-connection") {
+        debug("peer", `connectToPeer(${peer.nodeId}): self-connection, will not reconnect`);
+        return;
+      }
       if (!lastError) {
         lastError = classifyCloseReason(ev.code, ev.reason);
       }
@@ -397,19 +423,17 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     // Peer's persistent public key for TOFU identity binding
     const peerPublicKey = conn.remoteIdentityKey ?? undefined;
 
-    // Prevent self-connection: if outbound connection authenticated as our own nodeId
-    // (e.g. peer URL accidentally points to self), close immediately.
-    if (conn.role === "outbound" && nodeId === this.config.nodeId) {
-      debug("peer", `Self-connection detected (outbound to ${nodeId}), closing`);
+    // Prevent self-connection: close immediately if the remote side authenticated
+    // with our own nodeId.  For outbound this means the peer URL accidentally
+    // points to self; for inbound it means a remote node is (mis)using our nodeId.
+    // Exception: loopback connections with the same nodeId are local clients
+    // (Mac desktop app / iOS simulator) and are allowed through.
+    const isLocalClient = conn.role === "inbound" && nodeId === this.config.nodeId && isLoopback(ip);
+    if (nodeId === this.config.nodeId && !isLocalClient) {
+      debug("peer", `Self-connection detected (${conn.role}, nodeId=${nodeId}, ip=${ip}), closing`);
       conn.close(4002, "self-connection");
       return;
     }
-
-    // Peer approval check (inbound only — outbound peers are explicitly configured)
-    // Skip approval for same-nodeId connections from localhost (local clients
-    // like Mac desktop app / iOS simulator). An attacker would need to already
-    // be on the same machine to exploit this, which is outside our threat model.
-    const isLocalClient = nodeId === this.config.nodeId && isLoopback(ip);
     debug("approval", `onPeerAuthenticated: nodeId=${nodeId} role=${conn.role} isLocalClient=${isLocalClient} ip=${ip}`);
     if (conn.role === "inbound" && !isLocalClient) {
       // IP-level approval rate limiting (suppress noise from leaked tokens)
@@ -580,8 +604,11 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
 
   // ── Message handling ───────────────────────────────────────────
   private onFrame(frame: AnyClusterFrame, from: Connection) {
-    // Ignore self-echo: frames with our own nodeId that were relayed back to us
-    if (frame.from === this.config.nodeId) return;
+    // Ignore self-echo: frames with our own nodeId that were relayed back to us.
+    // Exception: frames from same-nodeId satellite connections (Mac/iOS client)
+    // are legitimate requests that must be processed or relayed.
+    const isSatellite = from.remoteNodeId === this.config.nodeId;
+    if (frame.from === this.config.nodeId && !isSatellite) return;
 
     // Validate from field: must be the direct peer or a known node (relayed)
     if (frame.from && frame.from !== from.remoteNodeId && !this.router.getRoute(frame.from)) {
@@ -589,32 +616,10 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       return;
     }
 
-    // Skip dedup for streaming and response frame types.
-    // Stream frames share one id across many chunks.
-    // Response frames (model_res, tool_res, handoff_res, etc.) share the same id
-    // as their request — without this exemption, a relay node that forwarded
-    // the request would mark the id as seen and then drop the returning response.
     // Skip dedup for streaming chunks (same id across many chunks) and response
     // frames (share id with their request — relay would otherwise drop the reply).
-    // handoff_input, handoff_cancel, and handoff_status reuse the original handoff_req id,
-    // so relay nodes that already forwarded the request would drop them as duplicates.
-    const skipDedup = frame.type === "model_stream" || frame.type === "handoff_stream"
-      || frame.type === "model_res" || frame.type === "tool_res"
-      || frame.type === "handoff_res" || frame.type === "handoff_status_res"
-      || frame.type === "handoff_input_required"
-      || frame.type === "handoff_input" || frame.type === "handoff_cancel"
-      || frame.type === "handoff_status"
-      || frame.type === "diagnostic_exec_res" || frame.type === "diagnostic_status_res"
-      || frame.type === "peer_approval_res"
-      || frame.type === "acp_stream" || frame.type === "acp_res"
-      || frame.type === "acp_close_res"
-      || frame.type === "acp_list_res" || frame.type === "acp_resume_res"
-      || frame.type === "acp_cancel_res"
-      || frame.type === "acp_set_mode_res" || frame.type === "acp_get_modes_res"
-      || frame.type === "terminal_open_res" || frame.type === "terminal_data"
-      || frame.type === "terminal_resize" || frame.type === "terminal_close"
-      || frame.type === "terminal_close_res";
-    if (frame.id && !skipDedup && this.router.isDuplicate(frame.id)) return;
+    // handoff_input/cancel/status reuse the original handoff_req id.
+    if (frame.id && !SKIP_DEDUP_TYPES.has(frame.type) && this.router.isDuplicate(frame.id)) return;
 
     // Handle peer approval responses locally (don't emit to cluster-service)
     if (frame.type === "peer_approval_res") {
@@ -672,8 +677,26 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     }
 
     if (frame.to && frame.to !== this.config.nodeId) {
-      this.router.tryRelay(frame);
-      return;
+      if (this.router.tryRelay(frame)) return;
+
+      // Relay failed — for model_req, try alternative nodes or fall through to local handling
+      if (frame.type === "model_req") {
+        const modelId = (frame as any).payload?.model;
+        if (modelId) {
+          const exclude = new Set([frame.to, this.config.nodeId]);
+          const alternatives = this.router.findNodesForModel(modelId, exclude);
+          for (const alt of alternatives) {
+            if (this.sendTo(alt.nodeId, { ...frame, to: alt.nodeId })) {
+              debug("peer", `model_req failover: ${frame.to} → ${alt.nodeId}`);
+              return;
+            }
+          }
+        }
+        // No remote alternative — fall through to local handling
+        // (model-proxy will handle locally or send error back)
+      } else {
+        return;
+      }
     }
 
     // Forward to same-nodeId satellite connection (e.g. Mac desktop app) so that

package/src/router.ts

@@ -3,6 +3,7 @@ import type { Connection } from "./connection.ts";
 
 const DEFAULT_TTL = 3;
 const MAX_SEEN_FRAMES = 10_000;
+const MAX_FAILED_REQUESTS = 5_000;
 const ROTATE_INTERVAL = 60_000; // rotate dedup maps every 60s
 
 export interface RouteEntry {
@@ -232,6 +233,31 @@ export class Router {
     return this.routes.get(target);
   }
 
+  /** Find reachable nodes that provide a specific model, sorted by latency.
+   *  Excludes nodes in the `exclude` set. */
+  findNodesForModel(modelId: string, exclude?: Set<string>): RouteEntry[] {
+    const candidates: RouteEntry[] = [];
+    for (const entry of this.routes.values()) {
+      if (exclude?.has(entry.nodeId)) continue;
+      if (!entry.models.some((m) => m.id === modelId)) continue;
+      // Check reachability
+      if (entry.connection?.isOpen) {
+        candidates.push(entry);
+      } else if (entry.reachableVia) {
+        const relay = this.connections.get(entry.reachableVia);
+        if (relay?.isOpen) candidates.push(entry);
+      }
+    }
+    // Sort: direct first, then by latency
+    candidates.sort((a, b) => {
+      const aDirect = a.connection ? 0 : 1;
+      const bDirect = b.connection ? 0 : 1;
+      if (aDirect !== bDirect) return aDirect - bDirect;
+      return a.latencyMs - b.latencyMs;
+    });
+    return candidates;
+  }
+
   // ── Message sending and relay ──────────────────────────────────
   /** Send a frame to a specific node, relaying if necessary. Returns true if sent. */
   sendTo(targetNodeId: string, frame: ClusterFrame | AnyClusterFrame): boolean {
@@ -305,6 +331,21 @@ export class Router {
    *  TTL defaults to 15 minutes — long enough for handoff timeouts. */
   markFailed(requestId: string, ttlMs = 900_000) {
     this.failedRequests.set(requestId, Date.now() + ttlMs);
+    // Evict entries when map grows too large: first expired, then FIFO
+    if (this.failedRequests.size > MAX_FAILED_REQUESTS) {
+      const now = Date.now();
+      // Pass 1: remove expired entries
+      for (const [id, expiresAt] of this.failedRequests) {
+        if (now > expiresAt) this.failedRequests.delete(id);
+      }
+      // Pass 2: if still over limit, remove oldest (insertion-order) entries
+      if (this.failedRequests.size > MAX_FAILED_REQUESTS) {
+        for (const [id] of this.failedRequests) {
+          if (this.failedRequests.size <= MAX_FAILED_REQUESTS) break;
+          this.failedRequests.delete(id);
+        }
+      }
+    }
   }
 
   isFailed(requestId: string): boolean {

package/src/sentinel.ts

@@ -569,6 +569,28 @@ process.on("exit", (code) => {
   try { process.stderr.write(`[svc ${ts}] Exit code=${code}\n`); } catch { /* ignore */ }
 });
 
+/** Connect to all configured peers (called when gateway dies). */
+function connectAllPeers() {
+  for (const peer of config.peers) {
+    if (!connections.has(peer.nodeId) && !reconnectTimers.has(peer.nodeId)) {
+      connectToPeer(peer);
+    }
+  }
+}
+
+/** Disconnect from all peers (called when gateway recovers). */
+function disconnectAllPeers() {
+  for (const [nodeId, conn] of connections) {
+    conn.close(1000, "gateway recovered");
+    connections.delete(nodeId);
+  }
+  for (const [nodeId, timer] of reconnectTimers) {
+    clearTimeout(timer);
+    reconnectTimers.delete(nodeId);
+  }
+  reconnectAttempts.clear();
+}
+
 /** Periodically check if the gateway process is still alive via kill(pid, 0). */
 function startGatewayHealthCheck() {
   if (healthCheckTimer || !gatewayPid) return;
@@ -581,11 +603,15 @@ function startGatewayHealthCheck() {
         log("Gateway process detected — back online");
         // Release the port so the gateway can reclaim it
         stopListening();
+        // Disconnect from peers — gateway handles mesh connections
+        disconnectAllPeers();
       }
     } catch {
       if (gatewayAlive) {
         gatewayAlive = false;
         log(`Gateway process (pid ${gatewayPid}) gone — entering standalone mode`);
+        // Connect to peers now that gateway is down
+        connectAllPeers();
         // Take over the gateway's listen port
         if (config.listenPort) {
           // Small delay to let the OS release the port from the dead process
@@ -608,11 +634,7 @@ function boot() {
   writePidFile();
   log(`Started (pid ${process.pid}, gateway ${gatewayPid}, nodeId ${sentinelNodeId()}, takeover port ${config.listenPort || "none"})`);
 
-  // Connect to all configured peers
-  for (const peer of config.peers) {
-    connectToPeer(peer);
-  }
-
-  // Note: we do NOT start listening here.
-  // Listening only starts when gateway dies (port takeover mode).
+  // Do NOT connect to peers on boot — gateway handles mesh connections.
+  // Sentinel only connects when gateway dies (standalone mode).
+  // Listening also only starts when gateway dies (port takeover mode).
 }

package/src/types.ts

@@ -411,6 +411,14 @@ export interface KnowledgeSyncFrame extends ClusterFrame {
   };
 }
 
+// ── Health sync ──────────────────────────────────────────────────
+export interface HealthSyncFrame extends ClusterFrame {
+  type: "health_sync";
+  payload: {
+    data: string; // base64-encoded Automerge sync message
+  };
+}
+
 // ── Diagnostic (sentinel) ────────────────────────────────────────
 export interface DiagnosticExec extends ClusterFrame {
   type: "diagnostic_exec";
@@ -831,4 +839,5 @@ export type AnyClusterFrame =
   | TerminalData
   | TerminalResize
   | TerminalCloseRequest
-  | TerminalCloseResponse;
+  | TerminalCloseResponse
+  | HealthSyncFrame;

package/src/web.ts

@@ -3,6 +3,7 @@ import type { PeerManager } from "./peer-manager.ts";
 import type { HandoffManager } from "./handoff.ts";
 import type { ClawMatrixConfig } from "./config.ts";
 import type { SatelliteContext, IngestedEvent } from "./types.ts";
+import type { HealthTracker } from "./health-tracker.ts";
 import { timingSafeEqual } from "./auth.ts";
 import { renderDashboard } from "./web-ui.ts";
 import { readBody } from "./http-utils.ts";
@@ -46,6 +47,7 @@ export class WebHandler {
   private ingestedEvents: IngestedEvent[] = []; // ring buffer for ingested events
   private loginAttempts = new Map<string, { count: number; resetAt: number }>(); // IP → rate limit
   private loginCleanupTimer: ReturnType<typeof setInterval> | null = null;
+  private healthTracker: HealthTracker | null = null;
   private onPeerConnected: (nodeId: string) => void;
   private onPeerDisconnected: (nodeId: string) => void;
 
@@ -91,6 +93,11 @@ export class WebHandler {
     peerManager.on("peerDisconnected", this.onPeerDisconnected);
   }
 
+  /** Set the health tracker for availability API. */
+  setHealthTracker(tracker: HealthTracker) {
+    this.healthTracker = tracker;
+  }
+
   /** Clean up timers and pending requests on shutdown. */
   destroy() {
     // Remove event listeners to prevent post-destroy callbacks
@@ -181,6 +188,11 @@ export class WebHandler {
       return;
     }
 
+    if (path === "/api/availability" && req.method === "GET") {
+      this.handleAvailability(req, res);
+      return;
+    }
+
     if (path === "/api/satellite/poll" && req.method === "GET") {
       this.handleSatellitePoll(req, res);
       return;
@@ -271,6 +283,27 @@ export class WebHandler {
     }
   }
 
+  private handleAvailability(req: IncomingMessage, res: ServerResponse) {
+    if (!this.healthTracker) {
+      res.writeHead(503, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ error: "Health tracker not available" }));
+      return;
+    }
+
+    const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`);
+    const range = (url.searchParams.get("range") ?? "24h") as "24h" | "7d" | "90d";
+
+    if (!["24h", "7d", "90d"].includes(range)) {
+      res.writeHead(400, { "Content-Type": "application/json" });
+      res.end(JSON.stringify({ error: "Invalid range. Use 24h, 7d, or 90d" }));
+      return;
+    }
+
+    const result = this.healthTracker.getAvailability(range);
+    res.writeHead(200, { "Content-Type": "application/json" });
+    res.end(JSON.stringify(result));
+  }
+
   private handleStatus(res: ServerResponse) {
     const peers = this.peerManager.router.getAllPeers();
     const localNode = {