clawmatrix 0.4.2 → 0.5.1

package/src/peer-manager.ts

@@ -1,4 +1,4 @@
-import { EventEmitter } from "node:events";
+import { EventEmitter } from "eventemitter3";
 import path from "node:path";
 import { homedir, tmpdir } from "node:os";
 import { createServer, type IncomingMessage, type ServerResponse, type Server } from "node:http";
@@ -22,10 +22,11 @@ import type {
   PeerSync,
 } from "./types.ts";
 import { PeerApprovalManager, type ChannelApi, type NotifyTarget } from "./peer-approval.ts";
+import { computeBackoffDelay, getReconnectBackoff } from "./retry.ts";
 import { loadOrCreateIdentity } from "./identity.ts";
 import type { KeyPair } from "./crypto.ts";
 
-const RECONNECT_BASE = 1_000;
+// Reconnect constants kept for RECONNECT_MAX reference in failover scheduling
 const RECONNECT_MAX = 60_000;
 
 /** Frame types that bypass dedup (streams share one id across chunks; responses share id with request). */
@@ -51,6 +52,8 @@ const SKIP_DEDUP_TYPES = new Set([
   "file_transfer_ack", "file_transfer_complete",
 ]);
 
+// Reconnect backoff params are now provided by retry.ts → getReconnectBackoff()
+
 /** Classify WebSocket close code into a human-readable reason. */
 function classifyCloseReason(code: number, reason: string): string {
   if (reason) return reason;
@@ -94,6 +97,8 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
   /** Reconnect timers keyed by `nodeId|url` for per-channel reconnection. */
   private reconnectTimers = new Map<string, ReturnType<typeof setTimeout>>();
   private reconnectAttempts = new Map<string, number>();
+  /** Last close code per channel key for adaptive reconnect backoff. */
+  private lastCloseCode = new Map<string, number>();
   /** Track which nodeIds have already completed the full peer join (for multi-channel). */
   private joinedPeers = new Set<string>();
   /** All configured URLs per peer (for multi-URL peers). */
@@ -104,6 +109,10 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
   private urlProbeLatencies = new Map<string, number>();
   /** Route probe interval timer. */
   private probeTimer: ReturnType<typeof setInterval> | null = null;
+  /** EMA latency baseline per peer (for spike detection). */
+  private latencyBaselines = new Map<string, number>();
+  /** Last time a latency-triggered probe was fired per peer (debounce). */
+  private lastProbeTime = new Map<string, number>();
   /** Deferred disconnect timers — grace period before broadcasting peer_leave. */
   private disconnectGraceTimers = new Map<string, ReturnType<typeof setTimeout>>();
   private stopped = false;
@@ -121,10 +130,10 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
   private rateLimiter: RateLimiter;
   readonly approvalManager: PeerApprovalManager;
 
-  constructor(config: ClawMatrixConfig, openclawVersion?: string) {
+  constructor(config: ClawMatrixConfig, openclawVersion?: string, openclawConfig?: Record<string, unknown>) {
     super();
     this.config = config;
-    this.localDeviceInfo = collectDeviceInfo(openclawVersion);
+    this.localDeviceInfo = collectDeviceInfo(openclawVersion, openclawConfig);
     const acpAgents = config.acp?.enabled ? config.acp.agents : undefined;
     this.localCapabilities = {
       nodeId: config.nodeId,
@@ -285,7 +294,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
   }
 
   // ── Route probing (for multi-URL peers) ──────────────────────────
-  private static readonly PROBE_INTERVAL = 3_600_000; // 1 hour
+  private static readonly PROBE_INTERVAL = 900_000; // 15 minutes
   /** Minimum improvement ratio to trigger a route switch. */
   private static readonly SWITCH_THRESHOLD = 0.7; // new must be ≤70% of current
 
@@ -385,6 +394,32 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     this.switchRoute(nodeId, bestUrl);
   }
 
+  /** Trigger an immediate route probe for a specific peer (debounced). */
+  private triggerProbeForPeer(nodeId: string) {
+    const urls = this.peerUrls.get(nodeId);
+    if (!urls || urls.length <= 1) return;
+
+    const now = Date.now();
+    const lastProbe = this.lastProbeTime.get(nodeId) ?? 0;
+    if (now - lastProbe < 60_000) return; // debounce: 1 per minute
+    this.lastProbeTime.set(nodeId, now);
+
+    // Probe non-active URLs for this peer
+    const activeUrl = this.activeUrls.get(nodeId);
+    (async () => {
+      for (const url of urls) {
+        if (url === activeUrl) continue;
+        const latency = await this.probeUrl(url);
+        if (latency !== null) this.urlProbeLatencies.set(url, latency);
+      }
+      if (activeUrl) {
+        const route = this.router.getRoute(nodeId);
+        if (route && route.latencyMs > 0) this.urlProbeLatencies.set(activeUrl, route.latencyMs);
+      }
+      this.evaluateRouteSwitch(nodeId);
+    })();
+  }
+
   /** Callback to check if there are active tasks involving a peer. Set by ClusterRuntime. */
   private activeTaskChecker: ((nodeId: string) => boolean) | null = null;
 
@@ -465,10 +500,13 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       this.inboundIps.set(ws, ip);
       this.handleInboundOpen(ws, ip);
 
-      ws.on("message", (data) => {
+      ws.on("message", (data, isBinary) => {
         const conn = this.inboundConnections.get(ws);
         if (conn) {
-          conn.feedMessage(typeof data === "string" ? data : String(data));
+          // ws package always delivers data as Buffer; use isBinary to distinguish frame type.
+          // Text frames (e.g. base64 encrypted envelopes) must be passed as strings
+          // so Connection routes them through decryptBinary, not decryptBinaryRaw.
+          conn.feedMessage(isBinary ? (Buffer.isBuffer(data) ? data : Buffer.from(data as any)) : data.toString());
         }
       });
 
@@ -492,7 +530,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
   private handleInboundOpen(ws: WsWebSocket, ip: string) {
     // Wrap ws WebSocket into our WsTransport interface
     const transport: WsTransport = {
-      send(data: string) {
+      send(data: string | Buffer) {
         ws.send(data);
       },
       close(code?: number, reason?: string) {
@@ -539,10 +577,44 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
   private connectToPeer(peer: PeerConfig) {
     const urls = Array.isArray(peer.url) ? peer.url : [peer.url];
     this.peerUrls.set(peer.nodeId, urls);
-    // Connect to the first URL (or best known from probes)
-    const bestUrl = this.pickBestUrl(peer.nodeId, urls);
-    this.activeUrls.set(peer.nodeId, bestUrl);
-    this.connectToChannel(peer.nodeId, bestUrl);
+    if (urls.length <= 1) {
+      this.activeUrls.set(peer.nodeId, urls[0]);
+      this.connectToChannel(peer.nodeId, urls[0]);
+    } else {
+      // Multi-URL: connect all simultaneously, prune to best after 10s
+      this.activeUrls.set(peer.nodeId, urls[0]);
+      for (const url of urls) {
+        this.connectToChannel(peer.nodeId, url);
+      }
+      setTimeout(() => this.pruneWarmupConnections(peer.nodeId), 10_000);
+    }
+  }
+
+  /** After warmup period, keep only the lowest-latency channel and close the rest. */
+  private pruneWarmupConnections(nodeId: string) {
+    const channels = this.router.getChannels(nodeId);
+    if (channels.length <= 1) return;
+
+    let best: Connection | null = null;
+    let bestLatency = Infinity;
+    for (const conn of channels) {
+      if (!conn.isOpen) continue;
+      const lat = conn.latencyMs > 0 ? conn.latencyMs : 10_000;
+      if (lat < bestLatency) {
+        bestLatency = lat;
+        best = conn;
+      }
+    }
+
+    for (const conn of channels) {
+      if (conn !== best && conn.isOpen) {
+        conn.close(1000, "warm-up pruned");
+      }
+    }
+
+    if (best) {
+      debug("peer", `pruneWarmupConnections(${nodeId}): kept channel with latency=${best.latencyMs}ms, pruned ${channels.length - 1} others`);
+    }
   }
 
   /** Pick the best URL for a peer based on probe latencies. Falls back to first URL. */
@@ -571,10 +643,14 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     const attempt = this.reconnectAttempts.get(channelKey) ?? 0;
     debug("peer", `connectToChannel(${nodeId}): attempt=${attempt} url=${url}`);
 
-    // Use a common WS subprotocol for traffic disguise
-    let ws: WebSocket;
+    // Use the `ws` package for outbound connections.
+    // Node.js 24+'s built-in WebSocket (undici) defaults binaryType to "blob",
+    // causing binary frames to arrive as Blob instead of Buffer — which
+    // onRawMessage cannot handle, silently dropping encrypted frames (including auth_ok).
+    // The `ws` package defaults to binaryType "nodebuffer", avoiding this issue.
+    let ws: InstanceType<typeof WsWebSocket>;
     try {
-      ws = new WebSocket(url, ["graphql-transport-ws"]);
+      ws = new WsWebSocket(url, ["graphql-transport-ws"]);
     } catch (err) {
       debug("peer", `connectToChannel(${nodeId}): WebSocket constructor threw: ${err}`);
       this.scheduleChannelReconnect(nodeId, url);
@@ -596,6 +672,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       conn.on("authenticated", (caps) => {
         debug("peer", `connectToChannel(${nodeId}): authenticated url=${url}`);
         this.reconnectAttempts.delete(channelKey);
+        this.lastCloseCode.delete(channelKey);
         this.onPeerAuthenticated(conn, caps);
       });
 
@@ -624,6 +701,14 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
         debug("peer", `connectToChannel(${nodeId}): self-connection, will not reconnect`);
         return;
       }
+      // Don't reconnect warm-up pruned channels
+      if (ev.reason === "warm-up pruned") {
+        debug("peer", `connectToChannel(${nodeId}): warm-up pruned, will not reconnect`);
+        return;
+      }
+      // Record close code for adaptive backoff
+      const channelKey = `${nodeId}|${url}`;
+      this.lastCloseCode.set(channelKey, ev.code);
       if (!lastError) {
         lastError = classifyCloseReason(ev.code, ev.reason);
       }
@@ -665,7 +750,8 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       }
     }
 
-    const delay = Math.min(RECONNECT_BASE * 2 ** attempt, RECONNECT_MAX);
+    const params = getReconnectBackoff(this.lastCloseCode.get(channelKey) ?? 1006);
+    const delay = computeBackoffDelay(attempt, params);
     this.reconnectAttempts.set(channelKey, attempt + 1);
     const tag = reason ? ` reason="${reason}"` : "";
     debug("peer", `scheduleChannelReconnect(${nodeId}): attempt=${attempt} delay=${delay}ms url=${url}${tag}`);
@@ -693,11 +779,62 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
   }
 
   // ── Peer lifecycle ─────────────────────────────────────────────
+
+  /** Migrate tracking maps when a URL-derived nodeId (starts with "_") resolves to the real nodeId. */
+  private migrateUrlDerivedPeer(derivedId: string, realId: string) {
+    // Migrate peerUrls, activeUrls
+    const urls = this.peerUrls.get(derivedId);
+    if (urls) {
+      this.peerUrls.delete(derivedId);
+      if (!this.peerUrls.has(realId)) this.peerUrls.set(realId, urls);
+    }
+    const activeUrl = this.activeUrls.get(derivedId);
+    if (activeUrl) {
+      this.activeUrls.delete(derivedId);
+      if (!this.activeUrls.has(realId)) this.activeUrls.set(realId, activeUrl);
+    }
+    // Migrate reconnect state: channelKey uses nodeId|url
+    for (const url of urls ?? []) {
+      const oldKey = `${derivedId}|${url}`;
+      const newKey = `${realId}|${url}`;
+      const timer = this.reconnectTimers.get(oldKey);
+      if (timer) { this.reconnectTimers.delete(oldKey); this.reconnectTimers.set(newKey, timer); }
+      const attempts = this.reconnectAttempts.get(oldKey);
+      if (attempts !== undefined) { this.reconnectAttempts.delete(oldKey); this.reconnectAttempts.set(newKey, attempts); }
+      const code = this.lastCloseCode.get(oldKey);
+      if (code !== undefined) { this.lastCloseCode.delete(oldKey); this.lastCloseCode.set(newKey, code); }
+    }
+    // Update the config peer entry in place for sentinel and other consumers
+    for (const peer of this.config.peers) {
+      if (peer.nodeId === derivedId) {
+        (peer as { nodeId: string }).nodeId = realId;
+        break;
+      }
+    }
+    debug("peer", `migrateUrlDerivedPeer: ${derivedId} → ${realId}`);
+  }
+
   private onPeerAuthenticated(conn: Connection, caps: NodeCapabilities, ip?: string) {
     const nodeId = conn.remoteNodeId!;
     // Peer's persistent public key for TOFU identity binding
     const peerPublicKey = conn.remoteIdentityKey ?? undefined;
 
+    // If this outbound peer was configured with URL string shorthand (nodeId starts
+    // with "_"), resolve to the real nodeId from the authenticated handshake.
+    if (conn.role === "outbound") {
+      for (const peer of this.config.peers) {
+        if (peer.nodeId.startsWith("_") && this.peerUrls.get(peer.nodeId)) {
+          const urls = this.peerUrls.get(peer.nodeId)!;
+          // Check if this connection belongs to this URL-derived peer
+          const activeUrl = this.activeUrls.get(peer.nodeId);
+          if (activeUrl && urls.length > 0) {
+            this.migrateUrlDerivedPeer(peer.nodeId, nodeId);
+            break;
+          }
+        }
+      }
+    }
+
     // Prevent self-connection: close immediately if the remote side authenticated
     // with our own nodeId.  For outbound this means the peer URL accidentally
     // points to self; for inbound it means a remote node is (mis)using our nodeId.
@@ -796,7 +933,16 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       // Additional channel — just add to the channel pool, no peer_join broadcast
       this.router.addChannel(nodeId, conn);
       conn.on("message", (frame) => this.onFrame(frame, conn));
-      conn.on("latency", () => this.router.updateActiveChannel(nodeId));
+      conn.on("latency", (latencyMs) => {
+        this.router.updateActiveChannel(nodeId);
+        // Update baseline and check for spike
+        const baseline = this.latencyBaselines.get(nodeId) ?? latencyMs;
+        const newBaseline = baseline * 0.8 + latencyMs * 0.2;
+        this.latencyBaselines.set(nodeId, newBaseline);
+        if (latencyMs > baseline * 2 && latencyMs > 200) {
+          this.triggerProbeForPeer(nodeId);
+        }
+      });
       conn.on("close", () => this.onChannelDisconnected(conn));
       const channelCount = this.router.getChannelCount(nodeId);
       debug("peer", `completePeerJoin(${nodeId}): additional channel added (total=${channelCount})`);
@@ -830,7 +976,16 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     this.joinedPeers.add(nodeId);
 
     conn.on("message", (frame) => this.onFrame(frame, conn));
-    conn.on("latency", () => this.router.updateActiveChannel(nodeId));
+    conn.on("latency", (latencyMs) => {
+      this.router.updateActiveChannel(nodeId);
+      // Update baseline and check for spike
+      const baseline = this.latencyBaselines.get(nodeId) ?? latencyMs;
+      const newBaseline = baseline * 0.8 + latencyMs * 0.2;
+      this.latencyBaselines.set(nodeId, newBaseline);
+      if (latencyMs > baseline * 2 && latencyMs > 200) {
+        this.triggerProbeForPeer(nodeId);
+      }
+    });
     conn.on("close", () => this.onChannelDisconnected(conn));
 
     this.sendPeerSync(conn);
@@ -862,6 +1017,9 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
 
     audit("peer_join", { nodeId, detail: `agents=${caps.agents.length} models=${caps.models.length}` });
     this.emit("peerConnected", nodeId);
+
+    // Probe alternative routes after reconnect (connectivity may have changed)
+    setTimeout(() => this.triggerProbeForPeer(nodeId), 5_000);
   }
 
   /** Handle a single channel disconnecting (multi-channel aware). */
@@ -901,6 +1059,10 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     if (nodeId === this.config.nodeId) {
       this.router.removePeer(nodeId);
       this.joinedPeers.delete(nodeId);
+      this.latencyBaselines.delete(nodeId);
+      this.lastProbeTime.delete(nodeId);
+      // 清除 delta sync 版本号，确保重连时发全量 peer_sync
+      this.peerSyncVersions.delete(nodeId);
       return;
     }
 
@@ -950,6 +1112,9 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     audit("peer_leave", { nodeId });
     this.router.removePeer(nodeId);
     this.joinedPeers.delete(nodeId);
+    this.latencyBaselines.delete(nodeId);
+    this.lastProbeTime.delete(nodeId);
+    this.peerSyncVersions.delete(nodeId);
 
     // Remove satellite contexts that were only reachable via this peer
     for (let i = this.satelliteContexts.length - 1; i >= 0; i--) {
@@ -1076,13 +1241,19 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     this.emit("frame", frame, from);
   }
 
+  /** Last sync version sent to each direct peer. */
+  private peerSyncVersions = new Map<string, number>();
+
   private sendPeerSync(conn: Connection) {
-    const peers = this.router.buildPeerSyncPayload();
-    const payload: Record<string, unknown> = { peers };
+    const remoteNodeId = conn.remoteNodeId ?? "";
+    const sinceVersion = this.peerSyncVersions.get(remoteNodeId) ?? 0;
+    const delta = this.router.buildPeerSyncDelta(sinceVersion);
+    const payload: Record<string, unknown> = { ...delta };
     if (this.satelliteContexts.length > 0) {
       payload.satellites = this.satelliteContexts;
     }
-    conn.send({
+    this.peerSyncVersions.set(remoteNodeId, delta.version);
+    conn.sendDirect({
       type: "peer_sync",
       from: this.config.nodeId,
       timestamp: Date.now(),
@@ -1105,34 +1276,53 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     }
 
     let changed = false;
-    for (const peer of frame.payload.peers) {
-      if (peer.nodeId === this.config.nodeId) continue;
-      if (peer.nodeId === from.remoteNodeId) {
-        const prev = this.router.getRoute(peer.nodeId);
-        const hadAgents = prev?.agents.length ?? 0;
-        const hadDirectPeers = prev?.directPeers.length ?? 0;
-        const hadDeviceInfo = prev?.deviceInfo?.hostname;
-        const hadAcpAgents = prev?.acpAgents?.length ?? 0;
-        const hadToolProxyEnabled = prev?.toolProxy?.enabled;
-        const hadToolProxyCatalogLen = prev?.toolProxy?.catalog?.length ?? 0;
-        const hadToolProxyAllowLen = prev?.toolProxy?.allow?.length ?? 0;
-        this.router.updatePeerCapabilities(peer.nodeId, peer);
-        if (peer.agents.length !== hadAgents || peer.models.length !== (prev?.models.length ?? 0)
-            || (peer.directPeers?.length ?? 0) !== hadDirectPeers
-            || peer.toolProxy?.enabled !== hadToolProxyEnabled
-            || (peer.toolProxy?.catalog?.length ?? 0) !== hadToolProxyCatalogLen
-            || (peer.toolProxy?.allow?.length ?? 0) !== hadToolProxyAllowLen
-            || peer.deviceInfo?.hostname !== hadDeviceInfo
-            || (peer.acpAgents?.length ?? 0) !== hadAcpAgents) {
+
+    // Delta sync: apply removed, then added/updated
+    if (frame.payload.version != null && frame.payload.removed) {
+      for (const nodeId of frame.payload.removed) {
+        if (nodeId === this.config.nodeId) continue;
+        // Only remove relay routes — don't remove direct connections
+        const route = this.router.getRoute(nodeId);
+        if (route && !route.connection) {
+          this.router.removePeer(nodeId);
           changed = true;
         }
-      } else {
-        // Skip if the remote peer only knows about this node through us —
-        // using them as relay would create a routing loop.
-        if (peer.reachableVia === this.config.nodeId) continue;
-        const existing = this.router.getRoute(peer.nodeId);
-        if (!existing) changed = true;
-        this.router.addRelayPeer(peer, from.remoteNodeId!);
+      }
+    }
+
+    // Process peers from either full sync or delta (updated field)
+    // Delta sets peers = updated for backward compat, so this works for both
+    const peersToProcess = frame.payload.peers;
+    if (peersToProcess) {
+      for (const peer of peersToProcess) {
+        if (peer.nodeId === this.config.nodeId) continue;
+        if (peer.nodeId === from.remoteNodeId) {
+          const prev = this.router.getRoute(peer.nodeId);
+          const hadAgents = prev?.agents.length ?? 0;
+          const hadDirectPeers = prev?.directPeers.length ?? 0;
+          const hadDeviceInfo = prev?.deviceInfo?.hostname;
+          const hadAcpAgents = prev?.acpAgents?.length ?? 0;
+          const hadToolProxyEnabled = prev?.toolProxy?.enabled;
+          const hadToolProxyCatalogLen = prev?.toolProxy?.catalog?.length ?? 0;
+          const hadToolProxyAllowLen = prev?.toolProxy?.allow?.length ?? 0;
+          this.router.updatePeerCapabilities(peer.nodeId, peer);
+          if (peer.agents.length !== hadAgents || peer.models.length !== (prev?.models.length ?? 0)
+              || (peer.directPeers?.length ?? 0) !== hadDirectPeers
+              || peer.toolProxy?.enabled !== hadToolProxyEnabled
+              || (peer.toolProxy?.catalog?.length ?? 0) !== hadToolProxyCatalogLen
+              || (peer.toolProxy?.allow?.length ?? 0) !== hadToolProxyAllowLen
+              || peer.deviceInfo?.hostname !== hadDeviceInfo
+              || (peer.acpAgents?.length ?? 0) !== hadAcpAgents) {
+            changed = true;
+          }
+        } else {
+          // Skip if the remote peer only knows about this node through us —
+          // using them as relay would create a routing loop.
+          if (peer.reachableVia === this.config.nodeId) continue;
+          const existing = this.router.getRoute(peer.nodeId);
+          if (!existing) changed = true;
+          this.router.addRelayPeer(peer, from.remoteNodeId!);
+        }
       }
     }
 

package/src/retry.ts

@@ -0,0 +1,81 @@
+/**
+ * Shared retry/backoff/circuit-breaker utilities backed by cockatiel.
+ *
+ * Provides:
+ * - Per-node circuit breakers for mesh health tracking
+ * - Adaptive backoff delay computation
+ */
+
+import { circuitBreaker, handleAll, SamplingBreaker, ConsecutiveBreaker } from "cockatiel";
+
+// ── Per-node circuit breakers ────────────────────────────────────
+
+export interface CircuitBreakerConfig {
+  /** Number of consecutive failures before opening the circuit (default: 5). */
+  threshold?: number;
+  /** How long to wait before trying a half-open probe in ms (default: 30000). */
+  halfOpenAfter?: number;
+}
+
+const DEFAULT_CB_CONFIG: Required<CircuitBreakerConfig> = {
+  threshold: 5,
+  halfOpenAfter: 30_000,
+};
+
+const nodeBreakers = new Map<string, ReturnType<typeof circuitBreaker>>();
+
+/** Get or create a circuit breaker for a node. Breakers are cached per nodeId. */
+export function getNodeCircuitBreaker(nodeId: string, config?: CircuitBreakerConfig) {
+  let cb = nodeBreakers.get(nodeId);
+  if (cb) return cb;
+
+  const cfg = { ...DEFAULT_CB_CONFIG, ...config };
+  cb = circuitBreaker(handleAll, {
+    halfOpenAfter: cfg.halfOpenAfter,
+    breaker: new ConsecutiveBreaker(cfg.threshold),
+  });
+  nodeBreakers.set(nodeId, cb);
+  return cb;
+}
+
+/** Remove a node's circuit breaker (e.g. when peer leaves). */
+export function removeNodeCircuitBreaker(nodeId: string) {
+  nodeBreakers.delete(nodeId);
+}
+
+/** Check if a node's circuit is currently open (unhealthy). */
+export function isNodeCircuitOpen(nodeId: string): boolean {
+  const cb = nodeBreakers.get(nodeId);
+  if (!cb) return false;
+  return cb.state === "open";
+}
+
+/** Reset all circuit breakers (e.g. on shutdown). */
+export function resetAllCircuitBreakers() {
+  nodeBreakers.clear();
+}
+
+// ── Adaptive backoff computation ─────────────────────────────────
+
+export interface BackoffParams {
+  /** Base delay in ms. */
+  base: number;
+  /** Maximum delay in ms. */
+  max: number;
+}
+
+/** Compute an exponential backoff delay with jitter.
+ *  Returns a delay in ms: min(base * 2^attempt, max) * random(0.5, 1.0). */
+export function computeBackoffDelay(attempt: number, params: BackoffParams): number {
+  const raw = Math.min(params.base * 2 ** attempt, params.max);
+  return Math.round(raw * (0.5 + Math.random() * 0.5));
+}
+
+/** Get adaptive backoff parameters based on WebSocket close code. */
+export function getReconnectBackoff(code: number): BackoffParams {
+  if (code === 4001) return { base: 30_000, max: 300_000 };  // auth failed → slow
+  if (code === 4003) return { base: 5_000, max: 60_000 };    // auth timeout → medium
+  if (code === 1006) return { base: 1_000, max: 10_000 };    // network error → fast
+  if (code === 1001) return { base: 2_000, max: 30_000 };    // going away → medium-fast
+  return { base: 1_000, max: 60_000 };                        // default
+}