clawmatrix 0.1.23 → 0.2.1

package/src/sentinel.ts

@@ -0,0 +1,618 @@
+/**
+ * Sentinel — lightweight detached subprocess that survives OpenClaw crashes.
+ *
+ * Maintains independent WS connections to peers and handles diagnostic
+ * commands (exec, status) so remote nodes can troubleshoot even when
+ * the main gateway process is down.
+ *
+ * When the gateway dies and `listenPort` is configured, sentinel takes over
+ * the same port so existing clients (e.g. iOS) can reconnect without any
+ * URL change. When the gateway comes back, sentinel releases the port.
+ *
+ * Spawned by SentinelManager with `detached: true` + `unref()`.
+ * Receives config via IPC from the parent process.
+ */
+
+import { spawn } from "node:child_process";
+import { readFileSync, writeFileSync, unlinkSync, existsSync } from "node:fs";
+import { createServer, type Server } from "node:http";
+import { WebSocketServer, WebSocket as WsWebSocket } from "ws";
+import path from "node:path";
+import { homedir, tmpdir } from "node:os";
+import { Connection, type WsTransport, type ConnectionE2eeOptions } from "./connection.ts";
+import { collectDeviceInfo } from "./device-info.ts";
+import { loadOrCreateIdentity } from "./identity.ts";
+import type { KeyPair } from "./crypto.ts";
+import type {
+  AnyClusterFrame,
+  NodeCapabilities,
+  DiagnosticExec,
+  DiagnosticStatus,
+} from "./types.ts";
+
+// ── Config received from parent via IPC ─────────────────────────
+interface SentinelConfig {
+  nodeId: string;
+  secret: string;
+  peers: Array<{ nodeId: string; url: string }>;
+  agents?: Array<{ id: string; description?: string; tags?: string[] }>;
+  models?: Array<{ id: string; provider: string; description?: string }>;
+  tags?: string[];
+  e2ee: boolean;
+  compression: boolean;
+  pidFile: string;
+  gatewayPid?: number;
+  /** Port to take over when gateway dies (typically the gateway's own listen port). */
+  listenPort?: number;
+  listenHost?: string;
+  /** Peer approval config (allowList + persistPath). */
+  peerApproval?: {
+    enabled: boolean;
+    allowList: string[];
+    persistPath: string;
+  };
+}
+
+// ── State ───────────────────────────────────────────────────────
+let config: SentinelConfig;
+let gatewayAlive = true;
+let gatewayPid: number | null = null;
+let healthCheckTimer: ReturnType<typeof setInterval> | null = null;
+const startTime = Date.now();
+const sentinelNodeId = () => `${config.nodeId}:sentinel`;
+
+// ── Peer approval (read-only — sentinel only accepts already-approved peers) ──
+let approvedNodeIds = new Set<string>();
+let identityKeyPair: KeyPair | null = null;
+/** Approved peer records with pinned public keys (TOFU). */
+let approvedPeerKeys = new Map<string, string>(); // nodeId → publicKey
+
+const connections = new Map<string, Connection>();
+const reconnectTimers = new Map<string, ReturnType<typeof setTimeout>>();
+const reconnectAttempts = new Map<string, number>();
+const RECONNECT_BASE = 2_000;
+const RECONNECT_MAX = 60_000;
+
+// ── Inbound listener state (port takeover) ──────────────────────
+let httpServer: Server | null = null;
+let wss: WebSocketServer | null = null;
+const inboundConnections = new Map<WsWebSocket, Connection>();
+let listening = false;
+
+// ── Rate limiting for diagnostic_exec ────────────────────────────
+const EXEC_RATE_WINDOW = 60_000; // 1 minute
+const EXEC_RATE_LIMIT = 20; // max execs per window
+const execTimestamps: number[] = [];
+
+// ── Approved peers loading ────────────────────────────────────────
+
+/**
+ * Load approved peers from the persisted JSON file.
+ * Sentinel only accepts connections from peers that were already approved
+ * by the gateway — it does NOT support approving new peers.
+ * This ensures that even if the gateway is down, unapproved devices
+ * (including those with a leaked token) cannot join.
+ */
+function loadApprovedPeers() {
+  const stateDir = path.join(homedir() || tmpdir(), ".openclaw", "clawmatrix");
+  const approval = config.peerApproval;
+
+  // Load identity key pair (shared with gateway via same state dir)
+  try {
+    identityKeyPair = loadOrCreateIdentity(stateDir);
+  } catch {
+    log("Failed to load identity key pair");
+  }
+
+  if (!approval?.enabled) return;
+
+  // Add allowList peers
+  for (const nodeId of approval.allowList) {
+    approvedNodeIds.add(nodeId);
+  }
+
+  // Load persisted approved peers
+  try {
+    const filePath = path.join(stateDir, approval.persistPath);
+    if (existsSync(filePath)) {
+      const raw = readFileSync(filePath, "utf-8");
+      const data = JSON.parse(raw);
+      if (data.approved) {
+        for (const [nodeId, record] of Object.entries(data.approved)) {
+          approvedNodeIds.add(nodeId);
+          const rec = record as { publicKey?: string };
+          if (rec.publicKey) {
+            approvedPeerKeys.set(nodeId, rec.publicKey);
+          }
+        }
+      }
+    }
+  } catch {
+    log("Failed to load approved peers — rejecting all inbound");
+  }
+
+  log(`Loaded ${approvedNodeIds.size} approved peers`);
+}
+
+/**
+ * Check if an inbound peer should be accepted by sentinel.
+ * Only allows: same-nodeId from loopback, allowList, or approved peers
+ * with matching TOFU public key.
+ */
+function isSentinelPeerAllowed(nodeId: string, publicKey: string | null): boolean {
+  if (!config.peerApproval?.enabled) return true;
+
+  // Always allow configured peer nodeIds (outbound targets connecting back)
+  if (config.peers.some(p => p.nodeId === nodeId)) return true;
+
+  // Allow allowList peers
+  if (approvedNodeIds.has(nodeId)) {
+    // TOFU check: if we have a pinned key, verify it matches
+    const pinnedKey = approvedPeerKeys.get(nodeId);
+    if (pinnedKey && publicKey && pinnedKey !== publicKey) {
+      log(`TOFU mismatch for ${nodeId} — rejecting`);
+      return false;
+    }
+    return true;
+  }
+
+  return false;
+}
+
+// ── Peer connection ─────────────────────────────────────────────
+function buildCapabilities(): NodeCapabilities {
+  return {
+    nodeId: sentinelNodeId(),
+    agents: config.agents ?? [],
+    models: config.models ?? [],
+    tags: [...(config.tags ?? []), "sentinel"],
+    deviceInfo: collectDeviceInfo(),
+  };
+}
+
+function connectToPeer(peer: { nodeId: string; url: string }) {
+  const ws = new WebSocket(peer.url, ["graphql-transport-ws"]);
+  const e2eeOpts: ConnectionE2eeOptions = {
+    e2ee: config.e2ee,
+    compression: config.compression,
+    identityKeyPair: config.e2ee && identityKeyPair ? identityKeyPair : undefined,
+  };
+
+  // Create Connection only after WS is open (matches PeerManager pattern).
+  // Otherwise the 10s auth timer starts before the TCP handshake completes.
+  ws.addEventListener("open", () => {
+    const conn = new Connection(
+      ws as unknown as WsTransport,
+      "outbound",
+      sentinelNodeId(),
+      config.secret,
+      buildCapabilities(),
+      e2eeOpts,
+    );
+    conn.bindWebSocket(ws);
+
+    conn.on("authenticated", () => {
+      reconnectAttempts.delete(peer.nodeId);
+      connections.set(peer.nodeId, conn);
+      log(`Peer connected: ${peer.nodeId}`);
+    });
+
+    conn.on("message", (frame) => handleFrame(frame, conn));
+
+    conn.on("error", () => { /* close will follow */ });
+  });
+
+  let reconnectScheduled = false;
+  const tryReconnect = () => {
+    if (!reconnectScheduled) {
+      reconnectScheduled = true;
+      connections.delete(peer.nodeId);
+      scheduleReconnect(peer);
+    }
+  };
+
+  ws.addEventListener("error", tryReconnect);
+  ws.addEventListener("close", tryReconnect);
+}
+
+function scheduleReconnect(peer: { nodeId: string; url: string }) {
+  if (reconnectTimers.has(peer.nodeId)) return;
+  const attempt = reconnectAttempts.get(peer.nodeId) ?? 0;
+  const delay = Math.min(RECONNECT_BASE * 2 ** attempt, RECONNECT_MAX);
+  reconnectAttempts.set(peer.nodeId, attempt + 1);
+
+  const timer = setTimeout(() => {
+    reconnectTimers.delete(peer.nodeId);
+    connectToPeer(peer);
+  }, delay);
+  reconnectTimers.set(peer.nodeId, timer);
+}
+
+// ── Frame handling ──────────────────────────────────────────────
+function handleFrame(frame: AnyClusterFrame, conn: Connection) {
+  switch (frame.type) {
+    case "diagnostic_exec":
+      handleDiagnosticExec(frame as DiagnosticExec, conn);
+      break;
+    case "diagnostic_status":
+      handleDiagnosticStatus(frame as DiagnosticStatus, conn);
+      break;
+    // Silently ignore peer protocol frames — sentinel is not a full node
+    case "peer_sync":
+    case "ping":
+      conn.send({ type: "pong", from: sentinelNodeId(), timestamp: Date.now() } as AnyClusterFrame);
+      break;
+    case "pong":
+      break;
+  }
+}
+
+function handleDiagnosticExec(frame: DiagnosticExec, conn: Connection) {
+  // Rate limiting
+  const now = Date.now();
+  while (execTimestamps.length > 0 && now - execTimestamps[0]! > EXEC_RATE_WINDOW) {
+    execTimestamps.shift();
+  }
+  if (execTimestamps.length >= EXEC_RATE_LIMIT) {
+    conn.send({
+      type: "diagnostic_exec_res",
+      id: frame.id,
+      from: sentinelNodeId(),
+      to: frame.from,
+      timestamp: now,
+      payload: { success: false, error: "Rate limit exceeded" },
+    } as AnyClusterFrame);
+    return;
+  }
+  execTimestamps.push(now);
+
+  const { command, timeout = 30 } = frame.payload;
+  const timeoutMs = timeout * 1000;
+
+  log(`Exec from ${frame.from}: ${command}`);
+
+  const child = spawn("sh", ["-c", command], {
+    stdio: ["ignore", "pipe", "pipe"],
+    timeout: timeoutMs,
+  });
+
+  let stdout = "";
+  let stderr = "";
+  let responded = false;
+  const MAX_OUTPUT = 512 * 1024; // 512KB
+
+  const sendResponse = (payload: Record<string, unknown>) => {
+    if (responded) return;
+    responded = true;
+    conn.send({
+      type: "diagnostic_exec_res",
+      id: frame.id,
+      from: sentinelNodeId(),
+      to: frame.from,
+      timestamp: Date.now(),
+      payload,
+    } as AnyClusterFrame);
+  };
+
+  child.stdout?.on("data", (chunk: Buffer) => {
+    if (stdout.length < MAX_OUTPUT) stdout += chunk.toString();
+  });
+  child.stderr?.on("data", (chunk: Buffer) => {
+    if (stderr.length < MAX_OUTPUT) stderr += chunk.toString();
+  });
+
+  child.on("close", (code) => {
+    sendResponse({
+      success: code === 0,
+      exitCode: code ?? 1,
+      stdout: stdout.slice(0, MAX_OUTPUT),
+      stderr: stderr.slice(0, MAX_OUTPUT),
+    });
+  });
+
+  child.on("error", (err) => {
+    sendResponse({ success: false, error: err.message });
+  });
+}
+
+function handleDiagnosticStatus(frame: DiagnosticStatus, conn: Connection) {
+  conn.send({
+    type: "diagnostic_status_res",
+    id: frame.id,
+    from: sentinelNodeId(),
+    to: frame.from,
+    timestamp: Date.now(),
+    payload: {
+      gatewayAlive,
+      uptimeMs: Date.now() - startTime,
+      pid: process.pid,
+      gatewayPid: gatewayAlive && gatewayPid ? gatewayPid : undefined,
+      listening,
+    },
+  } as AnyClusterFrame);
+}
+
+// ── Port takeover: listen when gateway dies, release when it returns ──
+
+function startListening() {
+  if (listening || !config.listenPort) return;
+  // If we've been replaced by a new sentinel, exit instead of competing for the port
+  if (isReplaced()) {
+    log("PID file replaced — another sentinel is active, exiting");
+    cleanup();
+    return;
+  }
+  const port = config.listenPort;
+  const host = config.listenHost ?? "0.0.0.0";
+
+  const e2eeOpts: ConnectionE2eeOptions = {
+    e2ee: config.e2ee,
+    compression: config.compression,
+    identityKeyPair: config.e2ee && identityKeyPair ? identityKeyPair : undefined,
+    deferAuthOk: !!config.peerApproval?.enabled,
+  };
+
+  httpServer = createServer((_req, res) => {
+    res.writeHead(200, { "Content-Type": "text/html", "Server": "nginx" });
+    res.end("<!DOCTYPE html><html><head><title>Welcome</title></head><body><p>It works!</p></body></html>");
+  });
+
+  wss = new WebSocketServer({
+    server: httpServer,
+    handleProtocols(protocols) {
+      if (protocols.size > 0) return protocols.values().next().value!;
+      return false;
+    },
+  });
+
+  wss.on("connection", (ws) => {
+    const transport: WsTransport = {
+      send(data: string) { ws.send(data); },
+      close(code?: number, reason?: string) { ws.close(code, reason); },
+      get readyState() { return ws.readyState; },
+    };
+
+    const conn = new Connection(
+      transport,
+      "inbound",
+      sentinelNodeId(),
+      config.secret,
+      buildCapabilities(),
+      e2eeOpts,
+    );
+
+    inboundConnections.set(ws, conn);
+
+    conn.on("authenticated", (caps) => {
+      const nodeId = caps.nodeId;
+      const peerPublicKey = conn.remoteIdentityKey;
+
+      // Sentinel only accepts already-approved peers — no approval flow
+      if (!isSentinelPeerAllowed(nodeId, peerPublicKey)) {
+        log(`Rejected unapproved peer: ${nodeId}`);
+        conn.close(4005, "not approved");
+        return;
+      }
+
+      conn.completeAuth();
+      connections.set(nodeId, conn);
+      log(`Inbound peer authenticated: ${nodeId}`);
+
+      // Send peer_sync so the client can see this sentinel in its peer list
+      conn.send({
+        type: "peer_sync",
+        from: sentinelNodeId(),
+        timestamp: Date.now(),
+        payload: { peers: [buildCapabilities()] },
+      } as AnyClusterFrame);
+    });
+
+    conn.on("message", (frame) => handleFrame(frame, conn));
+
+    conn.on("close", () => {
+      inboundConnections.delete(ws);
+    });
+
+    conn.on("error", () => { /* close will follow */ });
+
+    ws.on("message", (data) => {
+      conn.feedMessage(typeof data === "string" ? data : String(data));
+    });
+
+    ws.on("close", (code, reason) => {
+      conn.feedClose(code, reason.toString());
+      inboundConnections.delete(ws);
+    });
+  });
+
+  httpServer.on("error", (err) => {
+    log(`Listener error on port ${port}: ${err.message}`);
+    httpServer?.close();
+    httpServer = null;
+    wss?.close();
+    wss = null;
+    listening = false;
+
+    // If we've been replaced by a new sentinel, exit gracefully
+    if (isReplaced()) {
+      log("PID file replaced — exiting");
+      cleanup();
+      return;
+    }
+    // Port may still be held briefly by the dying gateway — retry after a delay
+    setTimeout(() => {
+      if (!gatewayAlive && config.listenPort) startListening();
+    }, 3_000);
+  });
+
+  httpServer.listen(port, host, () => {
+    listening = true;
+    log(`Port takeover: listening on ${host}:${port}`);
+  });
+}
+
+function stopListening() {
+  if (!listening) return;
+  // Gracefully close all inbound connections
+  for (const [ws, conn] of inboundConnections) {
+    conn.close(1001, "gateway recovered");
+    ws.close(1001, "gateway recovered");
+  }
+  inboundConnections.clear();
+  wss?.close();
+  wss = null;
+  httpServer?.close();
+  httpServer = null;
+  listening = false;
+  log("Port released — gateway is back");
+}
+
+// ── PID file management ─────────────────────────────────────────
+function writePidFile() {
+  writeFileSync(config.pidFile, String(process.pid));
+}
+
+/** Check if another sentinel has replaced us (PID file contains a different PID). */
+function isReplaced(): boolean {
+  try {
+    if (!existsSync(config.pidFile)) return true;
+    const filePid = parseInt(readFileSync(config.pidFile, "utf-8").trim(), 10);
+    return filePid !== process.pid;
+  } catch {
+    return false;
+  }
+}
+
+function killOldSentinel() {
+  if (!existsSync(config.pidFile)) return;
+  try {
+    const oldPid = parseInt(readFileSync(config.pidFile, "utf-8").trim(), 10);
+    if (oldPid && oldPid !== process.pid) {
+      try {
+        process.kill(oldPid, 0); // existence check
+        process.kill(oldPid, "SIGTERM");
+        log(`Killing old sentinel (pid ${oldPid})`);
+        // Wait for the old process to actually exit (up to 5s)
+        const deadline = Date.now() + 5_000;
+        while (Date.now() < deadline) {
+          try {
+            process.kill(oldPid, 0);
+            // Still alive — busy-wait briefly
+            const waitUntil = Date.now() + 100;
+            while (Date.now() < waitUntil) { /* spin */ }
+          } catch {
+            // Process exited
+            log(`Old sentinel (pid ${oldPid}) exited`);
+            break;
+          }
+        }
+      } catch {
+        // Process already gone
+      }
+    }
+  } catch {
+    // Malformed PID file
+  }
+}
+
+function cleanup() {
+  if (healthCheckTimer) { clearInterval(healthCheckTimer); healthCheckTimer = null; }
+  try { unlinkSync(config.pidFile); } catch { /* ignore */ }
+  for (const conn of connections.values()) conn.close(1000, "shutdown");
+  for (const timer of reconnectTimers.values()) clearTimeout(timer);
+  stopListening();
+  process.exit(0);
+}
+
+// ── Logging ─────────────────────────────────────────────────────
+function log(msg: string) {
+  const ts = new Date().toISOString();
+  // Generic prefix to avoid endpoint detection fingerprinting
+  process.stderr.write(`[svc ${ts}] ${msg}\n`);
+}
+
+// ── Bootstrap ───────────────────────────────────────────────────
+process.on("message", (msg: unknown) => {
+  const m = msg as { type: string; config?: SentinelConfig };
+  if (m.type === "init" && m.config) {
+    config = m.config;
+    boot();
+  } else if (m.type === "shutdown") {
+    cleanup();
+  }
+});
+
+// Parent IPC disconnect — by design (SentinelManager disconnects after init).
+// Switch to PID-based health checks instead of treating disconnect as crash.
+process.on("disconnect", () => {
+  log("IPC disconnected — switching to PID-based gateway health check");
+  startGatewayHealthCheck();
+});
+
+process.on("SIGTERM", () => { log("Received SIGTERM"); cleanup(); });
+process.on("SIGINT", () => { log("Received SIGINT"); cleanup(); });
+process.on("SIGHUP", () => { log("Received SIGHUP (ignored)"); });
+process.on("uncaughtException", (err) => {
+  log(`Uncaught exception: ${err.stack || err.message}`);
+  // EADDRINUSE from a listen call means the port is taken — if we've been
+  // replaced by a new sentinel/gateway, exit cleanly instead of looping.
+  if ((err as NodeJS.ErrnoException).code === "EADDRINUSE" && isReplaced()) {
+    log("Port in use and PID file replaced — exiting");
+    cleanup();
+  }
+});
+process.on("unhandledRejection", (reason) => { log(`Unhandled rejection: ${reason}`); });
+process.on("beforeExit", (code) => { log(`beforeExit code=${code}`); });
+process.on("exit", (code) => {
+  // Sync write since event loop is draining
+  const ts = new Date().toISOString();
+  try { process.stderr.write(`[svc ${ts}] Exit code=${code}\n`); } catch { /* ignore */ }
+});
+
+/** Periodically check if the gateway process is still alive via kill(pid, 0). */
+function startGatewayHealthCheck() {
+  if (healthCheckTimer || !gatewayPid) return;
+  healthCheckTimer = setInterval(() => {
+    if (!gatewayPid) return;
+    try {
+      process.kill(gatewayPid, 0); // signal 0 = existence check
+      if (!gatewayAlive) {
+        gatewayAlive = true;
+        log("Gateway process detected — back online");
+        // Release the port so the gateway can reclaim it
+        stopListening();
+      }
+    } catch {
+      if (gatewayAlive) {
+        gatewayAlive = false;
+        log(`Gateway process (pid ${gatewayPid}) gone — entering standalone mode`);
+        // Take over the gateway's listen port
+        if (config.listenPort) {
+          // Small delay to let the OS release the port from the dead process
+          setTimeout(() => {
+            if (!gatewayAlive && !isReplaced()) startListening();
+          }, 2_000);
+        }
+      }
+    }
+  }, 5_000);
+}
+
+function boot() {
+  // Prefer explicit gatewayPid from config (sent by SentinelManager),
+  // fall back to ppid (may be inaccurate if forked indirectly).
+  gatewayPid = config.gatewayPid ?? process.ppid;
+
+  loadApprovedPeers();
+  killOldSentinel();
+  writePidFile();
+  log(`Started (pid ${process.pid}, gateway ${gatewayPid}, nodeId ${sentinelNodeId()}, takeover port ${config.listenPort || "none"})`);
+
+  // Connect to all configured peers
+  for (const peer of config.peers) {
+    connectToPeer(peer);
+  }
+
+  // Note: we do NOT start listening here.
+  // Listening only starts when gateway dies (port takeover mode).
+}

package/src/task-activity.ts

@@ -0,0 +1,74 @@
+import type { PeerManager } from "./peer-manager.ts";
+import type { ClawMatrixConfig } from "./config.ts";
+import type { TaskActivityFrame, TaskActivityStatus } from "./types.ts";
+
+/**
+ * Broadcasts task_activity frames to mobile peers (tagged mobile/ios/phone).
+ * Shared between AcpProxy and HandoffManager to avoid duplication.
+ */
+export class TaskActivityBroadcaster {
+  private config: ClawMatrixConfig;
+  private peerManager: PeerManager;
+  private throttles = new Map<string, number>();
+
+  constructor(config: ClawMatrixConfig, peerManager: PeerManager) {
+    this.config = config;
+    this.peerManager = peerManager;
+  }
+
+  broadcast(
+    taskId: string,
+    taskType: "acp" | "handoff",
+    status: TaskActivityStatus,
+    agent: string,
+    startedAt: number,
+    detail?: string,
+    tool?: string,
+    toolDone?: boolean,
+  ) {
+    // Throttle progress updates to at most once per 3 seconds per task
+    if (status === "progress") {
+      const now = Date.now();
+      const lastSent = this.throttles.get(taskId) ?? 0;
+      if (now - lastSent < 3_000) return;
+      this.throttles.set(taskId, now);
+    } else {
+      this.throttles.delete(taskId);
+    }
+
+    const peers = this.peerManager.router.getAllPeers();
+    const mobileTargets = peers.filter((p) =>
+      p.tags.some((t) => t === "mobile" || t === "ios" || t === "phone"),
+    );
+    if (mobileTargets.length === 0) return;
+
+    const now = Date.now();
+    const frame: TaskActivityFrame = {
+      type: "task_activity",
+      from: this.config.nodeId,
+      timestamp: now,
+      payload: {
+        taskId,
+        taskType,
+        status,
+        agent,
+        nodeId: this.config.nodeId,
+        title: agent,
+        detail,
+        startedAt,
+        elapsedMs: now - startedAt,
+        tool,
+        toolDone,
+      },
+    };
+
+    for (const target of mobileTargets) {
+      this.peerManager.sendTo(target.nodeId, { ...frame, to: target.nodeId });
+    }
+  }
+
+  /** Clean up throttle state for a completed/failed task. */
+  cleanup(taskId: string) {
+    this.throttles.delete(taskId);
+  }
+}