@mooncompany/uplink-chat 0.5.0

package/server/watchdog.js

@@ -0,0 +1,318 @@
+#!/usr/bin/env node
+
+/**
+ * Uplink Watchdog
+ * 
+ * Monitors the server process and restarts it on crash with exponential backoff.
+ * Spawned by `uplink-chat -d` — runs detached, manages the server lifecycle.
+ * 
+ * Features:
+ * - Exponential backoff on crash (1s, 2s, 4s... max 60s)
+ * - Clean exit detection (code 0, SIGTERM) → watchdog exits too
+ * - Crash loop protection (5 crashes in 2 minutes → stop)
+ * - State file for status reporting (.uplink-watchdog.json)
+ * - Log file for restart history (.uplink-watchdog.log)
+ */
+
+import { spawn, execSync } from 'child_process';
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+import { writeFileSync, appendFileSync, readFileSync, existsSync, unlinkSync, openSync } from 'fs';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const ROOT = join(__dirname, '..');
+
+const PID_FILE = join(ROOT, '.uplink-watchdog.pid');
+const STATE_FILE = join(ROOT, '.uplink-watchdog.json');
+const LOG_FILE = join(ROOT, '.uplink-watchdog.log');
+const SERVER_PATH = join(ROOT, 'server.js');
+
+// Configuration
+const MAX_BACKOFF_MS = 60000;       // 60 seconds max backoff
+const INITIAL_BACKOFF_MS = 1000;    // 1 second initial backoff
+const STABLE_THRESHOLD_MS = 30000;  // 30s of stable running resets backoff
+const CRASH_WINDOW_MS = 120000;     // 2 minute window for crash loop detection
+const MAX_CRASHES_IN_WINDOW = 5;    // 5 crashes in window = stop
+
+// State
+let serverProcess = null;
+let backoffMs = INITIAL_BACKOFF_MS;
+let restartCount = 0;
+let startedAt = Date.now();
+let serverStartedAt = null;
+let crashTimestamps = [];
+let shuttingDown = false;
+
+// Parse environment from argv (passed as JSON)
+const envArg = process.argv[2];
+let serverEnv = { ...process.env };
+if (envArg) {
+  try {
+    const extraEnv = JSON.parse(envArg);
+    Object.assign(serverEnv, extraEnv);
+  } catch {
+    // Ignore parse errors
+  }
+}
+
+// Read config.json for server settings (networkAccess, etc.)
+try {
+  const configPath = join(ROOT, 'config.json');
+  const configData = readFileSync(configPath, 'utf8');
+  const config = JSON.parse(configData);
+  
+  // Apply networkAccess setting if not already set by env/CLI
+  if (!serverEnv.UPLINK_HOST && config.networkAccess === true) {
+    serverEnv.UPLINK_HOST = '0.0.0.0';
+    logToFile('Network access enabled via config.json — binding to 0.0.0.0');
+  }
+} catch {
+  // Config not found or invalid — use defaults
+}
+
+/**
+ * Append a timestamped line to the watchdog log
+ */
+function logToFile(message) {
+  const timestamp = new Date().toISOString();
+  const line = `[${timestamp}] ${message}\n`;
+  try {
+    appendFileSync(LOG_FILE, line);
+  } catch {
+    // Can't log — ignore
+  }
+}
+
+/**
+ * Write current watchdog state to JSON file
+ */
+function writeState() {
+  const state = {
+    status: shuttingDown ? 'stopping' : (serverProcess ? 'running' : 'stopped'),
+    watchdogPid: process.pid,
+    serverPid: serverProcess?.pid || null,
+    startedAt,
+    serverStartedAt,
+    restartCount,
+    backoffMs,
+    lastUpdated: Date.now(),
+  };
+  try {
+    writeFileSync(STATE_FILE, JSON.stringify(state, null, 2));
+  } catch {
+    // Ignore write errors
+  }
+}
+
+/**
+ * Write watchdog PID file
+ */
+function writePidFile() {
+  try {
+    writeFileSync(PID_FILE, String(process.pid));
+  } catch {
+    // Ignore
+  }
+}
+
+/**
+ * Clean up PID and state files
+ */
+function cleanup() {
+  try { unlinkSync(PID_FILE); } catch {}
+  // Update state to stopped
+  const state = {
+    status: 'stopped',
+    watchdogPid: null,
+    serverPid: null,
+    startedAt,
+    serverStartedAt: null,
+    restartCount,
+    backoffMs: INITIAL_BACKOFF_MS,
+    lastUpdated: Date.now(),
+  };
+  try {
+    writeFileSync(STATE_FILE, JSON.stringify(state, null, 2));
+  } catch {}
+}
+
+/**
+ * Check if we're in a crash loop
+ */
+function isInCrashLoop() {
+  const now = Date.now();
+  // Remove old timestamps outside the window
+  crashTimestamps = crashTimestamps.filter(t => now - t < CRASH_WINDOW_MS);
+  return crashTimestamps.length >= MAX_CRASHES_IN_WINDOW;
+}
+
+/**
+ * Kill the server process gracefully
+ */
+function killServer() {
+  if (!serverProcess) return;
+  
+  try {
+    if (process.platform === 'win32') {
+      try {
+        execSync(`taskkill /PID ${serverProcess.pid} /T /F`, { stdio: 'ignore' });
+      } catch {
+        serverProcess.kill();
+      }
+    } else {
+      serverProcess.kill('SIGTERM');
+    }
+  } catch {
+    // Process might already be dead
+  }
+}
+
+/**
+ * Spawn the server process and monitor it
+ */
+function spawnServer() {
+  if (shuttingDown) return;
+  
+  serverStartedAt = Date.now();
+  
+  logToFile(`Starting server (attempt ${restartCount + 1}, backoff: ${backoffMs}ms)`);
+  
+  const serverLogFd = openSync(join(ROOT, '.uplink-server.log'), 'a');
+  serverProcess = spawn(process.execPath, [SERVER_PATH], {
+    cwd: ROOT,
+    env: serverEnv,
+    stdio: ['ignore', serverLogFd, serverLogFd],
+    detached: false,
+  });
+  
+  try {
+    writeFileSync(join(ROOT, '.uplink.pid'), String(serverProcess.pid));
+  } catch {}
+  
+  writeState();
+  
+  const stableTimer = setTimeout(() => {
+    // Server has been running stably for STABLE_THRESHOLD_MS — reset backoff
+    backoffMs = INITIAL_BACKOFF_MS;
+    logToFile('Server stable — backoff reset');
+    writeState();
+  }, STABLE_THRESHOLD_MS);
+  
+  serverProcess.on('exit', (code, signal) => {
+    clearTimeout(stableTimer);
+    serverProcess = null;
+    
+    if (shuttingDown) {
+      logToFile(`Server exited during shutdown (code: ${code}, signal: ${signal})`);
+      writeState();
+      return;
+    }
+    
+    // Clean exit (code 0) → watchdog exits too
+    if (code === 0) {
+      logToFile(`Server exited cleanly (code: 0) — watchdog exiting`);
+      cleanup();
+      process.exit(0);
+      return;
+    }
+    if (signal === 'SIGTERM') {
+      logToFile(`Server killed by SIGTERM — treating as crash, will restart`);
+    }
+    
+    // Crash — record it
+    restartCount++;
+    crashTimestamps.push(Date.now());
+    logToFile(`Server crashed (code: ${code}, signal: ${signal}) — restart #${restartCount}`);
+    
+    // Check for crash loop
+    if (isInCrashLoop()) {
+      logToFile(`CRASH LOOP DETECTED: ${MAX_CRASHES_IN_WINDOW} crashes in ${CRASH_WINDOW_MS / 1000}s — stopping watchdog`);
+      cleanup();
+      process.exit(1);
+      return;
+    }
+    
+    // Schedule restart with backoff
+    logToFile(`Restarting in ${backoffMs}ms...`);
+    writeState();
+    
+    setTimeout(() => {
+      // Double backoff for next time (capped)
+      backoffMs = Math.min(backoffMs * 2, MAX_BACKOFF_MS);
+      spawnServer();
+    }, backoffMs);
+  });
+  
+  serverProcess.on('error', (err) => {
+    clearTimeout(stableTimer);
+    logToFile(`Server process error: ${err.message}`);
+    serverProcess = null;
+    
+    if (shuttingDown) return;
+    
+    restartCount++;
+    crashTimestamps.push(Date.now());
+    
+    if (isInCrashLoop()) {
+      logToFile(`CRASH LOOP DETECTED — stopping watchdog`);
+      cleanup();
+      process.exit(1);
+      return;
+    }
+    
+    setTimeout(() => {
+      backoffMs = Math.min(backoffMs * 2, MAX_BACKOFF_MS);
+      spawnServer();
+    }, backoffMs);
+  });
+}
+
+/**
+ * Graceful shutdown handler
+ */
+function gracefulShutdown(signal) {
+  if (shuttingDown) return;
+  shuttingDown = true;
+  
+  logToFile(`Watchdog received ${signal} — shutting down`);
+  
+  if (serverProcess) {
+    killServer();
+    
+    // Give server 10 seconds to exit gracefully
+    const forceKillTimer = setTimeout(() => {
+      logToFile('Force killing server after timeout');
+      try {
+        serverProcess?.kill('SIGKILL');
+      } catch {}
+      cleanup();
+      process.exit(0);
+    }, 10000);
+    forceKillTimer.unref();
+    
+    // Wait for server to exit
+    serverProcess.on('exit', () => {
+      clearTimeout(forceKillTimer);
+      cleanup();
+      process.exit(0);
+    });
+  } else {
+    cleanup();
+    process.exit(0);
+  }
+}
+
+// Register signal handlers
+process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
+process.on('SIGINT', () => gracefulShutdown('SIGINT'));
+
+// Windows: handle Ctrl+C
+if (process.platform === 'win32') {
+  process.on('SIGHUP', () => gracefulShutdown('SIGHUP'));
+}
+
+// Write PID file and start
+writePidFile();
+logToFile(`Watchdog started (PID: ${process.pid})`);
+spawnServer();

package/server/websocket/broadcast.js

@@ -0,0 +1,359 @@
+/**
+ * WebSocket Broadcast Module
+ * Broadcasting utilities for all clients, sync, and OpenClaw push
+ */
+
+import { WebSocket } from 'ws';
+import { log } from '../utils.js';
+import { WEBSOCKET } from '../config.js';
+import { broadcastToProxyClients } from '../gateway-proxy.js';
+import { wsClients, safeSend } from './connections.js';
+
+// ============================================
+// Broadcast Circuit Breaker
+// ============================================
+
+const broadcastCircuitBreaker = {
+  failures: 0,
+  lastFailure: 0,
+  isOpen: false,
+  threshold: WEBSOCKET.broadcastCircuitBreaker.threshold,
+  resetTimeMs: WEBSOCKET.broadcastCircuitBreaker.resetTimeMs,
+};
+
+function checkCircuitBreaker() {
+  if (!broadcastCircuitBreaker.isOpen) return true;
+
+  // Check if reset time has passed
+  if (Date.now() - broadcastCircuitBreaker.lastFailure > broadcastCircuitBreaker.resetTimeMs) {
+    broadcastCircuitBreaker.isOpen = false;
+    broadcastCircuitBreaker.failures = 0;
+    log('info', '[WS] Broadcast circuit breaker reset');
+    return true;
+  }
+
+  return false;
+}
+
+function recordBroadcastFailure() {
+  broadcastCircuitBreaker.failures++;
+  broadcastCircuitBreaker.lastFailure = Date.now();
+
+  if (broadcastCircuitBreaker.failures >= broadcastCircuitBreaker.threshold) {
+    broadcastCircuitBreaker.isOpen = true;
+    log('warn', `[WS] Broadcast circuit breaker OPEN after ${broadcastCircuitBreaker.failures} failures`);
+  }
+}
+
+// ============================================
+// Message ID Generation
+// ============================================
+
+/**
+ * Generate a unique message ID for deduplication
+ * Format: msg_<timestamp>_<random>
+ */
+export function generateMessageId() {
+  const timestamp = Date.now();
+  const random = Math.random().toString(36).substring(2, 10);
+  return `msg_${timestamp}_${random}`;
+}
+
+// ============================================
+// Deduplication
+// ============================================
+
+// Deduplication cache for recently broadcast messages (prevents sync + push duplicates)
+const recentBroadcasts = new Map(); // hash -> timestamp
+const DEDUP_WINDOW_MS = WEBSOCKET.dedupWindowMs;
+const MAX_RECENT_BROADCASTS = WEBSOCKET.maxRecentBroadcasts;
+
+function getMessageHash(content, satelliteId) {
+  // DJB2 hash of full content string for reliable deduplication
+  const str = `${satelliteId}:${content || ''}`;
+  let hash = 5381;
+  for (let i = 0; i < str.length; i++) {
+    hash = ((hash << 5) + hash + str.charCodeAt(i)) | 0;
+  }
+  return hash.toString(36);
+}
+
+function cleanupRecentBroadcasts() {
+  const now = Date.now();
+  for (const [hash, timestamp] of recentBroadcasts) {
+    if (now - timestamp > DEDUP_WINDOW_MS) {
+      recentBroadcasts.delete(hash);
+    }
+  }
+  // Hard cap to prevent unbounded growth if cleanup can't keep up
+  if (recentBroadcasts.size > MAX_RECENT_BROADCASTS) {
+    const entries = Array.from(recentBroadcasts.entries());
+    entries.sort((a, b) => a[1] - b[1]);
+    const toRemove = recentBroadcasts.size - MAX_RECENT_BROADCASTS;
+    for (let i = 0; i < toRemove; i++) {
+      recentBroadcasts.delete(entries[i][0]);
+    }
+  }
+}
+
+// Track a broadcast for deduplication
+export function trackBroadcast(content, satelliteId) {
+  cleanupRecentBroadcasts();
+  const hash = getMessageHash(content, satelliteId);
+  recentBroadcasts.set(hash, Date.now());
+}
+
+// ============================================
+// Core Broadcasting
+// ============================================
+
+/**
+ * Broadcast a message to all clients except one (optional exclusion)
+ */
+export function broadcast(message, excludeClientId = null) {
+  // Check circuit breaker
+  if (!checkCircuitBreaker()) {
+    log('debug', '[WS] Broadcast skipped - circuit breaker open');
+    return { sent: 0, failed: 0, skipped: true };
+  }
+
+  const data = typeof message === 'string' ? message : JSON.stringify(message);
+
+  // Snapshot clients to avoid race condition during iteration
+  const clientsSnapshot = Array.from(wsClients.entries());
+
+  let sent = 0;
+  let failed = 0;
+
+  for (const [clientId, client] of clientsSnapshot) {
+    if (clientId === excludeClientId) continue;
+
+    try {
+      if (client.ws.readyState === WebSocket.OPEN) {
+        if (safeSend(client.ws, data)) {
+          sent++;
+        } else {
+          failed++;
+          recordBroadcastFailure();
+        }
+      }
+    } catch (err) {
+      failed++;
+      recordBroadcastFailure();
+      log('warn', `[WS] Broadcast failed for ${clientId}: ${err.message}`);
+    }
+  }
+
+  if (failed > 0) {
+    log('debug', `[WS] Broadcast: ${sent} sent, ${failed} failed`);
+  }
+
+  return { sent, failed, skipped: false };
+}
+
+/**
+ * Broadcast a message to ALL connected clients (no exclusion)
+ */
+export function broadcastToAll(data) {
+  // Check circuit breaker
+  if (!checkCircuitBreaker()) {
+    log('debug', '[WS] BroadcastToAll skipped - circuit breaker open');
+    return { sent: 0, failed: 0, skipped: true };
+  }
+
+  const message = typeof data === 'string' ? data : JSON.stringify(data);
+
+  // Snapshot clients to avoid race condition during iteration
+  const clientsSnapshot = Array.from(wsClients.values());
+
+  let sent = 0;
+  let failed = 0;
+
+  for (const client of clientsSnapshot) {
+    try {
+      if (client.ws.readyState === WebSocket.OPEN) {
+        if (safeSend(client.ws, message)) {
+          sent++;
+        } else {
+          failed++;
+          recordBroadcastFailure();
+        }
+      }
+    } catch (err) {
+      failed++;
+      recordBroadcastFailure();
+      log('warn', `[WS] BroadcastToAll failed: ${err.message}`);
+    }
+  }
+
+  if (failed > 0) {
+    log('debug', `[WS] BroadcastToAll: ${sent} sent, ${failed} failed`);
+  }
+
+  return { sent, failed, skipped: false };
+}
+
+/**
+ * Send a message to a specific client by ID
+ */
+export function sendToClient(clientId, message) {
+  const client = wsClients.get(clientId);
+  if (client && client.ws.readyState === WebSocket.OPEN) {
+    return safeSend(client.ws, message);
+  }
+  return false;
+}
+
+// ============================================
+// Sync Broadcasting (Cross-Device Sync)
+// ============================================
+
+// Rate limiting for sync broadcasts
+const syncBroadcastRateLimit = {
+  count: 0,
+  windowStart: Date.now(),
+  maxPerSecond: WEBSOCKET.syncBroadcastRateLimit.maxPerSecond,
+};
+
+/**
+ * Broadcast a sync message to all clients except the sender
+ * Used for real-time cross-device message synchronization
+ */
+export function broadcastSyncMessage(role, content, satelliteId, messageId, excludeClientId = null, requestId = null) {
+  // Rate limiting: max N broadcasts per second
+  const now = Date.now();
+  if (now - syncBroadcastRateLimit.windowStart > 1000) {
+    syncBroadcastRateLimit.count = 0;
+    syncBroadcastRateLimit.windowStart = now;
+  }
+
+  if (syncBroadcastRateLimit.count >= syncBroadcastRateLimit.maxPerSecond) {
+    log('warn', '[WS] Sync broadcast rate limit exceeded');
+    return { sent: 0, failed: 0, rateLimited: true };
+  }
+
+  syncBroadcastRateLimit.count++;
+
+  // Track assistant messages for deduplication (prevents OpenClaw push duplicates)
+  if (role === 'assistant' && content) {
+    trackBroadcast(content, satelliteId || 'main');
+  }
+
+  const syncMessage = {
+    type: 'event',
+    event: 'sync',
+    payload: {
+      messageId: messageId || generateMessageId(),
+      role,
+      content,
+      satelliteId: satelliteId || 'main',
+      timestamp: now,
+      ...(requestId ? { requestId } : {}),
+    },
+  };
+
+  // Broadcast to direct WebSocket clients (/ws)
+  const directResult = broadcast(syncMessage, excludeClientId);
+
+  // Also broadcast to Gateway proxy clients (/ws/gateway) for mobile sync
+  const proxyResult = broadcastToProxyClients(syncMessage);
+
+  const totalSent = directResult.sent + proxyResult.sent;
+  const totalFailed = directResult.failed + proxyResult.failed;
+
+  log('debug', `[WS] Sync broadcast: ${role} message to ${totalSent} clients (direct: ${directResult.sent}, proxy: ${proxyResult.sent}, satellite: ${satelliteId})`);
+
+  return { sent: totalSent, failed: totalFailed, skipped: directResult.skipped };
+}
+
+/**
+ * Broadcast sync.thinking event to all clients except the sender
+ */
+export function broadcastSyncThinking(requestId, satelliteId, excludeClientId = null) {
+  const message = {
+    type: 'event',
+    event: 'sync.thinking',
+    payload: { requestId, satelliteId: satelliteId || 'main' },
+  };
+  broadcast(message, excludeClientId);
+  broadcastToProxyClients(message);
+}
+
+/**
+ * Broadcast sync.tool event to all clients except the sender
+ */
+export function broadcastSyncTool(requestId, tool, satelliteId, excludeClientId = null) {
+  const message = {
+    type: 'event',
+    event: 'sync.tool',
+    payload: { requestId, tool, satelliteId: satelliteId || 'main' },
+  };
+  broadcast(message, excludeClientId);
+  broadcastToProxyClients(message);
+}
+
+/**
+ * Broadcast sync.complete event with usage stats to all clients
+ * Sent when a response finishes so WS-streaming clients can update token display
+ */
+export function broadcastSyncComplete(requestId, usage, satelliteId, excludeClientId = null) {
+  if (!usage) return;
+  const message = {
+    type: 'event',
+    event: 'sync.complete',
+    payload: { requestId, usage, satelliteId: satelliteId || 'main' },
+  };
+  broadcast(message, excludeClientId);
+  broadcastToProxyClients(message);
+}
+
+// ============================================
+// OpenClaw Push Broadcasting
+// ============================================
+
+/**
+ * Broadcast OpenClaw push messages to all WebSocket clients
+ * Maps payload.type to prefixed WebSocket message type
+ */
+export function broadcastOpenClawPush(payload) {
+  // Skip if this message was recently broadcast via sync (dedup)
+  if (payload.type === 'message' && payload.content) {
+    cleanupRecentBroadcasts();
+    const hash = getMessageHash(payload.content, payload.satelliteId);
+    if (recentBroadcasts.has(hash)) {
+      log('debug', `[WS] OpenClaw push: skipping duplicate for satellite ${payload.satelliteId}`);
+      return;
+    }
+  }
+
+  const payloadData = {
+    satelliteId: payload.satelliteId,
+    content: payload.content,
+    tool: payload.tool,
+    error: payload.error,
+    audioUrl: payload.audioUrl,
+    usage: payload.usage,
+    requestId: payload.requestId,
+    timestamp: payload.timestamp
+  };
+
+  // Remove undefined fields from payload
+  Object.keys(payloadData).forEach(key => {
+    if (payloadData[key] === undefined) delete payloadData[key];
+  });
+
+  const wsMessage = {
+    type: 'event',
+    event: `openclaw.${payload.type}`,
+    payload: payloadData,
+  };
+
+  // Broadcast to direct /ws clients
+  const directResult = broadcastToAll(wsMessage);
+
+  // Also broadcast to Gateway proxy clients (/ws/gateway)
+  const proxyResult = broadcastToProxyClients(wsMessage);
+
+  const hasSent = directResult.sent > 0 || proxyResult.sent > 0;
+  log('info', `[WS] OpenClaw push: ${payload.type} for satellite ${payload.satelliteId} (direct: ${directResult.sent}, proxy: ${proxyResult.sent})`);
+}