@mooncompany/uplink-chat 0.5.0

package/server/realtime/index.js

@@ -0,0 +1,349 @@
+/**
+ * Realtime Voice Relay — WebSocket proxy to OpenAI Realtime API
+ *
+ * Accepts client WebSocket connections on `/api/realtime`, opens a
+ * corresponding WebSocket to `wss://api.openai.com/v1/realtime`,
+ * and bidirectionally relays all JSON events between the two.
+ *
+ * On connection, sends a `session.update` to OpenAI with:
+ *   - voice from runtime config (`realtimeVoice`)
+ *   - `semantic_vad` turn detection
+ *   - PCM 24kHz audio format (input + output)
+ *   - agent instructions (from the main agent's identity.theme or a default)
+ *
+ * Usage (in server.js):
+ *   import { setupRealtimeRelay } from './server/realtime/index.js';
+ *   setupRealtimeRelay(httpServer);
+ */
+
+import { WebSocketServer, WebSocket } from 'ws';
+import fs from 'fs/promises';
+import path from 'path';
+import os from 'os';
+import { createLogger } from '../logger.js';
+import { loadConfig } from '../runtime-config.js';
+import { ALLOWED_ORIGINS } from '../config.js';
+
+const log = createLogger('realtime');
+
+// ─── Constants ──────────────────────────────────────────────────────────────
+const OPENAI_REALTIME_BASE = 'wss://api.openai.com/v1/realtime';
+const DEFAULT_MODEL = 'gpt-4o-mini-realtime-preview';
+
+// Connection bookkeeping
+const MAX_REALTIME_CONNECTIONS = 5;
+const activeSessions = new Map(); // sessionId → { clientWs, openaiWs }
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+/**
+ * Read agent identity/theme from the OpenClaw config file.
+ * Falls back to a sensible default if anything goes wrong.
+ */
+async function getAgentInstructions() {
+  const DEFAULT_INSTRUCTIONS =
+    'You are a helpful voice assistant. Be concise, friendly, and conversational. ' +
+    'Keep responses short since this is a voice conversation.';
+
+  try {
+    const stateDir = process.env.OPENCLAW_STATE_DIR || path.join(os.homedir(), '.openclaw');
+    const configPath = path.join(stateDir, 'openclaw.json');
+    const raw = await fs.readFile(configPath, 'utf8');
+    const config = JSON.parse(raw);
+
+    // Find the main agent (id === 'main') and pull identity.theme
+    const mainAgent = config.agents?.list?.find(a => a.id === 'main');
+    const name = mainAgent?.identity?.name;
+    const theme = mainAgent?.identity?.theme;
+
+    if (theme) {
+      return `Your name is ${name || 'Assistant'}. ${theme}. ` +
+        'Keep responses concise — this is a real-time voice conversation.';
+    }
+    if (name) {
+      return `Your name is ${name}. ${DEFAULT_INSTRUCTIONS}`;
+    }
+  } catch {
+    // Config not found or unparseable — fall through to default
+  }
+
+  return DEFAULT_INSTRUCTIONS;
+}
+
+/**
+ * Verify that the WebSocket upgrade origin is allowed.
+ */
+function verifyOrigin(origin) {
+  if (!origin) return true; // Non-browser clients don't send Origin
+  try {
+    const url = new URL(origin);
+    const host = url.hostname;
+    if (host === 'localhost' || host === '127.0.0.1' || host === '::1') return true;
+    if (host.endsWith('.ts.net')) return true;
+    return ALLOWED_ORIGINS.some(allowed => {
+      try { return new URL(allowed).hostname === host; } catch { return false; }
+    });
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Build the `session.update` event sent to OpenAI on connection.
+ */
+function buildSessionUpdate(voice, instructions) {
+  return JSON.stringify({
+    type: 'session.update',
+    session: {
+      modalities: ['text', 'audio'],
+      instructions,
+      voice: voice || 'marin',
+      input_audio_format: 'pcm16',
+      output_audio_format: 'pcm16',
+      input_audio_transcription: {
+        model: 'whisper-1',
+      },
+      turn_detection: {
+        type: 'server_vad',
+        threshold: 0.5,
+        prefix_padding_ms: 300,
+        silence_duration_ms: 500,
+      },
+    },
+  });
+}
+
+// ─── Public API ─────────────────────────────────────────────────────────────
+
+/**
+ * Set up the Realtime API WebSocket relay on the given HTTP server.
+ *
+ * Listens for upgrade requests on `/api/realtime` using noServer mode
+ * (same pattern as gateway-proxy.js) so it coexists with the main
+ * WebSocket server and Gateway proxy.
+ *
+ * @param {import('http').Server} server - The HTTP server instance
+ */
+export function setupRealtimeRelay(server) {
+  const wss = new WebSocketServer({ noServer: true, perMessageDeflate: false });
+
+  // ── Handle upgrade on /api/realtime ───────────────────────────────────
+  server.on('upgrade', (request, socket, head) => {
+    const { pathname } = new URL(request.url, `http://${request.headers.host}`);
+
+    if (pathname !== '/api/realtime') return; // Let other handlers take it
+
+    // Agent mode is handled by bridge.js's own upgrade handler
+    const url = new URL(request.url, `http://${request.headers.host}`);
+    if (url.searchParams.get('mode') === 'agent') return;
+
+    // Origin check
+    const origin = request.headers.origin;
+    if (!verifyOrigin(origin)) {
+      log.warn(`Rejected upgrade from invalid origin: ${origin}`);
+      socket.write('HTTP/1.1 403 Forbidden\r\n\r\n');
+      socket.destroy();
+      return;
+    }
+
+    // Connection limit
+    if (activeSessions.size >= MAX_REALTIME_CONNECTIONS) {
+      log.warn(`Rejected upgrade — at capacity (${activeSessions.size}/${MAX_REALTIME_CONNECTIONS})`);
+      socket.write('HTTP/1.1 429 Too Many Requests\r\n\r\n');
+      socket.destroy();
+      return;
+    }
+
+    wss.handleUpgrade(request, socket, head, (ws) => {
+      wss.emit('connection', ws, request);
+    });
+  });
+
+  // ── New client connection ─────────────────────────────────────────────
+  wss.on('connection', async (clientWs, req) => {
+    const sessionId = `rt-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+    const clientIp = req.socket.remoteAddress;
+    log.info(`Client connected: ${sessionId} from ${clientIp}`);
+
+    // ── Standalone mode (agent mode handled by bridge.js) ─────────────
+
+    // ── Read runtime config ─────────────────────────────────────────────
+    let config;
+    try {
+      config = await loadConfig();
+    } catch (err) {
+      log.error('Failed to load runtime config:', err);
+      clientWs.close(1011, 'Server configuration error');
+      return;
+    }
+
+    const apiKey = config.openaiApiKey;
+    if (!apiKey) {
+      log.error('No OpenAI API key configured — cannot open realtime session');
+      clientWs.send(JSON.stringify({
+        type: 'error',
+        error: 'No OpenAI API key configured. Add your key in Settings → Voice.',
+      }));
+      clientWs.close(1008, 'Missing API key');
+      return;
+    }
+
+    const model = config.realtimeModel || DEFAULT_MODEL;
+    const voice = config.realtimeVoice || 'marin';
+
+    // ── Open outbound WebSocket to OpenAI ───────────────────────────────
+    const openaiUrl = `${OPENAI_REALTIME_BASE}?model=${encodeURIComponent(model)}`;
+    let openaiWs;
+    try {
+      openaiWs = new WebSocket(openaiUrl, {
+        headers: {
+          'Authorization': `Bearer ${apiKey}`,
+          'OpenAI-Beta': 'realtime=v1',
+        },
+        perMessageDeflate: true,
+      });
+    } catch (err) {
+      log.error('Failed to create OpenAI WebSocket:', err);
+      clientWs.close(1011, 'Failed to connect to OpenAI');
+      return;
+    }
+
+    // Track session
+    let clientClosed = false;
+    let openaiClosed = false;
+    activeSessions.set(sessionId, { clientWs, openaiWs });
+
+    // Keepalive ping
+    const keepalive = setInterval(() => {
+      try {
+        if (clientWs.readyState === WebSocket.OPEN) clientWs.ping();
+        if (openaiWs.readyState === WebSocket.OPEN) openaiWs.ping();
+      } catch { /* swallow */ }
+    }, 15_000);
+
+    function cleanup() {
+      clearInterval(keepalive);
+      activeSessions.delete(sessionId);
+      if (!clientClosed && clientWs.readyState === WebSocket.OPEN) {
+        clientWs.close();
+      }
+      if (!openaiClosed && openaiWs.readyState === WebSocket.OPEN) {
+        openaiWs.close();
+      }
+    }
+
+    // ── OpenAI WebSocket events ─────────────────────────────────────────
+
+    openaiWs.on('open', async () => {
+      log.info(`OpenAI connected for session ${sessionId} (model=${model}, voice=${voice})`);
+
+      // Send session.update with agent instructions
+      try {
+        const instructions = await getAgentInstructions();
+        const sessionUpdate = buildSessionUpdate(voice, instructions);
+        log.info(`Sending session.update for ${sessionId}: ${sessionUpdate.substring(0, 500)}`);
+        openaiWs.send(sessionUpdate);
+      } catch (err) {
+        log.error(`Failed to send session.update for ${sessionId}:`, err);
+      }
+
+      // Notify client that the relay is ready
+      if (clientWs.readyState === WebSocket.OPEN) {
+        clientWs.send(JSON.stringify({ type: 'relay.ready' }));
+      }
+    });
+
+    openaiWs.on('message', (data, isBinary) => {
+      // Log OpenAI events for debugging
+      if (!isBinary) {
+        try {
+          const event = JSON.parse(data.toString());
+          if (event.type === 'error') {
+            log.error(`OpenAI error event for ${sessionId}:`, JSON.stringify(event));
+          } else {
+            log.info(`OpenAI event for ${sessionId}: ${event.type}`);
+          }
+        } catch { /* not JSON, skip */ }
+      }
+
+      // Forward OpenAI → Client
+      if (clientWs.readyState === WebSocket.OPEN) {
+        try {
+          if (isBinary) {
+            clientWs.send(data, { binary: true });
+          } else {
+            clientWs.send(data.toString());
+          }
+        } catch (err) {
+          log.error(`Error forwarding OpenAI → client (${sessionId}):`, err.message);
+        }
+      }
+    });
+
+    openaiWs.on('close', (code, reason) => {
+      openaiClosed = true;
+      const reasonStr = reason?.toString() || '';
+      log.info(`OpenAI disconnected for ${sessionId} (code=${code}${reasonStr ? ', reason=' + reasonStr : ''})`);
+      cleanup();
+    });
+
+    openaiWs.on('error', (err) => {
+      log.error(`OpenAI error for ${sessionId}:`, err.message);
+
+      // Forward a structured error to the client so it can display something useful
+      if (clientWs.readyState === WebSocket.OPEN) {
+        try {
+          clientWs.send(JSON.stringify({
+            type: 'error',
+            error: `OpenAI connection error: ${err.message}`,
+          }));
+        } catch { /* swallow */ }
+      }
+      cleanup();
+    });
+
+    // ── Client WebSocket events ─────────────────────────────────────────
+
+    clientWs.on('message', (data, isBinary) => {
+      // Log what the client is sending
+      if (!isBinary) {
+        try {
+          const clientMsg = JSON.parse(data.toString());
+          log.info(`Client event for ${sessionId}: ${clientMsg.type} (len=${data.length})`);
+        } catch { /* not JSON */ }
+      } else {
+        log.info(`Client binary for ${sessionId}: len=${data.length}`);
+      }
+
+      // Forward Client → OpenAI
+      // IMPORTANT: Send text frames as strings, not Buffers, so ws sends them
+      // as text frames (opcode 0x1) instead of binary frames (opcode 0x2).
+      // OpenAI Realtime API expects JSON as text frames.
+      if (openaiWs.readyState === WebSocket.OPEN) {
+        try {
+          if (isBinary) {
+            openaiWs.send(data);
+          } else {
+            openaiWs.send(data.toString());
+          }
+        } catch (err) {
+          log.error(`Error forwarding client → OpenAI (${sessionId}):`, err.message);
+        }
+      }
+    });
+
+    clientWs.on('close', (code) => {
+      clientClosed = true;
+      log.info(`Client disconnected for ${sessionId} (code=${code})`);
+      cleanup();
+    });
+
+    clientWs.on('error', (err) => {
+      log.error(`Client error for ${sessionId}:`, err.message);
+      cleanup();
+    });
+  });
+
+  log.info('Realtime voice relay ready at /api/realtime');
+  return wss;
+}