@mooncompany/uplink-chat 0.5.0

package/server/realtime/bridge.js

@@ -0,0 +1,837 @@
+/**
+ * Agent Voice Bridge — Routes voice through OpenClaw Gateway
+ *
+ * Triangle architecture:
+ *   Browser Mic → OpenAI Realtime API (ears: VAD + transcription only)
+ *                   ↓ transcript
+ *               Uplink Bridge (this module)
+ *                   ↓ text message
+ *               OpenClaw Gateway (brain)
+ *                   ↓ streaming text response
+ *               TTS Engine (mouth)
+ *                   ↓ PCM audio chunks
+ *               Browser Speaker (via WebSocket)
+ *
+ * Key difference from index.js (standalone relay):
+ *   - OpenAI Realtime opened with `modalities: ['text']` — no audio output from OpenAI
+ *   - Transcription events intercepted → routed to Gateway instead of letting OpenAI respond
+ *   - Gateway response streamed → TTS → PCM audio sent back to client
+ *   - Client plays audio directly via AudioContext (does NOT inject back into OpenAI)
+ *
+ * Custom events sent to client:
+ *   { type: 'bridge.transcript', text }         — user's transcribed speech
+ *   { type: 'bridge.response.delta', text }     — streaming text chunk from gateway
+ *   { type: 'bridge.audio', audio: base64PCM }  — TTS audio chunk
+ *   { type: 'bridge.response.done' }            — response complete
+ *   { type: 'bridge.status', status }            — 'thinking', 'speaking', etc.
+ *   { type: 'bridge.error', error }             — error message
+ *
+ * Usage (in server.js):
+ *   import { setupAgentVoiceBridge } from './server/realtime/bridge.js';
+ *   setupAgentVoiceBridge(httpServer);
+ */
+
+import { WebSocketServer, WebSocket } from 'ws';
+import { createLogger } from '../logger.js';
+import { loadConfig } from '../runtime-config.js';
+import { ALLOWED_ORIGINS, STREAM_READ_TIMEOUT_MS } from '../config.js';
+import { streamTTS, splitSentences } from './tts-stream.js';
+
+const log = createLogger('voice-bridge');
+
+// ─── Constants ──────────────────────────────────────────────────────────────
+
+const OPENAI_REALTIME_BASE = 'wss://api.openai.com/v1/realtime';
+const DEFAULT_MODEL = 'gpt-4o-mini-realtime-preview';
+const MAX_BRIDGE_CONNECTIONS = 5;
+
+// Gateway streaming constants
+const GATEWAY_FETCH_TIMEOUT_MS = 300_000; // 5 minutes for long responses
+const GATEWAY_STREAM_TIMEOUT_MS = STREAM_READ_TIMEOUT_MS || 300_000;
+
+// Connection bookkeeping
+const activeSessions = new Map(); // sessionId → session state
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+/**
+ * Verify that the WebSocket upgrade origin is allowed.
+ */
+function verifyOrigin(origin) {
+  if (!origin) return true; // Non-browser clients don't send Origin
+  try {
+    const url = new URL(origin);
+    const host = url.hostname;
+    if (host === 'localhost' || host === '127.0.0.1' || host === '::1') return true;
+    if (host.endsWith('.ts.net')) return true;
+    return ALLOWED_ORIGINS.some(allowed => {
+      try { return new URL(allowed).hostname === host; } catch { return false; }
+    });
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Send a JSON event to the client WebSocket if it's open.
+ */
+function sendToClient(ws, event) {
+  if (ws.readyState === WebSocket.OPEN) {
+    try {
+      ws.send(JSON.stringify(event));
+    } catch (err) {
+      log.error('Failed to send to client:', err.message);
+    }
+  }
+}
+
+/**
+ * Build the `session.update` for OpenAI Realtime in bridge mode.
+ * Key difference: modalities=['text'] only — no audio output from OpenAI.
+ * We keep input_audio_format=pcm16 so mic audio flows through for transcription.
+ */
+function buildBridgeSessionUpdate(voice, vadSilenceDurationMs) {
+  const silenceMs = Math.max(200, Math.min(1500, vadSilenceDurationMs || 400));
+  return JSON.stringify({
+    type: 'session.update',
+    session: {
+      modalities: ['text'],
+      instructions: 'Transcribe the user\'s speech accurately. Do not generate responses.',
+      voice: voice || 'marin',
+      input_audio_format: 'pcm16',
+      input_audio_transcription: {
+        model: 'whisper-1',
+      },
+      turn_detection: {
+        type: 'server_vad',
+        threshold: 0.65,
+        prefix_padding_ms: 200,
+        silence_duration_ms: silenceMs,
+      },
+    },
+  });
+}
+
+/**
+ * Build the TTS config from runtime config.
+ * @param {Object} runtimeConfig - The loaded runtime config
+ * @returns {Object} TTS config for streamTTS()
+ */
+function buildTTSConfig(runtimeConfig) {
+  // Use agent voice settings (from Settings → Voice → Agent Voice)
+  const engine = runtimeConfig.agentVoiceTtsEngine || 
+    (runtimeConfig.openaiApiKey ? 'openai' : 'edge');
+
+  // Map agentVoiceTtsVoice to the right engine field
+  const agentVoice = runtimeConfig.agentVoiceTtsVoice || 'echo';
+
+  return {
+    engine,
+    openaiApiKey: runtimeConfig.openaiApiKey,
+    openaiTtsVoice: engine === 'openai' ? agentVoice : (runtimeConfig.openaiTtsVoice || 'nova'),
+    openaiTtsModel: runtimeConfig.openaiTtsModel || 'tts-1',
+    edgeTtsVoice: runtimeConfig.edgeTtsVoice || 'en-US-GuyNeural',
+  };
+}
+
+/**
+ * Stream a message to the OpenClaw gateway and process the SSE response.
+ * Returns the full assembled response text.
+ *
+ * @param {Object} params
+ * @param {string} params.message - User's transcribed message
+ * @param {Object} params.config - Runtime config
+ * @param {string} params.sessionId - Session ID for logging
+ * @param {WebSocket} params.clientWs - Client WebSocket for streaming events
+ * @param {AbortSignal} params.signal - Abort signal for cancellation
+ * @param {Function} [params.onSentence] - Callback fired with each complete sentence during streaming
+ * @returns {Promise<{ response: string, tools: string[] }>}
+ */
+async function streamGatewayResponse({ message, config, sessionId, clientWs, signal, onSentence }) {
+  const gatewayUrl = config.gatewayUrl;
+  const gatewayToken = config.gatewayToken;
+
+  if (!gatewayUrl) {
+    throw new Error('Gateway URL not configured');
+  }
+
+  // Build session key — bridge shares the main session
+  const sessionKey = 'agent:main:main';
+
+  const url = `${gatewayUrl}/v1/chat/completions`;
+  log.info(`[${sessionId}] Gateway request: ${url}, session=${sessionKey}`);
+
+  const response = await fetch(url, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${gatewayToken}`,
+      'x-openclaw-session-key': sessionKey,
+    },
+    body: JSON.stringify({
+      model: config.voiceModel || 'openclaw',
+      user: 'uplink-voice',
+      stream: true,
+      max_tokens: 100,
+      stream_options: { include_usage: true },
+      messages: [
+        {
+          role: 'user',
+          content: `[Voice chat via Agent Voice Bridge — keep response conversational and concise] ${message}`,
+        },
+      ],
+    }),
+    signal,
+  });
+
+  if (!response.ok) {
+    const errText = await response.text().catch(() => 'Unknown error');
+    throw new Error(`Gateway error ${response.status}: ${errText}`);
+  }
+
+  if (!response.body) {
+    throw new Error('Gateway response has no body');
+  }
+
+  // Parse the SSE stream
+  const reader = response.body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = '';
+  let fullResponse = '';
+  let sentenceAccum = '';
+  const detectedTools = [];
+  let sentThinking = false;
+
+  const readWithTimeout = () => {
+    return Promise.race([
+      reader.read(),
+      new Promise((_, reject) =>
+        setTimeout(() => reject(new Error('Gateway stream read timed out')), GATEWAY_STREAM_TIMEOUT_MS)
+      ),
+    ]);
+  };
+
+  try {
+    while (true) {
+      if (signal?.aborted) break;
+
+      const { done, value } = await readWithTimeout();
+      if (done) break;
+
+      buffer += decoder.decode(value, { stream: true });
+      const lines = buffer.split('\n');
+      buffer = lines.pop() || '';
+
+      for (const line of lines) {
+        if (!line.startsWith('data: ')) continue;
+        const data = line.slice(6);
+        if (data === '[DONE]') continue;
+
+        try {
+          const parsed = JSON.parse(data);
+          const delta = parsed.choices?.[0]?.delta;
+
+          // Detect tool calls
+          const toolCalls = delta?.tool_calls;
+          if (toolCalls && toolCalls.length > 0) {
+            const toolName = toolCalls[0]?.function?.name;
+            if (toolName && !detectedTools.includes(toolName)) {
+              detectedTools.push(toolName);
+              if (!sentThinking) {
+                sendToClient(clientWs, { type: 'bridge.status', status: 'thinking' });
+                sentThinking = true;
+              }
+              log.info(`[${sessionId}] Tool detected: ${toolName}`);
+            }
+          }
+
+          // Stream text content
+          if (delta?.content) {
+            fullResponse += delta.content;
+            sendToClient(clientWs, {
+              type: 'bridge.response.delta',
+              text: delta.content,
+            });
+
+            // Real-time clause detection for pipelined TTS
+            // Fire on sentence ends AND clause boundaries for faster first audio
+            if (onSentence) {
+              sentenceAccum = (sentenceAccum || '') + delta.content;
+              // Match sentence-ending punctuation OR clause breaks (after 15+ chars)
+              // Clause breaks: comma, semicolon, colon, dash, newline
+              let match;
+              while (sentenceAccum.length > 0) {
+                // Prefer sentence boundaries
+                match = sentenceAccum.match(/^(.*?[.!?])\s/s);
+                if (!match && sentenceAccum.length >= 20) {
+                  // Fall back to clause boundaries for faster streaming
+                  match = sentenceAccum.match(/^(.{15,}?[,;:\-—\n])\s*/s);
+                }
+                if (match) {
+                  const clause = match[1].trim();
+                  sentenceAccum = sentenceAccum.slice(match[0].length);
+                  if (clause) onSentence(clause);
+                } else {
+                  break;
+                }
+              }
+            }
+          }
+        } catch {
+          // Skip unparseable chunks
+        }
+      }
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  // Flush any remaining text as a final sentence
+  if (onSentence && sentenceAccum.trim()) {
+    onSentence(sentenceAccum.trim());
+  }
+
+  return { response: fullResponse, tools: detectedTools };
+}
+
+/**
+ * Process TTS for a complete response: split into sentences, stream TTS for each,
+ * send audio chunks to client.
+ *
+ * @param {Object} params
+ * @param {string} params.text - Full response text
+ * @param {Object} params.ttsConfig - TTS engine configuration
+ * @param {WebSocket} params.clientWs - Client WebSocket
+ * @param {string} params.sessionId - Session ID for logging
+ * @param {AbortSignal} params.signal - Abort signal
+ */
+async function processResponseTTS({ text, ttsConfig, clientWs, sessionId, signal }) {
+  const sentences = splitSentences(text);
+
+  if (sentences.length === 0) {
+    log.warn(`[${sessionId}] No sentences to synthesize`);
+    return;
+  }
+
+  log.info(`[${sessionId}] TTS: ${sentences.length} sentence(s), engine=${ttsConfig.engine}`);
+  sendToClient(clientWs, { type: 'bridge.status', status: 'speaking' });
+
+  for (const sentence of sentences) {
+    if (signal?.aborted) break;
+    if (clientWs.readyState !== WebSocket.OPEN) break;
+
+    try {
+      for await (const pcmChunk of streamTTS(sentence, ttsConfig)) {
+        if (signal?.aborted) break;
+        if (clientWs.readyState !== WebSocket.OPEN) break;
+
+        // Send PCM chunk as base64 to client
+        sendToClient(clientWs, {
+          type: 'bridge.audio',
+          audio: pcmChunk.toString('base64'),
+        });
+      }
+    } catch (err) {
+      log.error(`[${sessionId}] TTS error for sentence:`, err.message);
+      sendToClient(clientWs, {
+        type: 'bridge.error',
+        error: `TTS failed: ${err.message}`,
+      });
+      // Continue with remaining sentences — client gets text even if audio fails
+    }
+  }
+}
+
+/**
+ * Save a message pair (user + assistant) to the sync file for chat history.
+ * Uses a direct HTTP POST to our own server's message-saving endpoint
+ * or imports the function directly.
+ *
+ * @param {string} userText - User's transcribed message
+ * @param {string} assistantText - Agent's response
+ * @param {string} sessionId - Session ID for logging
+ */
+async function saveToHistory(userText, assistantText, sessionId) {
+  try {
+    // Lazy import to avoid circular dependencies
+    // The saveMessageToSync function writes to the messages-sync.json file
+    const { saveMessageToSync } = await import('../routes.js');
+
+    if (userText) {
+      await saveMessageToSync('user', userText);
+      log.debug(`[${sessionId}] Saved user message to history`);
+    }
+    if (assistantText) {
+      await saveMessageToSync('assistant', assistantText);
+      log.debug(`[${sessionId}] Saved assistant message to history`);
+    }
+  } catch (err) {
+    log.error(`[${sessionId}] Failed to save to history:`, err.message);
+  }
+}
+
+/**
+ * Broadcast sync messages to WebSocket clients for cross-device sync.
+ *
+ * @param {string} role - 'user' or 'assistant'
+ * @param {string} text - Message text
+ */
+async function broadcastSync(role, text) {
+  try {
+    const { broadcastSyncMessage, generateMessageId } = await import('../websocket/index.js');
+    const msgId = generateMessageId();
+    broadcastSyncMessage(role, text, 'main', msgId);
+  } catch (err) {
+    log.warn('Failed to broadcast sync message:', err.message);
+  }
+}
+
+// ─── Main Bridge Logic ─────────────────────────────────────────────────────
+
+/**
+ * Handle a single bridge session: client WS + OpenAI Realtime WS + Gateway.
+ *
+ * @param {WebSocket} clientWs - Client WebSocket connection
+ * @param {import('http').IncomingMessage} req - HTTP upgrade request
+ */
+async function handleBridgeConnection(clientWs, req) {
+  const sessionId = `bridge-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  const clientIp = req.socket.remoteAddress;
+  log.info(`Bridge client connected: ${sessionId} from ${clientIp}`);
+
+  // ── Load runtime config ─────────────────────────────────────────────
+  let config;
+  try {
+    config = await loadConfig();
+  } catch (err) {
+    log.error('Failed to load runtime config:', err);
+    clientWs.close(1011, 'Server configuration error');
+    return;
+  }
+
+  const apiKey = config.openaiApiKey;
+  if (!apiKey) {
+    log.error('No OpenAI API key — cannot open realtime session');
+    sendToClient(clientWs, {
+      type: 'bridge.error',
+      error: 'No OpenAI API key configured. Add your key in Settings → Voice.',
+    });
+    clientWs.close(1008, 'Missing API key');
+    return;
+  }
+
+  if (!config.gatewayUrl || !config.gatewayToken) {
+    log.error('Gateway not configured — cannot use agent voice bridge');
+    sendToClient(clientWs, {
+      type: 'bridge.error',
+      error: 'OpenClaw Gateway not configured. Check Settings.',
+    });
+    clientWs.close(1008, 'Gateway not configured');
+    return;
+  }
+
+  const model = config.realtimeModel || DEFAULT_MODEL;
+  const voice = config.realtimeVoice || 'marin';
+  const vadSilenceDurationMs = config.vadSilenceDurationMs || 400;
+  const ttsConfig = buildTTSConfig(config);
+
+  // ── Open OpenAI Realtime WebSocket ──────────────────────────────────
+  const openaiUrl = `${OPENAI_REALTIME_BASE}?model=${encodeURIComponent(model)}`;
+  let openaiWs;
+  try {
+    openaiWs = new WebSocket(openaiUrl, {
+      headers: {
+        'Authorization': `Bearer ${apiKey}`,
+        'OpenAI-Beta': 'realtime=v1',
+      },
+      perMessageDeflate: true,
+    });
+  } catch (err) {
+    log.error('Failed to create OpenAI WebSocket:', err);
+    clientWs.close(1011, 'Failed to connect to OpenAI');
+    return;
+  }
+
+  // ── Session state ───────────────────────────────────────────────────
+  let clientClosed = false;
+  let openaiClosed = false;
+  let processingResponse = false;
+  let activeAbort = null; // AbortController for current gateway request
+
+  activeSessions.set(sessionId, { clientWs, openaiWs });
+
+  // Keepalive
+  const keepalive = setInterval(() => {
+    try {
+      if (clientWs.readyState === WebSocket.OPEN) clientWs.ping();
+      if (openaiWs.readyState === WebSocket.OPEN) openaiWs.ping();
+    } catch { /* swallow */ }
+  }, 15_000);
+
+  function cleanup() {
+    clearInterval(keepalive);
+    activeSessions.delete(sessionId);
+    if (activeAbort) {
+      activeAbort.abort();
+      activeAbort = null;
+    }
+    if (!clientClosed && clientWs.readyState === WebSocket.OPEN) {
+      clientWs.close();
+    }
+    if (!openaiClosed && openaiWs.readyState === WebSocket.OPEN) {
+      openaiWs.close();
+    }
+  }
+
+  // ── Process a user transcript through the gateway ───────────────────
+  
+  const MIN_BARGE_IN_WORDS = 3; // Don't barge-in for short/ambiguous transcripts
+  
+  async function processTranscript(transcript) {
+    if (!transcript || !transcript.trim()) return;
+
+    const trimmed = transcript.trim();
+    const wordCount = trimmed.split(/\s+/).filter(w => w.length > 0).length;
+    
+    log.info(`[${sessionId}] Transcript: "${trimmed.substring(0, 200)}" (${wordCount} words)`);
+
+    // If a response is in progress, only barge-in for substantial speech
+    // Short fragments (1-2 words) during a response are likely noise/echo
+    if (activeAbort) {
+      if (wordCount < MIN_BARGE_IN_WORDS) {
+        log.info(`[${sessionId}] Skipping short transcript during active response: "${trimmed}"`);
+        return;
+      }
+      log.info(`[${sessionId}] Barge-in: aborting previous response`);
+      activeAbort.abort();
+    }
+
+    processingResponse = true;
+    activeAbort = new AbortController();
+    const { signal } = activeAbort;
+
+    // Send transcript to client
+    sendToClient(clientWs, { type: 'bridge.transcript', text: trimmed });
+
+    // Broadcast user message for cross-device sync
+    broadcastSync('user', trimmed);
+
+    try {
+      // Send to gateway and stream response with pipelined TTS
+      sendToClient(clientWs, { type: 'bridge.status', status: 'thinking' });
+
+      // Queue for sentences that arrive during streaming
+      const sentenceQueue = [];
+      let ttsRunning = false;
+      let streamDone = false;
+      let sentenceCount = 0;
+
+      // TTS consumer — processes sentences from queue as they arrive
+      const processTTSQueue = async () => {
+        if (ttsRunning) return;
+        ttsRunning = true;
+        while (sentenceQueue.length > 0) {
+          if (signal?.aborted) break;
+          if (clientWs.readyState !== WebSocket.OPEN) break;
+
+          const sentence = sentenceQueue.shift();
+          const clean = sentence.replace(/MEDIA:.+?(?:\n|$)/g, '').trim();
+          if (!clean) continue;
+
+          sentenceCount++;
+          if (sentenceCount === 1) {
+            sendToClient(clientWs, { type: 'bridge.status', status: 'speaking' });
+          }
+
+          try {
+            for await (const pcmChunk of streamTTS(clean, ttsConfig)) {
+              if (signal?.aborted) break;
+              if (clientWs.readyState !== WebSocket.OPEN) break;
+              sendToClient(clientWs, {
+                type: 'bridge.audio',
+                audio: pcmChunk.toString('base64'),
+              });
+            }
+          } catch (err) {
+            log.error(`[${sessionId}] TTS error:`, err.message);
+          }
+        }
+        ttsRunning = false;
+      };
+
+      // Stream gateway with sentence callback — TTS starts on first sentence
+      const { response: gatewayResponse } = await streamGatewayResponse({
+        message: trimmed,
+        config,
+        sessionId,
+        clientWs,
+        signal,
+        onSentence: (sentence) => {
+          sentenceQueue.push(sentence);
+          processTTSQueue(); // kick TTS if not already running
+        },
+      });
+
+      if (signal.aborted) {
+        log.info(`[${sessionId}] Response aborted (barge-in)`);
+        return;
+      }
+
+      // Wait for TTS queue to drain
+      while (sentenceQueue.length > 0 || ttsRunning) {
+        if (signal?.aborted) break;
+        await new Promise(r => setTimeout(r, 50));
+      }
+
+      const cleanResponse = (gatewayResponse || '').replace(/MEDIA:.+?(?:\n|$)/g, '').trim();
+      log.info(`[${sessionId}] Complete: ${cleanResponse.length} chars, ${sentenceCount} sentences TTS'd`);
+
+      // Signal response complete
+      sendToClient(clientWs, { type: 'bridge.response.done' });
+
+      // Save to chat history (async, don't block)
+      if (cleanResponse) {
+        saveToHistory(trimmed, cleanResponse, sessionId);
+        broadcastSync('assistant', cleanResponse);
+      }
+
+    } catch (err) {
+      if (err.name === 'AbortError' || signal.aborted) {
+        log.info(`[${sessionId}] Response processing aborted`);
+      } else {
+        log.error(`[${sessionId}] Error processing transcript:`, err.message);
+        sendToClient(clientWs, {
+          type: 'bridge.error',
+          error: `Processing failed: ${err.message}`,
+        });
+        sendToClient(clientWs, { type: 'bridge.response.done' });
+      }
+    } finally {
+      processingResponse = false;
+      if (activeAbort?.signal === signal) {
+        activeAbort = null;
+      }
+    }
+  }
+
+  // ── OpenAI Realtime WebSocket events ────────────────────────────────
+
+  openaiWs.on('open', () => {
+    log.info(`[${sessionId}] OpenAI Realtime connected (model=${model})`);
+
+    // Send bridge-mode session update (text-only modalities, no audio output)
+    try {
+      const sessionUpdate = buildBridgeSessionUpdate(voice, vadSilenceDurationMs);
+      log.debug(`[${sessionId}] Sending bridge session.update (VAD silence: ${vadSilenceDurationMs}ms)`);
+      openaiWs.send(sessionUpdate);
+    } catch (err) {
+      log.error(`[${sessionId}] Failed to send session.update:`, err);
+    }
+
+    // Notify client that bridge is ready
+    sendToClient(clientWs, { type: 'relay.ready' });
+  });
+
+  openaiWs.on('message', (data, isBinary) => {
+    if (isBinary) {
+      // In bridge mode we shouldn't get binary audio from OpenAI (modalities=['text']),
+      // but if we do, just drop it
+      return;
+    }
+
+    try {
+      const event = JSON.parse(data.toString());
+      const eventType = event.type;
+
+      // ── Intercept transcription events ──────────────────────────────
+      if (eventType === 'conversation.item.input_audio_transcription.completed') {
+        // This is the main event — user speech has been transcribed
+        const transcript = event.transcript;
+        log.info(`[${sessionId}] OpenAI transcription completed: "${(transcript || '').substring(0, 100)}"`);
+        // Process async — don't block the WebSocket message handler
+        processTranscript(transcript);
+        // Do NOT forward to client — we handle it ourselves
+        return;
+      }
+
+      // ── Forward select events to client (for UI state) ─────────────
+      // These events are useful for the client's VAD/speech indicators
+      switch (eventType) {
+        case 'session.created':
+        case 'session.updated':
+          // Let client know session is configured
+          sendToClient(clientWs, event);
+          break;
+
+        case 'input_audio_buffer.speech_started':
+          // User started speaking — useful for UI animation
+          sendToClient(clientWs, event);
+          // If agent is speaking, this is a barge-in
+          if (processingResponse && activeAbort) {
+            log.info(`[${sessionId}] Barge-in detected: user started speaking`);
+            activeAbort.abort();
+          }
+          break;
+
+        case 'input_audio_buffer.speech_stopped':
+        case 'input_audio_buffer.committed':
+          // User stopped speaking — forward for UI
+          sendToClient(clientWs, event);
+          break;
+
+        case 'conversation.item.input_audio_transcription.failed':
+          // Transcription failed
+          log.warn(`[${sessionId}] Transcription failed:`, JSON.stringify(event.error));
+          sendToClient(clientWs, {
+            type: 'bridge.error',
+            error: 'Speech transcription failed — please try again',
+          });
+          break;
+
+        case 'error':
+          // OpenAI error — forward to client
+          log.error(`[${sessionId}] OpenAI error:`, JSON.stringify(event));
+          sendToClient(clientWs, {
+            type: 'bridge.error',
+            error: event.error?.message || 'OpenAI Realtime error',
+          });
+          break;
+
+        case 'rate_limits.updated':
+          // Log rate limits but don't forward
+          log.debug(`[${sessionId}] Rate limits: ${JSON.stringify(event.rate_limits)}`);
+          break;
+
+        default:
+          // Drop all other events (response.*, conversation.item.created, etc.)
+          // In bridge mode, OpenAI shouldn't be generating responses
+          log.debug(`[${sessionId}] Dropping OpenAI event: ${eventType}`);
+          break;
+      }
+    } catch (err) {
+      log.error(`[${sessionId}] Error parsing OpenAI message:`, err.message);
+    }
+  });
+
+  openaiWs.on('close', (code, reason) => {
+    openaiClosed = true;
+    const reasonStr = reason?.toString() || '';
+    log.info(`[${sessionId}] OpenAI disconnected (code=${code}${reasonStr ? ', reason=' + reasonStr : ''})`);
+    sendToClient(clientWs, {
+      type: 'bridge.error',
+      error: 'Voice connection lost — please reconnect',
+    });
+    cleanup();
+  });
+
+  openaiWs.on('error', (err) => {
+    log.error(`[${sessionId}] OpenAI error:`, err.message);
+    sendToClient(clientWs, {
+      type: 'bridge.error',
+      error: `Voice connection error: ${err.message}`,
+    });
+    cleanup();
+  });
+
+  // ── Client WebSocket events ─────────────────────────────────────────
+
+  clientWs.on('message', (data, isBinary) => {
+    // Forward everything from client to OpenAI (mic audio, session updates, etc.)
+    if (openaiWs.readyState === WebSocket.OPEN) {
+      try {
+        if (isBinary) {
+          // Binary = mic audio — forward directly
+          openaiWs.send(data);
+        } else {
+          // Text = JSON commands from client
+          const msgStr = data.toString();
+          try {
+            const msg = JSON.parse(msgStr);
+            log.debug(`[${sessionId}] Client event: ${msg.type}`);
+
+            // Intercept bridge-specific commands
+            if (msg.type === 'bridge.cancel') {
+              // Client wants to cancel current response
+              if (activeAbort) {
+                log.info(`[${sessionId}] Client cancelled response`);
+                activeAbort.abort();
+              }
+              return;
+            }
+          } catch { /* not JSON, just forward */ }
+
+          openaiWs.send(msgStr);
+        }
+      } catch (err) {
+        log.error(`[${sessionId}] Error forwarding client → OpenAI:`, err.message);
+      }
+    }
+  });
+
+  clientWs.on('close', (code) => {
+    clientClosed = true;
+    log.info(`[${sessionId}] Client disconnected (code=${code})`);
+    cleanup();
+  });
+
+  clientWs.on('error', (err) => {
+    log.error(`[${sessionId}] Client error:`, err.message);
+    cleanup();
+  });
+}
+
+// ─── Public API ─────────────────────────────────────────────────────────────
+
+/**
+ * Set up the Agent Voice Bridge WebSocket endpoint on the given HTTP server.
+ *
+ * Listens for upgrade requests on `/api/realtime/bridge` using noServer mode
+ * (same pattern as setupRealtimeRelay in index.js) so it coexists with other
+ * WebSocket servers.
+ *
+ * @param {import('http').Server} server - The HTTP server instance
+ * @returns {WebSocketServer} The WebSocket server instance
+ */
+export function setupAgentVoiceBridge(server) {
+  const wss = new WebSocketServer({ noServer: true, perMessageDeflate: false });
+
+  // ── Handle upgrade on /api/realtime/bridge ──────────────────────────
+  server.on('upgrade', (request, socket, head) => {
+    const { pathname } = new URL(request.url, `http://${request.headers.host}`);
+
+    // Handle /api/realtime?mode=agent OR /api/realtime/bridge
+    const params = new URL(request.url, `http://${request.headers.host}`).searchParams;
+    const isAgentMode = pathname === '/api/realtime' && params.get('mode') === 'agent';
+    const isBridgePath = pathname === '/api/realtime/bridge';
+    if (!isAgentMode && !isBridgePath) return; // Let other handlers take it
+
+    // Origin check
+    const origin = request.headers.origin;
+    if (!verifyOrigin(origin)) {
+      log.warn(`Rejected upgrade from invalid origin: ${origin}`);
+      socket.write('HTTP/1.1 403 Forbidden\r\n\r\n');
+      socket.destroy();
+      return;
+    }
+
+    // Connection limit
+    if (activeSessions.size >= MAX_BRIDGE_CONNECTIONS) {
+      log.warn(`Rejected upgrade — at capacity (${activeSessions.size}/${MAX_BRIDGE_CONNECTIONS})`);
+      socket.write('HTTP/1.1 429 Too Many Requests\r\n\r\n');
+      socket.destroy();
+      return;
+    }
+
+    wss.handleUpgrade(request, socket, head, (ws) => {
+      wss.emit('connection', ws, request);
+    });
+  });
+
+  // ── New client connection ───────────────────────────────────────────
+  wss.on('connection', (clientWs, req) => {
+    handleBridgeConnection(clientWs, req);
+  });
+
+  log.info('Agent Voice Bridge ready at /api/realtime?mode=agent');
+  return wss;
+}