@cohortapp/agent-sdk 2.3.0

package/scripts/cloud-relay/voice/server.mjs

@@ -0,0 +1,532 @@
+/**
+ * Maestro — OpenAI Realtime Voice Cloud Relay (Railway)
+ *
+ * Public HTTPS service deployed on Railway that bridges OpenAI's
+ * `realtime.call.incoming` webhook into a Mac-mini-resident maestro
+ * daemon over a buffer-and-poll channel.
+ *
+ * Why this exists: cloudflared "trycloudflare.com" URLs are ephemeral
+ * (rotate on every restart). For a production OpenAI webhook config
+ * we need a stable URL that never changes.
+ *
+ * Architecture:
+ *   1. OpenAI POSTs realtime.call.incoming → this service
+ *   2. Service verifies signature (HMAC-SHA256 against OPENAI_WEBHOOK_SECRET)
+ *   3. Service buffers the event in memory (TTL 10 min, dedup by call_id)
+ *   4. Service returns 200 immediately so OpenAI's call setup proceeds
+ *   5. Local Mac polls /openai/realtime/calls every 2-5s, sees new
+ *      events, calls OpenAI's /v1/realtime/calls/<id>/accept directly
+ *      from the Mac, then opens the WebSocket from the Mac. Tool
+ *      bridge / RAG / context-loading all happen on the Mac with
+ *      zero round-trips through Railway.
+ *
+ * Endpoints:
+ *   POST /openai/realtime/incoming  — OpenAI webhook receiver. Verifies
+ *                                      signature, buffers event, ack 200.
+ *   GET  /openai/realtime/calls     — Drain buffer (auth-required: bearer
+ *                                      VOICE_RELAY_POLL_TOKEN).
+ *   POST /openai/realtime/test      — Echo back; for connectivity probes.
+ *   GET  /health                    — Liveness probe.
+ *
+ * Env (set via Railway dashboard or `railway variables`):
+ *   PORT                       — provided by Railway
+ *   OPENAI_WEBHOOK_SECRET      — copy from OpenAI dashboard "signing secret"
+ *   VOICE_RELAY_POLL_TOKEN     — random hex; the Mac uses this as bearer
+ *   BUFFER_TTL_MS              — optional, default 600000 (10 min)
+ */
+
+import http from "node:http";
+import crypto from "node:crypto";
+import { WebSocketServer, WebSocket } from "ws";
+
+const PORT = parseInt(process.env.PORT || "8080", 10);
+const OPENAI_WEBHOOK_SECRET = process.env.OPENAI_WEBHOOK_SECRET || "";
+const POLL_TOKEN = process.env.VOICE_RELAY_POLL_TOKEN || "";
+const OPENAI_API_KEY = process.env.OPENAI_API_KEY || "";
+const VOICE_TWIML_TOKEN = process.env.VOICE_TWIML_TOKEN || ""; // bearer for /twilio/twiml endpoints
+const BUFFER_TTL_MS = parseInt(process.env.BUFFER_TTL_MS || "600000", 10);
+const MAX_BUFFER_SIZE = parseInt(process.env.MAX_BUFFER_SIZE || "500", 10);
+
+if (!OPENAI_WEBHOOK_SECRET) console.warn("[warn] OPENAI_WEBHOOK_SECRET not set — webhook signature verification disabled");
+if (!POLL_TOKEN) console.warn("[warn] VOICE_RELAY_POLL_TOKEN not set — poll endpoint is unauthenticated");
+if (!OPENAI_API_KEY) console.warn("[warn] OPENAI_API_KEY not set — outbound Media Streams bridge disabled");
+
+// ── In-memory buffer ───────────────────────────────────────────────────────
+
+class EventBuffer {
+  constructor(name) { this.name = name; this.events = []; this.seen = new Map(); }
+  push(event) {
+    const id = event.id;
+    if (id && this.seen.has(id)) return false;
+    if (id) this.seen.set(id, Date.now());
+    this.events.push(event);
+    this.gc();
+    return true;
+  }
+  drain() { return this.events.splice(0, this.events.length); }
+  gc() {
+    const now = Date.now();
+    while (this.events.length && now - new Date(this.events[0].received_at).getTime() > BUFFER_TTL_MS) {
+      this.events.shift();
+    }
+    while (this.events.length > MAX_BUFFER_SIZE) this.events.shift();
+    for (const [id, ts] of this.seen) if (now - ts > BUFFER_TTL_MS) this.seen.delete(id);
+  }
+}
+const buffer = new EventBuffer("openai_voice");
+setInterval(() => buffer.gc(), 60000).unref();
+
+// ── HTTP helpers ───────────────────────────────────────────────────────────
+
+function readBody(req) {
+  return new Promise((resolve, reject) => {
+    let data = "";
+    req.setEncoding("utf-8");
+    req.on("data", (c) => { data += c; });
+    req.on("end", () => resolve(data));
+    req.on("error", reject);
+  });
+}
+function json(res, status, payload) {
+  res.writeHead(status, { "content-type": "application/json" });
+  res.end(JSON.stringify(payload));
+}
+function text(res, status, body) {
+  res.writeHead(status, { "content-type": "text/plain" });
+  res.end(body);
+}
+
+// ── OpenAI webhook signature verification ──────────────────────────────────
+// OpenAI sends `openai-signature: <hex>` (or sometimes `webhook-signature:
+// t=<ts>,v1=<hex>` Stripe-style). We accept either.
+
+function verifyOpenAISignature(rawBody, headers) {
+  if (!OPENAI_WEBHOOK_SECRET) return true; // dev mode
+  const headerSig = headers["openai-signature"] || headers["webhook-signature"] || "";
+  if (!headerSig) return false;
+  let providedHex = headerSig;
+  if (headerSig.includes(",")) {
+    const parts = Object.fromEntries(
+      headerSig.split(",").map((kv) => kv.split("=").map((s) => s.trim()))
+    );
+    providedHex = parts.v1 || parts.v0 || "";
+  }
+  const expected = crypto.createHmac("sha256", OPENAI_WEBHOOK_SECRET).update(rawBody || "").digest("hex");
+  if (providedHex.length !== expected.length) return false;
+  try {
+    return crypto.timingSafeEqual(Buffer.from(providedHex), Buffer.from(expected));
+  } catch {
+    return false;
+  }
+}
+
+function checkPollAuth(req, res) {
+  if (!POLL_TOKEN) return true;
+  const auth = req.headers["authorization"] || "";
+  if (auth === `Bearer ${POLL_TOKEN}`) return true;
+  text(res, 401, "unauthorized");
+  return false;
+}
+
+// ── Handlers ───────────────────────────────────────────────────────────────
+
+async function handleIncoming(req, res) {
+  const rawBody = await readBody(req);
+  if (!verifyOpenAISignature(rawBody, req.headers)) {
+    console.warn("[openai-voice] signature mismatch");
+    return json(res, 403, { ok: false, error: "openai_signature_invalid" });
+  }
+  let parsed;
+  try { parsed = JSON.parse(rawBody); }
+  catch { return json(res, 400, { ok: false, error: "invalid_json" }); }
+
+  const callId = parsed.data?.call_id || parsed.call_id || `nocallid-${Date.now()}`;
+  const eventType = parsed.type || "unknown";
+
+  // Critical-path: OpenAI gives us roughly 5 seconds to POST /accept on a
+  // realtime.call.incoming webhook. The Mac-mini poller runs at ~750ms
+  // intervals, plus there's a ~50–200ms hop home, plus accept latency —
+  // worst-case we miss the window. So accept FROM RAILWAY immediately;
+  // the local handler then just attaches the event WebSocket whenever
+  // its next poll picks the event up. Audio is bridged inside OpenAI, so
+  // the local WS attach can lag by hundreds of ms without harm.
+  let acceptedByRelay = false;
+  let acceptError = null;
+  if (eventType === "realtime.call.incoming" && OPENAI_API_KEY && callId && !callId.startsWith("nocallid-")) {
+    try {
+      const acceptResp = await fetch(`https://api.openai.com/v1/realtime/calls/${encodeURIComponent(callId)}/accept`, {
+        method: "POST",
+        headers: { Authorization: `Bearer ${OPENAI_API_KEY}` },
+      });
+      if (acceptResp.ok) {
+        acceptedByRelay = true;
+        console.log(`[openai-voice] relay-accepted call=${callId} (status=${acceptResp.status})`);
+      } else {
+        acceptError = `accept ${acceptResp.status}: ${(await acceptResp.text()).slice(0, 200)}`;
+        console.warn(`[openai-voice] relay-accept failed call=${callId}: ${acceptError}`);
+      }
+    } catch (err) {
+      acceptError = err.message;
+      console.warn(`[openai-voice] relay-accept threw call=${callId}: ${err.message}`);
+    }
+  }
+
+  const accepted = buffer.push({
+    id: `${callId}-${eventType}`,
+    received_at: new Date().toISOString(),
+    event_type: eventType,
+    call_id: callId,
+    accepted_by_relay: acceptedByRelay,
+    accept_error: acceptError,
+    // Store the parsed object for human inspection AND the raw body
+    // bytes so the local poller can re-verify HMAC against what OpenAI
+    // actually signed (re-stringifying parsed loses key-order/whitespace
+    // and breaks signature validation downstream).
+    payload: parsed,
+    raw_body: rawBody,
+  });
+  console.log(`[openai-voice] ${accepted ? "buffered" : "dedup"} ${eventType} call=${callId} accepted_by_relay=${acceptedByRelay}`);
+  return json(res, 200, { ok: true, dedup: !accepted, call_id: callId, accepted_by_relay: acceptedByRelay });
+}
+
+function handlePoll(req, res) {
+  if (!checkPollAuth(req, res)) return;
+  const events = buffer.drain();
+  return json(res, 200, {
+    ok: true,
+    count: events.length,
+    events,
+    server_time: new Date().toISOString(),
+  });
+}
+
+function handleTest(req, res) {
+  return json(res, 200, {
+    ok: true,
+    service: "openai-voice-cloud-relay",
+    signature_required: !!OPENAI_WEBHOOK_SECRET,
+    poll_auth_required: !!POLL_TOKEN,
+    server_time: new Date().toISOString(),
+  });
+}
+
+function handleHealth(req, res) {
+  return json(res, 200, {
+    ok: true,
+    service: "openai-voice-cloud-relay",
+    uptime_sec: Math.round(process.uptime()),
+    buffer: { events: buffer.events.length, seen: buffer.seen.size },
+    bridges: { active: activeBridges.size },
+    config: {
+      signature_required: !!OPENAI_WEBHOOK_SECRET,
+      poll_auth_required: !!POLL_TOKEN,
+      twiml_auth_required: !!VOICE_TWIML_TOKEN,
+      openai_key_loaded: !!OPENAI_API_KEY,
+      media_streams_enabled: !!OPENAI_API_KEY,
+      buffer_ttl_ms: BUFFER_TTL_MS,
+    },
+    server_time: new Date().toISOString(),
+  });
+}
+
+// ── Outbound bridge state ──────────────────────────────────────────────────
+// When a call is queued via /twilio/queue-outbound, we store its brief +
+// session config keyed by callSid (Twilio's identifier) so the WebSocket
+// handler can fetch it when Twilio connects with that callSid.
+const queuedOutbound = new Map();   // callSid → { brief, voice, instructions, identity }
+const QUEUED_TTL_MS = 5 * 60_000;
+function gcQueued() {
+  const now = Date.now();
+  for (const [k, v] of queuedOutbound) {
+    if (now - v.queued_at > QUEUED_TTL_MS) queuedOutbound.delete(k);
+  }
+}
+setInterval(gcQueued, 60_000).unref();
+
+// Active bridges (Twilio WS ↔ OpenAI WS), keyed by Twilio streamSid.
+const activeBridges = new Map();
+
+// ── Twilio outbound endpoints ──────────────────────────────────────────────
+
+function checkTwimlAuth(req, res, url) {
+  // Bearer token via Authorization header OR ?token= query param (Twilio
+  // may not set Authorization on TwiML callbacks, so query param is the
+  // fallback).
+  if (!VOICE_TWIML_TOKEN) return true;
+  const tokenQ = url.searchParams.get("token");
+  const auth = req.headers["authorization"] || "";
+  if (auth === `Bearer ${VOICE_TWIML_TOKEN}`) return true;
+  if (tokenQ && tokenQ === VOICE_TWIML_TOKEN) return true;
+  text(res, 401, "unauthorized");
+  return false;
+}
+
+/**
+ * POST /twilio/queue-outbound
+ * Body: { call_sid, brief, voice?, identity?, session?, peer?, peer_name? }
+ *
+ * Called by the LOCAL agent right before / after it tells Twilio to dial
+ * the user's PSTN number. Stores the session config keyed by the Twilio
+ * call_sid so when Twilio connects to /twilio/stream with that streamSid,
+ * we know what to inject as session instructions.
+ */
+async function handleQueueOutbound(req, res, url) {
+  if (!checkTwimlAuth(req, res, url)) return;
+  const rawBody = await readBody(req);
+  let parsed;
+  try { parsed = JSON.parse(rawBody); }
+  catch { return json(res, 400, { ok: false, error: "invalid_json" }); }
+
+  const callSid = parsed.call_sid;
+  if (!callSid) return json(res, 400, { ok: false, error: "missing call_sid" });
+  queuedOutbound.set(callSid, { ...parsed, queued_at: Date.now() });
+  console.log(`[twilio-outbound] queued config for call_sid=${callSid} (${parsed.peer_name || parsed.peer || "?"})`);
+  return json(res, 200, { ok: true, call_sid: callSid });
+}
+
+/**
+ * GET/POST /twilio/twiml
+ * Returns TwiML that tells Twilio to <Connect><Stream> to our WSS endpoint.
+ * Called by Twilio when an outbound call is answered (we set this URL as
+ * the call's TwiML when invoking the Twilio Voice API).
+ */
+async function handleTwiml(req, res, url) {
+  if (!checkTwimlAuth(req, res, url)) return;
+  const host = req.headers["x-forwarded-host"] || req.headers.host;
+  const proto = (req.headers["x-forwarded-proto"] || "https") === "http" ? "ws" : "wss";
+  const streamUrl = `${proto}://${host}/twilio/stream${VOICE_TWIML_TOKEN ? `?token=${encodeURIComponent(VOICE_TWIML_TOKEN)}` : ""}`;
+  const twiml = `<?xml version="1.0" encoding="UTF-8"?>
+<Response>
+  <Connect>
+    <Stream url="${streamUrl.replace(/&/g, "&amp;")}" />
+  </Connect>
+</Response>`;
+  console.log(`[twilio-twiml] served TwiML (req from ${req.socket.remoteAddress}, ua=${(req.headers["user-agent"] || "?").slice(0, 60)})`);
+  res.writeHead(200, { "content-type": "text/xml" });
+  res.end(twiml);
+}
+
+// ── Server ────────────────────────────────────────────────────────────────
+
+const server = http.createServer(async (req, res) => {
+  try {
+    const url = new URL(req.url, `http://${req.headers.host || "localhost"}`);
+    const path = url.pathname;
+    if (req.method === "POST" && path === "/openai/realtime/incoming") return handleIncoming(req, res);
+    if (req.method === "GET"  && path === "/openai/realtime/calls")    return handlePoll(req, res);
+    if (req.method === "POST" && path === "/openai/realtime/test")     return handleTest(req, res);
+    if (req.method === "POST" && path === "/twilio/queue-outbound")    return handleQueueOutbound(req, res, url);
+    if ((req.method === "GET" || req.method === "POST") && path === "/twilio/twiml") return handleTwiml(req, res, url);
+    if (req.method === "GET"  && (path === "/health" || path === "/")) return handleHealth(req, res);
+    return text(res, 404, "not found");
+  } catch (err) {
+    console.error("[error]", err);
+    return text(res, 500, "internal error");
+  }
+});
+
+// ── WebSocket: Twilio Media Streams ⇄ OpenAI Realtime ──────────────────────
+// Twilio opens a WSS connection to /twilio/stream. We:
+//   1. Wait for Twilio's `start` event (gives us the streamSid + callSid)
+//   2. Look up the queued session config for that callSid
+//   3. Open a WSS to OpenAI's realtime endpoint
+//   4. session.update with input/output audio = g711_ulaw + the queued
+//      instructions / tools
+//   5. Bridge audio frames bidirectionally (no transcoding — both sides
+//      speak g711_ulaw natively)
+
+const wss = new WebSocketServer({ noServer: true });
+
+server.on("upgrade", (req, socket, head) => {
+  const url = new URL(req.url, `http://${req.headers.host || "localhost"}`);
+  if (url.pathname !== "/twilio/stream") {
+    socket.destroy();
+    return;
+  }
+  // Twilio Media Streams strips query params on WS upgrade, so we can't
+  // require ?token=. The endpoint is rate-limited by OpenAI's per-minute
+  // billing for any connection that does land here, and we log
+  // remoteAddress + user-agent so abuse is detectable. The TwiML
+  // endpoint that publishes the stream URL still requires
+  // VOICE_TWIML_TOKEN, so a malicious caller would need to know
+  // the WSS URL out-of-band to abuse this endpoint.
+  console.log(`[bridge] WS upgrade from ${req.socket.remoteAddress} ua=${(req.headers["user-agent"] || "?").slice(0, 60)} path=${url.pathname} qs=${url.search}`);
+  wss.handleUpgrade(req, socket, head, (ws) => {
+    handleTwilioStream(ws, req);
+  });
+});
+
+function handleTwilioStream(twilioWs, req) {
+  if (!OPENAI_API_KEY) {
+    console.warn("[bridge] cannot bridge — OPENAI_API_KEY not set");
+    twilioWs.close(1011, "openai key missing");
+    return;
+  }
+
+  let openaiWs = null;
+  let streamSid = null;
+  let callSid = null;
+  let openaiReady = false;
+  let bufferedFrames = [];   // Twilio frames received before OpenAI is ready
+  let responseInProgress = false; // tracks whether a response.create is mid-flight
+
+  console.log(`[bridge] Twilio Media Stream connected from ${req.socket.remoteAddress}`);
+
+  const cleanup = (reason) => {
+    if (openaiWs && openaiWs.readyState !== WebSocket.CLOSED) {
+      try { openaiWs.close(1000, reason); } catch {}
+    }
+    if (twilioWs && twilioWs.readyState !== WebSocket.CLOSED) {
+      try { twilioWs.close(1000, reason); } catch {}
+    }
+    if (streamSid) activeBridges.delete(streamSid);
+    console.log(`[bridge] closed (${reason}) stream=${streamSid || "?"} call=${callSid || "?"}`);
+  };
+
+  twilioWs.on("error", (err) => {
+    console.error(`[bridge] twilio ws error: ${err.message}`);
+  });
+
+  twilioWs.on("close", () => cleanup("twilio_closed"));
+
+  twilioWs.on("message", (raw) => {
+    let evt;
+    try { evt = JSON.parse(raw.toString("utf-8")); }
+    catch { return; }
+
+    if (evt.event === "start") {
+      streamSid = evt.start?.streamSid;
+      callSid = evt.start?.callSid;
+      activeBridges.set(streamSid, { callSid, started: Date.now() });
+      console.log(`[bridge] start stream=${streamSid} call=${callSid}`);
+      const queued = queuedOutbound.get(callSid) || {};
+      openOpenAIWs(queued);
+    } else if (evt.event === "media") {
+      // Twilio sends 20ms of µ-law @ 8kHz, base64-encoded.
+      // OpenAI accepts the same bytes when session.audio.input.format = g711_ulaw.
+      const payload = evt.media?.payload;
+      if (!payload) return;
+      if (openaiReady) {
+        openaiWs.send(JSON.stringify({ type: "input_audio_buffer.append", audio: payload }));
+      } else {
+        bufferedFrames.push(payload);
+        if (bufferedFrames.length > 1500) bufferedFrames.shift(); // ~30s cap
+      }
+    } else if (evt.event === "stop") {
+      console.log(`[bridge] twilio stop stream=${streamSid}`);
+      // Don't close OpenAI yet — let the model finish current response if any
+      setTimeout(() => cleanup("twilio_stop"), 1500);
+    } else if (evt.event === "mark") {
+      // Twilio playback marker (we don't use these but log for debugging)
+    }
+  });
+
+  function openOpenAIWs(queued) {
+    const model = queued.model || "gpt-realtime-2";
+    const url = `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(model)}`;
+    openaiWs = new WebSocket(url, {
+      headers: { Authorization: `Bearer ${OPENAI_API_KEY}` },
+    });
+
+    openaiWs.on("open", () => {
+      console.log(`[bridge] openai ws open model=${model}`);
+      // Send session config — use g711_ulaw on both sides so we don't transcode.
+      const sessionUpdate = buildSessionUpdate(queued);
+      openaiWs.send(JSON.stringify(sessionUpdate));
+      openaiReady = true;
+      // Drain any frames Twilio sent before OpenAI was ready
+      for (const payload of bufferedFrames) {
+        openaiWs.send(JSON.stringify({ type: "input_audio_buffer.append", audio: payload }));
+      }
+      bufferedFrames = [];
+      // Nudge the model to greet first.
+      openaiWs.send(JSON.stringify({ type: "response.create" }));
+    });
+
+    openaiWs.on("error", (err) => {
+      console.error(`[bridge] openai ws error: ${err.message}`);
+    });
+
+    openaiWs.on("close", (code, reason) => {
+      console.log(`[bridge] openai ws closed code=${code} reason=${reason}`);
+      cleanup("openai_closed");
+    });
+
+    openaiWs.on("message", (raw) => {
+      let evt;
+      try { evt = JSON.parse(raw.toString("utf-8")); }
+      catch { return; }
+
+      // Forward the model's audio back to Twilio.
+      if (evt.type === "response.audio.delta" || evt.type === "response.output_audio.delta") {
+        if (twilioWs.readyState === WebSocket.OPEN && streamSid && evt.delta) {
+          twilioWs.send(JSON.stringify({
+            event: "media",
+            streamSid,
+            media: { payload: evt.delta },
+          }));
+        }
+      } else if (evt.type === "input_audio_buffer.speech_started") {
+        // User started speaking → barge-in: tell Twilio to clear playback buffer.
+        // Only send response.cancel if a response is ACTUALLY running, otherwise
+        // OpenAI rejects with "response_cancel_not_active" (harmless but noisy).
+        if (twilioWs.readyState === WebSocket.OPEN && streamSid) {
+          twilioWs.send(JSON.stringify({ event: "clear", streamSid }));
+        }
+        if (responseInProgress) {
+          openaiWs.send(JSON.stringify({ type: "response.cancel" }));
+          responseInProgress = false;
+        }
+      } else if (evt.type === "response.created") {
+        responseInProgress = true;
+      } else if (evt.type === "error") {
+        console.error(`[bridge] openai error event: ${JSON.stringify(evt).slice(0, 300)}`);
+      } else if (evt.type === "session.created" || evt.type === "session.updated") {
+        // Quiet — no action needed
+      } else if (evt.type === "response.done") {
+        responseInProgress = false;
+      }
+    });
+  }
+}
+
+function buildSessionUpdate(queued) {
+  const voice = queued.voice || "cedar";
+  const identity = queued.identity || {};
+  // PREFER `instructions` (the full pre-loaded contextual prompt the
+  // local agent shipped — includes identity, memory, RAG, peer info,
+  // voice-mode rules) over the bare `brief`. The `brief` describes
+  // the task; `instructions` describes the whole agent.
+  const instructions = queued.instructions || queued.brief ||
+    `You are ${identity.name || "the agent"}, ${identity.role || "an executive operator"} at ${identity.company || "Northwind"}. Reply briefly in your voice.`;
+  return {
+    type: "session.update",
+    session: {
+      type: "realtime",
+      output_modalities: ["audio"],
+      instructions,
+      audio: {
+        input: {
+          format: { type: "audio/pcmu" }, // g711_ulaw — matches Twilio
+          turn_detection: {
+            type: "server_vad",
+            threshold: 0.5,
+            prefix_padding_ms: 300,
+            silence_duration_ms: 500,
+          },
+        },
+        output: {
+          format: { type: "audio/pcmu" },
+          voice,
+        },
+      },
+      tools: queued.tools || [],
+      tool_choice: queued.tool_choice || "auto",
+    },
+  };
+}
+
+server.listen(PORT, () => {
+  console.log(`[start] openai-voice-cloud-relay listening on :${PORT}`);
+  console.log(`[start] sig=${!!OPENAI_WEBHOOK_SECRET} poll_auth=${!!POLL_TOKEN} openai_key=${!!OPENAI_API_KEY} twiml_auth=${!!VOICE_TWIML_TOKEN}`);
+});