@vellumai/assistant 0.3.0

package/src/calls/relay-server.ts

@@ -0,0 +1,390 @@
+/**
+ * WebSocket handler for Twilio ConversationRelay protocol.
+ *
+ * Manages real-time voice conversations over WebSocket. Each active call
+ * has a single RelayConnection instance that processes inbound messages
+ * from Twilio and can send text tokens back for TTS.
+ */
+
+import type { ServerWebSocket } from 'bun';
+import { getLogger } from '../util/logger.js';
+import {
+  getCallSession,
+  updateCallSession,
+  recordCallEvent,
+  expirePendingQuestions,
+} from './call-store.js';
+import { CallOrchestrator } from './call-orchestrator.js';
+import { fireCallTranscriptNotifier, fireCallCompletionNotifier } from './call-state.js';
+import {
+  extractPromptSpeakerMetadata,
+  SpeakerIdentityTracker,
+  type PromptSpeakerContext,
+} from './speaker-identification.js';
+import { isTerminalState } from './call-state-machine.js';
+
+const log = getLogger('relay-server');
+
+// ── ConversationRelay message types ──────────────────────────────────
+
+// Messages FROM Twilio
+export interface RelaySetupMessage {
+  type: 'setup';
+  callSid: string;
+  from: string;
+  to: string;
+  customParameters?: Record<string, string>;
+}
+
+export interface RelayPromptMessage {
+  type: 'prompt';
+  voicePrompt: string;
+  lang: string;
+  last: boolean;
+  speakerId?: string;
+  speakerLabel?: string;
+  speakerName?: string;
+  speakerConfidence?: number;
+  participantId?: string;
+  participant?: {
+    id?: string;
+    name?: string;
+  };
+  speaker?: {
+    id?: string;
+    label?: string;
+    name?: string;
+    confidence?: number;
+  };
+  metadata?: Record<string, unknown>;
+  providerMetadata?: Record<string, unknown>;
+}
+
+export interface RelayInterruptMessage {
+  type: 'interrupt';
+  utteranceUntilInterrupt: string;
+}
+
+export interface RelayDtmfMessage {
+  type: 'dtmf';
+  digit: string;
+}
+
+export interface RelayErrorMessage {
+  type: 'error';
+  description: string;
+}
+
+export type RelayInboundMessage =
+  | RelaySetupMessage
+  | RelayPromptMessage
+  | RelayInterruptMessage
+  | RelayDtmfMessage
+  | RelayErrorMessage;
+
+// Messages TO Twilio
+export interface RelayTextMessage {
+  type: 'text';
+  token: string;
+  last: boolean;
+}
+
+export interface RelayEndMessage {
+  type: 'end';
+  handoffData?: string;
+}
+
+// ── WebSocket data type ──────────────────────────────────────────────
+
+export interface RelayWebSocketData {
+  callSessionId: string;
+}
+
+// ── Module-level state ───────────────────────────────────────────────
+
+/** Active relay connections keyed by callSessionId. */
+export const activeRelayConnections = new Map<string, RelayConnection>();
+
+// ── RelayConnection ──────────────────────────────────────────────────
+
+/**
+ * Manages a single WebSocket connection for one call.
+ */
+export class RelayConnection {
+  private ws: ServerWebSocket<RelayWebSocketData>;
+  private callSessionId: string;
+  private conversationHistory: Array<{
+    role: 'caller' | 'assistant';
+    text: string;
+    timestamp: number;
+    speaker?: PromptSpeakerContext;
+  }>;
+  private abortController: AbortController;
+  private orchestrator: CallOrchestrator | null = null;
+  private speakerIdentityTracker: SpeakerIdentityTracker;
+
+  constructor(ws: ServerWebSocket<RelayWebSocketData>, callSessionId: string) {
+    this.ws = ws;
+    this.callSessionId = callSessionId;
+    this.conversationHistory = [];
+    this.abortController = new AbortController();
+    this.speakerIdentityTracker = new SpeakerIdentityTracker();
+  }
+
+  /**
+   * Handle an inbound message from Twilio via the ConversationRelay WebSocket.
+   */
+  async handleMessage(data: string): Promise<void> {
+    let parsed: RelayInboundMessage;
+    try {
+      parsed = JSON.parse(data) as RelayInboundMessage;
+    } catch {
+      log.warn({ callSessionId: this.callSessionId, data }, 'Failed to parse relay message');
+      return;
+    }
+
+    switch (parsed.type) {
+      case 'setup':
+        await this.handleSetup(parsed);
+        break;
+      case 'prompt':
+        await this.handlePrompt(parsed);
+        break;
+      case 'interrupt':
+        this.handleInterrupt(parsed);
+        break;
+      case 'dtmf':
+        this.handleDtmf(parsed);
+        break;
+      case 'error':
+        this.handleError(parsed);
+        break;
+      default:
+        log.warn({ callSessionId: this.callSessionId, type: (parsed as Record<string, unknown>).type }, 'Unknown relay message type');
+    }
+  }
+
+  /**
+   * Send a text token to the caller for TTS playback.
+   */
+  sendTextToken(token: string, last: boolean): void {
+    const message: RelayTextMessage = { type: 'text', token, last };
+    try {
+      this.ws.send(JSON.stringify(message));
+    } catch (err) {
+      log.error({ err, callSessionId: this.callSessionId }, 'Failed to send text token');
+    }
+  }
+
+  /**
+   * End the ConversationRelay session.
+   */
+  endSession(reason?: string): void {
+    const message: RelayEndMessage = { type: 'end' };
+    if (reason) {
+      message.handoffData = JSON.stringify({ reason });
+    }
+    try {
+      this.ws.send(JSON.stringify(message));
+    } catch (err) {
+      log.error({ err, callSessionId: this.callSessionId }, 'Failed to send end message');
+    }
+  }
+
+  /**
+   * Get the conversation history for context.
+   */
+  getConversationHistory(): Array<{ role: string; text: string; speaker?: PromptSpeakerContext }> {
+    return this.conversationHistory.map(({ role, text, speaker }) => ({ role, text, speaker }));
+  }
+
+  /**
+   * Get the call session ID for this connection.
+   */
+  getCallSessionId(): string {
+    return this.callSessionId;
+  }
+
+  /**
+   * Set the orchestrator for this connection.
+   */
+  setOrchestrator(orchestrator: CallOrchestrator): void {
+    this.orchestrator = orchestrator;
+  }
+
+  /**
+   * Get the orchestrator for this connection.
+   */
+  getOrchestrator(): CallOrchestrator | null {
+    return this.orchestrator;
+  }
+
+  /**
+   * Clean up resources on disconnect.
+   */
+  destroy(): void {
+    if (this.orchestrator) {
+      this.orchestrator.destroy();
+      this.orchestrator = null;
+    }
+    this.abortController.abort();
+    log.info({ callSessionId: this.callSessionId }, 'RelayConnection destroyed');
+  }
+
+  /**
+   * Handle transport-level close from the relay websocket.
+   *
+   * Twilio status callbacks are best-effort; if they are delayed or absent,
+   * we still finalize the call lifecycle from the relay close signal.
+   */
+  handleTransportClosed(code?: number, reason?: string): void {
+    const session = getCallSession(this.callSessionId);
+    if (!session) return;
+    if (isTerminalState(session.status)) return;
+
+    const isNormalClose = code === 1000;
+    if (isNormalClose) {
+      updateCallSession(this.callSessionId, {
+        status: 'completed',
+        endedAt: Date.now(),
+      });
+      recordCallEvent(this.callSessionId, 'call_ended', {
+        reason: reason || 'relay_closed',
+        closeCode: code,
+      });
+    } else {
+      const detail = reason || (code ? `relay_closed_${code}` : 'relay_closed_abnormal');
+      updateCallSession(this.callSessionId, {
+        status: 'failed',
+        endedAt: Date.now(),
+        lastError: `Relay websocket closed unexpectedly: ${detail}`,
+      });
+      recordCallEvent(this.callSessionId, 'call_failed', {
+        reason: detail,
+        closeCode: code,
+      });
+    }
+
+    expirePendingQuestions(this.callSessionId);
+    fireCallCompletionNotifier(session.conversationId, this.callSessionId);
+  }
+
+  // ── Private handlers ─────────────────────────────────────────────
+
+  private async handleSetup(msg: RelaySetupMessage): Promise<void> {
+    log.info(
+      { callSessionId: this.callSessionId, callSid: msg.callSid, from: msg.from, to: msg.to },
+      'ConversationRelay setup received',
+    );
+
+    // Store the callSid association on the call session
+    const session = getCallSession(this.callSessionId);
+    if (session) {
+      const updates: Parameters<typeof updateCallSession>[1] = {
+        providerCallSid: msg.callSid,
+      };
+      if (!isTerminalState(session.status) && session.status !== 'in_progress' && session.status !== 'waiting_on_user') {
+        updates.status = 'in_progress';
+        if (!session.startedAt) {
+          updates.startedAt = Date.now();
+        }
+      }
+      updateCallSession(this.callSessionId, updates);
+    }
+
+    recordCallEvent(this.callSessionId, 'call_connected', {
+      callSid: msg.callSid,
+      from: msg.from,
+      to: msg.to,
+      customParameters: msg.customParameters,
+    });
+
+    // Create and attach the LLM-driven orchestrator
+    const orchestrator = new CallOrchestrator(this.callSessionId, this, session?.task ?? null);
+    this.setOrchestrator(orchestrator);
+  }
+
+  private async handlePrompt(msg: RelayPromptMessage): Promise<void> {
+    if (!msg.last) {
+      // Partial transcript, wait for final
+      return;
+    }
+
+    log.info(
+      { callSessionId: this.callSessionId, transcript: msg.voicePrompt, lang: msg.lang },
+      'Caller transcript received (final)',
+    );
+
+    const speakerMetadata = extractPromptSpeakerMetadata(msg as unknown as Record<string, unknown>);
+    const speaker = this.speakerIdentityTracker.identifySpeaker(speakerMetadata);
+
+    // Record in conversation history
+    this.conversationHistory.push({
+      role: 'caller',
+      text: msg.voicePrompt,
+      timestamp: Date.now(),
+      speaker,
+    });
+
+    // Record event
+    recordCallEvent(this.callSessionId, 'caller_spoke', {
+      transcript: msg.voicePrompt,
+      lang: msg.lang,
+      speakerId: speaker.speakerId,
+      speakerLabel: speaker.speakerLabel,
+      speakerConfidence: speaker.speakerConfidence,
+      speakerSource: speaker.source,
+    });
+
+    const session = getCallSession(this.callSessionId);
+    if (session) {
+      fireCallTranscriptNotifier(session.conversationId, this.callSessionId, 'caller', msg.voicePrompt);
+    }
+
+    // Route to orchestrator for LLM-driven response
+    if (this.orchestrator) {
+      await this.orchestrator.handleCallerUtterance(msg.voicePrompt, speaker);
+    } else {
+      // Fallback if orchestrator not yet initialized
+      this.sendTextToken('I\'m still setting up. Please hold.', true);
+    }
+  }
+
+  private handleInterrupt(msg: RelayInterruptMessage): void {
+    log.info(
+      { callSessionId: this.callSessionId, utteranceUntilInterrupt: msg.utteranceUntilInterrupt },
+      'Caller interrupted assistant',
+    );
+
+    // Abort any in-flight processing
+    this.abortController.abort();
+    this.abortController = new AbortController();
+
+    // Notify the orchestrator of the interruption
+    if (this.orchestrator) {
+      this.orchestrator.handleInterrupt();
+    }
+  }
+
+  private handleDtmf(msg: RelayDtmfMessage): void {
+    log.info(
+      { callSessionId: this.callSessionId, digit: msg.digit },
+      'DTMF digit received',
+    );
+
+    recordCallEvent(this.callSessionId, 'caller_spoke', {
+      dtmfDigit: msg.digit,
+    });
+  }
+
+  private handleError(msg: RelayErrorMessage): void {
+    log.error(
+      { callSessionId: this.callSessionId, description: msg.description },
+      'ConversationRelay error',
+    );
+
+    recordCallEvent(this.callSessionId, 'call_failed', {
+      error: msg.description,
+    });
+  }
+}

package/src/calls/speaker-identification.ts

@@ -0,0 +1,213 @@
+export interface PromptSpeakerContext {
+  speakerId: string;
+  speakerLabel: string;
+  speakerConfidence: number | null;
+  source: 'provider' | 'inferred';
+}
+
+export interface PromptSpeakerMetadata {
+  speakerId?: string;
+  speakerLabel?: string;
+  speakerName?: string;
+  speakerConfidence?: number;
+  participantId?: string;
+}
+
+interface SpeakerProfile {
+  speakerId: string;
+  speakerLabel: string;
+  speakerConfidence: number | null;
+  source: 'provider' | 'inferred';
+  utteranceCount: number;
+  firstSeenAt: number;
+  lastSeenAt: number;
+}
+
+function toCleanString(value: unknown): string | null {
+  if (typeof value !== 'string') return null;
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function toNumber(value: unknown): number | null {
+  if (typeof value === 'number' && Number.isFinite(value)) return value;
+  if (typeof value === 'string') {
+    const parsed = Number(value);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  return null;
+}
+
+function getObject(value: unknown): Record<string, unknown> | null {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) return null;
+  return value as Record<string, unknown>;
+}
+
+function normalizeSpeakerLabel(metadata: PromptSpeakerMetadata, fallbackIndex: number): string {
+  const preferredLabel = toCleanString(metadata.speakerName) ?? toCleanString(metadata.speakerLabel);
+  if (preferredLabel) return preferredLabel;
+  return `Speaker ${fallbackIndex}`;
+}
+
+export function extractPromptSpeakerMetadata(message: Record<string, unknown>): PromptSpeakerMetadata {
+  const providerMetadata = getObject(message.providerMetadata);
+  const metadata = getObject(message.metadata);
+  const participant = getObject(message.participant);
+  const speaker = getObject(message.speaker);
+
+  const pick = (...values: unknown[]): string | undefined => {
+    for (const value of values) {
+      const cleaned = toCleanString(value);
+      if (cleaned) return cleaned;
+    }
+    return undefined;
+  };
+
+  const pickNumber = (...values: unknown[]): number | undefined => {
+    for (const value of values) {
+      const parsed = toNumber(value);
+      if (parsed !== null) return parsed;
+    }
+    return undefined;
+  };
+
+  return {
+    speakerId: pick(
+      message.speakerId,
+      message.speaker_id,
+      speaker?.id,
+      speaker?.speakerId,
+      metadata?.speakerId,
+      providerMetadata?.speakerId,
+      metadata?.speaker_id,
+      providerMetadata?.speaker_id,
+    ),
+    speakerLabel: pick(
+      message.speakerLabel,
+      message.speaker_label,
+      message.speaker,
+      speaker?.label,
+      speaker?.name,
+      metadata?.speakerLabel,
+      providerMetadata?.speakerLabel,
+      metadata?.speaker_label,
+      providerMetadata?.speaker_label,
+    ),
+    speakerName: pick(
+      message.speakerName,
+      message.speaker_name,
+      participant?.name,
+      metadata?.speakerName,
+      providerMetadata?.speakerName,
+      metadata?.speaker_name,
+      providerMetadata?.speaker_name,
+    ),
+    speakerConfidence: pickNumber(
+      message.speakerConfidence,
+      message.speaker_confidence,
+      message.confidence,
+      speaker?.confidence,
+      metadata?.speakerConfidence,
+      providerMetadata?.speakerConfidence,
+      metadata?.speaker_confidence,
+      providerMetadata?.speaker_confidence,
+    ),
+    participantId: pick(
+      message.participantId,
+      message.participant_id,
+      participant?.id,
+      metadata?.participantId,
+      providerMetadata?.participantId,
+      metadata?.participant_id,
+      providerMetadata?.participant_id,
+    ),
+  };
+}
+
+export class SpeakerIdentityTracker {
+  private profiles = new Map<string, SpeakerProfile>();
+  private nextInferredIndex = 1;
+
+  identifySpeaker(metadata: PromptSpeakerMetadata): PromptSpeakerContext {
+    const providerSpeakerId =
+      toCleanString(metadata.speakerId)
+      ?? toCleanString(metadata.participantId)
+      ?? null;
+
+    if (providerSpeakerId) {
+      const existing = this.profiles.get(providerSpeakerId);
+      if (existing) {
+        existing.lastSeenAt = Date.now();
+        existing.utteranceCount += 1;
+        if (metadata.speakerConfidence !== undefined) {
+          existing.speakerConfidence = metadata.speakerConfidence;
+        }
+        return {
+          speakerId: existing.speakerId,
+          speakerLabel: existing.speakerLabel,
+          speakerConfidence: existing.speakerConfidence,
+          source: existing.source,
+        };
+      }
+
+      const profile: SpeakerProfile = {
+        speakerId: providerSpeakerId,
+        speakerLabel: normalizeSpeakerLabel(metadata, this.nextInferredIndex),
+        speakerConfidence: metadata.speakerConfidence ?? null,
+        source: 'provider',
+        utteranceCount: 1,
+        firstSeenAt: Date.now(),
+        lastSeenAt: Date.now(),
+      };
+      this.profiles.set(providerSpeakerId, profile);
+      this.nextInferredIndex += 1;
+      return {
+        speakerId: profile.speakerId,
+        speakerLabel: profile.speakerLabel,
+        speakerConfidence: profile.speakerConfidence,
+        source: profile.source,
+      };
+    }
+
+    const inferredSpeakerId = 'primary-speaker';
+    const existingPrimary = this.profiles.get(inferredSpeakerId);
+    if (existingPrimary) {
+      existingPrimary.lastSeenAt = Date.now();
+      existingPrimary.utteranceCount += 1;
+      return {
+        speakerId: existingPrimary.speakerId,
+        speakerLabel: existingPrimary.speakerLabel,
+        speakerConfidence: existingPrimary.speakerConfidence,
+        source: existingPrimary.source,
+      };
+    }
+
+    const inferredProfile: SpeakerProfile = {
+      speakerId: inferredSpeakerId,
+      speakerLabel: normalizeSpeakerLabel(metadata, this.nextInferredIndex),
+      speakerConfidence: metadata.speakerConfidence ?? null,
+      source: 'inferred',
+      utteranceCount: 1,
+      firstSeenAt: Date.now(),
+      lastSeenAt: Date.now(),
+    };
+    this.profiles.set(inferredSpeakerId, inferredProfile);
+    this.nextInferredIndex += 1;
+
+    return {
+      speakerId: inferredProfile.speakerId,
+      speakerLabel: inferredProfile.speakerLabel,
+      speakerConfidence: inferredProfile.speakerConfidence,
+      source: inferredProfile.source,
+    };
+  }
+
+  listProfiles(): PromptSpeakerContext[] {
+    return [...this.profiles.values()].map((profile) => ({
+      speakerId: profile.speakerId,
+      speakerLabel: profile.speakerLabel,
+      speakerConfidence: profile.speakerConfidence,
+      source: profile.source,
+    }));
+  }
+}

package/src/calls/twilio-config.ts

@@ -0,0 +1,45 @@
+import { getSecureKey } from '../security/secure-keys.js';
+import { getLogger } from '../util/logger.js';
+import { loadConfig } from '../config/loader.js';
+import { getPublicBaseUrl, getTwilioRelayUrl } from '../inbound/public-ingress-urls.js';
+
+const log = getLogger('twilio-config');
+
+export interface TwilioConfig {
+  accountSid: string;
+  authToken: string;
+  phoneNumber: string;
+  webhookBaseUrl: string;
+  wssBaseUrl: string;
+}
+
+export function getTwilioConfig(): TwilioConfig {
+  const accountSid = getSecureKey('credential:twilio:account_sid');
+  const authToken = getSecureKey('credential:twilio:auth_token');
+  const phoneNumber = process.env.TWILIO_PHONE_NUMBER || getSecureKey('credential:twilio:phone_number') || '';
+  const config = loadConfig();
+  const webhookBaseUrl = getPublicBaseUrl(config);
+
+  // Always use the centralized relay URL derived from the public ingress base URL.
+  // TWILIO_WSS_BASE_URL is ignored.
+  let wssBaseUrl: string;
+  if (process.env.TWILIO_WSS_BASE_URL) {
+    log.warn('TWILIO_WSS_BASE_URL env var is ignored. Relay URL is derived from ingress.publicBaseUrl.');
+  }
+  try {
+    wssBaseUrl = getTwilioRelayUrl(config);
+  } catch {
+    wssBaseUrl = '';
+  }
+
+  if (!accountSid || !authToken) {
+    throw new Error('Twilio credentials not configured. Set credential:twilio:account_sid and credential:twilio:auth_token via the credential_store tool.');
+  }
+  if (!phoneNumber) {
+    throw new Error('TWILIO_PHONE_NUMBER not configured.');
+  }
+
+  log.debug('Twilio config loaded successfully');
+
+  return { accountSid, authToken, phoneNumber, webhookBaseUrl, wssBaseUrl };
+}