@vellumai/assistant 0.6.3 → 0.6.4

package/src/calls/twilio-routes.ts

@@ -1,13 +1,32 @@
 /**
  * HTTP route handlers for Twilio voice webhooks.
  *
- * - handleVoiceWebhook: initial voice webhook; returns TwiML to connect ConversationRelay
+ * - handleVoiceWebhook: initial voice webhook; returns TwiML to connect
+ *   ConversationRelay (Twilio-native STT) or Stream (custom media-stream STT)
  * - handleStatusCallback: call status updates (ringing, in-progress, completed, etc.)
  * - handleConnectAction: called when the ConversationRelay connection ends
+ *
+ * ## STT routing
+ *
+ * TwiML generation is driven by `services.stt.provider` via
+ * {@link resolveTelephonySttRouting}. The resolver returns a discriminated
+ * strategy that determines which TwiML path to use:
+ *
+ * - **`conversation-relay-native`** (deepgram, google-gemini) — emits
+ *   `<Connect><ConversationRelay>` with Twilio-native `transcriptionProvider`
+ *   and `speechModel` attributes.
+ *
+ * - **`media-stream-custom`** (openai-whisper) — emits
+ *   `<Connect><Stream>` so the daemon receives raw audio and transcribes
+ *   server-side.
  */
 
 import { loadConfig } from "../config/loader.js";
-import { getTwilioRelayUrl } from "../inbound/public-ingress-urls.js";
+import {
+  getTwilioMediaStreamUrl,
+  getTwilioRelayUrl,
+} from "../inbound/public-ingress-urls.js";
+import { getProviderEntry } from "../providers/speech-to-text/provider-catalog.js";
 import { mintEdgeRelayToken } from "../runtime/auth/token-service.js";
 import { getLogger } from "../util/logger.js";
 import { persistCallCompletionMessage } from "./call-conversation-messages.js";
@@ -26,12 +45,33 @@ import {
   releaseCallbackClaim,
   updateCallSession,
 } from "./call-store.js";
+import { routeSetup } from "./relay-setup-router.js";
 import { resolveCallHints } from "./stt-hints.js";
+import { resolveTelephonySttRouting } from "./telephony-stt-routing.js";
 import type { CallStatus } from "./types.js";
 import { resolveVoiceQualityProfile } from "./voice-quality.js";
 
 const log = getLogger("twilio-routes");
 
+// ── Speech config type ───────────────────────────────────────────────
+
+/**
+ * Twilio ConversationRelay speech-to-text attributes.
+ *
+ * All values are pre-formatted strings ready for direct insertion into
+ * TwiML XML attribute values (XML escaping is the caller's responsibility).
+ */
+export interface TwilioRelaySpeechConfig {
+  /** STT provider name (e.g. "Deepgram", "Google"). */
+  transcriptionProvider: string;
+  /** ASR model identifier, or undefined when the provider default should be used. */
+  speechModel: string | undefined;
+  /** Comma-separated vocabulary hints for the STT provider, or undefined when no hints are available. */
+  hints: string | undefined;
+  /** How aggressively the provider detects the start of caller speech. */
+  interruptSensitivity: string;
+}
+
 // ── Helpers ──────────────────────────────────────────────────────────
 
 function escapeXml(str: string): string {
@@ -49,15 +89,12 @@ export function generateTwiML(
   welcomeGreeting: string | null,
   profile: {
     language: string;
-    transcriptionProvider: string;
-    speechModel?: string;
     ttsProvider: string;
     voice: string;
-    interruptSensitivity: string;
   },
+  speechConfig: TwilioRelaySpeechConfig,
   relayToken?: string,
   customParameters?: Record<string, string>,
-  hints?: string,
 ): string {
   const greetingAttr =
     welcomeGreeting && welcomeGreeting.trim().length > 0
@@ -95,16 +132,76 @@ export function generateTwiML(
 ${greetingAttr}
       voice="${escapeXml(profile.voice)}"
       language="${escapeXml(profile.language)}"
-      transcriptionProvider="${escapeXml(profile.transcriptionProvider)}"${profile.speechModel ? `\n      speechModel="${escapeXml(profile.speechModel)}"` : ""}
+      transcriptionProvider="${escapeXml(speechConfig.transcriptionProvider)}"${speechConfig.speechModel ? `\n      speechModel="${escapeXml(speechConfig.speechModel)}"` : ""}
       ttsProvider="${escapeXml(profile.ttsProvider)}"
       interruptible="true"
       dtmfDetection="true"
-      interruptSensitivity="${escapeXml(profile.interruptSensitivity)}"${hints ? `\n      hints="${escapeXml(hints)}"` : ""}
+      interruptSensitivity="${escapeXml(speechConfig.interruptSensitivity)}"${speechConfig.hints ? `\n      hints="${escapeXml(speechConfig.hints)}"` : ""}
     ${relayClose}
   </Connect>
 </Response>`;
 }
 
+/**
+ * Generate `<Connect><Stream>` TwiML for the media-stream STT path.
+ *
+ * Used when the telephony STT routing resolver selects `media-stream-custom`
+ * (e.g. OpenAI Whisper). Twilio opens a WebSocket to `streamUrl` and sends
+ * raw audio frames; the daemon transcribes server-side.
+ *
+ * `callSessionId` and `token` are encoded as **path segments** on the
+ * WebSocket URL so the gateway can validate and route the upgrade request
+ * before any Twilio `start` frame arrives. Twilio Media Streams does not
+ * reliably preserve URL query parameters across the WebSocket upgrade, so
+ * path-based encoding is the primary transport for handshake metadata.
+ *
+ * Both values are also propagated as `<Parameter>` children so Twilio
+ * includes them in the `start` event's `customParameters` object for
+ * downstream observability.
+ */
+export function generateStreamTwiML(
+  callSessionId: string,
+  streamUrl: string,
+  relayToken?: string,
+  customParameters?: Record<string, string>,
+): string {
+  // Build the WebSocket URL with callSessionId and token as path segments.
+  // Twilio Media Streams does not reliably preserve query parameters
+  // across the WebSocket upgrade, so path-based encoding is the primary
+  // transport. The gateway extracts metadata from path segments first,
+  // falling back to query parameters for legacy compatibility.
+  let fullStreamUrl = streamUrl.replace(/\/+$/, "");
+  fullStreamUrl += `/${encodeURIComponent(callSessionId)}`;
+  if (relayToken) {
+    fullStreamUrl += `/${encodeURIComponent(relayToken)}`;
+  }
+
+  // Build <Parameter> elements for the Twilio start event payload.
+  // Spread customParameters first so callSessionId and token cannot be
+  // overridden by caller-supplied values.
+  const allParams: Record<string, string> = {
+    ...customParameters,
+    callSessionId,
+  };
+
+  if (relayToken) {
+    allParams.token = relayToken;
+  }
+
+  let parameterElements = "";
+  for (const [key, value] of Object.entries(allParams)) {
+    parameterElements += `\n      <Parameter name="${escapeXml(key)}" value="${escapeXml(value)}" />`;
+  }
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<Response>
+  <Connect>
+    <Stream url="${escapeXml(fullStreamUrl)}">${parameterElements}
+    </Stream>
+  </Connect>
+</Response>`;
+}
+
 export function buildWelcomeGreeting(
   task: string | null,
   configuredGreeting?: string,
@@ -233,8 +330,17 @@ export async function handleVoiceWebhook(req: Request): Promise<Response> {
 
 /**
  * Shared TwiML generation for both inbound and outbound voice webhooks.
- * Resolves voice quality profile, relay URL, and welcome greeting,
- * then returns a Response with the generated TwiML.
+ *
+ * Resolves the telephony STT routing strategy from `services.stt.provider`
+ * and branches:
+ *
+ * - **`conversation-relay-native`** — emits `<Connect><ConversationRelay>`
+ *   TwiML with Twilio-native STT attributes (`transcriptionProvider`,
+ *   `speechModel`). Used for deepgram and google-gemini.
+ *
+ * - **`media-stream-custom`** — emits `<Connect><Stream>` TwiML so the
+ *   daemon receives raw audio for server-side transcription. Used for
+ *   openai-whisper.
  *
  * When `verificationSessionId` is provided, it is included as a
  * `<Parameter>` in the TwiML for observability and compatibility with
@@ -253,18 +359,137 @@ function buildVoiceWebhookTwiml(
   } | null,
   verificationSessionId?: string | null,
 ): Response {
-  const profile = resolveVoiceQualityProfile(loadConfig());
-
-  const hints = resolveCallHints(sessionContext, profile.hints);
+  const cfg = loadConfig();
+  const profile = resolveVoiceQualityProfile(cfg);
 
   log.info(
     { callSessionId, ttsProvider: profile.ttsProvider, voice: profile.voice },
     "Voice quality profile resolved",
   );
 
-  const relayUrl = getTwilioRelayUrl(loadConfig());
-  const welcomeGreeting = buildWelcomeGreeting(sessionContext?.task ?? null);
+  // Resolve telephony STT strategy from services.stt.provider.
+  // The routing resolver handles speech-model defaults internally per provider.
+  const routingResult = resolveTelephonySttRouting();
+
+  // Derive Deepgram fallback values from the provider catalog so they stay
+  // in sync with the single source of truth. Hardcoded strings are kept only
+  // as a final safety net in case the catalog entry is missing.
+  const dgEntry = getProviderEntry("deepgram");
+  const fallbackProvider =
+    dgEntry?.telephonyRouting.twilioNativeMapping?.provider ?? "Deepgram";
+  const fallbackModel =
+    dgEntry?.telephonyRouting.twilioNativeMapping?.defaultSpeechModel ??
+    "nova-3";
+
+  if (routingResult.status === "unknown-provider") {
+    log.error(
+      {
+        callSessionId,
+        providerId: routingResult.providerId,
+        reason: routingResult.reason,
+      },
+      "Telephony STT routing failed — unknown provider; falling back to ConversationRelay with Deepgram",
+    );
+    // Graceful degradation: fall back to Deepgram ConversationRelay so
+    // calls don't fail entirely on a misconfigured provider.
+    return buildConversationRelayResponse(
+      callSessionId,
+      cfg,
+      profile,
+      sessionContext,
+      verificationSessionId,
+      { transcriptionProvider: fallbackProvider, speechModel: fallbackModel },
+    );
+  }
+
+  const { strategy } = routingResult;
+
+  if (strategy.strategy === "conversation-relay-native") {
+    return buildConversationRelayResponse(
+      callSessionId,
+      cfg,
+      profile,
+      sessionContext,
+      verificationSessionId,
+      {
+        transcriptionProvider: strategy.transcriptionProvider,
+        speechModel: strategy.speechModel,
+      },
+    );
+  }
+
+  // media-stream-custom path — preflight check to reject interactive setup
+  // flows that the media-stream transport cannot support. The media-stream
+  // server has a defensive fallback for these cases, but catching them here
+  // avoids bootstrapping a WebSocket session that will immediately fail.
+  const session = getCallSession(callSessionId);
+  const from = sessionContext?.fromNumber ?? "";
+  const to = sessionContext?.toNumber ?? "";
 
+  const { outcome } = routeSetup({
+    callSessionId,
+    session: session ?? null,
+    from,
+    to,
+  });
+
+  // The media-stream transport supports normal_call and deny (which speaks
+  // a message and tears down). All other outcomes require interactive
+  // sub-flows (DTMF entry, name capture, guardian wait) that media-stream
+  // cannot perform. Reject these deterministically before stream bootstrap.
+  if (outcome.action !== "normal_call" && outcome.action !== "deny") {
+    log.warn(
+      {
+        callSessionId,
+        setupAction: outcome.action,
+        strategy: "media-stream-custom",
+      },
+      "Media-stream preflight rejected unsupported interactive setup flow — falling back to ConversationRelay with Deepgram",
+    );
+    // Fall back to ConversationRelay so the interactive flow can proceed
+    // through the relay server which supports it natively.
+    return buildConversationRelayResponse(
+      callSessionId,
+      cfg,
+      profile,
+      sessionContext,
+      verificationSessionId,
+      { transcriptionProvider: fallbackProvider, speechModel: fallbackModel },
+    );
+  }
+
+  return buildMediaStreamResponse(callSessionId, cfg, verificationSessionId);
+}
+
+/**
+ * Build a ConversationRelay TwiML response for Twilio-native STT providers.
+ */
+function buildConversationRelayResponse(
+  callSessionId: string,
+  cfg: ReturnType<typeof loadConfig>,
+  profile: ReturnType<typeof resolveVoiceQualityProfile>,
+  sessionContext: {
+    task: string | null;
+    toNumber: string;
+    fromNumber: string;
+    direction: "inbound" | "outbound";
+    inviteFriendName: string | null;
+    inviteGuardianName: string | null;
+  } | null,
+  verificationSessionId: string | null | undefined,
+  sttAttrs: { transcriptionProvider: string; speechModel: string | undefined },
+): Response {
+  const rawHints = resolveCallHints(sessionContext, profile.hints);
+
+  const speechConfig: TwilioRelaySpeechConfig = {
+    transcriptionProvider: sttAttrs.transcriptionProvider,
+    speechModel: sttAttrs.speechModel,
+    hints: rawHints && rawHints.length > 0 ? rawHints : undefined,
+    interruptSensitivity: profile.interruptSensitivity,
+  };
+
+  const relayUrl = getTwilioRelayUrl(cfg);
+  const welcomeGreeting = buildWelcomeGreeting(sessionContext?.task ?? null);
   const relayToken = mintEdgeRelayToken();
 
   // Propagate verificationSessionId as a TwiML <Parameter> for
@@ -278,12 +503,51 @@ function buildVoiceWebhookTwiml(
     relayUrl,
     welcomeGreeting,
     profile,
+    speechConfig,
     relayToken,
     customParameters,
-    hints || undefined,
   );
 
-  log.info({ callSessionId }, "Returning ConversationRelay TwiML");
+  log.info(
+    {
+      callSessionId,
+      strategy: "conversation-relay-native",
+      transcriptionProvider: sttAttrs.transcriptionProvider,
+    },
+    "Returning ConversationRelay TwiML",
+  );
+
+  return new Response(twiml, {
+    status: 200,
+    headers: { "Content-Type": "text/xml" },
+  });
+}
+
+/**
+ * Build a Stream TwiML response for custom media-stream STT providers.
+ */
+function buildMediaStreamResponse(
+  callSessionId: string,
+  cfg: ReturnType<typeof loadConfig>,
+  verificationSessionId: string | null | undefined,
+): Response {
+  const streamUrl = getTwilioMediaStreamUrl(cfg);
+  const relayToken = mintEdgeRelayToken();
+
+  const customParameters: Record<string, string> | undefined =
+    verificationSessionId ? { verificationSessionId } : undefined;
+
+  const twiml = generateStreamTwiML(
+    callSessionId,
+    streamUrl,
+    relayToken,
+    customParameters,
+  );
+
+  log.info(
+    { callSessionId, strategy: "media-stream-custom" },
+    "Returning Stream TwiML",
+  );
 
   return new Response(twiml, {
     status: 200,

package/src/calls/voice-quality.ts

@@ -1,9 +1,14 @@
 import { loadConfig } from "../config/loader.js";
+import { DEFAULT_ELEVENLABS_VOICE_ID } from "../config/schemas/elevenlabs.js";
+import { getTtsProvider } from "../tts/provider-registry.js";
+import { resolveTtsConfig } from "../tts/tts-config-resolver.js";
+import {
+  getNativeTwilioVoiceSpec,
+  resolveCallStrategy,
+} from "./tts-call-strategy.js";
 
 export interface VoiceQualityProfile {
   language: string;
-  transcriptionProvider: string;
-  speechModel?: string;
   ttsProvider: string;
   voice: string;
   interruptSensitivity: string;
@@ -45,49 +50,87 @@ export function buildElevenLabsVoiceSpec(config: {
 /**
  * Resolve the effective voice quality profile from config.
  *
- * Supports ElevenLabs (default) and Fish Audio TTS providers.
- * When Fish Audio is selected, `ttsProvider` is set to `"Google"` as a
- * placeholder — ConversationRelay requires a valid provider in TwiML, but
- * actual audio is delivered via `play` messages from the call-controller.
- * The voice string is left empty since it is unused in that mode.
+ * Uses the explicit call strategy from the TTS provider catalog to
+ * determine the call path. The catalog's `callMode` field
+ * (`"native-twilio"` vs `"synthesized-play"`) drives the decision
+ * rather than inferring behavior from runtime `supportsStreaming`
+ * capability.
  *
- * For ElevenLabs, the voice ID comes from the shared `elevenlabs.voiceId`
- * config (defaults to Amelia — ZF6FPAbjXT4488VcRRnw).
+ * For **synthesized-play** providers (e.g. Fish Audio),
+ * ConversationRelay needs a valid TTS provider in TwiML, so we set
+ * `ttsProvider` to `"Google"` as a placeholder and leave `voice` empty
+ * since actual audio is delivered via `play` messages.
+ *
+ * For **native-twilio** providers (e.g. ElevenLabs), `ttsProvider` and
+ * `voice` are populated via the provider's registered
+ * {@link NativeTwilioVoiceSpec} builder so Twilio handles TTS natively.
+ * New native providers plug in by registering a voice-spec builder --
+ * no edits to this module required.
+ *
+ * NOTE: STT provider and speech model are intentionally NOT part of this
+ * profile. STT resolution is handled once in the voice webhook route
+ * (`twilio-routes.ts`) via `resolveTelephonySttRouting()` to maintain a
+ * single point of ownership.
  */
 export function resolveVoiceQualityProfile(
   config?: ReturnType<typeof loadConfig>,
 ): VoiceQualityProfile {
   const cfg = config ?? loadConfig();
   const voice = cfg.calls.voice;
-  const configuredTts = voice.ttsProvider ?? "elevenlabs";
-  const fishAudio = configuredTts === "fish-audio";
-  const isGoogle = voice.transcriptionProvider === "Google";
-  // Treat the legacy Deepgram default ("nova-3") as unset when provider is
-  // Google — upgraded workspaces may still have it persisted from prior defaults.
-  const effectiveSpeechModel =
-    voice.speechModel == null ||
-    (voice.speechModel === "nova-3" && isGoogle)
-      ? isGoogle
-        ? undefined
-        : "nova-3"
-      : voice.speechModel;
+
+  // Resolve the call strategy from catalog metadata.
+  // Falls back to native ElevenLabs when config/catalog is unavailable.
+  const strategy = resolveCallStrategy(cfg);
+
+  // Before committing to the catalog-derived strategy, verify the
+  // runtime provider is actually registered. If the provider registry
+  // hasn't been initialised (early startup, test mocks), fall back to
+  // native mode so this function and resolveCallTtsProvider agree on
+  // the same degraded-mode path.
+  let runtimeAvailable = false;
+  try {
+    const resolved = resolveTtsConfig(cfg);
+    getTtsProvider(resolved.provider);
+    runtimeAvailable = true;
+  } catch {
+    // Provider not registered — will fall through to native path below.
+  }
+
+  let ttsProvider: string;
+  let voiceSpec: string;
+
+  if (runtimeAvailable && strategy.callMode === "synthesized-play") {
+    // Synthesized providers stream audio via `play` messages.
+    // Twilio still needs a valid ttsProvider in TwiML, so use a
+    // placeholder and leave voice empty.
+    ttsProvider = "Google";
+    voiceSpec = "";
+  } else {
+    // Native providers: delegate voice-spec building to the
+    // provider's registered builder.
+    try {
+      const spec = getNativeTwilioVoiceSpec(strategy.providerId);
+      ttsProvider = spec.twilioProviderName;
+
+      const resolved = resolveTtsConfig(cfg);
+      voiceSpec = spec.buildVoiceSpec(resolved.providerConfig);
+    } catch {
+      // Voice-spec builder not registered or config unavailable --
+      // fall back to ElevenLabs using the config's elevenlabs block.
+      ttsProvider = "ElevenLabs";
+      voiceSpec = buildElevenLabsVoiceSpec(
+        cfg.services?.tts?.providers?.elevenlabs ?? {
+          voiceId: DEFAULT_ELEVENLABS_VOICE_ID,
+        },
+      );
+    }
+  }
+
   return {
     language: voice.language,
-    transcriptionProvider: voice.transcriptionProvider,
-    speechModel: effectiveSpeechModel,
-    ttsProvider: fishAudio ? "Google" : "ElevenLabs",
-    voice: fishAudio ? "" : buildElevenLabsVoiceSpec(cfg.elevenlabs),
+    ttsProvider,
+    voice: voiceSpec,
     interruptSensitivity: voice.interruptSensitivity ?? "low",
     hints: voice.hints ?? [],
   };
 }
-
-/**
- * Check whether Fish Audio TTS is configured for phone calls.
- */
-export function isFishAudioTts(
-  config?: ReturnType<typeof loadConfig>,
-): boolean {
-  const cfg = config ?? loadConfig();
-  return cfg.calls.voice?.ttsProvider === "fish-audio";
-}

package/src/calls/voice-session-bridge.ts

@@ -21,6 +21,7 @@ import { buildAssistantEvent } from "../runtime/assistant-event.js";
 import { assistantEventHub } from "../runtime/assistant-event-hub.js";
 import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
 import { computeToolApprovalDigest } from "../security/tool-approval-digest.js";
+import { createAbortReason } from "../util/abort-reasons.js";
 import { getLogger } from "../util/logger.js";
 import {
   CALL_OPENING_MARKER,
@@ -539,7 +540,13 @@ export async function startVoiceTurn(
 
   const abortFn = () => {
     if (conversation.currentRequestId === requestId) {
-      conversation.abort();
+      conversation.abort(
+        createAbortReason(
+          "voice_session_aborted",
+          "voice-session-bridge.abortFn",
+          conversation.conversationId,
+        ),
+      );
     }
   };
 

package/src/channels/types.ts

@@ -145,6 +145,22 @@ export function supportsHostProxy(
   return false;
 }
 
+/**
+ * Whether the interface can service host_browser frames via the
+ * ChromeExtensionRegistry (WebSocket) when an extension connection exists.
+ *
+ * Returns `true` for interfaces where the daemon should set
+ * `hostBrowserSenderOverride` and provision a `HostBrowserProxy` when the
+ * guardian has an active extension connection — currently `chrome-extension`
+ * and `macos`.
+ *
+ * Use this instead of hard-coding interface checks so new desktop interfaces
+ * only need to be added in one place.
+ */
+export function canServiceRegistryBrowser(id: InterfaceId): boolean {
+  return id === "chrome-extension" || id === "macos";
+}
+
 export interface TurnInterfaceContext {
   userMessageInterface: InterfaceId;
   assistantMessageInterface: InterfaceId;

package/src/cli/__tests__/run-assistant-command.ts

@@ -25,10 +25,20 @@ export async function runAssistantCommandFull(
 
   const stdoutChunks: string[] = [];
   const originalWrite = process.stdout.write;
-  process.stdout.write = ((chunk: string | Uint8Array) => {
+  // Override must invoke the callback (when provided) so that `Writable` streams
+  // piped into `process.stdout` (e.g. pino's CLI destination) can drain.
+  // Without this, only the first write lands and subsequent writes hang in
+  // backpressure. The second arg can be either an encoding string or the callback.
+  process.stdout.write = ((
+    chunk: string | Uint8Array,
+    encoding?: unknown,
+    cb?: (err?: Error | null) => void,
+  ) => {
     stdoutChunks.push(
       typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk),
     );
+    const callback = typeof encoding === "function" ? encoding : cb;
+    if (typeof callback === "function") callback();
     return true;
   }) as typeof process.stdout.write;