@vellumai/assistant 0.10.4 → 0.10.5-dev.202607022329.d89bf2f

package/src/calls/call-speech-output.ts

@@ -8,7 +8,8 @@
  *
  * Two output paths (determined by the catalog's `callMode`):
  * - **Native**: `callMode: "native-twilio"` — text is sent via
- *   `sendTextToken()` for Twilio's built-in TTS engine.
+ *   `sendTextToken()`, which the media-stream transport re-synthesizes
+ *   through daemon TTS.
  * - **Synthesized**: `callMode: "synthesized-play"` — text is synthesized
  *   via the provider API, streamed through the audio store, and played
  *   via `sendPlayUrl()`.
@@ -17,11 +18,15 @@
 import { loadConfig } from "../config/loader.js";
 import { getPublicBaseUrl } from "../inbound/public-ingress-urls.js";
 import { getCatalogProvider } from "../tts/provider-catalog.js";
+import { getTtsProvider } from "../tts/provider-registry.js";
 import type { TtsProvider, TtsProviderId } from "../tts/types.js";
 import { getLogger } from "../util/logger.js";
 import { createStreamingEntry } from "./audio-store.js";
 import type { CallTransport } from "./call-transport.js";
-import { resolveCallTtsProvider } from "./resolve-call-tts-provider.js";
+import {
+  findPlayableTelephonyTtsFallback,
+  resolveCallTtsProvider,
+} from "./resolve-call-tts-provider.js";
 
 const log = getLogger("call-speech-output");
 
@@ -42,7 +47,7 @@ const log = getLogger("call-speech-output");
  * Twilio before the session ends. Interactive mid-call callers can
  * fire-and-forget with `void speakSystemPrompt(...)`.
  */
-export function speakSystemPrompt(
+export async function speakSystemPrompt(
   relay: CallTransport,
   text: string,
 ): Promise<void> {
@@ -50,14 +55,15 @@ export function speakSystemPrompt(
   // the audio store entry contains PCM that audioBufferToFrames can
   // transcode to mu-law. Without this, compressed formats (mp3, opus)
   // are fetched by processFetchUrlItem and produce garbled audio.
-  const { provider, useSynthesizedPath, audioFormat } = resolveCallTtsProvider({
-    preferWav: relay.requiresWavAudio,
-  });
+  const { provider, useSynthesizedPath, audioFormat } =
+    await resolveCallTtsProvider({
+      preferWav: relay.requiresWavAudio,
+    });
 
   if (!useSynthesizedPath || !provider) {
-    // Native path — send text for Twilio's built-in TTS.
+    // Native path — send text tokens through the transport.
     relay.sendTextToken(text, true);
-    return Promise.resolve();
+    return;
   }
 
   // Synthesized path — synthesize audio and send play URL.
@@ -70,20 +76,26 @@ export function speakSystemPrompt(
 
 /**
  * Synthesize text via a streaming TTS provider and send the play URL
- * to the relay.
+ * to the transport.
  *
- * On synthesis failure the behavior depends on the provider:
- * - Providers with a native Twilio TTS fallback (e.g. Fish Audio) fall
- *   back to `sendTextToken(text)` so the caller still hears the message.
- * - Providers without a native fallback (e.g. Deepgram) log the error
- *   and send only an empty end-of-turn signal — the caller hears nothing
- *   but the relay transitions back to listening state.
+ * On synthesis failure the behavior depends on the transport:
+ * - WAV-requiring transports (media-stream) retry once with a playable
+ *   fallback provider — native token TTS cannot rescue the prompt there
+ *   because text tokens are themselves synthesized via the same provider
+ *   path. When no fallback exists (or the retry also fails), only an
+ *   empty end-of-turn signal is sent so the transport transitions back
+ *   to listening state.
+ * - On non-WAV transports, providers with a native Twilio TTS fallback
+ *   (e.g. Fish Audio) fall back to `sendTextToken(text)` so the caller
+ *   still hears the message; providers without one (e.g. Deepgram) log
+ *   the error and send only the end-of-turn signal.
  */
 async function synthesizeAndPlay(
   relay: CallTransport,
   provider: TtsProvider,
   text: string,
   format: "mp3" | "wav" | "opus",
+  isFallbackRetry = false,
 ): Promise<void> {
   let handle: ReturnType<typeof createStreamingEntry> | null = null;
   let playUrlSent = false;
@@ -140,11 +152,11 @@ async function synthesizeAndPlay(
     }
 
     // Signal end of this turn's speech.  An empty token with `last: true`
-    // tells ConversationRelay to start listening — it does NOT trigger TTS
+    // tells the transport to start listening — it does NOT trigger TTS
     // synthesis.  This is required even when a synthesized provider handled
-    // all audio playback, because ConversationRelay still needs the
-    // end-of-turn signal to transition from "assistant speaking" to
-    // "caller speaking" state.
+    // all audio playback, because the transport still needs the end-of-turn
+    // signal to transition from "assistant speaking" to "caller speaking"
+    // state.
     relay.sendTextToken("", true);
   } catch (err) {
     // Extract error class and code for diagnosable log entries.
@@ -154,6 +166,49 @@ async function synthesizeAndPlay(
         ? (err as Error & { code?: string }).code
         : undefined;
 
+    if (relay.requiresWavAudio) {
+      // WAV-requiring transport (media-stream): native token TTS routes
+      // back through the same synthesis path, so retry once with a
+      // playable fallback provider before degrading to end-of-turn only.
+      if (!isFallbackRetry) {
+        const fallbackId = await findPlayableTelephonyTtsFallback(
+          provider.id as TtsProviderId,
+        );
+        const fallbackProvider = fallbackId
+          ? lookupRegisteredProvider(fallbackId)
+          : null;
+        if (fallbackProvider) {
+          log.warn(
+            {
+              err,
+              provider: provider.id,
+              fallbackProvider: fallbackProvider.id,
+              errName,
+              errCode,
+            },
+            "System prompt TTS synthesis failed — retrying with fallback provider",
+          );
+          handle?.finalize();
+          handle = null;
+          return await synthesizeAndPlay(
+            relay,
+            fallbackProvider,
+            text,
+            format,
+            true,
+          );
+        }
+      }
+      log.error(
+        { err, provider: provider.id, errName, errCode },
+        "System prompt TTS synthesis failed on WAV-requiring transport — sending end-of-turn only",
+      );
+      // Send the end-of-turn signal so the transport transitions from
+      // "assistant speaking" to "caller speaking" state.
+      relay.sendTextToken("", true);
+      return;
+    }
+
     // `allowNativeFallback` controls whether the system prompt text
     // should be sent via native Twilio token-based TTS when synthesis
     // fails. When false (e.g. Deepgram), the design choice is to
@@ -167,10 +222,10 @@ async function synthesizeAndPlay(
         { err, provider: provider.id, errName, errCode },
         "System prompt TTS synthesis failed — native fallback disabled for this provider",
       );
-      // Send the end-of-turn signal so ConversationRelay transitions from
-      // "assistant speaking" to "caller speaking" state. Without this, the
-      // relay hangs waiting for the prompt to complete and the caller
-      // cannot interact.
+      // Send the end-of-turn signal so the transport transitions from
+      // "assistant speaking" to "caller speaking" state. Without this,
+      // the session hangs waiting for the prompt to complete and the
+      // caller cannot interact.
       relay.sendTextToken("", true);
       return;
     }
@@ -188,3 +243,12 @@ async function synthesizeAndPlay(
     handle?.finalize();
   }
 }
+
+/** Look up a registered provider, returning null instead of throwing. */
+function lookupRegisteredProvider(id: TtsProviderId): TtsProvider | null {
+  try {
+    return getTtsProvider(id);
+  } catch {
+    return null;
+  }
+}

package/src/calls/call-store.ts

@@ -401,6 +401,7 @@ export function claimCallback(
 ): string | null {
   // Clear any expired orphaned claims so they can be reprocessed
   rawRun(
+    "calls:claimCallback:clearExpired",
     `DELETE FROM processed_callbacks WHERE dedupe_key = ? AND created_at < ?`,
     dedupeKey,
     Date.now() - CLAIM_EXPIRY_MS,
@@ -408,6 +409,7 @@ export function claimCallback(
 
   const claimId = uuid();
   const changes = rawRun(
+    "calls:claimCallback:insert",
     `INSERT OR IGNORE INTO processed_callbacks (id, dedupe_key, call_session_id, claim_id, created_at) VALUES (?, ?, ?, ?, ?)`,
     uuid(),
     dedupeKey,
@@ -427,6 +429,7 @@ export function claimCallback(
  */
 export function releaseCallbackClaim(dedupeKey: string, claimId: string): void {
   rawRun(
+    "calls:releaseCallbackClaim",
     `DELETE FROM processed_callbacks WHERE dedupe_key = ? AND claim_id = ?`,
     dedupeKey,
     claimId,
@@ -454,6 +457,7 @@ export function finalizeCallbackClaim(
   // Set created_at far in the future so expiry check never matches
   const NEVER_EXPIRE = Date.now() + 100 * 365 * 24 * 60 * 60 * 1000; // ~100 years
   const changes = rawRun(
+    "calls:finalizeCallbackClaim",
     `UPDATE processed_callbacks SET created_at = ? WHERE dedupe_key = ? AND claim_id = ?`,
     NEVER_EXPIRE,
     dedupeKey,

package/src/calls/call-transport.ts

@@ -2,9 +2,9 @@
  * Transport interface consumed by CallController for sending voice output
  * and controlling call lifecycle.
  *
- * Decouples the controller from any specific wire protocol (e.g.
- * ConversationRelay) so that alternative transports (media-stream, etc.)
- * can be introduced without modifying controller logic.
+ * Decouples the controller from any specific wire protocol (e.g. Twilio
+ * Media Streams) so that alternative transports can be introduced without
+ * modifying controller logic.
  */
 
 // ── Transport interface ──────────────────────────────────────────────
@@ -30,12 +30,6 @@ export interface CallTransport {
    */
   endSession(reason?: string): void;
 
-  /**
-   * Return the current connection-level state. The controller uses this
-   * to suppress silence nudges during guardian wait states.
-   */
-  getConnectionState(): string;
-
   /**
    * When true, the transport requires WAV (PCM) audio for playback.
    *
@@ -45,33 +39,25 @@ export interface CallTransport {
    * WAV from TTS providers and the audio store.
    */
   readonly requiresWavAudio?: boolean;
-}
-
-// ── ConversationRelay adapter ────────────────────────────────────────
-
-import type { RelayConnection } from "./relay-server.js";
-
-/**
- * Adapts a RelayConnection (Twilio ConversationRelay WebSocket) to the
- * CallTransport interface. All calls are forwarded 1:1 — no behavioral
- * changes from the pre-abstraction path.
- */
-export class ConversationRelayTransport implements CallTransport {
-  constructor(private relay: RelayConnection) {}
 
-  sendTextToken(token: string, last: boolean): void {
-    this.relay.sendTextToken(token, last);
-  }
-
-  sendPlayUrl(url: string): void {
-    this.relay.sendPlayUrl(url);
-  }
-
-  endSession(reason?: string): void {
-    this.relay.endSession(reason);
-  }
+  /**
+   * Arm a one-shot callback invoked when the transport sends the first
+   * audio frame of queued playback to the caller.
+   *
+   * Transports that buffer text and synthesize asynchronously (e.g.
+   * media-stream) implement this so the controller can flip to the
+   * `speaking` state only when real outbound audio starts, rather than
+   * when tokens are merely buffered. Passing `null` disarms the signal.
+   * Transports that emit audio immediately may omit this.
+   */
+  setAudioStartCallback?(cb: (() => void) | null): void;
 
-  getConnectionState(): string {
-    return this.relay.getConnectionState();
-  }
+  /**
+   * Discard any buffered, not-yet-queued text held by the transport.
+   *
+   * Called by the controller when it aborts an in-flight turn so the
+   * aborted turn's unsent text cannot leak into the next turn's
+   * synthesis. Transports that don't buffer text may omit this.
+   */
+  discardPendingText?(): void;
 }

package/src/calls/fish-audio-client.ts

@@ -18,6 +18,8 @@ const IDLE_TIMEOUT_MS = 5_000;
 interface SynthesizeOptions {
   onChunk?: (chunk: Uint8Array) => void;
   signal?: AbortSignal;
+  /** PCM sample rate to request (Fish `sample_rate`). Omitted = API default. */
+  sampleRate?: number;
 }
 
 /**
@@ -47,6 +49,7 @@ export async function synthesizeWithFishAudio(
     reference_id: config.referenceId || undefined,
     model: "s2-pro",
     format: config.format,
+    sample_rate: options?.sampleRate,
     mp3_bitrate: 192,
     chunk_length: config.chunkLength,
     normalize: true,

package/src/calls/gateway-invite-reader.ts

@@ -0,0 +1,98 @@
+/**
+ * Gateway-backed voice-invite reader for the phone channel.
+ *
+ * Resolves voice-invite state from the gateway's canonical `ingress_invites`
+ * table via the `get_active_voice_invite` / `redeem_voice_invite` IPC routes.
+ *
+ * Failure semantics are asymmetric by design:
+ * - Detection fails SOFT — `null` on ANY failure (transport error,
+ *   `undefined`, malformed shape), so the caller falls to the unverified
+ *   path rather than stalling call setup.
+ * - Redemption fails CLOSED — any IPC failure is the generic
+ *   `invalid_or_expired` outcome. There is no local fallback: the gateway
+ *   row is the single lifecycle authority for voice invites.
+ */
+
+import {
+  type ActiveVoiceInvite,
+  ActiveVoiceInviteSchema,
+  InviteRedemptionOutcomeSchema,
+} from "@vellumai/gateway-client";
+import { z } from "zod";
+
+import { ipcCall } from "../ipc/gateway-client.js";
+
+// Short IPC timeout so detection resolves promptly rather than stalling call
+// setup on a gateway that accepts the socket but hangs.
+const DETECTION_IPC_TIMEOUT_MS = 2_000;
+// Redemption performs the atomic claim plus ACL/mirror writes; give it more
+// headroom — a timeout still fails closed.
+const REDEMPTION_IPC_TIMEOUT_MS = 10_000;
+
+const GetActiveVoiceInviteResponseSchema = z.object({
+  invite: ActiveVoiceInviteSchema.nullable(),
+});
+
+const RedeemVoiceInviteResponseSchema = z.discriminatedUnion("ok", [
+  z.object({ ok: z.literal(true), outcome: InviteRedemptionOutcomeSchema }),
+  z.object({ ok: z.literal(false), reason: z.literal("invalid_or_expired") }),
+]);
+
+export type GatewayVoiceRedemptionResult = z.infer<
+  typeof RedeemVoiceInviteResponseSchema
+>;
+
+const REDEMPTION_FAILURE: GatewayVoiceRedemptionResult = {
+  ok: false,
+  reason: "invalid_or_expired",
+};
+
+/**
+ * Resolve the active voice invite awaiting `fromNumber`, or `null` when there
+ * is none — or when the gateway can't answer (fail-soft).
+ */
+export async function getActiveVoiceInvite(
+  fromNumber: string | undefined,
+): Promise<ActiveVoiceInvite | null> {
+  if (!fromNumber) {
+    return null;
+  }
+  try {
+    const result = await ipcCall(
+      "get_active_voice_invite",
+      { callerExternalUserId: fromNumber },
+      DETECTION_IPC_TIMEOUT_MS,
+    );
+    const parsed = GetActiveVoiceInviteResponseSchema.safeParse(result);
+    if (!parsed.success) {
+      return null;
+    }
+    return parsed.data.invite;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Redeem a spoken voice code for `fromNumber` at the gateway. Any transport
+ * failure or malformed response is the generic failure outcome (fail-closed).
+ */
+export async function redeemVoiceInviteViaGateway(
+  fromNumber: string,
+  code: string,
+): Promise<GatewayVoiceRedemptionResult> {
+  try {
+    const result = await ipcCall(
+      "redeem_voice_invite",
+      { callerExternalUserId: fromNumber, code },
+      REDEMPTION_IPC_TIMEOUT_MS,
+    );
+    const parsed = RedeemVoiceInviteResponseSchema.safeParse(result);
+    if (!parsed.success) {
+      return REDEMPTION_FAILURE;
+    }
+    return parsed.data;
+  } catch {
+    return REDEMPTION_FAILURE;
+  }
+}