@vellumai/assistant 0.5.6 → 0.5.8

package/src/calls/call-controller.ts

@@ -9,8 +9,10 @@
  */
 
 import { getGatewayInternalBaseUrl } from "../config/env.js";
+import { loadConfig } from "../config/loader.js";
 import type { TrustContext } from "../daemon/conversation-runtime-assembly.js";
 import type { ServerMessage } from "../daemon/message-protocol.js";
+import { getPublicBaseUrl } from "../inbound/public-ingress-urls.js";
 import {
   expireCanonicalGuardianRequest,
   getCanonicalRequestByPendingQuestionId,
@@ -22,6 +24,7 @@ import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
 import { mintDaemonDeliveryToken } from "../runtime/auth/token-service.js";
 import { computeToolApprovalDigest } from "../security/tool-approval-digest.js";
 import { getLogger } from "../util/logger.js";
+import { createStreamingEntry } from "./audio-store.js";
 import {
   getMaxCallDurationMs,
   getSilenceTimeoutMs,
@@ -42,10 +45,12 @@ import {
   updateCallSession,
 } from "./call-store.js";
 import { finalizeCall } from "./finalize-call.js";
+import { synthesizeWithFishAudio } from "./fish-audio-client.js";
 import { sendGuardianExpiryNotices } from "./guardian-action-sweep.js";
 import { dispatchGuardianQuestion } from "./guardian-dispatch.js";
 import type { RelayConnection } from "./relay-server.js";
 import type { PromptSpeakerContext } from "./speaker-identification.js";
+import { sanitizeForTts } from "./tts-text-sanitizer.js";
 import {
   ASK_GUARDIAN_CAPTURE_REGEX,
   CALL_OPENING_ACK_MARKER,
@@ -56,6 +61,7 @@ import {
   extractBalancedJson,
   stripInternalSpeechMarkers,
 } from "./voice-control-protocol.js";
+import { isFishAudioTts } from "./voice-quality.js";
 import {
   startVoiceTurn,
   type VoiceTurnHandle,
@@ -101,6 +107,8 @@ export class CallController {
   private task: string | null;
   /** True when the call session was created via the inbound path (no outbound task). */
   private isInbound: boolean;
+  /** When true, the disclosure announcement is skipped for this call. */
+  private skipDisclosure: boolean;
   /** Instructions queued while an LLM turn is in-flight or during pending guardian input */
   private pendingInstructions: string[] = [];
   /** Ensures the call opener is triggered at most once per call. */
@@ -131,6 +139,8 @@ export class CallController {
    * without blocking the caller.
    */
   private guardianUnavailableForCall = false;
+  /** Active Fish Audio session — tracked so interrupt handling can close it. */
+  private activeFishAbort: AbortController | null = null;
 
   constructor(
     callSessionId: string,
@@ -150,9 +160,10 @@ export class CallController {
     this.assistantId = opts?.assistantId ?? DAEMON_INTERNAL_ASSISTANT_ID;
     this.trustContext = opts?.trustContext ?? null;
 
-    // Resolve the conversation ID from the call session
+    // Resolve the conversation ID and skipDisclosure from the call session
     const session = getCallSession(callSessionId);
     this.conversationId = session?.conversationId ?? callSessionId;
+    this.skipDisclosure = session?.skipDisclosure ?? false;
 
     this.startDurationTimer();
     this.resetSilenceTimer();
@@ -340,6 +351,11 @@ export class CallController {
     const wasSpeaking = this.state === "speaking";
     this.abortCurrentTurn();
     this.llmRunVersion++;
+    // Cancel in-flight Fish Audio synthesis on barge-in
+    if (this.activeFishAbort) {
+      this.activeFishAbort.abort();
+      this.activeFishAbort = null;
+    }
     // Explicitly terminate the in-progress TTS turn so the relay can
     // immediately hand control back to the caller after barge-in.
     if (wasSpeaking) {
@@ -370,6 +386,10 @@ export class CallController {
     this.pendingInstructions = [];
     this.llmRunVersion++;
     this.abortCurrentTurn();
+    if (this.activeFishAbort) {
+      this.activeFishAbort.abort();
+      this.activeFishAbort = null;
+    }
     this.currentTurnPromise = null;
     unregisterCallController(this.callSessionId);
 
@@ -516,24 +536,45 @@ export class CallController {
     runVersion: number,
     runSignal: AbortSignal,
   ): Promise<string> {
+    // Fish Audio TTS routing: when configured, buffer text by sentence
+    // boundaries and synthesize via Fish Audio instead of streaming text
+    // tokens for ElevenLabs TTS.
+    const config = loadConfig();
+    const useFishAudio = isFishAudioTts(config);
+
     // Buffer incoming tokens so we can strip control markers ([ASK_GUARDIAN:...], [END_CALL])
     // before they reach TTS. We hold text whenever an unmatched '[' appears, since it
     // could be the start of a control marker.
     let ttsBuffer = "";
     let fullResponseText = "";
 
+    // When using Fish Audio, we accumulate all text and synthesize
+    // the complete response at the end of the turn (better prosody).
+    let fishAudioTextBuffer = "";
+
+    /** Emit a chunk of safe text to the appropriate TTS backend. */
+    const emitSafeChunk = (safeText: string): void => {
+      const cleaned = sanitizeForTts(safeText);
+      if (cleaned.length === 0) return;
+      if (useFishAudio) {
+        fishAudioTextBuffer += cleaned;
+      } else {
+        this.relay.sendTextToken(cleaned, false);
+      }
+    };
+
     const flushSafeText = (): void => {
       if (!this.isCurrentRun(runVersion)) return;
       if (ttsBuffer.length === 0) return;
       const bracketIdx = ttsBuffer.indexOf("[");
       if (bracketIdx === -1) {
         // No bracket at all — safe to flush everything
-        this.relay.sendTextToken(ttsBuffer, false);
+        emitSafeChunk(ttsBuffer);
         ttsBuffer = "";
       } else {
         // Flush everything before the bracket
         if (bracketIdx > 0) {
-          this.relay.sendTextToken(ttsBuffer.slice(0, bracketIdx), false);
+          emitSafeChunk(ttsBuffer.slice(0, bracketIdx));
           ttsBuffer = ttsBuffer.slice(bracketIdx);
         }
 
@@ -547,10 +588,10 @@ export class CallController {
           // Not a control marker prefix — flush up to the next '[' (if any)
           const nextBracket = ttsBuffer.indexOf("[", 1);
           if (nextBracket === -1) {
-            this.relay.sendTextToken(ttsBuffer, false);
+            emitSafeChunk(ttsBuffer);
             ttsBuffer = "";
           } else {
-            this.relay.sendTextToken(ttsBuffer.slice(0, nextBracket), false);
+            emitSafeChunk(ttsBuffer.slice(0, nextBracket));
             ttsBuffer = ttsBuffer.slice(nextBracket);
           }
         }
@@ -585,6 +626,7 @@ export class CallController {
         trustContext: this.trustContext ?? undefined,
         isInbound: this.isInbound,
         task: this.task,
+        skipDisclosure: this.skipDisclosure,
         onTextDelta,
         onComplete,
         onError,
@@ -625,10 +667,50 @@ export class CallController {
     // Final sweep: strip any remaining control markers from the buffer
     ttsBuffer = stripInternalSpeechMarkers(ttsBuffer);
     if (ttsBuffer.length > 0) {
-      this.relay.sendTextToken(ttsBuffer, false);
+      emitSafeChunk(ttsBuffer);
+    }
+
+    // When using Fish Audio, synthesize the complete response text in a
+    // single REST API call. The full text gives Fish Audio better context
+    // for prosody and intonation. Audio streams back via chunked transfer
+    // encoding and is forwarded to Twilio as it arrives.
+    const sanitizedFishText = sanitizeForTts(fishAudioTextBuffer.trim());
+    if (useFishAudio && sanitizedFishText.length > 0) {
+      if (!this.isCurrentRun(runVersion)) return fullResponseText;
+      let handle: ReturnType<typeof createStreamingEntry> | null = null;
+      try {
+        const format = config.fishAudio.format ?? "mp3";
+        handle = createStreamingEntry(format as "mp3" | "wav" | "opus");
+        const baseUrl = getPublicBaseUrl(config);
+        const url = `${baseUrl}/v1/audio/${handle.audioId}`;
+        this.relay.sendPlayUrl(url);
+        const abortController = new AbortController();
+        this.activeFishAbort = abortController;
+        await synthesizeWithFishAudio(
+          sanitizedFishText,
+          config.fishAudio,
+          {
+            onChunk: (chunk) => handle!.push(chunk),
+            signal: abortController.signal,
+          },
+        );
+      } catch (err) {
+        if (err instanceof DOMException && err.name === "AbortError") {
+          log.debug("Fish Audio synthesis aborted (barge-in)");
+        } else {
+          log.error({ err }, "Fish Audio synthesis failed — skipping");
+        }
+      } finally {
+        this.activeFishAbort = null;
+        handle?.finalize();
+      }
     }
 
-    // Signal end of this turn's speech
+    // Signal end of this turn's speech.  An empty token with `last: true`
+    // tells ConversationRelay to start listening — it does NOT trigger TTS
+    // synthesis.  This is required even when Fish Audio handled all audio
+    // playback, because ConversationRelay still needs the end-of-turn signal
+    // to transition from "assistant speaking" to "caller speaking" state.
     this.relay.sendTextToken("", true);
 
     // Mark the greeting's first response as awaiting ack
@@ -652,7 +734,7 @@ export class CallController {
     recordCallEvent(this.callSessionId, "assistant_spoke", {
       text: responseText,
     });
-    const spokenText = stripInternalSpeechMarkers(responseText).trim();
+    const spokenText = sanitizeForTts(stripInternalSpeechMarkers(responseText)).trim();
     if (spokenText.length > 0) {
       const session = getCallSession(this.callSessionId);
       if (session) {

package/src/calls/call-domain.ts

@@ -85,6 +85,7 @@ export type StartCallInput = {
   conversationId: string;
   assistantId?: string;
   callerIdentityMode?: "assistant_number" | "user_number";
+  skipDisclosure?: boolean;
 };
 
 export type CancelCallInput = {
@@ -364,6 +365,7 @@ export async function startCall(
     context: callContext,
     conversationId,
     callerIdentityMode,
+    skipDisclosure,
     assistantId = DAEMON_INTERNAL_ASSISTANT_ID,
   } = input;
 
@@ -440,6 +442,7 @@ export async function startCall(
       task: callContext ? `${task}\n\nContext: ${callContext}` : task,
       callerIdentityMode: identityResult.mode,
       callerIdentitySource: identityResult.source,
+      skipDisclosure,
       initiatedFromConversationId: conversationId,
     });
     sessionId = session.id;

package/src/calls/call-store.ts

@@ -41,6 +41,10 @@ const parseCallSession = createRowMapper<
   inviteGuardianName: "inviteGuardianName",
   callerIdentityMode: "callerIdentityMode",
   callerIdentitySource: "callerIdentitySource",
+  skipDisclosure: {
+    from: "skipDisclosure",
+    transform: (v: unknown) => v === 1,
+  },
   initiatedFromConversationId: "initiatedFromConversationId",
   startedAt: "startedAt",
   endedAt: "endedAt",
@@ -87,11 +91,13 @@ export function createCallSession(opts: {
   inviteGuardianName?: string;
   callerIdentityMode?: string;
   callerIdentitySource?: string;
+  skipDisclosure?: boolean;
   initiatedFromConversationId?: string;
 }): CallSession {
   const db = getDb();
   const now = Date.now();
-  const session = {
+  const skipDisclosure = opts.skipDisclosure ?? false;
+  const row = {
     id: uuid(),
     conversationId: opts.conversationId,
     provider: opts.provider,
@@ -106,6 +112,7 @@ export function createCallSession(opts: {
     inviteGuardianName: opts.inviteGuardianName ?? null,
     callerIdentityMode: opts.callerIdentityMode ?? null,
     callerIdentitySource: opts.callerIdentitySource ?? null,
+    skipDisclosure: skipDisclosure ? 1 : 0,
     initiatedFromConversationId: opts.initiatedFromConversationId ?? null,
     startedAt: null,
     endedAt: null,
@@ -113,8 +120,8 @@ export function createCallSession(opts: {
     createdAt: now,
     updatedAt: now,
   };
-  db.insert(callSessions).values(session).run();
-  return session;
+  db.insert(callSessions).values(row).run();
+  return { ...row, skipDisclosure };
 }
 
 export function getCallSession(id: string): CallSession | null {

package/src/calls/fish-audio-client.ts

@@ -0,0 +1,129 @@
+import type { FishAudioConfig } from "../config/schemas/fish-audio.js";
+import { credentialKey } from "../security/credential-key.js";
+import { getSecureKeyAsync } from "../security/secure-keys.js";
+import { getLogger } from "../util/logger.js";
+
+const log = getLogger("fish-audio-client");
+
+/** Timeout waiting for the first chunk from Fish Audio (ms). */
+const FIRST_CHUNK_TIMEOUT_MS = 10_000;
+
+/** Timeout waiting between consecutive chunks (ms). */
+const IDLE_TIMEOUT_MS = 5_000;
+
+// ---------------------------------------------------------------------------
+// Fish Audio REST API (POST /v1/tts)
+// ---------------------------------------------------------------------------
+
+interface SynthesizeOptions {
+  onChunk?: (chunk: Uint8Array) => void;
+  signal?: AbortSignal;
+}
+
+/**
+ * Synthesize text to audio using the Fish Audio REST API with the s2-pro
+ * model. Streams audio chunks via the optional `onChunk` callback as they
+ * arrive from the server's chunked transfer-encoded response. Returns the
+ * complete audio buffer when the response finishes.
+ *
+ * Pass an `AbortSignal` to cancel in-flight synthesis (e.g. on barge-in).
+ */
+export async function synthesizeWithFishAudio(
+  text: string,
+  config: FishAudioConfig,
+  options?: SynthesizeOptions,
+): Promise<Buffer> {
+  const apiKey = await getSecureKeyAsync(
+    credentialKey("fish-audio", "api_key"),
+  );
+  if (!apiKey) {
+    throw new Error(
+      "Fish Audio API key not configured. Store it via: assistant credentials set --service fish-audio --field api_key <key>",
+    );
+  }
+
+  const body = {
+    text,
+    reference_id: config.referenceId || undefined,
+    model: "s2-pro",
+    format: config.format,
+    mp3_bitrate: 192,
+    chunk_length: config.chunkLength,
+    normalize: true,
+    latency: config.latency,
+    temperature: 1.0,
+    prosody: config.speed !== 1.0 ? { speed: config.speed } : undefined,
+  };
+
+  log.info(
+    {
+      referenceId: config.referenceId,
+      format: config.format,
+      textLength: text.length,
+    },
+    "Starting Fish Audio synthesis",
+  );
+
+  const response = await fetch("https://api.fish.audio/v1/tts", {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${apiKey}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify(body),
+    signal: options?.signal,
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(`Fish Audio API error (${response.status}): ${errorText}`);
+  }
+
+  if (!response.body) {
+    throw new Error("Fish Audio API returned no body");
+  }
+
+  const chunks: Uint8Array[] = [];
+  const reader = response.body.getReader();
+  let isFirstChunk = true;
+
+  try {
+    while (true) {
+      const timeoutMs = isFirstChunk ? FIRST_CHUNK_TIMEOUT_MS : IDLE_TIMEOUT_MS;
+      let timerId: ReturnType<typeof setTimeout>;
+      const timeout = new Promise<never>((_, reject) => {
+        timerId = setTimeout(
+          () => reject(new Error(`Fish Audio read timed out after ${timeoutMs}ms`)),
+          timeoutMs,
+        );
+      });
+      let done: boolean;
+      let value: Uint8Array | undefined;
+      try {
+        ({ done, value } = await Promise.race([reader.read(), timeout]));
+      } finally {
+        clearTimeout(timerId!);
+      }
+      if (done) break;
+      if (value) {
+        isFirstChunk = false;
+        chunks.push(value);
+        options?.onChunk?.(value);
+      }
+    }
+  } catch (err) {
+    try { await reader.cancel(); } catch { /* Ignore cancellation errors */ }
+    throw err;
+  }
+
+  const totalLength = chunks.reduce((sum, c) => sum + c.byteLength, 0);
+  const merged = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    merged.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+
+  log.debug({ bytes: totalLength }, "Fish Audio synthesis complete");
+  return Buffer.from(merged);
+}

package/src/calls/relay-server.ts

@@ -139,6 +139,12 @@ export interface RelayEndMessage {
   handoffData?: string;
 }
 
+export interface RelayPlayMessage {
+  type: "play";
+  source: string;
+  interruptible: boolean;
+}
+
 // ── WebSocket data type ──────────────────────────────────────────────
 
 export interface RelayWebSocketData {
@@ -323,6 +329,27 @@ export class RelayConnection {
     }
   }
 
+  /**
+   * Send a play-audio URL to the caller. Used when the assistant handles
+   * TTS synthesis itself (e.g. Fish Audio) instead of relying on
+   * ConversationRelay's built-in TTS.
+   */
+  sendPlayUrl(url: string): void {
+    const message: RelayPlayMessage = {
+      type: "play",
+      source: url,
+      interruptible: true,
+    };
+    try {
+      this.ws.send(JSON.stringify(message));
+    } catch (err) {
+      log.error(
+        { err, callSessionId: this.callSessionId },
+        "Failed to send play URL",
+      );
+    }
+  }
+
   /**
    * End the ConversationRelay session.
    */

package/src/calls/stt-hints.ts

@@ -0,0 +1,189 @@
+import {
+  findContactByAddress,
+  findGuardianForChannel,
+  listContacts,
+  listGuardianChannels,
+} from "../contacts/contact-store.js";
+import { getAssistantName } from "../daemon/identity-helpers.js";
+import { DEFAULT_USER_REFERENCE, resolveGuardianName } from "../prompts/user-reference.js";
+import { getLogger } from "../util/logger.js";
+
+const logger = getLogger("stt-hints");
+
+export interface SttHintsInput {
+  staticHints: string[];
+  assistantName: string | null;
+  guardianName: string | null;
+  taskDescription: string | null;
+  targetContactName: string | null;
+  callerContactName: string | null;
+  inviteFriendName: string | null;
+  inviteGuardianName: string | null;
+  recentContactNames: string[];
+}
+
+const MAX_HINTS_LENGTH = 500;
+
+/**
+ * Assemble STT vocabulary hints from multiple sources into a single
+ * comma-separated string suitable for speech-to-text provider hint APIs.
+ *
+ * Pure function — no DB or filesystem dependencies.
+ */
+export function buildSttHints(input: SttHintsInput): string {
+  const hints: string[] = [...input.staticHints];
+
+  if (input.assistantName != null && input.assistantName.trim().length > 0) {
+    hints.push(input.assistantName.trim());
+  }
+
+  if (
+    input.guardianName != null &&
+    input.guardianName.trim().length > 0 &&
+    input.guardianName.trim() !== DEFAULT_USER_REFERENCE
+  ) {
+    hints.push(input.guardianName.trim());
+  }
+
+  if (input.inviteFriendName != null && input.inviteFriendName.trim().length > 0) {
+    hints.push(input.inviteFriendName.trim());
+  }
+
+  if (input.inviteGuardianName != null && input.inviteGuardianName.trim().length > 0) {
+    hints.push(input.inviteGuardianName.trim());
+  }
+
+  if (input.targetContactName != null && input.targetContactName.trim().length > 0) {
+    hints.push(input.targetContactName.trim());
+  }
+
+  if (input.callerContactName != null && input.callerContactName.trim().length > 0) {
+    hints.push(input.callerContactName.trim());
+  }
+
+  // Extract potential proper nouns from task description.
+  // Split on sentence boundaries, then for each sentence take words
+  // after the first that start with an uppercase letter.
+  if (input.taskDescription != null && input.taskDescription.trim().length > 0) {
+    // Split on sentence-ending punctuation followed by whitespace, but avoid
+    // splitting on periods after common abbreviations (Dr., Mr., etc.) so that
+    // names like "Dr. Smith" aren't fragmented and dropped by the first-word skip.
+    const sentences = input.taskDescription.split(
+      /(?<!\b(?:Mr|Mrs|Ms|Dr|Jr|Sr|St|Rev|Prof|Gen|Sgt|Lt|Col))[.]\s+|[!?]\s+/,
+    );
+    for (const sentence of sentences) {
+      const words = sentence.trim().split(/\s+/);
+      // Skip the first word (always capitalized at sentence start)
+      for (let i = 1; i < words.length; i++) {
+        // Use Unicode-aware \p{L} to preserve accented/non-Latin letters (José, Łukasz, etc.)
+        const word = words[i].replace(/[^\p{L}'-]/gu, "");
+        if (word.length > 0 && /^\p{Lu}/u.test(word)) {
+          hints.push(word);
+        }
+      }
+    }
+  }
+
+  hints.push(...input.recentContactNames);
+
+  // Deduplicate (case-insensitive), filter empty/whitespace-only, trim each
+  const seen = new Set<string>();
+  const deduped: string[] = [];
+  for (const hint of hints) {
+    const trimmed = hint.trim();
+    if (trimmed.length === 0) continue;
+    const key = trimmed.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    deduped.push(trimmed);
+  }
+
+  const joined = deduped.join(",");
+
+  if (joined.length <= MAX_HINTS_LENGTH) {
+    return joined;
+  }
+
+  // Truncate at the last comma before the limit to avoid partial words
+  const truncated = joined.slice(0, MAX_HINTS_LENGTH);
+  const lastComma = truncated.lastIndexOf(",");
+  if (lastComma === -1) {
+    // Single hint that exceeds the limit — return it truncated
+    return truncated;
+  }
+  return truncated.slice(0, lastComma);
+}
+
+/**
+ * Wire real data sources (contacts DB, identity helpers, config) into
+ * {@link buildSttHints}. All DB lookups are best-effort — errors are
+ * logged but never propagate so hints can never fail a call.
+ */
+export function resolveCallHints(
+  session: {
+    task: string | null;
+    toNumber: string;
+    fromNumber: string;
+    direction: "inbound" | "outbound";
+    inviteFriendName: string | null;
+    inviteGuardianName: string | null;
+  } | null,
+  staticHints: string[],
+): string {
+  const assistantName = getAssistantName();
+
+  // Look up the guardian contact for a displayName fallback (mirrors relay-server pattern)
+  let guardianDisplayName: string | undefined;
+  try {
+    const voiceGuardian = findGuardianForChannel("phone");
+    const guardianChannels = voiceGuardian ? null : listGuardianChannels();
+    const guardianContact = voiceGuardian?.contact ?? guardianChannels?.contact;
+    guardianDisplayName = guardianContact?.displayName;
+  } catch (err) {
+    logger.warn({ err }, "Failed to look up guardian contact for STT hints");
+  }
+  const guardianName = resolveGuardianName(guardianDisplayName);
+
+  let targetContactName: string | null = null;
+  let callerContactName: string | null = null;
+  let recentContactNames: string[] = [];
+
+  // For inbound calls, fromNumber is the caller (the interesting party);
+  // toNumber is the assistant's own Twilio number (not useful for contact lookup).
+  // For outbound calls, toNumber is who we're calling.
+  try {
+    if (session) {
+      const otherPartyNumber =
+        session.direction === "inbound" ? session.fromNumber : session.toNumber;
+      const otherPartyContact = findContactByAddress("phone", otherPartyNumber);
+      if (otherPartyContact) {
+        if (session.direction === "inbound") {
+          callerContactName = otherPartyContact.displayName;
+        } else {
+          targetContactName = otherPartyContact.displayName;
+        }
+      }
+    }
+  } catch (err) {
+    logger.warn({ err }, "Failed to look up contact for STT hints");
+  }
+
+  try {
+    const recentContacts = listContacts(15);
+    recentContactNames = recentContacts.map((c) => c.displayName);
+  } catch (err) {
+    logger.warn({ err }, "Failed to list recent contacts for STT hints");
+  }
+
+  return buildSttHints({
+    staticHints,
+    assistantName,
+    guardianName,
+    taskDescription: session?.task ?? null,
+    targetContactName,
+    callerContactName,
+    inviteFriendName: session?.inviteFriendName ?? null,
+    inviteGuardianName: session?.inviteGuardianName ?? null,
+    recentContactNames,
+  });
+}