@vellumai/assistant 0.5.7 → 0.5.9

package/src/calls/__tests__/tts-text-sanitizer.test.ts

@@ -0,0 +1,254 @@
+import { describe, expect, test } from "bun:test";
+
+import { sanitizeForTts } from "../tts-text-sanitizer.js";
+
+describe("sanitizeForTts", () => {
+  describe("markdown links", () => {
+    test("strips markdown links, keeping link text", () => {
+      expect(sanitizeForTts("Check [this link](https://example.com)")).toBe(
+        "Check this link",
+      );
+    });
+
+    test("handles multiple links", () => {
+      expect(
+        sanitizeForTts("See [foo](http://a.com) and [bar](http://b.com)"),
+      ).toBe("See foo and bar");
+    });
+
+    test("preserves Fish Audio S2 bracket annotations", () => {
+      expect(sanitizeForTts("Hello [laughter] world")).toBe(
+        "Hello [laughter] world",
+      );
+      expect(sanitizeForTts("[breath] ok")).toBe("[breath] ok");
+    });
+
+    test("handles URLs with balanced parentheses (e.g. Wikipedia)", () => {
+      expect(
+        sanitizeForTts(
+          "See [Function](https://en.wikipedia.org/wiki/Function_(mathematics))",
+        ),
+      ).toBe("See Function");
+    });
+
+    test("handles URLs with multiple balanced parentheses groups", () => {
+      expect(
+        sanitizeForTts("[link](http://example.com/a_(b)_c_(d))"),
+      ).toBe("link");
+    });
+  });
+
+  describe("bold and italic", () => {
+    test("strips bold (asterisks)", () => {
+      expect(sanitizeForTts("Hello **world**")).toBe("Hello world");
+    });
+
+    test("strips bold (underscores)", () => {
+      expect(sanitizeForTts("Hello __world__")).toBe("Hello world");
+    });
+
+    test("strips italic (asterisks)", () => {
+      expect(sanitizeForTts("Hello *world*")).toBe("Hello world");
+    });
+
+    test("strips italic (underscores)", () => {
+      expect(sanitizeForTts("Hello _world_")).toBe("Hello world");
+    });
+
+    test("strips bold+italic (asterisks)", () => {
+      expect(sanitizeForTts("Hello ***world***")).toBe("Hello world");
+    });
+
+    test("strips bold+italic (underscores)", () => {
+      expect(sanitizeForTts("Hello ___world___")).toBe("Hello world");
+    });
+
+    test("preserves arithmetic asterisks", () => {
+      expect(sanitizeForTts("5 * 3 = 15")).toBe("5 * 3 = 15");
+    });
+
+    test("preserves identifiers with underscores", () => {
+      expect(sanitizeForTts("The my_var variable")).toBe(
+        "The my_var variable",
+      );
+    });
+
+    test("preserves snake_case identifiers", () => {
+      expect(sanitizeForTts("use some_function_name here")).toBe(
+        "use some_function_name here",
+      );
+    });
+  });
+
+  describe("headers", () => {
+    test("strips h1", () => {
+      expect(sanitizeForTts("# Header\n\nSome text")).toBe(
+        "Header\n\nSome text",
+      );
+    });
+
+    test("strips h2", () => {
+      expect(sanitizeForTts("## Sub Header")).toBe("Sub Header");
+    });
+
+    test("strips h3 through h6", () => {
+      expect(sanitizeForTts("### H3")).toBe("H3");
+      expect(sanitizeForTts("#### H4")).toBe("H4");
+      expect(sanitizeForTts("##### H5")).toBe("H5");
+      expect(sanitizeForTts("###### H6")).toBe("H6");
+    });
+
+    test("does not strip # in middle of line", () => {
+      expect(sanitizeForTts("Issue #42")).toBe("Issue #42");
+    });
+  });
+
+  describe("code", () => {
+    test("strips code fences, keeping content", () => {
+      const input = "Here:\n```js\nconst x = 1;\n```\nDone.";
+      expect(sanitizeForTts(input)).toBe("Here:\nconst x = 1;\nDone.");
+    });
+
+    test("strips inline code backticks", () => {
+      expect(sanitizeForTts("Use `console.log` here")).toBe(
+        "Use console.log here",
+      );
+    });
+
+    test("preserves # comments inside code fences", () => {
+      const input = "Example:\n```python\n# This is a comment\nprint('hi')\n```\nDone.";
+      expect(sanitizeForTts(input)).toBe(
+        "Example:\n# This is a comment\nprint('hi')\nDone.",
+      );
+    });
+
+    test("preserves shell comments inside code fences", () => {
+      const input = "Run:\n```bash\n## Install deps\napt-get install curl\n```";
+      expect(sanitizeForTts(input)).toBe(
+        "Run:\n## Install deps\napt-get install curl\n",
+      );
+    });
+  });
+
+  describe("bullet markers", () => {
+    test("strips dash bullets", () => {
+      expect(sanitizeForTts("- First\n- Second")).toBe("First\nSecond");
+    });
+
+    test("strips asterisk bullets", () => {
+      expect(sanitizeForTts("* First\n* Second")).toBe("First\nSecond");
+    });
+
+    test("does not strip dashes mid-line", () => {
+      expect(sanitizeForTts("well-known fact")).toBe("well-known fact");
+    });
+  });
+
+  describe("emojis", () => {
+    test("strips simple emojis", () => {
+      expect(sanitizeForTts("Hello world 👋")).toBe("Hello world ");
+    });
+
+    test("strips compound emojis (ZWJ sequences)", () => {
+      // Family emoji: man + ZWJ + woman + ZWJ + girl
+      expect(sanitizeForTts("Family: 👨‍👩‍👧")).toBe("Family: ");
+    });
+
+    test("strips emojis with skin tone modifiers", () => {
+      expect(sanitizeForTts("Wave 👋🏽 hello")).toBe("Wave hello");
+    });
+
+    test("strips flag emojis", () => {
+      // Flags are regional indicator sequences (Extended_Pictographic)
+      expect(sanitizeForTts("Hello 🇺🇸 world")).toBe("Hello world");
+    });
+
+    test("strips emojis with variation selectors", () => {
+      // Heart with variation selector
+      expect(sanitizeForTts("Love ❤️ you")).toBe("Love you");
+    });
+
+    test("preserves numbers and currency", () => {
+      expect(sanitizeForTts("$100.50 and €200")).toBe("$100.50 and €200");
+    });
+
+    test("preserves punctuation", () => {
+      expect(sanitizeForTts("Hello, world! How are you?")).toBe(
+        "Hello, world! How are you?",
+      );
+    });
+  });
+
+  describe("whitespace collapsing", () => {
+    test("collapses multiple spaces to single space", () => {
+      expect(sanitizeForTts("Hello    world")).toBe("Hello world");
+    });
+
+    test("collapses multiple blank lines to single newline", () => {
+      expect(sanitizeForTts("Hello\n\n\n\nworld")).toBe("Hello\n\nworld");
+    });
+  });
+
+  describe("combined transformations", () => {
+    test("acceptance: Hello **world** with emoji", () => {
+      expect(sanitizeForTts("Hello **world** 👋")).toBe("Hello world ");
+    });
+
+    test("acceptance: markdown link", () => {
+      expect(
+        sanitizeForTts("Check [this link](https://example.com)"),
+      ).toBe("Check this link");
+    });
+
+    test("acceptance: arithmetic preserved", () => {
+      expect(sanitizeForTts("5 * 3 = 15")).toBe("5 * 3 = 15");
+    });
+
+    test("acceptance: header with text", () => {
+      expect(sanitizeForTts("# Header\n\nSome text")).toBe(
+        "Header\n\nSome text",
+      );
+    });
+
+    test("complex mixed markdown and emojis", () => {
+      const input =
+        "# Welcome 🎉\n\nHello **world**! Check [docs](http://x.com).\n\n- Item *one*\n- Item `two`";
+      const expected =
+        "Welcome \n\nHello world! Check docs.\n\nItem one\nItem two";
+      expect(sanitizeForTts(input)).toBe(expected);
+    });
+  });
+
+  describe("edge cases", () => {
+    test("empty string", () => {
+      expect(sanitizeForTts("")).toBe("");
+    });
+
+    test("already-clean text", () => {
+      const clean = "Hello, this is plain text.";
+      expect(sanitizeForTts(clean)).toBe(clean);
+    });
+
+    test("nested markdown (bold inside italic)", () => {
+      expect(sanitizeForTts("*Hello **world***")).toBe("Hello world");
+    });
+
+    test("partial markdown (unmatched asterisks)", () => {
+      expect(sanitizeForTts("Hello **world")).toBe("Hello **world");
+    });
+
+    test("idempotency: applying twice gives same result", () => {
+      const input = "# Hello **world** 👋\n\n- Item *one*\n- [link](http://x.com)";
+      const once = sanitizeForTts(input);
+      const twice = sanitizeForTts(once);
+      expect(twice).toBe(once);
+    });
+
+    test("preserves trailing whitespace for streaming chunks", () => {
+      // Streaming chunks must keep trailing spaces so word boundaries survive
+      // concatenation: "Hello " + "world" = "Hello world", not "Helloworld"
+      expect(sanitizeForTts("Hello ")).toBe("Hello ");
+      expect(sanitizeForTts("the quick ")).toBe("the quick ");
+    });
+  });
+});

package/src/calls/call-controller.ts

@@ -50,6 +50,7 @@ import { sendGuardianExpiryNotices } from "./guardian-action-sweep.js";
 import { dispatchGuardianQuestion } from "./guardian-dispatch.js";
 import type { RelayConnection } from "./relay-server.js";
 import type { PromptSpeakerContext } from "./speaker-identification.js";
+import { sanitizeForTts } from "./tts-text-sanitizer.js";
 import {
   ASK_GUARDIAN_CAPTURE_REGEX,
   CALL_OPENING_ACK_MARKER,
@@ -553,10 +554,12 @@ export class CallController {
 
     /** Emit a chunk of safe text to the appropriate TTS backend. */
     const emitSafeChunk = (safeText: string): void => {
+      const cleaned = sanitizeForTts(safeText);
+      if (cleaned.length === 0) return;
       if (useFishAudio) {
-        fishAudioTextBuffer += safeText;
+        fishAudioTextBuffer += cleaned;
       } else {
-        this.relay.sendTextToken(safeText, false);
+        this.relay.sendTextToken(cleaned, false);
       }
     };
 
@@ -671,7 +674,8 @@ export class CallController {
     // single REST API call. The full text gives Fish Audio better context
     // for prosody and intonation. Audio streams back via chunked transfer
     // encoding and is forwarded to Twilio as it arrives.
-    if (useFishAudio && fishAudioTextBuffer.trim().length > 0) {
+    const sanitizedFishText = sanitizeForTts(fishAudioTextBuffer.trim());
+    if (useFishAudio && sanitizedFishText.length > 0) {
       if (!this.isCurrentRun(runVersion)) return fullResponseText;
       let handle: ReturnType<typeof createStreamingEntry> | null = null;
       try {
@@ -683,7 +687,7 @@ export class CallController {
         const abortController = new AbortController();
         this.activeFishAbort = abortController;
         await synthesizeWithFishAudio(
-          fishAudioTextBuffer.trim(),
+          sanitizedFishText,
           config.fishAudio,
           {
             onChunk: (chunk) => handle!.push(chunk),
@@ -730,7 +734,7 @@ export class CallController {
     recordCallEvent(this.callSessionId, "assistant_spoke", {
       text: responseText,
     });
-    const spokenText = stripInternalSpeechMarkers(responseText).trim();
+    const spokenText = sanitizeForTts(stripInternalSpeechMarkers(responseText)).trim();
     if (spokenText.length > 0) {
       const session = getCallSession(this.callSessionId);
       if (session) {

package/src/calls/fish-audio-client.ts

@@ -87,21 +87,33 @@ export async function synthesizeWithFishAudio(
   const reader = response.body.getReader();
   let isFirstChunk = true;
 
-  while (true) {
-    const timeoutMs = isFirstChunk ? FIRST_CHUNK_TIMEOUT_MS : IDLE_TIMEOUT_MS;
-    const timeout = new Promise<never>((_, reject) =>
-      setTimeout(
-        () => reject(new Error(`Fish Audio read timed out after ${timeoutMs}ms`)),
-        timeoutMs,
-      ),
-    );
-    const { done, value } = await Promise.race([reader.read(), timeout]);
-    if (done) break;
-    if (value) {
-      isFirstChunk = false;
-      chunks.push(value);
-      options?.onChunk?.(value);
+  try {
+    while (true) {
+      const timeoutMs = isFirstChunk ? FIRST_CHUNK_TIMEOUT_MS : IDLE_TIMEOUT_MS;
+      let timerId: ReturnType<typeof setTimeout>;
+      const timeout = new Promise<never>((_, reject) => {
+        timerId = setTimeout(
+          () => reject(new Error(`Fish Audio read timed out after ${timeoutMs}ms`)),
+          timeoutMs,
+        );
+      });
+      let done: boolean;
+      let value: Uint8Array | undefined;
+      try {
+        ({ done, value } = await Promise.race([reader.read(), timeout]));
+      } finally {
+        clearTimeout(timerId!);
+      }
+      if (done) break;
+      if (value) {
+        isFirstChunk = false;
+        chunks.push(value);
+        options?.onChunk?.(value);
+      }
     }
+  } catch (err) {
+    try { await reader.cancel(); } catch { /* Ignore cancellation errors */ }
+    throw err;
   }
 
   const totalLength = chunks.reduce((sum, c) => sum + c.byteLength, 0);

package/src/calls/stt-hints.ts

@@ -0,0 +1,189 @@
+import {
+  findContactByAddress,
+  findGuardianForChannel,
+  listContacts,
+  listGuardianChannels,
+} from "../contacts/contact-store.js";
+import { getAssistantName } from "../daemon/identity-helpers.js";
+import { DEFAULT_USER_REFERENCE, resolveGuardianName } from "../prompts/user-reference.js";
+import { getLogger } from "../util/logger.js";
+
+const logger = getLogger("stt-hints");
+
+export interface SttHintsInput {
+  staticHints: string[];
+  assistantName: string | null;
+  guardianName: string | null;
+  taskDescription: string | null;
+  targetContactName: string | null;
+  callerContactName: string | null;
+  inviteFriendName: string | null;
+  inviteGuardianName: string | null;
+  recentContactNames: string[];
+}
+
+const MAX_HINTS_LENGTH = 500;
+
+/**
+ * Assemble STT vocabulary hints from multiple sources into a single
+ * comma-separated string suitable for speech-to-text provider hint APIs.
+ *
+ * Pure function — no DB or filesystem dependencies.
+ */
+export function buildSttHints(input: SttHintsInput): string {
+  const hints: string[] = [...input.staticHints];
+
+  if (input.assistantName != null && input.assistantName.trim().length > 0) {
+    hints.push(input.assistantName.trim());
+  }
+
+  if (
+    input.guardianName != null &&
+    input.guardianName.trim().length > 0 &&
+    input.guardianName.trim() !== DEFAULT_USER_REFERENCE
+  ) {
+    hints.push(input.guardianName.trim());
+  }
+
+  if (input.inviteFriendName != null && input.inviteFriendName.trim().length > 0) {
+    hints.push(input.inviteFriendName.trim());
+  }
+
+  if (input.inviteGuardianName != null && input.inviteGuardianName.trim().length > 0) {
+    hints.push(input.inviteGuardianName.trim());
+  }
+
+  if (input.targetContactName != null && input.targetContactName.trim().length > 0) {
+    hints.push(input.targetContactName.trim());
+  }
+
+  if (input.callerContactName != null && input.callerContactName.trim().length > 0) {
+    hints.push(input.callerContactName.trim());
+  }
+
+  // Extract potential proper nouns from task description.
+  // Split on sentence boundaries, then for each sentence take words
+  // after the first that start with an uppercase letter.
+  if (input.taskDescription != null && input.taskDescription.trim().length > 0) {
+    // Split on sentence-ending punctuation followed by whitespace, but avoid
+    // splitting on periods after common abbreviations (Dr., Mr., etc.) so that
+    // names like "Dr. Smith" aren't fragmented and dropped by the first-word skip.
+    const sentences = input.taskDescription.split(
+      /(?<!\b(?:Mr|Mrs|Ms|Dr|Jr|Sr|St|Rev|Prof|Gen|Sgt|Lt|Col))[.]\s+|[!?]\s+/,
+    );
+    for (const sentence of sentences) {
+      const words = sentence.trim().split(/\s+/);
+      // Skip the first word (always capitalized at sentence start)
+      for (let i = 1; i < words.length; i++) {
+        // Use Unicode-aware \p{L} to preserve accented/non-Latin letters (José, Łukasz, etc.)
+        const word = words[i].replace(/[^\p{L}'-]/gu, "");
+        if (word.length > 0 && /^\p{Lu}/u.test(word)) {
+          hints.push(word);
+        }
+      }
+    }
+  }
+
+  hints.push(...input.recentContactNames);
+
+  // Deduplicate (case-insensitive), filter empty/whitespace-only, trim each
+  const seen = new Set<string>();
+  const deduped: string[] = [];
+  for (const hint of hints) {
+    const trimmed = hint.trim();
+    if (trimmed.length === 0) continue;
+    const key = trimmed.toLowerCase();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    deduped.push(trimmed);
+  }
+
+  const joined = deduped.join(",");
+
+  if (joined.length <= MAX_HINTS_LENGTH) {
+    return joined;
+  }
+
+  // Truncate at the last comma before the limit to avoid partial words
+  const truncated = joined.slice(0, MAX_HINTS_LENGTH);
+  const lastComma = truncated.lastIndexOf(",");
+  if (lastComma === -1) {
+    // Single hint that exceeds the limit — return it truncated
+    return truncated;
+  }
+  return truncated.slice(0, lastComma);
+}
+
+/**
+ * Wire real data sources (contacts DB, identity helpers, config) into
+ * {@link buildSttHints}. All DB lookups are best-effort — errors are
+ * logged but never propagate so hints can never fail a call.
+ */
+export function resolveCallHints(
+  session: {
+    task: string | null;
+    toNumber: string;
+    fromNumber: string;
+    direction: "inbound" | "outbound";
+    inviteFriendName: string | null;
+    inviteGuardianName: string | null;
+  } | null,
+  staticHints: string[],
+): string {
+  const assistantName = getAssistantName();
+
+  // Look up the guardian contact for a displayName fallback (mirrors relay-server pattern)
+  let guardianDisplayName: string | undefined;
+  try {
+    const voiceGuardian = findGuardianForChannel("phone");
+    const guardianChannels = voiceGuardian ? null : listGuardianChannels();
+    const guardianContact = voiceGuardian?.contact ?? guardianChannels?.contact;
+    guardianDisplayName = guardianContact?.displayName;
+  } catch (err) {
+    logger.warn({ err }, "Failed to look up guardian contact for STT hints");
+  }
+  const guardianName = resolveGuardianName(guardianDisplayName);
+
+  let targetContactName: string | null = null;
+  let callerContactName: string | null = null;
+  let recentContactNames: string[] = [];
+
+  // For inbound calls, fromNumber is the caller (the interesting party);
+  // toNumber is the assistant's own Twilio number (not useful for contact lookup).
+  // For outbound calls, toNumber is who we're calling.
+  try {
+    if (session) {
+      const otherPartyNumber =
+        session.direction === "inbound" ? session.fromNumber : session.toNumber;
+      const otherPartyContact = findContactByAddress("phone", otherPartyNumber);
+      if (otherPartyContact) {
+        if (session.direction === "inbound") {
+          callerContactName = otherPartyContact.displayName;
+        } else {
+          targetContactName = otherPartyContact.displayName;
+        }
+      }
+    }
+  } catch (err) {
+    logger.warn({ err }, "Failed to look up contact for STT hints");
+  }
+
+  try {
+    const recentContacts = listContacts(15);
+    recentContactNames = recentContacts.map((c) => c.displayName);
+  } catch (err) {
+    logger.warn({ err }, "Failed to list recent contacts for STT hints");
+  }
+
+  return buildSttHints({
+    staticHints,
+    assistantName,
+    guardianName,
+    taskDescription: session?.task ?? null,
+    targetContactName,
+    callerContactName,
+    inviteFriendName: session?.inviteFriendName ?? null,
+    inviteGuardianName: session?.inviteGuardianName ?? null,
+    recentContactNames,
+  });
+}

package/src/calls/tts-text-sanitizer.ts

@@ -0,0 +1,61 @@
+/**
+ * Sanitizes text for TTS synthesis by stripping markdown formatting and emojis.
+ *
+ * Preserves arithmetic expressions (e.g. `5 * 3`), identifiers with underscores
+ * (e.g. `my_var`), and Fish Audio S2 bracket annotations (e.g. `[laughter]`).
+ */
+export function sanitizeForTts(text: string): string {
+  let result = text;
+
+  // 1. Markdown links: [text](url) → text
+  //    Only matches the full [...](...) pattern — plain brackets like
+  //    Fish Audio S2 annotations ([laughter], [breath]) pass through.
+  //    Handles one level of balanced parentheses in URLs (e.g. Wikipedia links).
+  result = result.replace(
+    /\[([^\]]+)\]\([^()]*(?:\([^()]*\))*[^()]*\)/g,
+    "$1",
+  );
+
+  // 2. Bold+italic: ***text*** or ___text___ → text
+  result = result.replace(/\*{3}(.+?)\*{3}/g, "$1");
+  result = result.replace(/_{3}(.+?)_{3}/g, "$1");
+
+  // 3. Bold: **text** or __text__ → text
+  result = result.replace(/\*{2}(.+?)\*{2}/g, "$1");
+  result = result.replace(/_{2}(.+?)_{2}/g, "$1");
+
+  // 4. Code fences: strip ```...``` fences but keep content
+  //    Must run before header stripping so # comments inside code blocks are preserved.
+  result = result.replace(/```[^\n]*\n([\s\S]*?)```\n?/g, "$1");
+
+  // 5. Headers: strip leading # characters at line starts
+  result = result.replace(/^#{1,6}\s+/gm, "");
+
+  // 6. Inline code: strip single backticks
+  result = result.replace(/`([^`]+)`/g, "$1");
+
+  // 7. Bullet markers: strip `- ` or `* ` at line starts
+  //    Must run before italic stripping so `* item` is treated as a bullet.
+  result = result.replace(/^[-*]\s+/gm, "");
+
+  // 8. Italic: *text* or _text_ → text
+  //    Word-boundary-aware to preserve arithmetic like `5 * 3` and identifiers like `my_var`.
+  result = result.replace(/(?<!\w)\*([^*]+)\*(?!\w)/g, "$1");
+  result = result.replace(/(?<!\w)_([^_]+)_(?!\w)/g, "$1");
+
+  // 9. Emojis: strip extended pictographic characters, variation selectors,
+  //    zero-width joiners, skin tone modifiers, and regional indicator symbols (flags).
+  result = result.replace(/[\u200D\uFE00-\uFE0F]/gu, "");
+  result = result.replace(/[\u{1F3FB}-\u{1F3FF}]/gu, "");
+  result = result.replace(/\p{Extended_Pictographic}/gu, "");
+  result = result.replace(/[\u{1F1E6}-\u{1F1FF}]/gu, "");
+
+  // 10. Collapse whitespace: multiple spaces → single space,
+  //     multiple blank lines → single newline.
+  //     Does NOT trim trailing whitespace — callers handle trimming so that
+  //     streaming chunks preserve inter-word spaces (e.g. "Hello " + "world").
+  result = result.replace(/ {2,}/g, " ");
+  result = result.replace(/\n{3,}/g, "\n\n");
+
+  return result;
+}