@vellumai/assistant 0.5.6 → 0.5.8

package/src/calls/tts-text-sanitizer.ts

@@ -0,0 +1,61 @@
+/**
+ * Sanitizes text for TTS synthesis by stripping markdown formatting and emojis.
+ *
+ * Preserves arithmetic expressions (e.g. `5 * 3`), identifiers with underscores
+ * (e.g. `my_var`), and Fish Audio S2 bracket annotations (e.g. `[laughter]`).
+ */
+export function sanitizeForTts(text: string): string {
+  let result = text;
+
+  // 1. Markdown links: [text](url) → text
+  //    Only matches the full [...](...) pattern — plain brackets like
+  //    Fish Audio S2 annotations ([laughter], [breath]) pass through.
+  //    Handles one level of balanced parentheses in URLs (e.g. Wikipedia links).
+  result = result.replace(
+    /\[([^\]]+)\]\([^()]*(?:\([^()]*\))*[^()]*\)/g,
+    "$1",
+  );
+
+  // 2. Bold+italic: ***text*** or ___text___ → text
+  result = result.replace(/\*{3}(.+?)\*{3}/g, "$1");
+  result = result.replace(/_{3}(.+?)_{3}/g, "$1");
+
+  // 3. Bold: **text** or __text__ → text
+  result = result.replace(/\*{2}(.+?)\*{2}/g, "$1");
+  result = result.replace(/_{2}(.+?)_{2}/g, "$1");
+
+  // 4. Code fences: strip ```...``` fences but keep content
+  //    Must run before header stripping so # comments inside code blocks are preserved.
+  result = result.replace(/```[^\n]*\n([\s\S]*?)```\n?/g, "$1");
+
+  // 5. Headers: strip leading # characters at line starts
+  result = result.replace(/^#{1,6}\s+/gm, "");
+
+  // 6. Inline code: strip single backticks
+  result = result.replace(/`([^`]+)`/g, "$1");
+
+  // 7. Bullet markers: strip `- ` or `* ` at line starts
+  //    Must run before italic stripping so `* item` is treated as a bullet.
+  result = result.replace(/^[-*]\s+/gm, "");
+
+  // 8. Italic: *text* or _text_ → text
+  //    Word-boundary-aware to preserve arithmetic like `5 * 3` and identifiers like `my_var`.
+  result = result.replace(/(?<!\w)\*([^*]+)\*(?!\w)/g, "$1");
+  result = result.replace(/(?<!\w)_([^_]+)_(?!\w)/g, "$1");
+
+  // 9. Emojis: strip extended pictographic characters, variation selectors,
+  //    zero-width joiners, skin tone modifiers, and regional indicator symbols (flags).
+  result = result.replace(/[\u200D\uFE00-\uFE0F]/gu, "");
+  result = result.replace(/[\u{1F3FB}-\u{1F3FF}]/gu, "");
+  result = result.replace(/\p{Extended_Pictographic}/gu, "");
+  result = result.replace(/[\u{1F1E6}-\u{1F1FF}]/gu, "");
+
+  // 10. Collapse whitespace: multiple spaces → single space,
+  //     multiple blank lines → single newline.
+  //     Does NOT trim trailing whitespace — callers handle trimming so that
+  //     streaming chunks preserve inter-word spaces (e.g. "Hello " + "world").
+  result = result.replace(/ {2,}/g, " ");
+  result = result.replace(/\n{3,}/g, "\n\n");
+
+  return result;
+}

package/src/calls/twilio-routes.ts

@@ -26,6 +26,7 @@ import {
   releaseCallbackClaim,
   updateCallSession,
 } from "./call-store.js";
+import { resolveCallHints } from "./stt-hints.js";
 import type { CallStatus } from "./types.js";
 import { resolveVoiceQualityProfile } from "./voice-quality.js";
 
@@ -49,11 +50,14 @@ export function generateTwiML(
   profile: {
     language: string;
     transcriptionProvider: string;
+    speechModel?: string;
     ttsProvider: string;
     voice: string;
+    interruptSensitivity: string;
   },
   relayToken?: string,
   customParameters?: Record<string, string>,
+  hints?: string,
 ): string {
   const greetingAttr =
     welcomeGreeting && welcomeGreeting.trim().length > 0
@@ -91,10 +95,11 @@ export function generateTwiML(
 ${greetingAttr}
       voice="${escapeXml(profile.voice)}"
       language="${escapeXml(profile.language)}"
-      transcriptionProvider="${escapeXml(profile.transcriptionProvider)}"
+      transcriptionProvider="${escapeXml(profile.transcriptionProvider)}"${profile.speechModel ? `\n      speechModel="${escapeXml(profile.speechModel)}"` : ""}
       ttsProvider="${escapeXml(profile.ttsProvider)}"
       interruptible="true"
       dtmfDetection="true"
+      interruptSensitivity="${escapeXml(profile.interruptSensitivity)}"${hints ? `\n      hints="${escapeXml(hints)}"` : ""}
     ${relayClose}
   </Connect>
 </Response>`;
@@ -179,7 +184,14 @@ export async function handleVoiceWebhook(req: Request): Promise<Response> {
 
     return buildVoiceWebhookTwiml(
       session.id,
-      session.task,
+      {
+        task: session.task,
+        toNumber: callerTo,
+        fromNumber: callerFrom,
+        direction: "inbound",
+        inviteFriendName: null,
+        inviteGuardianName: null,
+      },
       session.verificationSessionId,
     );
   }
@@ -207,7 +219,14 @@ export async function handleVoiceWebhook(req: Request): Promise<Response> {
 
   return buildVoiceWebhookTwiml(
     callSessionId,
-    session.task,
+    {
+      task: session.task,
+      toNumber: session.toNumber,
+      fromNumber: session.fromNumber,
+      direction: "outbound",
+      inviteFriendName: session.inviteFriendName,
+      inviteGuardianName: session.inviteGuardianName,
+    },
     session.verificationSessionId,
   );
 }
@@ -224,18 +243,27 @@ export async function handleVoiceWebhook(req: Request): Promise<Response> {
  */
 function buildVoiceWebhookTwiml(
   callSessionId: string,
-  task: string | null,
+  sessionContext: {
+    task: string | null;
+    toNumber: string;
+    fromNumber: string;
+    direction: "inbound" | "outbound";
+    inviteFriendName: string | null;
+    inviteGuardianName: string | null;
+  } | null,
   verificationSessionId?: string | null,
 ): Response {
   const profile = resolveVoiceQualityProfile(loadConfig());
 
+  const hints = resolveCallHints(sessionContext, profile.hints);
+
   log.info(
     { callSessionId, ttsProvider: profile.ttsProvider, voice: profile.voice },
     "Voice quality profile resolved",
   );
 
   const relayUrl = getTwilioRelayUrl(loadConfig());
-  const welcomeGreeting = buildWelcomeGreeting(task);
+  const welcomeGreeting = buildWelcomeGreeting(sessionContext?.task ?? null);
 
   const relayToken = mintEdgeRelayToken();
 
@@ -252,6 +280,7 @@ function buildVoiceWebhookTwiml(
     profile,
     relayToken,
     customParameters,
+    hints || undefined,
   );
 
   log.info({ callSessionId }, "Returning ConversationRelay TwiML");

package/src/calls/types.ts

@@ -75,6 +75,7 @@ export interface CallSession {
   inviteGuardianName: string | null;
   callerIdentityMode: string | null;
   callerIdentitySource: string | null;
+  skipDisclosure: boolean;
   initiatedFromConversationId?: string | null;
   startedAt: number | null;
   endedAt: number | null;

package/src/calls/voice-ingress-preflight.ts

@@ -29,18 +29,6 @@ function fail(error: string): VoiceIngressPreflightFailure {
   };
 }
 
-function buildGatewayUnhealthyMessage(
-  target: string,
-  error: string | undefined,
-  afterRecoveryAttempt: boolean,
-): string {
-  const detail = error ?? "Unknown gateway health check failure";
-  if (afterRecoveryAttempt) {
-    return `Voice callback gateway is still unhealthy at ${target} after a local recovery attempt: ${detail}`;
-  }
-  return `Voice callback gateway is unhealthy at ${target}: ${detail}`;
-}
-
 export async function preflightVoiceIngress(): Promise<VoiceIngressPreflightResult> {
   const ingressConfig = loadConfig();
 
@@ -65,36 +53,6 @@ export async function preflightVoiceIngress(): Promise<VoiceIngressPreflightResu
     );
   }
 
-  const { ensureLocalGatewayReady, probeLocalGatewayHealth } =
-    await import("../runtime/local-gateway-health.js");
-
-  const initialHealth = await probeLocalGatewayHealth();
-  if (!initialHealth.healthy && !initialHealth.localDeployment) {
-    return fail(
-      buildGatewayUnhealthyMessage(
-        initialHealth.target,
-        initialHealth.error,
-        false,
-      ),
-    );
-  }
-
-  if (initialHealth.localDeployment) {
-    const recovery = await ensureLocalGatewayReady();
-    // Re-probe after the wake flow so the dial path only continues when the
-    // current gateway process is demonstrably serving the callback stack.
-    const confirmedHealth = await probeLocalGatewayHealth();
-    if (!confirmedHealth.healthy) {
-      return fail(
-        buildGatewayUnhealthyMessage(
-          confirmedHealth.target,
-          confirmedHealth.error ?? recovery.error,
-          recovery.recoveryAttempted,
-        ),
-      );
-    }
-  }
-
   return {
     ok: true,
     ingressConfig: {

package/src/calls/voice-quality.ts

@@ -3,8 +3,11 @@ import { loadConfig } from "../config/loader.js";
 export interface VoiceQualityProfile {
   language: string;
   transcriptionProvider: string;
+  speechModel?: string;
   ttsProvider: string;
   voice: string;
+  interruptSensitivity: string;
+  hints: string[];
 }
 
 /**
@@ -42,19 +45,49 @@ export function buildElevenLabsVoiceSpec(config: {
 /**
  * Resolve the effective voice quality profile from config.
  *
- * Always uses ElevenLabs TTS via Twilio ConversationRelay.
- * The voice ID comes from the shared `elevenlabs.voiceId` config
- * (defaults to Amelia — ZF6FPAbjXT4488VcRRnw).
+ * Supports ElevenLabs (default) and Fish Audio TTS providers.
+ * When Fish Audio is selected, `ttsProvider` is set to `"Google"` as a
+ * placeholder — ConversationRelay requires a valid provider in TwiML, but
+ * actual audio is delivered via `play` messages from the call-controller.
+ * The voice string is left empty since it is unused in that mode.
+ *
+ * For ElevenLabs, the voice ID comes from the shared `elevenlabs.voiceId`
+ * config (defaults to Amelia — ZF6FPAbjXT4488VcRRnw).
  */
 export function resolveVoiceQualityProfile(
   config?: ReturnType<typeof loadConfig>,
 ): VoiceQualityProfile {
   const cfg = config ?? loadConfig();
   const voice = cfg.calls.voice;
+  const configuredTts = voice.ttsProvider ?? "elevenlabs";
+  const fishAudio = configuredTts === "fish-audio";
+  const isGoogle = voice.transcriptionProvider === "Google";
+  // Treat the legacy Deepgram default ("nova-3") as unset when provider is
+  // Google — upgraded workspaces may still have it persisted from prior defaults.
+  const effectiveSpeechModel =
+    voice.speechModel == null ||
+    (voice.speechModel === "nova-3" && isGoogle)
+      ? isGoogle
+        ? undefined
+        : "nova-3"
+      : voice.speechModel;
   return {
     language: voice.language,
     transcriptionProvider: voice.transcriptionProvider,
-    ttsProvider: "ElevenLabs",
-    voice: buildElevenLabsVoiceSpec(cfg.elevenlabs),
+    speechModel: effectiveSpeechModel,
+    ttsProvider: fishAudio ? "Google" : "ElevenLabs",
+    voice: fishAudio ? "" : buildElevenLabsVoiceSpec(cfg.elevenlabs),
+    interruptSensitivity: voice.interruptSensitivity ?? "low",
+    hints: voice.hints ?? [],
   };
 }
+
+/**
+ * Check whether Fish Audio TTS is configured for phone calls.
+ */
+export function isFishAudioTts(
+  config?: ReturnType<typeof loadConfig>,
+): boolean {
+  const cfg = config ?? loadConfig();
+  return cfg.calls.voice?.ttsProvider === "fish-audio";
+}

package/src/calls/voice-session-bridge.ts

@@ -20,9 +20,7 @@ import type { ServerMessage } from "../daemon/message-protocol.js";
 import { buildAssistantEvent } from "../runtime/assistant-event.js";
 import { assistantEventHub } from "../runtime/assistant-event-hub.js";
 import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
-import { checkIngressForSecrets } from "../security/secret-ingress.js";
 import { computeToolApprovalDigest } from "../security/tool-approval-digest.js";
-import { IngressBlockedError } from "../util/errors.js";
 import { getLogger } from "../util/logger.js";
 import {
   CALL_OPENING_MARKER,
@@ -95,6 +93,8 @@ export interface VoiceTurnOptions {
   isInbound: boolean;
   /** The outbound call task, if any. */
   task?: string | null;
+  /** When true, skip the disclosure announcement for this call. */
+  skipDisclosure?: boolean;
   /** Called for each streaming text token from the agent loop. */
   onTextDelta: (text: string) => void;
   /** Called when the agent loop completes a full response. */
@@ -128,9 +128,11 @@ function buildVoiceCallControlPrompt(opts: {
   isInbound: boolean;
   task?: string | null;
   isCallerGuardian?: boolean;
+  skipDisclosure?: boolean;
 }): string {
   const config = getConfig();
-  const disclosureEnabled = config.calls?.disclosure?.enabled === true;
+  const disclosureEnabled =
+    config.calls?.disclosure?.enabled === true && !opts.skipDisclosure;
   const disclosureText = config.calls?.disclosure?.text?.trim();
   const disclosureRule =
     disclosureEnabled && disclosureText
@@ -210,6 +212,7 @@ function buildVoiceCallControlPrompt(opts: {
   lines.push(
     "9. After the opening greeting turn, treat the Task field as background context only — do not re-execute its instructions on subsequent turns.",
     '10. Do not make up information. If you are unsure, use [ASK_GUARDIAN: your question] to consult your guardian. For tool permission requests, use [ASK_GUARDIAN_APPROVAL: {"question":"...","toolName":"...","input":{...}}].',
+    `11. Your text is sent directly to a text-to-speech engine. Never use markdown formatting (asterisks, headers, backticks, links) or emojis in your spoken responses. Write plain conversational text only. Protocol markers like ${opts.isCallerGuardian ? "[END_CALL]" : "[ASK_GUARDIAN: ...] and [END_CALL]"} are not spoken text and should still be used normally.`,
     "</voice_call_control>",
   );
 
@@ -240,15 +243,6 @@ export async function startVoiceTurn(
     );
   }
 
-  // Block inbound content that contains secrets
-  const ingressCheck = checkIngressForSecrets(opts.content);
-  if (ingressCheck.blocked) {
-    throw new IngressBlockedError(
-      ingressCheck.userNotice!,
-      ingressCheck.detectedTypes,
-    );
-  }
-
   const eventSink: VoiceRunEventSink = {
     onTextDelta: opts.onTextDelta,
     onMessageComplete: opts.onComplete,
@@ -286,6 +280,7 @@ export async function startVoiceTurn(
     isInbound: opts.isInbound,
     task: opts.task,
     isCallerGuardian,
+    skipDisclosure: opts.skipDisclosure,
   });
 
   // Get or create the conversation

package/src/cli/commands/avatar.ts

@@ -11,9 +11,9 @@ import {
 } from "../../avatar/traits-png-sync.js";
 import { setPlatformBaseUrl } from "../../config/env.js";
 import { credentialKey } from "../../security/credential-key.js";
-import { getSecureKeyAsync } from "../../security/secure-keys.js";
 import { generateAndSaveAvatar } from "../../tools/system/avatar-generator.js";
 import { getWorkspaceDir } from "../../util/platform.js";
+import { getSecureKeyViaDaemon } from "../lib/daemon-credential-client.js";
 import { log } from "../logger.js";
 import { writeOutput } from "../output.js";
 
@@ -74,7 +74,7 @@ Examples:
       // without the daemon's in-memory state.
       try {
         const key = credentialKey("vellum", "platform_base_url");
-        const persisted = await getSecureKeyAsync(key);
+        const persisted = await getSecureKeyViaDaemon(key);
         if (persisted) {
           setPlatformBaseUrl(persisted);
         }

package/src/cli/commands/config.ts

@@ -6,7 +6,6 @@ import {
   loadRawConfig,
   saveRawConfig,
   setNestedValue,
-  syncConfigToLockfile,
 } from "../../config/loader.js";
 import { AssistantConfigSchema } from "../../config/schema.js";
 import { getSchemaAtPath } from "../../config/schema-utils.js";
@@ -73,8 +72,7 @@ Arguments:
           true, "42" becomes number 42). Falls back to plain string if JSON
           parsing fails.
 
-After writing the value to config.json, the lockfile is automatically synced
-to reflect the updated configuration.
+After writing the value to config.json, the change takes effect immediately.
 
 To manage API keys, use "assistant keys set <provider> <key>" instead.
 
@@ -93,7 +91,6 @@ Examples:
       }
       setNestedValue(raw, key, parsed);
       saveRawConfig(raw);
-      syncConfigToLockfile();
       log.info(`Set ${key} = ${JSON.stringify(parsed)}`);
     });