@vellumai/assistant 0.12.1-staging.1 → 0.12.1

package/src/calls/call-transport.ts

@@ -98,20 +98,4 @@ export interface CallTransport {
    * this — the controller then treats playback as already drained.
    */
   awaitPlaybackDrained?(): Promise<void>;
-
-  /**
-   * Whether no assistant speech is buffered, queued, in flight, or
-   * (estimatedly) still playing to the caller. Progress narration speaks
-   * only into audible silence, so it asks before every update. Transports
-   * that play speech synchronously may omit this; the controller then
-   * treats the transport as idle.
-   */
-  isPlaybackIdle?(): boolean;
-
-  /**
-   * Wall-clock estimate of when audio already sent finishes playing to the
-   * caller, 0 when nothing is playing. Anchors the dead-air countdown for
-   * progress narration to when the caller last heard something.
-   */
-  playbackTailUntilMs?(): number;
 }

package/src/calls/media-stream-output.ts

@@ -327,14 +327,6 @@ export class MediaStreamOutput implements CallTransport {
   /** True when the queue drain loop is actively running. */
   private draining = false;
 
-  /**
-   * Estimated wall-clock instant the audio already sent to Twilio finishes
-   * playing to the caller. Frames leave faster than real time and Twilio
-   * buffers them, so the only local view of "still audible" is this running
-   * sum of sent frame durations; a `clear` drops the buffer and resets it.
-   */
-  private playbackTailAtMs = 0;
-
   /** Abort controller for the currently in-flight synthesis/fetch. */
   private activePlaybackAbort: AbortController | null = null;
 
@@ -667,7 +659,6 @@ export class MediaStreamOutput implements CallTransport {
       return;
     }
     this.sendClearCommand();
-    this.playbackTailAtMs = 0;
     // Twilio drops its buffered audio (and the marks within it) on `clear`, so
     // any end-of-turn mark already sent will never echo. Treat those as drained
     // so a pending end-call drain wait resolves instead of stalling on the cap.
@@ -725,24 +716,6 @@ export class MediaStreamOutput implements CallTransport {
     return this.playbackQueue.length;
   }
 
-  /**
-   * Whether no speech is buffered toward a sentence, queued, being
-   * synthesized or fetched, or (estimatedly) still playing at Twilio.
-   */
-  isPlaybackIdle(): boolean {
-    return (
-      this.textBuffer.length === 0 &&
-      this.playbackQueue.length === 0 &&
-      !this.draining &&
-      Date.now() >= this.playbackTailAtMs
-    );
-  }
-
-  /** See {@link CallTransport.playbackTailUntilMs}. */
-  playbackTailUntilMs(): number {
-    return this.playbackTailAtMs;
-  }
-
   /**
    * Runtime check for closed state. Used instead of direct property access
    * in async methods because TypeScript's control flow analysis cannot
@@ -799,7 +772,6 @@ export class MediaStreamOutput implements CallTransport {
     this.playbackQueue.length = 0;
     this.playbackVersion++;
     this.audioStartCallback = null;
-    this.playbackTailAtMs = 0;
     if (this.activePlaybackAbort) {
       this.activePlaybackAbort.abort();
       this.activePlaybackAbort = null;
@@ -882,10 +854,6 @@ export class MediaStreamOutput implements CallTransport {
     for (const frame of frames) {
       this.sendAudioPayload(frame);
     }
-    const durationMs =
-      (frames.length * MULAW_FRAME_SIZE * 1000) / TELEPHONY_SAMPLE_RATE_HZ;
-    this.playbackTailAtMs =
-      Math.max(Date.now(), this.playbackTailAtMs) + durationMs;
   }
 
   /**

package/src/calls/media-stream-server.ts

@@ -47,11 +47,6 @@ import { addMessage } from "../persistence/conversation-crud.js";
 import { resolveGuardianName } from "../prompts/user-reference.js";
 import { DAEMON_INTERNAL_ASSISTANT_ID } from "../runtime/assistant-scope.js";
 import { getLogger } from "../util/logger.js";
-import {
-  type BargeInGuard,
-  createBargeInGuard,
-  DEFAULT_BARGE_IN_MIN_SPEECH_MS,
-} from "./barge-in-guard.js";
 import { CallController } from "./call-controller.js";
 import {
   formatDuration,
@@ -193,8 +188,6 @@ export class MediaStreamCallSession {
   private bargeInAccepted = 0;
   /** Number of barge-in attempts that were ignored (assistant not speaking). */
   private bargeInIgnored = 0;
-  /** Armed by caller speech while a turn or its tail is interruptible. */
-  private bargeInGuard: BargeInGuard | null = null;
   /** Number of turn-start transitions detected by the STT session. */
   private turnStarts = 0;
   /** Number of transcript finals produced (non-empty). */
@@ -222,9 +215,6 @@ export class MediaStreamCallSession {
     // Create STT session with callbacks wired to the controller.
     const callbacks: MediaStreamSttSessionCallbacks = {
       onSpeechStart: () => this.handleSpeechStart(),
-      onMediaFrame: (hasSpeech, durationMs) =>
-        this.handleMediaFrame(hasSpeech, durationMs),
-      onSpeechEnd: () => this.handleSpeechEnd(),
       onTranscriptFinal: (text, durationMs) =>
         this.handleTranscriptFinal(text, durationMs),
       onDtmf: (digit) => this.handleDtmf(digit),
@@ -778,80 +768,38 @@ export class MediaStreamCallSession {
 
   private handleSpeechStart(): void {
     this.turnStarts++;
-  }
 
-  // Sustained-speech barge-in guard, fed by every inbound frame. Speech only
-  // counts while there is something to interrupt: an assistant turn in
-  // flight (thinking or speaking) or a completed turn's tail still playing
-  // from Twilio's buffer. Outside that window the guard is dropped, so
-  // speech from the caller's own utterance never carries into a turn that
-  // starts before the local VAD has ended the utterance (a streaming final
-  // can start the turn first). A speech frame inside the window arms the
-  // guard; a fired guard reaches the controller. A line click, a cough, or
-  // TTS bleed never gets that far.
-  private handleMediaFrame(hasSpeech: boolean, durationMs: number): void {
-    if (!this.output || !this.controller) {
-      return;
-    }
-    const bargeable =
-      this.controller.getState() !== "idle" || !this.output.isPlaybackIdle();
-    if (!bargeable) {
-      this.bargeInGuard = null;
-      return;
-    }
-    if (this.bargeInGuard === null) {
-      if (!hasSpeech) {
-        return;
+    // Barge-in: clear queued outbound audio and abort the in-flight LLM
+    // turn only when the assistant is actively speaking. Uses the gated
+    // handleBargeIn path so initial inbound audio frames do not cancel a
+    // still-starting initial turn.
+    //
+    // clearAudio runs via the onAccepted hook so it only fires when the
+    // barge-in passes the speaking gate — an ignored barge-in (controller
+    // idle/processing) must not flush queued/in-flight TTS such as a
+    // buffered greeting. The hook runs before handleInterrupt so the
+    // end-of-turn mark it enqueues survives the queue flush.
+    if (this.output && this.controller) {
+      const output = this.output;
+      const accepted = this.controller.handleBargeIn(() => output.clearAudio());
+      if (accepted) {
+        this.bargeInAccepted++;
+        log.info(
+          { callSessionId: this.callSessionId },
+          "Media-stream barge-in accepted — cleared outbound audio",
+        );
+      } else {
+        // No turn to abort, but a completed turn's tail can still be
+        // playing from Twilio's buffer — flush only that buffer so the
+        // caller isn't talked over. Queued speech that hasn't reached
+        // Twilio yet (greeting, handoff prompt) is preserved.
+        output.clearBufferedAudio();
+        this.bargeInIgnored++;
+        log.debug(
+          { callSessionId: this.callSessionId },
+          "Media-stream barge-in ignored — assistant not speaking",
+        );
       }
-      this.bargeInGuard = createBargeInGuard(DEFAULT_BARGE_IN_MIN_SPEECH_MS);
-    }
-    const step = this.bargeInGuard.track(
-      hasSpeech ? "speech" : "silence",
-      durationMs,
-    );
-    if (step === "fired") {
-      this.bargeInGuard = null;
-      this.fireBargeIn();
-    }
-  }
-
-  private handleSpeechEnd(): void {
-    // An untripped guard was noise or a fragment: the run never reached the
-    // threshold before the caller's turn ended.
-    this.bargeInGuard = null;
-  }
-
-  // Sustained caller speech landed: clear queued outbound audio and abort
-  // the in-flight LLM turn when one is in flight (thinking or speaking).
-  //
-  // clearAudio runs via the onAccepted hook so it only fires when the
-  // barge-in passes the controller's gate; an ignored barge-in (controller
-  // idle) must not flush queued/in-flight TTS such as a buffered greeting.
-  // The hook runs before handleInterrupt so the end-of-turn mark it
-  // enqueues survives the queue flush.
-  private fireBargeIn(): void {
-    if (!this.output || !this.controller) {
-      return;
-    }
-    const output = this.output;
-    const accepted = this.controller.handleBargeIn(() => output.clearAudio());
-    if (accepted) {
-      this.bargeInAccepted++;
-      log.info(
-        { callSessionId: this.callSessionId },
-        "Media-stream barge-in accepted: cleared outbound audio",
-      );
-    } else {
-      // No turn to abort, but a completed turn's tail can still be
-      // playing from Twilio's buffer, so flush only that buffer so the
-      // caller isn't talked over. Queued speech that hasn't reached
-      // Twilio yet (greeting, handoff prompt) is preserved.
-      output.clearBufferedAudio();
-      this.bargeInIgnored++;
-      log.debug(
-        { callSessionId: this.callSessionId },
-        "Media-stream barge-in ignored: no turn in flight",
-      );
     }
   }
 

package/src/calls/media-stream-stt-session.ts

@@ -120,16 +120,6 @@ export interface MediaStreamSttSessionCallbacks {
   /** Called when the turn detector transitions to active (first speech-bearing chunk). */
   onSpeechStart?: () => void;
 
-  /**
-   * Called for every inbound frame with the local VAD's verdict on it and
-   * the frame's duration. Drives the sustained-speech barge-in guard, which
-   * needs the silence frames inside a detector turn as well as the speech.
-   */
-  onMediaFrame?: (hasSpeech: boolean, durationMs: number) => void;
-
-  /** Called when the turn detector ends the caller's turn (silence or cap). */
-  onSpeechEnd?: () => void;
-
   /**
    * Called when a completed caller utterance has been transcribed.
    *
@@ -265,7 +255,6 @@ export class MediaStreamSttSession {
         this.callbacks.onSpeechStart?.();
       },
       onTurnEnd: (reason, durationMs) => {
-        this.callbacks.onSpeechEnd?.();
         void this.handleTurnEnd(reason, durationMs);
       },
     });
@@ -388,10 +377,6 @@ export class MediaStreamSttSession {
     const raw = Buffer.from(event.media.payload, "base64");
     const hasSpeech = detectSpeechActivity(raw);
     this.turnDetector.onMediaChunk(hasSpeech);
-    this.callbacks.onMediaFrame?.(
-      hasSpeech,
-      raw.length / (TELEPHONY_SOURCE_SAMPLE_RATE / 1000),
-    );
 
     if (this.mode === "batch") {
       this.currentTurnChunks.push(event.media.payload);

package/src/calls/resolve-call-tts-provider.ts

@@ -8,7 +8,6 @@
  */
 
 import { loadConfig } from "../config/loader.js";
-import { resolveEffectiveSpeechProviders } from "../config/managed-speech-defaults.js";
 import {
   getCatalogProvider,
   getTtsProvider,
@@ -74,13 +73,6 @@ export interface ResolveCallTtsOptions {
 /**
  * Resolve the active TTS provider via the global provider abstraction.
  *
- * The provider is the effective one after managed-speech defaulting
- * ({@link resolveEffectiveSpeechProviders}): a configured BYOK provider
- * whose credential does not resolve is stood in for by managed speech
- * while the platform connection is usable, the same substitution live
- * voice makes, so a fresh Vellum connection speaks on the phone with zero
- * TTS configuration.
- *
  * The native-vs-synthesized decision is driven by the catalog's
  * `callMode` field via {@link resolveCallStrategy}. Providers with
  * `callMode: "synthesized-play"` have their audio streamed through the
@@ -102,11 +94,10 @@ export async function resolveCallTtsProvider(
 ): Promise<ResolvedCallTts> {
   try {
     const config = loadConfig();
-    const effectiveProviderId = await effectiveTtsProviderId(config);
-    const resolved = resolveTtsConfig(config, effectiveProviderId);
+    const resolved = resolveTtsConfig(config);
 
     // Use the catalog's callMode to decide the call path.
-    const strategy = resolveCallStrategy(config, effectiveProviderId);
+    const strategy = resolveCallStrategy(config);
 
     let providerId = resolved.provider;
     let useSynthesizedPath = strategy.callMode === "synthesized-play";
@@ -184,28 +175,6 @@ export async function resolveCallTtsProvider(
   }
 }
 
-/**
- * The TTS provider the call actually uses after managed-speech defaulting.
- * A failure inside the managed lookup (platform client, credential store)
- * keeps the configured provider: a managed-speech hiccup must not demote a
- * working BYOK provider, and the outer resolver's own fallback is the
- * silent native path.
- */
-async function effectiveTtsProviderId(
-  config: ReturnType<typeof loadConfig>,
-): Promise<TtsProviderId | undefined> {
-  try {
-    const { tts } = await resolveEffectiveSpeechProviders(config);
-    return tts;
-  } catch (err) {
-    log.warn(
-      { err },
-      "Managed-speech defaulting failed for call TTS; keeping the configured provider",
-    );
-    return undefined;
-  }
-}
-
 /**
  * Decode a resolved call audio format into the provider request format
  * and the audio-store format.

package/src/calls/speaker-identification.ts

@@ -0,0 +1,233 @@
+export interface PromptSpeakerContext {
+  speakerId: string;
+  speakerLabel: string;
+  speakerConfidence: number | null;
+  source: "provider" | "inferred";
+}
+
+export interface PromptSpeakerMetadata {
+  speakerId?: string;
+  speakerLabel?: string;
+  speakerName?: string;
+  speakerConfidence?: number;
+  participantId?: string;
+}
+
+interface SpeakerProfile {
+  speakerId: string;
+  speakerLabel: string;
+  speakerConfidence: number | null;
+  source: "provider" | "inferred";
+  utteranceCount: number;
+  firstSeenAt: number;
+  lastSeenAt: number;
+}
+
+function toCleanString(value: unknown): string | null {
+  if (typeof value !== "string") {
+    return null;
+  }
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function toNumber(value: unknown): number | null {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    return value;
+  }
+  if (typeof value === "string") {
+    const parsed = Number(value);
+    if (Number.isFinite(parsed)) {
+      return parsed;
+    }
+  }
+  return null;
+}
+
+function getObject(value: unknown): Record<string, unknown> | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    return null;
+  }
+  return value as Record<string, unknown>;
+}
+
+function normalizeSpeakerLabel(
+  metadata: PromptSpeakerMetadata,
+  fallbackIndex: number,
+): string {
+  const preferredLabel =
+    toCleanString(metadata.speakerName) ?? toCleanString(metadata.speakerLabel);
+  if (preferredLabel) {
+    return preferredLabel;
+  }
+  return `Speaker ${fallbackIndex}`;
+}
+
+export function extractPromptSpeakerMetadata(
+  message: Record<string, unknown>,
+): PromptSpeakerMetadata {
+  const providerMetadata = getObject(message.providerMetadata);
+  const metadata = getObject(message.metadata);
+  const participant = getObject(message.participant);
+  const speaker = getObject(message.speaker);
+
+  const pick = (...values: unknown[]): string | undefined => {
+    for (const value of values) {
+      const cleaned = toCleanString(value);
+      if (cleaned) {
+        return cleaned;
+      }
+    }
+    return undefined;
+  };
+
+  const pickNumber = (...values: unknown[]): number | undefined => {
+    for (const value of values) {
+      const parsed = toNumber(value);
+      if (parsed != null) {
+        return parsed;
+      }
+    }
+    return undefined;
+  };
+
+  return {
+    speakerId: pick(
+      message.speakerId,
+      message.speaker_id,
+      speaker?.id,
+      speaker?.speakerId,
+      metadata?.speakerId,
+      providerMetadata?.speakerId,
+      metadata?.speaker_id,
+      providerMetadata?.speaker_id,
+    ),
+    speakerLabel: pick(
+      message.speakerLabel,
+      message.speaker_label,
+      message.speaker,
+      speaker?.label,
+      speaker?.name,
+      metadata?.speakerLabel,
+      providerMetadata?.speakerLabel,
+      metadata?.speaker_label,
+      providerMetadata?.speaker_label,
+    ),
+    speakerName: pick(
+      message.speakerName,
+      message.speaker_name,
+      participant?.name,
+      metadata?.speakerName,
+      providerMetadata?.speakerName,
+      metadata?.speaker_name,
+      providerMetadata?.speaker_name,
+    ),
+    speakerConfidence: pickNumber(
+      message.speakerConfidence,
+      message.speaker_confidence,
+      message.confidence,
+      speaker?.confidence,
+      metadata?.speakerConfidence,
+      providerMetadata?.speakerConfidence,
+      metadata?.speaker_confidence,
+      providerMetadata?.speaker_confidence,
+    ),
+    participantId: pick(
+      message.participantId,
+      message.participant_id,
+      participant?.id,
+      metadata?.participantId,
+      providerMetadata?.participantId,
+      metadata?.participant_id,
+      providerMetadata?.participant_id,
+    ),
+  };
+}
+
+export class SpeakerIdentityTracker {
+  private profiles = new Map<string, SpeakerProfile>();
+  private nextInferredIndex = 1;
+
+  identifySpeaker(metadata: PromptSpeakerMetadata): PromptSpeakerContext {
+    const providerSpeakerId =
+      toCleanString(metadata.speakerId) ??
+      toCleanString(metadata.participantId) ??
+      null;
+
+    if (providerSpeakerId) {
+      const existing = this.profiles.get(providerSpeakerId);
+      if (existing) {
+        existing.lastSeenAt = Date.now();
+        existing.utteranceCount += 1;
+        if (metadata.speakerConfidence !== undefined) {
+          existing.speakerConfidence = metadata.speakerConfidence;
+        }
+        return {
+          speakerId: existing.speakerId,
+          speakerLabel: existing.speakerLabel,
+          speakerConfidence: existing.speakerConfidence,
+          source: existing.source,
+        };
+      }
+
+      const profile: SpeakerProfile = {
+        speakerId: providerSpeakerId,
+        speakerLabel: normalizeSpeakerLabel(metadata, this.nextInferredIndex),
+        speakerConfidence: metadata.speakerConfidence ?? null,
+        source: "provider",
+        utteranceCount: 1,
+        firstSeenAt: Date.now(),
+        lastSeenAt: Date.now(),
+      };
+      this.profiles.set(providerSpeakerId, profile);
+      this.nextInferredIndex += 1;
+      return {
+        speakerId: profile.speakerId,
+        speakerLabel: profile.speakerLabel,
+        speakerConfidence: profile.speakerConfidence,
+        source: profile.source,
+      };
+    }
+
+    const inferredSpeakerId = "primary-speaker";
+    const existingPrimary = this.profiles.get(inferredSpeakerId);
+    if (existingPrimary) {
+      existingPrimary.lastSeenAt = Date.now();
+      existingPrimary.utteranceCount += 1;
+      return {
+        speakerId: existingPrimary.speakerId,
+        speakerLabel: existingPrimary.speakerLabel,
+        speakerConfidence: existingPrimary.speakerConfidence,
+        source: existingPrimary.source,
+      };
+    }
+
+    const inferredProfile: SpeakerProfile = {
+      speakerId: inferredSpeakerId,
+      speakerLabel: normalizeSpeakerLabel(metadata, this.nextInferredIndex),
+      speakerConfidence: metadata.speakerConfidence ?? null,
+      source: "inferred",
+      utteranceCount: 1,
+      firstSeenAt: Date.now(),
+      lastSeenAt: Date.now(),
+    };
+    this.profiles.set(inferredSpeakerId, inferredProfile);
+    this.nextInferredIndex += 1;
+
+    return {
+      speakerId: inferredProfile.speakerId,
+      speakerLabel: inferredProfile.speakerLabel,
+      speakerConfidence: inferredProfile.speakerConfidence,
+      source: inferredProfile.source,
+    };
+  }
+
+  listProfiles(): PromptSpeakerContext[] {
+    return [...this.profiles.values()].map((profile) => ({
+      speakerId: profile.speakerId,
+      speakerLabel: profile.speakerLabel,
+      speakerConfidence: profile.speakerConfidence,
+      source: profile.source,
+    }));
+  }
+}

package/src/calls/tts-call-strategy.ts

@@ -40,21 +40,17 @@ export interface TtsCallStrategy {
 }
 
 /**
- * Resolve the call strategy for the active TTS provider.
+ * Resolve the call strategy for the currently configured TTS provider.
  *
- * Reads the provider from config via {@link resolveTtsConfig} (or takes
- * `providerOverride`, the provider a caller already resolved after
- * managed-speech defaulting), then looks up its `callMode` in the catalog.
+ * Reads the active provider from config via {@link resolveTtsConfig},
+ * then looks up the provider's `callMode` in the catalog.
  *
  * Falls back to `native-twilio` with `"elevenlabs"` when the config
  * or catalog is unavailable (e.g. test mocks, pre-migration configs).
  */
-export function resolveCallStrategy(
-  config: AssistantConfig,
-  providerOverride?: TtsProviderId,
-): TtsCallStrategy {
+export function resolveCallStrategy(config: AssistantConfig): TtsCallStrategy {
   try {
-    const resolved = resolveTtsConfig(config, providerOverride);
+    const resolved = resolveTtsConfig(config);
     const catalogEntry = getCatalogProvider(resolved.provider);
     return {
       providerId: catalogEntry.id,

package/src/calls/types.ts

@@ -11,8 +11,6 @@ export type CallEventType =
   | "call_connected"
   | "caller_spoke"
   | "assistant_spoke"
-  | "tool_use_started"
-  | "tool_use_completed"
   | "user_question_asked"
   | "user_answered"
   | "user_instruction_relayed"