@vellumai/assistant 0.11.3-staging.3 → 0.11.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.11.3-staging.3",
+  "version": "0.11.3",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/config-schema.test.ts

@@ -932,6 +932,9 @@ describe("AssistantConfigSchema", () => {
         silenceThresholdMs: 1200,
         maxTurnDurationMs: 30000,
         bargeInMinSpeechMs: 250,
+        echoBargeInMargin: 1.5,
+        echoEmaHalfLifeMs: 400,
+        echoDrainSlackMs: 300,
       },
       frontModel: {
         endpointDecisionTimeoutMs: 1200,

package/src/config/schemas/__tests__/live-voice.test.ts

@@ -34,6 +34,9 @@ describe("LiveVoiceVadConfigSchema", () => {
       silenceThresholdMs: 1200,
       maxTurnDurationMs: 30_000,
       bargeInMinSpeechMs: 250,
+      echoBargeInMargin: 1.5,
+      echoEmaHalfLifeMs: 400,
+      echoDrainSlackMs: 300,
     });
   });
 
@@ -75,6 +78,35 @@ describe("LiveVoiceVadConfigSchema", () => {
     });
     expect(result.success).toBe(false);
   });
+
+  test("accepts echo gate overrides", () => {
+    const parsed = LiveVoiceVadConfigSchema.parse({
+      echoBargeInMargin: 2.25,
+      echoEmaHalfLifeMs: 250,
+      echoDrainSlackMs: 500,
+    });
+    expect(parsed.echoBargeInMargin).toBe(2.25);
+    expect(parsed.echoEmaHalfLifeMs).toBe(250);
+    expect(parsed.echoDrainSlackMs).toBe(500);
+  });
+
+  test("rejects an echo margin that cannot exceed its reference", () => {
+    expect(
+      LiveVoiceVadConfigSchema.safeParse({ echoBargeInMargin: 1 }).success,
+    ).toBe(false);
+  });
+
+  test("rejects invalid echo timing values", () => {
+    expect(
+      LiveVoiceVadConfigSchema.safeParse({ echoEmaHalfLifeMs: 0 }).success,
+    ).toBe(false);
+    expect(
+      LiveVoiceVadConfigSchema.safeParse({ echoEmaHalfLifeMs: 250.5 }).success,
+    ).toBe(false);
+    expect(
+      LiveVoiceVadConfigSchema.safeParse({ echoDrainSlackMs: -1 }).success,
+    ).toBe(false);
+  });
 });
 
 describe("LiveVoiceFrontModelConfigSchema", () => {
@@ -191,6 +223,9 @@ describe("LiveVoiceConfigSchema", () => {
         silenceThresholdMs: 1200,
         maxTurnDurationMs: 30_000,
         bargeInMinSpeechMs: 250,
+        echoBargeInMargin: 1.5,
+        echoEmaHalfLifeMs: 400,
+        echoDrainSlackMs: 300,
       },
       frontModel: FRONT_MODEL_DEFAULTS,
       maxSessionDurationSeconds: 1800,

package/src/config/schemas/live-voice.ts

@@ -42,6 +42,31 @@ export const LiveVoiceVadConfigSchema = z
       .describe(
         "Sustained speech (ms) required before speech during assistant playback interrupts it — the default 'interrupt sensitivity' (higher = harder to interrupt). 0 disables the guard. Clients may override it per-session via the start frame. Raised from 60 so brief TTS bleed through imperfect echo cancellation no longer self-interrupts the assistant.",
       ),
+    echoBargeInMargin: z
+      .number({ error: "liveVoice.vad.echoBargeInMargin must be a number" })
+      .gt(1, "liveVoice.vad.echoBargeInMargin must be greater than 1")
+      .default(1.5)
+      .describe(
+        "Multiplier over the learned playback echo level that microphone input must exceed to count as speech during playback. Higher values reduce false interruptions but require louder barge-in speech.",
+      ),
+    echoEmaHalfLifeMs: z
+      .number({ error: "liveVoice.vad.echoEmaHalfLifeMs must be a number" })
+      .int("liveVoice.vad.echoEmaHalfLifeMs must be an integer")
+      .positive("liveVoice.vad.echoEmaHalfLifeMs must be a positive integer")
+      .default(400)
+      .describe(
+        "Half-life (ms) of the learned playback echo level. Smaller values adapt faster to changing speaker volume; larger values are steadier against transients.",
+      ),
+    echoDrainSlackMs: z
+      .number({ error: "liveVoice.vad.echoDrainSlackMs must be a number" })
+      .int("liveVoice.vad.echoDrainSlackMs must be an integer")
+      .nonnegative(
+        "liveVoice.vad.echoDrainSlackMs must be a nonnegative integer",
+      )
+      .default(300)
+      .describe(
+        "Time (ms) after the estimated client playback tail during which microphone input can still be classified as playback echo.",
+      ),
   })
   .describe(
     "Voice-activity-detection tuning for live voice sessions (open-mic turn segmentation)",

package/src/live-voice/__tests__/live-voice-integration.test.ts

@@ -254,6 +254,9 @@ function createMultiCycleHarness(startVoiceTurn: LiveVoiceTurnStarter) {
   const session = createLiveVoiceSession(context, {
     // Credential-free harness: every leg is injected, so skip the preflight.
     resolveCredentialReadiness: null,
+    // These cycle mechanics use one discrete mic chunk per utterance. Keep
+    // the adaptive playback classifier out of their timing model.
+    echoBargeInMargin: 1,
     resolveTranscriber,
     startVoiceTurn,
     streamTtsAudio,
@@ -546,6 +549,8 @@ describe("LiveVoiceSession integration smoke harness", () => {
     let turnCount = 0;
     const session = createLiveVoiceSession(context, {
       resolveCredentialReadiness: null,
+      // This cycle mechanic uses one discrete mic chunk per utterance.
+      echoBargeInMargin: 1,
       resolveTranscriber,
       startVoiceTurn,
       streamTtsAudio,

package/src/live-voice/__tests__/live-voice-vad.test.ts

@@ -83,6 +83,21 @@ function pcm(amplitude: number, sampleCount = 240): Uint8Array {
   return new Uint8Array(buffer);
 }
 
+function tonePcm(
+  amplitude: number,
+  frequencyHz: number,
+  sampleCount = 240,
+): Uint8Array {
+  const buffer = Buffer.alloc(sampleCount * 2);
+  for (let index = 0; index < sampleCount; index += 1) {
+    const sample = Math.round(
+      amplitude * Math.sin((2 * Math.PI * frequencyHz * index) / SAMPLE_RATE),
+    );
+    buffer.writeInt16LE(sample, index * 2);
+  }
+  return new Uint8Array(buffer);
+}
+
 // 10 ms of speech at 24 kHz.
 const LOUD_CHUNK = pcm(8_000);
 // 300 ms of speech at 24 kHz — comfortably exceeds the default sustained-speech
@@ -164,6 +179,9 @@ function createHarness(options: {
   turnDetectorConfig?: TurnDetectorConfig;
   speechEnergyThreshold?: number;
   bargeInMinSpeechMs?: number;
+  echoBargeInMargin?: number;
+  echoEmaHalfLifeMs?: number;
+  echoDrainSlackMs?: number;
   frontDecider?: VoiceFrontDecider | null;
   frontModelConfig?: Partial<LiveVoiceFrontModelConfig>;
   emitMetrics?: boolean;
@@ -242,6 +260,10 @@ function createHarness(options: {
       (options.viaFactory ? undefined : { silenceThresholdMs: 40 }),
     speechEnergyThreshold: options.speechEnergyThreshold,
     bargeInMinSpeechMs: options.bargeInMinSpeechMs,
+    echoBargeInMargin:
+      options.echoBargeInMargin ?? (options.viaFactory ? undefined : 1),
+    echoEmaHalfLifeMs: options.echoEmaHalfLifeMs,
+    echoDrainSlackMs: options.echoDrainSlackMs,
     ...(options.frontDecider !== undefined
       ? { frontDecider: options.frontDecider }
       : {}),
@@ -291,6 +313,15 @@ function makeTtsChunk(text: string): LiveVoiceTtsAudioChunk {
   };
 }
 
+function makePcmTtsChunk(audio: Uint8Array): LiveVoiceTtsAudioChunk {
+  return {
+    type: "tts_audio",
+    contentType: "audio/pcm",
+    sampleRate: SAMPLE_RATE,
+    dataBase64: Buffer.from(audio).toString("base64"),
+  };
+}
+
 function makeTtsResult(text: string): LiveVoiceTtsResult {
   return {
     provider: "fish-audio",
@@ -3209,6 +3240,9 @@ describe("LiveVoiceSession VAD threshold configuration", () => {
       silenceThresholdMs: 1200,
       maxTurnDurationMs: 30_000,
       bargeInMinSpeechMs: 250,
+      echoBargeInMargin: 1.5,
+      echoEmaHalfLifeMs: 400,
+      echoDrainSlackMs: 300,
     });
 
     const { frames, session } = createHarness({ viaFactory: true });
@@ -3322,6 +3356,10 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
   // detector timers stay out of the guard's audio-duration accounting.
   function createSpeakingTurnHarness(options: {
     bargeInMinSpeechMs: number;
+    echoEmaHalfLifeMs?: number;
+    echoBargeInMargin?: number;
+    echoDrainSlackMs?: number;
+    ttsAudio?: Uint8Array;
     finals?: string[];
     startFrame?: LiveVoiceClientStartFrame;
   }) {
@@ -3332,7 +3370,11 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
       return { turnId: "bridge-turn", abort };
     });
     const streamTtsAudio = mock(async (ttsOptions: LiveVoiceTtsOptions) => {
-      ttsOptions.onAudioChunk(makeTtsChunk("assistant audio"));
+      ttsOptions.onAudioChunk(
+        options.ttsAudio
+          ? makePcmTtsChunk(options.ttsAudio)
+          : makeTtsChunk("assistant audio"),
+      );
       return makeTtsResult("assistant audio");
     });
     const harness = createHarness({
@@ -3340,6 +3382,9 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
       startVoiceTurn,
       streamTtsAudio,
       bargeInMinSpeechMs: options.bargeInMinSpeechMs,
+      echoEmaHalfLifeMs: options.echoEmaHalfLifeMs ?? 4,
+      echoBargeInMargin: options.echoBargeInMargin ?? 1,
+      echoDrainSlackMs: options.echoDrainSlackMs ?? 60_000,
       turnDetectorConfig: { silenceThresholdMs: 5_000 },
       ...(options.startFrame ? { startFrame: options.startFrame } : {}),
     });
@@ -3653,6 +3698,238 @@ describe("LiveVoiceSession sustained-speech barge-in guard", () => {
     ).toMatchObject({ type: "turn_cancelled", turnId: "live-turn-1" });
     await waitFor(() => abort.mock.calls.length === 1);
   });
+
+  describe("echo-adaptive barge-in", () => {
+    const playbackEchoChunk = tonePcm(4_700, 200);
+    const bargeInSpeechChunk = tonePcm(9_400, 530);
+    const playbackReference = tonePcm(4_700, 200, SAMPLE_RATE * 2);
+
+    test("steady loud playback echo does not interrupt the turn", async () => {
+      const { frames, session, abort, speakFirstReply } =
+        createSpeakingTurnHarness({
+          bargeInMinSpeechMs: 60,
+          echoBargeInMargin: 1.5,
+          echoEmaHalfLifeMs: 40,
+          ttsAudio: playbackReference,
+        });
+      await speakFirstReply();
+      const speechStartedBaseline = countType(frames, "speech_started");
+
+      for (let index = 0; index < 40; index += 1) {
+        await session.handleBinaryAudio(playbackEchoChunk);
+      }
+      await flushAsyncCallbacks();
+
+      expect(countType(frames, "speech_started")).toBe(speechStartedBaseline);
+      expect(countType(frames, "turn_cancelled")).toBe(0);
+      expect(abort).not.toHaveBeenCalled();
+    });
+
+    test("speech above the learned echo margin still interrupts", async () => {
+      const { frames, session, abort, speakFirstReply } =
+        createSpeakingTurnHarness({
+          bargeInMinSpeechMs: 60,
+          echoBargeInMargin: 1.5,
+          echoEmaHalfLifeMs: 400,
+          ttsAudio: playbackReference,
+        });
+      await speakFirstReply();
+
+      for (let index = 0; index < 25; index += 1) {
+        await session.handleBinaryAudio(playbackEchoChunk);
+      }
+      for (let index = 0; index < 8; index += 1) {
+        await session.handleBinaryAudio(bargeInSpeechChunk);
+      }
+
+      await waitFor(() => countType(frames, "turn_cancelled") === 1);
+      await waitFor(() => abort.mock.calls.length === 1);
+    });
+
+    test("classified echo resets a partial guard run immediately", async () => {
+      const { frames, session, abort, speakFirstReply } =
+        createSpeakingTurnHarness({
+          bargeInMinSpeechMs: 60,
+          echoBargeInMargin: 1.5,
+          echoEmaHalfLifeMs: 400,
+          ttsAudio: playbackReference,
+        });
+      await speakFirstReply();
+
+      for (let index = 0; index < 25; index += 1) {
+        await session.handleBinaryAudio(playbackEchoChunk);
+      }
+      for (let index = 0; index < 5; index += 1) {
+        await session.handleBinaryAudio(bargeInSpeechChunk);
+      }
+      await session.handleBinaryAudio(playbackEchoChunk);
+      await session.handleBinaryAudio(bargeInSpeechChunk);
+      await flushAsyncCallbacks();
+
+      expect(countType(frames, "turn_cancelled")).toBe(0);
+      expect(abort).not.toHaveBeenCalled();
+
+      for (let index = 0; index < 5; index += 1) {
+        await session.handleBinaryAudio(bargeInSpeechChunk);
+      }
+      await waitFor(() => countType(frames, "turn_cancelled") === 1);
+    });
+
+    test("quiet playback keeps fixed-threshold barge-in sensitivity", async () => {
+      const { frames, session, abort, speakFirstReply } =
+        createSpeakingTurnHarness({
+          bargeInMinSpeechMs: 60,
+          echoBargeInMargin: 1.5,
+          echoEmaHalfLifeMs: 40,
+          ttsAudio: playbackReference,
+        });
+      await speakFirstReply();
+
+      for (let index = 0; index < 31; index += 1) {
+        await session.handleBinaryAudio(pcm(200));
+      }
+      for (let index = 0; index < 7; index += 1) {
+        await session.handleBinaryAudio(bargeInSpeechChunk);
+      }
+
+      await waitFor(() => countType(frames, "turn_cancelled") === 1);
+      await waitFor(() => abort.mock.calls.length === 1);
+    });
+
+    test("playback echo is not forwarded as transcription pre-roll", async () => {
+      const { frames, session, transcribers, speakFirstReply } =
+        createSpeakingTurnHarness({
+          bargeInMinSpeechMs: 60,
+          echoBargeInMargin: 1.5,
+          echoEmaHalfLifeMs: 40,
+          ttsAudio: playbackReference,
+        });
+      await speakFirstReply();
+
+      const echoChunk = playbackEchoChunk;
+      for (let index = 0; index < 5; index += 1) {
+        await session.handleBinaryAudio(echoChunk);
+      }
+      for (let index = 0; index < 7; index += 1) {
+        await session.handleBinaryAudio(bargeInSpeechChunk);
+      }
+      await waitFor(() => countType(frames, "turn_cancelled") === 1);
+
+      const echoBuffer = Buffer.from(echoChunk);
+      expect(
+        transcribers.some((transcriber) =>
+          transcriber.received.some((buffer) => buffer.equals(echoBuffer)),
+        ),
+      ).toBe(false);
+    });
+
+    test("instant barge-in remains protected from onset echo", async () => {
+      const { frames, session, abort, speakFirstReply } =
+        createSpeakingTurnHarness({
+          bargeInMinSpeechMs: 0,
+          echoBargeInMargin: 1.5,
+          echoEmaHalfLifeMs: 40,
+          ttsAudio: playbackReference,
+        });
+      await speakFirstReply();
+
+      for (let index = 0; index < 30; index += 1) {
+        await session.handleBinaryAudio(playbackEchoChunk);
+      }
+      await flushAsyncCallbacks();
+      expect(countType(frames, "turn_cancelled")).toBe(0);
+
+      await session.handleBinaryAudio(bargeInSpeechChunk);
+      await waitFor(() => countType(frames, "turn_cancelled") === 1);
+      await waitFor(() => abort.mock.calls.length === 1);
+    });
+
+    test("echo suppression covers the client playback tail", async () => {
+      const { frames, session, abort, speakFirstReply, completeFirstReply } =
+        createSpeakingTurnHarness({
+          bargeInMinSpeechMs: 60,
+          echoBargeInMargin: 1.5,
+          echoEmaHalfLifeMs: 40,
+          ttsAudio: playbackReference,
+        });
+      await speakFirstReply();
+      completeFirstReply();
+      await waitFor(() => frames.some((frame) => frame.type === "tts_done"));
+      const speechStartedBaseline = countType(frames, "speech_started");
+
+      for (let index = 0; index < 40; index += 1) {
+        await session.handleBinaryAudio(playbackEchoChunk);
+      }
+      await flushAsyncCallbacks();
+
+      expect(countType(frames, "speech_started")).toBe(speechStartedBaseline);
+      expect(countType(frames, "turn_cancelled")).toBe(0);
+      expect(abort).not.toHaveBeenCalled();
+    });
+
+    test("speech at playback onset cannot seed its own echo threshold", async () => {
+      const { frames, session, abort, speakFirstReply, transcribers } =
+        createSpeakingTurnHarness({
+          bargeInMinSpeechMs: 250,
+          echoBargeInMargin: 1.5,
+          echoEmaHalfLifeMs: 400,
+          ttsAudio: playbackReference,
+        });
+      await speakFirstReply();
+
+      const onsetSpeech = tonePcm(9_400, 530, 7_200);
+      await session.handleBinaryAudio(onsetSpeech);
+
+      await waitFor(() => countType(frames, "turn_cancelled") === 1);
+      await waitFor(() => abort.mock.calls.length === 1);
+      expect(
+        transcribers.some((transcriber) =>
+          transcriber.received.some((buffer) =>
+            buffer.equals(Buffer.from(onsetSpeech)),
+          ),
+        ),
+      ).toBe(true);
+    });
+
+    test("speech already in progress bypasses playback warm-up", async () => {
+      let callbacks: VoiceTurnCallbacks | undefined;
+      const abort = mock();
+      const startVoiceTurn = mock(async (options: VoiceTurnOptions) => {
+        callbacks ??= options.callbacks;
+        return { turnId: "bridge-turn", abort };
+      });
+      const streamTtsAudio = mock(async (options: LiveVoiceTtsOptions) => {
+        options.onAudioChunk(makeTtsChunk("assistant audio"));
+        return makeTtsResult("assistant audio");
+      });
+      const { frames, session } = createHarness({
+        finals: ["what's the weather", "actually never mind"],
+        startVoiceTurn,
+        streamTtsAudio,
+        bargeInMinSpeechMs: 60,
+        echoBargeInMargin: 1.5,
+        echoEmaHalfLifeMs: 400,
+        echoDrainSlackMs: 60_000,
+        turnDetectorConfig: { silenceThresholdMs: 5_000 },
+      });
+
+      await session.start();
+      await session.handleBinaryAudio(LOUD_CHUNK);
+      await session.handleClientFrame({ type: "ptt_release" });
+      await waitFor(() => frames.some((frame) => frame.type === "thinking"));
+      for (let index = 0; index < 3; index += 1) {
+        await session.handleBinaryAudio(pcm(3_000));
+      }
+      callbacks?.assistant_text_delta?.(makeTextDelta("It is sunny today."));
+      await waitFor(() => frames.some((frame) => frame.type === "tts_audio"));
+      for (let index = 0; index < 3; index += 1) {
+        await session.handleBinaryAudio(pcm(3_000));
+      }
+
+      await waitFor(() => countType(frames, "turn_cancelled") === 1);
+      await waitFor(() => abort.mock.calls.length === 1);
+    });
+  });
 });
 
 describe("LiveVoiceSession unified front-door endpointing", () => {

package/src/live-voice/live-voice-session.ts

@@ -57,7 +57,11 @@ import {
   dominantLanguageTag,
   voteDominantLanguage,
 } from "../stt/language-metadata.js";
-import { detectPcm16SpeechActivity } from "../stt/speech-energy.js";
+import {
+  DEFAULT_SPEECH_ENERGY_THRESHOLD,
+  pcm16MaxNormalizedCorrelation,
+  pcm16MeanAmplitude,
+} from "../stt/speech-energy.js";
 import type {
   StreamingTranscriber,
   SttProviderId,
@@ -132,6 +136,13 @@ type LiveVoiceSessionState =
   | "failed"
   | "closed";
 
+type VadEnergyClassification = "speech" | "silence" | "echo";
+
+interface VadClassifiedChunk {
+  readonly chunk: Buffer;
+  readonly classification: VadEnergyClassification;
+}
+
 // Cap on audio buffered while a server-VAD utterance waits for its
 // transcriber (PCM16 mono seconds; oldest chunks are dropped past the cap).
 const SERVER_VAD_PENDING_AUDIO_MAX_SECONDS = 10;
@@ -151,6 +162,25 @@ const FINALIZE_GRACE_MS = 1_000;
 // liveVoice.vad.bargeInMinSpeechMs schema default; 0 disables the guard for
 // instant barge-in.
 const DEFAULT_BARGE_IN_MIN_SPEECH_MS = 250;
+// The playback echo gate learns microphone energy while assistant audio is
+// expected at the speaker. Input must rise above the learned level by this
+// margin to count as user speech.
+const DEFAULT_ECHO_BARGE_IN_MARGIN = 1.5;
+const DEFAULT_ECHO_EMA_HALF_LIFE_MS = 400;
+// Before learning a microphone power baseline, compare a short input window
+// with the PCM sent to the speaker. This keeps a user's first interruption
+// from becoming its own echo threshold.
+const ECHO_CORRELATION_PROBE_MS = 100;
+const ECHO_CORRELATION_MIN_MS = 50;
+const ECHO_CORRELATION_THRESHOLD = 0.65;
+const ECHO_REFERENCE_MAX_MS = 10_000;
+// Echo should reach the microphone near playback onset. If no signal arrives
+// within this much input audio, the gate returns to the fixed base threshold.
+// The same interval expires a learned reference after a real silent gap.
+const ECHO_ONSET_ELIGIBILITY_MS = 300;
+// Client buffering makes audible playback trail the server's send-time
+// estimate. Keep the echo window open briefly past that estimate.
+const DEFAULT_ECHO_DRAIN_SLACK_MS = 300;
 // Mirrors MediaTurnDetector's DEFAULT_SILENCE_THRESHOLD_MS: the session
 // tracks the effective trailing-silence threshold (the detector keeps its own
 // copy private) so the endpoint decider can report the pause length.
@@ -281,6 +311,17 @@ export interface LiveVoiceSessionOptions {
    * defaults to `DEFAULT_BARGE_IN_MIN_SPEECH_MS`.
    */
   bargeInMinSpeechMs?: number;
+  /**
+   * Multiplier over the learned playback echo level that input must exceed
+   * to count as speech while assistant audio is playing. Values at or below
+   * 1 disable adaptation for internal fixed-gate callers; workspace config
+   * requires a value greater than 1.
+   */
+  echoBargeInMargin?: number;
+  /** Half-life in milliseconds for the learned playback echo level. */
+  echoEmaHalfLifeMs?: number;
+  /** Extra time after the playback estimate during which echo is expected. */
+  echoDrainSlackMs?: number;
   /**
    * Overrides the bounded wait for the shared transcriber's finalize
    * flush in persistent mode (test hook). Defaults to `FINALIZE_GRACE_MS`.
@@ -1033,9 +1074,29 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
   private failureCode: LiveVoiceProtocolErrorCode | null = null;
   // Non-null iff the start frame requested turnDetection "server_vad".
   private readonly turnDetector: MediaTurnDetector | null;
-  // Energy gate for server-VAD speech classification; undefined defers to
-  // DEFAULT_SPEECH_ENERGY_THRESHOLD.
+  // Base energy gate for server-VAD speech classification. During estimated
+  // playback, classifyVadEnergy raises this above the learned echo level.
   private readonly speechEnergyThreshold: number | undefined;
+  private readonly echoBargeInMargin: number;
+  private readonly echoEmaHalfLifeMs: number;
+  private readonly echoDrainSlackMs: number;
+  // Learned microphone energy attributable to assistant playback.
+  private echoEnergyEma = 0;
+  // Signal-bearing microphone audio held until it can be compared with the
+  // assistant PCM. A nonmatch is replayed through VAD in original order.
+  private echoProbeChunks: Buffer[] = [];
+  // Recent raw assistant PCM from the current playback burst.
+  private echoReferenceAudio = Buffer.alloc(0);
+  private echoWindowTotalAudioMs = 0;
+  // Consecutive sub-base input expires a reference that can no longer
+  // describe audible playback.
+  private echoSubBaseRunMs = 0;
+  // Once onset eligibility lapses, later user speech cannot seed a new echo
+  // reference in the same playback window.
+  private echoOnsetLapsed = false;
+  // A live speech run that predates playback belongs to the user and bypasses
+  // echo warm-up until that run genuinely resets.
+  private echoWindowGuardCarryover = false;
   // Mutable so a mid-session `update_config` frame can retune "interrupt
   // sensitivity" live (see applyConfigUpdate).
   private bargeInMinSpeechMs: number;
@@ -1208,6 +1269,12 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       context.startFrame.bargeInMinSpeechMs ??
       options.bargeInMinSpeechMs ??
       DEFAULT_BARGE_IN_MIN_SPEECH_MS;
+    this.echoBargeInMargin =
+      options.echoBargeInMargin ?? DEFAULT_ECHO_BARGE_IN_MARGIN;
+    this.echoEmaHalfLifeMs =
+      options.echoEmaHalfLifeMs ?? DEFAULT_ECHO_EMA_HALF_LIFE_MS;
+    this.echoDrainSlackMs =
+      options.echoDrainSlackMs ?? DEFAULT_ECHO_DRAIN_SLACK_MS;
     this.finalizeGraceMs = options.finalizeGraceMs ?? FINALIZE_GRACE_MS;
     this.frontDecider = options.frontDecider ?? null;
     this.frontModelConfig = LiveVoiceFrontModelConfigSchema.parse(
@@ -1683,12 +1750,26 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       return;
     }
 
-    const hasSpeech = detectPcm16SpeechActivity(
-      chunk,
-      this.speechEnergyThreshold,
-    );
+    for (const classified of this.classifyVadEnergy(chunk)) {
+      await this.handleClassifiedVadAudio(detector, classified);
+    }
+  }
+
+  private async handleClassifiedVadAudio(
+    detector: MediaTurnDetector,
+    classified: VadClassifiedChunk,
+  ): Promise<void> {
+    const { chunk, classification: energyClassification } = classified;
+    const hasSpeech = energyClassification === "speech";
     detector.onMediaChunk(hasSpeech);
-    this.trackBargeInGuard(hasSpeech, chunk);
+    this.trackBargeInGuard(energyClassification, chunk);
+
+    // Playback echo is neither user audio nor useful pre-roll. Dropping it
+    // prevents the assistant's reply from reaching transcription as a ghost
+    // follow-up turn.
+    if (energyClassification === "echo") {
+      return;
+    }
 
     // Idle mic: hold silent chunks in the bounded pre-roll instead of
     // collecting or streaming them; flushed on speech onset so the
@@ -1766,6 +1847,193 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     await this.routeVadAudio(utterance, chunk);
   }
 
+  /**
+   * Classify microphone energy while keeping assistant playback echo out of
+   * barge-in, turn detection, pre-roll, and transcription.
+   *
+   * A short onset probe must correlate with PCM sent to the speaker before its
+   * microphone power can seed the adaptive threshold. Nonmatching probe audio
+   * is replayed through VAD in original order, so a user who talks at playback
+   * onset is neither learned as echo nor lost. Once seeded, the EMA follows
+   * confirmed echo while speech above the learned margin remains frozen out.
+   */
+  private classifyVadEnergy(chunk: Buffer): VadClassifiedChunk[] {
+    const baseThreshold =
+      this.speechEnergyThreshold ?? DEFAULT_SPEECH_ENERGY_THRESHOLD;
+    const meanAmplitude = pcm16MeanAmplitude(chunk);
+    if (
+      this.echoBargeInMargin <= 1 ||
+      !this.isAssistantPlaybackEchoPossible()
+    ) {
+      this.resetEchoReference();
+      return [
+        this.classifyAtFixedThreshold(chunk, baseThreshold, meanAmplitude),
+      ];
+    }
+
+    if (this.echoWindowTotalAudioMs === 0) {
+      this.echoWindowGuardCarryover =
+        this.pendingBargeIn !== null && this.pendingBargeIn.speechMs > 0;
+    } else if (this.pendingBargeIn === null) {
+      this.echoWindowGuardCarryover = false;
+    }
+
+    const chunkMs = pcm16DurationMs(
+      chunk.byteLength,
+      this.context.startFrame.audio.sampleRate,
+    );
+    const onsetWasEligible =
+      !this.echoOnsetLapsed &&
+      this.echoWindowTotalAudioMs < ECHO_ONSET_ELIGIBILITY_MS;
+    this.echoWindowTotalAudioMs += chunkMs;
+
+    if (this.echoProbeChunks.length > 0) {
+      this.echoProbeChunks.push(Buffer.from(chunk));
+      return this.resolveEchoProbe(baseThreshold);
+    }
+
+    if (meanAmplitude <= baseThreshold) {
+      this.echoSubBaseRunMs += chunkMs;
+      if (this.echoSubBaseRunMs >= ECHO_ONSET_ELIGIBILITY_MS) {
+        this.echoEnergyEma = 0;
+        this.echoOnsetLapsed = true;
+      }
+      return [{ chunk, classification: "silence" }];
+    }
+
+    this.echoSubBaseRunMs = 0;
+    if (
+      this.echoEnergyEma === 0 &&
+      onsetWasEligible &&
+      !this.echoWindowGuardCarryover
+    ) {
+      this.echoProbeChunks.push(Buffer.from(chunk));
+      return this.resolveEchoProbe(baseThreshold);
+    }
+
+    if (this.echoEnergyEma === 0) {
+      this.echoOnsetLapsed = true;
+      return [{ chunk, classification: "speech" }];
+    }
+
+    const speechThreshold = Math.max(
+      baseThreshold,
+      this.echoBargeInMargin * this.echoEnergyEma,
+    );
+    if (meanAmplitude > speechThreshold) {
+      const guardHasSpeech =
+        this.pendingBargeIn !== null && this.pendingBargeIn.speechMs > 0;
+      if (!guardHasSpeech && this.echoMatchesAssistant(chunk)) {
+        this.updateEchoEnergy(meanAmplitude, chunkMs);
+        return [{ chunk, classification: "echo" }];
+      }
+      return [{ chunk, classification: "speech" }];
+    }
+
+    this.updateEchoEnergy(meanAmplitude, chunkMs);
+    return [{ chunk, classification: "echo" }];
+  }
+
+  private resolveEchoProbe(baseThreshold: number): VadClassifiedChunk[] {
+    const probe = Buffer.concat(this.echoProbeChunks);
+    const probeAudioMs = pcm16DurationMs(
+      probe.byteLength,
+      this.context.startFrame.audio.sampleRate,
+    );
+    if (
+      probeAudioMs >= ECHO_CORRELATION_MIN_MS &&
+      this.echoMatchesAssistant(probe)
+    ) {
+      this.echoEnergyEma = Math.max(baseThreshold, pcm16MeanAmplitude(probe));
+      const chunks = this.echoProbeChunks.splice(0);
+      return chunks.map((chunk) => ({ chunk, classification: "echo" }));
+    }
+    if (probeAudioMs < ECHO_CORRELATION_PROBE_MS) {
+      return [];
+    }
+
+    this.echoOnsetLapsed = true;
+    const chunks = this.echoProbeChunks.splice(0);
+    return chunks.map((chunk) =>
+      this.classifyAtFixedThreshold(chunk, baseThreshold),
+    );
+  }
+
+  private echoMatchesAssistant(chunk: Buffer): boolean {
+    const sampleRate = this.context.startFrame.audio.sampleRate;
+    const minimumBytes = Math.ceil(
+      (sampleRate * ECHO_CORRELATION_MIN_MS * 2) / 1_000,
+    );
+    if (
+      chunk.byteLength < minimumBytes ||
+      this.echoReferenceAudio.byteLength < minimumBytes
+    ) {
+      return false;
+    }
+    const probeByteLength = Math.min(
+      chunk.byteLength,
+      Math.ceil((sampleRate * ECHO_CORRELATION_PROBE_MS * 2) / 1_000),
+    );
+    return (
+      pcm16MaxNormalizedCorrelation(
+        chunk.subarray(0, probeByteLength),
+        this.echoReferenceAudio,
+      ) >= ECHO_CORRELATION_THRESHOLD
+    );
+  }
+
+  private updateEchoEnergy(meanAmplitude: number, chunkMs: number): void {
+    const alpha = 1 - 0.5 ** (chunkMs / this.echoEmaHalfLifeMs);
+    this.echoEnergyEma =
+      alpha * meanAmplitude + (1 - alpha) * this.echoEnergyEma;
+  }
+
+  private classifyAtFixedThreshold(
+    chunk: Buffer,
+    baseThreshold: number,
+    meanAmplitude = pcm16MeanAmplitude(chunk),
+  ): VadClassifiedChunk {
+    return {
+      chunk,
+      classification: meanAmplitude > baseThreshold ? "speech" : "silence",
+    };
+  }
+
+  private isAssistantPlaybackEchoPossible(): boolean {
+    return (
+      Date.now() < this.assistantPlaybackTailUntilMs + this.echoDrainSlackMs
+    );
+  }
+
+  private resetEchoReference(): void {
+    this.echoEnergyEma = 0;
+    this.echoProbeChunks = [];
+    this.echoReferenceAudio = Buffer.alloc(0);
+    this.echoWindowTotalAudioMs = 0;
+    this.echoSubBaseRunMs = 0;
+    this.echoOnsetLapsed = false;
+    this.echoWindowGuardCarryover = false;
+  }
+
+  private appendEchoReference(chunk: LiveVoiceTtsAudioChunk): void {
+    if (
+      chunk.contentType.split(";", 1)[0]?.trim().toLowerCase() !==
+        "audio/pcm" ||
+      chunk.sampleRate !== this.context.startFrame.audio.sampleRate
+    ) {
+      return;
+    }
+    const audio = Buffer.from(chunk.dataBase64, "base64");
+    const maxBytes = Math.ceil(
+      (chunk.sampleRate * ECHO_REFERENCE_MAX_MS * 2) / 1_000,
+    );
+    const combined = Buffer.concat([this.echoReferenceAudio, audio]);
+    this.echoReferenceAudio =
+      combined.byteLength > maxBytes
+        ? combined.subarray(combined.byteLength - maxBytes)
+        : combined;
+  }
+
   private async routeVadAudio(
     utterance: UtteranceCycle,
     chunk: Buffer,
@@ -1913,7 +2181,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     // The client can still be draining audible playback after tts_done
     // (the turn is already cleared server-side) — that tail deserves the
     // same guard, or a noise blip clips the reply's last words.
-    const drainingPlayback = Date.now() < this.assistantPlaybackTailUntilMs;
+    const drainingPlayback = this.isAssistantPlaybackEchoPossible();
 
     if ((bargeableTurn || drainingPlayback) && this.bargeInMinSpeechMs > 0) {
       // Onset audio keeps flowing into the cycle/pre-roll while the guard
@@ -1935,13 +2203,14 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     }
   }
 
-  // Advances the sustained-speech barge-in guard by one server-VAD chunk:
-  // above-gate speech accumulates toward bargeInMinSpeechMs while brief
-  // sub-threshold gaps are tolerated (a single continuous silence longer than
-  // BARGE_IN_GAP_TOLERANCE_MS, or cumulative tolerated silence past the
-  // duty-cycle ceiling, zeroes the run), and once met the deferred
-  // speech_started + barge-in fire.
-  private trackBargeInGuard(hasSpeech: boolean, chunk: Buffer): void {
+  // Advance the sustained-speech barge-in guard by one server-VAD chunk.
+  // Speech accumulates toward bargeInMinSpeechMs, short true-silence gaps are
+  // tolerated, and classified playback echo resets the run immediately.
+  // Longer or mostly silent runs reset through the existing gap limits.
+  private trackBargeInGuard(
+    classification: VadEnergyClassification,
+    chunk: Buffer,
+  ): void {
     const guard = this.pendingBargeIn;
     if (!guard) {
       return;
@@ -1950,7 +2219,11 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
       chunk.byteLength,
       this.context.startFrame.audio.sampleRate,
     );
-    if (!hasSpeech) {
+    if (classification === "echo") {
+      this.resetBargeInGuardRun();
+      return;
+    }
+    if (classification === "silence") {
       guard.silenceMs += chunkMs;
       guard.toleratedSilenceMs += chunkMs;
       // Strictly greater on the per-gap check: a gap of exactly
@@ -1964,9 +2237,7 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
         guard.toleratedSilenceMs >
           this.bargeInMinSpeechMs * BARGE_IN_MAX_TOLERATED_SILENCE_RATIO
       ) {
-        guard.speechMs = 0;
-        guard.silenceMs = 0;
-        guard.toleratedSilenceMs = 0;
+        this.resetBargeInGuardRun();
       }
       return;
     }
@@ -1984,6 +2255,21 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
     }
   }
 
+  private resetBargeInGuardRun(): void {
+    const guard = this.pendingBargeIn;
+    if (!guard) {
+      return;
+    }
+    guard.speechMs = 0;
+    guard.silenceMs = 0;
+    guard.toleratedSilenceMs = 0;
+    if (this.echoWindowGuardCarryover) {
+      this.echoWindowGuardCarryover = false;
+      this.echoEnergyEma = 0;
+      this.echoProbeChunks = [];
+    }
+  }
+
   private bargeIn(turn: ActiveAssistantTurn): void {
     // Abort synchronously so no tts_audio frame can follow turn_cancelled,
     // and settle the cancelled turn's metrics so the next utterance's marks
@@ -5328,6 +5614,10 @@ export class LiveVoiceSession implements LiveVoiceSessionContract {
         chunk.sampleRate,
       );
       const now = Date.now();
+      if (!this.isAssistantPlaybackEchoPossible()) {
+        this.resetEchoReference();
+      }
+      this.appendEchoReference(chunk);
       this.assistantPlaybackTailUntilMs =
         Math.max(now, this.assistantPlaybackTailUntilMs) + chunkMs;
       const turnAfterSend = this.activeAssistantTurn;
@@ -5832,6 +6122,11 @@ export function createLiveVoiceSession(
       options.speechEnergyThreshold ?? vadConfig?.speechEnergyThreshold,
     bargeInMinSpeechMs:
       options.bargeInMinSpeechMs ?? vadConfig?.bargeInMinSpeechMs,
+    echoBargeInMargin:
+      options.echoBargeInMargin ?? vadConfig?.echoBargeInMargin,
+    echoEmaHalfLifeMs:
+      options.echoEmaHalfLifeMs ?? vadConfig?.echoEmaHalfLifeMs,
+    echoDrainSlackMs: options.echoDrainSlackMs ?? vadConfig?.echoDrainSlackMs,
     frontModelConfig,
     // Eager construction is safe even when the `liveVoice.frontModel` config
     // namespace is absent — schema defaults fill the tunables. An explicit

package/src/stt/__tests__/speech-energy.test.ts

@@ -3,6 +3,8 @@ import { describe, expect, test } from "bun:test";
 import {
   DEFAULT_SPEECH_ENERGY_THRESHOLD,
   detectPcm16SpeechActivity,
+  pcm16MaxNormalizedCorrelation,
+  pcm16MeanAmplitude,
 } from "../speech-energy.js";
 
 /** Build a PCM16LE buffer from an array of sample values. */
@@ -66,3 +68,80 @@ describe("detectPcm16SpeechActivity", () => {
     expect(detectPcm16SpeechActivity(quiet, 500)).toBe(false);
   });
 });
+
+describe("pcm16MeanAmplitude", () => {
+  test("returns 0 for an empty buffer", () => {
+    expect(pcm16MeanAmplitude(Buffer.alloc(0))).toBe(0);
+  });
+
+  test("returns the exact mean absolute amplitude", () => {
+    expect(pcm16MeanAmplitude(pcm16([1_000, -2_000, 3_000, -4_000]))).toBe(
+      2_500,
+    );
+  });
+
+  test("ignores a trailing odd byte", () => {
+    const samples = pcm16([3_000, -3_000]);
+    expect(
+      pcm16MeanAmplitude(Buffer.concat([samples, Buffer.from([0x01])])),
+    ).toBe(3_000);
+  });
+
+  test("matches the detector's threshold comparison", () => {
+    const chunks = [
+      Buffer.alloc(0),
+      pcm16([0, 0]),
+      pcm16([500, -500]),
+      pcm16([DEFAULT_SPEECH_ENERGY_THRESHOLD]),
+      pcm16([DEFAULT_SPEECH_ENERGY_THRESHOLD + 1]),
+    ];
+    for (const chunk of chunks) {
+      expect(detectPcm16SpeechActivity(chunk)).toBe(
+        pcm16MeanAmplitude(chunk) > DEFAULT_SPEECH_ENERGY_THRESHOLD,
+      );
+    }
+  });
+});
+
+describe("pcm16MaxNormalizedCorrelation", () => {
+  const wave = (frequency: number, count: number): Buffer =>
+    pcm16(
+      Array.from({ length: count }, (_, index) =>
+        Math.round(
+          8_000 * Math.sin((2 * Math.PI * frequency * index) / 16_000),
+        ),
+      ),
+    );
+
+  test("finds a matching waveform at an arbitrary reference offset", () => {
+    const input = wave(240, 800);
+    const reference = Buffer.concat([wave(410, 400), input, wave(610, 400)]);
+
+    expect(pcm16MaxNormalizedCorrelation(input, reference)).toBeGreaterThan(
+      0.99,
+    );
+  });
+
+  test("is invariant to gain and polarity", () => {
+    const inputSamples = Array.from({ length: 800 }, (_, index) =>
+      Math.round(5_000 * Math.sin((2 * Math.PI * index) / 83)),
+    );
+    const inverted = pcm16(inputSamples.map((sample) => -2 * sample));
+
+    expect(
+      pcm16MaxNormalizedCorrelation(pcm16(inputSamples), inverted),
+    ).toBeGreaterThan(0.99);
+  });
+
+  test("rejects an unrelated waveform and flat power", () => {
+    expect(
+      pcm16MaxNormalizedCorrelation(wave(240, 800), wave(610, 1_600)),
+    ).toBeLessThan(0.3);
+    expect(
+      pcm16MaxNormalizedCorrelation(
+        pcm16(new Array(800).fill(3_000)),
+        pcm16(new Array(1_600).fill(3_000)),
+      ),
+    ).toBe(0);
+  });
+});

package/src/stt/speech-energy.ts

@@ -18,6 +18,120 @@
  */
 export const DEFAULT_SPEECH_ENERGY_THRESHOLD = 800;
 
+/**
+ * Return the mean absolute amplitude of little-endian signed PCM16 audio.
+ * Empty buffers return 0, and a trailing odd byte is ignored.
+ */
+export function pcm16MeanAmplitude(chunk: Buffer): number {
+  const sampleCount = Math.floor(chunk.length / 2);
+  if (sampleCount === 0) {
+    return 0;
+  }
+
+  let totalAmplitude = 0;
+  for (let i = 0; i < sampleCount; i += 1) {
+    totalAmplitude += Math.abs(chunk.readInt16LE(i * 2));
+  }
+  return totalAmplitude / sampleCount;
+}
+
+/**
+ * Find the strongest normalized correlation between a PCM16 input window and
+ * any same-length window in a PCM16 reference. The samples are block-averaged
+ * before matching so this stays cheap enough for live audio and remains
+ * tolerant of the low-pass filtering introduced by speakers and microphones.
+ *
+ * The absolute coefficient makes polarity inversion harmless. A flat input or
+ * reference has no identifying waveform and returns 0 instead of being treated
+ * as a match based on level alone.
+ */
+export function pcm16MaxNormalizedCorrelation(
+  input: Buffer,
+  reference: Buffer,
+  downsampleFactor = 8,
+): number {
+  if (!Number.isInteger(downsampleFactor) || downsampleFactor <= 0) {
+    return 0;
+  }
+
+  let inputSamples = blockAveragePcm16(input, downsampleFactor);
+  const referenceSamples = blockAveragePcm16(reference, downsampleFactor);
+  if (inputSamples.length > referenceSamples.length) {
+    inputSamples = inputSamples.subarray(0, referenceSamples.length);
+  }
+  if (inputSamples.length < 2) {
+    return 0;
+  }
+
+  let inputSum = 0;
+  for (let index = 0; index < inputSamples.length; index += 1) {
+    inputSum += inputSamples[index]!;
+  }
+  const inputMean = inputSum / inputSamples.length;
+  const centeredInput = new Float64Array(inputSamples.length);
+  let inputEnergy = 0;
+  for (let index = 0; index < inputSamples.length; index += 1) {
+    const centered = inputSamples[index]! - inputMean;
+    centeredInput[index] = centered;
+    inputEnergy += centered * centered;
+  }
+  if (inputEnergy === 0) {
+    return 0;
+  }
+
+  const prefixSum = new Float64Array(referenceSamples.length + 1);
+  const prefixSquareSum = new Float64Array(referenceSamples.length + 1);
+  for (let index = 0; index < referenceSamples.length; index += 1) {
+    const sample = referenceSamples[index]!;
+    prefixSum[index + 1] = prefixSum[index]! + sample;
+    prefixSquareSum[index + 1] = prefixSquareSum[index]! + sample * sample;
+  }
+
+  let best = 0;
+  const windowLength = inputSamples.length;
+  for (
+    let offset = 0;
+    offset + windowLength <= referenceSamples.length;
+    offset += 1
+  ) {
+    const referenceSum = prefixSum[offset + windowLength]! - prefixSum[offset]!;
+    const referenceSquareSum =
+      prefixSquareSum[offset + windowLength]! - prefixSquareSum[offset]!;
+    const referenceEnergy =
+      referenceSquareSum - (referenceSum * referenceSum) / windowLength;
+    if (referenceEnergy <= 0) {
+      continue;
+    }
+
+    let dotProduct = 0;
+    for (let index = 0; index < windowLength; index += 1) {
+      dotProduct += centeredInput[index]! * referenceSamples[offset + index]!;
+    }
+    const correlation =
+      Math.abs(dotProduct) / Math.sqrt(inputEnergy * referenceEnergy);
+    best = Math.max(best, Math.min(correlation, 1));
+  }
+  return best;
+}
+
+function blockAveragePcm16(
+  chunk: Buffer,
+  downsampleFactor: number,
+): Float64Array {
+  const sampleCount = Math.floor(chunk.length / 2);
+  const blockCount = Math.floor(sampleCount / downsampleFactor);
+  const result = new Float64Array(blockCount);
+  for (let block = 0; block < blockCount; block += 1) {
+    let sum = 0;
+    const firstSample = block * downsampleFactor;
+    for (let offset = 0; offset < downsampleFactor; offset += 1) {
+      sum += chunk.readInt16LE((firstSample + offset) * 2);
+    }
+    result[block] = sum / downsampleFactor;
+  }
+  return result;
+}
+
 /**
  * Detect speech activity in a chunk of little-endian signed 16-bit mono
  * PCM samples.
@@ -34,16 +148,5 @@ export function detectPcm16SpeechActivity(
   chunk: Buffer,
   threshold = DEFAULT_SPEECH_ENERGY_THRESHOLD,
 ): boolean {
-  const sampleCount = Math.floor(chunk.length / 2);
-  if (sampleCount === 0) {
-    return false;
-  }
-
-  let totalAmplitude = 0;
-  for (let i = 0; i < sampleCount; i++) {
-    totalAmplitude += Math.abs(chunk.readInt16LE(i * 2));
-  }
-  const avgAmplitude = totalAmplitude / sampleCount;
-
-  return avgAmplitude > threshold;
+  return pcm16MeanAmplitude(chunk) > threshold;
 }