@elizaos/plugin-local-inference 2.0.3-beta.2 → 2.0.3-beta.3

package/src/services/voice/audio-frame-consumer.test.ts

@@ -16,6 +16,7 @@
  */
 
 import { describe, expect, it } from "vitest";
+import type { HandleLiveVoiceAttributionOptions } from "../../runtime/voice-entity-binding";
 import {
 	type AttributionPipelineLike,
 	AudioFrameConsumer,
@@ -23,8 +24,12 @@ import {
 	type AudioFrameEvent,
 	decodeAudioFramePcm,
 	type RuntimeEventSink,
+	type SelfVoiceSimilarityResolver,
+	type TurnTranscriber,
+	type VadSegmenter,
 } from "./audio-frame-consumer";
 import type { VoiceAttributionOutput } from "./speaker/attribution-pipeline";
+import type { PcmFrame, VadEvent } from "./types";
 import { VadDetector } from "./vad";
 
 const SR = 16_000;
@@ -119,6 +124,9 @@ function makeFrame(opts: {
 function buildHarness(
 	probs: readonly number[],
 	entityId: string | null = "entity-x",
+	transcribe?: TurnTranscriber,
+	attributionOptions?: Partial<HandleLiveVoiceAttributionOptions>,
+	resolveSelfVoiceSimilarity?: SelfVoiceSimilarityResolver,
 ) {
 	const silero = new ScriptedSilero(probs);
 	const vad = new VadDetector(silero, {
@@ -131,13 +139,20 @@ function buildHarness(
 	const pipeline = new FakePipeline(entityId);
 	const runtime = new FakeRuntime();
 	const consumer = new AudioFrameConsumer(
-		{ vad, pipeline, runtime },
+		{
+			vad,
+			pipeline,
+			runtime,
+			...(transcribe ? { transcribe } : {}),
+			...(resolveSelfVoiceSimilarity ? { resolveSelfVoiceSimilarity } : {}),
+		},
 		{
 			source: { kind: "device", deviceId: "pixel" },
 			attributionOptions: {
 				ownerEntityId: "entity-x",
 				knownSpeakerEntityIds: ["entity-x"],
 				endOfTurnProbability: 0.95,
+				...attributionOptions,
 			},
 			preRollSeconds: 0, // deterministic buffering for assertions
 			maxTurnSeconds: 30,
@@ -146,6 +161,26 @@ function buildHarness(
 	return { consumer, pipeline, runtime, vad };
 }
 
+/** Feed one loud turn (speech then silence) and flush, so exactly one turn
+ *  finalizes. Shared by the transcript-join tests. */
+async function driveOneTurn(consumer: AudioFrameConsumer): Promise<void> {
+	let ts = 1000;
+	let idx = 0;
+	for (let i = 0; i < 40; i++) {
+		await consumer.onAudioFrame(
+			makeFrame({ amplitude: 0.6, timestamp: ts, frameIndex: idx++ }),
+		);
+		ts += 20;
+	}
+	for (let i = 0; i < 24; i++) {
+		await consumer.onAudioFrame(
+			makeFrame({ amplitude: 0.0, timestamp: ts, frameIndex: idx++ }),
+		);
+		ts += 20;
+	}
+	await consumer.flush();
+}
+
 describe("decodeAudioFramePcm", () => {
 	it("decodes base64 LE-s16 mono → Float32 [-1,1]", () => {
 		const frame = makeFrame({ amplitude: 0.5, timestamp: 0, frameIndex: 0 });
@@ -284,6 +319,41 @@ describe("AudioFrameConsumer", () => {
 		expect(metaSignal).toBeTruthy();
 	});
 
+	it("passes live selfVoiceSimilarity into the gate and suppresses agent echo", async () => {
+		const probs = [...Array(24).fill(0.9), ...Array(12).fill(0.0)];
+		const resolveSelfVoiceSimilarity: SelfVoiceSimilarityResolver = (
+			embedding,
+			output,
+		) => {
+			expect(embedding).toBe(output.observation?.embedding);
+			expect(embedding.length).toBe(256);
+			return 0.91;
+		};
+		const { consumer } = buildHarness(
+			probs,
+			"entity-x",
+			undefined,
+			{ agentSpeaking: true },
+			resolveSelfVoiceSimilarity,
+		);
+		let signal: {
+			agentShouldSpeak: boolean | null;
+			nextSpeaker: string;
+			metadata?: { provenance?: string; selfVoiceSimilarity?: number };
+		} | null = null;
+		consumer.onTurn((t) => {
+			signal = t.signal as typeof signal;
+		});
+
+		await driveOneTurn(consumer);
+
+		expect(signal).not.toBeNull();
+		expect(signal?.agentShouldSpeak).toBe(false);
+		expect(signal?.nextSpeaker).toBe("user");
+		expect(signal?.metadata?.provenance).toBe("voice-bridge+self-voice");
+		expect(signal?.metadata?.selfVoiceSimilarity).toBeCloseTo(0.91);
+	});
+
 	it("does not segment a turn from pure silence", async () => {
 		const { consumer, pipeline } = buildHarness(Array(40).fill(0.0));
 		let ts = 1000;
@@ -341,3 +411,259 @@ describe("AudioFrameConsumer", () => {
 		}
 	});
 });
+
+describe("AudioFrameConsumer — ASR transcript join (#8786)", () => {
+	const TURN_PROBS = [...Array(24).fill(0.9), ...Array(12).fill(0.0)];
+
+	it("joins the per-turn ASR transcript onto VOICE_TURN_OBSERVED", async () => {
+		const seen: Array<{ length: number; sampleRate: number }> = [];
+		const transcribe: TurnTranscriber = (pcm, sampleRate) => {
+			seen.push({ length: pcm.length, sampleRate });
+			return "  I'm Jill  ";
+		};
+		const { consumer, runtime } = buildHarness(
+			TURN_PROBS,
+			"entity-x",
+			transcribe,
+		);
+		await driveOneTurn(consumer);
+
+		// The transcriber saw the real buffered turn PCM at 16 kHz.
+		expect(seen.length).toBe(1);
+		expect(seen[0].sampleRate).toBe(16_000);
+		expect(seen[0].length).toBeGreaterThan(SR * 0.4);
+		// VOICE_TURN_OBSERVED now carries the trimmed transcript (was "" before).
+		expect(runtime.emitted.length).toBe(1);
+		expect(runtime.emitted[0].payload.text).toBe("I'm Jill");
+	});
+
+	it("stays diarization-only (empty text) when no transcriber is wired", async () => {
+		const { consumer, runtime } = buildHarness(TURN_PROBS, "entity-x");
+		await driveOneTurn(consumer);
+		expect(runtime.emitted.length).toBe(1);
+		expect(runtime.emitted[0].payload.text).toBe("");
+	});
+
+	it("degrades to a transcript-less turn when ASR throws (turn kept)", async () => {
+		const transcribe: TurnTranscriber = () => {
+			throw new Error("asr decode failed");
+		};
+		const { consumer, runtime } = buildHarness(
+			TURN_PROBS,
+			"entity-x",
+			transcribe,
+		);
+		await driveOneTurn(consumer);
+		// The diarized turn still emits; only the transcript is dropped, counted.
+		expect(runtime.emitted.length).toBe(1);
+		expect(runtime.emitted[0].payload.text).toBe("");
+		expect(consumer.transcriptionErrors).toBe(1);
+	});
+
+	it("ignores an empty/whitespace transcript (no text stamped)", async () => {
+		const transcribe: TurnTranscriber = () => "   ";
+		const { consumer, runtime } = buildHarness(
+			TURN_PROBS,
+			"entity-x",
+			transcribe,
+		);
+		await driveOneTurn(consumer);
+		expect(runtime.emitted.length).toBe(1);
+		expect(runtime.emitted[0].payload.text).toBe("");
+	});
+});
+
+// --- echo cancellation wiring (#9455) ----------------------------------------
+
+/** VadSegmenter that just records every frame the consumer pushes downstream. */
+class RecordingVad implements VadSegmenter {
+	readonly frames: Float32Array[] = [];
+	get inSpeech(): boolean {
+		return false;
+	}
+	onVadEvent(_listener: (event: VadEvent) => void): () => void {
+		return () => {};
+	}
+	async pushFrame(frame: PcmFrame): Promise<void> {
+		this.frames.push(frame.pcm);
+	}
+	async flush(): Promise<void> {}
+	reset(): void {}
+}
+
+describe("AudioFrameConsumer — echo cancellation (#9455)", () => {
+	const SR = 16000;
+	const BLOCK = 320;
+	function farSignal(n: number, seed = 1): Float32Array {
+		const x = new Float32Array(n);
+		let s = seed >>> 0;
+		let p1 = 0;
+		let p2 = 0;
+		for (let i = 0; i < n; i++) {
+			s = (s * 1103515245 + 12345) & 0x7fffffff;
+			const w = s / 0x3fffffff - 1;
+			p1 = 0.92 * p1 + 0.08 * w;
+			p2 = 0.85 * p2 + 0.15 * p1;
+			x[i] = p2 * 3;
+		}
+		return x;
+	}
+	function echoOf(x: Float32Array): Float32Array {
+		const delay = 35;
+		const tail = 90;
+		const h = new Float32Array(delay + tail);
+		for (let k = 0; k < tail; k++)
+			h[delay + k] = Math.exp(-k / 25) * (k % 2 ? -0.6 : 0.8) * 0.22;
+		const y = new Float32Array(x.length);
+		for (let n = 0; n < x.length; n++) {
+			let acc = 0;
+			for (let k = 0; k < h.length; k++) if (n - k >= 0) acc += h[k] * x[n - k];
+			y[n] = acc;
+		}
+		return y;
+	}
+	const power = (a: Float32Array) =>
+		a.reduce((p, v) => p + v * v, 0) / Math.max(1, a.length);
+
+	function makeConsumer(vad: RecordingVad, far: Float32Array | null) {
+		return new AudioFrameConsumer(
+			{
+				vad,
+				pipeline: new FakePipeline("e"),
+				runtime: new FakeRuntime(),
+				...(far
+					? {
+							echoReference: (ts: number, samples: number) => {
+								const off = Math.round((ts - 1000) / 20) * BLOCK;
+								return far.subarray(off, off + samples);
+							},
+						}
+					: {}),
+			},
+			{ source: { kind: "device", deviceId: "pixel" }, preRollSeconds: 0 },
+		);
+	}
+
+	it("cancels the agent's echo on the mic before VAD when echoReference is wired", async () => {
+		const N = SR * 3;
+		const far = farSignal(N);
+		const echo = echoOf(far); // the agent's TTS leaking into the mic
+		const vad = new RecordingVad();
+		const consumer = makeConsumer(vad, far);
+		let ts = 1000;
+		for (let off = 0; off + BLOCK <= N; off += BLOCK) {
+			await consumer.pushDecodedFrame(echo.subarray(off, off + BLOCK), ts);
+			ts += 20;
+		}
+		// after convergence, the recorded (post-AEC) frames carry far less echo
+		// energy than the raw mic frames did.
+		const lateOut = vad.frames[vad.frames.length - 1];
+		const lateRawOff = (vad.frames.length - 1) * BLOCK;
+		const lateRaw = echo.subarray(lateRawOff, lateRawOff + BLOCK);
+		expect(power(lateOut)).toBeLessThan(power(lateRaw) * 0.1); // >10 dB
+	});
+
+	it("leaves the mic untouched when no echoReference is wired", async () => {
+		const vad = new RecordingVad();
+		const consumer = makeConsumer(vad, null);
+		const frame = farSignal(BLOCK, 7);
+		await consumer.pushDecodedFrame(frame, 1000);
+		expect(Array.from(vad.frames[0])).toEqual(Array.from(frame));
+	});
+
+	it("skips the canceller entirely while the agent is silent (#9649 fast path)", async () => {
+		// The reference provider returns far PCM while the agent plays, then null
+		// once it stops. Frames during silence must be EXACT passthrough — proving
+		// the canceller is not invoked at all (so it can't subtract a stale echo
+		// estimate against converged weights) — and must not increment the
+		// cancelled-frame counter.
+		const N = SR * 2;
+		const far = farSignal(N);
+		const echo = echoOf(far);
+		const PLAYBACK_FRAMES = 40; // agent plays for the first 40 frames, then stops
+		const vad = new RecordingVad();
+		const consumer = new AudioFrameConsumer(
+			{
+				vad,
+				pipeline: new FakePipeline("skip"),
+				runtime: new FakeRuntime(),
+				echoReference: (ts: number, samples: number) => {
+					const idx = Math.round((ts - 1000) / 20);
+					if (idx >= PLAYBACK_FRAMES) return null; // agent silent
+					const off = idx * BLOCK;
+					return far.subarray(off, off + samples);
+				},
+			},
+			{ source: { kind: "device", deviceId: "pixel" }, preRollSeconds: 0 },
+		);
+
+		// Mic carries echo while the agent plays, then pure (distinct) near speech.
+		const nearSilentEra = farSignal(N, 555);
+		let ts = 1000;
+		let frameIdx = 0;
+		const silentInputs: Float32Array[] = [];
+		for (let off = 0; off + BLOCK <= N; off += BLOCK, frameIdx++) {
+			const mic =
+				frameIdx < PLAYBACK_FRAMES
+					? echo.subarray(off, off + BLOCK)
+					: nearSilentEra.subarray(off, off + BLOCK);
+			if (frameIdx >= PLAYBACK_FRAMES) silentInputs.push(mic);
+			await consumer.pushDecodedFrame(mic, ts);
+			ts += 20;
+		}
+
+		// Only the playback frames were cancelled; silent frames took the fast path.
+		expect(consumer.echoFramesCancelled).toBe(PLAYBACK_FRAMES);
+
+		// Every silent-era frame is bit-identical to its input (no canceller touch).
+		const silentOutputs = vad.frames.slice(PLAYBACK_FRAMES);
+		expect(silentOutputs.length).toBe(silentInputs.length);
+		for (let i = 0; i < silentOutputs.length; i++) {
+			expect(Array.from(silentOutputs[i])).toEqual(Array.from(silentInputs[i]));
+		}
+	});
+
+	it("clears stale far-end state before playback resumes after silence", async () => {
+		const PLAYBACK_FRAMES = 40;
+		const SILENT_FRAMES = 5;
+		const restartFrame = PLAYBACK_FRAMES + SILENT_FRAMES;
+		const totalFrames = restartFrame + 1;
+		const N = totalFrames * BLOCK;
+		const far = farSignal(N);
+		const echo = echoOf(far);
+		const zeroReference = new Float32Array(BLOCK);
+		const zeroMic = new Float32Array(BLOCK);
+		const vad = new RecordingVad();
+		const consumer = new AudioFrameConsumer(
+			{
+				vad,
+				pipeline: new FakePipeline("restart"),
+				runtime: new FakeRuntime(),
+				echoReference: (ts: number, samples: number) => {
+					const idx = Math.round((ts - 1000) / 20);
+					if (idx < PLAYBACK_FRAMES) {
+						const off = idx * BLOCK;
+						return far.subarray(off, off + samples);
+					}
+					if (idx < restartFrame) return null;
+					return zeroReference.subarray(0, samples);
+				},
+			},
+			{ source: { kind: "device", deviceId: "pixel" }, preRollSeconds: 0 },
+		);
+
+		let ts = 1000;
+		for (let frameIdx = 0; frameIdx < totalFrames; frameIdx++) {
+			const off = frameIdx * BLOCK;
+			const mic =
+				frameIdx < PLAYBACK_FRAMES ? echo.subarray(off, off + BLOCK) : zeroMic;
+			await consumer.pushDecodedFrame(mic, ts);
+			ts += 20;
+		}
+
+		// The post-silence non-empty reference frame should not inherit any
+		// stale far-end samples from the previous playback burst.
+		expect(consumer.echoFramesCancelled).toBe(PLAYBACK_FRAMES + 1);
+		expect(Array.from(vad.frames[restartFrame])).toEqual(Array.from(zeroMic));
+	});
+});

package/src/services/voice/audio-frame-consumer.ts

@@ -36,6 +36,10 @@ import {
 	handleLiveVoiceAttribution,
 } from "../../runtime/voice-entity-binding.js";
 import type { VoiceTurnSignal } from "./eot-classifier.js";
+import {
+	NlmsEchoCanceller,
+	type ResidualSuppressionOptions,
+} from "./nlms-echo-canceller.js";
 import type {
 	VoiceAttributionOutput,
 	VoiceAttributionPipeline,
@@ -176,8 +180,36 @@ export interface AttributionPipelineLike {
  */
 export interface RuntimeEventSink {
 	emitEvent(type: unknown, payload: Record<string, unknown>): Promise<void>;
+	/**
+	 * Optional host-supplied far-end (agent TTS playback) reference for the live
+	 * AEC path (#9583). When a host wires this, the live diarization route threads
+	 * it into the session's NLMS echo canceller instead of relying on the
+	 * playback-frames ingest route. Absent on headless/core runtimes.
+	 */
+	voiceEchoReferenceProvider?: EchoReferenceProvider;
 }
 
+/**
+ * Transcribe a finalized turn's buffered PCM to text (#8786). When injected, the
+ * consumer joins the ASR transcript into the diarization attribution so
+ * `VOICE_TURN_OBSERVED` carries the real text — previously the live audio-frame
+ * path attributed *who* spoke but always emitted `text: ""`, so name/partner
+ * extraction (`VoiceObserver.ingestTurn`) could never fire from live audio.
+ *
+ * Returns the transcript, or `null`/empty for silence / no decode. Best-effort:
+ * the consumer swallows a rejection (counted in `transcriptionErrors`) and falls
+ * back to a transcript-less turn rather than dropping the diarized turn.
+ */
+export type TurnTranscriber = (
+	pcm: Float32Array,
+	sampleRate: number,
+) => Promise<string | null> | string | null;
+
+export type SelfVoiceSimilarityResolver = (
+	embedding: Float32Array,
+	output: VoiceAttributionOutput,
+) => Promise<number | null | undefined> | number | null | undefined;
+
 // ---------------------------------------------------------------------------
 // Consumer
 // ---------------------------------------------------------------------------
@@ -189,8 +221,39 @@ export interface AudioFrameConsumerDeps {
 	pipeline: AttributionPipelineLike;
 	/** Runtime event sink for VOICE_TURN_OBSERVED. */
 	runtime: RuntimeEventSink;
+	/**
+	 * Optional ASR for the finalized turn's PCM (#8786). When present, its text
+	 * rides on `VOICE_TURN_OBSERVED` so live name/entity extraction runs. When
+	 * absent the path stays diarization-only (transcript `""`, as before).
+	 */
+	transcribe?: TurnTranscriber;
+	/**
+	 * Optional live acoustic self-voice resolver. When wired, the consumer passes
+	 * the turn's WeSpeaker embedding to the host's agent-TTS centroid matcher and
+	 * forwards the resulting cosine into the ambient gate.
+	 */
+	resolveSelfVoiceSimilarity?: SelfVoiceSimilarityResolver;
+	/**
+	 * Optional agent-playback (far-end) reference for acoustic echo cancellation
+	 * (#9455). Given a mic frame's clock timestamp and sample count, returns the
+	 * agent's TTS playback PCM for that exact window (Float32 16 kHz), or null
+	 * when the agent is not playing. When wired, the consumer runs an NLMS echo
+	 * canceller on every mic frame BEFORE VAD/attribution so the agent never
+	 * transcribes its own TTS. Absent → no AEC (unchanged behavior). The caller
+	 * owns the playback capture + the playback→mic delay calibration.
+	 */
+	echoReference?: EchoReferenceProvider;
 }
 
+/**
+ * Returns the agent's TTS playback PCM (the far-end echo reference) aligned to a
+ * mic frame's time window, or null when the agent is silent. See #9455.
+ */
+export type EchoReferenceProvider = (
+	timestampMs: number,
+	samples: number,
+) => Float32Array | null;
+
 export interface AudioFrameConsumerConfig {
 	/** Source metadata stamped onto every attributed turn. */
 	source?: VoiceInputSource;
@@ -208,6 +271,12 @@ export interface AudioFrameConsumerConfig {
 	 * out of the attribution buffer. Default 0.3 s.
 	 */
 	preRollSeconds?: number;
+	/**
+	 * Opt-in nonlinear residual-echo suppressor forwarded to the NLMS canceller
+	 * (#9583/#9649). Default-off; only meaningful when an `echoReference` is wired
+	 * (no canceller exists otherwise). See {@link NlmsEchoCancellerOptions.residualSuppression}.
+	 */
+	residualSuppression?: boolean | ResidualSuppressionOptions;
 }
 
 /** A finalized, attributed turn the consumer surfaces to its caller. */
@@ -237,6 +306,11 @@ export class AudioFrameConsumer {
 	private readonly vad: VadSegmenter;
 	private readonly pipeline: AttributionPipelineLike;
 	private readonly runtime: RuntimeEventSink;
+	private readonly transcribe: TurnTranscriber | null;
+	private readonly resolveSelfVoiceSimilarity: SelfVoiceSimilarityResolver | null;
+	private readonly echoReference: EchoReferenceProvider | null;
+	/** NLMS echo canceller, instantiated only when an `echoReference` is wired. */
+	private readonly echoCanceller: NlmsEchoCanceller | null;
 	private readonly source: VoiceInputSource | undefined;
 	private readonly attributionOptions: HandleLiveVoiceAttributionOptions;
 	private readonly maxTurnSamples: number;
@@ -261,6 +335,15 @@ export class AudioFrameConsumer {
 	/** Count of frames that failed to decode (surfaced via getters, not thrown). */
 	droppedFrames = 0;
 
+	/** Count of turns whose ASR transcribe threw (degraded to a transcript-less
+	 *  turn rather than dropping the diarized turn). */
+	transcriptionErrors = 0;
+
+	/** Count of mic frames the echo canceller actually processed (i.e. the agent
+	 *  was playing). Frames skipped while the agent is silent do not count, so
+	 *  this also measures how often AEC took the cheap passthrough path. */
+	echoFramesCancelled = 0;
+
 	constructor(
 		deps: AudioFrameConsumerDeps,
 		config: AudioFrameConsumerConfig = {},
@@ -268,6 +351,16 @@ export class AudioFrameConsumer {
 		this.vad = deps.vad;
 		this.pipeline = deps.pipeline;
 		this.runtime = deps.runtime;
+		this.transcribe = deps.transcribe ?? null;
+		this.resolveSelfVoiceSimilarity = deps.resolveSelfVoiceSimilarity ?? null;
+		this.echoReference = deps.echoReference ?? null;
+		this.echoCanceller = this.echoReference
+			? new NlmsEchoCanceller(
+					config.residualSuppression
+						? { residualSuppression: config.residualSuppression }
+						: {},
+				)
+			: null;
 		this.source = config.source;
 		this.attributionOptions = config.attributionOptions ?? {};
 		const sr = AUDIO_FRAME_PIPELINE_SAMPLE_RATE;
@@ -329,20 +422,44 @@ export class AudioFrameConsumer {
 		timestampMs: number,
 	): Promise<void> {
 		if (this.closed) return;
+		// #9455/#9649: cancel the agent's TTS echo before VAD/attribution so the
+		// agent never transcribes its own playback. When the reference provider
+		// returns null/empty the agent is silent — skip the FIR canceller so AEC
+		// is cheap and exactly passthrough on the common no-playback path.
+		const micPcm = this.cancelEcho(pcm, timestampMs);
 		this.lastFrameEndMs =
-			timestampMs + (pcm.length / AUDIO_FRAME_PIPELINE_SAMPLE_RATE) * 1000;
+			timestampMs + (micPcm.length / AUDIO_FRAME_PIPELINE_SAMPLE_RATE) * 1000;
 		if (this.capturing) {
-			this.appendTurnChunk(pcm);
+			this.appendTurnChunk(micPcm);
 		} else {
-			this.appendPreRoll(pcm);
+			this.appendPreRoll(micPcm);
 		}
 		await this.vad.pushFrame({
-			pcm,
+			pcm: micPcm,
 			sampleRate: AUDIO_FRAME_PIPELINE_SAMPLE_RATE,
 			timestampMs,
 		});
 	}
 
+	/**
+	 * Run the echo canceller on one mic frame when (and only when) the agent is
+	 * playing. The reference provider returns null while the agent is silent, in
+	 * which case the mic frame is passed through verbatim and the FIR
+	 * `process()` loop is not invoked. The canceller still observes the silent
+	 * far-end so stale playback history is cleared before playback resumes.
+	 * Returns the echo-cancelled (or untouched) mic frame.
+	 */
+	private cancelEcho(pcm: Float32Array, timestampMs: number): Float32Array {
+		if (!this.echoCanceller || !this.echoReference) return pcm;
+		const reference = this.echoReference(timestampMs, pcm.length);
+		if (!reference || reference.length === 0) {
+			this.echoCanceller.observeFarEndSilence(pcm);
+			return pcm;
+		}
+		this.echoFramesCancelled += 1;
+		return this.echoCanceller.process(pcm, reference);
+	}
+
 	/**
 	 * Flush the VAD (finalize any open segment) and await all pending
 	 * attribution. Call at end-of-capture so a trailing utterance is not lost.
@@ -426,10 +543,15 @@ export class AudioFrameConsumer {
 			endedAtMs: args.endedAtMs,
 			...(this.source ? { source: this.source } : {}),
 		});
+		// Join the ASR transcript for this turn (#8786) so VOICE_TURN_OBSERVED
+		// carries the real text and live name/entity extraction can fire. ASR is
+		// best-effort: a decode failure degrades to a transcript-less turn (the
+		// diarized speaker is still emitted), never a dropped turn.
+		const opts = await this.resolveTurnOptions(args.pcm, output);
 		const signal = await handleLiveVoiceAttribution(
 			this.runtime as Parameters<typeof handleLiveVoiceAttribution>[0],
 			output,
-			this.attributionOptions,
+			opts,
 		);
 		const turn: AttributedTurn = {
 			turnId: args.turnId,
@@ -442,6 +564,44 @@ export class AudioFrameConsumer {
 		for (const listener of this.turnListeners) listener(turn);
 	}
 
+	/**
+	 * Merge the per-turn ASR transcript into the attribution options. Returns the
+	 * base options unchanged when no transcriber is wired or the decode yields no
+	 * text; a thrown decode is swallowed (counted in `transcriptionErrors`) so a
+	 * diarized turn is never dropped over an ASR failure.
+	 */
+	private async resolveTurnOptions(
+		pcm: Float32Array,
+		output: VoiceAttributionOutput,
+	): Promise<HandleLiveVoiceAttributionOptions> {
+		let options = this.attributionOptions;
+		try {
+			if (this.transcribe) {
+				const transcript = await this.transcribe(
+					pcm,
+					AUDIO_FRAME_PIPELINE_SAMPLE_RATE,
+				);
+				const trimmed = transcript?.trim();
+				if (trimmed) {
+					options = { ...options, transcript: trimmed };
+				}
+			}
+		} catch {
+			this.transcriptionErrors += 1;
+		}
+		const embedding = output.observation?.embedding;
+		if (this.resolveSelfVoiceSimilarity && embedding) {
+			const similarity = await this.resolveSelfVoiceSimilarity(
+				embedding,
+				output,
+			);
+			if (typeof similarity === "number" && Number.isFinite(similarity)) {
+				options = { ...options, selfVoiceSimilarity: similarity };
+			}
+		}
+		return options;
+	}
+
 	// ---- buffering ---------------------------------------------------------
 
 	private appendTurnChunk(pcm: Float32Array): void {

package/src/services/voice/barge-in.ts

@@ -87,9 +87,8 @@ function tripToken(
 	token: BargeInCancelToken,
 	reason: BargeInCancelToken["reason"],
 ): void {
-	const trip = (
-		token as unknown as { __trip?: (r: BargeInCancelToken["reason"]) => void }
-	).__trip;
+	const trip = (token as { __trip?: (r: BargeInCancelToken["reason"]) => void })
+		.__trip;
 	if (trip) trip(reason);
 }