@kuralle-syrinx/core 4.0.0 → 4.2.0

package/src/voice-agent-session.ts

@@ -41,6 +41,8 @@ import type {
   SpeechToTextAudioPacket,
   SttResultPacket,
   SttInterimPacket,
+  SttPartialPacket,
+  TurnChangePacket,
   VadAudioPacket,
   VadSpeechStartedPacket,
   VadSpeechActivityPacket,
@@ -64,6 +66,11 @@ import { PrimarySpeakerGate } from "./primary-speaker-gate.js";
 import { takeCompleteVoiceText, isCompleteVoiceText, appendVoiceText } from "./voice-text.js";
 import { TtsPlayoutClock } from "./tts-playout-clock.js";
 import { TurnArbiter, isBackchannel } from "./turn-arbiter.js";
+import { InteractionCoordinator } from "./interaction-coordinator.js";
+import { isLifecycleInteractionPolicy, type InteractionPolicy, type WordTiming } from "./interaction-policy.js";
+import { pcm16BytesToSamples } from "./audio/pcm.js";
+import { DeferInteractionPolicy } from "./policies/defer.js";
+import { RuleBasedInteractionPolicy } from "./policies/rule-based.js";
 import * as make from "./packet-factories.js";
 import { pluginStage, stageOrder, isAudioStage } from "./init-stage-order.js";
 import {
@@ -162,6 +169,21 @@ export interface VoiceAgentSessionConfig {
    * provider STT ownership; Smart Turn sessions must opt in explicitly.
    */
   endpointingOwner?: "provider_stt" | "smart_turn" | "timer";
+  /** The front model owns full-duplex interaction (turn-taking + barge-in). When true, the session's
+   *  InteractionPolicy runs observe-only (DeferInteractionPolicy) — Syrinx does not drive its own
+   *  turn/interrupt decisions; the front's native decisions stand. Default: false (Syrinx drives).
+   *  A realtime factory sets this from RealtimeAdapter.caps.supportsFullDuplex. */
+  fullDuplex?: boolean;
+  /** When true, Syrinx suppresses policy-timed backchannel cue packets (front/provider owns them). */
+  emitsBackchannel?: boolean;
+  /**
+   * Optional interaction policy injected by the caller (learned controllers, Smart Turn policy, etc.).
+   * When omitted, the session uses RuleBasedInteractionPolicy. When `fullDuplex` is true, the
+   * coordinator runs observe-only via DeferInteractionPolicy regardless of this setting.
+   */
+  interactionPolicy?: InteractionPolicy;
+  /** Config passed to `interactionPolicy.initialize` when the injected policy is lifecycle-capable. */
+  interactionPolicyConfig?: Record<string, unknown>;
   readonly metricsExporter?: MetricsExporter;
   readonly scheduler?: Scheduler;
   readonly observability?: {
@@ -190,6 +212,24 @@ export interface VoiceAgentSessionEvents {
    * these as `tool_call_*` wire messages — the standard "thinking" cue.
    */
   tool_call_cue: (event: { tsMs: number; turnId: string; phase: "started" | "delayed" | "complete" | "failed"; toolId: string; toolName: string; afterMs?: number }) => void;
+  /**
+   * Per-turn latency decomposition, emitted once at the turn's first TTS audio.
+   * `ttfaMs` is anchored to the real end of user speech (VAD speech-end, falling
+   * back to the endpoint decision) — `fillerUsed` flags turns where a latency filler spoke
+   * first, and `backchannelUsed` flags turns where a wait-gap cue played before the answer.
+   * Decomposition: eouDelayMs (speech end → endpoint) + llmTtftMs (endpoint →
+   * first LLM delta) + ttsTtfbMs (first TTS text dispatched → first audio).
+   */
+  turn_latency: (event: {
+    tsMs: number;
+    turnId: string;
+    ttfaMs: number;
+    eouDelayMs?: number;
+    llmTtftMs?: number;
+    ttsTtfbMs?: number;
+    fillerUsed: boolean;
+    backchannelUsed: boolean;
+  }) => void;
   agent_first_audio: (event: { tsMs: number; turnId: string }) => void;
   agent_finished: (event: { tsMs: number; turnId: string } & Record<string, unknown>) => void;
   error: (event: { tsMs: number; stage: string; category: string; message: string }) => void;
@@ -204,6 +244,15 @@ interface TtsTextBuffer {
   emitted: string;
 }
 
+interface TurnTiming {
+  speechEndedMs?: number;
+  eosMs?: number;
+  firstLlmDeltaMs?: number;
+  firstTtsTextMs?: number;
+  fillerUsed?: boolean;
+  backchannelUsed?: boolean;
+}
+
 /** Suffix marking a context created to speak an error fallback, so it never recurses. */
 const FALLBACK_CONTEXT_SUFFIX = ":error-fallback";
 
@@ -212,6 +261,10 @@ function toolCueTimerKey(contextId: string, toolId: string): string {
   return `tool_cue:${contextId}:${toolId}`;
 }
 
+function interactionPlayoutTimerKey(contextId: string): string {
+  return `interaction.playout:${contextId}`;
+}
+
 // =============================================================================
 // Session Implementation
 // =============================================================================
@@ -243,7 +296,10 @@ export class VoiceAgentSession {
   private ttsTextBuffers = new Map<string, TtsTextBuffer>();
   private readonly minInterruptionMs: number;
   private readonly primarySpeakerGate: PrimarySpeakerGate;
-  private readonly turnArbiter!: TurnArbiter;
+  private readonly ruleBasedPolicy!: RuleBasedInteractionPolicy;
+  private readonly injectedInteractionPolicy: InteractionPolicy | null;
+  private readonly activeInteractionPolicy: InteractionPolicy;
+  private readonly interaction!: InteractionCoordinator;
   private readonly latencyFiller: LatencyFillerController;
   private firstLlmDeltaReceived = new Set<string>();
   private readonly vaqiMissedResponseMs: number;
@@ -252,15 +308,21 @@ export class VoiceAgentSession {
   private readonly watchdogs!: VoiceSessionWatchdogs;
   private readonly observabilityObserver: ObservabilityObserver;
   private turnUserStoppedAtMs = new Map<string, number>();
+  private turnTimings = new Map<string, TurnTiming>();
   private speakerEnrollmentContextId: string | null = null;
   private firstTtsAudioFired = new Set<string>();
+  private readonly pendingInteractionPlayoutTimers = new Set<string>();
   private readonly errorFallbackText: string;
   private fallbackInjectedContexts = new Set<string>();
   // G3: pending tool calls per context (toolId → toolName) driving the tool_call_cue lifecycle.
   private readonly delayCueAfterMs: number;
   private pendingToolCues = new Map<string, Map<string, string>>();
   private readonly endpointingOwner: "provider_stt" | "smart_turn" | "timer";
+  private readonly fullDuplex: boolean;
+  private readonly emitsBackchannel: boolean;
+  private userSpeaking = false;
   private lastFinalizedContextId = "";
+  private readonly sttPartialWordTimings = new Map<string, readonly WordTiming[]>();
 
   constructor(config: VoiceAgentSessionConfig) {
     const owner = config.endpointingOwner;
@@ -268,6 +330,8 @@ export class VoiceAgentSession {
       throw new Error(`Unsupported endpointingOwner: ${owner}`);
     }
     this.endpointingOwner = owner ?? "provider_stt";
+    this.fullDuplex = config.fullDuplex === true;
+    this.emitsBackchannel = config.emitsBackchannel === true;
     this.config = config;
     this.scheduler = config.scheduler ?? new TimerScheduler();
     this.ttsPlayout = new TtsPlayoutClock(this.scheduler);
@@ -318,12 +382,29 @@ export class VoiceAgentSession {
       },
     });
 
-    this.turnArbiter = new TurnArbiter({
+    this.injectedInteractionPolicy = config.interactionPolicy ?? null;
+    this.ruleBasedPolicy = new RuleBasedInteractionPolicy({
       bus: this.bus,
       primarySpeakerGate: this.primarySpeakerGate,
       ttsPlayout: this.ttsPlayout,
       minInterruptionMs: this.minInterruptionMs,
     });
+    const coordinatorPolicy: InteractionPolicy = this.fullDuplex
+      ? new DeferInteractionPolicy()
+      : (this.injectedInteractionPolicy ?? this.ruleBasedPolicy);
+    this.activeInteractionPolicy = coordinatorPolicy;
+    this.interaction = new InteractionCoordinator({
+      bus: this.bus,
+      policy: coordinatorPolicy,
+      executor: this.ruleBasedPolicy.arbiter,
+      scheduler: this.scheduler,
+      caps: { emitsBackchannel: this.emitsBackchannel },
+      isUserSpeaking: () => this.userSpeaking,
+      isTtsActive: () => this.ttsPlayout.activeContexts().length > 0,
+      onBackchannelEmitted: (contextId) => {
+        this.timingFor(contextId).backchannelUsed = true;
+      },
+    });
     this.watchdogs = new VoiceSessionWatchdogs({
       bus: this.bus,
       plugins: this.plugins,
@@ -360,6 +441,10 @@ export class VoiceAgentSession {
     this.modeSwitcher = new ModeSwitcher(this.bus);
   }
 
+  private get turnArbiter(): TurnArbiter {
+    return this.ruleBasedPolicy.arbiter;
+  }
+
   // =========================================================================
   // Public API
   // =========================================================================
@@ -386,6 +471,7 @@ export class VoiceAgentSession {
 
     // 1. Wire all bus handlers
     this.wireBusHandlers();
+    this.interaction.initialize();
 
     // 2. Start bus drain loop
     this.busStartPromise = this.bus.start();
@@ -417,16 +503,23 @@ export class VoiceAgentSession {
 
     // 1. Stop idle timeout
     this.idleTimeout.dispose();
+    this.interaction.dispose();
     this.watchdogs.dispose();
     this.observabilityObserver.dispose();
     this.ttsPlayout.clear();
+    for (const contextId of this.pendingInteractionPlayoutTimers) {
+      this.scheduler.cancel(interactionPlayoutTimerKey(contextId));
+    }
+    this.pendingInteractionPlayoutTimers.clear();
     this.turnArbiter.clear();
     this.turnUserStoppedAtMs.clear();
+    this.turnTimings.clear();
     this.firstTtsAudioFired.clear();
     this.fallbackInjectedContexts.clear();
     this.ttsTextBuffers.clear();
     this.interruptedGenerationContextIds.clear();
     this.firstLlmDeltaReceived.clear();
+    this.sttPartialWordTimings.clear();
     for (const [contextId, pending] of this.pendingToolCues) {
       for (const toolId of pending.keys()) this.scheduler.cancel(toolCueTimerKey(contextId, toolId));
     }
@@ -511,6 +604,7 @@ export class VoiceAgentSession {
     // STT results
     this.bus.on("stt.audio", this.handleSttAudio.bind(this));
     this.bus.on("stt.interim", this.handleSttInterim.bind(this));
+    this.bus.on("stt.partial", this.handleSttPartial.bind(this));
     this.bus.on("stt.result", this.handleSttResult.bind(this));
 
     // VAD
@@ -520,8 +614,11 @@ export class VoiceAgentSession {
     this.bus.on("vad.audio", this.handleVadAudioForSpeakerGate.bind(this));
 
     // EOS
-    this.bus.on("turn.change", () => {
+    this.bus.on("turn.change", (pkt: TurnChangePacket) => {
       this.lastFinalizedContextId = "";
+      if (pkt.previousContextId) {
+        this.sttPartialWordTimings.delete(pkt.previousContextId);
+      }
     });
     this.bus.on("eos.turn_complete", this.handleTurnComplete.bind(this));
     this.bus.on("eos.interim", this.handleEosInterim.bind(this));
@@ -608,9 +705,22 @@ export class VoiceAgentSession {
       timestampMs: pkt.timestampMs,
     });
 
+    this.observeAudioFrame(pkt);
+
     this.watchdogs.scheduleInputCadenceWatchdog(pkt.contextId);
   }
 
+  private observeAudioFrame(pkt: UserAudioReceivedPacket): void {
+    if (pkt.audio.byteLength < 2 || pkt.audio.byteLength % 2 !== 0) return;
+    this.interaction.observe({
+      kind: "audio_frame",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+      audio: pcm16BytesToSamples(pkt.audio),
+      sampleRateHz: pkt.sampleRateHz ?? 16_000,
+    });
+  }
+
   private handleSttAudio(pkt: SpeechToTextAudioPacket): void {
     this.watchdogs.scheduleSttForceFinalize(pkt.contextId);
   }
@@ -620,9 +730,14 @@ export class VoiceAgentSession {
     this.bus.push(Route.Main, make.eosTurnComplete(pkt.contextId, pkt.timestampMs, pkt.text, []));
   }
 
+  private handleSttPartial(pkt: SttPartialPacket): void {
+    if (pkt.wordTimings) {
+      this.sttPartialWordTimings.set(pkt.contextId, pkt.wordTimings);
+    }
+  }
+
   private handleSttInterim(pkt: SttInterimPacket): void {
-    this.turnArbiter.noteInterimEvidence(pkt.text);
-    this.maybeBargeInFromProviderStt(pkt.contextId, pkt.text, pkt.timestampMs);
+    this.observeSttForBargeIn(pkt.contextId, pkt.timestampMs, pkt.text, "stt_partial");
     this.currentTurnId = pkt.contextId;
     this.emit("user_input_partial", {
       tsMs: pkt.timestampMs,
@@ -639,8 +754,7 @@ export class VoiceAgentSession {
 
   private handleSttResult(pkt: SttResultPacket): void {
     this.watchdogs.clearSttForceFinalizeIfContext(pkt.contextId);
-    this.turnArbiter.noteInterimEvidence(pkt.text, pkt.confidence);
-    this.maybeBargeInFromProviderStt(pkt.contextId, pkt.text, pkt.timestampMs);
+    this.observeSttForBargeIn(pkt.contextId, pkt.timestampMs, pkt.text, "stt_final", pkt.confidence);
     this.currentTurnId = pkt.contextId;
     this.emit("user_input_final", {
       tsMs: pkt.timestampMs,
@@ -661,7 +775,7 @@ export class VoiceAgentSession {
   }
 
   private handleVadAudioForSpeakerGate(pkt: VadAudioPacket): void {
-    if (this.turnArbiter.observeBargeInAudio(pkt)) return;
+    if (this.interaction.observeBargeInAudio(pkt)) return;
 
     if (this.shouldEnrollPrimarySpeaker(pkt.contextId)) {
       this.primarySpeakerGate.enrollUserTurnChunk(pkt.audio);
@@ -680,16 +794,43 @@ export class VoiceAgentSession {
   // Provider transcripts arriving while TTS playout is active are the speech
   // evidence instead (echo of our own playout is mitigated by client AEC plus
   // the arbiter's backchannel / low-confidence suppression).
-  private maybeBargeInFromProviderStt(contextId: string, text: string, timestampMs: number): void {
-    if (this.endpointingOwner !== "provider_stt") return;
-    if (!text.trim()) return;
-    const interruptedContextId = this.latestActiveTtsContextId();
-    if (!interruptedContextId) return;
-    this.turnArbiter.onProviderSttEvidence(contextId, timestampMs, interruptedContextId);
+  private observeSttForBargeIn(
+    contextId: string,
+    timestampMs: number,
+    text: string,
+    kind: "stt_partial" | "stt_final",
+    confidence?: number,
+  ): void {
+    const interruptedContextId =
+      this.endpointingOwner === "provider_stt" && text.trim()
+        ? this.latestActiveTtsContextId() ?? undefined
+        : undefined;
+    const wordTimings = this.sttPartialWordTimings.get(contextId);
+    if (kind === "stt_partial") {
+      this.interaction.observe({
+        kind: "stt_partial",
+        contextId,
+        timestampMs,
+        text,
+        interruptedContextId,
+        ...(wordTimings ? { wordTimings } : {}),
+      });
+      return;
+    }
+    this.interaction.observe({
+      kind: "stt_final",
+      contextId,
+      timestampMs,
+      text,
+      confidence,
+      interruptedContextId,
+      ...(wordTimings ? { wordTimings } : {}),
+    });
   }
 
   private handleVadSpeechStarted(pkt: VadSpeechStartedPacket): void {
     this.lastFinalizedContextId = "";
+    this.userSpeaking = true;
 
     if (this.latencyFiller.isFillerOnly(this.currentTurnId)) {
       this.cancelLatencyFillerTurn(this.currentTurnId, pkt.timestampMs);
@@ -712,17 +853,27 @@ export class VoiceAgentSession {
     const interruptedContextId = this.latestActiveTtsContextId();
     if (!interruptedContextId) {
       this.speakerEnrollmentContextId = pkt.contextId;
-      return;
     }
 
-    this.turnArbiter.onSpeechStarted(pkt, interruptedContextId);
+    this.interaction.observe({
+      kind: "vad_speech_started",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+      confidence: pkt.confidence,
+      ...(interruptedContextId ? { interruptedContextId } : {}),
+    });
   }
 
   private handleVadSpeechActivity(pkt: VadSpeechActivityPacket): void {
-    this.turnArbiter.onSpeechActivity(pkt);
+    this.interaction.observe({
+      kind: "vad_speech_activity",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+    });
   }
 
   private handleVadSpeechEnded(pkt: VadSpeechEndedPacket): void {
+    this.userSpeaking = false;
     this.emit("user_stopped_speaking", {
       tsMs: pkt.timestampMs,
       turnId: pkt.contextId,
@@ -738,12 +889,51 @@ export class VoiceAgentSession {
       this.speakerEnrollmentContextId = null;
     }
 
-    this.turnArbiter.onSpeechEnded(pkt, Boolean(this.latestActiveTtsContextId()));
+    this.interaction.observe({
+      kind: "vad_speech_ended",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+      hasActiveTts: Boolean(this.latestActiveTtsContextId()),
+    });
 
     this.turnUserStoppedAtMs.set(pkt.contextId, pkt.timestampMs);
+    this.timingFor(pkt.contextId).speechEndedMs = pkt.timestampMs;
     this.watchdogs.startVaqiMissedResponseTimer(pkt.contextId, pkt.timestampMs);
   }
 
+  private timingFor(contextId: string): TurnTiming {
+    let timing = this.turnTimings.get(contextId);
+    if (!timing) {
+      timing = {};
+      this.turnTimings.set(contextId, timing);
+    }
+    return timing;
+  }
+
+  private emitTurnLatency(contextId: string, firstAudioMs: number): void {
+    const timing = this.turnTimings.get(contextId);
+    this.turnTimings.delete(contextId);
+    if (!timing) return;
+    const anchorMs = timing.speechEndedMs ?? timing.eosMs;
+    if (anchorMs === undefined) return; // text-injected or fallback turn — not a voice TTFA
+    this.emit("turn_latency", {
+      tsMs: firstAudioMs,
+      turnId: contextId,
+      ttfaMs: firstAudioMs - anchorMs,
+      ...(timing.speechEndedMs !== undefined && timing.eosMs !== undefined
+        ? { eouDelayMs: timing.eosMs - timing.speechEndedMs }
+        : {}),
+      ...(timing.eosMs !== undefined && timing.firstLlmDeltaMs !== undefined
+        ? { llmTtftMs: timing.firstLlmDeltaMs - timing.eosMs }
+        : {}),
+      ...(timing.firstTtsTextMs !== undefined
+        ? { ttsTtfbMs: firstAudioMs - timing.firstTtsTextMs }
+        : {}),
+      fillerUsed: timing.fillerUsed === true,
+      backchannelUsed: timing.backchannelUsed === true,
+    });
+  }
+
   private handleTurnComplete(pkt: EndOfSpeechPacket): void {
     if (this.lastFinalizedContextId === pkt.contextId) {
       this.bus.push(Route.Background, make.metric(pkt.contextId, "eos.duplicate_dropped", "1"));
@@ -764,6 +954,7 @@ export class VoiceAgentSession {
     }
 
     this.lastFinalizedContextId = pkt.contextId;
+    this.interaction.reset(pkt.contextId);
 
     // Re-arm per-turn guard state for the next turn. Transports with a stable
     // per-call contextId (telephony callSid) reuse one id across turns, so these
@@ -807,8 +998,13 @@ export class VoiceAgentSession {
     // disconnect prompt later in the call.
     this.bus.push(Route.Main, make.stopIdleTimeout(pkt.contextId, Date.now(), true));
 
+    const timing = this.timingFor(pkt.contextId);
+    timing.eosMs = pkt.timestampMs;
+
     const fillerText = this.latencyFiller.start(pkt.contextId, pkt.text, pkt.timestampMs);
     if (fillerText) {
+      timing.fillerUsed = true;
+      timing.firstTtsTextMs ??= pkt.timestampMs;
       this.bus.push(Route.Main, make.ttsText(pkt.contextId, Date.now(), fillerText));
       this.bus.push(Route.Background, make.metric(pkt.contextId, "filler.started", fillerText));
     }
@@ -840,6 +1036,8 @@ export class VoiceAgentSession {
     let deltaText = pkt.text;
     if (!this.firstLlmDeltaReceived.has(pkt.contextId)) {
       this.firstLlmDeltaReceived.add(pkt.contextId);
+      const timing = this.turnTimings.get(pkt.contextId);
+      if (timing) timing.firstLlmDeltaMs ??= pkt.timestampMs;
       if (this.latencyFiller.isActive(pkt.contextId)) {
         deltaText = this.latencyFiller.spliceLlmDelta(pkt.contextId, deltaText);
         this.bus.push(Route.Background, make.metric(pkt.contextId, "filler.spliced", "1"));
@@ -858,7 +1056,7 @@ export class VoiceAgentSession {
       timestampMs: pkt.timestampMs,
     });
 
-    this.bufferTtsText(pkt.contextId, deltaText);
+    this.bufferTtsText(pkt.contextId, deltaText, pkt.timestampMs);
   }
 
   private handleLlmDone(pkt: LlmResponseDonePacket): void {
@@ -869,7 +1067,7 @@ export class VoiceAgentSession {
       return;
     }
 
-    const spokenText = this.flushTtsText(pkt.contextId);
+    const spokenText = this.flushTtsText(pkt.contextId, pkt.timestampMs);
     this.emit("agent_finished", {
       tsMs: pkt.timestampMs,
       turnId: pkt.contextId,
@@ -887,11 +1085,13 @@ export class VoiceAgentSession {
     this.bus.push(Route.Main, make.ttsDone(pkt.contextId, Date.now(), spokenText));
   }
 
-  private bufferTtsText(contextId: string, text: string): void {
+  private bufferTtsText(contextId: string, text: string, tsMs?: number): void {
     const buffer = this.ttsTextBuffers.get(contextId) ?? { pending: "", emitted: "" };
     buffer.pending += text;
     const complete = takeCompleteVoiceText(buffer.pending);
     if (complete.text) {
+      const timing = this.turnTimings.get(contextId);
+      if (timing) timing.firstTtsTextMs ??= tsMs ?? Date.now();
       this.bus.push(Route.Main, make.ttsText(contextId, Date.now(), complete.text));
       buffer.emitted = appendVoiceText(buffer.emitted, complete.text);
     }
@@ -899,11 +1099,13 @@ export class VoiceAgentSession {
     this.ttsTextBuffers.set(contextId, buffer);
   }
 
-  private flushTtsText(contextId: string): string {
+  private flushTtsText(contextId: string, tsMs?: number): string {
     const buffer = this.ttsTextBuffers.get(contextId);
     if (!buffer) return "";
     const tail = buffer.pending.trim();
     if (tail) {
+      const timing = this.turnTimings.get(contextId);
+      if (timing) timing.firstTtsTextMs ??= tsMs ?? Date.now();
       this.bus.push(Route.Main, make.ttsText(contextId, Date.now(), tail));
       buffer.emitted = appendVoiceText(buffer.emitted, tail);
       buffer.pending = "";
@@ -965,14 +1167,22 @@ export class VoiceAgentSession {
     toolName: string,
     afterMs?: number,
   ): void {
+    const tsMs = Date.now();
     this.emit("tool_call_cue", {
-      tsMs: Date.now(),
+      tsMs,
       turnId: contextId,
       phase,
       toolId,
       toolName,
       ...(afterMs !== undefined ? { afterMs } : {}),
     });
+    this.interaction.observe({
+      kind: "delegate_state",
+      contextId,
+      timestampMs: tsMs,
+      delegateInFlight: phase === "started" || phase === "delayed",
+      toolCallPhase: phase,
+    });
   }
 
   /** G3: resolve one pending tool call with a terminal cue phase. */
@@ -1077,6 +1287,7 @@ export class VoiceAgentSession {
         );
         this.turnUserStoppedAtMs.delete(pkt.contextId);
       }
+      this.emitTurnLatency(pkt.contextId, pkt.timestampMs);
     }
 
     this.primarySpeakerGate.observeAssistantPlayout(pkt.audio);
@@ -1089,6 +1300,14 @@ export class VoiceAgentSession {
     const audioDurationMs = estimatePcm16Duration(pkt.audio, sampleRateHz);
     const now = Date.now();
     this.ttsPlayout.noteAudio(pkt.contextId, audioDurationMs, now);
+    this.interaction.observe({
+      kind: "playout_tick",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+      ttsActive: true,
+      audio: pcm16BytesToSamples(pkt.audio),
+      sampleRateHz,
+    });
 
     // Anchor the idle timer to when playout actually *ends* (P2), not to chunk
     // arrival. TTS streams faster than realtime, so extending by each chunk's
@@ -1115,7 +1334,11 @@ export class VoiceAgentSession {
     // Generation finished, but the streamed audio is still playing out. Keep the
     // context interruptible until its playout estimate elapses, then release it.
     this.generatingContextIds.delete(pkt.contextId);
-    this.ttsPlayout.scheduleRelease(pkt.contextId, Date.now());
+    const now = Date.now();
+    this.ttsPlayout.scheduleRelease(pkt.contextId, now);
+    const playoutEndMs = this.ttsPlayout.playoutEnd(pkt.contextId);
+    const remainingMs = playoutEndMs === undefined ? 0 : Math.max(0, playoutEndMs - now);
+    this.scheduleInteractionPlayoutTick(pkt.contextId, remainingMs);
     this.watchdogs.clearTtsStallTimerFor(pkt.contextId);
     this.debugPush({
       component: "tts",
@@ -1125,14 +1348,43 @@ export class VoiceAgentSession {
     });
   }
 
+  private scheduleInteractionPlayoutTick(contextId: string, delayMs: number): void {
+    this.pendingInteractionPlayoutTimers.add(contextId);
+    this.scheduler.schedule(interactionPlayoutTimerKey(contextId), delayMs, () => {
+      this.pendingInteractionPlayoutTimers.delete(contextId);
+      if (this.ttsPlayout.isActive(contextId)) return;
+      this.interaction.observe({
+        kind: "playout_tick",
+        contextId,
+        timestampMs: Date.now(),
+        ttsActive: false,
+      });
+    });
+  }
+
+  private cancelInteractionPlayoutTick(contextId: string): void {
+    if (!this.pendingInteractionPlayoutTimers.delete(contextId)) return;
+    this.scheduler.cancel(interactionPlayoutTimerKey(contextId));
+  }
+
   private handleTtsPlayoutProgress(pkt: TextToSpeechPlayoutProgressPacket): void {
+    this.cancelInteractionPlayoutTick(pkt.contextId);
     this.ttsPlayout.noteProgress(pkt.contextId, pkt.complete, pkt.playedOutMs);
+    this.interaction.observe({
+      kind: "playout_tick",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+      playedOutMs: pkt.playedOutMs,
+      ttsActive: !pkt.complete,
+    });
   }
 
   private handleInterruptDetected(pkt: InterruptionDetectedPacket): void {
+    this.cancelInteractionPlayoutTick(pkt.contextId);
     this.interruptedGenerationContextIds.add(pkt.contextId);
     this.failPendingToolCues(pkt.contextId); // G3: the aborted delegate's cue fails (R5)
     this.latencyFiller.cancel(pkt.contextId);
+    this.turnTimings.delete(pkt.contextId);
     this.firstLlmDeltaReceived.delete(pkt.contextId);
     this.ttsTextBuffers.delete(pkt.contextId);
     this.ttsPlayout.release(pkt.contextId);
@@ -1178,10 +1430,12 @@ export class VoiceAgentSession {
    * the LLM, and drops late deltas/audio for the stale context.
    */
   private cancelStaleGeneration(contextId: string, timestampMs: number): void {
+    this.cancelInteractionPlayoutTick(contextId);
     this.interruptedGenerationContextIds.add(contextId);
     this.failPendingToolCues(contextId); // G3: a superseded turn's pending cue fails
     this.generatingContextIds.delete(contextId);
     this.latencyFiller.cancel(contextId);
+    this.turnTimings.delete(contextId);
     this.firstLlmDeltaReceived.delete(contextId);
     this.ttsTextBuffers.delete(contextId);
     this.ttsPlayout.release(contextId);
@@ -1298,6 +1552,19 @@ export class VoiceAgentSession {
       });
     }
 
+    if (
+      this.activeInteractionPolicy === this.injectedInteractionPolicy &&
+      isLifecycleInteractionPolicy(this.activeInteractionPolicy)
+    ) {
+      const policy = this.activeInteractionPolicy;
+      steps.push({
+        name: "interaction_policy",
+        stage: InitStage.EOS,
+        run: () => policy.initialize(this.config.interactionPolicyConfig ?? {}),
+        cleanup: () => policy.close(),
+      });
+    }
+
     const orderedPluginSteps = steps.sort(
       (a, b) => stageOrder(a.stage) - stageOrder(b.stage),
     );
@@ -1327,6 +1594,20 @@ export class VoiceAgentSession {
   }
 
   private applyEndpointingOwnerInvariant(): void {
+    if (this.activeInteractionPolicy === this.injectedInteractionPolicy) {
+      for (const [name, plugin] of this.plugins) {
+        const capability = plugin.endpointingCapability;
+        if (!capability || capability.owner !== "provider_stt") continue;
+        if (!capability.disableConfig) {
+          throw new Error(`interactionPolicy requires ${name} to expose endpointingCapability.disableConfig`);
+        }
+        this.config.plugins[name] = {
+          ...(this.config.plugins[name] ?? {}),
+          ...capability.disableConfig,
+        };
+      }
+      return;
+    }
     if (this.endpointingOwner === "timer") return;
     const owner: EndpointingOwner = this.endpointingOwner;
     const finalizers = [...this.plugins.entries()]
@@ -1352,6 +1633,9 @@ export class VoiceAgentSession {
   private shouldInitializePlugin(plugin: VoicePlugin): boolean {
     const capability = plugin.endpointingCapability;
     if (!capability) return true;
+    if (this.activeInteractionPolicy === this.injectedInteractionPolicy) {
+      return capability.owner === "provider_stt";
+    }
     if (this.endpointingOwner === "timer") return false;
     if (capability.owner === "smart_turn") return capability.owner === this.endpointingOwner;
     return true;