@kuralle-syrinx/core 2.1.0

package/src/voice-agent-session.ts

@@ -0,0 +1,1175 @@
+// SPDX-License-Identifier: MIT
+//
+// Syrinx Kernel v2 — Voice Agent Session
+//
+// The central orchestrator. Wires together PipelineBus, plugins, init chain,
+// error handler, idle timeout, mode switcher, and debug event stream.
+//
+// Breaking changes from v0.1:
+//   - Plugins accept PipelineBus directly (no callbacks).
+//   - Explicit init/finalize chains with stage-level error reporting.
+//   - Categorized errors with recoverable flag.
+//   - Priority bus with Critical/Main/Background routes.
+//   - Unified ConversationEvent debug stream.
+//   - Idle timeout with consecutive backoff.
+//   - Mode switching (text ↔ audio).
+//
+// Backward compat: on/off event emitter stays for client-side consumers
+// (bridges bus packets to legacy event names like "user_started_speaking").
+
+import { PipelineBusImpl, Route, type PipelineBus } from "./pipeline-bus.js";
+import { runInitChain, runFinalizeChain, type InitStep } from "./init-chain.js";
+import type { VoicePlugin, PluginConfig, EndpointingOwner } from "./plugin-contract.js";
+import { IdleTimeoutManager, type IdleTimeoutConfig } from "./idle-timeout.js";
+import { ModeSwitcher } from "./mode-switcher.js";
+import { createConversationEventStream, type ConversationEvent } from "./conversation-event.js";
+import { isRecoverable } from "./error-handler.js";
+import type {
+  VoiceErrorPacket,
+  UserAudioReceivedPacket,
+  UserTextReceivedPacket,
+  InterruptionDetectedPacket,
+  LlmDeltaPacket,
+  LlmResponseDonePacket,
+  LlmToolCallPacket,
+  LlmToolResultPacket,
+  TextToSpeechAudioPacket,
+  TextToSpeechEndPacket,
+  TextToSpeechPlayoutProgressPacket,
+  SpeechToTextAudioPacket,
+  SttResultPacket,
+  SttInterimPacket,
+  VadAudioPacket,
+  VadSpeechStartedPacket,
+  VadSpeechActivityPacket,
+  VadSpeechEndedPacket,
+  EndOfSpeechPacket,
+  InterimEndOfSpeechPacket,
+  InjectMessagePacket,
+  DisconnectRequestedPacket,
+  InitializationFailedPacket,
+  ModeSwitchRequestedPacket,
+  StartIdleTimeoutPacket,
+  StopIdleTimeoutPacket,
+} from "./packets.js";
+import {
+  SessionState,
+  InitStage,
+  ErrorCategory,
+} from "./packets.js";
+import { LatencyFillerController } from "./latency-filler.js";
+import { PrimarySpeakerGate } from "./primary-speaker-gate.js";
+import { takeCompleteVoiceText, isCompleteVoiceText, appendVoiceText } from "./voice-text.js";
+import { TtsPlayoutClock } from "./tts-playout-clock.js";
+import { TurnArbiter } from "./turn-arbiter.js";
+import * as make from "./packet-factories.js";
+import { pluginStage, stageOrder, isAudioStage } from "./init-stage-order.js";
+import {
+  estimatePcm16Duration,
+  languageFromTranscripts,
+  requireTtsAudioSampleRate,
+  VoiceSessionWatchdogs,
+} from "./voice-agent-session-util.js";
+import { noopMetricsExporter, type MetricsExporter } from "./observability.js";
+import { ObservabilityObserver } from "./observability-observer.js";
+import { TimerScheduler, type Scheduler } from "./scheduler.js";
+
+// =============================================================================
+// Types
+// =============================================================================
+
+export interface VoiceAgentSessionConfig {
+  /** Plugin configurations, keyed by plugin name. */
+  plugins: Record<string, PluginConfig>;
+  /** Idle timeout configuration. */
+  idleTimeout?: Partial<IdleTimeoutConfig>;
+  /** PipelineBus configuration. */
+  busConfig?: {
+    mainCapacity?: number;
+    bgCapacity?: number;
+    criticalBatchSize?: number;
+  };
+  /**
+   * Maximum ms to wait for an STT final transcript after audio injection stops.
+   * When this timer fires, asks the streaming STT provider to flush buffered audio.
+   * Default: 7000 (endpointing + 2s grace)
+   */
+  sttForceFinalizeTimeoutMs?: number;
+  /**
+   * Minimum sustained user-speech duration (ms) during assistant playback before a
+   * barge-in is committed. Filters transient noise, clicks, and very short blips that
+   * would otherwise falsely cut off the agent. The agent keeps speaking until the
+   * user's speech is sustained past this threshold, then interruption fires
+   * immediately. Set to 0 to disable the gate and interrupt on the first VAD speech
+   * frame (legacy behavior). Default: 280 ms.
+   */
+  minInterruptionMs?: number;
+  /**
+   * When true (default), barge-in requires sustained speech from the enrolled
+   * primary speaker (first user turn fingerprint) in addition to G1's time gate.
+   * Non-primary / echo speech emits `interrupt.suppressed_non_primary`. When no
+   * profile is enrolled yet, falls back to G1-only behavior.
+   */
+  primarySpeakerBargeInEnabled?: boolean;
+  /**
+   * When true, emit a short discourse connective via TTS at endpoint (before LLM
+   * TTFB) and splice the real response in when it arrives. Off by default.
+   */
+  latencyFillerEnabled?: boolean;
+  /**
+   * Maximum ms after a user turn ends to wait for first assistant audio before
+   * emitting a vaqi.missed_response metric (VAQI-M). 0 disables the check.
+   * Default: 4000.
+   */
+  vaqiMissedResponseMs?: number;
+  /**
+   * Max ms of silence from the TTS provider AFTER it has begun producing audio for a
+   * turn before the output is treated as a stalled provider. Guards against a TTS
+   * provider that goes silent mid-utterance without `tts.end` or an error (dead air).
+   * Armed only after the first `tts.audio`, so first-audio latency (which can be many
+   * seconds on some providers, e.g. Gemini) is never watchdogged. On breach, a
+   * recoverable `tts.error` (NetworkTimeout) is emitted so the turn fails visibly
+   * instead of hanging. 0 disables. Default: 15000.
+   */
+  ttsStallMs?: number;
+  /**
+   * Max ms of silence on inbound user audio while the session is Ready before a
+   * recoverable transport warning is emitted. Continuous streams (telephony, open
+   * mic) should set this; push-to-talk / headless sessions leave at 0 (disabled).
+   * Default: 0.
+   */
+  inputCadenceTimeoutMs?: number;
+  /**
+   * Spoken fallback when the reasoning (LLM) layer fails a turn with a recoverable
+   * error. "Never fail silently" (Deepgram guide Ch3): rather than ending the turn in
+   * unexplained silence, the agent speaks this line via the normal TTS path (which is
+   * unaffected by an LLM failure). Empty string disables. Default: a brief apology.
+   * (TTS/STT-failure fallback needs canned audio / a clarification prompt — out of scope.)
+   */
+  errorFallbackText?: string;
+  /**
+   * Which component owns turn boundary (EOS) for this session. Defaults to
+   * provider STT ownership; Smart Turn sessions must opt in explicitly.
+   */
+  endpointingOwner?: "provider_stt" | "smart_turn" | "timer";
+  readonly metricsExporter?: MetricsExporter;
+  readonly scheduler?: Scheduler;
+  readonly observability?: {
+    readonly sessionId?: string;
+    readonly provider?: string;
+    readonly model?: string;
+    readonly region?: string;
+  };
+}
+
+export interface VoiceAgentSessionEvents {
+  user_started_speaking: (event: { tsMs: number; turnId: string }) => void;
+  user_stopped_speaking: (event: { tsMs: number; turnId: string }) => void;
+  user_input_partial: (event: { tsMs: number; turnId: string; text: string }) => void;
+  user_input_final: (event: { tsMs: number; turnId: string; text: string; confidence: number }) => void;
+  agent_text_delta: (event: { tsMs: number; turnId: string; delta: string }) => void;
+  agent_tool_call: (event: { tsMs: number; turnId: string; id: string; name: string; args: Record<string, unknown> }) => void;
+  agent_tool_result: (event: { tsMs: number; turnId: string; id: string; result: string; durationMs: number }) => void;
+  agent_first_audio: (event: { tsMs: number; turnId: string }) => void;
+  agent_finished: (event: { tsMs: number; turnId: string } & Record<string, unknown>) => void;
+  error: (event: { tsMs: number; stage: string; category: string; message: string }) => void;
+  closed: (event: { tsMs: number; reason: string }) => void;
+  state_changed: (event: { tsMs: number; from: SessionState; to: SessionState }) => void;
+}
+
+type EventHandler<T> = (event: T) => void;
+
+interface TtsTextBuffer {
+  pending: string;
+  emitted: string;
+}
+
+/** Suffix marking a context created to speak an error fallback, so it never recurses. */
+const FALLBACK_CONTEXT_SUFFIX = ":error-fallback";
+
+// =============================================================================
+// Session Implementation
+// =============================================================================
+
+export class VoiceAgentSession {
+  readonly bus: PipelineBus;
+  readonly debugEvents: ReadableStream<ConversationEvent>;
+  private readonly config: VoiceAgentSessionConfig;
+  private readonly sttForceFinalizeTimeoutMs: number;
+  private _state: SessionState = SessionState.Uninitialized;
+  private plugins: Map<string, VoicePlugin> = new Map();
+  private initSteps: InitStep[] = [];
+  private idleTimeout: IdleTimeoutManager;
+  private modeSwitcher: ModeSwitcher;
+  private debugPush: (event: ConversationEvent) => void;
+  private eventListeners = new Map<string, Set<EventHandler<unknown>>>();
+  private currentTurnId = "";
+  private busStartPromise: Promise<void> | null = null;
+  private closePromise: Promise<void> | null = null;
+  // Tracks which contexts are still playing out their TTS audio; turn-taking and
+  // the stall watchdog key on this. Pure state — see TtsPlayoutClock.
+  private readonly scheduler: Scheduler;
+  private readonly ttsPlayout: TtsPlayoutClock;
+  private interruptedGenerationContextIds = new Set<string>();
+  private ttsTextBuffers = new Map<string, TtsTextBuffer>();
+  private readonly minInterruptionMs: number;
+  private readonly primarySpeakerGate: PrimarySpeakerGate;
+  private readonly turnArbiter!: TurnArbiter;
+  private readonly latencyFiller: LatencyFillerController;
+  private firstLlmDeltaReceived = new Set<string>();
+  private readonly vaqiMissedResponseMs: number;
+  private readonly ttsStallMs: number;
+  private readonly inputCadenceTimeoutMs: number;
+  private readonly watchdogs!: VoiceSessionWatchdogs;
+  private readonly observabilityObserver: ObservabilityObserver;
+  private turnUserStoppedAtMs = new Map<string, number>();
+  private speakerEnrollmentContextId: string | null = null;
+  private firstTtsAudioFired = new Set<string>();
+  private readonly errorFallbackText: string;
+  private fallbackInjectedContexts = new Set<string>();
+  private readonly endpointingOwner: "provider_stt" | "smart_turn" | "timer";
+  private lastFinalizedContextId = "";
+
+  constructor(config: VoiceAgentSessionConfig) {
+    const owner = config.endpointingOwner;
+    if (owner !== undefined && owner !== "provider_stt" && owner !== "smart_turn" && owner !== "timer") {
+      throw new Error(`Unsupported endpointingOwner: ${owner}`);
+    }
+    this.endpointingOwner = owner ?? "provider_stt";
+    this.config = config;
+    this.scheduler = config.scheduler ?? new TimerScheduler();
+    this.ttsPlayout = new TtsPlayoutClock(this.scheduler);
+    this.sttForceFinalizeTimeoutMs = config.sttForceFinalizeTimeoutMs ?? 7000;
+    this.minInterruptionMs = config.minInterruptionMs ?? 280;
+    this.primarySpeakerGate = new PrimarySpeakerGate({
+      enabled: config.primarySpeakerBargeInEnabled !== false,
+    });
+    this.latencyFiller = new LatencyFillerController({
+      enabled: config.latencyFillerEnabled === true,
+    });
+    this.vaqiMissedResponseMs = config.vaqiMissedResponseMs ?? 4000;
+    this.ttsStallMs = config.ttsStallMs ?? 15000;
+    this.inputCadenceTimeoutMs = config.inputCadenceTimeoutMs ?? 0;
+    this.errorFallbackText = config.errorFallbackText ?? "Sorry, I'm having trouble right now. Could you try again?";
+
+    // Debug events
+    const [stream, push] = createConversationEventStream();
+    this.debugEvents = stream;
+    this.debugPush = push;
+
+    // PipelineBus
+    this.bus = new PipelineBusImpl({
+      ...config.busConfig,
+      onPacket: (route, packet) => {
+        this.debugPush({
+          component: "bus",
+          type: "packet",
+          data: {
+            context_id: packet.contextId,
+            route: Route[route] ?? String(route),
+            kind: packet.kind,
+          },
+          timestampMs: packet.timestampMs,
+        });
+      },
+      onBackgroundDrop: (dropped) => {
+        this.debugPush({
+          component: "pipeline",
+          type: "background_dropped",
+          data: {
+            context_id: dropped.contextId,
+            kind: dropped.kind,
+          },
+          timestampMs: Date.now(),
+        });
+      },
+    });
+
+    this.turnArbiter = new TurnArbiter({
+      bus: this.bus,
+      primarySpeakerGate: this.primarySpeakerGate,
+      ttsPlayout: this.ttsPlayout,
+      minInterruptionMs: this.minInterruptionMs,
+    });
+    this.watchdogs = new VoiceSessionWatchdogs({
+      bus: this.bus,
+      plugins: this.plugins,
+      ttsPlayout: this.ttsPlayout,
+      sttForceFinalizeTimeoutMs: this.sttForceFinalizeTimeoutMs,
+      vaqiMissedResponseMs: this.vaqiMissedResponseMs,
+      ttsStallMs: this.ttsStallMs,
+      inputCadenceTimeoutMs: this.inputCadenceTimeoutMs,
+      getSessionState: () => this._state,
+      isGenerationInterrupted: (contextId) => this.interruptedGenerationContextIds.has(contextId),
+      onVaqiMissedResponseFired: (contextId) => {
+        this.turnUserStoppedAtMs.delete(contextId);
+      },
+      scheduler: this.scheduler,
+    });
+
+    const obs = config.observability;
+    this.observabilityObserver = new ObservabilityObserver({
+      bus: this.bus,
+      exporter: config.metricsExporter ?? noopMetricsExporter,
+      sessionId: obs?.sessionId ?? "",
+      dims: {
+        provider: obs?.provider ?? "unknown",
+        model: obs?.model ?? "unknown",
+        region: obs?.region ?? "unknown",
+      },
+      getContextId: () => this.currentContextId,
+    });
+
+    // Idle timeout — starts after bus handlers are wired
+    this.idleTimeout = new IdleTimeoutManager(this.bus, config.idleTimeout, this.scheduler);
+
+    // Mode switcher
+    this.modeSwitcher = new ModeSwitcher(this.bus);
+  }
+
+  // =========================================================================
+  // Public API
+  // =========================================================================
+
+  get state(): SessionState {
+    return this._state;
+  }
+
+  get currentContextId(): string {
+    return this.currentTurnId;
+  }
+
+  /** Register a plugin. Must be called before start(). */
+  registerPlugin(name: string, plugin: VoicePlugin): void {
+    this.plugins.set(name, plugin);
+  }
+
+  /** Start the session. Runs init chain, starts bus draining. */
+  async start(): Promise<void> {
+    if (this._state !== SessionState.Uninitialized) {
+      throw new Error(`Cannot start session in state ${this._state}`);
+    }
+    this._state = SessionState.Initializing;
+
+    // 1. Wire all bus handlers
+    this.wireBusHandlers();
+
+    // 2. Start bus drain loop
+    this.busStartPromise = this.bus.start();
+
+    // 3. Build init chain from registered plugins
+    this.buildInitChain();
+
+    // 4. Run init chain
+    try {
+      await runInitChain(this.bus, this.initSteps);
+    } catch (err) {
+      this._state = SessionState.Failed;
+      throw err;
+    }
+
+    this._state = SessionState.Ready;
+  }
+
+  /** Shut down the session. Runs finalize chain in reverse order. */
+  async close(): Promise<void> {
+    if (this._state === SessionState.Closed) return;
+    if (this.closePromise) return await this.closePromise;
+    this.closePromise = this.closeOnce();
+    return await this.closePromise;
+  }
+
+  private async closeOnce(): Promise<void> {
+    this._state = SessionState.Finalizing;
+
+    // 1. Stop idle timeout
+    this.idleTimeout.dispose();
+    this.watchdogs.dispose();
+    this.observabilityObserver.dispose();
+    this.ttsPlayout.clear();
+    this.turnArbiter.clear();
+    this.turnUserStoppedAtMs.clear();
+    this.firstTtsAudioFired.clear();
+    this.fallbackInjectedContexts.clear();
+    this.ttsTextBuffers.clear();
+    this.interruptedGenerationContextIds.clear();
+    this.firstLlmDeltaReceived.clear();
+
+    // 2. Run finalize chain (reverse order)
+    await runFinalizeChain(this.initSteps);
+
+    // 3. Stop bus
+    this.bus.stop();
+    await this.busStartPromise;
+
+    // 4. Emit debug event
+    this.emitDebug("session", "disconnected", { reason: "close" });
+
+    this._state = SessionState.Closed;
+  }
+
+  /** Switch between text and audio mode. */
+  async switchMode(mode: "text" | "audio"): Promise<void> {
+    this.bus.push(Route.Main, make.modeSwitchRequested(this.currentTurnId, Date.now(), mode));
+  }
+
+  requestClientInterrupt(contextId: string): void {
+    this.turnArbiter.commitClientInterrupt(contextId);
+  }
+
+  // =========================================================================
+  // Legacy Event Emitter (client-side consumers)
+  // =========================================================================
+
+  on<K extends keyof VoiceAgentSessionEvents>(
+    event: K,
+    handler: VoiceAgentSessionEvents[K],
+  ): void {
+    if (!this.eventListeners.has(event)) {
+      this.eventListeners.set(event, new Set());
+    }
+    this.eventListeners.get(event)!.add(handler as EventHandler<unknown>);
+  }
+
+  off<K extends keyof VoiceAgentSessionEvents>(
+    event: K,
+    handler: VoiceAgentSessionEvents[K],
+  ): void {
+    this.eventListeners.get(event)?.delete(handler as EventHandler<unknown>);
+  }
+
+  private emit<K extends keyof VoiceAgentSessionEvents>(
+    event: K,
+    payload: Parameters<VoiceAgentSessionEvents[K]>[0],
+  ): void {
+    const listeners = this.eventListeners.get(event);
+    if (!listeners) return;
+    for (const fn of listeners) {
+      try {
+        (fn as EventHandler<unknown>)(payload);
+      } catch {
+        // Don't let listener errors crash the session
+      }
+    }
+  }
+
+  // =========================================================================
+  // Bus Handler Wiring
+  // =========================================================================
+
+  private wireBusHandlers(): void {
+    // Input pipeline
+    this.bus.on("user.audio_received", this.handleUserAudio.bind(this));
+    this.bus.on("user.text_received", this.handleUserText.bind(this));
+
+    // STT results
+    this.bus.on("stt.audio", this.handleSttAudio.bind(this));
+    this.bus.on("stt.interim", this.handleSttInterim.bind(this));
+    this.bus.on("stt.result", this.handleSttResult.bind(this));
+
+    // VAD
+    this.bus.on("vad.speech_started", this.handleVadSpeechStarted.bind(this));
+    this.bus.on("vad.speech_activity", this.handleVadSpeechActivity.bind(this));
+    this.bus.on("vad.speech_ended", this.handleVadSpeechEnded.bind(this));
+    this.bus.on("vad.audio", this.handleVadAudioForSpeakerGate.bind(this));
+
+    // EOS
+    this.bus.on("turn.change", () => {
+      this.lastFinalizedContextId = "";
+    });
+    this.bus.on("eos.turn_complete", this.handleTurnComplete.bind(this));
+    this.bus.on("eos.interim", this.handleEosInterim.bind(this));
+
+    // LLM
+    this.bus.on("llm.delta", this.handleLlmDelta.bind(this));
+    this.bus.on("llm.done", this.handleLlmDone.bind(this));
+    this.bus.on("llm.tool_call", this.handleLlmToolCall.bind(this));
+    this.bus.on("llm.tool_result", this.handleLlmToolResult.bind(this));
+
+    // TTS
+    this.bus.on("tts.audio", this.handleTtsAudio.bind(this));
+    this.bus.on("tts.end", this.handleTtsEnd.bind(this));
+    this.bus.on("tts.playout_progress", this.handleTtsPlayoutProgress.bind(this));
+
+    // Interrupts
+    this.bus.on("interrupt.detected", this.handleInterruptDetected.bind(this));
+
+    // Errors
+    this.bus.on("stt.error", this.handleComponentError.bind(this));
+    this.bus.on("tts.error", this.handleComponentError.bind(this));
+    this.bus.on("vad.error", this.handleComponentError.bind(this));
+    this.bus.on("llm.error", this.handleComponentError.bind(this));
+    this.bus.on("pipeline.error", this.handleComponentError.bind(this));
+
+    // Lifecycle
+    this.bus.on("init.failed", this.handleInitFailed.bind(this));
+
+    // Behavior
+    this.bus.on<StartIdleTimeoutPacket>("behavior.idle_timeout_start", (pkt) => {
+      this.idleTimeout.handleStart(pkt);
+    });
+    this.bus.on<StopIdleTimeoutPacket>("behavior.idle_timeout_stop", (pkt) => {
+      this.idleTimeout.handleStop(pkt);
+    });
+
+    // Injected messages — push through LLM path for natural TTS
+    this.bus.on("inject.message", this.handleInjectMessage.bind(this));
+
+    // Disconnect
+    this.bus.on("session.disconnect", this.handleDisconnect.bind(this));
+
+    // Mode switching
+    this.bus.on<ModeSwitchRequestedPacket>("mode.switch_requested", async (pkt) => {
+      await this.modeSwitcher.handleSwitchRequested(pkt);
+    });
+
+    this.observabilityObserver.wire();
+  }
+
+  // =========================================================================
+  // Handler Implementations
+  // =========================================================================
+
+  private handleUserAudio(pkt: UserAudioReceivedPacket): void {
+    if (this.shouldEnrollPrimarySpeaker(pkt.contextId)) {
+      this.primarySpeakerGate.enrollUserTurnChunk(pkt.audio);
+    }
+    if (this.endpointingOwner === "provider_stt") {
+      this.bus.push(
+        Route.Main,
+        make.recordUserAudio(pkt.contextId, pkt.timestampMs, pkt.audio),
+        make.vadAudio(pkt.contextId, pkt.timestampMs, pkt.audio),
+        make.sttAudio(pkt.contextId, pkt.timestampMs, pkt.audio),
+      );
+    } else {
+      this.bus.push(
+        Route.Main,
+        make.recordUserAudio(pkt.contextId, pkt.timestampMs, pkt.audio),
+        make.vadAudio(pkt.contextId, pkt.timestampMs, pkt.audio),
+        make.sttAudio(pkt.contextId, pkt.timestampMs, pkt.audio),
+        make.eosAudio(pkt.contextId, pkt.timestampMs, pkt.audio),
+      );
+    }
+
+    this.debugPush({
+      component: "input",
+      type: "audio_received",
+      data: { context_id: pkt.contextId, bytes: String(pkt.audio.length) },
+      timestampMs: pkt.timestampMs,
+    });
+
+    this.watchdogs.scheduleInputCadenceWatchdog(pkt.contextId);
+  }
+
+  private handleSttAudio(pkt: SpeechToTextAudioPacket): void {
+    this.watchdogs.scheduleSttForceFinalize(pkt.contextId);
+  }
+
+  private handleUserText(pkt: UserTextReceivedPacket): void {
+    // Treat text input as an immediate EOS turn complete
+    this.bus.push(Route.Main, make.eosTurnComplete(pkt.contextId, pkt.timestampMs, pkt.text, []));
+  }
+
+  private handleSttInterim(pkt: SttInterimPacket): void {
+    this.turnArbiter.noteInterimEvidence(pkt.text);
+    this.maybeBargeInFromProviderStt(pkt.contextId, pkt.text, pkt.timestampMs);
+    this.currentTurnId = pkt.contextId;
+    this.emit("user_input_partial", {
+      tsMs: pkt.timestampMs,
+      turnId: pkt.contextId,
+      text: pkt.text,
+    });
+    this.debugPush({
+      component: "stt",
+      type: "interim",
+      data: { context_id: pkt.contextId, text: pkt.text },
+      timestampMs: pkt.timestampMs,
+    });
+  }
+
+  private handleSttResult(pkt: SttResultPacket): void {
+    this.watchdogs.clearSttForceFinalizeIfContext(pkt.contextId);
+    this.turnArbiter.noteInterimEvidence(pkt.text, pkt.confidence);
+    this.maybeBargeInFromProviderStt(pkt.contextId, pkt.text, pkt.timestampMs);
+    this.currentTurnId = pkt.contextId;
+    this.emit("user_input_final", {
+      tsMs: pkt.timestampMs,
+      turnId: pkt.contextId,
+      text: pkt.text,
+      confidence: pkt.confidence,
+    });
+    this.debugPush({
+      component: "stt",
+      type: "final",
+      data: {
+        context_id: pkt.contextId,
+        text: pkt.text,
+        confidence: String(pkt.confidence),
+      },
+      timestampMs: pkt.timestampMs,
+    });
+  }
+
+  private handleVadAudioForSpeakerGate(pkt: VadAudioPacket): void {
+    if (this.turnArbiter.observeBargeInAudio(pkt)) return;
+
+    if (this.shouldEnrollPrimarySpeaker(pkt.contextId)) {
+      this.primarySpeakerGate.enrollUserTurnChunk(pkt.audio);
+    }
+  }
+
+  private shouldEnrollPrimarySpeaker(contextId: string): boolean {
+    return (
+      !this.latestActiveTtsContextId() &&
+      this.speakerEnrollmentContextId === contextId
+    );
+  }
+
+  // Deployments that delegate endpointing to the provider STT register no VAD
+  // plugin, so vad.speech_started never fires and barge-in would stay dormant.
+  // Provider transcripts arriving while TTS playout is active are the speech
+  // evidence instead (echo of our own playout is mitigated by client AEC plus
+  // the arbiter's backchannel / low-confidence suppression).
+  private maybeBargeInFromProviderStt(contextId: string, text: string, timestampMs: number): void {
+    if (this.endpointingOwner !== "provider_stt") return;
+    if (!text.trim()) return;
+    const interruptedContextId = this.latestActiveTtsContextId();
+    if (!interruptedContextId) return;
+    this.turnArbiter.onProviderSttEvidence(contextId, timestampMs, interruptedContextId);
+  }
+
+  private handleVadSpeechStarted(pkt: VadSpeechStartedPacket): void {
+    this.lastFinalizedContextId = "";
+
+    if (this.latencyFiller.isFillerOnly(this.currentTurnId)) {
+      this.cancelLatencyFillerTurn(this.currentTurnId, pkt.timestampMs);
+    }
+
+    this.emit("user_started_speaking", {
+      tsMs: pkt.timestampMs,
+      turnId: pkt.contextId,
+    });
+    this.debugPush({
+      component: "vad",
+      type: "speech_started",
+      data: {
+        context_id: pkt.contextId,
+        confidence: String(pkt.confidence),
+      },
+      timestampMs: pkt.timestampMs,
+    });
+
+    const interruptedContextId = this.latestActiveTtsContextId();
+    if (!interruptedContextId) {
+      this.speakerEnrollmentContextId = pkt.contextId;
+      return;
+    }
+
+    this.turnArbiter.onSpeechStarted(pkt, interruptedContextId);
+  }
+
+  private handleVadSpeechActivity(pkt: VadSpeechActivityPacket): void {
+    this.turnArbiter.onSpeechActivity(pkt);
+  }
+
+  private handleVadSpeechEnded(pkt: VadSpeechEndedPacket): void {
+    this.emit("user_stopped_speaking", {
+      tsMs: pkt.timestampMs,
+      turnId: pkt.contextId,
+    });
+    this.debugPush({
+      component: "vad",
+      type: "speech_ended",
+      data: { context_id: pkt.contextId },
+      timestampMs: pkt.timestampMs,
+    });
+
+    if (this.speakerEnrollmentContextId === pkt.contextId) {
+      this.speakerEnrollmentContextId = null;
+    }
+
+    this.turnArbiter.onSpeechEnded(pkt, Boolean(this.latestActiveTtsContextId()));
+
+    this.turnUserStoppedAtMs.set(pkt.contextId, pkt.timestampMs);
+    this.watchdogs.startVaqiMissedResponseTimer(pkt.contextId, pkt.timestampMs);
+  }
+
+  private handleTurnComplete(pkt: EndOfSpeechPacket): void {
+    if (this.lastFinalizedContextId === pkt.contextId) {
+      this.bus.push(Route.Background, make.metric(pkt.contextId, "eos.duplicate_dropped", "1"));
+      return;
+    }
+    this.lastFinalizedContextId = pkt.contextId;
+
+    // Re-arm per-turn guard state for the next turn. Transports with a stable
+    // per-call contextId (telephony callSid) reuse one id across turns, so these
+    // Sets must be cleared at the turn boundary or turn 2+ inherits stale flags:
+    // - firstTtsAudioFired: else vaqi.latency_ms is never emitted again
+    // - interruptedGenerationContextIds: else turn N+1's LLM/TTS packets are dropped after a prior barge-in
+    // - fallbackInjectedContexts: else only one error fallback can ever be spoken per call
+    this.firstTtsAudioFired.delete(pkt.contextId);
+    this.interruptedGenerationContextIds.delete(pkt.contextId);
+    this.fallbackInjectedContexts.delete(pkt.contextId);
+
+    this.currentTurnId = pkt.contextId;
+    this.idleTimeout.setContextId(pkt.contextId);
+
+    this.emit("user_input_final", {
+      tsMs: pkt.timestampMs,
+      turnId: pkt.contextId,
+      text: pkt.text,
+      confidence: 1.0,
+    });
+    this.debugPush({
+      component: "eos",
+      type: "turn_complete",
+      data: { context_id: pkt.contextId, text: pkt.text },
+      timestampMs: pkt.timestampMs,
+    });
+
+    // Stop idle timeout while LLM is processing
+    this.bus.push(Route.Main, make.stopIdleTimeout(pkt.contextId, Date.now(), false));
+
+    const fillerText = this.latencyFiller.start(pkt.contextId, pkt.text, pkt.timestampMs);
+    if (fillerText) {
+      this.bus.push(Route.Main, make.ttsText(pkt.contextId, Date.now(), fillerText));
+      this.bus.push(Route.Background, make.metric(pkt.contextId, "filler.started", fillerText));
+    }
+
+    this.bus.push(
+      Route.Main,
+      make.userInput(pkt.contextId, Date.now(), pkt.text, languageFromTranscripts(pkt.transcripts)),
+    );
+  }
+
+  private handleEosInterim(pkt: InterimEndOfSpeechPacket): void {
+    this.debugPush({
+      component: "eos",
+      type: "interim",
+      data: { context_id: pkt.contextId, text: pkt.text },
+      timestampMs: pkt.timestampMs,
+    });
+  }
+
+  private handleLlmDelta(pkt: LlmDeltaPacket): void {
+    if (this.interruptedGenerationContextIds.has(pkt.contextId)) {
+      this.bus.push(
+        Route.Background,
+        make.metric(pkt.contextId, "llm.delta_ignored_after_interrupt", String(pkt.text.length)),
+      );
+      return;
+    }
+
+    let deltaText = pkt.text;
+    if (!this.firstLlmDeltaReceived.has(pkt.contextId)) {
+      this.firstLlmDeltaReceived.add(pkt.contextId);
+      if (this.latencyFiller.isActive(pkt.contextId)) {
+        deltaText = this.latencyFiller.spliceLlmDelta(pkt.contextId, deltaText);
+        this.bus.push(Route.Background, make.metric(pkt.contextId, "filler.spliced", "1"));
+      }
+    }
+
+    this.emit("agent_text_delta", {
+      tsMs: pkt.timestampMs,
+      turnId: pkt.contextId,
+      delta: deltaText,
+    });
+    this.debugPush({
+      component: "llm",
+      type: "delta",
+      data: { context_id: pkt.contextId, text: deltaText },
+      timestampMs: pkt.timestampMs,
+    });
+
+    this.bufferTtsText(pkt.contextId, deltaText);
+  }
+
+  private handleLlmDone(pkt: LlmResponseDonePacket): void {
+    if (this.interruptedGenerationContextIds.has(pkt.contextId)) {
+      this.ttsTextBuffers.delete(pkt.contextId);
+      this.bus.push(Route.Background, make.metric(pkt.contextId, "llm.done_ignored_after_interrupt", "1"));
+      return;
+    }
+
+    const spokenText = this.flushTtsText(pkt.contextId);
+    this.emit("agent_finished", {
+      tsMs: pkt.timestampMs,
+      turnId: pkt.contextId,
+    });
+    this.debugPush({
+      component: "llm",
+      type: "done",
+      data: { context_id: pkt.contextId },
+      timestampMs: pkt.timestampMs,
+    });
+
+    // Start idle timeout after agent finishes
+    this.bus.push(Route.Main, make.startIdleTimeout(pkt.contextId, Date.now()));
+
+    this.bus.push(Route.Main, make.ttsDone(pkt.contextId, Date.now(), spokenText));
+  }
+
+  private bufferTtsText(contextId: string, text: string): void {
+    const buffer = this.ttsTextBuffers.get(contextId) ?? { pending: "", emitted: "" };
+    buffer.pending += text;
+    const complete = takeCompleteVoiceText(buffer.pending);
+    if (complete.text) {
+      this.bus.push(Route.Main, make.ttsText(contextId, Date.now(), complete.text));
+      buffer.emitted = appendVoiceText(buffer.emitted, complete.text);
+    }
+    buffer.pending = complete.remaining;
+    this.ttsTextBuffers.set(contextId, buffer);
+  }
+
+  private flushTtsText(contextId: string): string {
+    const buffer = this.ttsTextBuffers.get(contextId);
+    if (!buffer) return "";
+    const tail = buffer.pending.trim();
+    if (tail) {
+      this.bus.push(Route.Main, make.ttsText(contextId, Date.now(), tail));
+      buffer.emitted = appendVoiceText(buffer.emitted, tail);
+      buffer.pending = "";
+      this.bus.push(
+        Route.Background,
+        make.metric(contextId, isCompleteVoiceText(tail) ? "tts.final_text_flushed" : "tts.final_tail_flushed", tail),
+      );
+    }
+    this.ttsTextBuffers.delete(contextId);
+    this.latencyFiller.clear(contextId);
+    this.firstLlmDeltaReceived.delete(contextId);
+    return buffer.emitted.trim();
+  }
+
+  private cancelLatencyFillerTurn(contextId: string, timestampMs: number): void {
+    const cancelled = this.latencyFiller.cancel(contextId);
+    if (!cancelled) return;
+    this.bus.push(Route.Background, make.metric(contextId, "filler.cancelled", cancelled.text, timestampMs));
+    this.turnArbiter.emitInterruptDetected(contextId);
+  }
+
+  private handleLlmToolCall(pkt: LlmToolCallPacket): void {
+    this.emit("agent_tool_call", {
+      tsMs: pkt.timestampMs,
+      turnId: pkt.contextId,
+      id: pkt.toolId,
+      name: pkt.toolName,
+      args: pkt.toolArgs,
+    });
+    this.debugPush({
+      component: "tool",
+      type: "call_started",
+      data: {
+        context_id: pkt.contextId,
+        tool_id: pkt.toolId,
+        tool_name: pkt.toolName,
+      },
+      timestampMs: pkt.timestampMs,
+    });
+  }
+
+  private handleLlmToolResult(pkt: LlmToolResultPacket): void {
+    this.emit("agent_tool_result", {
+      tsMs: pkt.timestampMs,
+      turnId: pkt.contextId,
+      id: pkt.toolId,
+      result: pkt.result,
+      durationMs: 0,
+    });
+    this.debugPush({
+      component: "tool",
+      type: "call_completed",
+      data: {
+        context_id: pkt.contextId,
+        tool_id: pkt.toolId,
+        tool_name: pkt.toolName,
+      },
+      timestampMs: pkt.timestampMs,
+    });
+  }
+
+  private handleTtsAudio(pkt: TextToSpeechAudioPacket): void {
+    if (this.interruptedGenerationContextIds.has(pkt.contextId)) {
+      this.bus.push(
+        Route.Background,
+        make.metric(pkt.contextId, "tts.audio_ignored_after_interrupt", String(pkt.audio.byteLength)),
+      );
+      return;
+    }
+
+    if (!this.firstTtsAudioFired.has(pkt.contextId)) {
+      this.firstTtsAudioFired.add(pkt.contextId);
+      this.watchdogs.clearVaqiIfContext(pkt.contextId);
+      const userStoppedMs = this.turnUserStoppedAtMs.get(pkt.contextId);
+      if (userStoppedMs !== undefined) {
+        this.bus.push(
+          Route.Background,
+          make.metric(pkt.contextId, "vaqi.latency_ms", String(pkt.timestampMs - userStoppedMs)),
+        );
+        this.turnUserStoppedAtMs.delete(pkt.contextId);
+      }
+    }
+
+    this.primarySpeakerGate.observeAssistantPlayout(pkt.audio);
+    // Audio just arrived — (re)arm the stall watchdog for this turn's TTS output.
+    this.watchdogs.armTtsStallTimer(pkt.contextId);
+
+    // Extend idle timeout by audio duration to prevent timeout during playback.
+    const sampleRateHz = requireTtsAudioSampleRate(pkt.sampleRateHz);
+    const audioDurationMs = estimatePcm16Duration(pkt.audio, sampleRateHz);
+    this.idleTimeout.extend(audioDurationMs);
+
+    // Mark active and advance this context's playout cursor by the chunk's
+    // realtime duration.
+    this.ttsPlayout.noteAudio(pkt.contextId, audioDurationMs, Date.now());
+
+    this.debugPush({
+      component: "tts",
+      type: "audio",
+      data: {
+        context_id: pkt.contextId,
+        bytes: String(pkt.audio.length),
+      },
+      timestampMs: pkt.timestampMs,
+    });
+
+    this.bus.push(Route.Main, make.recordAssistantAudio(pkt.contextId, Date.now(), pkt.audio, sampleRateHz));
+  }
+
+  private handleTtsEnd(pkt: TextToSpeechEndPacket): void {
+    // Generation finished, but the streamed audio is still playing out. Keep the
+    // context interruptible until its playout estimate elapses, then release it.
+    this.ttsPlayout.scheduleRelease(pkt.contextId, Date.now());
+    this.watchdogs.clearTtsStallTimerFor(pkt.contextId);
+    this.debugPush({
+      component: "tts",
+      type: "end",
+      data: {},
+      timestampMs: Date.now(),
+    });
+  }
+
+  private handleTtsPlayoutProgress(pkt: TextToSpeechPlayoutProgressPacket): void {
+    this.ttsPlayout.noteProgress(pkt.contextId, pkt.complete, pkt.playedOutMs);
+  }
+
+  private handleInterruptDetected(pkt: InterruptionDetectedPacket): void {
+    this.interruptedGenerationContextIds.add(pkt.contextId);
+    this.latencyFiller.cancel(pkt.contextId);
+    this.firstLlmDeltaReceived.delete(pkt.contextId);
+    this.ttsTextBuffers.delete(pkt.contextId);
+    this.ttsPlayout.release(pkt.contextId);
+    this.watchdogs.clearTtsStallTimerFor(pkt.contextId);
+    this.debugPush({
+      component: "turn",
+      type: "interrupt_detected",
+      data: {
+        context_id: pkt.contextId,
+        source: pkt.source,
+      },
+      timestampMs: pkt.timestampMs,
+    });
+
+    this.bus.push(
+      Route.Background,
+      make.metric(
+        pkt.contextId,
+        "interrupt.onset_to_logic_cancel_ms",
+        String(Math.max(0, Date.now() - pkt.timestampMs)),
+      ),
+    );
+
+    // Stop idle timeout
+    this.bus.push(Route.Critical, make.stopIdleTimeout(pkt.contextId, Date.now(), true));
+
+    this.bus.push(Route.Critical, make.recordAssistantTruncate(pkt.contextId, Date.now()));
+
+    // Interrupt TTS, then LLM
+    this.bus.push(Route.Critical, make.interruptTts(pkt.contextId, pkt.timestampMs));
+    this.bus.push(Route.Critical, make.interruptLlm(pkt.contextId, pkt.timestampMs));
+    // Reset STT transcript state too, so a barge-in cannot leak stale finalized
+    // segments into the next turn when a client reuses the same contextId
+    // (the provider STT plugins listen for interrupt.stt; previously unfired).
+    this.bus.push(Route.Critical, make.interruptStt(pkt.contextId, pkt.timestampMs));
+  }
+
+  private handleComponentError(pkt: VoiceErrorPacket): void {
+    this.emit("error", {
+      tsMs: pkt.timestampMs,
+      stage: `${pkt.component}.error`,
+      category: pkt.category,
+      message: pkt.cause.message,
+    });
+
+    this.debugPush({
+      component: pkt.component,
+      type: "error",
+      data: {
+        category: pkt.category,
+        message: pkt.cause.message,
+        recoverable: String(pkt.isRecoverable),
+      },
+      timestampMs: pkt.timestampMs,
+    });
+
+    this.latencyFiller.clear(pkt.contextId);
+    // The packet's own recoverability verdict is authoritative when present: the
+    // bus marks handler exceptions (pipeline.error) recoverable by design — one
+    // misbehaving handler must degrade the turn, not kill the whole call.
+    const recoverable = typeof pkt.isRecoverable === "boolean" ? pkt.isRecoverable : isRecoverable(pkt.category);
+    if (!recoverable) {
+      // Fatal error — close session
+      void this.close().catch(() => {
+        // Best effort
+      });
+      return;
+    }
+    // Recoverable errors are handled by individual component retry logic. But never
+    // leave the caller in unexplained silence: if the reasoning layer failed the turn,
+    // speak a graceful fallback (G4 — Deepgram guide "never fail silently").
+    this.maybeSpeakErrorFallback(pkt);
+  }
+
+  private maybeSpeakErrorFallback(pkt: VoiceErrorPacket): void {
+    if (!this.errorFallbackText) return;
+    // Only the reasoning layer: a TTS/STT failure can't reliably use the same TTS path
+    // for a fallback (that needs canned audio / a clarification prompt — out of scope).
+    if (pkt.component !== "llm") return;
+    const contextId = pkt.contextId;
+    if (contextId.endsWith(FALLBACK_CONTEXT_SUFFIX)) return; // never fall back for a fallback
+    if (this.fallbackInjectedContexts.has(contextId)) return; // at most once per turn
+    this.fallbackInjectedContexts.add(contextId);
+    this.bus.push(Route.Background, make.metric(contextId, "error.fallback_spoken", pkt.component));
+    this.bus.push(
+      Route.Main,
+      make.injectMessage(`${contextId}${FALLBACK_CONTEXT_SUFFIX}`, Date.now(), this.errorFallbackText),
+    );
+  }
+
+  private handleInitFailed(pkt: InitializationFailedPacket): void {
+    this.emit("error", {
+      tsMs: pkt.timestampMs,
+      stage: `init.${pkt.stage}`,
+      category: ErrorCategory.InternalFault,
+      message: `Initialization failed: ${pkt.stage}/${pkt.component} — ${pkt.cause.message}`,
+    });
+  }
+
+  private handleInjectMessage(pkt: InjectMessagePacket): void {
+    // Inject as synthetic LLM output — goes through normal TTS path
+    this.bus.push(Route.Main, make.llmDelta(pkt.contextId, Date.now(), pkt.text));
+    this.bus.push(Route.Main, make.llmDone(pkt.contextId, Date.now(), pkt.text));
+  }
+
+  private handleDisconnect(pkt: DisconnectRequestedPacket): void {
+    this.emit("closed", { tsMs: pkt.timestampMs, reason: pkt.reason });
+    this.debugPush({
+      component: "session",
+      type: "disconnect_requested",
+      data: { reason: pkt.reason },
+      timestampMs: pkt.timestampMs,
+    });
+    void this.close().catch(() => {
+      // Best effort
+    });
+  }
+
+  // =========================================================================
+  // Init Chain
+  // =========================================================================
+
+  private buildInitChain(): void {
+    const steps: InitStep[] = [];
+    this.applyEndpointingOwnerInvariant();
+
+    for (const [name, plugin] of this.plugins) {
+      if (!this.shouldInitializePlugin(plugin)) continue;
+      steps.push({
+        name,
+        stage: pluginStage(name),
+        run: () => {
+          return plugin.initialize(this.bus, this.config.plugins[name] ?? {});
+        },
+        cleanup: () => plugin.close(),
+      });
+    }
+
+    const orderedPluginSteps = steps.sort(
+      (a, b) => stageOrder(a.stage) - stageOrder(b.stage),
+    );
+    this.modeSwitcher.register({
+      textToAudioSteps: orderedPluginSteps,
+      audioToTextCleanups: [...orderedPluginSteps]
+        .reverse()
+        .filter((step) => isAudioStage(step.stage))
+        .map((step) => async () => {
+          await step.cleanup?.();
+        }),
+    });
+
+    // Behavior is always last (idle timeout, max session)
+    orderedPluginSteps.push({
+      name: "idle_timeout",
+      stage: InitStage.Behavior,
+      run: async () => {
+        this.idleTimeout.start();
+      },
+      cleanup: async () => {
+        this.idleTimeout.dispose();
+      },
+    });
+
+    this.initSteps = orderedPluginSteps;
+  }
+
+  private applyEndpointingOwnerInvariant(): void {
+    if (this.endpointingOwner === "timer") return;
+    const owner: EndpointingOwner = this.endpointingOwner;
+    const finalizers = [...this.plugins.entries()]
+      .filter(([, plugin]) => plugin.endpointingCapability !== undefined);
+    const enabledFinalizers = finalizers
+      .filter(([, plugin]) => plugin.endpointingCapability?.owner === owner);
+    if (finalizers.length > 0 && enabledFinalizers.length !== 1) {
+      throw new Error(
+        `endpointingOwner=${owner} requires exactly one registered ${owner} EOS finalizer; found ${String(enabledFinalizers.length)}`,
+      );
+    }
+    for (const [name, plugin] of finalizers) {
+      if (plugin.endpointingCapability?.owner === owner) continue;
+      const disabled = plugin.endpointingCapability?.disableConfig;
+      if (!disabled) continue;
+      this.config.plugins[name] = {
+        ...(this.config.plugins[name] ?? {}),
+        ...disabled,
+      };
+    }
+  }
+
+  private shouldInitializePlugin(plugin: VoicePlugin): boolean {
+    const capability = plugin.endpointingCapability;
+    if (!capability) return true;
+    if (this.endpointingOwner === "timer") return false;
+    if (capability.owner === "smart_turn") return capability.owner === this.endpointingOwner;
+    return true;
+  }
+
+  // =========================================================================
+  // Helpers
+  // =========================================================================
+
+  private emitDebug(
+    component: string,
+    type: string,
+    data: Record<string, string>,
+  ): void {
+    this.debugPush({
+      component,
+      type,
+      data,
+      timestampMs: Date.now(),
+    });
+  }
+
+  private latestActiveTtsContextId(): string {
+    return this.ttsPlayout.latestActive();
+  }
+}