@kuralle-syrinx/core 4.1.0 → 4.2.0

package/src/voice-agent-session.ts

@@ -41,6 +41,8 @@ import type {
   SpeechToTextAudioPacket,
   SttResultPacket,
   SttInterimPacket,
+  SttPartialPacket,
+  TurnChangePacket,
   VadAudioPacket,
   VadSpeechStartedPacket,
   VadSpeechActivityPacket,
@@ -64,6 +66,11 @@ import { PrimarySpeakerGate } from "./primary-speaker-gate.js";
 import { takeCompleteVoiceText, isCompleteVoiceText, appendVoiceText } from "./voice-text.js";
 import { TtsPlayoutClock } from "./tts-playout-clock.js";
 import { TurnArbiter, isBackchannel } from "./turn-arbiter.js";
+import { InteractionCoordinator } from "./interaction-coordinator.js";
+import { isLifecycleInteractionPolicy, type InteractionPolicy, type WordTiming } from "./interaction-policy.js";
+import { pcm16BytesToSamples } from "./audio/pcm.js";
+import { DeferInteractionPolicy } from "./policies/defer.js";
+import { RuleBasedInteractionPolicy } from "./policies/rule-based.js";
 import * as make from "./packet-factories.js";
 import { pluginStage, stageOrder, isAudioStage } from "./init-stage-order.js";
 import {
@@ -162,6 +169,21 @@ export interface VoiceAgentSessionConfig {
    * provider STT ownership; Smart Turn sessions must opt in explicitly.
    */
   endpointingOwner?: "provider_stt" | "smart_turn" | "timer";
+  /** The front model owns full-duplex interaction (turn-taking + barge-in). When true, the session's
+   *  InteractionPolicy runs observe-only (DeferInteractionPolicy) — Syrinx does not drive its own
+   *  turn/interrupt decisions; the front's native decisions stand. Default: false (Syrinx drives).
+   *  A realtime factory sets this from RealtimeAdapter.caps.supportsFullDuplex. */
+  fullDuplex?: boolean;
+  /** When true, Syrinx suppresses policy-timed backchannel cue packets (front/provider owns them). */
+  emitsBackchannel?: boolean;
+  /**
+   * Optional interaction policy injected by the caller (learned controllers, Smart Turn policy, etc.).
+   * When omitted, the session uses RuleBasedInteractionPolicy. When `fullDuplex` is true, the
+   * coordinator runs observe-only via DeferInteractionPolicy regardless of this setting.
+   */
+  interactionPolicy?: InteractionPolicy;
+  /** Config passed to `interactionPolicy.initialize` when the injected policy is lifecycle-capable. */
+  interactionPolicyConfig?: Record<string, unknown>;
   readonly metricsExporter?: MetricsExporter;
   readonly scheduler?: Scheduler;
   readonly observability?: {
@@ -193,8 +215,8 @@ export interface VoiceAgentSessionEvents {
   /**
    * Per-turn latency decomposition, emitted once at the turn's first TTS audio.
    * `ttfaMs` is anchored to the real end of user speech (VAD speech-end, falling
-   * back to the endpoint decision) — and `fillerUsed` flags turns where a latency
-   * filler spoke first, so TTFA on those turns is not mistaken for reasoning speed.
+   * back to the endpoint decision) — `fillerUsed` flags turns where a latency filler spoke
+   * first, and `backchannelUsed` flags turns where a wait-gap cue played before the answer.
    * Decomposition: eouDelayMs (speech end → endpoint) + llmTtftMs (endpoint →
    * first LLM delta) + ttsTtfbMs (first TTS text dispatched → first audio).
    */
@@ -206,6 +228,7 @@ export interface VoiceAgentSessionEvents {
     llmTtftMs?: number;
     ttsTtfbMs?: number;
     fillerUsed: boolean;
+    backchannelUsed: boolean;
   }) => void;
   agent_first_audio: (event: { tsMs: number; turnId: string }) => void;
   agent_finished: (event: { tsMs: number; turnId: string } & Record<string, unknown>) => void;
@@ -227,6 +250,7 @@ interface TurnTiming {
   firstLlmDeltaMs?: number;
   firstTtsTextMs?: number;
   fillerUsed?: boolean;
+  backchannelUsed?: boolean;
 }
 
 /** Suffix marking a context created to speak an error fallback, so it never recurses. */
@@ -237,6 +261,10 @@ function toolCueTimerKey(contextId: string, toolId: string): string {
   return `tool_cue:${contextId}:${toolId}`;
 }
 
+function interactionPlayoutTimerKey(contextId: string): string {
+  return `interaction.playout:${contextId}`;
+}
+
 // =============================================================================
 // Session Implementation
 // =============================================================================
@@ -268,7 +296,10 @@ export class VoiceAgentSession {
   private ttsTextBuffers = new Map<string, TtsTextBuffer>();
   private readonly minInterruptionMs: number;
   private readonly primarySpeakerGate: PrimarySpeakerGate;
-  private readonly turnArbiter!: TurnArbiter;
+  private readonly ruleBasedPolicy!: RuleBasedInteractionPolicy;
+  private readonly injectedInteractionPolicy: InteractionPolicy | null;
+  private readonly activeInteractionPolicy: InteractionPolicy;
+  private readonly interaction!: InteractionCoordinator;
   private readonly latencyFiller: LatencyFillerController;
   private firstLlmDeltaReceived = new Set<string>();
   private readonly vaqiMissedResponseMs: number;
@@ -280,13 +311,18 @@ export class VoiceAgentSession {
   private turnTimings = new Map<string, TurnTiming>();
   private speakerEnrollmentContextId: string | null = null;
   private firstTtsAudioFired = new Set<string>();
+  private readonly pendingInteractionPlayoutTimers = new Set<string>();
   private readonly errorFallbackText: string;
   private fallbackInjectedContexts = new Set<string>();
   // G3: pending tool calls per context (toolId → toolName) driving the tool_call_cue lifecycle.
   private readonly delayCueAfterMs: number;
   private pendingToolCues = new Map<string, Map<string, string>>();
   private readonly endpointingOwner: "provider_stt" | "smart_turn" | "timer";
+  private readonly fullDuplex: boolean;
+  private readonly emitsBackchannel: boolean;
+  private userSpeaking = false;
   private lastFinalizedContextId = "";
+  private readonly sttPartialWordTimings = new Map<string, readonly WordTiming[]>();
 
   constructor(config: VoiceAgentSessionConfig) {
     const owner = config.endpointingOwner;
@@ -294,6 +330,8 @@ export class VoiceAgentSession {
       throw new Error(`Unsupported endpointingOwner: ${owner}`);
     }
     this.endpointingOwner = owner ?? "provider_stt";
+    this.fullDuplex = config.fullDuplex === true;
+    this.emitsBackchannel = config.emitsBackchannel === true;
     this.config = config;
     this.scheduler = config.scheduler ?? new TimerScheduler();
     this.ttsPlayout = new TtsPlayoutClock(this.scheduler);
@@ -344,12 +382,29 @@ export class VoiceAgentSession {
       },
     });
 
-    this.turnArbiter = new TurnArbiter({
+    this.injectedInteractionPolicy = config.interactionPolicy ?? null;
+    this.ruleBasedPolicy = new RuleBasedInteractionPolicy({
       bus: this.bus,
       primarySpeakerGate: this.primarySpeakerGate,
       ttsPlayout: this.ttsPlayout,
       minInterruptionMs: this.minInterruptionMs,
     });
+    const coordinatorPolicy: InteractionPolicy = this.fullDuplex
+      ? new DeferInteractionPolicy()
+      : (this.injectedInteractionPolicy ?? this.ruleBasedPolicy);
+    this.activeInteractionPolicy = coordinatorPolicy;
+    this.interaction = new InteractionCoordinator({
+      bus: this.bus,
+      policy: coordinatorPolicy,
+      executor: this.ruleBasedPolicy.arbiter,
+      scheduler: this.scheduler,
+      caps: { emitsBackchannel: this.emitsBackchannel },
+      isUserSpeaking: () => this.userSpeaking,
+      isTtsActive: () => this.ttsPlayout.activeContexts().length > 0,
+      onBackchannelEmitted: (contextId) => {
+        this.timingFor(contextId).backchannelUsed = true;
+      },
+    });
     this.watchdogs = new VoiceSessionWatchdogs({
       bus: this.bus,
       plugins: this.plugins,
@@ -386,6 +441,10 @@ export class VoiceAgentSession {
     this.modeSwitcher = new ModeSwitcher(this.bus);
   }
 
+  private get turnArbiter(): TurnArbiter {
+    return this.ruleBasedPolicy.arbiter;
+  }
+
   // =========================================================================
   // Public API
   // =========================================================================
@@ -412,6 +471,7 @@ export class VoiceAgentSession {
 
     // 1. Wire all bus handlers
     this.wireBusHandlers();
+    this.interaction.initialize();
 
     // 2. Start bus drain loop
     this.busStartPromise = this.bus.start();
@@ -443,9 +503,14 @@ export class VoiceAgentSession {
 
     // 1. Stop idle timeout
     this.idleTimeout.dispose();
+    this.interaction.dispose();
     this.watchdogs.dispose();
     this.observabilityObserver.dispose();
     this.ttsPlayout.clear();
+    for (const contextId of this.pendingInteractionPlayoutTimers) {
+      this.scheduler.cancel(interactionPlayoutTimerKey(contextId));
+    }
+    this.pendingInteractionPlayoutTimers.clear();
     this.turnArbiter.clear();
     this.turnUserStoppedAtMs.clear();
     this.turnTimings.clear();
@@ -454,6 +519,7 @@ export class VoiceAgentSession {
     this.ttsTextBuffers.clear();
     this.interruptedGenerationContextIds.clear();
     this.firstLlmDeltaReceived.clear();
+    this.sttPartialWordTimings.clear();
     for (const [contextId, pending] of this.pendingToolCues) {
       for (const toolId of pending.keys()) this.scheduler.cancel(toolCueTimerKey(contextId, toolId));
     }
@@ -538,6 +604,7 @@ export class VoiceAgentSession {
     // STT results
     this.bus.on("stt.audio", this.handleSttAudio.bind(this));
     this.bus.on("stt.interim", this.handleSttInterim.bind(this));
+    this.bus.on("stt.partial", this.handleSttPartial.bind(this));
     this.bus.on("stt.result", this.handleSttResult.bind(this));
 
     // VAD
@@ -547,8 +614,11 @@ export class VoiceAgentSession {
     this.bus.on("vad.audio", this.handleVadAudioForSpeakerGate.bind(this));
 
     // EOS
-    this.bus.on("turn.change", () => {
+    this.bus.on("turn.change", (pkt: TurnChangePacket) => {
       this.lastFinalizedContextId = "";
+      if (pkt.previousContextId) {
+        this.sttPartialWordTimings.delete(pkt.previousContextId);
+      }
     });
     this.bus.on("eos.turn_complete", this.handleTurnComplete.bind(this));
     this.bus.on("eos.interim", this.handleEosInterim.bind(this));
@@ -635,9 +705,22 @@ export class VoiceAgentSession {
       timestampMs: pkt.timestampMs,
     });
 
+    this.observeAudioFrame(pkt);
+
     this.watchdogs.scheduleInputCadenceWatchdog(pkt.contextId);
   }
 
+  private observeAudioFrame(pkt: UserAudioReceivedPacket): void {
+    if (pkt.audio.byteLength < 2 || pkt.audio.byteLength % 2 !== 0) return;
+    this.interaction.observe({
+      kind: "audio_frame",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+      audio: pcm16BytesToSamples(pkt.audio),
+      sampleRateHz: pkt.sampleRateHz ?? 16_000,
+    });
+  }
+
   private handleSttAudio(pkt: SpeechToTextAudioPacket): void {
     this.watchdogs.scheduleSttForceFinalize(pkt.contextId);
   }
@@ -647,9 +730,14 @@ export class VoiceAgentSession {
     this.bus.push(Route.Main, make.eosTurnComplete(pkt.contextId, pkt.timestampMs, pkt.text, []));
   }
 
+  private handleSttPartial(pkt: SttPartialPacket): void {
+    if (pkt.wordTimings) {
+      this.sttPartialWordTimings.set(pkt.contextId, pkt.wordTimings);
+    }
+  }
+
   private handleSttInterim(pkt: SttInterimPacket): void {
-    this.turnArbiter.noteInterimEvidence(pkt.text);
-    this.maybeBargeInFromProviderStt(pkt.contextId, pkt.text, pkt.timestampMs);
+    this.observeSttForBargeIn(pkt.contextId, pkt.timestampMs, pkt.text, "stt_partial");
     this.currentTurnId = pkt.contextId;
     this.emit("user_input_partial", {
       tsMs: pkt.timestampMs,
@@ -666,8 +754,7 @@ export class VoiceAgentSession {
 
   private handleSttResult(pkt: SttResultPacket): void {
     this.watchdogs.clearSttForceFinalizeIfContext(pkt.contextId);
-    this.turnArbiter.noteInterimEvidence(pkt.text, pkt.confidence);
-    this.maybeBargeInFromProviderStt(pkt.contextId, pkt.text, pkt.timestampMs);
+    this.observeSttForBargeIn(pkt.contextId, pkt.timestampMs, pkt.text, "stt_final", pkt.confidence);
     this.currentTurnId = pkt.contextId;
     this.emit("user_input_final", {
       tsMs: pkt.timestampMs,
@@ -688,7 +775,7 @@ export class VoiceAgentSession {
   }
 
   private handleVadAudioForSpeakerGate(pkt: VadAudioPacket): void {
-    if (this.turnArbiter.observeBargeInAudio(pkt)) return;
+    if (this.interaction.observeBargeInAudio(pkt)) return;
 
     if (this.shouldEnrollPrimarySpeaker(pkt.contextId)) {
       this.primarySpeakerGate.enrollUserTurnChunk(pkt.audio);
@@ -707,16 +794,43 @@ export class VoiceAgentSession {
   // Provider transcripts arriving while TTS playout is active are the speech
   // evidence instead (echo of our own playout is mitigated by client AEC plus
   // the arbiter's backchannel / low-confidence suppression).
-  private maybeBargeInFromProviderStt(contextId: string, text: string, timestampMs: number): void {
-    if (this.endpointingOwner !== "provider_stt") return;
-    if (!text.trim()) return;
-    const interruptedContextId = this.latestActiveTtsContextId();
-    if (!interruptedContextId) return;
-    this.turnArbiter.onProviderSttEvidence(contextId, timestampMs, interruptedContextId);
+  private observeSttForBargeIn(
+    contextId: string,
+    timestampMs: number,
+    text: string,
+    kind: "stt_partial" | "stt_final",
+    confidence?: number,
+  ): void {
+    const interruptedContextId =
+      this.endpointingOwner === "provider_stt" && text.trim()
+        ? this.latestActiveTtsContextId() ?? undefined
+        : undefined;
+    const wordTimings = this.sttPartialWordTimings.get(contextId);
+    if (kind === "stt_partial") {
+      this.interaction.observe({
+        kind: "stt_partial",
+        contextId,
+        timestampMs,
+        text,
+        interruptedContextId,
+        ...(wordTimings ? { wordTimings } : {}),
+      });
+      return;
+    }
+    this.interaction.observe({
+      kind: "stt_final",
+      contextId,
+      timestampMs,
+      text,
+      confidence,
+      interruptedContextId,
+      ...(wordTimings ? { wordTimings } : {}),
+    });
   }
 
   private handleVadSpeechStarted(pkt: VadSpeechStartedPacket): void {
     this.lastFinalizedContextId = "";
+    this.userSpeaking = true;
 
     if (this.latencyFiller.isFillerOnly(this.currentTurnId)) {
       this.cancelLatencyFillerTurn(this.currentTurnId, pkt.timestampMs);
@@ -739,17 +853,27 @@ export class VoiceAgentSession {
     const interruptedContextId = this.latestActiveTtsContextId();
     if (!interruptedContextId) {
       this.speakerEnrollmentContextId = pkt.contextId;
-      return;
     }
 
-    this.turnArbiter.onSpeechStarted(pkt, interruptedContextId);
+    this.interaction.observe({
+      kind: "vad_speech_started",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+      confidence: pkt.confidence,
+      ...(interruptedContextId ? { interruptedContextId } : {}),
+    });
   }
 
   private handleVadSpeechActivity(pkt: VadSpeechActivityPacket): void {
-    this.turnArbiter.onSpeechActivity(pkt);
+    this.interaction.observe({
+      kind: "vad_speech_activity",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+    });
   }
 
   private handleVadSpeechEnded(pkt: VadSpeechEndedPacket): void {
+    this.userSpeaking = false;
     this.emit("user_stopped_speaking", {
       tsMs: pkt.timestampMs,
       turnId: pkt.contextId,
@@ -765,7 +889,12 @@ export class VoiceAgentSession {
       this.speakerEnrollmentContextId = null;
     }
 
-    this.turnArbiter.onSpeechEnded(pkt, Boolean(this.latestActiveTtsContextId()));
+    this.interaction.observe({
+      kind: "vad_speech_ended",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+      hasActiveTts: Boolean(this.latestActiveTtsContextId()),
+    });
 
     this.turnUserStoppedAtMs.set(pkt.contextId, pkt.timestampMs);
     this.timingFor(pkt.contextId).speechEndedMs = pkt.timestampMs;
@@ -801,6 +930,7 @@ export class VoiceAgentSession {
         ? { ttsTtfbMs: firstAudioMs - timing.firstTtsTextMs }
         : {}),
       fillerUsed: timing.fillerUsed === true,
+      backchannelUsed: timing.backchannelUsed === true,
     });
   }
 
@@ -824,6 +954,7 @@ export class VoiceAgentSession {
     }
 
     this.lastFinalizedContextId = pkt.contextId;
+    this.interaction.reset(pkt.contextId);
 
     // Re-arm per-turn guard state for the next turn. Transports with a stable
     // per-call contextId (telephony callSid) reuse one id across turns, so these
@@ -1036,14 +1167,22 @@ export class VoiceAgentSession {
     toolName: string,
     afterMs?: number,
   ): void {
+    const tsMs = Date.now();
     this.emit("tool_call_cue", {
-      tsMs: Date.now(),
+      tsMs,
       turnId: contextId,
       phase,
       toolId,
       toolName,
       ...(afterMs !== undefined ? { afterMs } : {}),
     });
+    this.interaction.observe({
+      kind: "delegate_state",
+      contextId,
+      timestampMs: tsMs,
+      delegateInFlight: phase === "started" || phase === "delayed",
+      toolCallPhase: phase,
+    });
   }
 
   /** G3: resolve one pending tool call with a terminal cue phase. */
@@ -1161,6 +1300,14 @@ export class VoiceAgentSession {
     const audioDurationMs = estimatePcm16Duration(pkt.audio, sampleRateHz);
     const now = Date.now();
     this.ttsPlayout.noteAudio(pkt.contextId, audioDurationMs, now);
+    this.interaction.observe({
+      kind: "playout_tick",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+      ttsActive: true,
+      audio: pcm16BytesToSamples(pkt.audio),
+      sampleRateHz,
+    });
 
     // Anchor the idle timer to when playout actually *ends* (P2), not to chunk
     // arrival. TTS streams faster than realtime, so extending by each chunk's
@@ -1187,7 +1334,11 @@ export class VoiceAgentSession {
     // Generation finished, but the streamed audio is still playing out. Keep the
     // context interruptible until its playout estimate elapses, then release it.
     this.generatingContextIds.delete(pkt.contextId);
-    this.ttsPlayout.scheduleRelease(pkt.contextId, Date.now());
+    const now = Date.now();
+    this.ttsPlayout.scheduleRelease(pkt.contextId, now);
+    const playoutEndMs = this.ttsPlayout.playoutEnd(pkt.contextId);
+    const remainingMs = playoutEndMs === undefined ? 0 : Math.max(0, playoutEndMs - now);
+    this.scheduleInteractionPlayoutTick(pkt.contextId, remainingMs);
     this.watchdogs.clearTtsStallTimerFor(pkt.contextId);
     this.debugPush({
       component: "tts",
@@ -1197,11 +1348,39 @@ export class VoiceAgentSession {
     });
   }
 
+  private scheduleInteractionPlayoutTick(contextId: string, delayMs: number): void {
+    this.pendingInteractionPlayoutTimers.add(contextId);
+    this.scheduler.schedule(interactionPlayoutTimerKey(contextId), delayMs, () => {
+      this.pendingInteractionPlayoutTimers.delete(contextId);
+      if (this.ttsPlayout.isActive(contextId)) return;
+      this.interaction.observe({
+        kind: "playout_tick",
+        contextId,
+        timestampMs: Date.now(),
+        ttsActive: false,
+      });
+    });
+  }
+
+  private cancelInteractionPlayoutTick(contextId: string): void {
+    if (!this.pendingInteractionPlayoutTimers.delete(contextId)) return;
+    this.scheduler.cancel(interactionPlayoutTimerKey(contextId));
+  }
+
   private handleTtsPlayoutProgress(pkt: TextToSpeechPlayoutProgressPacket): void {
+    this.cancelInteractionPlayoutTick(pkt.contextId);
     this.ttsPlayout.noteProgress(pkt.contextId, pkt.complete, pkt.playedOutMs);
+    this.interaction.observe({
+      kind: "playout_tick",
+      contextId: pkt.contextId,
+      timestampMs: pkt.timestampMs,
+      playedOutMs: pkt.playedOutMs,
+      ttsActive: !pkt.complete,
+    });
   }
 
   private handleInterruptDetected(pkt: InterruptionDetectedPacket): void {
+    this.cancelInteractionPlayoutTick(pkt.contextId);
     this.interruptedGenerationContextIds.add(pkt.contextId);
     this.failPendingToolCues(pkt.contextId); // G3: the aborted delegate's cue fails (R5)
     this.latencyFiller.cancel(pkt.contextId);
@@ -1251,6 +1430,7 @@ export class VoiceAgentSession {
    * the LLM, and drops late deltas/audio for the stale context.
    */
   private cancelStaleGeneration(contextId: string, timestampMs: number): void {
+    this.cancelInteractionPlayoutTick(contextId);
     this.interruptedGenerationContextIds.add(contextId);
     this.failPendingToolCues(contextId); // G3: a superseded turn's pending cue fails
     this.generatingContextIds.delete(contextId);
@@ -1372,6 +1552,19 @@ export class VoiceAgentSession {
       });
     }
 
+    if (
+      this.activeInteractionPolicy === this.injectedInteractionPolicy &&
+      isLifecycleInteractionPolicy(this.activeInteractionPolicy)
+    ) {
+      const policy = this.activeInteractionPolicy;
+      steps.push({
+        name: "interaction_policy",
+        stage: InitStage.EOS,
+        run: () => policy.initialize(this.config.interactionPolicyConfig ?? {}),
+        cleanup: () => policy.close(),
+      });
+    }
+
     const orderedPluginSteps = steps.sort(
       (a, b) => stageOrder(a.stage) - stageOrder(b.stage),
     );
@@ -1401,6 +1594,20 @@ export class VoiceAgentSession {
   }
 
   private applyEndpointingOwnerInvariant(): void {
+    if (this.activeInteractionPolicy === this.injectedInteractionPolicy) {
+      for (const [name, plugin] of this.plugins) {
+        const capability = plugin.endpointingCapability;
+        if (!capability || capability.owner !== "provider_stt") continue;
+        if (!capability.disableConfig) {
+          throw new Error(`interactionPolicy requires ${name} to expose endpointingCapability.disableConfig`);
+        }
+        this.config.plugins[name] = {
+          ...(this.config.plugins[name] ?? {}),
+          ...capability.disableConfig,
+        };
+      }
+      return;
+    }
     if (this.endpointingOwner === "timer") return;
     const owner: EndpointingOwner = this.endpointingOwner;
     const finalizers = [...this.plugins.entries()]
@@ -1426,6 +1633,9 @@ export class VoiceAgentSession {
   private shouldInitializePlugin(plugin: VoicePlugin): boolean {
     const capability = plugin.endpointingCapability;
     if (!capability) return true;
+    if (this.activeInteractionPolicy === this.injectedInteractionPolicy) {
+      return capability.owner === "provider_stt";
+    }
     if (this.endpointingOwner === "timer") return false;
     if (capability.owner === "smart_turn") return capability.owner === this.endpointingOwner;
     return true;