@kuralle-syrinx/core 2.1.0

package/src/turn-arbiter.ts

@@ -0,0 +1,283 @@
+// SPDX-License-Identifier: MIT
+//
+// CR-09 — explicit turn-taking state for barge-in decisions (G1 + VE-02).
+
+import { Route, type PipelineBus } from "./pipeline-bus.js";
+import type {
+  VadAudioPacket,
+  VadSpeechActivityPacket,
+  VadSpeechEndedPacket,
+  VadSpeechStartedPacket,
+} from "./packets.js";
+import { PrimarySpeakerGate } from "./primary-speaker-gate.js";
+import { TtsPlayoutClock } from "./tts-playout-clock.js";
+import * as make from "./packet-factories.js";
+
+type TurnInterruptionState =
+  | { kind: "idle" }
+  | {
+      kind: "pending";
+      userContextId: string;
+      interruptedContextId: string;
+      firstSpeechMs: number;
+      awaitingAudio: boolean;
+    };
+
+type PendingTurnInterruption = Extract<TurnInterruptionState, { kind: "pending" }>;
+
+const BACKCHANNELS = new Set([
+  "yeah",
+  "yep",
+  "yup",
+  "uh huh",
+  "uhhuh",
+  "uh-huh",
+  "mhm",
+  "mm hmm",
+  "mm-hmm",
+  "mmhmm",
+  "okay",
+  "ok",
+  "right",
+  "sure",
+  "uh",
+  "um",
+  "hmm",
+  "i see",
+  "got it",
+  "gotcha",
+  "oh",
+]);
+
+function isBackchannel(text: string): boolean {
+  const norm = text
+    .toLowerCase()
+    .replace(/[^a-z\s'-]/g, "")
+    .trim()
+    .replace(/\s+/g, " ");
+  if (!norm) return false;
+  return BACKCHANNELS.has(norm);
+}
+
+export interface TurnArbiterDeps {
+  readonly bus: PipelineBus;
+  readonly primarySpeakerGate: PrimarySpeakerGate;
+  readonly ttsPlayout: TtsPlayoutClock;
+  readonly minInterruptionMs: number;
+}
+
+export class TurnArbiter {
+  private turnInterruption: TurnInterruptionState = { kind: "idle" };
+  private latestInterimText = "";
+  private latestInterimConfidence: number | null = null;
+
+  constructor(private readonly deps: TurnArbiterDeps) {}
+
+  noteInterimEvidence(text: string, confidence?: number): void {
+    this.latestInterimText = text;
+    this.latestInterimConfidence = typeof confidence === "number" ? confidence : null;
+  }
+
+  onSpeechStarted(pkt: VadSpeechStartedPacket, interruptedContextId: string): void {
+    const { minInterruptionMs, bus, primarySpeakerGate } = this.deps;
+
+    // Idempotent for an already-pending barge-in: with both a local VAD and a
+    // provider STT emitting speech-start (vad_events), the later event must not
+    // reset firstSpeechMs and delay the commit.
+    if (this.pendingFor(pkt.contextId)) return;
+
+    if (minInterruptionMs <= 0) {
+      if (this.shouldDeferImmediateBargeInForSpeakerGate()) {
+        this.transitionToPending(pkt, interruptedContextId, true);
+        return;
+      }
+      bus.push(Route.Background, make.metric(interruptedContextId, "vaqi.interruption", "1"));
+      this.emitInterruptDetected(interruptedContextId);
+      return;
+    }
+
+    this.transitionToPending(pkt, interruptedContextId, false);
+  }
+
+  onSpeechActivity(pkt: VadSpeechActivityPacket): void {
+    const pending = this.pendingFor(pkt.contextId);
+    if (!pending) return;
+    if (pkt.timestampMs - pending.firstSpeechMs < this.deps.minInterruptionMs) return;
+    this.tryCommit(pkt.timestampMs);
+  }
+
+  // Barge-in evidence for deployments where the provider STT owns endpointing and
+  // no VAD plugin emits vad.speech_started: interim/final transcripts arriving
+  // while TTS playout is active are the speech signal. First evidence opens the
+  // pending window; later evidence commits once sustained past minInterruptionMs,
+  // through the same backchannel / low-confidence / speaker-gate suppression.
+  onProviderSttEvidence(userContextId: string, timestampMs: number, interruptedContextId: string): void {
+    const state = this.turnInterruption;
+    if (state.kind === "pending") {
+      if (state.userContextId !== userContextId) return;
+      this.tryCommit(timestampMs);
+      return;
+    }
+    this.transitionToPending({ contextId: userContextId, timestampMs }, interruptedContextId, false);
+  }
+
+  onSpeechEnded(pkt: VadSpeechEndedPacket, hasActiveTts: boolean): void {
+    const pending = this.pendingFor(pkt.contextId);
+    if (pending) {
+      const durationMs = pkt.timestampMs - pending.firstSpeechMs;
+      if (durationMs >= this.deps.minInterruptionMs) {
+        if (
+          this.deps.primarySpeakerGate.isEnabled() &&
+          this.deps.primarySpeakerGate.hasProfile() &&
+          !this.deps.primarySpeakerGate.shouldCommitBargeIn()
+        ) {
+          this.suppress(pending, "interrupt.suppressed_non_primary", durationMs);
+        } else {
+          this.tryCommit(pkt.timestampMs);
+        }
+      } else {
+        this.suppress(pending, "interrupt.suppressed_short_speech", durationMs);
+      }
+      return;
+    }
+
+    if (!hasActiveTts) {
+      this.deps.primarySpeakerGate.lockProfileFromFirstTurn();
+    }
+  }
+
+  observeBargeInAudio(pkt: VadAudioPacket): boolean {
+    const pending = this.pendingFor(pkt.contextId);
+    if (!pending) return false;
+
+    this.deps.primarySpeakerGate.observeBargeInChunk(pkt.audio);
+    if (pending.awaitingAudio) {
+      this.setAwaitingAudio(false);
+      this.tryCommit(pkt.timestampMs);
+    }
+    return true;
+  }
+
+  emitInterruptDetected(interruptedContextId: string): void {
+    this.deps.bus.push(Route.Critical, make.interruptDetected(interruptedContextId, Date.now(), "vad"));
+  }
+
+  commitClientInterrupt(interruptedContextId: string): void {
+    if (!this.deps.ttsPlayout.isActive(interruptedContextId)) return;
+    this.turnInterruption = { kind: "idle" };
+    this.deps.primarySpeakerGate.resetBargeInWindow();
+    this.deps.bus.push(Route.Background, make.metric(interruptedContextId, "interrupt.committed_after_ms", "0"));
+    this.deps.bus.push(Route.Background, make.metric(interruptedContextId, "vaqi.interruption", "1"));
+    this.deps.bus.push(Route.Background, make.metric(interruptedContextId, "interrupt.latency_ms", "0"));
+    this.deps.bus.push(Route.Critical, make.interruptDetected(interruptedContextId, Date.now(), "client"));
+  }
+
+  clear(): void {
+    this.turnInterruption = { kind: "idle" };
+    this.latestInterimText = "";
+    this.latestInterimConfidence = null;
+  }
+
+  private pendingFor(userContextId: string): PendingTurnInterruption | null {
+    const state = this.turnInterruption;
+    if (state.kind !== "pending" || state.userContextId !== userContextId) return null;
+    return state;
+  }
+
+  private transitionToPending(
+    pkt: Pick<VadSpeechStartedPacket, "contextId" | "timestampMs">,
+    interruptedContextId: string,
+    awaitingAudio: boolean,
+  ): void {
+    this.deps.primarySpeakerGate.beginBargeInWindow();
+    // Reset interim evidence so a stale low-confidence/backchannel interim from a
+    // previous turn cannot suppress this new turn's barge-in. The current turn's
+    // own interims (noteInterimEvidence) repopulate it before tryCommit reads it.
+    this.latestInterimText = "";
+    this.latestInterimConfidence = null;
+    this.turnInterruption = {
+      kind: "pending",
+      userContextId: pkt.contextId,
+      interruptedContextId,
+      firstSpeechMs: pkt.timestampMs,
+      awaitingAudio,
+    };
+  }
+
+  private setAwaitingAudio(awaitingAudio: boolean): void {
+    const state = this.turnInterruption;
+    if (state.kind !== "pending") return;
+    this.turnInterruption = { ...state, awaitingAudio };
+  }
+
+  private shouldDeferImmediateBargeInForSpeakerGate(): boolean {
+    const gate = this.deps.primarySpeakerGate;
+    return gate.isEnabled() && gate.hasProfile();
+  }
+
+  private tryCommit(nowMs: number): void {
+    const pending = this.turnInterruption.kind === "pending" ? this.turnInterruption : null;
+    if (!pending) return;
+    if (nowMs - pending.firstSpeechMs < this.deps.minInterruptionMs) return;
+
+    const gate = this.deps.primarySpeakerGate;
+    if (gate.isEnabled() && gate.hasProfile() && !gate.shouldCommitBargeIn()) {
+      this.suppress(pending, "interrupt.suppressed_non_primary", nowMs - pending.firstSpeechMs);
+      return;
+    }
+
+    const sustainedMs = nowMs - pending.firstSpeechMs;
+    if (this.latestInterimConfidence !== null && this.latestInterimConfidence < 0.5) {
+      this.suppress(pending, "interrupt.suppressed_low_confidence", sustainedMs);
+      return;
+    }
+    if (this.latestInterimText && isBackchannel(this.latestInterimText)) {
+      this.suppress(pending, "interrupt.suppressed_backchannel", sustainedMs);
+      return;
+    }
+    this.turnInterruption = { kind: "idle" };
+    gate.resetBargeInWindow();
+
+    const { bus, ttsPlayout } = this.deps;
+    if (!ttsPlayout.isActive(pending.interruptedContextId)) {
+      bus.push(
+        Route.Background,
+        make.metric(pending.interruptedContextId, "interrupt.gate_resolved_after_tts_end", String(sustainedMs)),
+      );
+      return;
+    }
+
+    bus.push(
+      Route.Background,
+      make.metric(pending.interruptedContextId, "interrupt.committed_after_ms", String(sustainedMs)),
+    );
+    bus.push(Route.Background, make.metric(pending.interruptedContextId, "vaqi.interruption", "1"));
+    bus.push(
+      Route.Background,
+      make.metric(pending.interruptedContextId, "interrupt.latency_ms", String(sustainedMs)),
+    );
+    this.emitInterruptDetected(pending.interruptedContextId);
+    this.latestInterimText = "";
+    this.latestInterimConfidence = null;
+  }
+
+  private suppress(
+    pending: PendingTurnInterruption,
+    metricName: string,
+    durationMs: number,
+  ): void {
+    if (metricName === "interrupt.suppressed_non_primary") {
+      this.deps.primarySpeakerGate.beginBargeInWindow();
+      this.turnInterruption = { ...pending };
+    } else {
+      this.turnInterruption = { kind: "idle" };
+      this.deps.primarySpeakerGate.resetBargeInWindow();
+    }
+    this.deps.bus.push(
+      Route.Background,
+      make.metric(pending.interruptedContextId, metricName, String(durationMs)),
+    );
+    this.latestInterimText = "";
+    this.latestInterimConfidence = null;
+  }
+}

package/src/voice-agent-session-util.ts

@@ -0,0 +1,240 @@
+// SPDX-License-Identifier: MIT
+
+import { Route, type PipelineBus } from "./pipeline-bus.js";
+import type { SttResultPacket } from "./packets.js";
+import { ErrorCategory, SessionState } from "./packets.js";
+import type { VoicePlugin } from "./plugin-contract.js";
+import { TimerScheduler, type Scheduler } from "./scheduler.js";
+import { TtsPlayoutClock } from "./tts-playout-clock.js";
+import * as make from "./packet-factories.js";
+
+export function estimatePcm16Duration(audio: Uint8Array, sampleRate: number): number {
+  const samples = audio.length / 2;
+  return (samples / sampleRate) * 1000;
+}
+
+export function requireTtsAudioSampleRate(value: unknown): number {
+  if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) {
+    throw new Error("tts.audio sampleRateHz must be a positive integer");
+  }
+  return value;
+}
+
+export function languageFromTranscripts(transcripts: readonly SttResultPacket[]): string {
+  for (const transcript of transcripts) {
+    if (transcript.language) {
+      return transcript.language;
+    }
+  }
+  return "";
+}
+
+interface ForceFinalizableSttPlugin extends VoicePlugin {
+  forceFinalize(contextId?: string): void;
+}
+
+export function findForceFinalizableSttPlugin(
+  plugins: ReadonlyMap<string, VoicePlugin>,
+): ForceFinalizableSttPlugin | null {
+  for (const name of ["stt", "deepgram"]) {
+    const plugin = plugins.get(name);
+    if (isForceFinalizableSttPlugin(plugin)) {
+      return plugin;
+    }
+  }
+
+  for (const plugin of plugins.values()) {
+    if (isForceFinalizableSttPlugin(plugin)) {
+      return plugin;
+    }
+  }
+
+  return null;
+}
+
+function isForceFinalizableSttPlugin(
+  plugin: VoicePlugin | undefined,
+): plugin is ForceFinalizableSttPlugin {
+  return (
+    typeof plugin === "object" &&
+    plugin !== null &&
+    "forceFinalize" in plugin &&
+    typeof plugin.forceFinalize === "function"
+  );
+}
+
+export interface VoiceSessionWatchdogsDeps {
+  readonly bus: PipelineBus;
+  readonly plugins: ReadonlyMap<string, VoicePlugin>;
+  readonly ttsPlayout: TtsPlayoutClock;
+  readonly sttForceFinalizeTimeoutMs: number;
+  readonly vaqiMissedResponseMs: number;
+  readonly ttsStallMs: number;
+  readonly inputCadenceTimeoutMs: number;
+  readonly getSessionState: () => SessionState;
+  readonly isGenerationInterrupted: (contextId: string) => boolean;
+  readonly onVaqiMissedResponseFired: (contextId: string) => void;
+  readonly scheduler?: Scheduler;
+}
+
+export class VoiceSessionWatchdogs {
+  private readonly scheduler: Scheduler;
+  private sttForceFinalizeScheduled = false;
+  private pendingSttContextId = "";
+  private vaqiMissedResponseScheduled = false;
+  private vaqiMissedResponseContextId = "";
+  private vaqiMissedResponseStartMs = 0;
+  private ttsStallScheduled = false;
+  private ttsStallContextId = "";
+  private inputCadenceScheduled = false;
+  private inputCadenceContextId = "";
+
+  constructor(private readonly deps: VoiceSessionWatchdogsDeps) {
+    this.scheduler = deps.scheduler ?? new TimerScheduler();
+  }
+
+  dispose(): void {
+    this.clearSttForceFinalizeTimer();
+    this.clearVaqiMissedResponseTimer();
+    this.clearTtsStallTimer();
+    this.clearInputCadenceWatchdog();
+  }
+
+  scheduleSttForceFinalize(contextId: string): void {
+    if (this.deps.getSessionState() !== SessionState.Ready) return;
+    if (this.deps.sttForceFinalizeTimeoutMs <= 0) return;
+
+    this.pendingSttContextId = contextId;
+    this.clearSttForceFinalizeTimer(false);
+    this.sttForceFinalizeScheduled = true;
+    this.scheduler.schedule("voice.watchdog.stt_force_finalize", this.deps.sttForceFinalizeTimeoutMs, () => {
+      this.sttForceFinalizeScheduled = false;
+      const plugin = findForceFinalizableSttPlugin(this.deps.plugins);
+      plugin?.forceFinalize(contextId);
+    });
+  }
+
+  clearSttForceFinalizeIfContext(contextId: string): void {
+    if (this.pendingSttContextId === contextId) {
+      this.clearSttForceFinalizeTimer();
+    }
+  }
+
+  startVaqiMissedResponseTimer(contextId: string, startMs: number): void {
+    if (this.deps.vaqiMissedResponseMs <= 0) return;
+    this.clearVaqiMissedResponseTimer();
+    this.vaqiMissedResponseContextId = contextId;
+    this.vaqiMissedResponseStartMs = startMs;
+    this.vaqiMissedResponseScheduled = true;
+    this.scheduler.schedule("voice.watchdog.vaqi_missed_response", this.deps.vaqiMissedResponseMs, () => {
+      this.vaqiMissedResponseScheduled = false;
+      const cid = this.vaqiMissedResponseContextId;
+      const elapsedMs = Date.now() - this.vaqiMissedResponseStartMs;
+      this.vaqiMissedResponseContextId = "";
+      this.vaqiMissedResponseStartMs = 0;
+      this.deps.onVaqiMissedResponseFired(cid);
+      this.deps.bus.push(Route.Background, make.metric(cid, "vaqi.missed_response", String(elapsedMs)));
+    });
+  }
+
+  clearVaqiIfContext(contextId: string): void {
+    if (this.vaqiMissedResponseContextId === contextId) {
+      this.clearVaqiMissedResponseTimer();
+    }
+  }
+
+  armTtsStallTimer(contextId: string): void {
+    if (this.deps.ttsStallMs <= 0) return;
+    this.clearTtsStallTimer();
+    this.ttsStallContextId = contextId;
+    this.ttsStallScheduled = true;
+    this.scheduler.schedule("voice.watchdog.tts_stall", this.deps.ttsStallMs, () => {
+      this.ttsStallScheduled = false;
+      const cid = this.ttsStallContextId;
+      this.ttsStallContextId = "";
+      if (this.deps.isGenerationInterrupted(cid)) return;
+      if (!this.deps.ttsPlayout.isActive(cid)) return;
+      this.deps.ttsPlayout.release(cid);
+      this.deps.bus.push(Route.Background, make.metric(cid, "tts.stall_detected", String(this.deps.ttsStallMs)));
+      this.deps.bus.push(
+        Route.Critical,
+        make.ttsError(
+          cid,
+          Date.now(),
+          new Error(`TTS output stalled: no audio or tts.end for ${String(this.deps.ttsStallMs)}ms`),
+          ErrorCategory.NetworkTimeout,
+          true,
+        ),
+      );
+    });
+  }
+
+  clearTtsStallTimerFor(contextId: string): void {
+    if (this.ttsStallContextId === contextId) this.clearTtsStallTimer();
+  }
+
+  scheduleInputCadenceWatchdog(contextId: string): void {
+    if (this.deps.inputCadenceTimeoutMs <= 0) return;
+    if (this.deps.getSessionState() !== SessionState.Ready) return;
+
+    this.clearInputCadenceWatchdog();
+    this.inputCadenceContextId = contextId;
+    this.inputCadenceScheduled = true;
+    this.scheduler.schedule("voice.watchdog.input_cadence", this.deps.inputCadenceTimeoutMs, () => {
+      this.inputCadenceScheduled = false;
+      const cid = this.inputCadenceContextId;
+      if (this.deps.getSessionState() !== SessionState.Ready) return;
+
+      this.deps.bus.push(
+        Route.Background,
+        make.metric(cid, "input.cadence_stall_ms", String(this.deps.inputCadenceTimeoutMs)),
+      );
+      this.deps.bus.push(Route.Critical, {
+        kind: "pipeline.error",
+        contextId: cid,
+        timestampMs: Date.now(),
+        component: "pipeline",
+        category: ErrorCategory.NetworkTimeout,
+        cause: new Error("inbound audio stalled"),
+        isRecoverable: true,
+      });
+
+      this.scheduleInputCadenceWatchdog(cid);
+    });
+  }
+
+  clearInputCadenceWatchdog(): void {
+    if (this.inputCadenceScheduled) {
+      this.scheduler.cancel("voice.watchdog.input_cadence");
+      this.inputCadenceScheduled = false;
+    }
+    this.inputCadenceContextId = "";
+  }
+
+  private clearSttForceFinalizeTimer(clearContext = true): void {
+    if (this.sttForceFinalizeScheduled) {
+      this.scheduler.cancel("voice.watchdog.stt_force_finalize");
+      this.sttForceFinalizeScheduled = false;
+    }
+    if (clearContext) {
+      this.pendingSttContextId = "";
+    }
+  }
+
+  private clearVaqiMissedResponseTimer(): void {
+    if (this.vaqiMissedResponseScheduled) {
+      this.scheduler.cancel("voice.watchdog.vaqi_missed_response");
+      this.vaqiMissedResponseScheduled = false;
+    }
+    this.vaqiMissedResponseContextId = "";
+    this.vaqiMissedResponseStartMs = 0;
+  }
+
+  private clearTtsStallTimer(): void {
+    if (this.ttsStallScheduled) {
+      this.scheduler.cancel("voice.watchdog.tts_stall");
+      this.ttsStallScheduled = false;
+    }
+    this.ttsStallContextId = "";
+  }
+}