@kuralle-syrinx/core 4.0.0 → 4.2.0

package/src/voice-agent-session.test.ts

@@ -2,14 +2,25 @@
 
 import { describe, it, expect, vi } from "vitest";
 import { VoiceAgentSession } from "./voice-agent-session.js";
-import { Route, type PipelineBus, type PluginConfig, type VoicePlugin } from "./index.js";
-import { ErrorCategory } from "./packets.js";
+import {
+  Route,
+  type InteractionDecision,
+  type InteractionObservation,
+  type InteractionPolicy,
+  type PipelineBus,
+  type PluginConfig,
+  type VoicePlugin,
+} from "./index.js";
+import { InteractionCoordinator } from "./interaction-coordinator.js";
+import type { Scheduler, ScheduledCallback } from "./scheduler.js";
+import { ErrorCategory, type InteractionBackchannelPacket } from "./packets.js";
 import type {
   EndOfSpeechAudioPacket,
   RecordAssistantAudioPacket,
   RecordUserAudioPacket,
   SpeechToTextAudioPacket,
   SttInterimPacket,
+  SttPartialPacket,
   SttResultPacket,
   EndOfSpeechPacket,
   LlmDeltaPacket,
@@ -104,6 +115,50 @@ class EndpointingPlugin extends CapturingPlugin {
   }
 }
 
+class CapturingInteractionPolicy implements InteractionPolicy {
+  readonly observations: InteractionObservation[] = [];
+  readonly resetContextIds: string[] = [];
+  initializeCount = 0;
+  closeCount = 0;
+  initializedConfig: Record<string, unknown> | null = null;
+
+  constructor(
+    private readonly decide: (observation: InteractionObservation) => readonly InteractionDecision[] = () => [],
+  ) {}
+
+  async initialize(config: Record<string, unknown>): Promise<void> {
+    this.initializeCount += 1;
+    this.initializedConfig = config;
+  }
+
+  observe(observation: InteractionObservation): readonly InteractionDecision[] {
+    this.observations.push(observation);
+    return this.decide(observation);
+  }
+
+  reset(contextId: string): void {
+    this.resetContextIds.push(contextId);
+  }
+
+  async close(): Promise<void> {
+    this.closeCount += 1;
+  }
+}
+
+/** Records scheduled/cancelled timers without firing them, for leak assertions. */
+class RecordingScheduler implements Scheduler {
+  readonly timers = new Map<string, { delayMs: number; cb: ScheduledCallback }>();
+  schedule(key: string, delayMs: number, cb: ScheduledCallback): void {
+    this.timers.set(key, { delayMs, cb });
+  }
+  cancel(key: string): void {
+    this.timers.delete(key);
+  }
+  pendingKeys(prefix: string): string[] {
+    return [...this.timers.keys()].filter((k) => k.startsWith(prefix));
+  }
+}
+
 class InterruptAwareStreamingTtsPlugin implements VoicePlugin {
   private bus: PipelineBus | null = null;
   private interval: ReturnType<typeof setInterval> | null = null;
@@ -192,6 +247,147 @@ async function enrollPrimarySpeaker(
 }
 
 describe("VoiceAgentSession", () => {
+  it("owns an injected policy lifecycle and feeds decoded audio plus playout observations", async () => {
+    const policy = new CapturingInteractionPolicy();
+    const session = new VoiceAgentSession({
+      plugins: {},
+      interactionPolicy: policy,
+      interactionPolicyConfig: { model_path: "/tmp/policy.onnx" },
+    });
+    await session.start();
+
+    session.bus.push(Route.Main, {
+      kind: "user.audio_received",
+      contextId: "turn-policy",
+      timestampMs: 1000,
+      audio: new Uint8Array([0x34, 0x12, 0xcc, 0xff]),
+    } satisfies UserAudioReceivedPacket);
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "turn-policy",
+      timestampMs: 1100,
+      audio: new Uint8Array(640),
+      sampleRateHz: 16000,
+    } satisfies TextToSpeechAudioPacket);
+    session.bus.push(Route.Main, {
+      kind: "tts.playout_progress",
+      contextId: "turn-policy",
+      timestampMs: 1200,
+      playedOutMs: 20,
+      complete: true,
+    } satisfies TextToSpeechPlayoutProgressPacket);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(policy.initializeCount).toBe(1);
+    expect(policy.initializedConfig).toEqual({ model_path: "/tmp/policy.onnx" });
+    const audioFrame = policy.observations.find((observation) => observation.kind === "audio_frame");
+    expect(audioFrame).toMatchObject({ contextId: "turn-policy", timestampMs: 1000, sampleRateHz: 16000 });
+    expect(audioFrame?.kind === "audio_frame" ? [...(audioFrame.audio ?? [])] : []).toEqual([4660, -52]);
+    expect(policy.observations.filter((observation) => observation.kind === "playout_tick")).toEqual([
+      expect.objectContaining({
+        contextId: "turn-policy",
+        ttsActive: true,
+        sampleRateHz: 16000,
+        audio: expect.any(Int16Array),
+      }),
+      expect.objectContaining({ contextId: "turn-policy", playedOutMs: 20, ttsActive: false }),
+    ]);
+
+    await closeSession(session);
+    expect(policy.closeCount).toBe(1);
+  });
+
+  it("threads the packet's true sample rate to the injected policy audio_frame (not a hardcoded 16k)", async () => {
+    const policy = new CapturingInteractionPolicy();
+    const session = new VoiceAgentSession({ plugins: {}, interactionPolicy: policy });
+    await session.start();
+
+    session.bus.push(Route.Main, {
+      kind: "user.audio_received",
+      contextId: "turn-rate",
+      timestampMs: 2000,
+      audio: new Uint8Array([0x34, 0x12, 0xcc, 0xff]),
+      sampleRateHz: 24000,
+    } satisfies UserAudioReceivedPacket);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    const audioFrame = policy.observations.find((o) => o.kind === "audio_frame");
+    expect(audioFrame).toMatchObject({ contextId: "turn-rate", sampleRateHz: 24000 });
+
+    await closeSession(session);
+  });
+
+  it("cancels a pending interaction playout timer on close even after its turn completed", async () => {
+    const scheduler = new RecordingScheduler();
+    const policy = new CapturingInteractionPolicy();
+    const session = new VoiceAgentSession({ plugins: {}, interactionPolicy: policy, scheduler });
+    await session.start();
+
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "turn-x",
+      timestampMs: 1000,
+      audio: new Uint8Array(640),
+      sampleRateHz: 16000,
+    } satisfies TextToSpeechAudioPacket);
+    session.bus.push(Route.Main, {
+      kind: "tts.end",
+      contextId: "turn-x",
+      timestampMs: 1100,
+    } satisfies TextToSpeechEndPacket);
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    expect(scheduler.pendingKeys("interaction.playout:")).toEqual(["interaction.playout:turn-x"]);
+
+    // Turn completes (telephony reuses the contextId) — this deletes turn-x from
+    // firstTtsAudioFired, orphaning the timer from the old stop() cancel loop.
+    session.bus.push(Route.Main, {
+      kind: "eos.turn_complete",
+      contextId: "turn-x",
+      timestampMs: 1200,
+      text: "done",
+      transcripts: [],
+    } satisfies EndOfSpeechPacket);
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    expect(scheduler.pendingKeys("interaction.playout:")).toEqual(["interaction.playout:turn-x"]);
+
+    await session.close();
+    expect(scheduler.pendingKeys("interaction.playout:")).toEqual([]);
+  });
+
+  it("does not initialize an injected policy when full-duplex defer mode owns interaction", async () => {
+    const policy = new CapturingInteractionPolicy();
+    const session = new VoiceAgentSession({ plugins: {}, interactionPolicy: policy, fullDuplex: true });
+    await session.start();
+    await closeSession(session);
+
+    expect(policy.initializeCount).toBe(0);
+    expect(policy.closeCount).toBe(0);
+    expect(policy.observations).toEqual([]);
+  });
+
+  it("makes an injected policy the sole endpoint owner while keeping provider STT initialized", async () => {
+    const policy = new CapturingInteractionPolicy();
+    const provider = new EndpointingPlugin({
+      owner: "provider_stt",
+      disableConfig: { emit_eos_on_final: false },
+    });
+    const legacySmartTurn = new EndpointingPlugin({ owner: "smart_turn" });
+    const session = new VoiceAgentSession({
+      plugins: { stt: { emit_eos_on_final: true }, eos: {} },
+      endpointingOwner: "smart_turn",
+      interactionPolicy: policy,
+    });
+    session.registerPlugin("stt", provider);
+    session.registerPlugin("eos", legacySmartTurn);
+    await session.start();
+
+    expect(provider.initializeCount).toBe(1);
+    expect(provider.config).toEqual({ emit_eos_on_final: false });
+    expect(legacySmartTurn.initializeCount).toBe(0);
+    expect(policy.initializeCount).toBe(1);
+    await closeSession(session);
+  });
+
   it("passes configured plugin options to each plugin during initialization", async () => {
     const plugin = new CapturingPlugin();
     const session = new VoiceAgentSession({
@@ -370,6 +566,151 @@ describe("VoiceAgentSession", () => {
     await closeSession(session);
   });
 
+  it("emits a decomposed turn_latency session event on first TTS audio", async () => {
+    const session = new VoiceAgentSession({ plugins: {} });
+    const events: Array<Record<string, unknown>> = [];
+    session.on("turn_latency", (event) => {
+      events.push(event);
+    });
+    await session.start();
+
+    const t0 = 100_000;
+    session.bus.push(Route.Main, { kind: "vad.speech_ended", contextId: "turn-1", timestampMs: t0 });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    session.bus.push(Route.Main, {
+      kind: "eos.turn_complete",
+      contextId: "turn-1",
+      timestampMs: t0 + 200,
+      text: "what are the lab fees",
+      transcripts: [],
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    session.bus.push(Route.Main, { kind: "llm.delta", contextId: "turn-1", timestampMs: t0 + 900, text: "The fee is ten dollars." });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "turn-1",
+      timestampMs: t0 + 1150,
+      audio: new Uint8Array(320),
+      sampleRateHz: 16000,
+    });
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(events).toHaveLength(1);
+    expect(events[0]).toEqual({
+      tsMs: expect.any(Number),
+      turnId: "turn-1",
+      ttfaMs: 1150,
+      eouDelayMs: 200,
+      llmTtftMs: 700,
+      ttsTtfbMs: 250,
+      fillerUsed: false,
+      backchannelUsed: false,
+    });
+
+    await closeSession(session);
+  });
+
+  it("turn_latency anchors TTFA to eos when no VAD speech-end was seen, and marks filler turns", async () => {
+    const session = new VoiceAgentSession({ plugins: {}, latencyFillerEnabled: true });
+    const events: Array<Record<string, unknown>> = [];
+    session.on("turn_latency", (event) => {
+      events.push(event);
+    });
+    await session.start();
+
+    const t0 = 200_000;
+    session.bus.push(Route.Main, {
+      kind: "eos.turn_complete",
+      contextId: "turn-2",
+      timestampMs: t0,
+      text: "tell me about registration holds",
+      transcripts: [],
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "turn-2",
+      timestampMs: t0 + 400,
+      audio: new Uint8Array(320),
+      sampleRateHz: 16000,
+    });
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(events).toHaveLength(1);
+    expect(events[0]).toMatchObject({
+      turnId: "turn-2",
+      ttfaMs: 400,
+      fillerUsed: true,
+      backchannelUsed: false,
+    });
+    expect(events[0]!["eouDelayMs"]).toBeUndefined();
+
+    await closeSession(session);
+  });
+
+  it("turn_latency is not emitted for an interrupted turn", async () => {
+    const session = new VoiceAgentSession({ plugins: {} });
+    const events: unknown[] = [];
+    session.on("turn_latency", (event) => {
+      events.push(event);
+    });
+    await session.start();
+
+    const t0 = 300_000;
+    session.bus.push(Route.Main, {
+      kind: "eos.turn_complete",
+      contextId: "turn-3",
+      timestampMs: t0,
+      text: "hello there",
+      transcripts: [],
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    session.bus.push(Route.Critical, {
+      kind: "interrupt.detected",
+      contextId: "turn-3",
+      timestampMs: t0 + 100,
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "turn-3",
+      timestampMs: t0 + 300,
+      audio: new Uint8Array(320),
+      sampleRateHz: 16000,
+    });
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(events).toHaveLength(0);
+
+    await closeSession(session);
+  });
+
+  it("IP-C3: tool_call_cue.delayed emits interaction.backchannel end-to-end", async () => {
+    const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 30 });
+    const backchannels: InteractionBackchannelPacket[] = [];
+    session.bus.on("interaction.backchannel", (pkt) => {
+      backchannels.push(pkt as InteractionBackchannelPacket);
+    });
+    await session.start();
+
+    session.bus.push(Route.Main, {
+      kind: "llm.tool_call",
+      contextId: "turn-1",
+      timestampMs: Date.now(),
+      toolId: "t1",
+      toolName: "consult_knowledge",
+      toolArgs: { query: "fees" },
+    });
+
+    await new Promise((resolve) => setTimeout(resolve, 70));
+
+    expect(backchannels).toHaveLength(1);
+    expect(backchannels[0]).toMatchObject({ contextId: "turn-1", cue: "mm_hmm" });
+
+    await closeSession(session);
+  });
+
   it("G3/WBS-3: tool-call lifecycle cues fire started → delayed → complete", async () => {
     const session = new VoiceAgentSession({ plugins: {}, delayCueAfterMs: 30 });
     const cues: Array<{ phase: string; turnId: string; toolId: string; toolName: string; afterMs?: number }> = [];
@@ -1075,6 +1416,126 @@ describe("VoiceAgentSession", () => {
     await closeSession(session);
   });
 
+  it("fullDuplex:true runs the interaction policy observe-only (no VAD-driven interrupt)", async () => {
+    const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280, fullDuplex: true });
+    const interrupts: InterruptTtsPacket[] = [];
+
+    await session.start();
+    session.bus.on("interrupt.tts", (pkt) => {
+      interrupts.push(pkt as InterruptTtsPacket);
+    });
+
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "assistant-turn",
+      timestampMs: Date.now(),
+      audio: new Uint8Array([1, 2, 3, 4]),
+      sampleRateHz: 16000,
+    } satisfies TextToSpeechAudioPacket);
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    const t0 = Date.now();
+    session.bus.push(Route.Main, {
+      kind: "vad.speech_started",
+      contextId: "user",
+      timestampMs: t0,
+      confidence: 0.99,
+    } satisfies VadSpeechStartedPacket);
+    session.bus.push(Route.Main, {
+      kind: "vad.speech_activity",
+      contextId: "user",
+      timestampMs: t0 + 100,
+      isAsync: true,
+    } satisfies VadSpeechActivityPacket);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+    expect(interrupts).toEqual([]);
+
+    session.bus.push(Route.Main, {
+      kind: "vad.speech_activity",
+      contextId: "user",
+      timestampMs: t0 + 300,
+      isAsync: true,
+    } satisfies VadSpeechActivityPacket);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(interrupts).toEqual([]);
+
+    await closeSession(session);
+  });
+
+  it("fullDuplex:true still honors a direct client interrupt (executor survives defer mode) (IP-C2 regression)", async () => {
+    // Defer mode swaps only the coordinator's DRIVE policy to observe-only; the executor stays the
+    // rule policy's arbiter, so a client-initiated "stop" (requestClientInterrupt) must still fire —
+    // the front owning turn-taking does not disable the user's explicit interrupt.
+    const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280, fullDuplex: true });
+    const interrupts: InterruptTtsPacket[] = [];
+
+    await session.start();
+    session.bus.on("interrupt.tts", (pkt) => {
+      interrupts.push(pkt as InterruptTtsPacket);
+    });
+
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "assistant-turn",
+      timestampMs: Date.now(),
+      audio: new Uint8Array([1, 2, 3, 4]),
+      sampleRateHz: 16000,
+    } satisfies TextToSpeechAudioPacket);
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    session.requestClientInterrupt("assistant-turn");
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(interrupts).toEqual([
+      expect.objectContaining({ kind: "interrupt.tts", contextId: "assistant-turn" }),
+    ]);
+
+    await closeSession(session);
+  });
+
+  it("stt.partial carries wordTimings but does NOT itself drive barge-in (IP-C4 no-double-drive guard)", async () => {
+    // IP-C4: stt.partial is the rich-seam carrier; the session caches its wordTimings for the
+    // observation but barge-in stays driven ONLY by stt.interim/stt.result. Pushing sustained
+    // stt.partial (no interim) during active TTS must NOT interrupt — proving no second barge-in driver.
+    const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
+    const interrupts: InterruptTtsPacket[] = [];
+
+    await session.start();
+    session.bus.on("interrupt.tts", (pkt) => {
+      interrupts.push(pkt as InterruptTtsPacket);
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "assistant-turn",
+      timestampMs: Date.now(),
+      audio: new Uint8Array([1, 2, 3, 4]),
+      sampleRateHz: 16000,
+    } satisfies TextToSpeechAudioPacket);
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    const t0 = Date.now();
+    session.bus.push(Route.Main, {
+      kind: "stt.partial",
+      contextId: "user",
+      timestampMs: t0,
+      text: "wait actually I need",
+      wordTimings: [{ word: "wait", startMs: 0, endMs: 200, confidence: 0.9 }],
+    } satisfies SttPartialPacket);
+    session.bus.push(Route.Main, {
+      kind: "stt.partial",
+      contextId: "user",
+      timestampMs: t0 + 400,
+      text: "wait actually I need something else",
+      wordTimings: [{ word: "wait", startMs: 0, endMs: 200, confidence: 0.9 }],
+    } satisfies SttPartialPacket);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(interrupts).toEqual([]);
+
+    await closeSession(session);
+  });
+
   it("commits a barge-in from provider STT interim transcripts when no VAD plugin is registered", async () => {
     // Cascade deployments with endpointingOwner "provider_stt" (the default) have
     // no vad.speech_started producer — interim transcripts during TTS playout are
@@ -1127,6 +1588,100 @@ describe("VoiceAgentSession", () => {
     await closeSession(session);
   });
 
+  it("attaches cached stt.partial wordTimings to the next barge-in observation (IP-C4)", async () => {
+    const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
+    const observations: InteractionObservation[] = [];
+    await session.start();
+
+    const interaction = (session as unknown as { interaction: InteractionCoordinator }).interaction;
+    const originalObserve = interaction.observe.bind(interaction);
+    interaction.observe = (obs) => {
+      observations.push(obs);
+      originalObserve(obs);
+    };
+
+    session.bus.push(Route.Main, {
+      kind: "stt.partial",
+      contextId: "user",
+      timestampMs: Date.now(),
+      text: "wait actually",
+      wordTimings: [{ word: "wait", startMs: 100, endMs: 300, confidence: 0.9 }],
+    } satisfies SttPartialPacket);
+
+    session.bus.push(Route.Main, {
+      kind: "stt.interim",
+      contextId: "user",
+      timestampMs: Date.now(),
+      text: "wait actually",
+    } satisfies SttInterimPacket);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(observations).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({
+          kind: "stt_partial",
+          text: "wait actually",
+          wordTimings: [{ word: "wait", startMs: 100, endMs: 300, confidence: 0.9 }],
+        }),
+      ]),
+    );
+
+    await closeSession(session);
+  });
+
+  it("suppresses a backchannel provider-STT interim through the interaction seam (IP-C1 regression)", async () => {
+    // IP-C1 routed provider-STT barge-in through InteractionCoordinator ->
+    // RuleBasedInteractionPolicy -> TurnArbiter. This pins that the reshaped
+    // chain still SUPPRESSES a sustained backchannel ("okay") rather than cutting
+    // the assistant — the session-level suppression case the policy-unit test
+    // (rule-based.test.ts, vad-driven) and the commit test above do not cover.
+    const session = new VoiceAgentSession({ plugins: {}, minInterruptionMs: 280 });
+    const interrupts: InterruptTtsPacket[] = [];
+    const metrics: string[] = [];
+
+    await session.start();
+    session.bus.on("interrupt.tts", (pkt) => {
+      interrupts.push(pkt as InterruptTtsPacket);
+    });
+    session.bus.on("metric.conversation", (pkt) => {
+      metrics.push((pkt as unknown as { name: string }).name);
+    });
+
+    // Assistant is speaking.
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "assistant-turn",
+      timestampMs: Date.now(),
+      audio: new Uint8Array([1, 2, 3, 4]),
+      sampleRateHz: 16000,
+    } satisfies TextToSpeechAudioPacket);
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    // Sustained-past-minInterruptionMs backchannel evidence: opens the pending
+    // window, then the second interim (past 280ms) reaches tryCommit -> the
+    // arbiter's backchannel suppression fires instead of an interrupt decision.
+    const t0 = Date.now();
+    session.bus.push(Route.Main, {
+      kind: "stt.interim",
+      contextId: "user",
+      timestampMs: t0,
+      text: "okay",
+    } satisfies SttInterimPacket);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+    session.bus.push(Route.Main, {
+      kind: "stt.interim",
+      contextId: "user",
+      timestampMs: t0 + 300,
+      text: "okay",
+    } satisfies SttInterimPacket);
+    await new Promise((resolve) => setTimeout(resolve, 20));
+
+    expect(interrupts).toEqual([]);
+    expect(metrics).toContain("interrupt.suppressed_backchannel");
+
+    await closeSession(session);
+  });
+
   it("emits interrupt.onset_to_logic_cancel_ms and stamps interrupt.tts/llm with detected onset", async () => {
     const session = new VoiceAgentSession({ plugins: {} });
     const onsetMetrics: Array<{ name: string; value: string }> = [];