@kuralle-syrinx/server-websocket 2.1.0

package/src/turn-metrics.test.ts

@@ -0,0 +1,327 @@
+// SPDX-License-Identifier: MIT
+
+import { describe, expect, it, vi } from "vitest";
+import { Route, VoiceAgentSession } from "@kuralle-syrinx/core";
+import {
+  buildBrowserMetricsMessage,
+  TurnMetricsTracker,
+} from "./turn-metrics.js";
+import { waitForCondition } from "./test-helpers.js";
+
+describe("turn metrics", () => {
+  it("computes stage latencies from synthetic timestamps", () => {
+    const message = buildBrowserMetricsMessage("turn-a", {
+      speechEndMs: 1000,
+      sttFinalMs: 1200,
+      eosMs: 0,
+      vadStopHangoverMs: 0,
+      textReadyMs: 1500,
+      firstAudioByteMs: 1700,
+      firstAudioPlayedMs: 1900,
+      lastAudioPlayedMs: 2500,
+    });
+
+    expect(message).toEqual({
+      type: "metrics",
+      turnId: "turn-a",
+      correlationId: "turn-a",
+      speechEndMs: 1000,
+      textReadyMs: 1500,
+      firstAudioByteMs: 1700,
+      firstAudioPlayedMs: 1900,
+      lastAudioPlayedMs: 2500,
+      sttMs: 200,
+      llmTTFTMs: 300,
+      ttsTTFBMs: 200,
+      e2eMs: 900,
+      eouBudgetMs: {
+        sttFinalDelayMs: 200,
+        totalMs: 200,
+      },
+    });
+  });
+
+  it("buildBrowserMetricsMessage eou budget sums hangover, stt-final, and endpoint delays", () => {
+    const message = buildBrowserMetricsMessage("turn-eou-unit", {
+      speechEndMs: 1000,
+      sttFinalMs: 1250,
+      eosMs: 1300,
+      vadStopHangoverMs: 80,
+      textReadyMs: 0,
+      firstAudioByteMs: 0,
+      firstAudioPlayedMs: 0,
+      lastAudioPlayedMs: 0,
+    });
+
+    expect(message.eouBudgetMs).toEqual({
+      vadStopHangoverMs: 80,
+      sttFinalDelayMs: 250,
+      endpointDelayMs: 50,
+      totalMs: 380,
+    });
+    expect(message.sttMs).toBe(250);
+  });
+
+  it("keeps correlation id stable for the turn context", async () => {
+    const session = new VoiceAgentSession({ plugins: {} });
+    const emitted: unknown[] = [];
+    const tracker = new TurnMetricsTracker(session.bus, (message) => emitted.push(message));
+    const disposers: Array<() => void> = [];
+    tracker.wire(disposers);
+    void session.start();
+
+    session.bus.push(Route.Main, {
+      kind: "vad.speech_ended",
+      contextId: "turn-correlation",
+      timestampMs: 500,
+    });
+    session.bus.push(Route.Main, {
+      kind: "stt.result",
+      contextId: "turn-correlation",
+      timestampMs: 700,
+      text: "hello",
+      confidence: 0.99,
+    });
+    session.bus.push(Route.Main, {
+      kind: "llm.delta",
+      contextId: "turn-correlation",
+      timestampMs: 900,
+      text: "hi",
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "turn-correlation",
+      timestampMs: 1100,
+      audio: new Uint8Array(640),
+      sampleRateHz: 16000,
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.playout_started",
+      contextId: "turn-correlation",
+      timestampMs: 1100,
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.playout_progress",
+      contextId: "turn-correlation",
+      timestampMs: 1300,
+      playedOutMs: 200,
+      complete: false,
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.playout_progress",
+      contextId: "turn-correlation",
+      timestampMs: 1800,
+      playedOutMs: 120,
+      complete: true,
+    });
+
+    await waitForCondition(() => emitted.length === 1);
+    expect(emitted[0]).toMatchObject({
+      type: "metrics",
+      turnId: "turn-correlation",
+      correlationId: "turn-correlation",
+      sttMs: 200,
+      llmTTFTMs: 200,
+      ttsTTFBMs: 200,
+      e2eMs: 600,
+    });
+
+    for (const dispose of disposers) dispose();
+    await session.close();
+  });
+
+  it("records firstAudioPlayedMs from playout_started, not throttled progress", async () => {
+    const session = new VoiceAgentSession({ plugins: {} });
+    const emitted: unknown[] = [];
+    const tracker = new TurnMetricsTracker(session.bus, (message) => emitted.push(message));
+    const disposers: Array<() => void> = [];
+    tracker.wire(disposers);
+    void session.start();
+
+    session.bus.push(Route.Main, {
+      kind: "vad.speech_ended",
+      contextId: "turn-throttle",
+      timestampMs: 1000,
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.playout_started",
+      contextId: "turn-throttle",
+      timestampMs: 1100,
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.playout_progress",
+      contextId: "turn-throttle",
+      timestampMs: 1300,
+      playedOutMs: 200,
+      complete: true,
+    });
+
+    await waitForCondition(() => emitted.length === 1);
+    expect(emitted[0]).toMatchObject({
+      firstAudioPlayedMs: 1100,
+      e2eMs: 100,
+    });
+
+    for (const dispose of disposers) dispose();
+    await session.close();
+  });
+
+  it("emits metrics once when playout completes for a wired browser session", async () => {
+    const session = new VoiceAgentSession({ plugins: {} });
+    const emitted: unknown[] = [];
+    const tracker = new TurnMetricsTracker(session.bus, (message) => emitted.push(message));
+    const disposers: Array<() => void> = [];
+    tracker.wire(disposers);
+    void session.start();
+
+    session.bus.push(Route.Main, {
+      kind: "vad.speech_ended",
+      contextId: "turn-live",
+      timestampMs: 10_000,
+    });
+    session.bus.push(Route.Main, {
+      kind: "stt.result",
+      contextId: "turn-live",
+      timestampMs: 10_200,
+      text: "hello",
+      confidence: 0.99,
+    });
+    session.bus.push(Route.Main, {
+      kind: "llm.delta",
+      contextId: "turn-live",
+      timestampMs: 10_450,
+      text: "hi",
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.audio",
+      contextId: "turn-live",
+      timestampMs: 10_600,
+      audio: new Uint8Array(640),
+      sampleRateHz: 16000,
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.playout_started",
+      contextId: "turn-live",
+      timestampMs: 10_600,
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.playout_progress",
+      contextId: "turn-live",
+      timestampMs: 10_800,
+      playedOutMs: 200,
+      complete: false,
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.playout_progress",
+      contextId: "turn-live",
+      timestampMs: 11_000,
+      playedOutMs: 120,
+      complete: true,
+    });
+
+    await waitForCondition(() => emitted.length === 1);
+    expect(emitted[0]).toMatchObject({
+      type: "metrics",
+      turnId: "turn-live",
+      correlationId: "turn-live",
+      sttMs: 200,
+      llmTTFTMs: 250,
+      ttsTTFBMs: 150,
+      e2eMs: 600,
+    });
+
+    for (const dispose of disposers) dispose();
+    await session.close();
+  });
+
+  it("eou_budget_breakdown: vad hangover, stt-final delay, endpoint delay, and total", async () => {
+    const session = new VoiceAgentSession({ plugins: {} });
+    const emitted: unknown[] = [];
+    const tracker = new TurnMetricsTracker(session.bus, (message) => emitted.push(message));
+    const disposers: Array<() => void> = [];
+    tracker.wire(disposers);
+    void session.start();
+
+    session.bus.push(Route.Main, {
+      kind: "vad.speech_ended",
+      contextId: "turn-eou",
+      timestampMs: 1000,
+    });
+    session.bus.push(Route.Main, {
+      kind: "metric.conversation",
+      contextId: "turn-eou",
+      timestampMs: 1005,
+      name: "vad.stop_hangover_ms",
+      value: "80",
+    });
+    session.bus.push(Route.Main, {
+      kind: "stt.result",
+      contextId: "turn-eou",
+      timestampMs: 1250,
+      text: "hello",
+      confidence: 0.99,
+    });
+    session.bus.push(Route.Main, {
+      kind: "eos.turn_complete",
+      contextId: "turn-eou",
+      timestampMs: 1300,
+      text: "hello",
+      transcripts: [],
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.playout_progress",
+      contextId: "turn-eou",
+      timestampMs: 1500,
+      playedOutMs: 100,
+      complete: true,
+    });
+
+    await waitForCondition(() => emitted.length === 1);
+    expect(emitted[0]).toMatchObject({
+      type: "metrics",
+      turnId: "turn-eou",
+      correlationId: "turn-eou",
+      sttMs: 250,
+      eouBudgetMs: {
+        vadStopHangoverMs: 80,
+        sttFinalDelayMs: 250,
+        endpointDelayMs: 50,
+        totalMs: 380,
+      },
+    });
+
+    for (const dispose of disposers) dispose();
+    await session.close();
+  });
+
+  it("drops partial turn state on interrupt without emitting metrics", async () => {
+    const session = new VoiceAgentSession({ plugins: {} });
+    const emit = vi.fn();
+    const tracker = new TurnMetricsTracker(session.bus, emit);
+    tracker.wire([]);
+    void session.start();
+
+    session.bus.push(Route.Main, {
+      kind: "vad.speech_ended",
+      contextId: "turn-interrupted",
+      timestampMs: 1000,
+    });
+    session.bus.push(Route.Critical, {
+      kind: "interrupt.tts",
+      contextId: "turn-interrupted",
+      timestampMs: 1100,
+    });
+    session.bus.push(Route.Main, {
+      kind: "tts.playout_progress",
+      contextId: "turn-interrupted",
+      timestampMs: 1200,
+      playedOutMs: 20,
+      complete: true,
+    });
+
+    await waitForCondition(() => emit.mock.calls.length === 0, 200);
+    expect(emit).not.toHaveBeenCalled();
+
+    await session.close();
+  });
+});

package/src/turn-metrics.ts

@@ -0,0 +1,193 @@
+// SPDX-License-Identifier: MIT
+
+import {
+  type ConversationMetricPacket,
+  type EndOfSpeechPacket,
+  type InterruptTtsPacket,
+  type LlmDeltaPacket,
+  type PipelineBus,
+  type SttResultPacket,
+  type TextToSpeechAudioPacket,
+  type TextToSpeechPlayoutProgressPacket,
+  type TextToSpeechPlayoutStartedPacket,
+  type VadSpeechEndedPacket,
+} from "@kuralle-syrinx/core";
+
+export interface TurnTimestampState {
+  speechEndMs: number;
+  sttFinalMs: number;
+  eosMs: number;
+  vadStopHangoverMs: number;
+  textReadyMs: number;
+  firstAudioByteMs: number;
+  firstAudioPlayedMs: number;
+  lastAudioPlayedMs: number;
+}
+
+export interface BrowserMetricsMessage {
+  readonly type: "metrics";
+  readonly turnId: string;
+  readonly correlationId: string;
+  readonly speechEndMs?: number;
+  readonly textReadyMs?: number;
+  readonly firstAudioByteMs?: number;
+  readonly firstAudioPlayedMs?: number;
+  readonly lastAudioPlayedMs?: number;
+  readonly sttMs?: number;
+  readonly llmTTFTMs?: number;
+  readonly ttsTTFBMs?: number;
+  readonly e2eMs?: number;
+  readonly eouBudgetMs?: {
+    readonly vadStopHangoverMs?: number;
+    readonly sttFinalDelayMs?: number;
+    readonly endpointDelayMs?: number;
+    readonly totalMs?: number;
+  };
+}
+
+function positiveDelta(endMs: number, startMs: number): number | undefined {
+  if (endMs <= 0 || startMs <= 0 || endMs < startMs) return undefined;
+  return endMs - startMs;
+}
+
+export function buildBrowserMetricsMessage(
+  turnId: string,
+  timestamps: TurnTimestampState,
+): BrowserMetricsMessage {
+  const sttMs = positiveDelta(timestamps.sttFinalMs, timestamps.speechEndMs);
+  const llmTTFTMs = positiveDelta(timestamps.textReadyMs, timestamps.sttFinalMs);
+  const ttsTTFBMs = positiveDelta(timestamps.firstAudioByteMs, timestamps.textReadyMs);
+  const e2eFromPlayout = positiveDelta(timestamps.firstAudioPlayedMs, timestamps.speechEndMs);
+  const e2eFromByte = positiveDelta(timestamps.firstAudioByteMs, timestamps.speechEndMs);
+
+  const sttFinalDelayMs = positiveDelta(timestamps.sttFinalMs, timestamps.speechEndMs);
+  const endpointDelayMs = positiveDelta(timestamps.eosMs, timestamps.sttFinalMs);
+  const vadStopHangoverMs =
+    timestamps.vadStopHangoverMs > 0 ? timestamps.vadStopHangoverMs : undefined;
+  const eouTotalMs =
+    (vadStopHangoverMs ?? 0) +
+      (positiveDelta(timestamps.eosMs, timestamps.speechEndMs) ?? sttFinalDelayMs ?? 0) ||
+    undefined;
+  const eouBudgetMs =
+    vadStopHangoverMs !== undefined ||
+    sttFinalDelayMs !== undefined ||
+    endpointDelayMs !== undefined ||
+    eouTotalMs !== undefined
+      ? {
+          ...(vadStopHangoverMs !== undefined ? { vadStopHangoverMs } : {}),
+          ...(sttFinalDelayMs !== undefined ? { sttFinalDelayMs } : {}),
+          ...(endpointDelayMs !== undefined ? { endpointDelayMs } : {}),
+          ...(eouTotalMs !== undefined ? { totalMs: eouTotalMs } : {}),
+        }
+      : undefined;
+
+  return {
+    type: "metrics",
+    turnId,
+    correlationId: turnId,
+    ...(timestamps.speechEndMs > 0 ? { speechEndMs: timestamps.speechEndMs } : {}),
+    ...(timestamps.textReadyMs > 0 ? { textReadyMs: timestamps.textReadyMs } : {}),
+    ...(timestamps.firstAudioByteMs > 0 ? { firstAudioByteMs: timestamps.firstAudioByteMs } : {}),
+    ...(timestamps.firstAudioPlayedMs > 0 ? { firstAudioPlayedMs: timestamps.firstAudioPlayedMs } : {}),
+    ...(timestamps.lastAudioPlayedMs > 0 ? { lastAudioPlayedMs: timestamps.lastAudioPlayedMs } : {}),
+    ...(sttMs !== undefined ? { sttMs } : {}),
+    ...(llmTTFTMs !== undefined ? { llmTTFTMs } : {}),
+    ...(ttsTTFBMs !== undefined ? { ttsTTFBMs } : {}),
+    ...(e2eFromPlayout !== undefined ? { e2eMs: e2eFromPlayout } : e2eFromByte !== undefined ? { e2eMs: e2eFromByte } : {}),
+    ...(eouBudgetMs !== undefined ? { eouBudgetMs } : {}),
+  };
+}
+
+function emptyTurnState(): TurnTimestampState {
+  return {
+    speechEndMs: 0,
+    sttFinalMs: 0,
+    eosMs: 0,
+    vadStopHangoverMs: 0,
+    textReadyMs: 0,
+    firstAudioByteMs: 0,
+    firstAudioPlayedMs: 0,
+    lastAudioPlayedMs: 0,
+  };
+}
+
+export class TurnMetricsTracker {
+  private readonly turns: Map<string, TurnTimestampState>;
+
+  constructor(
+    private readonly bus: PipelineBus,
+    private readonly onEmit: (message: BrowserMetricsMessage) => void,
+    persistedTurns?: Map<string, TurnTimestampState>,
+  ) {
+    this.turns = persistedTurns ?? new Map();
+  }
+
+  wire(disposers: Array<() => void>): void {
+    disposers.push(
+      this.bus.on("vad.speech_ended", (pkt) => {
+        const ended = pkt as VadSpeechEndedPacket;
+        const state = this.turnState(ended.contextId);
+        if (state.speechEndMs === 0) state.speechEndMs = ended.timestampMs;
+      }),
+      this.bus.on("stt.result", (pkt) => {
+        const result = pkt as SttResultPacket;
+        const state = this.turnState(result.contextId);
+        if (state.sttFinalMs === 0) state.sttFinalMs = result.timestampMs;
+      }),
+      this.bus.on("metric.conversation", (pkt) => {
+        const metric = pkt as ConversationMetricPacket;
+        if (metric.name !== "vad.stop_hangover_ms") return;
+        const hangoverMs = Number(metric.value);
+        if (Number.isNaN(hangoverMs)) return;
+        const state = this.turnState(metric.contextId);
+        if (state.vadStopHangoverMs === 0) state.vadStopHangoverMs = hangoverMs;
+      }),
+      this.bus.on("eos.turn_complete", (pkt) => {
+        const eos = pkt as EndOfSpeechPacket;
+        const state = this.turnState(eos.contextId);
+        if (state.eosMs === 0) state.eosMs = eos.timestampMs;
+      }),
+      this.bus.on("llm.delta", (pkt) => {
+        const delta = pkt as LlmDeltaPacket;
+        if (delta.text.length === 0) return;
+        const state = this.turnState(delta.contextId);
+        if (state.textReadyMs === 0) state.textReadyMs = delta.timestampMs;
+      }),
+      this.bus.on("tts.audio", (pkt) => {
+        const audio = pkt as TextToSpeechAudioPacket;
+        const state = this.turnState(audio.contextId);
+        if (state.firstAudioByteMs === 0) state.firstAudioByteMs = audio.timestampMs;
+      }),
+      this.bus.on("tts.playout_started", (pkt) => {
+        const started = pkt as TextToSpeechPlayoutStartedPacket;
+        const state = this.turns.get(started.contextId);
+        if (!state) return;
+        if (state.firstAudioPlayedMs === 0) {
+          state.firstAudioPlayedMs = started.timestampMs;
+        }
+      }),
+      this.bus.on("tts.playout_progress", (pkt) => {
+        const progress = pkt as TextToSpeechPlayoutProgressPacket;
+        const state = this.turns.get(progress.contextId);
+        if (!state) return;
+        if (progress.complete) {
+          state.lastAudioPlayedMs = progress.timestampMs;
+          this.onEmit(buildBrowserMetricsMessage(progress.contextId, state));
+          this.turns.delete(progress.contextId);
+        }
+      }),
+      this.bus.on("interrupt.tts", (pkt) => {
+        this.turns.delete((pkt as InterruptTtsPacket).contextId);
+      }),
+    );
+  }
+
+  private turnState(contextId: string): TurnTimestampState {
+    let state = this.turns.get(contextId);
+    if (!state) {
+      state = emptyTurnState();
+      this.turns.set(contextId, state);
+    }
+    return state;
+  }
+}