@kuralle-syrinx/browser-client 4.4.0 → 4.5.0

package/src/transport.test.ts

@@ -0,0 +1,233 @@
+// SPDX-License-Identifier: MIT
+
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { Decoder as OpusDecoder, Encoder as OpusEncoder } from "@evan/opus";
+import { decodeSyrinxAudioEnvelope } from "@kuralle-syrinx/core";
+import { pcm16BytesToSamples, pcm16SamplesToBytes } from "@kuralle-syrinx/core/audio";
+import { SyrinxBrowserClient } from "./index.js";
+import type { ClientTransport, ClientTransportHandlers } from "./transport.js";
+import * as browserOpus from "./browser-opus.js";
+import { pickBrowserWireCodec, createBrowserOpusCodec } from "./browser-opus.js";
+
+async function pollUntil(predicate: () => boolean, timeoutMs: number, label: string): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (predicate()) return;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+  throw new Error(`Timed out waiting for ${label}`);
+}
+
+class FakeTransport implements ClientTransport {
+  readonly sent: unknown[] = [];
+  private handlers: ClientTransportHandlers = {};
+  private open = false;
+
+  get connected(): boolean {
+    return this.open;
+  }
+
+  setHandlers(handlers: ClientTransportHandlers): void {
+    this.handlers = handlers;
+  }
+
+  connect(_url: string): void {
+    this.open = true;
+    this.handlers.onOpen?.();
+  }
+
+  disconnect(): void {
+    this.open = false;
+    this.handlers.onClose?.(1000, "");
+  }
+
+  sendAudio(data: Uint8Array | ArrayBuffer): void {
+    this.sent.push(data);
+  }
+
+  sendJson(value: unknown): void {
+    this.sent.push(JSON.stringify(value));
+  }
+
+  emitMessage(data: string): void {
+    this.handlers.onMessage?.(data);
+  }
+
+  emitAudio(data: ArrayBuffer): void {
+    this.handlers.onAudio?.(data);
+  }
+}
+
+describe("ClientTransport seam", () => {
+  it("drives SyrinxBrowserClient through an injected fake transport", () => {
+    const transport = new FakeTransport();
+    const client = new SyrinxBrowserClient({
+      url: "ws://unused/ws",
+      transport,
+      reconnect: false,
+      keepaliveIntervalMs: false,
+    });
+    const events: string[] = [];
+    client.on((event) => events.push(event.type));
+    client.connect();
+
+    expect(events).toContain("open");
+    transport.emitMessage(JSON.stringify({
+      type: "ready",
+      sessionId: "sess-fake",
+      audio: {
+        inputSampleRateHz: 16000,
+        outputSampleRateHz: 16000,
+        encoding: "pcm_s16le",
+        channels: 1,
+      },
+    }));
+    client.sendAudioPcm(new Uint8Array([1, 0, 2, 0]), 16000);
+    expect(transport.sent.length).toBeGreaterThan(0);
+  });
+});
+
+describe("browser opus negotiation", () => {
+  it("prefers opus when the server advertises it", () => {
+    expect(pickBrowserWireCodec(["pcm_s16le", "opus"], true)).toBe("opus");
+    expect(pickBrowserWireCodec(["pcm_s16le", "opus"], false)).toBe("pcm_s16le");
+    expect(pickBrowserWireCodec(["pcm_s16le"], true)).toBe("pcm_s16le");
+  });
+
+  it("produces opus frames and decodes them back to PCM16 samples", async () => {
+    const codec = await createBrowserOpusCodec(48000);
+    const pcm = new Int16Array(960);
+    pcm[0] = 1000;
+    const wire = codec.encodePcm16Frame(pcm, true)[0]!;
+    expect(wire.byteLength).toBeGreaterThan(0);
+    expect(codec.decodeOpusFrame(wire).length).toBeGreaterThan(0);
+  });
+
+  it("flushes pre-ready uplink audio in negotiated opus, not pcm", async () => {
+    const transport = new FakeTransport();
+    const client = new SyrinxBrowserClient({
+      url: "ws://unused/ws",
+      transport,
+      reconnect: false,
+      keepaliveIntervalMs: false,
+    });
+    client.connect();
+
+    const pcm = new Uint8Array(640);
+    pcm[0] = 1;
+    pcm[1] = 0;
+    client.sendAudioPcm(pcm, 16000, { contextId: "turn-early" });
+
+    expect(transport.sent.filter((entry) => entry instanceof Uint8Array)).toHaveLength(0);
+
+    transport.emitMessage(JSON.stringify({
+      type: "ready",
+      sessionId: "sess-opus-early",
+      audio: {
+        inputSampleRateHz: 16000,
+        outputSampleRateHz: 16000,
+        encoding: "opus",
+        supportedInputCodecs: ["pcm_s16le", "opus"],
+        channels: 1,
+        binaryEnvelope: "syrinx.audio.v1",
+      },
+    }));
+
+    await pollUntil(
+      () => transport.sent.some((entry) => {
+        if (typeof entry !== "string") return false;
+        return (JSON.parse(entry) as { type?: string }).type === "codec_capability";
+      }),
+      3_000,
+      "codec_capability",
+    );
+
+    const uplink = transport.sent.filter((entry): entry is Uint8Array => entry instanceof Uint8Array);
+    expect(uplink.length).toBeGreaterThan(0);
+    for (const frame of uplink) {
+      expect(decodeSyrinxAudioEnvelope(frame).header.encoding).toBe("opus");
+    }
+    expect(decodeSyrinxAudioEnvelope(uplink[0]!).header.contextId).toBe("turn-early");
+  });
+
+  it("encodes uplink envelopes as opus after ready negotiation", async () => {
+    const transport = new FakeTransport();
+    const client = new SyrinxBrowserClient({
+      url: "ws://unused/ws",
+      transport,
+      reconnect: false,
+      keepaliveIntervalMs: false,
+    });
+    client.connect();
+    transport.emitMessage(JSON.stringify({
+      type: "ready",
+      sessionId: "sess-opus",
+      audio: {
+        inputSampleRateHz: 16000,
+        outputSampleRateHz: 16000,
+        encoding: "opus",
+        supportedInputCodecs: ["pcm_s16le", "opus"],
+        channels: 1,
+        binaryEnvelope: "syrinx.audio.v1",
+      },
+    }));
+
+    await vi.waitFor(() => transport.sent.length > 0, { timeout: 50 }).catch(() => undefined);
+
+    const pcm = new Uint8Array(640);
+    pcm[0] = 1;
+    pcm[1] = 0;
+    client.sendAudioPcm(pcm, 16000, { contextId: "turn-opus" });
+
+    await vi.waitFor(() => transport.sent.some((entry) => {
+      if (!(entry instanceof Uint8Array)) return false;
+      return decodeSyrinxAudioEnvelope(entry).header.encoding === "opus";
+    }), { timeout: 3_000 });
+
+    const envelopeBytes = transport.sent.find((entry): entry is Uint8Array => {
+      if (!(entry instanceof Uint8Array)) return false;
+      return decodeSyrinxAudioEnvelope(entry).header.encoding === "opus";
+    })!;
+    const envelope = decodeSyrinxAudioEnvelope(envelopeBytes);
+    expect(envelope.header.sampleRateHz).toBe(48000);
+    const reference = pcm16BytesToSamples(
+      new OpusDecoder({ channels: 1, sample_rate: 48000 }).decode(envelope.audio),
+    );
+    expect(reference.length).toBeGreaterThan(0);
+  });
+});
+
+describe("browser downlink codec capability", () => {
+  it("reports pcm downlink when opus load fails", async () => {
+    vi.spyOn(browserOpus, "createBrowserOpusCodec").mockRejectedValue(new Error("opus unavailable"));
+
+    const transport = new FakeTransport();
+    const client = new SyrinxBrowserClient({
+      url: "ws://unused/ws",
+      transport,
+      reconnect: false,
+      keepaliveIntervalMs: false,
+    });
+    client.connect();
+    transport.emitMessage(JSON.stringify({
+      type: "ready",
+      sessionId: "sess-pcm-downlink",
+      audio: {
+        inputSampleRateHz: 16000,
+        outputSampleRateHz: 16000,
+        encoding: "opus",
+        supportedInputCodecs: ["pcm_s16le", "opus"],
+        channels: 1,
+        binaryEnvelope: "syrinx.audio.v1",
+      },
+    }));
+
+    await vi.waitFor(() => transport.sent.some((entry) => {
+      if (typeof entry !== "string") return false;
+      const message = JSON.parse(entry) as { type?: string; downlinkEncoding?: string };
+      return message.type === "codec_capability" && message.downlinkEncoding === "pcm_s16le";
+    }), { timeout: 3_000 });
+
+    vi.restoreAllMocks();
+  });
+});

package/src/turn-recorder.test.ts

@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: MIT
+
+import { describe, expect, it } from "vitest";
+
+import { encodeWav, TurnAudioRecorder } from "./turn-recorder.js";
+
+/** Decode a WAV the way a consumer would, so the test proves the bytes, not our own encoder. */
+function decodeWav(bytes: Uint8Array): { sampleRateHz: number; channels: number; bits: number; samples: Int16Array } {
+  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  const tag = (offset: number, length: number): string =>
+    String.fromCharCode(...bytes.subarray(offset, offset + length));
+  expect(tag(0, 4)).toBe("RIFF");
+  expect(tag(8, 4)).toBe("WAVE");
+  expect(tag(12, 4)).toBe("fmt ");
+  expect(tag(36, 4)).toBe("data");
+  expect(view.getUint32(4, true)).toBe(bytes.byteLength - 8);
+  const dataBytes = view.getUint32(40, true);
+  expect(44 + dataBytes).toBe(bytes.byteLength);
+  const samples = new Int16Array(dataBytes / 2);
+  for (let i = 0; i < samples.length; i += 1) samples[i] = view.getInt16(44 + i * 2, true);
+  return {
+    sampleRateHz: view.getUint32(24, true),
+    channels: view.getUint16(22, true),
+    bits: view.getUint16(34, true),
+    samples,
+  };
+}
+
+const frame = (value: number, length = 320): Int16Array => new Int16Array(length).fill(value);
+
+const ready = (hz: number) => ({ type: "ready" as const, audio: { inputSampleRateHz: hz } });
+
+/** Drive one whole turn: pre-speech silence, speech, then the boundary that names the turn. */
+function runTurn(rec: TurnAudioRecorder, turnId: string, value: number, frames = 5): void {
+  rec.onMessage({ type: "speech_started", turnId });
+  for (let i = 0; i < frames; i += 1) rec.pushFrame(frame(value));
+  rec.onMessage({ type: "turn_complete", turnId, transcript: "x" });
+}
+
+describe("encodeWav", () => {
+  it("writes a header a decoder accepts, little-endian regardless of host", () => {
+    const wav = encodeWav(Int16Array.from([0, 1, -1, 32767, -32768]), 16000);
+    const decoded = decodeWav(wav);
+    expect(decoded).toMatchObject({ sampleRateHz: 16000, channels: 1, bits: 16 });
+    expect([...decoded.samples]).toEqual([0, 1, -1, 32767, -32768]);
+  });
+
+  it("refuses an invalid sample rate rather than writing an undecodable file", () => {
+    expect(() => encodeWav(new Int16Array(4), 0)).toThrow(/invalid sampleRateHz/);
+  });
+});
+
+describe("TurnAudioRecorder", () => {
+  it("yields one decodable WAV per turn at the negotiated rate", () => {
+    const rec = new TurnAudioRecorder();
+    rec.onMessage(ready(24000));
+    runTurn(rec, "t1", 100);
+    runTurn(rec, "t2", 200);
+    runTurn(rec, "t3", 300);
+
+    expect(rec.list().map((t) => t.turnId)).toEqual(["t1", "t2", "t3"]);
+    for (const [turnId, value] of [["t1", 100], ["t2", 200], ["t3", 300]] as const) {
+      const wav = rec.getWav(turnId);
+      expect(wav, `${turnId} should have audio`).toBeDefined();
+      const decoded = decodeWav(wav as Uint8Array);
+      // The negotiated rate from `ready`, not the 16k default — a fixture written at the
+      // wrong rate transcribes as gibberish.
+      expect(decoded.sampleRateHz).toBe(24000);
+      expect(decoded.samples.length).toBe(5 * 320);
+      expect(decoded.samples[0]).toBe(value);
+    }
+  });
+
+  it("keeps the pre-roll so the speech onset is not clipped", () => {
+    const rec = new TurnAudioRecorder({ preRollMs: 20 });
+    rec.onMessage(ready(16000));
+    // 16000 Hz * 20ms = 320 samples of pre-roll retained.
+    rec.pushFrame(frame(7, 320));
+    rec.onMessage({ type: "speech_started", turnId: "t1" });
+    rec.pushFrame(frame(9, 320));
+    rec.onMessage({ type: "turn_complete", turnId: "t1", transcript: "x" });
+
+    const decoded = decodeWav(rec.getWav("t1") as Uint8Array);
+    expect(decoded.samples.length).toBe(640);
+    expect(decoded.samples[0]).toBe(7); // onset present
+    expect(decoded.samples[320]).toBe(9);
+  });
+
+  it("attributes a turn whose id only arrives on the closing message", () => {
+    const rec = new TurnAudioRecorder();
+    rec.onMessage(ready(16000));
+    rec.onMessage({ type: "speech_started" }); // no turnId yet
+    rec.pushFrame(frame(5));
+    rec.onMessage({ type: "speech_ended" }); // still none — must not be lost
+    rec.onMessage({ type: "stt_output", turnId: "late", transcript: "hi" });
+    expect(rec.getWav("late")).toBeDefined();
+  });
+
+  it("evicts oldest audio past maxTurns and does not grow unbounded", () => {
+    const rec = new TurnAudioRecorder({ maxTurns: 3 });
+    rec.onMessage(ready(16000));
+    for (let i = 0; i < 25; i += 1) runTurn(rec, `t${String(i)}`, i + 1);
+
+    expect(rec.list()).toHaveLength(3);
+    expect(rec.list().map((t) => t.turnId)).toEqual(["t22", "t23", "t24"]);
+    expect(rec.getWav("t0")).toBeUndefined(); // evicted, reported as absent not as silence
+    // 3 turns * 5 frames * 320 samples * 2 bytes — flat regardless of session length.
+    expect(rec.retainedBytes()).toBe(3 * 5 * 320 * 2);
+  });
+
+  it("caps a single runaway turn and says the tail was dropped", () => {
+    const rec = new TurnAudioRecorder({ maxTurnMs: 100 }); // 1600 samples at 16k
+    rec.onMessage(ready(16000));
+    rec.onMessage({ type: "speech_started", turnId: "stuck" });
+    for (let i = 0; i < 100; i += 1) rec.pushFrame(frame(1, 320));
+    rec.onMessage({ type: "turn_complete", turnId: "stuck", transcript: "x" });
+
+    const entry = rec.list()[0];
+    expect(entry?.truncated).toBe(true);
+    expect(entry?.byteLength).toBeLessThanOrEqual(1600 * 2 + 320 * 2);
+  });
+
+  it("reports duration from the sample count, not from wall clock", () => {
+    const rec = new TurnAudioRecorder();
+    rec.onMessage(ready(16000));
+    runTurn(rec, "t1", 1, 50); // 50 * 320 = 16000 samples = exactly 1s
+    expect(rec.list()[0]?.durationMs).toBe(1000);
+  });
+
+  it("reset drops every retained blob", () => {
+    const rec = new TurnAudioRecorder();
+    rec.onMessage(ready(16000));
+    runTurn(rec, "t1", 1);
+    rec.reset();
+    expect(rec.list()).toHaveLength(0);
+    expect(rec.retainedBytes()).toBe(0);
+  });
+});

package/src/turn-timeline.test.ts

@@ -0,0 +1,151 @@
+// SPDX-License-Identifier: MIT
+
+import { describe, expect, it } from "vitest";
+
+import { buildSessionRecord, type TurnRecord } from "./session-record.js";
+import { buildTurnTimeline, buildTimelines, endpointingMarker, FAST_TURN_FLOOR_MS } from "./turn-timeline.js";
+
+const turnWith = (timings: TurnRecord["timings"]): TurnRecord => ({
+  turnId: "t1",
+  startedAtMs: 0,
+  events: [],
+  agentText: "",
+  toolCalls: [],
+  ttsAudioBytes: 0,
+  errors: [],
+  complete: true,
+  droppedEvents: 0,
+  ...(timings ? { timings } : {}),
+});
+
+describe("turn timeline — segments", () => {
+  it("derives consecutive segments from the marks", () => {
+    const tl = buildTurnTimeline(
+      turnWith({ speechEndMs: 1000, textReadyMs: 1300, firstAudioByteMs: 2500, firstAudioPlayedMs: 2700, lastAudioPlayedMs: 6300, e2eMs: 5300 }),
+    );
+    expect(tl.segments.map((s) => [s.label, s.durationMs])).toEqual([
+      ["Deciding you're done, transcribing, and thinking", 300],
+      ["Voice (to first audio)", 1200],
+      ["First audio out", 200],
+      ["Agent speaking", 3600],
+    ]);
+    expect(tl.totalMs).toBe(5300);
+  });
+
+  it("starts each segment where the previous ended", () => {
+    const tl = buildTurnTimeline(
+      turnWith({ speechEndMs: 100, textReadyMs: 200, firstAudioByteMs: 500 }),
+    );
+    expect(tl.segments.map((s) => s.startMs)).toEqual([0, 100]);
+  });
+
+  it("marks the longest segment as slowest — where the time actually went", () => {
+    const tl = buildTurnTimeline(
+      turnWith({ speechEndMs: 0, textReadyMs: 100, firstAudioByteMs: 2000, firstAudioPlayedMs: 2100 }),
+    );
+    expect(tl.segments.filter((s) => s.slowest).map((s) => s.label)).toEqual(["Voice (to first audio)"]);
+  });
+
+  it("uses plain language, never packet names", () => {
+    const tl = buildTurnTimeline(turnWith({ speechEndMs: 0, textReadyMs: 100 }));
+    const labels = tl.segments.map((s) => s.label).join(" ");
+    expect(labels).not.toMatch(/eos|stt_|tts_|turn_complete|packet/i);
+    expect(labels).toMatch(/deciding you're done/i);
+  });
+});
+
+describe("turn timeline — missing data is stated, not faked", () => {
+  it("reports no-metrics rather than a zero-length lane", () => {
+    // The Workers path emits no `metrics` (LDT-18). An empty lane would read as
+    // a zero-latency turn, which is the opposite of the truth.
+    const tl = buildTurnTimeline(turnWith(undefined));
+    expect(tl.unavailable).toBe("no-metrics");
+    expect(tl.segments).toHaveLength(0);
+  });
+
+  it("reports insufficient-marks when only one mark arrived", () => {
+    const tl = buildTurnTimeline(turnWith({ speechEndMs: 100, e2eMs: 900 }));
+    expect(tl.unavailable).toBe("insufficient-marks");
+    expect(tl.totalMs).toBe(900); // still report what is known
+  });
+
+  it("collapses a missing middle mark without corrupting neighbours", () => {
+    const tl = buildTurnTimeline(
+      turnWith({ speechEndMs: 0, firstAudioByteMs: 1000, lastAudioPlayedMs: 3000 }),
+    );
+    expect(tl.segments.map((s) => [s.label, s.durationMs])).toEqual([
+      ["Voice (to first audio)", 1000],
+      ["Agent speaking", 2000],
+    ]);
+  });
+
+  it("never produces a negative duration from out-of-order marks", () => {
+    const tl = buildTurnTimeline(turnWith({ speechEndMs: 1000, textReadyMs: 400 }));
+    expect(tl.segments[0]?.durationMs).toBe(0);
+  });
+});
+
+describe("turn timeline — the fast-turn floor", () => {
+  it("flags an implausibly fast reply instead of celebrating it", () => {
+    // 480ms is not "fast" — the endpointer fired while the caller was still talking.
+    const tl = buildTurnTimeline(turnWith({ speechEndMs: 0, textReadyMs: 60, firstAudioByteMs: 400, e2eMs: 480 }));
+    expect(tl.suspiciouslyFast).toEqual({ totalMs: 480, floorMs: FAST_TURN_FLOOR_MS });
+  });
+
+  it("does not flag a normal turn", () => {
+    const tl = buildTurnTimeline(turnWith({ speechEndMs: 0, textReadyMs: 300, firstAudioByteMs: 1500, e2eMs: 2000 }));
+    expect(tl.suspiciouslyFast).toBeUndefined();
+  });
+
+  it("does not flag a turn with no measurable total", () => {
+    expect(buildTurnTimeline(turnWith({ speechEndMs: 0, textReadyMs: 0, e2eMs: 0 })).suspiciouslyFast).toBeUndefined();
+  });
+});
+
+describe("turn timeline — from a real recorded session", () => {
+  it("builds one lane per turn off a SessionRecord", () => {
+    const rec = buildSessionRecord([
+      { message: { type: "agent_chunk", turnId: "t1", text: "a" } as never, atMs: 0 },
+      { message: { type: "metrics", turnId: "t1", speechEndMs: 0, textReadyMs: 200, firstAudioByteMs: 900, e2eMs: 1100 } as never, atMs: 1 },
+      { message: { type: "agent_chunk", turnId: "t2", text: "b" } as never, atMs: 2 },
+    ]);
+    const tls = buildTimelines(rec.turns);
+    expect(tls).toHaveLength(2);
+    expect(tls[0]?.segments).toHaveLength(2);
+    expect(tls[1]?.unavailable).toBe("no-metrics"); // t2 never got metrics
+  });
+});
+
+describe("turn timeline — endpointing marker", () => {
+  it("names the owner in plain language, never the raw enum value", () => {
+    expect(endpointingMarker("provider_stt", "end_of_speech").text).toMatch(/speech-to-text provider/i);
+    expect(endpointingMarker("smart_turn", "end_of_speech").text).toMatch(/smart turn/i);
+    // "timer" covers both a realtime front and a session timer fallback, so the
+    // label stays honest for either — never assumes the realtime model specifically.
+    expect(endpointingMarker("timer", "end_of_speech").text).toMatch(/timer/i);
+    // No packet/message names leak into user-facing text.
+    for (const owner of ["provider_stt", "smart_turn", "timer"] as const) {
+      expect(endpointingMarker(owner, "end_of_speech").text).not.toMatch(/provider_stt|smart_turn|eos\.turn/);
+    }
+  });
+
+  it("says force-finalized was a timeout, not a natural endpoint", () => {
+    const m = endpointingMarker("provider_stt", "force_finalized");
+    expect(m.kind).toBe("endpoint");
+    expect(m.text).toMatch(/force-finalized.*timeout/i);
+  });
+
+  it("flags a typed turn and never claims an endpointer fired", () => {
+    const m = endpointingMarker("text", "typed");
+    expect(m.kind).toBe("typed");
+    expect(m.text).toMatch(/typed/i);
+    // It must not say an endpointer decided the turn ended — only that none ran.
+    expect(m.text).not.toMatch(/decided you had finished|force-finalized|marked the end/i);
+  });
+
+  it("says the cause is unknown rather than guessing when the owner is absent", () => {
+    const m = endpointingMarker(undefined, undefined);
+    expect(m.kind).toBe("unknown");
+    expect(m.text).toMatch(/unknown/i);
+  });
+});