@kuralle-syrinx/browser-client 2.1.0

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@kuralle-syrinx/browser-client",
+  "version": "2.1.0",
+  "private": false,
+  "type": "module",
+  "license": "MIT",
+  "main": "./src/index.ts",
+  "types": "./src/index.ts",
+  "dependencies": {
+    "@evan/opus": "1.0.3",
+    "@kuralle-syrinx/core": "2.1.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.7.0",
+    "vite": "^6.0.0",
+    "vitest": "^2.1.0"
+  },
+  "scripts": {
+    "dev": "vite --port 5173",
+    "build": "vite build",
+    "preview": "vite preview",
+    "typecheck": "tsc --noEmit",
+    "test": "vitest run"
+  }
+}

package/src/audio.test.ts

@@ -0,0 +1,130 @@
+// SPDX-License-Identifier: MIT
+
+import { describe, expect, it } from "vitest";
+import { encodeSyrinxAudioEnvelope } from "@kuralle-syrinx/core";
+import {
+  decodeBrowserAssistantAudio,
+  encodeBrowserAudioEnvelopeFrame,
+  encodeBrowserAudioFrame,
+  float32ToPcm16,
+  pcm16FrameSampleCount,
+  resampleFloat32Linear,
+} from "./audio.js";
+
+describe("browser audio utilities", () => {
+  it("resamples 48 kHz microphone input to 16 kHz PCM without trusting AudioContext rate", () => {
+    const input = new Float32Array(480);
+    for (let i = 0; i < input.length; i += 1) input[i] = i / input.length;
+
+    const output = resampleFloat32Linear(input, {
+      fromSampleRateHz: 48000,
+      toSampleRateHz: 16000,
+    });
+
+    expect(output.length).toBe(160);
+    expect(output[0]).toBeCloseTo(0);
+    expect(output[1]).toBeCloseTo(input[3]!);
+    expect(output.at(-1)).toBeCloseTo(input[477]!);
+  });
+
+  it("resamples 44.1 kHz microphone input to 16 kHz with stable frame sizing", () => {
+    const input = new Float32Array(441);
+    input.fill(0.25);
+
+    const output = resampleFloat32Linear(input, {
+      fromSampleRateHz: 44100,
+      toSampleRateHz: 16000,
+    });
+
+    expect(output.length).toBe(160);
+    expect(pcm16FrameSampleCount(16000)).toBe(320);
+    expect(output.every((sample) => Math.abs(sample - 0.25) < 0.0001)).toBe(true);
+  });
+
+  it("encodes browser Float32 audio as turn-scoped 16 kHz PCM16 JSON frames", () => {
+    const frame = encodeBrowserAudioFrame(new Float32Array([-2, -1, 0, 0.5, 1, 2]), {
+      fromSampleRateHz: 48000,
+      toSampleRateHz: 16000,
+      contextId: "review-turn",
+    });
+    const bytes = Buffer.from(frame.audio, "base64");
+    const pcm = new Int16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
+
+    expect(frame).toMatchObject({
+      type: "audio",
+      contextId: "review-turn",
+      sequence: undefined,
+      sampleRateHz: 16000,
+    });
+    expect(Array.from(pcm)).toEqual([-32768, 16384]);
+  });
+
+  it("preserves sequence metadata on encoded browser JSON audio frames", () => {
+    const frame = encodeBrowserAudioFrame(new Float32Array([0, 0.5, 1]), {
+      fromSampleRateHz: 48000,
+      toSampleRateHz: 16000,
+      contextId: "review-turn",
+      sequence: 17,
+    });
+
+    expect(frame).toMatchObject({
+      type: "audio",
+      contextId: "review-turn",
+      sampleRateHz: 16000,
+      sequence: 17,
+    });
+  });
+
+  it("encodes browser Float32 audio as turn-scoped binary envelope frames", () => {
+    const frame = encodeBrowserAudioEnvelopeFrame(new Float32Array([-2, -1, 0, 0.5, 1, 2]), {
+      fromSampleRateHz: 48000,
+      toSampleRateHz: 16000,
+      contextId: "review-turn",
+      sequence: 9,
+    });
+
+    const decoded = decodeBrowserAssistantAudio(frame);
+    const bytes = new Uint8Array(decoded.data);
+    const pcm = new Int16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2);
+
+    expect(decoded.metadata).toMatchObject({
+      type: "audio",
+      contextId: "review-turn",
+      sampleRateHz: 16000,
+      sequence: 9,
+      encoding: "pcm_s16le",
+      channels: 1,
+      byteLength: 4,
+    });
+    expect(Array.from(pcm)).toEqual([-32768, 16384]);
+  });
+
+  it("clamps Float32 samples before PCM16 conversion", () => {
+    expect(Array.from(float32ToPcm16(new Float32Array([-1.5, -1, 0, 1, 1.5])))).toEqual([
+      -32768,
+      -32768,
+      0,
+      32767,
+      32767,
+    ]);
+  });
+
+  it("decodes enveloped assistant audio before playback", () => {
+    const encoded = encodeSyrinxAudioEnvelope({
+      type: "audio",
+      contextId: "turn-tts",
+      sampleRateHz: 16000,
+      sequence: 2,
+      byteLength: 4,
+    }, new Uint8Array([1, 2, 3, 4]));
+
+    const decoded = decodeBrowserAssistantAudio(encoded);
+
+    expect(new Uint8Array(decoded.data)).toEqual(new Uint8Array([1, 2, 3, 4]));
+    expect(decoded.metadata).toMatchObject({
+      contextId: "turn-tts",
+      sampleRateHz: 16000,
+      sequence: 2,
+    });
+  });
+});

package/src/audio.ts

@@ -0,0 +1,308 @@
+// SPDX-License-Identifier: MIT
+
+import {
+  decodeSyrinxAudioEnvelope,
+  encodeSyrinxAudioEnvelope,
+  hasSyrinxAudioEnvelope,
+  type SyrinxAudioEnvelopeHeader,
+} from "@kuralle-syrinx/core";
+import { pcm16SamplesToBytes } from "@kuralle-syrinx/core/audio";
+import type { BrowserOpusCodec } from "./browser-opus.js";
+
+export interface ResampleFloat32Options {
+  readonly fromSampleRateHz: number;
+  readonly toSampleRateHz: number;
+}
+
+export interface EncodeBrowserAudioOptions extends ResampleFloat32Options {
+  readonly contextId?: string;
+  readonly sequence?: number;
+}
+
+export interface SyrinxAudioJsonFrame {
+  readonly type: "audio";
+  readonly audio: string;
+  readonly sampleRateHz: number;
+  readonly contextId?: string;
+  readonly sequence?: number;
+}
+
+export interface BrowserAssistantAudio {
+  readonly data: ArrayBuffer;
+  readonly metadata?: SyrinxAudioEnvelopeHeader;
+}
+
+export interface AudioJitterBufferOptions {
+  readonly targetBufferMs?: number;
+  readonly sampleRateHz: number;
+}
+
+interface ScheduledAudioFrame {
+  readonly buffer: AudioBuffer;
+  readonly scheduledTime: number;
+  readonly contextId?: string;
+  source: AudioBufferSourceNode | null;
+}
+
+export function resampleFloat32Linear(input: Float32Array, options: ResampleFloat32Options): Float32Array {
+  const fromRate = readPositiveSampleRate(options.fromSampleRateHz, "fromSampleRateHz");
+  const toRate = readPositiveSampleRate(options.toSampleRateHz, "toSampleRateHz");
+  if (fromRate === toRate) return input;
+
+  const outLength = Math.max(1, Math.round((input.length * toRate) / fromRate));
+  const output = new Float32Array(outLength);
+  const ratio = fromRate / toRate;
+
+  for (let i = 0; i < outLength; i += 1) {
+    const src = i * ratio;
+    const lo = Math.floor(src);
+    const hi = Math.min(input.length - 1, lo + 1);
+    const frac = src - lo;
+    output[i] = input[lo]! * (1 - frac) + input[hi]! * frac;
+  }
+
+  return output;
+}
+
+export function float32ToPcm16(samples: Float32Array): Int16Array {
+  const pcm = new Int16Array(samples.length);
+  for (let i = 0; i < samples.length; i += 1) {
+    const clamped = Math.max(-1, Math.min(1, samples[i]!));
+    pcm[i] = clamped < 0 ? Math.round(clamped * 32768) : Math.round(clamped * 32767);
+  }
+  return pcm;
+}
+
+export function encodeBrowserAudioFrame(input: Float32Array, options: EncodeBrowserAudioOptions): SyrinxAudioJsonFrame {
+  const targetRate = readPositiveSampleRate(options.toSampleRateHz, "toSampleRateHz");
+  const resampled = resampleFloat32Linear(input, options);
+  const pcm = float32ToPcm16(resampled);
+  return {
+    type: "audio",
+    contextId: options.contextId,
+    sequence: options.sequence,
+    sampleRateHz: targetRate,
+    audio: bytesToBase64(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength)),
+  };
+}
+
+export function encodeBrowserAudioEnvelopeFrame(input: Float32Array, options: EncodeBrowserAudioOptions): Uint8Array {
+  const targetRate = readPositiveSampleRate(options.toSampleRateHz, "toSampleRateHz");
+  const resampled = resampleFloat32Linear(input, options);
+  const pcm = float32ToPcm16(resampled);
+  const audio = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
+  return encodeSyrinxAudioEnvelope({
+    type: "audio",
+    contextId: options.contextId,
+    sampleRateHz: targetRate,
+    sequence: options.sequence,
+    encoding: "pcm_s16le",
+    channels: 1,
+    byteLength: audio.byteLength,
+    durationMs: Math.round((pcm.length / targetRate) * 1000),
+  }, audio);
+}
+
+export function pcm16FrameSampleCount(sampleRateHz: number, frameDurationMs = 20): number {
+  const sampleRate = readPositiveSampleRate(sampleRateHz, "sampleRateHz");
+  if (!Number.isFinite(frameDurationMs) || frameDurationMs <= 0) {
+    throw new Error("frameDurationMs must be a positive number");
+  }
+  return Math.max(1, Math.round(sampleRate * (frameDurationMs / 1000)));
+}
+
+export function decodeBrowserAssistantAudio(
+  input: ArrayBuffer | ArrayBufferView,
+  opusCodec: BrowserOpusCodec | null = null,
+): BrowserAssistantAudio {
+  const bytes = ArrayBuffer.isView(input)
+    ? new Uint8Array(input.buffer, input.byteOffset, input.byteLength)
+    : new Uint8Array(input);
+  if (!hasSyrinxAudioEnvelope(bytes)) {
+    return { data: copyArrayBuffer(bytes) };
+  }
+  const envelope = decodeSyrinxAudioEnvelope(bytes);
+  if (envelope.header.encoding === "opus") {
+    if (!opusCodec) {
+      return { data: new ArrayBuffer(0), metadata: envelope.header };
+    }
+    const wireRate = envelope.header.sampleRateHz;
+    const decoded = opusCodec.decodeOpusFrame(envelope.audio);
+    const pcm = pcm16SamplesToBytes(decoded);
+    return {
+      data: copyArrayBuffer(pcm),
+      metadata: {
+        ...envelope.header,
+        encoding: "pcm_s16le",
+        sampleRateHz: wireRate,
+        byteLength: pcm.byteLength,
+      },
+    };
+  }
+  return {
+    data: copyArrayBuffer(envelope.audio),
+    metadata: envelope.header,
+  };
+}
+
+export function bytesToBase64(bytes: Uint8Array): string {
+  let binary = "";
+  for (let i = 0; i < bytes.length; i += 0x8000) {
+    binary += String.fromCharCode(...bytes.subarray(i, i + 0x8000));
+  }
+  return btoa(binary);
+}
+
+function readPositiveSampleRate(value: number, name: string): number {
+  if (!Number.isFinite(value) || value <= 0) throw new Error(`${name} must be a positive sample rate`);
+  return value;
+}
+
+function copyArrayBuffer(bytes: Uint8Array): ArrayBuffer {
+  const copy = new Uint8Array(bytes.byteLength);
+  copy.set(bytes);
+  return copy.buffer;
+}
+
+export class AudioJitterBuffer {
+  private readonly context: AudioContext;
+  private readonly scheduledFrames = new Set<ScheduledAudioFrame>();
+  private nextScheduledTime = 0;
+  private readonly targetBufferMs: number;
+  private readonly sampleRateHz: number;
+  private readonly contextIds = new Set<string>();
+  private lastEnqueuedContextId: string | null = null;
+
+  /** True while assistant audio is scheduled or playing on the playout clock (not the generation clock). */
+  get isPlayingOut(): boolean {
+    return this.scheduledFrames.size > 0;
+  }
+
+  /** ContextId of the most recently scheduled assistant audio while any audio is still playing out. */
+  get activeContextId(): string | null {
+    return this.scheduledFrames.size > 0 ? this.lastEnqueuedContextId : null;
+  }
+
+  constructor(context: AudioContext, options: AudioJitterBufferOptions) {
+    this.context = context;
+    this.sampleRateHz = options.sampleRateHz;
+    this.targetBufferMs = options.targetBufferMs ?? 100;
+  }
+
+  enqueue(pcm16Data: ArrayBuffer, contextId?: string): void {
+    try {
+      const pcm16Array = new Int16Array(pcm16Data);
+      
+      // Skip empty or invalid audio data
+      if (pcm16Array.length === 0) {
+        return;
+      }
+      
+      const float32Array = new Float32Array(pcm16Array.length);
+      
+      // Convert PCM16 to Float32
+      for (let i = 0; i < pcm16Array.length; i++) {
+        float32Array[i] = pcm16Array[i]! / (pcm16Array[i]! < 0 ? 32768 : 32767);
+      }
+
+      const audioBuffer = this.context.createBuffer(1, float32Array.length, this.sampleRateHz);
+      audioBuffer.copyToChannel(float32Array, 0);
+
+      const now = this.context.currentTime;
+      
+      // If this is the first frame or we've fallen behind, establish baseline
+      if (this.nextScheduledTime === 0 || this.nextScheduledTime < now) {
+        this.nextScheduledTime = now + (this.targetBufferMs / 1000);
+      }
+
+      const frame: ScheduledAudioFrame = {
+        buffer: audioBuffer,
+        scheduledTime: this.nextScheduledTime,
+        contextId,
+        source: null,
+      };
+
+      this.scheduledFrames.add(frame);
+      if (contextId) {
+        this.contextIds.add(contextId);
+        this.lastEnqueuedContextId = contextId;
+      }
+
+      // Schedule the audio
+      const source = this.context.createBufferSource();
+      source.buffer = audioBuffer;
+      source.connect(this.context.destination);
+      
+      frame.source = source;
+      source.start(frame.scheduledTime);
+      
+      // Clean up when done
+      source.onended = () => {
+        this.scheduledFrames.delete(frame);
+        frame.source = null;
+      };
+
+      this.nextScheduledTime += audioBuffer.duration;
+    } catch (error) {
+      console.warn("AudioJitterBuffer: Failed to enqueue audio frame:", error);
+    }
+  }
+
+  clear(contextId?: string): void {
+    if (contextId) {
+      for (const frame of this.scheduledFrames) {
+        if (frame.contextId === contextId) {
+          if (frame.source) {
+            try {
+              frame.source.stop();
+            } catch {
+              // Ignore if already stopped
+            }
+          }
+          this.scheduledFrames.delete(frame);
+        }
+      }
+      this.contextIds.delete(contextId);
+      if (this.lastEnqueuedContextId === contextId) this.lastEnqueuedContextId = null;
+      this.recomputeNextScheduledTime();
+    } else {
+      // Clear all frames
+      for (const frame of this.scheduledFrames) {
+        if (frame.source) {
+          try {
+            frame.source.stop();
+          } catch {
+            // Ignore if already stopped
+          }
+        }
+      }
+      this.scheduledFrames.clear();
+      this.contextIds.clear();
+      this.lastEnqueuedContextId = null;
+      this.nextScheduledTime = 0;
+    }
+  }
+
+  get bufferedDurationMs(): number {
+    if (this.scheduledFrames.size === 0) return 0;
+    const now = this.context.currentTime;
+    return Math.max(0, (this.nextScheduledTime - now) * 1000);
+  }
+
+  get activeContextIds(): readonly string[] {
+    return [...this.contextIds];
+  }
+
+  private recomputeNextScheduledTime(): void {
+    if (this.scheduledFrames.size === 0) {
+      this.nextScheduledTime = 0;
+      return;
+    }
+    let maxEnd = 0;
+    for (const frame of this.scheduledFrames) {
+      const end = frame.scheduledTime + frame.buffer.duration;
+      if (end > maxEnd) maxEnd = end;
+    }
+    this.nextScheduledTime = maxEnd;
+  }
+}

package/src/browser-opus.ts

@@ -0,0 +1,118 @@
+// SPDX-License-Identifier: MIT
+
+import { encodeSyrinxAudioEnvelope } from "@kuralle-syrinx/core";
+import { pcm16BytesToSamples, pcm16SamplesToBytes } from "@kuralle-syrinx/core/audio";
+
+export const BROWSER_OPUS_SAMPLE_RATE_HZ = 48_000;
+export const BROWSER_OPUS_FRAME_DURATION_MS = 20;
+export const BROWSER_SUPPORTED_INPUT_CODECS = ["pcm_s16le", "opus"] as const;
+
+export type BrowserWireCodec = "pcm_s16le" | "opus";
+
+type OpusEncoder = { encode(pcm: Uint8Array): Uint8Array };
+type OpusDecoder = { decode(opus: Uint8Array): Uint8Array };
+
+type OpusModule = {
+  readonly Encoder: new (options: { channels: 1; sample_rate: number; application: "voip" }) => OpusEncoder;
+  readonly Decoder: new (options: { channels: 1; sample_rate: number }) => OpusDecoder;
+};
+
+export interface BrowserOpusCodec {
+  readonly sampleRateHz: number;
+  readonly frameSamples: number;
+  encodePcm16Frame(samples: Int16Array, flush?: boolean): Uint8Array[];
+  decodeOpusFrame(wire: Uint8Array): Int16Array;
+  reset(): void;
+}
+
+let opusModulePromise: Promise<OpusModule> | null = null;
+
+export function loadBrowserOpusModule(): Promise<OpusModule> {
+  opusModulePromise ??= import("@evan/opus") as Promise<OpusModule>;
+  return opusModulePromise;
+}
+
+export async function createBrowserOpusCodec(
+  sampleRateHz = BROWSER_OPUS_SAMPLE_RATE_HZ,
+): Promise<BrowserOpusCodec> {
+  const { Encoder, Decoder } = await loadBrowserOpusModule();
+  const encoder = new Encoder({ channels: 1, sample_rate: sampleRateHz, application: "voip" });
+  const decoder = new Decoder({ channels: 1, sample_rate: sampleRateHz });
+  const frameSamples = Math.max(1, Math.round((sampleRateHz * BROWSER_OPUS_FRAME_DURATION_MS) / 1000));
+  let encodeRemainder = new Int16Array(0);
+
+  return {
+    sampleRateHz,
+    frameSamples,
+    reset() {
+      encodeRemainder = new Int16Array(0);
+    },
+    decodeOpusFrame(wire: Uint8Array): Int16Array {
+      return pcm16BytesToSamples(decoder.decode(wire));
+    },
+    encodePcm16Frame(samples: Int16Array, flush = false): Uint8Array[] {
+      const pending = new Int16Array(encodeRemainder.length + samples.length);
+      pending.set(encodeRemainder);
+      pending.set(samples, encodeRemainder.length);
+      const completeFrames = Math.floor(pending.length / frameSamples);
+      const frames: Uint8Array[] = [];
+      for (let index = 0; index < completeFrames; index += 1) {
+        const frame = pending.subarray(index * frameSamples, (index + 1) * frameSamples);
+        frames.push(encoder.encode(pcm16SamplesToBytes(frame)));
+      }
+      const consumed = completeFrames * frameSamples;
+      const remainder = pending.subarray(consumed);
+      if (flush && remainder.length > 0) {
+        const padded = new Int16Array(frameSamples);
+        padded.set(remainder);
+        frames.push(encoder.encode(pcm16SamplesToBytes(padded)));
+        encodeRemainder = new Int16Array(0);
+      } else {
+        encodeRemainder = new Int16Array(remainder);
+      }
+      return frames;
+    },
+  };
+}
+
+export function pickBrowserWireCodec(
+  supportedInputCodecs: readonly string[] | undefined,
+  opusAvailable: boolean,
+): BrowserWireCodec {
+  if (opusAvailable && supportedInputCodecs?.includes("opus")) return "opus";
+  return "pcm_s16le";
+}
+
+export function encodeBrowserOpusEnvelope(
+  opusPayload: Uint8Array,
+  sampleRateHz: number,
+  options: { readonly contextId?: string; readonly sequence?: number },
+): Uint8Array {
+  return encodeSyrinxAudioEnvelope({
+    type: "audio",
+    contextId: options.contextId,
+    sampleRateHz,
+    sequence: options.sequence,
+    encoding: "opus",
+    channels: 1,
+    byteLength: opusPayload.byteLength,
+    durationMs: BROWSER_OPUS_FRAME_DURATION_MS,
+  }, opusPayload);
+}
+
+export function encodeBrowserPcmEnvelope(
+  audio: Uint8Array,
+  sampleRateHz: number,
+  options: { readonly contextId?: string; readonly sequence?: number },
+): Uint8Array {
+  return encodeSyrinxAudioEnvelope({
+    type: "audio",
+    contextId: options.contextId,
+    sampleRateHz,
+    sequence: options.sequence,
+    encoding: "pcm_s16le",
+    channels: 1,
+    byteLength: audio.byteLength,
+    durationMs: Math.round((audio.byteLength / 2 / sampleRateHz) * 1000),
+  }, audio);
+}