@lokutor/sdk 1.1.17 → 1.1.19

package/src/audio-utils.ts

@@ -0,0 +1,253 @@
+import { AUDIO_CONFIG } from './types';
+
+/**
+ * Audio utility functions for format conversion, resampling, and PCM processing
+ */
+
+/**
+ * Convert 16-bit PCM (Int16) to 32-bit Float
+ * @param int16Data Int16Array of PCM audio
+ * @returns Float32Array normalized to [-1, 1]
+ */
+export function pcm16ToFloat32(int16Data: Int16Array): Float32Array {
+  const float32 = new Float32Array(int16Data.length);
+  for (let i = 0; i < int16Data.length; i++) {
+    float32[i] = int16Data[i] / 32768.0;
+  }
+  return float32;
+}
+
+/**
+ * Convert 32-bit Float to 16-bit PCM (Int16)
+ * @param float32Data Float32Array normalized to [-1, 1]
+ * @returns Int16Array of PCM audio
+ */
+export function float32ToPcm16(float32Data: Float32Array): Int16Array {
+  const int16 = new Int16Array(float32Data.length);
+  for (let i = 0; i < float32Data.length; i++) {
+    // Clamp to [-1, 1]
+    const s = Math.max(-1, Math.min(1, float32Data[i]));
+    // Convert to Int16
+    int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
+  }
+  return int16;
+}
+
+/**
+ * Resample audio data from one sample rate to another using linear interpolation
+ * @param input Float32Array of input audio
+ * @param inputRate Original sample rate in Hz
+ * @param outputRate Target sample rate in Hz
+ * @returns Float32Array of resampled audio
+ */
+export function resample(
+  input: Float32Array,
+  inputRate: number,
+  outputRate: number
+): Float32Array {
+  if (inputRate === outputRate) {
+    return new Float32Array(input);
+  }
+
+  const ratio = inputRate / outputRate;
+  const outputLength = Math.round(input.length / ratio);
+  const output = new Float32Array(outputLength);
+
+  for (let i = 0; i < outputLength; i++) {
+    const pos = i * ratio;
+    const left = Math.floor(pos);
+    const right = Math.min(left + 1, input.length - 1);
+    const weight = pos - left;
+
+    // Linear interpolation
+    output[i] = input[left] * (1 - weight) + input[right] * weight;
+  }
+
+  return output;
+}
+
+/**
+ * Apply a simple low-pass filter for anti-aliasing during downsampling
+ * @param data Float32Array of audio
+ * @param cutoffFreq Cutoff frequency in Hz
+ * @param sampleRate Sample rate in Hz
+ * @returns Filtered Float32Array
+ */
+export function applyLowPassFilter(
+  data: Float32Array,
+  cutoffFreq: number,
+  sampleRate: number
+): Float32Array {
+  // Calculate filter coefficient using first-order IIR filter
+  const dt = 1 / sampleRate;
+  const rc = 1 / (2 * Math.PI * cutoffFreq);
+  const alpha = dt / (rc + dt);
+
+  const filtered = new Float32Array(data.length);
+  filtered[0] = data[0];
+
+  for (let i = 1; i < data.length; i++) {
+    filtered[i] = filtered[i - 1] + alpha * (data[i] - filtered[i - 1]);
+  }
+
+  return filtered;
+}
+
+/**
+ * Resample audio with anti-aliasing low-pass filter
+ * Best used when downsampling to prevent aliasing artifacts
+ * @param input Float32Array of input audio
+ * @param inputRate Original sample rate in Hz
+ * @param outputRate Target sample rate in Hz
+ * @returns Float32Array of resampled and filtered audio
+ */
+export function resampleWithAntiAliasing(
+  input: Float32Array,
+  inputRate: number,
+  outputRate: number
+): Float32Array {
+  if (inputRate === outputRate) {
+    return new Float32Array(input);
+  }
+
+  // If downsampling, apply low-pass filter first to prevent aliasing
+  let processed = input;
+  if (outputRate < inputRate) {
+    const nyquistFreq = outputRate / 2;
+    const cutoffFreq = nyquistFreq * 0.9; // 90% of Nyquist to be safe
+    processed = applyLowPassFilter(input, cutoffFreq, inputRate);
+  }
+
+  return resample(processed, inputRate, outputRate);
+}
+
+/**
+ * Convert raw audio samples to Uint8Array (bytes)
+ * @param data Int16Array of PCM audio
+ * @returns Uint8Array containing PCM bytes
+ */
+export function pcm16ToBytes(data: Int16Array): Uint8Array {
+  return new Uint8Array(data.buffer, data.byteOffset, data.byteLength);
+}
+
+/**
+ * Convert bytes to Int16Array
+ * @param bytes Uint8Array of PCM bytes
+ * @returns Int16Array of PCM audio
+ */
+export function bytesToPcm16(bytes: Uint8Array): Int16Array {
+  if (bytes.length % 2 !== 0) {
+    // Trim the last byte to avoid RangeError
+    bytes = bytes.slice(0, bytes.length - 1);
+  }
+  return new Int16Array(bytes.buffer, bytes.byteOffset, bytes.length / 2);
+}
+
+/**
+ * Normalize audio amplitude to prevent clipping
+ * @param data Float32Array of audio
+ * @param targetPeak Peak level to normalize to (0-1)
+ * @returns Normalized Float32Array
+ */
+export function normalizeAudio(data: Float32Array, targetPeak: number = 0.95): Float32Array {
+  let maxAbs = 0;
+  for (let i = 0; i < data.length; i++) {
+    maxAbs = Math.max(maxAbs, Math.abs(data[i]));
+  }
+
+  if (maxAbs === 0) return new Float32Array(data);
+
+  const scale = targetPeak / maxAbs;
+  const normalized = new Float32Array(data.length);
+  for (let i = 0; i < data.length; i++) {
+    normalized[i] = data[i] * scale;
+  }
+  return normalized;
+}
+
+/**
+ * Calculate RMS (Root Mean Square) amplitude
+ * @param data Float32Array or Uint8Array of audio
+ * @returns RMS value (0-1 for normalized float, 0-255 for byte data)
+ */
+export function calculateRMS(data: Float32Array | Uint8Array): number {
+  let sum = 0;
+  let length = data.length;
+
+  if (data instanceof Uint8Array) {
+    // For byte data (0-128 is silence, 128-255 is audio)
+    for (let i = 0; i < length; i++) {
+      const v = (data[i] - 128) / 128.0;
+      sum += v * v;
+    }
+  } else {
+    // For float data (-1 to 1)
+    for (let i = 0; i < length; i++) {
+      sum += data[i] * data[i];
+    }
+  }
+
+  return Math.sqrt(sum / length);
+}
+
+/**
+ * Create a resample function factory for streaming audio
+ * Useful for processing audio in chunks
+ */
+export class StreamResampler {
+  private inputBuffer: Float32Array = new Float32Array(0);
+  private inputRate: number;
+  private outputRate: number;
+
+  constructor(inputRate: number, outputRate: number) {
+    this.inputRate = inputRate;
+    this.outputRate = outputRate;
+  }
+
+  /**
+   * Process a chunk of audio and return resampled data
+   * @param inputChunk Float32Array chunk to process
+   * @param flush If true, output remaining buffered samples
+   * @returns Resampled Float32Array (may be empty if more data needed)
+   */
+  process(inputChunk: Float32Array, flush: boolean = false): Float32Array {
+    // Concatenate new data with buffered data
+    const combined = new Float32Array(this.inputBuffer.length + inputChunk.length);
+    combined.set(this.inputBuffer);
+    combined.set(inputChunk, this.inputBuffer.length);
+
+    const ratio = this.inputRate / this.outputRate;
+    let outputLength = Math.floor(combined.length / ratio);
+
+    if (outputLength === 0 && !flush) {
+      this.inputBuffer = combined;
+      return new Float32Array(0);
+    }
+
+    // On flush, output everything including partial frames
+    if (flush && outputLength === 0 && combined.length > 0) {
+      outputLength = 1;
+    }
+
+    const output = new Float32Array(outputLength);
+
+    for (let i = 0; i < outputLength; i++) {
+      const pos = i * ratio;
+      const left = Math.floor(pos);
+      const right = Math.min(left + 1, combined.length - 1);
+      const weight = pos - left;
+
+      output[i] = combined[left] * (1 - weight) + combined[right] * weight;
+    }
+
+    // Keep remaining samples in buffer
+    const consumed = Math.floor(outputLength * ratio);
+    this.inputBuffer = combined.slice(consumed);
+
+    return output;
+  }
+
+  reset(): void {
+    this.inputBuffer = new Float32Array(0);
+  }
+}

package/src/browser-audio.ts

@@ -0,0 +1,395 @@
+import { AUDIO_CONFIG } from './types';
+import { float32ToPcm16, pcm16ToFloat32, StreamResampler, calculateRMS } from './audio-utils';
+
+/**
+ * Configuration for browser audio handling
+ */
+export interface BrowserAudioConfig {
+  inputSampleRate?: number;
+  outputSampleRate?: number;
+  autoGainControl?: boolean;
+  echoCancellation?: boolean;
+  noiseSuppression?: boolean;
+  onInputError?: (error: Error) => void;
+}
+
+/**
+ * Analyser configuration for audio visualization
+ */
+export interface AnalyserConfig {
+  enabled?: boolean;
+  fftSize?: number;
+}
+
+/**
+ * Browser-based audio manager for Web Audio API operations
+ * Handles microphone input, speaker output, and visualization
+ */
+export class BrowserAudioManager {
+  private audioContext: AudioContext | null = null;
+  private mediaStreamAudioSourceNode: MediaStreamAudioSourceNode | null = null;
+  private scriptProcessor: ScriptProcessorNode | null = null;
+  private analyserNode: AnalyserNode | null = null;
+  private mediaStream: MediaStream | null = null;
+  private resampler: StreamResampler | null = null;
+
+  // Playback scheduling
+  private nextPlaybackTime: number = 0;
+  private activeSources: AudioBufferSourceNode[] = [];
+  private playbackQueue: AudioBuffer[] = [];
+
+  // Configuration
+  private inputSampleRate: number;
+  private outputSampleRate: number;
+  private autoGainControl: boolean;
+  private echoCancellation: boolean;
+  private noiseSuppression: boolean;
+
+  // Callbacks
+  private onAudioInput?: (pcm16Data: Uint8Array) => void;
+  private onInputError?: (error: Error) => void;
+
+  // Audio processing state
+  private isMuted: boolean = false;
+  private isListening: boolean = false;
+
+  constructor(config: BrowserAudioConfig = {}) {
+    this.inputSampleRate = config.inputSampleRate ?? AUDIO_CONFIG.SAMPLE_RATE;
+    this.outputSampleRate = config.outputSampleRate ?? AUDIO_CONFIG.SPEAKER_SAMPLE_RATE;
+    this.autoGainControl = config.autoGainControl ?? true;
+    this.echoCancellation = config.echoCancellation ?? true;
+    this.noiseSuppression = config.noiseSuppression ?? true;
+    this.onInputError = config.onInputError;
+  }
+
+  /**
+   * Initialize the AudioContext and analyser
+   */
+  async init(analyserConfig?: AnalyserConfig): Promise<void> {
+    if (this.audioContext) return; // Already initialized
+
+    const AudioContextClass =
+      (window as any).AudioContext || (window as any).webkitAudioContext;
+    if (!AudioContextClass) {
+      throw new Error('Web Audio API not supported in this browser');
+    }
+
+    this.audioContext = new AudioContextClass();
+
+    // Ensure AudioContext is running (not suspended)
+    if (!this.audioContext) {
+      throw new Error('Failed to initialize AudioContext');
+    }
+
+    if (this.audioContext.state === 'suspended') {
+      await this.audioContext.resume();
+      console.log('👂 AudioContext resumed');
+    }
+
+    // Setup analyser for visualization if enabled
+    if (analyserConfig?.enabled !== false) {
+      this.analyserNode = this.audioContext.createAnalyser();
+      this.analyserNode.fftSize = analyserConfig?.fftSize ?? 256;
+    }
+  }
+
+  /**
+   * Start capturing audio from the microphone
+   */
+  async startMicrophone(
+    onAudioInput: (pcm16Data: Uint8Array) => void
+  ): Promise<void> {
+    if (!this.audioContext) {
+      await this.init();
+    }
+
+    try {
+      this.onAudioInput = onAudioInput;
+      this.isListening = true;
+
+      // Request microphone access with constraints
+      this.mediaStream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          autoGainControl: this.autoGainControl,
+          echoCancellation: this.echoCancellation,
+          noiseSuppression: this.noiseSuppression,
+        },
+      });
+
+      // Create source from microphone stream
+      this.mediaStreamAudioSourceNode =
+        this.audioContext!.createMediaStreamSource(this.mediaStream);
+
+      // Create script processor for PCM extraction
+      // Note: ScriptProcessorNode is deprecated but widely supported.
+      // AudioWorklet would be better but requires additional setup.
+      const bufferSize = 4096;
+      this.scriptProcessor = this.audioContext!.createScriptProcessor(
+        bufferSize,
+        1, // input channels
+        1  // output channels
+      );
+
+      // Connect the audio graph
+      this.mediaStreamAudioSourceNode.connect(this.scriptProcessor);
+      this.scriptProcessor.connect(this.audioContext!.destination);
+
+      // Connect to analyser if available
+      if (this.analyserNode) {
+        this.mediaStreamAudioSourceNode.connect(this.analyserNode);
+      }
+
+      // Initialize stateful resampler if sample rates differ
+      const hardwareRate = this.audioContext!.sampleRate;
+      if (hardwareRate !== this.inputSampleRate) {
+        this.resampler = new StreamResampler(hardwareRate, this.inputSampleRate);
+      } else {
+        this.resampler = null;
+      }
+
+      // Handle audio processing
+      this.scriptProcessor.onaudioprocess = (event: AudioProcessingEvent) => {
+        this._processAudioInput(event);
+      };
+
+      console.log('🎤 Microphone started');
+    } catch (error) {
+      const err = error instanceof Error ? error : new Error(String(error));
+      if (this.onInputError) this.onInputError(err);
+      throw err;
+    }
+  }
+
+  /**
+   * Internal method to process microphone audio data
+   */
+  private _processAudioInput(event: AudioProcessingEvent): void {
+    if (!this.onAudioInput || !this.audioContext || !this.isListening) return;
+    if (this.isMuted) return;
+
+    const inputBuffer = event.inputBuffer;
+    const inputData = inputBuffer.getChannelData(0);
+
+    // Silence output to prevent feedback
+    const outputBuffer = event.outputBuffer;
+    for (let i = 0; i < outputBuffer.getChannelData(0).length; i++) {
+      outputBuffer.getChannelData(0)[i] = 0;
+    }
+
+    // Resample from hardware rate to target rate if needed
+    let processedData: Float32Array = new Float32Array(inputData);
+
+    if (this.resampler) {
+      processedData = this.resampler.process(processedData);
+    }
+
+    if (processedData.length === 0) return; // Need more data for resampler
+
+    // Convert Float32 to Int16 PCM
+    const int16Data = float32ToPcm16(processedData);
+    const uint8Data = new Uint8Array(
+      int16Data.buffer,
+      int16Data.byteOffset,
+      int16Data.byteLength
+    );
+
+    this.onAudioInput(uint8Data);
+  }
+
+  /**
+   * Stop capturing microphone input
+   */
+  stopMicrophone(): void {
+    this.isListening = false;
+
+    if (this.mediaStream) {
+      this.mediaStream.getTracks().forEach((track) => track.stop());
+      this.mediaStream = null;
+    }
+
+    if (this.scriptProcessor) {
+      this.scriptProcessor.disconnect();
+      this.scriptProcessor = null;
+    }
+
+    if (this.mediaStreamAudioSourceNode) {
+      this.mediaStreamAudioSourceNode.disconnect();
+      this.mediaStreamAudioSourceNode = null;
+    }
+
+    console.log('🎤 Microphone stopped');
+  }
+
+  /**
+   * Play back audio received from the server
+   * @param pcm16Data Int16 PCM audio data at SPEAKER_SAMPLE_RATE
+   */
+  playAudio(pcm16Data: Uint8Array): void {
+    if (!this.audioContext) {
+      console.warn('AudioContext not initialized');
+      return;
+    }
+
+    if (pcm16Data.length % 2 !== 0) {
+      console.warn(`Discarding odd-length PCM buffer (${pcm16Data.length} bytes)`);
+      return;
+    }
+
+    // Convert Int16 to Float32
+    const int16Array = new Int16Array(
+      pcm16Data.buffer,
+      pcm16Data.byteOffset,
+      pcm16Data.length / 2
+    );
+    const float32Data = pcm16ToFloat32(int16Array);
+
+    // Create audio buffer
+    const audioBuffer = this.audioContext.createBuffer(
+      1,
+      float32Data.length,
+      this.outputSampleRate
+    );
+    audioBuffer.getChannelData(0).set(float32Data);
+
+    // Schedule playback
+    this._schedulePlayback(audioBuffer);
+  }
+
+  /**
+   * Internal method to schedule and play audio with sample-accurate timing
+   */
+  private _schedulePlayback(audioBuffer: AudioBuffer): void {
+    if (!this.audioContext) return;
+
+    const currentTime = this.audioContext.currentTime;
+    const duration = audioBuffer.length / this.outputSampleRate;
+
+    // Schedule playback to occur seamlessly after previous audio
+    const startTime = Math.max(
+      currentTime + 0.01, // Minimum 10ms delay
+      this.nextPlaybackTime
+    );
+    this.nextPlaybackTime = startTime + duration;
+
+    // Create and configure source node
+    const source = this.audioContext.createBufferSource();
+    source.buffer = audioBuffer;
+    source.connect(this.audioContext.destination);
+
+    if (this.analyserNode) {
+      source.connect(this.analyserNode);
+    }
+
+    source.start(startTime);
+    this.activeSources.push(source);
+
+    // Clean up source reference when finished
+    source.onended = () => {
+      const index = this.activeSources.indexOf(source);
+      if (index > -1) {
+        this.activeSources.splice(index, 1);
+      }
+    };
+  }
+
+  /**
+   * Stop all currently playing audio and clear the queue
+   */
+  stopPlayback(): void {
+    this.activeSources.forEach((source) => {
+      try {
+        source.stop();
+      } catch (e) {
+        // Already stopped or other error
+      }
+    });
+    this.activeSources = [];
+    this.playbackQueue = [];
+    this.nextPlaybackTime = this.audioContext?.currentTime ?? 0;
+    console.log('🔇 Playback stopped');
+  }
+
+  /**
+   * Toggle mute state
+   */
+  setMuted(muted: boolean): void {
+    this.isMuted = muted;
+  }
+
+  /**
+   * Get current mute state
+   */
+  isMicMuted(): boolean {
+    return this.isMuted;
+  }
+
+  /**
+   * Get current amplitude from analyser (for visualization)
+   * Returns value between 0 and 1
+   */
+  getAmplitude(): number {
+    if (!this.analyserNode) return 0;
+
+    const dataArray = new Uint8Array(this.analyserNode.frequencyBinCount);
+    this.analyserNode.getByteTimeDomainData(dataArray);
+
+    const rms = calculateRMS(dataArray);
+    return Math.min(rms * 10, 1); // Boost for visualization
+  }
+
+  /**
+   * Get frequency data from analyser for visualization
+   */
+  getFrequencyData(): Uint8Array {
+    if (!this.analyserNode) {
+      return new Uint8Array(0);
+    }
+
+    const dataArray = new Uint8Array(this.analyserNode.frequencyBinCount);
+    this.analyserNode.getByteFrequencyData(dataArray);
+    return dataArray;
+  }
+
+  /**
+   * Get time-domain data from analyser for waveform visualization
+   */
+  getWaveformData(): Uint8Array {
+    if (!this.analyserNode) {
+      return new Uint8Array(0);
+    }
+
+    const dataArray = new Uint8Array(this.analyserNode.frequencyBinCount);
+    this.analyserNode.getByteTimeDomainData(dataArray);
+    return dataArray;
+  }
+
+  /**
+   * Cleanup and close AudioContext
+   */
+  cleanup(): void {
+    this.stopMicrophone();
+    this.stopPlayback();
+
+    if (this.analyserNode) {
+      this.analyserNode.disconnect();
+      this.analyserNode = null;
+    }
+
+    // Don't close AudioContext - it can be expensive to recreate
+    // Just leave it suspended if no longer needed
+  }
+
+  /**
+   * Get current audio context state
+   */
+  getState(): 'running' | 'suspended' | 'closed' | 'interrupted' | null {
+    return this.audioContext?.state ?? null;
+  }
+
+  /**
+   * Check if microphone is currently listening
+   */
+  isRecording(): boolean {
+    return this.isListening;
+  }
+}