@volley/recognition-client-sdk 0.1.200

package/src/recognition-client.ts

@@ -0,0 +1,595 @@
+/**
+ * RealTimeTwoWayWebSocketRecognitionClient - Clean, compact SDK for real-time speech recognition
+ *
+ * Features:
+ * - Ring buffer-based audio storage with fixed memory footprint
+ * - Automatic buffering when disconnected, immediate send when connected
+ * - Buffer persists after flush (for future retry/reconnection scenarios)
+ * - Built on WebSocketAudioClient for robust protocol handling
+ * - Simple API: connect() → sendAudio() → stopRecording()
+ * - Type-safe message handling with callbacks
+ * - Automatic backpressure management
+ * - Overflow detection with buffer state tracking
+ *
+ * Example:
+ * ```typescript
+ * const client = new RealTimeTwoWayWebSocketRecognitionClient({
+ *   url: 'ws://localhost:3101/ws/v1/recognize',
+ *   onTranscript: (result) => console.log(result.finalTranscript),
+ *   onError: (error) => console.error(error),
+ *   maxBufferDurationSec: 60  // Ring buffer for 60 seconds
+ * });
+ *
+ * await client.connect();
+ *
+ * // Send audio chunks - always stored in ring buffer, sent if connected
+ * micStream.on('data', (chunk) => client.sendAudio(chunk));
+ *
+ * // Signal end of audio and wait for final results
+ * await client.stopRecording();
+ *
+ * // Server will close connection after sending finals
+ * // No manual cleanup needed - browser handles it
+ * ```
+ */
+
+import { WebSocketAudioClient } from '@recog/websocket';
+import {
+  AudioEncoding,
+  RecognitionResultTypeV1,
+  ClientControlActionV1,
+  RecognitionContextTypeV1,
+  ControlSignalTypeV1,
+  type TranscriptionResultV1,
+  type FunctionCallResultV1,
+  type MetadataResultV1,
+  type ErrorResultV1,
+  type ClientControlMessageV1,
+  type ASRRequestConfig,
+  type ASRRequestV1,
+  type GameContextV1,
+  SampleRate
+} from '@recog/shared-types';
+import { v4 as uuidv4 } from 'uuid';
+import { ClientState } from './recognition-client.types.js';
+import type {
+  IRecognitionClient,
+  IRecognitionClientStats,
+  RealTimeTwoWayWebSocketRecognitionClientConfig,
+  RecognitionCallbackUrl
+} from './recognition-client.types.js';
+import { buildWebSocketUrl } from './utils/url-builder.js';
+import { AudioRingBuffer } from './utils/audio-ring-buffer.js';
+import { MessageHandler } from './utils/message-handler.js';
+
+// ============================================================================
+// UTILITIES
+// ============================================================================
+
+/**
+ * Check if a WebSocket close code indicates normal closure
+ * @param code - WebSocket close code
+ * @returns true if the disconnection was normal/expected, false if it was an error
+ */
+export function isNormalDisconnection(code: number): boolean {
+  return code === 1000; // 1000 is the only "normal" close code
+}
+
+// ============================================================================
+// TYPE DEFINITIONS
+// ============================================================================
+
+/**
+ * Re-export TranscriptionResultV1 as TranscriptionResult for backward compatibility
+ */
+export type TranscriptionResult = TranscriptionResultV1;
+
+// Re-export config interface from types file for backward compatibility
+export type { RealTimeTwoWayWebSocketRecognitionClientConfig } from './recognition-client.types.js';
+
+/**
+ * Internal config with processed values and defaults
+ */
+interface InternalConfig {
+  url: string;
+  readonly audioUtteranceId: string;  // Immutable - ensures one audio session per client instance
+  asrRequestConfig?: ASRRequestConfig;
+  gameContext?: GameContextV1;
+  callbackUrls?: RecognitionCallbackUrl[];
+  onTranscript: (result: TranscriptionResultV1) => void;
+  onFunctionCall: (result: FunctionCallResultV1) => void;
+  onMetadata: (metadata: MetadataResultV1) => void;
+  onError: (error: ErrorResultV1) => void;
+  onConnected: () => void;
+  onDisconnected: (code: number, reason: string) => void;
+  highWaterMark: number;
+  lowWaterMark: number;
+  maxBufferDurationSec: number;
+  chunksPerSecond: number;
+  logger?: (level: 'debug' | 'info' | 'warn' | 'error', message: string, data?: any) => void;
+}
+
+// ============================================================================
+// RECOGNITION CLIENT
+// ============================================================================
+
+/**
+ * RealTimeTwoWayWebSocketRecognitionClient - SDK-level client for real-time speech recognition
+ *
+ * Implements IRecognitionClient interface for dependency injection and testing.
+ * Extends WebSocketAudioClient with local audio buffering and simple callback-based API.
+ */
+export class RealTimeTwoWayWebSocketRecognitionClient
+  extends WebSocketAudioClient<number, any, any>
+  implements IRecognitionClient
+{
+  private static readonly PROTOCOL_VERSION = 1;
+
+  private config: InternalConfig;
+  private audioBuffer: AudioRingBuffer;
+  private messageHandler: MessageHandler;
+  private state: ClientState = ClientState.INITIAL;
+  private connectionPromise: Promise<void> | undefined;
+
+  // Debug control (internal state, controlled by debugCommand in ASRRequest)
+  private isDebugLogEnabled = false;
+
+  // Stats
+  private audioBytesSent = 0;
+  private audioChunksSent = 0;
+  private audioStatsLogInterval = 100;
+  private lastAudioStatsLog = 0;
+
+  constructor(config: RealTimeTwoWayWebSocketRecognitionClientConfig) {
+    // Generate UUID v4 for audioUtteranceId if not provided
+    const audioUtteranceId = config.audioUtteranceId || uuidv4();
+
+    // Build WebSocket URL with query parameters
+    const url = buildWebSocketUrl({
+      audioUtteranceId,
+      ...(config.url && { url: config.url }),
+      ...(config.callbackUrls && { callbackUrls: config.callbackUrls }),
+      ...(config.userId && { userId: config.userId }),
+      ...(config.gameSessionId && { gameSessionId: config.gameSessionId }),
+      ...(config.deviceId && { deviceId: config.deviceId }),
+      ...(config.accountId && { accountId: config.accountId }),
+      ...(config.questionAnswerId && { questionAnswerId: config.questionAnswerId }),
+      ...(config.platform && { platform: config.platform }),
+      ...(config.gameContext && { gameContext: config.gameContext })
+    });
+
+    // Initialize base WebSocketAudioClient
+    super({
+      url: url,
+      highWM: config.highWaterMark ?? 512_000,
+      lowWM: config.lowWaterMark ?? 128_000
+    });
+
+    // Process config with defaults
+    this.config = {
+      url,
+      audioUtteranceId,
+      ...(config.asrRequestConfig && { asrRequestConfig: config.asrRequestConfig }),
+      ...(config.gameContext && { gameContext: config.gameContext }),
+      ...(config.callbackUrls && { callbackUrls: config.callbackUrls }),
+      onTranscript: config.onTranscript || (() => {}),
+      onFunctionCall: config.onFunctionCall || (() => {}),
+      onMetadata: config.onMetadata || (() => {}),
+      onError: config.onError || (() => {}),
+      onConnected: config.onConnected || (() => {}),
+      onDisconnected: config.onDisconnected || (() => {}),
+      highWaterMark: config.highWaterMark ?? 512_000,
+      lowWaterMark: config.lowWaterMark ?? 128_000,
+      maxBufferDurationSec: config.maxBufferDurationSec ?? 60,
+      chunksPerSecond: config.chunksPerSecond ?? 100,
+      ...(config.logger && { logger: config.logger })
+    };
+
+    // Initialize audio buffer
+    this.audioBuffer = new AudioRingBuffer({
+      maxBufferDurationSec: this.config.maxBufferDurationSec,
+      chunksPerSecond: this.config.chunksPerSecond,
+      ...(this.config.logger && { logger: this.config.logger })
+    });
+
+    // Initialize message handler
+    this.messageHandler = new MessageHandler({
+      onTranscript: this.config.onTranscript,
+      onFunctionCall: this.config.onFunctionCall,
+      onMetadata: this.config.onMetadata,
+      onError: this.config.onError,
+      onControlMessage: this.handleControlMessage.bind(this),
+      ...(this.config.logger && { logger: this.config.logger })
+    });
+  }
+
+  // ==========================================================================
+  // PRIVATE HELPERS
+  // ==========================================================================
+
+  /**
+   * Internal logging helper - only logs if a logger was provided in config
+   * Debug logs are additionally gated by isDebugLogEnabled flag
+   * @param level - Log level: debug, info, warn, or error
+   * @param message - Message to log
+   * @param data - Optional additional data to log
+   */
+  private log(level: 'debug' | 'info' | 'warn' | 'error', message: string, data?: any): void {
+    // Skip debug logs if debug logging is not enabled
+    if (level === 'debug' && !this.isDebugLogEnabled) {
+      return;
+    }
+
+    if (this.config.logger) {
+      this.config.logger(level, `[SDK] ${message}`, data);
+    }
+  }
+
+  /**
+   * Clean up internal resources to free memory
+   * Called when connection closes (normally or abnormally)
+   */
+  private cleanup(): void {
+    this.log('debug', 'Cleaning up resources');
+
+    // Clear audio buffer to free memory
+    this.audioBuffer.clear();
+
+    // Reset stats
+    this.audioBytesSent = 0;
+    this.audioChunksSent = 0;
+    this.lastAudioStatsLog = 0;
+
+    // Clear connection promise so new connections can be made
+    this.connectionPromise = undefined;
+  }
+
+  // ==========================================================================
+  // PUBLIC API
+  // ==========================================================================
+
+  override async connect(): Promise<void> {
+    // FIRST: Check if we already have a connection promise (handles simultaneous calls)
+    if (this.connectionPromise) {
+      this.log('debug', 'Returning existing connection promise', {
+        state: this.state,
+        hasPromise: true
+      });
+      return this.connectionPromise;
+    }
+
+    // SECOND: Check state machine - prevent connections in wrong states
+    if (
+      this.state !== ClientState.INITIAL &&
+      this.state !== ClientState.FAILED &&
+      this.state !== ClientState.STOPPED
+    ) {
+      this.log('debug', 'Already connected or in wrong state', {
+        state: this.state
+      });
+      // If we're already connected/ready, return resolved promise
+      return Promise.resolve();
+    }
+
+    // RETRY HINT: Wrap this method with exponential backoff (e.g., 1s, 2s, 4s) on FAILED state
+    // Ensure audioBuffer persists between retries - same audioUtteranceId = same audio session
+
+    this.log('debug', 'Creating new connection to WebSocket', { url: this.config.url });
+    this.state = ClientState.CONNECTING;
+
+    const connectionStartTime = Date.now();
+
+    // Store the promise IMMEDIATELY so simultaneous calls will get the same promise
+    this.connectionPromise = new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        this.log('warn', 'Connection timeout', { timeout: 10000 });
+        this.state = ClientState.FAILED;
+        reject(new Error('Timeout'));
+      }, 10000);
+
+      const originalOnConnected = this.onConnected.bind(this);
+      this.onConnected = (): void => {
+        clearTimeout(timeout);
+        const connectionTime = Date.now() - connectionStartTime;
+        this.log('debug', 'Connection established successfully', {
+          connectionTimeMs: connectionTime,
+          url: this.config.url
+        });
+        this.state = ClientState.CONNECTED;
+        originalOnConnected();
+        resolve();
+      };
+
+      const originalOnError = this.onError.bind(this);
+      this.onError = (error): void => {
+        clearTimeout(timeout);
+        this.log('warn', 'Connection error', error);
+        this.state = ClientState.FAILED;
+        originalOnError(error);
+        reject(error);
+      };
+
+      super.connect();
+    });
+
+    return this.connectionPromise;
+  }
+
+  override sendAudio(audioData: ArrayBuffer | ArrayBufferView): void {
+    const bytes = ArrayBuffer.isView(audioData) ? audioData.byteLength : audioData.byteLength;
+    if (bytes === 0) return;
+
+    // BACKPRESSURE HINT: Return false or throw if audioBuffer.write() returns false (overflow)
+    // Caller should pause audio capture until buffer has space (check isBufferOverflowing())
+
+    // Always write to ring buffer
+    this.audioBuffer.write(audioData);
+
+    // Send immediately if ready and not backpressured
+    if (this.state === ClientState.READY && !super.isLocalBackpressured()) {
+      this.log('debug', 'Sending audio immediately', { bytes });
+      this.sendAudioNow(audioData);
+      this.audioBuffer.read(); // Remove from buffer since we sent it
+    } else {
+      this.log('debug', 'Buffering audio', {
+        bytes,
+        state: this.state,
+        backpressured: super.isLocalBackpressured()
+      });
+    }
+
+    // Log audio stats periodically (only if debug logging is enabled)
+    if (this.isDebugLogEnabled) {
+      const totalChunks = this.audioChunksSent + this.audioBuffer.getStats().chunksBuffered;
+      if (totalChunks - this.lastAudioStatsLog >= this.audioStatsLogInterval) {
+        const stats = this.audioBuffer.getStats();
+        this.log('debug', 'Audio statistics', {
+          totalBytesSent: this.audioBytesSent,
+          totalChunksSent: this.audioChunksSent,
+          ...stats
+        });
+        this.lastAudioStatsLog = totalChunks;
+      }
+    }
+  }
+
+  async stopRecording(): Promise<void> {
+    if (this.state !== ClientState.READY) {
+      this.log('warn', 'Cannot stop recording - not in READY state', { state: this.state });
+      return;
+    }
+
+    this.log('debug', 'Stopping recording');
+    this.state = ClientState.STOPPING;
+
+    super.sendMessage(RealTimeTwoWayWebSocketRecognitionClient.PROTOCOL_VERSION, 'message', {
+      type: RecognitionContextTypeV1.CONTROL_SIGNAL,
+      signal: ControlSignalTypeV1.STOP_RECORDING
+    });
+
+    return new Promise((resolve) => {
+      const timeout = setTimeout(() => {
+        this.state = ClientState.STOPPED;
+        resolve();
+      }, 5000);
+
+      const original = this.config.onTranscript;
+      this.config.onTranscript = (result): void => {
+        original(result);
+        if (result.is_finished) {
+          clearTimeout(timeout);
+          this.state = ClientState.STOPPED;
+          resolve();
+        }
+      };
+
+      // CRITICAL: Update MessageHandler's callback to use the wrapped version
+      // Otherwise it will keep calling the original and never detect is_finished
+      (this.messageHandler as any).callbacks.onTranscript = this.config.onTranscript;
+    });
+  }
+
+
+  getAudioUtteranceId(): string {
+    return this.config.audioUtteranceId;
+  }
+
+  getState(): ClientState {
+    return this.state;
+  }
+
+  isConnected(): boolean {
+    return this.state === ClientState.READY;
+  }
+
+  isConnecting(): boolean {
+    return this.state === ClientState.CONNECTING;
+  }
+
+  isStopping(): boolean {
+    return this.state === ClientState.STOPPING;
+  }
+
+  isTranscriptionFinished(): boolean {
+    return this.state === ClientState.STOPPED;
+  }
+
+  isBufferOverflowing(): boolean {
+    return this.audioBuffer.isOverflowing();
+  }
+
+  getStats(): IRecognitionClientStats {
+    const bufferStats = this.audioBuffer.getStats();
+    return {
+      audioBytesSent: this.audioBytesSent,
+      audioChunksSent: this.audioChunksSent,
+      audioChunksBuffered: bufferStats.chunksBuffered,
+      bufferOverflowCount: bufferStats.overflowCount,
+      currentBufferedChunks: bufferStats.currentBufferedChunks,
+      hasWrapped: bufferStats.hasWrapped
+    };
+  }
+
+  // ==========================================================================
+  // WEBSOCKET HOOKS (from WebSocketAudioClient)
+  // ==========================================================================
+
+  protected onConnected(): void {
+    this.log('debug', 'WebSocket onConnected callback');
+
+    // Send ASRRequest with configuration (if provided)
+    if (this.config.asrRequestConfig) {
+      // Extract debugCommand if present (with type safety for new field)
+      const debugCommand = (this.config.asrRequestConfig as any).debugCommand;
+      if (debugCommand?.enableDebugLog) {
+        this.isDebugLogEnabled = true;
+        this.log('debug', 'Debug logging enabled via debugCommand');
+      }
+
+      // Only generate debug log data if debug logging is enabled
+      if (this.isDebugLogEnabled) {
+        this.log('debug', 'Sending ASR request', this.config.asrRequestConfig);
+      }
+
+      const asrRequest: ASRRequestV1 = {
+        type: RecognitionContextTypeV1.ASR_REQUEST,
+        audioUtteranceId: this.config.audioUtteranceId,
+        provider: this.config.asrRequestConfig.provider.toString(),
+        model: this.config.asrRequestConfig.model,
+        language: this.config.asrRequestConfig.language?.toString() || 'en',
+        sampleRate:
+          typeof this.config.asrRequestConfig.sampleRate === 'number'
+            ? this.config.asrRequestConfig.sampleRate
+            : SampleRate.RATE_16000,
+        encoding:
+          typeof this.config.asrRequestConfig.encoding === 'number'
+            ? this.config.asrRequestConfig.encoding
+            : AudioEncoding.LINEAR16,
+        interimResults: this.config.asrRequestConfig.interimResults ?? false,
+        // Auto-enable useContext if gameContext is provided, or use explicit value if set
+        useContext: this.config.asrRequestConfig.useContext ?? !!this.config.gameContext,
+        ...(debugCommand && { debugCommand })
+      };
+
+      super.sendMessage(
+        RealTimeTwoWayWebSocketRecognitionClient.PROTOCOL_VERSION,
+        'message',
+        asrRequest
+      );
+    }
+
+    // Send GameContext if provided
+    if (this.config.gameContext) {
+      // Only pass gameContext object to log if debug logging is enabled
+      if (this.isDebugLogEnabled) {
+        this.log('debug', 'Sending game context', this.config.gameContext);
+      }
+      super.sendMessage(
+        RealTimeTwoWayWebSocketRecognitionClient.PROTOCOL_VERSION,
+        'message',
+        this.config.gameContext
+      );
+    }
+
+    this.log('debug', 'Waiting for server ready signal');
+    this.config.onConnected();
+  }
+
+  protected onDisconnected(code: number, reason: string): void {
+    this.log('debug', 'WebSocket disconnected', { code, reason, previousState: this.state });
+
+    // Update state based on disconnection type
+    if (this.state === ClientState.STOPPING) {
+      this.state = ClientState.STOPPED;
+    } else if (
+      this.state === ClientState.CONNECTED ||
+      this.state === ClientState.READY ||
+      this.state === ClientState.CONNECTING
+    ) {
+      this.log('error', 'Unexpected disconnection', { code, reason });
+      this.state = ClientState.FAILED;
+    }
+
+    // Clean up memory proactively when connection closes
+    this.cleanup();
+
+    this.config.onDisconnected(code, reason);
+  }
+
+  protected onError(error: Event): void {
+    this.state = ClientState.FAILED;
+
+    const errorResult: ErrorResultV1 = {
+      type: RecognitionResultTypeV1.ERROR,
+      audioUtteranceId: '',
+      message: 'WebSocket error',
+      description: error.type || 'Connection error'
+    };
+    this.config.onError(errorResult);
+  }
+
+  protected override onMessage(msg: { v: number; type: string; data: any }): void {
+    this.messageHandler.handleMessage(msg);
+  }
+
+  // ==========================================================================
+  // INTERNAL HELPERS
+  // ==========================================================================
+
+  /**
+   * Handle control messages from server
+   * @param msg - Control message containing server actions
+   */
+  private handleControlMessage(msg: ClientControlMessageV1): void {
+    switch (msg.action) {
+      case ClientControlActionV1.READY_FOR_UPLOADING_RECORDING: {
+        this.log('debug', 'Server ready for audio upload');
+        this.state = ClientState.READY;
+        this.messageHandler.setSessionStartTime(Date.now());
+
+        // Flush buffered audio now that server is ready
+        const bufferedChunks = this.audioBuffer.flush();
+        if (bufferedChunks.length > 0) {
+          this.log('debug', 'Flushing buffered audio', { chunks: bufferedChunks.length });
+          bufferedChunks.forEach((chunk) => this.sendAudioNow(chunk.data));
+        }
+        break;
+      }
+
+      case ClientControlActionV1.STOP_RECORDING:
+        this.log('debug', 'Received stop recording signal from server');
+        break;
+
+      default:
+        this.log('warn', 'Unknown control action', { action: msg.action });
+    }
+  }
+
+  /**
+   * Send audio immediately to the server (without buffering)
+   * @param audioData - Audio data to send
+   */
+  private sendAudioNow(audioData: ArrayBuffer | ArrayBufferView): void {
+    const byteLength = ArrayBuffer.isView(audioData)
+      ? audioData.byteLength
+      : audioData.byteLength;
+
+    const encodingId = (this.config.asrRequestConfig?.encoding ||
+      AudioEncoding.LINEAR16) as AudioEncoding;
+
+    const sampleRate =
+      typeof this.config.asrRequestConfig?.sampleRate === 'number'
+        ? this.config.asrRequestConfig.sampleRate
+        : SampleRate.RATE_16000;
+
+    super.sendAudio(
+      audioData,
+      RealTimeTwoWayWebSocketRecognitionClient.PROTOCOL_VERSION,
+      encodingId,
+      sampleRate
+    );
+
+    this.audioBytesSent += byteLength;
+    this.audioChunksSent++;
+  }
+}