@volley/recognition-client-sdk 0.1.200

package/src/simplified-vgf-recognition-client.ts

@@ -0,0 +1,339 @@
+/**
+ * Simplified VGF Recognition Client
+ *
+ * A thin wrapper around RealTimeTwoWayWebSocketRecognitionClient that maintains
+ * a VGF RecognitionState as a pure sink/output of recognition events.
+ *
+ * The VGF state is updated based on events but never influences client behavior.
+ * All functionality is delegated to the underlying client.
+ */
+
+import { RecognitionState } from './vgf-recognition-state.js';
+import {
+    IRecognitionClient,
+    IRecognitionClientConfig,
+    ClientState
+} from './recognition-client.types.js';
+import { RealTimeTwoWayWebSocketRecognitionClient } from './recognition-client.js';
+import {
+    createVGFStateFromConfig,
+    mapTranscriptionResultToState,
+    mapMetadataToState,
+    mapErrorToState,
+    updateStateOnStop
+} from './vgf-recognition-mapper.js';
+import { RecognitionContextTypeV1 } from '@recog/shared-types';
+
+/**
+ * Configuration for SimplifiedVGFRecognitionClient
+ */
+export interface SimplifiedVGFClientConfig extends IRecognitionClientConfig {
+    /**
+     * Callback invoked whenever the VGF state changes
+     * Use this to update your UI or React state
+     */
+    onStateChange?: (state: RecognitionState) => void;
+
+    /**
+     * Optional initial state to restore from a previous session
+     * If provided, audioUtteranceId will be extracted and used
+     */
+    initialState?: RecognitionState;
+}
+
+/**
+ * Interface for SimplifiedVGFRecognitionClient
+ *
+ * A simplified client that maintains VGF state for game developers.
+ * All methods from the underlying client are available, plus VGF state management.
+ */
+export interface ISimplifiedVGFRecognitionClient {
+    // ============= Core Connection Methods =============
+    /**
+     * Connect to the recognition service WebSocket
+     * @returns Promise that resolves when connected and ready
+     */
+    connect(): Promise<void>;
+
+    /**
+     * Send audio data for transcription
+     * @param audioData - PCM audio data as ArrayBuffer or typed array
+     */
+    sendAudio(audioData: ArrayBuffer | ArrayBufferView): void;
+
+    /**
+     * Stop recording and wait for final transcription
+     * @returns Promise that resolves when transcription is complete
+     */
+    stopRecording(): Promise<void>;
+
+    // ============= VGF State Methods =============
+    /**
+     * Get the current VGF recognition state
+     * @returns Current RecognitionState with all transcription data
+     */
+    getVGFState(): RecognitionState;
+
+    // ============= Status Check Methods =============
+    /**
+     * Check if connected to the WebSocket
+     */
+    isConnected(): boolean;
+
+    /**
+     * Check if currently connecting
+     */
+    isConnecting(): boolean;
+
+    /**
+     * Check if currently stopping
+     */
+    isStopping(): boolean;
+
+    /**
+     * Check if transcription has finished
+     */
+    isTranscriptionFinished(): boolean;
+
+    /**
+     * Check if the audio buffer has overflowed
+     */
+    isBufferOverflowing(): boolean;
+
+    // ============= Utility Methods =============
+    /**
+     * Get the audio utterance ID for this session
+     */
+    getAudioUtteranceId(): string;
+
+    /**
+     * Get the underlying client state (for advanced usage)
+     */
+    getState(): ClientState;
+
+}
+
+/**
+ * This wrapper ONLY maintains VGF state as a sink.
+ * All actual functionality is delegated to the underlying client.
+ */
+export class SimplifiedVGFRecognitionClient implements ISimplifiedVGFRecognitionClient {
+    private client: IRecognitionClient;
+    private state: RecognitionState;
+    private isRecordingAudio: boolean = false;
+    private stateChangeCallback: ((state: RecognitionState) => void) | undefined;
+
+    constructor(config: SimplifiedVGFClientConfig) {
+        const { onStateChange, initialState, ...clientConfig } = config;
+        this.stateChangeCallback = onStateChange;
+
+        // Use provided initial state or create from config
+        if (initialState) {
+            this.state = initialState;
+            // Override audioUtteranceId in config if state has one
+            if (initialState.audioUtteranceId && !clientConfig.audioUtteranceId) {
+                clientConfig.audioUtteranceId = initialState.audioUtteranceId;
+            }
+        } else {
+            // Initialize VGF state from config
+            this.state = createVGFStateFromConfig(clientConfig);
+        }
+
+        // Client is immediately ready to accept audio (will buffer if not connected)
+        this.state = { ...this.state, startRecordingStatus: 'READY' };
+
+        // If VGF state has promptSlotMap, configure gameContext to use it
+        if (this.state.promptSlotMap) {
+            // Set useContext=true in ASR config to enable context processing
+            if (clientConfig.asrRequestConfig) {
+                clientConfig.asrRequestConfig.useContext = true;
+            }
+
+            // Add promptSlotMap to gameContext
+            if (!clientConfig.gameContext) {
+                // Only create gameContext if we have gameId and gamePhase
+                // These should come from the game's configuration
+                if (clientConfig.logger) {
+                    clientConfig.logger('warn', '[VGF] promptSlotMap found but no gameContext provided. SlotMap will not be sent.');
+                }
+            } else {
+                // Merge promptSlotMap into existing gameContext
+                clientConfig.gameContext.slotMap = this.state.promptSlotMap;
+            }
+        }
+
+        // Create underlying client with callbacks that ONLY update VGF state
+        this.client = new RealTimeTwoWayWebSocketRecognitionClient({
+            ...clientConfig,
+
+            // These callbacks ONLY update the VGF state sink
+            onTranscript: (result) => {
+                // Update VGF state based on transcript
+                this.state = mapTranscriptionResultToState(this.state, result, this.isRecordingAudio);
+                this.notifyStateChange();
+
+                // Call original callback if provided
+                if (clientConfig.onTranscript) {
+                    clientConfig.onTranscript(result);
+                }
+            },
+
+            onMetadata: (metadata) => {
+                this.state = mapMetadataToState(this.state, metadata);
+                this.notifyStateChange();
+
+                if (clientConfig.onMetadata) {
+                    clientConfig.onMetadata(metadata);
+                }
+            },
+
+            onFunctionCall: (result) => {
+                // Pass through function call - no VGF state changes needed for P2 feature
+                if (clientConfig.onFunctionCall) {
+                    clientConfig.onFunctionCall(result);
+                }
+            },
+
+            onError: (error) => {
+                this.isRecordingAudio = false; // Reset on error
+                this.state = mapErrorToState(this.state, error);
+                this.notifyStateChange();
+
+                if (clientConfig.onError) {
+                    clientConfig.onError(error);
+                }
+            },
+
+            onConnected: () => {
+                // Don't update READY here - client can accept audio before connection
+                if (clientConfig.onConnected) {
+                    clientConfig.onConnected();
+                }
+            },
+
+            onDisconnected: (code, reason) => {
+                this.isRecordingAudio = false; // Reset on disconnect
+                if (clientConfig.onDisconnected) {
+                    clientConfig.onDisconnected(code, reason);
+                }
+            }
+        });
+    }
+
+    // DELEGATE ALL METHODS TO UNDERLYING CLIENT
+    // The wrapper ONLY updates VGF state, doesn't use it for decisions
+
+    async connect(): Promise<void> {
+        await this.client.connect();
+        // State will be updated via onConnected callback
+    }
+
+    sendAudio(audioData: ArrayBuffer | ArrayBufferView): void {
+        // Track recording for state updates
+        if (!this.isRecordingAudio) {
+            this.isRecordingAudio = true;
+            this.state = {
+                ...this.state,
+                startRecordingStatus: 'RECORDING',
+                startRecordingTimestamp: new Date().toISOString()
+            };
+            this.notifyStateChange();
+        }
+        this.client.sendAudio(audioData);
+    }
+
+    async stopRecording(): Promise<void> {
+        this.isRecordingAudio = false;
+        this.state = updateStateOnStop(this.state);
+        this.notifyStateChange();
+        await this.client.stopRecording();
+    }
+
+    // Pure delegation methods - no state logic
+    getAudioUtteranceId(): string {
+        return this.client.getAudioUtteranceId();
+    }
+
+    getState(): ClientState {
+        return this.client.getState();
+    }
+
+    isConnected(): boolean {
+        return this.client.isConnected();
+    }
+
+    isConnecting(): boolean {
+        return this.client.isConnecting();
+    }
+
+    isStopping(): boolean {
+        return this.client.isStopping();
+    }
+
+    isTranscriptionFinished(): boolean {
+        return this.client.isTranscriptionFinished();
+    }
+
+    isBufferOverflowing(): boolean {
+        return this.client.isBufferOverflowing();
+    }
+
+
+    // VGF State access (read-only for consumers)
+    getVGFState(): RecognitionState {
+        return { ...this.state };
+    }
+
+    private notifyStateChange(): void {
+        if (this.stateChangeCallback) {
+            this.stateChangeCallback({ ...this.state });
+        }
+    }
+}
+
+/**
+ * Factory function for creating simplified client
+ * Usage examples:
+ *
+ * // Basic usage
+ * const client = createSimplifiedVGFClient({
+ *   asrRequestConfig: { provider: 'deepgram', language: 'en' },
+ *   onStateChange: (state) => {
+ *     console.log('VGF State updated:', state);
+ *     // Update React state, game UI, etc.
+ *   }
+ * });
+ *
+ * // With initial state (e.g., restoring from previous session)
+ * const client = createSimplifiedVGFClient({
+ *   asrRequestConfig: { provider: 'deepgram', language: 'en' },
+ *   initialState: previousState, // Will use audioUtteranceId from state
+ *   onStateChange: (state) => setVGFState(state)
+ * });
+ *
+ * // With initial state containing promptSlotMap for enhanced recognition
+ * const stateWithSlots: RecognitionState = {
+ *   audioUtteranceId: 'session-123',
+ *   promptSlotMap: {
+ *     'song_title': ['one time', 'baby'],
+ *     'artists': ['justin bieber']
+ *   }
+ * };
+ * const client = createSimplifiedVGFClient({
+ *   asrRequestConfig: { provider: 'deepgram', language: 'en' },
+ *   gameContext: {
+ *     type: RecognitionContextTypeV1.GAME_CONTEXT,
+ *     gameId: 'music-quiz',  // Your game's ID
+ *     gamePhase: 'song-guessing'  // Current game phase
+ *   },
+ *   initialState: stateWithSlots, // promptSlotMap will be added to gameContext
+ *   onStateChange: (state) => setVGFState(state)
+ * });
+ *
+ * await client.connect();
+ * client.sendAudio(audioData);
+ * // VGF state automatically updates based on transcription results
+ */
+export function createSimplifiedVGFClient(config: SimplifiedVGFClientConfig): ISimplifiedVGFRecognitionClient {
+    return new SimplifiedVGFRecognitionClient(config);
+}

package/src/utils/audio-ring-buffer.ts

@@ -0,0 +1,170 @@
+/**
+ * Audio Ring Buffer
+ * Manages circular buffer for audio data with overflow detection
+ */
+
+export interface BufferedAudio {
+  data: ArrayBuffer | ArrayBufferView;
+  timestamp: number;
+}
+
+export interface AudioRingBufferConfig {
+  maxBufferDurationSec: number;
+  chunksPerSecond: number;
+  logger?: (level: 'debug' | 'info' | 'warn' | 'error', message: string, data?: any) => void;
+}
+
+export class AudioRingBuffer {
+  private buffer: BufferedAudio[] = [];
+  private bufferSize: number;
+  private writeIndex = 0;
+  private readIndex = 0;
+  private hasWrapped = false;
+  private totalBufferedBytes = 0;
+  private overflowCount = 0;
+  private chunksBuffered = 0;
+  private logger?: (level: 'debug' | 'info' | 'warn' | 'error', message: string, data?: any) => void;
+
+  constructor(config: AudioRingBufferConfig) {
+    this.bufferSize = config.maxBufferDurationSec * config.chunksPerSecond;
+    this.buffer = new Array(this.bufferSize);
+    if (config.logger) {
+      this.logger = config.logger;
+    }
+  }
+
+  /**
+   * Write audio chunk to ring buffer with overflow detection
+   */
+  write(audioData: ArrayBuffer | ArrayBufferView): void {
+    const bytes = ArrayBuffer.isView(audioData) ? audioData.byteLength : audioData.byteLength;
+
+    // Write to current position
+    this.buffer[this.writeIndex] = {
+      data: audioData,
+      timestamp: Date.now()
+    };
+
+    // Advance write pointer
+    const nextWriteIndex = (this.writeIndex + 1) % this.bufferSize;
+
+    // Detect overflow: write caught up to read
+    if (nextWriteIndex === this.readIndex && this.writeIndex !== this.readIndex) {
+      this.hasWrapped = true;
+      this.overflowCount++;
+
+      // Log buffer overflow event
+      if (this.logger) {
+        this.logger('debug', 'Buffer overflow detected', {
+          bufferSize: this.bufferSize,
+          totalOverflows: this.overflowCount,
+          droppedChunk: this.buffer[this.readIndex]?.timestamp
+        });
+      }
+
+      // Move read pointer forward to make room (drop oldest)
+      this.readIndex = (this.readIndex + 1) % this.bufferSize;
+    }
+
+    this.writeIndex = nextWriteIndex;
+    this.chunksBuffered++;
+    this.totalBufferedBytes += bytes;
+  }
+
+  /**
+   * Read next chunk from buffer
+   */
+  read(): BufferedAudio | null {
+    if (this.isEmpty()) {
+      return null;
+    }
+
+    const chunk = this.buffer[this.readIndex];
+    this.readIndex = (this.readIndex + 1) % this.bufferSize;
+    return chunk || null;
+  }
+
+  /**
+   * Read all buffered chunks without removing them
+   */
+  readAll(): BufferedAudio[] {
+    const chunks: BufferedAudio[] = [];
+    let index = this.readIndex;
+
+    while (index !== this.writeIndex) {
+      const chunk = this.buffer[index];
+      if (chunk) {
+        chunks.push(chunk);
+      }
+      index = (index + 1) % this.bufferSize;
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Flush all buffered data and advance read pointer
+   */
+  flush(): BufferedAudio[] {
+    const chunks = this.readAll();
+    this.readIndex = this.writeIndex;
+    return chunks;
+  }
+
+  /**
+   * Get count of buffered chunks
+   */
+  getBufferedCount(): number {
+    if (this.writeIndex >= this.readIndex) {
+      return this.writeIndex - this.readIndex;
+    } else {
+      // Wrapped around
+      return this.bufferSize - this.readIndex + this.writeIndex;
+    }
+  }
+
+  /**
+   * Check if buffer is empty
+   */
+  isEmpty(): boolean {
+    return this.readIndex === this.writeIndex;
+  }
+
+  /**
+   * Check if buffer has overflowed
+   */
+  isOverflowing(): boolean {
+    return this.hasWrapped;
+  }
+
+  /**
+   * Clear the buffer and reset all counters
+   * Frees memory by releasing all stored audio chunks
+   */
+  clear(): void {
+    this.buffer = [];
+    this.writeIndex = 0;
+    this.readIndex = 0;
+    this.hasWrapped = false;
+    this.overflowCount = 0;
+    this.chunksBuffered = 0;
+    this.totalBufferedBytes = 0;
+
+    if (this.logger) {
+      this.logger('debug', 'Audio buffer cleared');
+    }
+  }
+
+  /**
+   * Get buffer statistics
+   */
+  getStats() {
+    return {
+      chunksBuffered: this.chunksBuffered,
+      currentBufferedChunks: this.getBufferedCount(),
+      overflowCount: this.overflowCount,
+      hasWrapped: this.hasWrapped,
+      totalBufferedBytes: this.totalBufferedBytes
+    };
+  }
+}

package/src/utils/message-handler.ts

@@ -0,0 +1,131 @@
+/**
+ * Message Handler for Recognition Client
+ * Routes incoming WebSocket messages to appropriate callbacks
+ */
+
+import {
+  RecognitionResultTypeV1,
+  ClientControlActionV1,
+  type TranscriptionResultV1,
+  type FunctionCallResultV1,
+  type MetadataResultV1,
+  type ErrorResultV1,
+  type ClientControlMessageV1
+} from '@recog/shared-types';
+
+export interface MessageHandlerCallbacks {
+  onTranscript: (result: TranscriptionResultV1) => void;
+  onFunctionCall: (result: FunctionCallResultV1) => void;
+  onMetadata: (metadata: MetadataResultV1) => void;
+  onError: (error: ErrorResultV1) => void;
+  onControlMessage: (msg: ClientControlMessageV1) => void;
+  logger?: (level: 'debug' | 'info' | 'warn' | 'error', message: string, data?: any) => void;
+}
+
+export class MessageHandler {
+  private firstTranscriptTime: number | null = null;
+  private sessionStartTime: number | null = null;
+  private callbacks: MessageHandlerCallbacks;
+
+  constructor(callbacks: MessageHandlerCallbacks) {
+    this.callbacks = callbacks;
+  }
+
+  /**
+   * Set session start time for performance tracking
+   */
+  setSessionStartTime(time: number): void {
+    this.sessionStartTime = time;
+  }
+
+  /**
+   * Handle incoming WebSocket message
+   */
+  handleMessage(msg: { v: number; type: string; data: any }): void {
+    // Log ALL incoming messages for debugging
+    if (this.callbacks.logger) {
+      this.callbacks.logger('debug', 'Received WebSocket message', {
+        msgType: msg.type,
+        msgDataType: msg.data && typeof msg.data === 'object' && 'type' in msg.data ? msg.data.type : 'N/A',
+        fullMessage: msg
+      });
+    }
+
+    // Safely check for type in msg.data - guard against primitives
+    // Log error if we receive primitive data (indicates server issue)
+    if (msg.data && typeof msg.data !== 'object') {
+      if (this.callbacks.logger) {
+        this.callbacks.logger('error', 'Received primitive msg.data from server', {
+          dataType: typeof msg.data,
+          data: msg.data,
+          fullMessage: msg
+        });
+      }
+    }
+
+    const msgType = (msg.data && typeof msg.data === 'object' && 'type' in msg.data ? msg.data.type : undefined) || msg.type;
+    const msgData = msg.data || msg;
+
+    switch (msgType) {
+      case RecognitionResultTypeV1.TRANSCRIPTION:
+        this.handleTranscription(msgData as TranscriptionResultV1);
+        break;
+
+      case RecognitionResultTypeV1.FUNCTION_CALL:
+        this.callbacks.onFunctionCall(msgData as FunctionCallResultV1);
+        break;
+
+      case RecognitionResultTypeV1.METADATA:
+        this.callbacks.onMetadata(msgData as MetadataResultV1);
+        break;
+
+      case RecognitionResultTypeV1.ERROR:
+        this.callbacks.onError(msgData as ErrorResultV1);
+        break;
+
+      case RecognitionResultTypeV1.CLIENT_CONTROL_MESSAGE:
+        this.callbacks.onControlMessage(msgData as ClientControlMessageV1);
+        break;
+
+      default:
+        // Unknown message type - log if logger available
+        if (this.callbacks.logger) {
+          this.callbacks.logger('debug', 'Unknown message type', { type: msgType });
+        }
+    }
+  }
+
+  /**
+   * Handle transcript message and track performance metrics
+   * @param result - The transcription result from the server
+   */
+  private handleTranscription(result: TranscriptionResultV1): void {
+    // Track time to first transcript
+    if (!this.firstTranscriptTime && this.sessionStartTime) {
+      this.firstTranscriptTime = Date.now();
+      const timeToFirstTranscript = this.firstTranscriptTime - this.sessionStartTime;
+
+      if (this.callbacks.logger) {
+        this.callbacks.logger('debug', 'First transcript received', {
+          timeToFirstTranscriptMs: timeToFirstTranscript
+        });
+      }
+    }
+
+    this.callbacks.onTranscript(result);
+  }
+
+  /**
+   * Get performance metrics
+   */
+  getMetrics() {
+    return {
+      sessionStartTime: this.sessionStartTime,
+      firstTranscriptTime: this.firstTranscriptTime,
+      timeToFirstTranscript:
+        this.firstTranscriptTime && this.sessionStartTime
+          ? this.firstTranscriptTime - this.sessionStartTime
+          : null
+    };
+  }
+}

package/src/utils/url-builder.ts

@@ -0,0 +1,70 @@
+/**
+ * URL Builder for Recognition Client
+ * Handles WebSocket URL construction with query parameters
+ */
+
+import { getRecognitionServiceBase } from '@recog/shared-config';
+import type { GameContextV1 } from '@recog/shared-types';
+import type { RecognitionCallbackUrl } from '../recognition-client.types.js';
+
+export interface UrlBuilderConfig {
+  url?: string;
+  audioUtteranceId: string;
+  callbackUrls?: RecognitionCallbackUrl[];
+  userId?: string;
+  gameSessionId?: string;
+  deviceId?: string;
+  accountId?: string;
+  questionAnswerId?: string;
+  platform?: string;
+  gameContext?: GameContextV1;
+}
+
+/**
+ * Build WebSocket URL with all query parameters
+ */
+export function buildWebSocketUrl(config: UrlBuilderConfig): string {
+  // Use default URL if not provided (production recognition service)
+  const defaultBase = getRecognitionServiceBase('production');
+  const baseUrl = config.url || `${defaultBase.wsBase}/ws/v1/recognize`;
+
+  // Build URL - add all optional identification parameters
+  const url = new URL(baseUrl);
+
+  // Add audioUtteranceId as query parameter (required for server to recognize it)
+  url.searchParams.set('audioUtteranceId', config.audioUtteranceId);
+
+  // Add callback URLs if provided (for server-side notifications)
+  if (config.callbackUrls && config.callbackUrls.length > 0) {
+    // Serialize as JSON for complex structure
+    url.searchParams.set('callbackUrls', JSON.stringify(config.callbackUrls));
+  }
+
+  // Add user/session/device/account identification if provided
+  if (config.userId) {
+    url.searchParams.set('userId', config.userId);
+  }
+  if (config.gameSessionId) {
+    url.searchParams.set('gameSessionId', config.gameSessionId);
+  }
+  if (config.deviceId) {
+    url.searchParams.set('deviceId', config.deviceId);
+  }
+  if (config.accountId) {
+    url.searchParams.set('accountId', config.accountId);
+  }
+  if (config.questionAnswerId) {
+    url.searchParams.set('questionAnswerId', config.questionAnswerId);
+  }
+  if (config.platform) {
+    url.searchParams.set('platform', config.platform);
+  }
+
+  // Add gameId and gamePhase if provided from gameContext
+  if (config.gameContext) {
+    url.searchParams.set('gameId', config.gameContext.gameId);
+    url.searchParams.set('gamePhase', config.gameContext.gamePhase);
+  }
+
+  return url.toString();
+}