@volley/recognition-client-sdk 0.1.200 → 0.1.210

package/src/recognition-client.spec.ts

@@ -451,6 +451,185 @@ describe('RealTimeTwoWayWebSocketRecognitionClient', () => {
     });
   });
 
+  describe('Blob Audio Handling', () => {
+    it('should accept Blob as audio input', async () => {
+      const audioData = new Uint8Array([1, 2, 3, 4]);
+      const blob = new Blob([audioData], { type: 'audio/raw' });
+
+      // Should not throw
+      expect(() => client.sendAudio(blob)).not.toThrow();
+    });
+
+    it('should convert Blob to ArrayBuffer before buffering', async () => {
+      const audioData = new Uint8Array([1, 2, 3, 4]);
+      const blob = new Blob([audioData], { type: 'audio/raw' });
+
+      client.sendAudio(blob);
+
+      // Wait for async conversion
+      await new Promise(resolve => setTimeout(resolve, 100));
+
+      const stats = client.getStats();
+      // Should have buffered the converted data
+      expect(stats.currentBufferedChunks).toBeGreaterThan(0);
+    });
+
+    it('should handle empty Blob', async () => {
+      const blob = new Blob([], { type: 'audio/raw' });
+
+      client.sendAudio(blob);
+
+      // Wait for async conversion
+      await new Promise(resolve => setTimeout(resolve, 100));
+
+      const stats = client.getStats();
+      // Empty blob should not be buffered
+      expect(stats.currentBufferedChunks).toBe(0);
+    });
+
+    it('should handle large Blob', async () => {
+      const largeData = new Uint8Array(1024 * 1024); // 1MB
+      for (let i = 0; i < largeData.length; i++) {
+        largeData[i] = i % 256;
+      }
+      const blob = new Blob([largeData], { type: 'audio/raw' });
+
+      client.sendAudio(blob);
+
+      // Wait for async conversion
+      await new Promise(resolve => setTimeout(resolve, 100));
+
+      const stats = client.getStats();
+      expect(stats.currentBufferedChunks).toBe(1);
+    });
+
+    it('should handle multiple Blobs in sequence', async () => {
+      const blob1 = new Blob([new Uint8Array([1, 2, 3, 4])], { type: 'audio/raw' });
+      const blob2 = new Blob([new Uint8Array([5, 6, 7, 8])], { type: 'audio/raw' });
+      const blob3 = new Blob([new Uint8Array([9, 10, 11, 12])], { type: 'audio/raw' });
+
+      client.sendAudio(blob1);
+      client.sendAudio(blob2);
+      client.sendAudio(blob3);
+
+      // Wait for all async conversions
+      await new Promise(resolve => setTimeout(resolve, 200));
+
+      const stats = client.getStats();
+      expect(stats.currentBufferedChunks).toBe(3);
+    });
+
+    it('should handle mixed Blob and ArrayBuffer inputs', async () => {
+      const blob = new Blob([new Uint8Array([1, 2, 3, 4])], { type: 'audio/raw' });
+      const arrayBuffer = new ArrayBuffer(4);
+      const view = new Uint8Array(arrayBuffer);
+      view.set([5, 6, 7, 8]);
+
+      client.sendAudio(blob);
+      client.sendAudio(arrayBuffer);
+
+      // Wait for Blob conversion
+      await new Promise(resolve => setTimeout(resolve, 100));
+
+      const stats = client.getStats();
+      expect(stats.currentBufferedChunks).toBe(2);
+    });
+
+    it('should log error if Blob conversion fails', async () => {
+      const mockLogger = jest.fn();
+      const testClient = new RealTimeTwoWayWebSocketRecognitionClient({
+        url: 'ws://test.example.com/recognize',
+        logger: mockLogger
+      });
+
+      // Create a mock Blob that throws on arrayBuffer()
+      const badBlob: any = {
+        arrayBuffer: jest.fn().mockRejectedValue(new Error('Conversion failed'))
+      };
+
+      testClient.sendAudio(badBlob as Blob);
+
+      // Wait for error handling
+      await new Promise(resolve => setTimeout(resolve, 100));
+
+      // Should have logged an error
+      const errorCalls = mockLogger.mock.calls.filter(call => call[0] === 'error');
+      expect(errorCalls.length).toBeGreaterThan(0);
+    });
+
+    // Note: The blobToArrayBuffer() function has dual-path support:
+    // - Modern browsers: Uses blob.arrayBuffer() [Chrome 76+, Safari 14+]
+    // - Older Smart TVs: Falls back to FileReader [Tizen 2018-2019, webOS 3.0-4.x]
+
+    it('should use blob.arrayBuffer() when available (modern path)', async () => {
+      const audioData = new Uint8Array([1, 2, 3, 4]);
+      const blob = new Blob([audioData], { type: 'audio/raw' });
+
+      // Spy on blob.arrayBuffer to verify it's called
+      const arrayBufferSpy = jest.spyOn(blob, 'arrayBuffer');
+
+      client.sendAudio(blob);
+
+      // Wait for async conversion
+      await new Promise(resolve => setTimeout(resolve, 100));
+
+      // Should have used modern blob.arrayBuffer()
+      expect(arrayBufferSpy).toHaveBeenCalled();
+
+      // Should have buffered successfully
+      const stats = client.getStats();
+      expect(stats.currentBufferedChunks).toBe(1);
+    });
+
+    it('should use FileReader fallback when blob.arrayBuffer not available (Smart TV path)', async () => {
+      const audioData = new Uint8Array([1, 2, 3, 4]);
+
+      // Create a mock Blob-like object without arrayBuffer (simulates old Smart TV)
+      const blobLike = {
+        size: audioData.length,
+        type: 'audio/raw',
+        // No arrayBuffer method - will trigger FileReader path
+      };
+
+      // Mock FileReader
+      const mockReadAsArrayBuffer = jest.fn();
+      const originalFileReader = (global as any).FileReader;
+
+      (global as any).FileReader = jest.fn().mockImplementation(() => ({
+        readAsArrayBuffer: mockReadAsArrayBuffer,
+        onload: null,
+        onerror: null,
+        result: audioData.buffer
+      }));
+
+      // Trigger FileReader path by simulating onload after a delay
+      mockReadAsArrayBuffer.mockImplementation(function(this: any) {
+        setTimeout(() => {
+          if (this.onload) {
+            this.result = audioData.buffer;
+            this.onload();
+          }
+        }, 10);
+      });
+
+      client.sendAudio(blobLike as Blob);
+
+      // Wait for FileReader async conversion
+      await new Promise(resolve => setTimeout(resolve, 150));
+
+      // Should have used FileReader
+      expect((global as any).FileReader).toHaveBeenCalled();
+      expect(mockReadAsArrayBuffer).toHaveBeenCalledWith(blobLike);
+
+      // Should have buffered successfully
+      const stats = client.getStats();
+      expect(stats.currentBufferedChunks).toBe(1);
+
+      // Cleanup
+      (global as any).FileReader = originalFileReader;
+    });
+  });
+
   describe('Debug Logging', () => {
     it('should not log debug messages when debug logging is disabled (default)', () => {
       const mockLogger = jest.fn();

package/src/recognition-client.ts

@@ -75,6 +75,32 @@ export function isNormalDisconnection(code: number): boolean {
   return code === 1000; // 1000 is the only "normal" close code
 }
 
+/**
+ * Convert Blob to ArrayBuffer with Smart TV compatibility
+ *
+ * Browser Compatibility:
+ * - blob.arrayBuffer(): Newer TV
+ * - FileReader: All browsers, including older Smart TVs
+ *
+ * @see https://developer.samsung.com/smarttv/develop/specifications/web-engine-specifications.html
+ * @param blob - Blob to convert
+ * @returns Promise resolving to ArrayBuffer
+ */
+async function blobToArrayBuffer(blob: Blob): Promise<ArrayBuffer> {
+  // Modern approach (Chrome 76+, Safari 14+, Tizen 2020+, webOS 5.0+)
+  if (typeof blob.arrayBuffer === 'function') {
+    return await blob.arrayBuffer();
+  }
+
+  // Fallback for older Smart TVs (Tizen 2018-2019, webOS 3.0-4.x)
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = (): void => resolve(reader.result as ArrayBuffer);
+    reader.onerror = (): void => reject(reader.error);
+    reader.readAsArrayBuffer(blob);
+  });
+}
+
 // ============================================================================
 // TYPE DEFINITIONS
 // ============================================================================
@@ -315,7 +341,24 @@ export class RealTimeTwoWayWebSocketRecognitionClient
     return this.connectionPromise;
   }
 
-  override sendAudio(audioData: ArrayBuffer | ArrayBufferView): void {
+  override sendAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void {
+    // Handle Blob by converting to ArrayBuffer asynchronously
+    if (audioData instanceof Blob) {
+      blobToArrayBuffer(audioData)
+        .then((arrayBuffer) => {
+          this.sendAudioInternal(arrayBuffer);
+        })
+        .catch((error) => {
+          this.log('error', 'Failed to convert Blob to ArrayBuffer', error);
+        });
+      return;
+    }
+
+    // Handle ArrayBuffer and ArrayBufferView synchronously
+    this.sendAudioInternal(audioData);
+  }
+
+  private sendAudioInternal(audioData: ArrayBuffer | ArrayBufferView): void {
     const bytes = ArrayBuffer.isView(audioData) ? audioData.byteLength : audioData.byteLength;
     if (bytes === 0) return;
 

package/src/recognition-client.types.ts

@@ -164,9 +164,9 @@ export interface IRecognitionClient {
   /**
    * Send audio data to the recognition service
    * Audio is buffered locally and sent when connection is ready.
-   * @param audioData - PCM audio data as ArrayBuffer or typed array view
+   * @param audioData - PCM audio data as ArrayBuffer, typed array view, or Blob
    */
-  sendAudio(audioData: ArrayBuffer | ArrayBufferView): void;
+  sendAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
 
   /**
    * Stop recording and wait for final transcript

package/src/simplified-vgf-recognition-client.spec.ts

@@ -288,6 +288,12 @@ describe('SimplifiedVGFRecognitionClient', () => {
       expect(mockClient.sendAudio).toHaveBeenCalledWith(audioData);
     });
 
+    it('should delegate sendAudio() with Blob to underlying client', () => {
+      const blob = new Blob([new Uint8Array([1, 2, 3, 4])]);
+      simplifiedClient.sendAudio(blob);
+      expect(mockClient.sendAudio).toHaveBeenCalledWith(blob);
+    });
+
     it('should delegate stopRecording() to underlying client', async () => {
       await simplifiedClient.stopRecording();
       expect(mockClient.stopRecording).toHaveBeenCalled();

package/src/simplified-vgf-recognition-client.ts

@@ -57,9 +57,9 @@ export interface ISimplifiedVGFRecognitionClient {
 
     /**
      * Send audio data for transcription
-     * @param audioData - PCM audio data as ArrayBuffer or typed array
+     * @param audioData - PCM audio data as ArrayBuffer, typed array, or Blob
      */
-    sendAudio(audioData: ArrayBuffer | ArrayBufferView): void;
+    sendAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
 
     /**
      * Stop recording and wait for final transcription
@@ -228,7 +228,7 @@ export class SimplifiedVGFRecognitionClient implements ISimplifiedVGFRecognition
         // State will be updated via onConnected callback
     }
 
-    sendAudio(audioData: ArrayBuffer | ArrayBufferView): void {
+    sendAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void {
         // Track recording for state updates
         if (!this.isRecordingAudio) {
             this.isRecordingAudio = true;

package/src/utils/message-handler.spec.ts

@@ -0,0 +1,311 @@
+/**
+ * Unit tests for MessageHandler
+ */
+
+import { MessageHandler, MessageHandlerCallbacks } from './message-handler.js';
+import { RecognitionResultTypeV1, ClientControlActionV1 } from '@recog/shared-types';
+
+describe('MessageHandler', () => {
+  let callbacks: MessageHandlerCallbacks;
+  let handler: MessageHandler;
+  let mockLogger: jest.Mock;
+
+  beforeEach(() => {
+    mockLogger = jest.fn();
+    callbacks = {
+      onTranscript: jest.fn(),
+      onFunctionCall: jest.fn(),
+      onMetadata: jest.fn(),
+      onError: jest.fn(),
+      onControlMessage: jest.fn(),
+      logger: mockLogger
+    };
+    handler = new MessageHandler(callbacks);
+  });
+
+  describe('handleMessage', () => {
+    it('should handle transcription message', () => {
+      const msg = {
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.TRANSCRIPTION,
+          transcript: 'hello world',
+          isFinal: true
+        }
+      };
+
+      handler.handleMessage(msg);
+      expect(callbacks.onTranscript).toHaveBeenCalledWith(msg.data);
+    });
+
+    it('should handle function call message', () => {
+      const msg = {
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.FUNCTION_CALL,
+          functionName: 'testFunction',
+          arguments: { arg1: 'value1' }
+        }
+      };
+
+      handler.handleMessage(msg);
+      expect(callbacks.onFunctionCall).toHaveBeenCalledWith(msg.data);
+    });
+
+    it('should handle metadata message', () => {
+      const msg = {
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.METADATA,
+          metadata: { key: 'value' }
+        }
+      };
+
+      handler.handleMessage(msg);
+      expect(callbacks.onMetadata).toHaveBeenCalledWith(msg.data);
+    });
+
+    it('should handle error message', () => {
+      const msg = {
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.ERROR,
+          error: 'test error',
+          code: 'TEST_ERROR'
+        }
+      };
+
+      handler.handleMessage(msg);
+      expect(callbacks.onError).toHaveBeenCalledWith(msg.data);
+    });
+
+    it('should handle client control message', () => {
+      const msg = {
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.CLIENT_CONTROL_MESSAGE,
+          action: ClientControlActionV1.STOP_RECORDING,
+          audioUtteranceId: 'test-utterance'
+        }
+      };
+
+      handler.handleMessage(msg);
+      expect(callbacks.onControlMessage).toHaveBeenCalledWith(msg.data);
+    });
+
+    it('should handle unknown message type', () => {
+      const msg = {
+        v: 1,
+        type: 'unknown_type',
+        data: {
+          type: 'unknown',
+          content: 'test'
+        }
+      };
+
+      handler.handleMessage(msg);
+      expect(mockLogger).toHaveBeenCalledWith(
+        'debug',
+        'Unknown message type',
+        expect.objectContaining({ type: 'unknown' })
+      );
+    });
+
+    it('should log all incoming messages', () => {
+      const msg = {
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.TRANSCRIPTION,
+          transcript: 'test'
+        }
+      };
+
+      handler.handleMessage(msg);
+      expect(mockLogger).toHaveBeenCalledWith(
+        'debug',
+        'Received WebSocket message',
+        expect.objectContaining({
+          msgType: 'recognition_result',
+          msgDataType: RecognitionResultTypeV1.TRANSCRIPTION
+        })
+      );
+    });
+
+    it('should handle primitive msg.data', () => {
+      const msg = {
+        v: 1,
+        type: 'recognition_result',
+        data: 'primitive string'
+      };
+
+      handler.handleMessage(msg);
+      expect(mockLogger).toHaveBeenCalledWith(
+        'error',
+        'Received primitive msg.data from server',
+        expect.objectContaining({
+          dataType: 'string',
+          data: 'primitive string'
+        })
+      );
+    });
+
+    it('should handle message without data field', () => {
+      const msg = {
+        v: 1,
+        type: RecognitionResultTypeV1.METADATA,
+        data: {
+          type: RecognitionResultTypeV1.METADATA,
+          metadata: { key: 'value' }
+        }
+      };
+
+      handler.handleMessage(msg);
+      expect(callbacks.onMetadata).toHaveBeenCalled();
+    });
+
+    it('should work without logger', () => {
+      const callbacksNoLogger = {
+        onTranscript: jest.fn(),
+        onFunctionCall: jest.fn(),
+        onMetadata: jest.fn(),
+        onError: jest.fn(),
+        onControlMessage: jest.fn()
+      };
+      const handlerNoLogger = new MessageHandler(callbacksNoLogger);
+
+      const msg = {
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.TRANSCRIPTION,
+          transcript: 'test'
+        }
+      };
+
+      expect(() => handlerNoLogger.handleMessage(msg)).not.toThrow();
+      expect(callbacksNoLogger.onTranscript).toHaveBeenCalled();
+    });
+  });
+
+  describe('setSessionStartTime', () => {
+    it('should set session start time', () => {
+      const startTime = Date.now();
+      handler.setSessionStartTime(startTime);
+
+      const metrics = handler.getMetrics();
+      expect(metrics.sessionStartTime).toBe(startTime);
+    });
+  });
+
+  describe('getMetrics', () => {
+    it('should return initial metrics', () => {
+      const metrics = handler.getMetrics();
+      expect(metrics).toEqual({
+        sessionStartTime: null,
+        firstTranscriptTime: null,
+        timeToFirstTranscript: null
+      });
+    });
+
+    it('should track time to first transcript', () => {
+      const startTime = Date.now();
+      handler.setSessionStartTime(startTime);
+
+      // Simulate delay before first transcript
+      const transcriptMsg = {
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.TRANSCRIPTION,
+          transcript: 'first transcript',
+          isFinal: false
+        }
+      };
+
+      handler.handleMessage(transcriptMsg);
+
+      const metrics = handler.getMetrics();
+      expect(metrics.sessionStartTime).toBe(startTime);
+      expect(metrics.firstTranscriptTime).toBeGreaterThanOrEqual(startTime);
+      expect(metrics.timeToFirstTranscript).toBeGreaterThanOrEqual(0);
+    });
+
+    it('should log time to first transcript', () => {
+      const startTime = Date.now();
+      handler.setSessionStartTime(startTime);
+
+      const transcriptMsg = {
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.TRANSCRIPTION,
+          transcript: 'first transcript'
+        }
+      };
+
+      handler.handleMessage(transcriptMsg);
+
+      expect(mockLogger).toHaveBeenCalledWith(
+        'debug',
+        'First transcript received',
+        expect.objectContaining({
+          timeToFirstTranscriptMs: expect.any(Number)
+        })
+      );
+    });
+
+    it('should only track first transcript time once', () => {
+      const startTime = Date.now();
+      handler.setSessionStartTime(startTime);
+
+      // First transcript
+      handler.handleMessage({
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.TRANSCRIPTION,
+          transcript: 'first'
+        }
+      });
+
+      const firstMetrics = handler.getMetrics();
+      const firstTranscriptTime = firstMetrics.firstTranscriptTime;
+
+      // Second transcript
+      handler.handleMessage({
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.TRANSCRIPTION,
+          transcript: 'second'
+        }
+      });
+
+      const secondMetrics = handler.getMetrics();
+      expect(secondMetrics.firstTranscriptTime).toBe(firstTranscriptTime);
+    });
+
+    it('should not track transcript time without session start', () => {
+      const transcriptMsg = {
+        v: 1,
+        type: 'recognition_result',
+        data: {
+          type: RecognitionResultTypeV1.TRANSCRIPTION,
+          transcript: 'test'
+        }
+      };
+
+      handler.handleMessage(transcriptMsg);
+
+      const metrics = handler.getMetrics();
+      expect(metrics.firstTranscriptTime).toBeNull();
+      expect(metrics.timeToFirstTranscript).toBeNull();
+    });
+  });
+});