@volley/recognition-client-sdk 0.1.799 → 0.1.800

package/src/simplified-vgf-recognition-client.ts

@@ -17,12 +17,14 @@ import {
 import {
     IRecognitionClient,
     IRecognitionClientConfig,
+    IRecognitionClientStats,
     ClientState
 } from './recognition-client.types.js';
 import { RealTimeTwoWayWebSocketRecognitionClient } from './recognition-client.js';
 import {
     createVGFStateFromConfig,
     mapTranscriptionResultToState,
+    mapSessionConfiguredToState,
     mapErrorToState,
     updateStateOnStop,
     resetRecognitionVGFState
@@ -49,126 +51,22 @@ export interface SimplifiedVGFClientConfig extends IRecognitionClientConfig {
 /**
  * Interface for SimplifiedVGFRecognitionClient
  *
- * A simplified client that maintains VGF state for game developers.
- * All methods from the underlying client are available, plus VGF state management.
+ * Inherits the full IRecognitionClient surface (connect, sendAudio,
+ * sendAudioWithSampleRate, sendPrefixAudio, stopRecording, stopAbnormally,
+ * status checks, sendGameContext, getStats, getUrl, getState, getAudioUtteranceId)
+ * — see recognition-client.types.ts for those. Adds VGF-specific state access.
+ *
+ * Extending IRecognitionClient (rather than redeclaring methods) means
+ * TypeScript catches any base-client method that's not delegated by the
+ * VGF wrapper at compile time — keeps the two surfaces in sync.
  */
-export interface ISimplifiedVGFRecognitionClient {
-    // ============= Core Connection Methods =============
-    /**
-     * Connect to the recognition service WebSocket
-     * @returns Promise that resolves when connected and ready
-     */
-    connect(): Promise<void>;
-
-    /**
-     * Send audio data for transcription
-     * @param audioData - PCM audio data as ArrayBuffer, typed array, or Blob
-     */
-    sendAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
-
-    /**
-     * Send PCM16 mono audio captured at `sourceSampleRate`; the SDK
-     * downsamples to the session's target rate before transmitting. Use
-     * when capture is at the system's native rate (browser AudioContext is
-     * typically 44.1 kHz or 48 kHz). Audio must be signed 16-bit
-     * little-endian PCM, mono.
-     */
-    sendAudioWithSampleRate(
-        audioData: ArrayBuffer | ArrayBufferView | Blob,
-        sourceSampleRate: number
-    ): void;
-
-    /**
-     * Stop recording and wait for final transcription
-     * @returns Promise that resolves when transcription is complete
-     */
-    stopRecording(): Promise<void>;
-
-    /**
-     * Force stop and immediately close connection without waiting for server
-     *
-     * WARNING: This is an abnormal shutdown that bypasses the graceful stop flow:
-     * - Does NOT wait for server to process remaining audio
-     * - Does NOT receive final transcript from server (VGF state set to empty)
-     * - Immediately closes WebSocket connection
-     * - Cleans up resources (buffers, listeners)
-     *
-     * Use Cases:
-     * - User explicitly cancels/abandons the session
-     * - Timeout scenarios where waiting is not acceptable
-     * - Need immediate cleanup and can't wait for server
-     *
-     * RECOMMENDED: Use stopRecording() for normal shutdown.
-     * Only use this when immediate disconnection is required.
-     */
-    stopAbnormally(): void;
-
-    // ============= VGF State Methods =============
+export interface ISimplifiedVGFRecognitionClient extends IRecognitionClient {
     /**
-     * Get the current VGF recognition state
+     * Get the current VGF recognition state — the single shared store
+     * of inputs and outputs for this utterance.
      * @returns Current RecognitionState with all transcription data
      */
     getVGFState(): RecognitionState;
-
-    // ============= Status Check Methods =============
-    /**
-     * Check if connected to the WebSocket
-     */
-    isConnected(): boolean;
-
-    /**
-     * Check if currently connecting
-     */
-    isConnecting(): boolean;
-
-    /**
-     * Check if currently stopping
-     */
-    isStopping(): boolean;
-
-    /**
-     * Check if transcription has finished
-     */
-    isTranscriptionFinished(): boolean;
-
-    /**
-     * Check if the audio buffer has overflowed
-     */
-    isBufferOverflowing(): boolean;
-
-    // ============= Preconnect Methods =============
-    /**
-     * Send game context after connection is established (for preconnect flow).
-     *
-     * Preconnect flow: Create client with asrRequestConfig (useContext: true) but
-     * WITHOUT gameContext → call connect() → later call sendGameContext() with slotMap.
-     *
-     * @param context - Game context including slotMap for keyword boosting
-     */
-    sendGameContext(context: GameContextV1): void;
-
-    /**
-     * Check if server has sent READY signal (provider connected, ready for audio).
-     * In preconnect flow, this becomes true after sendGameContext() triggers provider attachment.
-     */
-    isServerReady(): boolean;
-
-    // ============= Utility Methods =============
-    /**
-     * Get the audio utterance ID for this session
-     */
-    getAudioUtteranceId(): string;
-
-    /**
-     * Get the WebSocket URL being used
-     */
-    getUrl(): string;
-
-    /**
-     * Get the underlying client state (for advanced usage)
-     */
-    getState(): ClientState;
-
 }
 
 /**
@@ -241,23 +139,38 @@ export class SimplifiedVGFRecognitionClient implements ISimplifiedVGFRecognition
         // Track the expected UUID for this session
         this.expectedUuid = this.state.audioUtteranceId;
 
-        // If VGF state has promptSlotMap, configure gameContext to use it
-        if (this.state.promptSlotMap) {
+        // Forward optional VGF inputs (promptSlotMap, promptSTT/STF/TTF) into the GameContext.
+        const hasPromptInputs =
+            this.state.promptSlotMap !== undefined ||
+            this.state.promptSTT !== undefined ||
+            this.state.promptSTF !== undefined ||
+            this.state.promptTTF !== undefined;
+
+        if (hasPromptInputs) {
             // Set useContext=true in ASR config to enable context processing
             if (clientConfig.asrRequestConfig) {
                 clientConfig.asrRequestConfig.useContext = true;
             }
 
-            // Add promptSlotMap to gameContext
             if (!clientConfig.gameContext) {
                 // Only create gameContext if we have gameId and gamePhase
                 // These should come from the game's configuration
                 if (clientConfig.logger) {
-                    clientConfig.logger('warn', '[VGF] promptSlotMap found but no gameContext provided. SlotMap will not be sent.');
+                    clientConfig.logger('warn', '[VGF] prompt inputs found but no gameContext provided. They will not be sent.');
                 }
             } else {
-                // Merge promptSlotMap into existing gameContext
-                clientConfig.gameContext.slotMap = this.state.promptSlotMap;
+                if (this.state.promptSlotMap !== undefined) {
+                    clientConfig.gameContext.slotMap = this.state.promptSlotMap;
+                }
+                if (this.state.promptSTT !== undefined) {
+                    clientConfig.gameContext.promptSTT = this.state.promptSTT;
+                }
+                if (this.state.promptSTF !== undefined) {
+                    clientConfig.gameContext.promptSTF = this.state.promptSTF;
+                }
+                if (this.state.promptTTF !== undefined) {
+                    clientConfig.gameContext.promptTTF = this.state.promptTTF;
+                }
             }
         }
 
@@ -303,6 +216,25 @@ export class SimplifiedVGFRecognitionClient implements ISimplifiedVGFRecognition
                 }
             },
 
+            onSessionConfigured: (sessionConfigured): void => {
+                // Skip update if UUID doesn't match (stale callback from previous session)
+                if (sessionConfigured.audioUtteranceId && sessionConfigured.audioUtteranceId !== this.expectedUuid) {
+                    if (this.logger) {
+                        this.logger('warn',
+                            `[RecogSDK:VGF] Skipping sessionConfigured update: UUID mismatch (expected: ${this.expectedUuid}, got: ${sessionConfigured.audioUtteranceId})`
+                        );
+                    }
+                    return;
+                }
+
+                this.state = mapSessionConfiguredToState(this.state, sessionConfigured);
+                this.notifyStateChange();
+
+                if (clientConfig.onSessionConfigured) {
+                    clientConfig.onSessionConfigured(sessionConfigured);
+                }
+            },
+
             onFunctionCall: (result): void => {
                 // Pass through function call - no VGF state changes needed for P2 feature
                 if (clientConfig.onFunctionCall) {
@@ -322,7 +254,7 @@ export class SimplifiedVGFRecognitionClient implements ISimplifiedVGFRecognition
                 }
 
                 this.isRecordingAudio = false; // Reset on error
-                this.state = mapErrorToState(this.state, error);
+                this.state = mapErrorToState(this.state);
                 this.notifyStateChange();
 
                 if (clientConfig.onError) {
@@ -367,6 +299,17 @@ export class SimplifiedVGFRecognitionClient implements ISimplifiedVGFRecognition
         this.client.sendAudioWithSampleRate(audioData, sourceSampleRate);
     }
 
+    sendPrefixAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void {
+        // Pure pass-through. Prefix audio is NOT user audio, so do not flip
+        // startRecordingStatus to RECORDING — that transition belongs to the
+        // first sendAudio() call.
+        this.client.sendPrefixAudio(audioData);
+    }
+
+    getStats(): IRecognitionClientStats {
+        return this.client.getStats();
+    }
+
     /**
      * Set VGF recording status to RECORDING on the first audio chunk.
      * Idempotent — subsequent calls are no-ops until disconnect/stop resets

package/src/vgf-recognition-mapper.spec.ts

@@ -3,6 +3,8 @@
  */
 
 import {
+    mapTranscriptionResultToState,
+    mapSessionConfiguredToState,
     resetRecognitionVGFState
 } from './vgf-recognition-mapper.js';
 import {
@@ -11,6 +13,13 @@ import {
     TranscriptionStatus,
     RecognitionActionProcessingState
 } from './vgf-recognition-state.js';
+import {
+    ASRApiType,
+    DetectionTypeV1,
+    RecognitionResultTypeV1,
+    SessionConfiguredV1,
+    TranscriptionResultV1
+} from '@recog/shared-types';
 
 describe('resetRecognitionVGFState', () => {
     it('should generate a new UUID', () => {
@@ -75,4 +84,138 @@ describe('resetRecognitionVGFState', () => {
         expect(newState.startRecordingStatus).toBe(RecordingStatus.READY);
         expect(newState.recognitionActionProcessingState).toBe(RecognitionActionProcessingState.NOT_STARTED);
     });
+
+    it('should preserve prompt input fields and clear detections', () => {
+        const originalState: RecognitionState = {
+            audioUtteranceId: 'old-uuid-123',
+            pendingTranscript: '',
+            promptSTT: 'hello,world',
+            promptSTF: 'map to play()',
+            promptTTF: 'extract title,artist',
+            detections: [
+                { type: DetectionTypeV1.SEARCH, query: 'one time', score: 0.92 }
+            ]
+        };
+
+        const newState = resetRecognitionVGFState(originalState);
+
+        expect(newState.promptSTT).toBe('hello,world');
+        expect(newState.promptSTF).toBe('map to play()');
+        expect(newState.promptTTF).toBe('extract title,artist');
+        expect(newState.detections).toBeUndefined();
+    });
+});
+
+describe('mapTranscriptionResultToState detections', () => {
+    const baseState: RecognitionState = {
+        audioUtteranceId: 'utt-1',
+        pendingTranscript: ''
+    };
+
+    it('should copy detections from a pending transcript result', () => {
+        const result: TranscriptionResultV1 = {
+            type: RecognitionResultTypeV1.TRANSCRIPTION,
+            audioUtteranceId: 'utt-1',
+            finalTranscript: '',
+            finalTranscriptRaw: '',
+            pendingTranscript: 'one',
+            is_finished: false,
+            detections: [
+                { type: DetectionTypeV1.SEARCH, query: 'one time', score: 0.81, startMs: 100, endMs: 900 }
+            ]
+        };
+
+        const newState = mapTranscriptionResultToState(baseState, result, true);
+
+        expect(newState.detections).toEqual([
+            { type: DetectionTypeV1.SEARCH, query: 'one time', score: 0.81, startMs: 100, endMs: 900 }
+        ]);
+    });
+
+    it('should copy detections from a final transcript result', () => {
+        const result: TranscriptionResultV1 = {
+            type: RecognitionResultTypeV1.TRANSCRIPTION,
+            audioUtteranceId: 'utt-1',
+            finalTranscript: 'one time',
+            finalTranscriptRaw: 'one time',
+            is_finished: true,
+            detections: [
+                { type: DetectionTypeV1.SEARCH, query: 'one time', score: 0.93 }
+            ]
+        };
+
+        const newState = mapTranscriptionResultToState(baseState, result, false);
+
+        expect(newState.detections).toHaveLength(1);
+        expect(newState.detections?.[0]?.query).toBe('one time');
+    });
+
+    it('should mirror SessionConfiguredV1 onto state and clear on reset', () => {
+        const sessionConfigured: SessionConfiguredV1 = {
+            type: RecognitionResultTypeV1.SESSION_CONFIGURED,
+            audioUtteranceId: 'utt-1',
+            provider: 'deepgram',
+            model: 'nova-3',
+            sampleRate: 16000,
+            encoding: 'linear16',
+            apiType: ASRApiType.STREAMING,
+            isFallback: false
+        };
+        const afterConfigured = mapSessionConfiguredToState(baseState, sessionConfigured);
+        expect(afterConfigured.sessionConfigured?.provider).toBe('deepgram');
+        expect(afterConfigured.sessionConfigured?.model).toBe('nova-3');
+        expect(afterConfigured.sessionConfigured?.isFallback).toBe(false);
+
+        const afterReset = resetRecognitionVGFState(afterConfigured);
+        expect(afterReset.sessionConfigured).toBeUndefined();
+    });
+
+    it('should copy accumulatedAudioTimeMs on pending and final transcripts and clear on reset', () => {
+        const pending: TranscriptionResultV1 = {
+            type: RecognitionResultTypeV1.TRANSCRIPTION,
+            audioUtteranceId: 'utt-1',
+            finalTranscript: '',
+            finalTranscriptRaw: '',
+            pendingTranscript: 'hi',
+            is_finished: false,
+            accumulatedAudioTimeMs: 1234
+        };
+        const afterPending = mapTranscriptionResultToState(baseState, pending, true);
+        expect(afterPending.accumulatedAudioTimeMs).toBe(1234);
+
+        const final: TranscriptionResultV1 = {
+            type: RecognitionResultTypeV1.TRANSCRIPTION,
+            audioUtteranceId: 'utt-1',
+            finalTranscript: 'hi',
+            finalTranscriptRaw: 'hi',
+            is_finished: true,
+            accumulatedAudioTimeMs: 5678
+        };
+        const afterFinal = mapTranscriptionResultToState(afterPending, final, false);
+        expect(afterFinal.accumulatedAudioTimeMs).toBe(5678);
+
+        const afterReset = resetRecognitionVGFState(afterFinal);
+        expect(afterReset.accumulatedAudioTimeMs).toBeUndefined();
+    });
+
+    it('should leave existing detections untouched when the result omits them', () => {
+        const stateWithDetections: RecognitionState = {
+            ...baseState,
+            detections: [{ type: DetectionTypeV1.SEARCH, query: 'prev', score: 0.5 }]
+        };
+        const result: TranscriptionResultV1 = {
+            type: RecognitionResultTypeV1.TRANSCRIPTION,
+            audioUtteranceId: 'utt-1',
+            finalTranscript: '',
+            finalTranscriptRaw: '',
+            pendingTranscript: 'hello',
+            is_finished: false
+        };
+
+        const newState = mapTranscriptionResultToState(stateWithDetections, result, true);
+
+        expect(newState.detections).toEqual([
+            { type: DetectionTypeV1.SEARCH, query: 'prev', score: 0.5 }
+        ]);
+    });
 });

package/src/vgf-recognition-mapper.ts

@@ -19,7 +19,7 @@ import {
 } from './recognition-client.types.js';
 import {
     TranscriptionResultV1,
-    ErrorResultV1
+    SessionConfiguredV1
 } from '@recog/shared-types';
 
 /**
@@ -94,6 +94,9 @@ export function mapTranscriptionResultToState(
         if (result.lastNonSilence !== undefined) {
             newState.lastNonSilence = result.lastNonSilence;
         }
+        if (result.accumulatedAudioTimeMs !== undefined) {
+            newState.accumulatedAudioTimeMs = result.accumulatedAudioTimeMs;
+        }
     } else {
         // Transcription is finished
         newState.transcriptionStatus = TranscriptionStatus.FINALIZED;
@@ -110,21 +113,46 @@ export function mapTranscriptionResultToState(
         if (result.lastNonSilence !== undefined) {
             newState.lastNonSilence = result.lastNonSilence;
         }
+        if (result.accumulatedAudioTimeMs !== undefined) {
+            newState.accumulatedAudioTimeMs = result.accumulatedAudioTimeMs;
+        }
 
         // Clear pending when we have final
         newState.pendingTranscript = "";
         newState.pendingConfidence = undefined;
     }
 
+    // Mirror provider-reported detections (currently Deepgram `search`; future entries
+    // may include keywords/keyterms/speech_contexts from other providers).
+    // Server pre-sorts by score descending, so [0] is the top hit — passed through as-is.
+    if (result.detections !== undefined) {
+        newState.detections = result.detections;
+    }
+
     return newState;
 }
 
+/**
+ * Mirrors the SessionConfiguredV1 message onto the VGF state.
+ * Carries the resolved provider/model/sampleRate/encoding/apiType/isFallback
+ * that the server actually chose (after circuit-breaker/fallback). Fires once
+ * per session, before audio streaming begins.
+ */
+export function mapSessionConfiguredToState(
+    currentState: RecognitionState,
+    sessionConfigured: SessionConfiguredV1
+): RecognitionState {
+    return {
+        ...currentState,
+        sessionConfigured
+    };
+}
+
 /**
  * Maps error to state
  */
 export function mapErrorToState(
-    currentState: RecognitionState,
-    error: ErrorResultV1
+    currentState: RecognitionState
 ): RecognitionState {
     return {
         ...currentState,
@@ -185,7 +213,10 @@ export function resetRecognitionVGFState(currentState: RecognitionState): Recogn
         recognitionActionProcessingState: RecognitionActionProcessingState.NOT_STARTED,
         finalTranscript: undefined,
         voiceEnd: undefined,
-        lastNonSilence: undefined
+        lastNonSilence: undefined,
+        accumulatedAudioTimeMs: undefined,
+        detections: undefined,
+        sessionConfigured: undefined
     };
 }
 
@@ -199,47 +230,6 @@ export function updateStateOnReady(currentState: RecognitionState): RecognitionS
     };
 }
 
-/**
- * Parses function call from transcript (STEP 3 support)
- * This is a placeholder - actual implementation would use NLP/LLM
- */
-export function extractFunctionCallFromTranscript(
-    transcript: string,
-    gameContext?: any
-): { metadata?: string; confidence?: number } | null {
-    // This would be replaced with actual function call extraction logic
-    // For example, using an LLM to parse intent from the transcript
-    // and map it to game actions
-
-    // Example stub implementation:
-    const lowerTranscript = transcript.toLowerCase();
-
-    // Simple pattern matching for demo
-    if (lowerTranscript.includes("play") && lowerTranscript.includes("artist")) {
-        return {
-            metadata: JSON.stringify({ action: "play", target: "artist" }),
-            confidence: 0.8
-        };
-    }
-
-    return null;
-}
-
-/**
- * Updates state with function call results (STEP 3)
- */
-export function updateStateWithFunctionCall(
-    currentState: RecognitionState,
-    functionCall: { metadata?: string; confidence?: number }
-): RecognitionState {
-    return {
-        ...currentState,
-        functionCallMetadata: functionCall.metadata,
-        functionCallConfidence: functionCall.confidence,
-        finalFunctionCallTimestamp: new Date().toISOString()
-    };
-}
-
 // Helper function to generate UUID (simplified version)
 function generateUUID(): string {
     return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {

package/src/vgf-recognition-state.ts

@@ -1,4 +1,5 @@
 import { z } from "zod"
+import { DetectionV1Schema, SessionConfiguredSchemaV1 } from "@recog/shared-types"
 
 /**
  * VGF-style state schema for game-side recognition state/results management.
@@ -25,9 +26,13 @@ export const RecognitionVGFStateSchema = z.object({
     // Voice timing (ms from stream start, prefix-adjusted)
     voiceEnd: z.number().optional(), // voice end time identified by ASR
     lastNonSilence: z.number().optional(), // last non-silence sample time from PCM analysis
+    accumulatedAudioTimeMs: z.number().optional(), // total user audio time watermark (ms) — mirrors TranscriptionResultV1.accumulatedAudioTimeMs
 
     // Tracking-only metadata
-    asrConfig: z.string().optional(), // Json format of the ASR config
+    asrConfig: z.string().optional(), // Json format of the *requested* ASR config (set once at construction).
+    // For the *resolved* truth — actual provider/model/sampleRate/encoding/apiType/isFallback chosen by the
+    // server after circuit-breaker/fallback — see `sessionConfigured` below.
+    sessionConfigured: SessionConfiguredSchemaV1.optional(), // Mirrors the SessionConfiguredV1 message; populated when the server emits it (before audio streams).
     startRecordingTimestamp: z.string().optional(), // Start of recording. Immutable after set.
     finalRecordingTimestamp: z.string().optional(), // End of recording. Immutable after set. Transcription may still be in progress.
     finalTranscriptionTimestamp: z.string().optional(), // When the final transcript was produced. Immutable after set.
@@ -44,6 +49,19 @@ export const RecognitionVGFStateSchema = z.object({
     // Support for prompt slot mapping - passed to recognition context when present
     promptSlotMap: z.record(z.string(), z.array(z.string())).optional(), // Optional map of slot names to prompt values for recognition context
 
+    // Optional prompt inputs - when set, forwarded into GameContext at client creation.
+    // Mirror the GameContextV1 fields: STT (ASR keywords/keyterms), STF (speech->function), TTF (text->function).
+    promptSTT: z.string().optional(),
+    promptSTF: z.string().optional(),
+    promptTTF: z.string().optional(),
+
+    // Provider-reported phrase detections from the last transcript message.
+    // Mirrors TranscriptionResultV1.detections — a heterogeneous list keyed by DetectionTypeV1
+    // (today only 'search' from Deepgram; future entries may include keywords/keyterms/speech_contexts).
+    // Sorted by `score` descending by the server (see deepgram/message-handlers/v1/transform-transcript.ts
+    // and provider-to-recognition-transformer.ts), so [0] is the top hit — no client-side re-rank needed.
+    detections: z.array(DetectionV1Schema).optional(),
+
     // Recognition action processing state - managed externally, SDK preserves but never modifies
     recognitionActionProcessingState: z.string().optional(), // "NOT_STARTED", "IN_PROGRESS", "COMPLETED"
 })