@volley/recognition-client-sdk 0.1.799 → 0.1.800

package/dist/browser.bundled.d.ts

@@ -1406,6 +1406,11 @@ interface IRecognitionClientConfig {
  *
  * Main interface for real-time speech recognition clients.
  * Provides methods for connection management, audio streaming, and session control.
+ *
+ * NOTE for maintainers: `ISimplifiedVGFRecognitionClient` extends this interface,
+ * so any method added here must also be implemented (typically as a delegate) by
+ * `SimplifiedVGFRecognitionClient`. TypeScript will flag missing delegates at
+ * compile time — do not work around the error, add the delegate.
  */
 interface IRecognitionClient {
     /**
@@ -1437,6 +1442,16 @@ interface IRecognitionClient {
      * @param sourceSampleRate - Source sample rate in Hz (e.g. 44100, 48000).
      */
     sendAudioWithSampleRate(audioData: ArrayBuffer | ArrayBufferView | Blob, sourceSampleRate: number): void;
+    /**
+     * Send prefix audio (e.g. a TTS prompt) that primes the provider's language
+     * model before user audio is streamed. Chunks accepted — the server buffers
+     * until the session is READY and flushes. Must be sent BEFORE the first
+     * `sendAudio()` to take effect. Only meaningful when
+     * `asrRequestConfig.prefixMode === PrefixMode.CLIENT`.
+     *
+     * @param audioData - PCM audio data as ArrayBuffer, typed array view, or Blob
+     */
+    sendPrefixAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
     /**
      * Stop recording and wait for final transcript
      * The server will close the connection after sending the final transcript.

package/dist/index.bundled.d.ts

@@ -2175,6 +2175,11 @@ interface IRecognitionClientConfig {
  *
  * Main interface for real-time speech recognition clients.
  * Provides methods for connection management, audio streaming, and session control.
+ *
+ * NOTE for maintainers: `ISimplifiedVGFRecognitionClient` extends this interface,
+ * so any method added here must also be implemented (typically as a delegate) by
+ * `SimplifiedVGFRecognitionClient`. TypeScript will flag missing delegates at
+ * compile time — do not work around the error, add the delegate.
  */
 interface IRecognitionClient {
     /**
@@ -2206,6 +2211,16 @@ interface IRecognitionClient {
      * @param sourceSampleRate - Source sample rate in Hz (e.g. 44100, 48000).
      */
     sendAudioWithSampleRate(audioData: ArrayBuffer | ArrayBufferView | Blob, sourceSampleRate: number): void;
+    /**
+     * Send prefix audio (e.g. a TTS prompt) that primes the provider's language
+     * model before user audio is streamed. Chunks accepted — the server buffers
+     * until the session is READY and flushes. Must be sent BEFORE the first
+     * `sendAudio()` to take effect. Only meaningful when
+     * `asrRequestConfig.prefixMode === PrefixMode.CLIENT`.
+     *
+     * @param audioData - PCM audio data as ArrayBuffer, typed array view, or Blob
+     */
+    sendPrefixAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
     /**
      * Stop recording and wait for final transcript
      * The server will close the connection after sending the final transcript.
@@ -2746,7 +2761,42 @@ declare const RecognitionVGFStateSchema: z.ZodObject<{
     finalConfidence: z.ZodOptional<z.ZodNumber>;
     voiceEnd: z.ZodOptional<z.ZodNumber>;
     lastNonSilence: z.ZodOptional<z.ZodNumber>;
+    accumulatedAudioTimeMs: z.ZodOptional<z.ZodNumber>;
     asrConfig: z.ZodOptional<z.ZodString>;
+    sessionConfigured: z.ZodOptional<z.ZodObject<{
+        type: z.ZodLiteral<RecognitionResultTypeV1.SESSION_CONFIGURED>;
+        audioUtteranceId: z.ZodString;
+        provider: z.ZodOptional<z.ZodString>;
+        model: z.ZodOptional<z.ZodString>;
+        sampleRate: z.ZodOptional<z.ZodNumber>;
+        encoding: z.ZodOptional<z.ZodString>;
+        apiType: z.ZodOptional<z.ZodNativeEnum<typeof ASRApiType>>;
+        isFallback: z.ZodOptional<z.ZodBoolean>;
+        asrRequest: z.ZodOptional<z.ZodString>;
+        providerConfig: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        type: RecognitionResultTypeV1.SESSION_CONFIGURED;
+        audioUtteranceId: string;
+        provider?: string | undefined;
+        model?: string | undefined;
+        sampleRate?: number | undefined;
+        encoding?: string | undefined;
+        apiType?: ASRApiType | undefined;
+        isFallback?: boolean | undefined;
+        asrRequest?: string | undefined;
+        providerConfig?: string | undefined;
+    }, {
+        type: RecognitionResultTypeV1.SESSION_CONFIGURED;
+        audioUtteranceId: string;
+        provider?: string | undefined;
+        model?: string | undefined;
+        sampleRate?: number | undefined;
+        encoding?: string | undefined;
+        apiType?: ASRApiType | undefined;
+        isFallback?: boolean | undefined;
+        asrRequest?: string | undefined;
+        providerConfig?: string | undefined;
+    }>>;
     startRecordingTimestamp: z.ZodOptional<z.ZodString>;
     finalRecordingTimestamp: z.ZodOptional<z.ZodString>;
     finalTranscriptionTimestamp: z.ZodOptional<z.ZodString>;
@@ -2756,6 +2806,28 @@ declare const RecognitionVGFStateSchema: z.ZodObject<{
     functionCallConfidence: z.ZodOptional<z.ZodNumber>;
     finalFunctionCallTimestamp: z.ZodOptional<z.ZodString>;
     promptSlotMap: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString, "many">>>;
+    promptSTT: z.ZodOptional<z.ZodString>;
+    promptSTF: z.ZodOptional<z.ZodString>;
+    promptTTF: z.ZodOptional<z.ZodString>;
+    detections: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        type: z.ZodNativeEnum<typeof DetectionTypeV1>;
+        query: z.ZodString;
+        score: z.ZodNumber;
+        startMs: z.ZodOptional<z.ZodNumber>;
+        endMs: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        type: DetectionTypeV1;
+        query: string;
+        score: number;
+        startMs?: number | undefined;
+        endMs?: number | undefined;
+    }, {
+        type: DetectionTypeV1;
+        query: string;
+        score: number;
+        startMs?: number | undefined;
+        endMs?: number | undefined;
+    }>, "many">>;
     recognitionActionProcessingState: z.ZodOptional<z.ZodString>;
 }, "strip", z.ZodTypeAny, {
     audioUtteranceId: string;
@@ -2766,7 +2838,20 @@ declare const RecognitionVGFStateSchema: z.ZodObject<{
     finalConfidence?: number | undefined;
     voiceEnd?: number | undefined;
     lastNonSilence?: number | undefined;
+    accumulatedAudioTimeMs?: number | undefined;
     asrConfig?: string | undefined;
+    sessionConfigured?: {
+        type: RecognitionResultTypeV1.SESSION_CONFIGURED;
+        audioUtteranceId: string;
+        provider?: string | undefined;
+        model?: string | undefined;
+        sampleRate?: number | undefined;
+        encoding?: string | undefined;
+        apiType?: ASRApiType | undefined;
+        isFallback?: boolean | undefined;
+        asrRequest?: string | undefined;
+        providerConfig?: string | undefined;
+    } | undefined;
     startRecordingTimestamp?: string | undefined;
     finalRecordingTimestamp?: string | undefined;
     finalTranscriptionTimestamp?: string | undefined;
@@ -2775,6 +2860,16 @@ declare const RecognitionVGFStateSchema: z.ZodObject<{
     functionCallConfidence?: number | undefined;
     finalFunctionCallTimestamp?: string | undefined;
     promptSlotMap?: Record<string, string[]> | undefined;
+    promptSTT?: string | undefined;
+    promptSTF?: string | undefined;
+    promptTTF?: string | undefined;
+    detections?: {
+        type: DetectionTypeV1;
+        query: string;
+        score: number;
+        startMs?: number | undefined;
+        endMs?: number | undefined;
+    }[] | undefined;
     recognitionActionProcessingState?: string | undefined;
 }, {
     audioUtteranceId: string;
@@ -2784,7 +2879,20 @@ declare const RecognitionVGFStateSchema: z.ZodObject<{
     finalConfidence?: number | undefined;
     voiceEnd?: number | undefined;
     lastNonSilence?: number | undefined;
+    accumulatedAudioTimeMs?: number | undefined;
     asrConfig?: string | undefined;
+    sessionConfigured?: {
+        type: RecognitionResultTypeV1.SESSION_CONFIGURED;
+        audioUtteranceId: string;
+        provider?: string | undefined;
+        model?: string | undefined;
+        sampleRate?: number | undefined;
+        encoding?: string | undefined;
+        apiType?: ASRApiType | undefined;
+        isFallback?: boolean | undefined;
+        asrRequest?: string | undefined;
+        providerConfig?: string | undefined;
+    } | undefined;
     startRecordingTimestamp?: string | undefined;
     finalRecordingTimestamp?: string | undefined;
     finalTranscriptionTimestamp?: string | undefined;
@@ -2794,6 +2902,16 @@ declare const RecognitionVGFStateSchema: z.ZodObject<{
     functionCallConfidence?: number | undefined;
     finalFunctionCallTimestamp?: string | undefined;
     promptSlotMap?: Record<string, string[]> | undefined;
+    promptSTT?: string | undefined;
+    promptSTF?: string | undefined;
+    promptTTF?: string | undefined;
+    detections?: {
+        type: DetectionTypeV1;
+        query: string;
+        score: number;
+        startMs?: number | undefined;
+        endMs?: number | undefined;
+    }[] | undefined;
     recognitionActionProcessingState?: string | undefined;
 }>;
 type RecognitionState = z.infer<typeof RecognitionVGFStateSchema>;
@@ -2843,102 +2961,22 @@ interface SimplifiedVGFClientConfig extends IRecognitionClientConfig {
 /**
  * Interface for SimplifiedVGFRecognitionClient
  *
- * A simplified client that maintains VGF state for game developers.
- * All methods from the underlying client are available, plus VGF state management.
+ * Inherits the full IRecognitionClient surface (connect, sendAudio,
+ * sendAudioWithSampleRate, sendPrefixAudio, stopRecording, stopAbnormally,
+ * status checks, sendGameContext, getStats, getUrl, getState, getAudioUtteranceId)
+ * — see recognition-client.types.ts for those. Adds VGF-specific state access.
+ *
+ * Extending IRecognitionClient (rather than redeclaring methods) means
+ * TypeScript catches any base-client method that's not delegated by the
+ * VGF wrapper at compile time — keeps the two surfaces in sync.
  */
-interface ISimplifiedVGFRecognitionClient {
+interface ISimplifiedVGFRecognitionClient extends IRecognitionClient {
     /**
-     * Connect to the recognition service WebSocket
-     * @returns Promise that resolves when connected and ready
-     */
-    connect(): Promise<void>;
-    /**
-     * Send audio data for transcription
-     * @param audioData - PCM audio data as ArrayBuffer, typed array, or Blob
-     */
-    sendAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
-    /**
-     * Send PCM16 mono audio captured at `sourceSampleRate`; the SDK
-     * downsamples to the session's target rate before transmitting. Use
-     * when capture is at the system's native rate (browser AudioContext is
-     * typically 44.1 kHz or 48 kHz). Audio must be signed 16-bit
-     * little-endian PCM, mono.
-     */
-    sendAudioWithSampleRate(audioData: ArrayBuffer | ArrayBufferView | Blob, sourceSampleRate: number): void;
-    /**
-     * Stop recording and wait for final transcription
-     * @returns Promise that resolves when transcription is complete
-     */
-    stopRecording(): Promise<void>;
-    /**
-     * Force stop and immediately close connection without waiting for server
-     *
-     * WARNING: This is an abnormal shutdown that bypasses the graceful stop flow:
-     * - Does NOT wait for server to process remaining audio
-     * - Does NOT receive final transcript from server (VGF state set to empty)
-     * - Immediately closes WebSocket connection
-     * - Cleans up resources (buffers, listeners)
-     *
-     * Use Cases:
-     * - User explicitly cancels/abandons the session
-     * - Timeout scenarios where waiting is not acceptable
-     * - Need immediate cleanup and can't wait for server
-     *
-     * RECOMMENDED: Use stopRecording() for normal shutdown.
-     * Only use this when immediate disconnection is required.
-     */
-    stopAbnormally(): void;
-    /**
-     * Get the current VGF recognition state
+     * Get the current VGF recognition state — the single shared store
+     * of inputs and outputs for this utterance.
      * @returns Current RecognitionState with all transcription data
      */
     getVGFState(): RecognitionState;
-    /**
-     * Check if connected to the WebSocket
-     */
-    isConnected(): boolean;
-    /**
-     * Check if currently connecting
-     */
-    isConnecting(): boolean;
-    /**
-     * Check if currently stopping
-     */
-    isStopping(): boolean;
-    /**
-     * Check if transcription has finished
-     */
-    isTranscriptionFinished(): boolean;
-    /**
-     * Check if the audio buffer has overflowed
-     */
-    isBufferOverflowing(): boolean;
-    /**
-     * Send game context after connection is established (for preconnect flow).
-     *
-     * Preconnect flow: Create client with asrRequestConfig (useContext: true) but
-     * WITHOUT gameContext → call connect() → later call sendGameContext() with slotMap.
-     *
-     * @param context - Game context including slotMap for keyword boosting
-     */
-    sendGameContext(context: GameContextV1): void;
-    /**
-     * Check if server has sent READY signal (provider connected, ready for audio).
-     * In preconnect flow, this becomes true after sendGameContext() triggers provider attachment.
-     */
-    isServerReady(): boolean;
-    /**
-     * Get the audio utterance ID for this session
-     */
-    getAudioUtteranceId(): string;
-    /**
-     * Get the WebSocket URL being used
-     */
-    getUrl(): string;
-    /**
-     * Get the underlying client state (for advanced usage)
-     */
-    getState(): ClientState;
 }
 /**
  * This wrapper ONLY maintains VGF state as a sink.
@@ -2956,6 +2994,8 @@ declare class SimplifiedVGFRecognitionClient implements ISimplifiedVGFRecognitio
     connect(): Promise<void>;
     sendAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
     sendAudioWithSampleRate(audioData: ArrayBuffer | ArrayBufferView | Blob, sourceSampleRate: number): void;
+    sendPrefixAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
+    getStats(): IRecognitionClientStats;
     /**
      * Set VGF recording status to RECORDING on the first audio chunk.
      * Idempotent — subsequent calls are no-ops until disconnect/stop resets

package/dist/index.js

@@ -6435,9 +6435,15 @@ var RecognitionVGFStateSchema = z.object({
   // voice end time identified by ASR
   lastNonSilence: z.number().optional(),
   // last non-silence sample time from PCM analysis
+  accumulatedAudioTimeMs: z.number().optional(),
+  // total user audio time watermark (ms) — mirrors TranscriptionResultV1.accumulatedAudioTimeMs
   // Tracking-only metadata
   asrConfig: z.string().optional(),
-  // Json format of the ASR config
+  // Json format of the *requested* ASR config (set once at construction).
+  // For the *resolved* truth — actual provider/model/sampleRate/encoding/apiType/isFallback chosen by the
+  // server after circuit-breaker/fallback — see `sessionConfigured` below.
+  sessionConfigured: SessionConfiguredSchemaV1.optional(),
+  // Mirrors the SessionConfiguredV1 message; populated when the server emits it (before audio streams).
   startRecordingTimestamp: z.string().optional(),
   // Start of recording. Immutable after set.
   finalRecordingTimestamp: z.string().optional(),
@@ -6458,6 +6464,17 @@ var RecognitionVGFStateSchema = z.object({
   // Support for prompt slot mapping - passed to recognition context when present
   promptSlotMap: z.record(z.string(), z.array(z.string())).optional(),
   // Optional map of slot names to prompt values for recognition context
+  // Optional prompt inputs - when set, forwarded into GameContext at client creation.
+  // Mirror the GameContextV1 fields: STT (ASR keywords/keyterms), STF (speech->function), TTF (text->function).
+  promptSTT: z.string().optional(),
+  promptSTF: z.string().optional(),
+  promptTTF: z.string().optional(),
+  // Provider-reported phrase detections from the last transcript message.
+  // Mirrors TranscriptionResultV1.detections — a heterogeneous list keyed by DetectionTypeV1
+  // (today only 'search' from Deepgram; future entries may include keywords/keyterms/speech_contexts).
+  // Sorted by `score` descending by the server (see deepgram/message-handlers/v1/transform-transcript.ts
+  // and provider-to-recognition-transformer.ts), so [0] is the top hit — no client-side re-rank needed.
+  detections: z.array(DetectionV1Schema).optional(),
   // Recognition action processing state - managed externally, SDK preserves but never modifies
   recognitionActionProcessingState: z.string().optional()
   // "NOT_STARTED", "IN_PROGRESS", "COMPLETED"
@@ -6529,6 +6546,9 @@ function mapTranscriptionResultToState(currentState, result, isRecording) {
     if (result.lastNonSilence !== void 0) {
       newState.lastNonSilence = result.lastNonSilence;
     }
+    if (result.accumulatedAudioTimeMs !== void 0) {
+      newState.accumulatedAudioTimeMs = result.accumulatedAudioTimeMs;
+    }
   } else {
     newState.transcriptionStatus = TranscriptionStatus.FINALIZED;
     newState.finalTranscript = result.finalTranscript || "";
@@ -6542,12 +6562,24 @@ function mapTranscriptionResultToState(currentState, result, isRecording) {
     if (result.lastNonSilence !== void 0) {
       newState.lastNonSilence = result.lastNonSilence;
     }
+    if (result.accumulatedAudioTimeMs !== void 0) {
+      newState.accumulatedAudioTimeMs = result.accumulatedAudioTimeMs;
+    }
     newState.pendingTranscript = "";
     newState.pendingConfidence = void 0;
   }
+  if (result.detections !== void 0) {
+    newState.detections = result.detections;
+  }
   return newState;
 }
-function mapErrorToState(currentState, error) {
+function mapSessionConfiguredToState(currentState, sessionConfigured) {
+  return {
+    ...currentState,
+    sessionConfigured
+  };
+}
+function mapErrorToState(currentState) {
   return {
     ...currentState,
     transcriptionStatus: TranscriptionStatus.ERROR,
@@ -6579,7 +6611,10 @@ function resetRecognitionVGFState(currentState) {
     recognitionActionProcessingState: RecognitionActionProcessingState.NOT_STARTED,
     finalTranscript: void 0,
     voiceEnd: void 0,
-    lastNonSilence: void 0
+    lastNonSilence: void 0,
+    accumulatedAudioTimeMs: void 0,
+    detections: void 0,
+    sessionConfigured: void 0
   };
 }
 function generateUUID() {
@@ -6624,16 +6659,28 @@ var SimplifiedVGFRecognitionClient = class {
     }
     this.state = { ...this.state, startRecordingStatus: "READY" };
     this.expectedUuid = this.state.audioUtteranceId;
-    if (this.state.promptSlotMap) {
+    const hasPromptInputs = this.state.promptSlotMap !== void 0 || this.state.promptSTT !== void 0 || this.state.promptSTF !== void 0 || this.state.promptTTF !== void 0;
+    if (hasPromptInputs) {
       if (clientConfig.asrRequestConfig) {
         clientConfig.asrRequestConfig.useContext = true;
       }
       if (!clientConfig.gameContext) {
         if (clientConfig.logger) {
-          clientConfig.logger("warn", "[VGF] promptSlotMap found but no gameContext provided. SlotMap will not be sent.");
+          clientConfig.logger("warn", "[VGF] prompt inputs found but no gameContext provided. They will not be sent.");
         }
       } else {
-        clientConfig.gameContext.slotMap = this.state.promptSlotMap;
+        if (this.state.promptSlotMap !== void 0) {
+          clientConfig.gameContext.slotMap = this.state.promptSlotMap;
+        }
+        if (this.state.promptSTT !== void 0) {
+          clientConfig.gameContext.promptSTT = this.state.promptSTT;
+        }
+        if (this.state.promptSTF !== void 0) {
+          clientConfig.gameContext.promptSTF = this.state.promptSTF;
+        }
+        if (this.state.promptTTF !== void 0) {
+          clientConfig.gameContext.promptTTF = this.state.promptTTF;
+        }
       }
     }
     this.client = new RealTimeTwoWayWebSocketRecognitionClient({
@@ -6669,6 +6716,22 @@ var SimplifiedVGFRecognitionClient = class {
           clientConfig.onMetadata(metadata);
         }
       },
+      onSessionConfigured: (sessionConfigured) => {
+        if (sessionConfigured.audioUtteranceId && sessionConfigured.audioUtteranceId !== this.expectedUuid) {
+          if (this.logger) {
+            this.logger(
+              "warn",
+              `[RecogSDK:VGF] Skipping sessionConfigured update: UUID mismatch (expected: ${this.expectedUuid}, got: ${sessionConfigured.audioUtteranceId})`
+            );
+          }
+          return;
+        }
+        this.state = mapSessionConfiguredToState(this.state, sessionConfigured);
+        this.notifyStateChange();
+        if (clientConfig.onSessionConfigured) {
+          clientConfig.onSessionConfigured(sessionConfigured);
+        }
+      },
       onFunctionCall: (result) => {
         if (clientConfig.onFunctionCall) {
           clientConfig.onFunctionCall(result);
@@ -6685,7 +6748,7 @@ var SimplifiedVGFRecognitionClient = class {
           return;
         }
         this.isRecordingAudio = false;
-        this.state = mapErrorToState(this.state, error);
+        this.state = mapErrorToState(this.state);
         this.notifyStateChange();
         if (clientConfig.onError) {
           clientConfig.onError(error);
@@ -6717,6 +6780,12 @@ var SimplifiedVGFRecognitionClient = class {
     this.markRecordingStarted();
     this.client.sendAudioWithSampleRate(audioData, sourceSampleRate);
   }
+  sendPrefixAudio(audioData) {
+    this.client.sendPrefixAudio(audioData);
+  }
+  getStats() {
+    return this.client.getStats();
+  }
   /**
    * Set VGF recording status to RECORDING on the first audio chunk.
    * Idempotent — subsequent calls are no-ops until disconnect/stop resets