@volley/recognition-client-sdk 0.1.767 → 0.1.799

package/README.md

@@ -152,9 +152,12 @@ builder
   .onError(error => {})            // Handle errors
   .onConnected(() => {})           // Connection established
   .onDisconnected((code) => {})   // Connection closed
-  .onMetadata(meta => {})          // Timing information
+  .onMetadata(meta => {})          // Timing information + final audio metrics (always-on)
+  .onAudioMetrics(m => {})         // Live audio-quality metrics (opt-in, since 0.1.767)
 ```
 
+> **Audio metrics**: every session delivers a final `audioMetrics` snapshot embedded in `Metadata` (volume, silence ratio, clipping, SNR — all derived from PCM, not from the ASR provider). To also receive live per-chunk updates while audio is flowing, set `asrRequestConfig.audioMetricsIntervalMs > 0` and register `.onAudioMetrics()`. Available in SDK **≥ 0.1.767**. See [audio-metrics-alpha.md](https://github.com/Volley-Inc/recognition-service/blob/dev/docs/design/functional-features/observability/audio-metrics-alpha.md) for the full schema, and the repo-root [CHANGELOG.md](https://github.com/Volley-Inc/recognition-service/blob/dev/CHANGELOG.md) for SDK release history.
+
 ### Optional Parameters
 
 ```typescript

package/dist/browser.bundled.d.ts

@@ -24,6 +24,7 @@ declare enum RecognitionProvider {
     BEDROCK = "bedrock",
     INWORLD_STT = "inworld-stt",
     AWS_TRANSCRIBE = "aws-transcribe",
+    AMAZON_NOVA_SONIC = "amazon-nova-sonic",
     TEST_ASR_PROVIDER_QUOTA = "test-asr-provider-quota",
     TEST_ASR_STREAMING = "test-asr-streaming"
 }
@@ -96,6 +97,7 @@ declare enum ElevenLabsModel {
  * @see https://platform.openai.com/docs/models/gpt-4o-transcribe
  */
 declare enum OpenAIRealtimeModel {
+    GPT_REALTIME_WHISPER = "gpt-realtime-whisper",
     GPT_4O_TRANSCRIBE = "gpt-4o-transcribe",
     GPT_4O_MINI_TRANSCRIBE = "gpt-4o-mini-transcribe"
 }
@@ -146,135 +148,28 @@ declare enum InworldSttModel {
 declare enum AwsTranscribeModel {
     DEFAULT = "default"
 }
+/**
+ * Amazon Nova Sonic bidirectional streaming model (Bedrock).
+ * Speech-to-speech model; we consume the USER FINAL transcript and discard the assistant text/audio output.
+ * @see https://docs.aws.amazon.com/nova/latest/userguide/speech-bidirection.html
+ */
+declare enum AmazonNovaSonicModel {
+    AMAZON_NOVA_SONIC_V1 = "amazon.nova-sonic-v1:0",
+    AMAZON_NOVA_2_SONIC = "amazon.nova-2-sonic-v1:0"
+}
 /**
  * Self-serve vLLM batch transcription models
- * Backed by recognition-inference / RunPod `/transcribe`
+ * Backed by recognition-inference / RunPod `/ws/transcribe`
  */
 declare enum SelfServeVllmModel {
+    QWEN3_ASR_0_6B = "qwen3-asr-0.6b",
+    QWEN3_ASR_0_6B_WOF_LETTER = "qwen3-asr-0.6b-wof-letter",
     QWEN3_ASR_1_7B = "qwen3-asr-1.7b"
 }
 /**
  * Type alias for any model from any provider
  */
-type RecognitionModel = DeepgramModel | GoogleModel | FireworksModel | GladiaModel | ElevenLabsModel | OpenAIRealtimeModel | MistralVoxtralModel | CartesiaModel | DashScopeModel | InworldSttModel | SelfServeVllmModel | BedrockModel | AwsTranscribeModel | string;
-
-/**
- * Audio encoding types
- */
-declare enum AudioEncoding {
-    ENCODING_UNSPECIFIED = 0,
-    LINEAR16 = 1,
-    OGG_OPUS = 2,
-    FLAC = 3,
-    MULAW = 4,
-    ALAW = 5
-}
-declare namespace AudioEncoding {
-    /**
-     * Convert numeric ID to AudioEncoding enum
-     * @param id - Numeric encoding identifier (0-5)
-     * @returns AudioEncoding enum value or undefined if invalid
-     */
-    function fromId(id: number): AudioEncoding | undefined;
-    /**
-     * Convert string name to AudioEncoding enum
-     * @param nameStr - String name like "linear16", "LINEAR16", "ogg_opus", "OGG_OPUS", etc. (case insensitive)
-     * @returns AudioEncoding enum value or undefined if invalid
-     */
-    function fromName(nameStr: string): AudioEncoding | undefined;
-    /**
-     * Convert AudioEncoding enum to numeric ID
-     * @param encoding - AudioEncoding enum value
-     * @returns Numeric ID (0-5)
-     */
-    function toId(encoding: AudioEncoding): number;
-    /**
-     * Convert AudioEncoding enum to string name
-     * @param encoding - AudioEncoding enum value
-     * @returns String name like "LINEAR16", "MULAW", etc.
-     */
-    function toName(encoding: AudioEncoding): string;
-    /**
-     * Check if a numeric ID is a valid encoding
-     * @param id - Numeric identifier to validate
-     * @returns true if valid encoding ID
-     */
-    function isIdValid(id: number): boolean;
-    /**
-     * Check if a string name is a valid encoding
-     * @param nameStr - String name to validate
-     * @returns true if valid encoding name
-     */
-    function isNameValid(nameStr: string): boolean;
-}
-/**
- * Common sample rates (in Hz)
- */
-declare enum SampleRate {
-    RATE_8000 = 8000,
-    RATE_16000 = 16000,
-    RATE_22050 = 22050,
-    RATE_24000 = 24000,
-    RATE_32000 = 32000,
-    RATE_44100 = 44100,
-    RATE_48000 = 48000
-}
-declare namespace SampleRate {
-    /**
-     * Convert Hz value to SampleRate enum
-     * @param hz - Sample rate in Hz (8000, 16000, etc.)
-     * @returns SampleRate enum value or undefined if invalid
-     */
-    function fromHz(hz: number): SampleRate | undefined;
-    /**
-     * Convert string name to SampleRate enum
-     * @param nameStr - String name like "rate_8000", "RATE_16000", etc. (case insensitive)
-     * @returns SampleRate enum value or undefined if invalid
-     */
-    function fromName(nameStr: string): SampleRate | undefined;
-    /**
-     * Convert SampleRate enum to Hz value
-     * @param rate - SampleRate enum value
-     * @returns Hz value (8000, 16000, etc.)
-     */
-    function toHz(rate: SampleRate): number;
-    /**
-     * Convert SampleRate enum to string name
-     * @param rate - SampleRate enum value
-     * @returns String name like "RATE_8000", "RATE_16000", etc.
-     */
-    function toName(rate: SampleRate): string;
-    /**
-     * Check if a numeric Hz value is a valid sample rate
-     * @param hz - Hz value to validate
-     * @returns true if valid sample rate
-     */
-    function isHzValid(hz: number): boolean;
-    /**
-     * Check if a string name is a valid sample rate
-     * @param nameStr - String name to validate
-     * @returns true if valid sample rate name
-     */
-    function isNameValid(nameStr: string): boolean;
-}
-/**
- * Supported languages for recognition
- * Using BCP-47 language tags
- */
-declare enum Language {
-    ENGLISH_US = "en-US",
-    ENGLISH_GB = "en-GB",
-    SPANISH_ES = "es-ES",
-    SPANISH_MX = "es-MX",
-    FRENCH_FR = "fr-FR",
-    GERMAN_DE = "de-DE",
-    ITALIAN_IT = "it-IT",
-    PORTUGUESE_BR = "pt-BR",
-    JAPANESE_JP = "ja-JP",
-    KOREAN_KR = "ko-KR",
-    CHINESE_CN = "zh-CN",
-    CHINESE_TW = "zh-TW"
-}
+type RecognitionModel = DeepgramModel | GoogleModel | FireworksModel | GladiaModel | ElevenLabsModel | OpenAIRealtimeModel | MistralVoxtralModel | CartesiaModel | DashScopeModel | InworldSttModel | SelfServeVllmModel | BedrockModel | AwsTranscribeModel | AmazonNovaSonicModel | string;
 
 /**
  * Recognition Result Types V1
@@ -294,6 +189,16 @@ declare enum RecognitionResultTypeV1 {
     AUDIO_METRICS = "AudioMetrics",
     SESSION_CONFIGURED = "SessionConfigured"
 }
+/**
+ * Source of a phrase detection — what kind of provider feature produced
+ * the hit. Currently only Deepgram's `search` parameter is wired up, so
+ * this enum has one value. New entries (e.g. KEYWORDS, KEYTERMS,
+ * SPEECH_CONTEXTS) get added when other providers join.
+ */
+declare enum DetectionTypeV1 {
+    /** Deepgram phonetic phrase match via the `search=…` request parameter */
+    SEARCH = "search"
+}
 /**
  * Transcription result V1 - contains transcript message
  * In the long run game side should not need to know it. In the short run it is send back to client.
@@ -318,6 +223,25 @@ declare const TranscriptionResultSchemaV1: z.ZodObject<{
     receivedAtMs: z.ZodOptional<z.ZodNumber>;
     accumulatedAudioTimeMs: z.ZodOptional<z.ZodNumber>;
     rawAudioTimeMs: z.ZodOptional<z.ZodNumber>;
+    detections: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        type: z.ZodNativeEnum<typeof DetectionTypeV1>;
+        query: z.ZodString;
+        score: z.ZodNumber;
+        startMs: z.ZodOptional<z.ZodNumber>;
+        endMs: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        type: DetectionTypeV1;
+        query: string;
+        score: number;
+        startMs?: number | undefined;
+        endMs?: number | undefined;
+    }, {
+        type: DetectionTypeV1;
+        query: string;
+        score: number;
+        startMs?: number | undefined;
+        endMs?: number | undefined;
+    }>, "many">>;
 }, "strip", z.ZodTypeAny, {
     type: RecognitionResultTypeV1.TRANSCRIPTION;
     audioUtteranceId: string;
@@ -337,6 +261,13 @@ declare const TranscriptionResultSchemaV1: z.ZodObject<{
     receivedAtMs?: number | undefined;
     accumulatedAudioTimeMs?: number | undefined;
     rawAudioTimeMs?: number | undefined;
+    detections?: {
+        type: DetectionTypeV1;
+        query: string;
+        score: number;
+        startMs?: number | undefined;
+        endMs?: number | undefined;
+    }[] | undefined;
 }, {
     type: RecognitionResultTypeV1.TRANSCRIPTION;
     audioUtteranceId: string;
@@ -356,6 +287,13 @@ declare const TranscriptionResultSchemaV1: z.ZodObject<{
     receivedAtMs?: number | undefined;
     accumulatedAudioTimeMs?: number | undefined;
     rawAudioTimeMs?: number | undefined;
+    detections?: {
+        type: DetectionTypeV1;
+        query: string;
+        score: number;
+        startMs?: number | undefined;
+        endMs?: number | undefined;
+    }[] | undefined;
 }>;
 type TranscriptionResultV1 = z.infer<typeof TranscriptionResultSchemaV1>;
 /**
@@ -667,6 +605,138 @@ declare const AudioMetricsResultSchemaV1: z.ZodObject<{
 }>;
 type AudioMetricsResultV1 = z.infer<typeof AudioMetricsResultSchemaV1>;
 
+/**
+ * Audio encoding types
+ */
+declare enum AudioEncoding {
+    ENCODING_UNSPECIFIED = 0,
+    LINEAR16 = 1,
+    OGG_OPUS = 2,
+    FLAC = 3,
+    MULAW = 4,
+    ALAW = 5
+}
+declare namespace AudioEncoding {
+    /**
+     * Convert numeric ID to AudioEncoding enum
+     * @param id - Numeric encoding identifier (0-5)
+     * @returns AudioEncoding enum value or undefined if invalid
+     */
+    function fromId(id: number): AudioEncoding | undefined;
+    /**
+     * Convert string name to AudioEncoding enum
+     * @param nameStr - String name like "linear16", "LINEAR16", "ogg_opus", "OGG_OPUS", etc. (case insensitive)
+     * @returns AudioEncoding enum value or undefined if invalid
+     */
+    function fromName(nameStr: string): AudioEncoding | undefined;
+    /**
+     * Convert AudioEncoding enum to numeric ID
+     * @param encoding - AudioEncoding enum value
+     * @returns Numeric ID (0-5)
+     */
+    function toId(encoding: AudioEncoding): number;
+    /**
+     * Convert AudioEncoding enum to string name
+     * @param encoding - AudioEncoding enum value
+     * @returns String name like "LINEAR16", "MULAW", etc.
+     */
+    function toName(encoding: AudioEncoding): string;
+    /**
+     * Check if a numeric ID is a valid encoding
+     * @param id - Numeric identifier to validate
+     * @returns true if valid encoding ID
+     */
+    function isIdValid(id: number): boolean;
+    /**
+     * Check if a string name is a valid encoding
+     * @param nameStr - String name to validate
+     * @returns true if valid encoding name
+     */
+    function isNameValid(nameStr: string): boolean;
+    /**
+     * Coerce a possibly-stringly-typed encoding value into the AudioEncoding enum.
+     *
+     * - enum / number → returned as-is (already AudioEncoding-shaped)
+     * - string (case-insensitive, e.g. 'linear16', 'LINEAR16') → converted via {@link fromName}.
+     *   Invokes `onStringInput` with a warning message so callers can route it
+     *   to their preferred logger.
+     * - invalid string → throws (preferred over silent fallback so typos surface)
+     * - undefined → defaults to {@link AudioEncoding.LINEAR16}
+     *
+     * Always normalize at the SDK / server boundary so downstream code can rely
+     * on a numeric AudioEncoding (the wire-level binary frame header is uint32).
+     */
+    function coerce(value: AudioEncoding | string | number | undefined, onStringInput?: (warning: string) => void): AudioEncoding;
+}
+/**
+ * Common sample rates (in Hz)
+ */
+declare enum SampleRate {
+    RATE_8000 = 8000,
+    RATE_16000 = 16000,
+    RATE_22050 = 22050,
+    RATE_24000 = 24000,
+    RATE_32000 = 32000,
+    RATE_44100 = 44100,
+    RATE_48000 = 48000
+}
+declare namespace SampleRate {
+    /**
+     * Convert Hz value to SampleRate enum
+     * @param hz - Sample rate in Hz (8000, 16000, etc.)
+     * @returns SampleRate enum value or undefined if invalid
+     */
+    function fromHz(hz: number): SampleRate | undefined;
+    /**
+     * Convert string name to SampleRate enum
+     * @param nameStr - String name like "rate_8000", "RATE_16000", etc. (case insensitive)
+     * @returns SampleRate enum value or undefined if invalid
+     */
+    function fromName(nameStr: string): SampleRate | undefined;
+    /**
+     * Convert SampleRate enum to Hz value
+     * @param rate - SampleRate enum value
+     * @returns Hz value (8000, 16000, etc.)
+     */
+    function toHz(rate: SampleRate): number;
+    /**
+     * Convert SampleRate enum to string name
+     * @param rate - SampleRate enum value
+     * @returns String name like "RATE_8000", "RATE_16000", etc.
+     */
+    function toName(rate: SampleRate): string;
+    /**
+     * Check if a numeric Hz value is a valid sample rate
+     * @param hz - Hz value to validate
+     * @returns true if valid sample rate
+     */
+    function isHzValid(hz: number): boolean;
+    /**
+     * Check if a string name is a valid sample rate
+     * @param nameStr - String name to validate
+     * @returns true if valid sample rate name
+     */
+    function isNameValid(nameStr: string): boolean;
+}
+/**
+ * Supported languages for recognition
+ * Using BCP-47 language tags
+ */
+declare enum Language {
+    ENGLISH_US = "en-US",
+    ENGLISH_GB = "en-GB",
+    SPANISH_ES = "es-ES",
+    SPANISH_MX = "es-MX",
+    FRENCH_FR = "fr-FR",
+    GERMAN_DE = "de-DE",
+    ITALIAN_IT = "it-IT",
+    PORTUGUESE_BR = "pt-BR",
+    JAPANESE_JP = "ja-JP",
+    KOREAN_KR = "ko-KR",
+    CHINESE_CN = "zh-CN",
+    CHINESE_TW = "zh-TW"
+}
+
 /**
  * Recognition Context Types V1
  * NOTE_TO_AI: DO NOT CHANGE THIS UNLESS EXPLICITLY ASKED. Always ask before making any changes.
@@ -852,10 +922,10 @@ interface ASRRequestConfig {
      * doesn't respond with is_final=true after stopRecording().
      *
      * - aggressive: 100ms - fast response, may cut off slow providers
-     * - balanced: 500ms - current default, good for most cases
-     * - conservative: 1000ms - wait longer for complex utterances
+     * - balanced: 500ms - good for most cases
+     * - conservative: 1000ms - current default, wait longer for complex utterances
      *
-     * @default 'balanced'
+     * @default 'conservative'
      * @see FinalTranscriptStability enum for detailed descriptions
      */
     finalTranscriptStability?: FinalTranscriptStability | string;
@@ -949,6 +1019,29 @@ interface ASRRequestConfig {
      * @example 500
      */
     audioMetricsIntervalMs?: number;
+    /**
+     * Opt-in: round-trip Deepgram `search` phrase hits into the transcript.
+     *
+     * When `true` AND the resolved provider/model is **deepgram nova-2** AND the
+     * GameContext `gamePhase` is `'Solve Puzzle'`, every Deepgram Results event
+     * with a `channel.search` hit at confidence ≥ 0.6 has the original query
+     * prepended to the transcript text delivered to the client. This restores
+     * parity with the legacy Roku→Deepgram WoF Puzzle-Solve path where the
+     * phrase round-trip lets downstream NLU match multi-word puzzle solutions
+     * even when nova-2's primary transcription drifts.
+     *
+     * Default: `false` (no prepend; transcript is whatever nova-2 produces).
+     *
+     * Scope guard rationale:
+     * - nova-2 only: nova-3 / flux do not need this (they handle phrase
+     *   spotting differently and the prepend would only add noise).
+     * - Solve-Puzzle scene only: other WoF scenes (Letter-Guess,
+     *   Bonus-Round, etc.) do NOT want the slotMap phrase prepended — only
+     *   Puzzle-Solve depends on the phrase round-trip.
+     *
+     * @default false
+     */
+    appendSearch?: boolean;
     /**
      * Optional fallback ASR configurations
      *
@@ -1327,6 +1420,23 @@ interface IRecognitionClient {
      * @param audioData - PCM audio data as ArrayBuffer, typed array view, or Blob
      */
     sendAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
+    /**
+     * Send PCM16 mono audio captured at `sourceSampleRate`; the SDK
+     * downsamples to the session's target rate (currently 16 kHz, set by the
+     * server validator) before transmitting.
+     *
+     * Use this when your capture pipeline produces audio at the system's
+     * native rate (browser `AudioContext` is typically 44.1 kHz or 48 kHz).
+     * If your audio is already at the target rate, prefer `sendAudio()` to
+     * skip the resample step.
+     *
+     * Audio must be signed 16-bit little-endian PCM, mono. Stereo must be
+     * mixed to mono by the caller.
+     *
+     * @param audioData - PCM16 mono audio at `sourceSampleRate`.
+     * @param sourceSampleRate - Source sample rate in Hz (e.g. 44100, 48000).
+     */
+    sendAudioWithSampleRate(audioData: ArrayBuffer | ArrayBufferView | Blob, sourceSampleRate: number): void;
     /**
      * Stop recording and wait for final transcript
      * The server will close the connection after sending the final transcript.
@@ -1526,6 +1636,29 @@ declare class RealTimeTwoWayWebSocketRecognitionClient extends WebSocketAudioCli
      */
     private connectWithRetry;
     sendAudio(audioData: ArrayBuffer | ArrayBufferView | Blob): void;
+    /**
+     * Send PCM16 mono audio captured at any sample rate. The SDK downsamples
+     * to the session's target rate (currently 16 kHz per server validator)
+     * before sending.
+     *
+     * Use this when your capture pipeline produces audio at the system's
+     * native rate — `AudioContext` defaults to 44.1 kHz or 48 kHz on most
+     * desktop/mobile hardware — and you don't want to bring your own
+     * resampler. If your audio is already at the target rate, prefer
+     * `sendAudio()` to skip the resample step.
+     *
+     * Algorithm: box-filter averaging (see audio-resampler.ts). Cheap, no
+     * dependencies, has a built-in low-pass effect so aliasing stays out of
+     * the speech band. Suitable for ASR; not a substitute for a high-quality
+     * resampler if you're doing music or full-fidelity processing.
+     *
+     * Audio must be signed 16-bit little-endian PCM, mono. Stereo must be
+     * mixed to mono by the caller.
+     *
+     * @param audioData - PCM16 mono audio at `sourceSampleRate`.
+     * @param sourceSampleRate - Source sample rate in Hz (e.g. 44100, 48000).
+     */
+    sendAudioWithSampleRate(audioData: ArrayBuffer | ArrayBufferView | Blob, sourceSampleRate: number): void;
     private sendAudioInternal;
     /**
      * Only active ehwne client is in READY state. otherwise it will return immediately.