@volley/recognition-client-sdk 0.1.767 → 0.1.799

package/dist/recog-client-sdk.browser.js

@@ -3749,6 +3749,7 @@ var RecognitionProvider;
   RecognitionProvider2["BEDROCK"] = "bedrock";
   RecognitionProvider2["INWORLD_STT"] = "inworld-stt";
   RecognitionProvider2["AWS_TRANSCRIBE"] = "aws-transcribe";
+  RecognitionProvider2["AMAZON_NOVA_SONIC"] = "amazon-nova-sonic";
   RecognitionProvider2["TEST_ASR_PROVIDER_QUOTA"] = "test-asr-provider-quota";
   RecognitionProvider2["TEST_ASR_STREAMING"] = "test-asr-streaming";
 })(RecognitionProvider || (RecognitionProvider = {}));
@@ -3799,6 +3800,7 @@ var ElevenLabsModel;
 })(ElevenLabsModel || (ElevenLabsModel = {}));
 var OpenAIRealtimeModel;
 (function(OpenAIRealtimeModel2) {
+  OpenAIRealtimeModel2["GPT_REALTIME_WHISPER"] = "gpt-realtime-whisper";
   OpenAIRealtimeModel2["GPT_4O_TRANSCRIBE"] = "gpt-4o-transcribe";
   OpenAIRealtimeModel2["GPT_4O_MINI_TRANSCRIBE"] = "gpt-4o-mini-transcribe";
 })(OpenAIRealtimeModel || (OpenAIRealtimeModel = {}));
@@ -3829,8 +3831,15 @@ var AwsTranscribeModel;
 (function(AwsTranscribeModel2) {
   AwsTranscribeModel2["DEFAULT"] = "default";
 })(AwsTranscribeModel || (AwsTranscribeModel = {}));
+var AmazonNovaSonicModel;
+(function(AmazonNovaSonicModel2) {
+  AmazonNovaSonicModel2["AMAZON_NOVA_SONIC_V1"] = "amazon.nova-sonic-v1:0";
+  AmazonNovaSonicModel2["AMAZON_NOVA_2_SONIC"] = "amazon.nova-2-sonic-v1:0";
+})(AmazonNovaSonicModel || (AmazonNovaSonicModel = {}));
 var SelfServeVllmModel;
 (function(SelfServeVllmModel2) {
+  SelfServeVllmModel2["QWEN3_ASR_0_6B"] = "qwen3-asr-0.6b";
+  SelfServeVllmModel2["QWEN3_ASR_0_6B_WOF_LETTER"] = "qwen3-asr-0.6b-wof-letter";
   SelfServeVllmModel2["QWEN3_ASR_1_7B"] = "qwen3-asr-1.7b";
 })(SelfServeVllmModel || (SelfServeVllmModel = {}));
 
@@ -3845,6 +3854,18 @@ var RecognitionResultTypeV1;
   RecognitionResultTypeV12["AUDIO_METRICS"] = "AudioMetrics";
   RecognitionResultTypeV12["SESSION_CONFIGURED"] = "SessionConfigured";
 })(RecognitionResultTypeV1 || (RecognitionResultTypeV1 = {}));
+var DetectionTypeV1;
+(function(DetectionTypeV12) {
+  DetectionTypeV12["SEARCH"] = "search";
+})(DetectionTypeV1 || (DetectionTypeV1 = {}));
+var DetectionV1Schema = z.object({
+  type: z.nativeEnum(DetectionTypeV1),
+  query: z.string(),
+  score: z.number().min(0).max(1),
+  startMs: z.number().optional(),
+  endMs: z.number().optional()
+  // Audio time (ms from stream start) where the hit ends
+});
 var TranscriptionResultSchemaV1 = z.object({
   type: z.literal(RecognitionResultTypeV1.TRANSCRIPTION),
   audioUtteranceId: z.string(),
@@ -3863,8 +3884,9 @@ var TranscriptionResultSchemaV1 = z.object({
   endTimestamp: z.number().optional(),
   receivedAtMs: z.number().optional(),
   accumulatedAudioTimeMs: z.number().optional(),
-  rawAudioTimeMs: z.number().optional()
-  // Total audio duration sent to provider (includes prefix)
+  rawAudioTimeMs: z.number().optional(),
+  detections: z.array(DetectionV1Schema).optional()
+  // Provider-reported phrase detections (query + score, optionally startMs/endMs). Always populated when the provider returns hits, regardless of `appendSearch`. Other providers leave this undefined.
 });
 var FunctionCallResultSchemaV1 = z.object({
   type: z.literal(RecognitionResultTypeV1.FUNCTION_CALL),
@@ -4117,7 +4139,15 @@ var TranscriptMessageSchema = z.object({
    * @example true
    * @default false
    */
-  is_fallback: z.boolean().optional()
+  is_fallback: z.boolean().optional(),
+  /**
+   * Provider-reported phrase detections (query + score, optionally
+   * startMs/endMs). Always populated when the provider returns hits,
+   * regardless of `appendSearch` or scene gating. Other providers leave
+   * this undefined.
+   * @example [{ query: 'justin bieber one time', score: 0.78, startMs: 1200, endMs: 2800 }]
+   */
+  detections: z.array(DetectionV1Schema).optional()
 });
 var VADEndSignalSchema = z.object({
   type: z.literal(ProviderMessageType.VAD_END_SIGNAL),
@@ -4434,6 +4464,9 @@ var ASRRequestSchemaV1 = z.object({
   // Streaming audio metrics opt-in: when > 0, server emits AudioMetrics results throttled to this interval (ms).
   // Undefined / 0 disables streaming audio metrics (final metrics still embedded in Metadata).
   audioMetricsIntervalMs: z.number().optional(),
+  // Opt-in: round-trip Deepgram `search` phrase hits into the transcript.
+  // Active only when (model = deepgram nova-2) AND (GameContext.gamePhase = 'Solve Puzzle'). See ASRRequestConfig.appendSearch in asr-config.types.ts for full semantics.
+  appendSearch: z.boolean().optional(),
   // Debug options (FOR DEBUG/TESTING ONLY - not for production use)
   debugCommand: RequestDebugCommandSchema
 });
@@ -4588,6 +4621,21 @@ var AudioEncoding;
     return NAME_TO_ENUM.has(nameStr.toUpperCase());
   }
   AudioEncoding2.isNameValid = isNameValid;
+  function coerce2(value, onStringInput) {
+    if (value === void 0) {
+      return AudioEncoding2.LINEAR16;
+    }
+    if (typeof value === "number") {
+      return value;
+    }
+    const result = fromName(value);
+    if (result === void 0) {
+      throw new Error(`Invalid encoding string: '${value}'. Use AudioEncoding enum or one of: LINEAR16, OGG_OPUS, FLAC, MULAW, ALAW (case insensitive)`);
+    }
+    onStringInput?.(`encoding passed as string '${value}'; prefer AudioEncoding.${toName(result)} enum for type safety`);
+    return result;
+  }
+  AudioEncoding2.coerce = coerce2;
 })(AudioEncoding || (AudioEncoding = {}));
 var PREFIX_AUDIO_ENCODING_OFFSET = 128;
 var SampleRate;
@@ -5313,6 +5361,37 @@ var MessageHandler = class {
   }
 };
 
+// src/utils/audio-resampler.ts
+function downsamplePcm16(input, srcRate, targetRate) {
+  if (targetRate > srcRate) {
+    throw new Error(
+      `downsamplePcm16: cannot upsample from ${srcRate}Hz to ${targetRate}Hz; capture audio at \u2265 ${targetRate}Hz instead.`
+    );
+  }
+  const buffer = ArrayBuffer.isView(input) ? input.buffer.slice(input.byteOffset, input.byteOffset + input.byteLength) : input;
+  const src = new Int16Array(buffer);
+  if (srcRate === targetRate || src.length === 0) {
+    return src.slice().buffer;
+  }
+  const ratio = srcRate / targetRate;
+  const dstLen = Math.floor(src.length / ratio);
+  const dst = new Int16Array(dstLen);
+  for (let i = 0; i < dstLen; i++) {
+    const startPos = i * ratio;
+    const endPos = (i + 1) * ratio;
+    const startIdx = Math.floor(startPos);
+    const endIdx = Math.min(Math.ceil(endPos), src.length);
+    let sum = 0;
+    let count = 0;
+    for (let j = startIdx; j < endIdx; j++) {
+      sum += src[j] ?? 0;
+      count++;
+    }
+    dst[i] = count > 0 ? Math.round(sum / count) : 0;
+  }
+  return dst.buffer;
+}
+
 // src/errors.ts
 var RecognitionError = class extends Error {
   constructor(errorType, message) {
@@ -5387,10 +5466,17 @@ var RealTimeTwoWayWebSocketRecognitionClient = class _RealTimeTwoWayWebSocketRec
     const retryConfig = config.connectionRetry || {};
     const maxAttempts = Math.max(1, Math.min(5, retryConfig.maxAttempts ?? 4));
     const delayMs = retryConfig.delayMs ?? 200;
+    const normalizedASRConfig = config.asrRequestConfig ? {
+      ...config.asrRequestConfig,
+      encoding: AudioEncoding.coerce(
+        config.asrRequestConfig.encoding,
+        (warning) => config.logger?.("warn", warning)
+      )
+    } : void 0;
     this.config = {
       url,
       audioUtteranceId,
-      ...config.asrRequestConfig && { asrRequestConfig: config.asrRequestConfig },
+      ...normalizedASRConfig && { asrRequestConfig: normalizedASRConfig },
       ...config.gameContext && { gameContext: config.gameContext },
       ...config.callbackUrls && { callbackUrls: config.callbackUrls },
       onTranscript: config.onTranscript || (() => {
@@ -5588,6 +5674,42 @@ var RealTimeTwoWayWebSocketRecognitionClient = class _RealTimeTwoWayWebSocketRec
     }
     this.sendAudioInternal(audioData);
   }
+  /**
+   * Send PCM16 mono audio captured at any sample rate. The SDK downsamples
+   * to the session's target rate (currently 16 kHz per server validator)
+   * before sending.
+   *
+   * Use this when your capture pipeline produces audio at the system's
+   * native rate — `AudioContext` defaults to 44.1 kHz or 48 kHz on most
+   * desktop/mobile hardware — and you don't want to bring your own
+   * resampler. If your audio is already at the target rate, prefer
+   * `sendAudio()` to skip the resample step.
+   *
+   * Algorithm: box-filter averaging (see audio-resampler.ts). Cheap, no
+   * dependencies, has a built-in low-pass effect so aliasing stays out of
+   * the speech band. Suitable for ASR; not a substitute for a high-quality
+   * resampler if you're doing music or full-fidelity processing.
+   *
+   * Audio must be signed 16-bit little-endian PCM, mono. Stereo must be
+   * mixed to mono by the caller.
+   *
+   * @param audioData - PCM16 mono audio at `sourceSampleRate`.
+   * @param sourceSampleRate - Source sample rate in Hz (e.g. 44100, 48000).
+   */
+  sendAudioWithSampleRate(audioData, sourceSampleRate) {
+    const targetRate = typeof this.config.asrRequestConfig?.sampleRate === "number" ? this.config.asrRequestConfig.sampleRate : SampleRate.RATE_16000;
+    if (audioData instanceof Blob) {
+      blobToArrayBuffer(audioData).then((arrayBuffer) => {
+        this.sendAudioInternal(
+          downsamplePcm16(arrayBuffer, sourceSampleRate, targetRate)
+        );
+      }).catch((error) => {
+        this.log("warn", "Failed to convert Blob to ArrayBuffer", error);
+      });
+      return;
+    }
+    this.sendAudioInternal(downsamplePcm16(audioData, sourceSampleRate, targetRate));
+  }
   sendAudioInternal(audioData) {
     const bytes = ArrayBuffer.isView(audioData) ? audioData.byteLength : audioData.byteLength;
     if (bytes === 0) return;
@@ -5734,7 +5856,7 @@ var RealTimeTwoWayWebSocketRecognitionClient = class _RealTimeTwoWayWebSocketRec
         model: this.config.asrRequestConfig.model,
         language: this.config.asrRequestConfig.language?.toString() || "en",
         sampleRate: typeof this.config.asrRequestConfig.sampleRate === "number" ? this.config.asrRequestConfig.sampleRate : SampleRate.RATE_16000,
-        encoding: typeof this.config.asrRequestConfig.encoding === "number" ? this.config.asrRequestConfig.encoding : AudioEncoding.LINEAR16,
+        encoding: this.config.asrRequestConfig.encoding,
         interimResults: this.config.asrRequestConfig.interimResults ?? false,
         // Auto-enable useContext if gameContext is provided, or use explicit value if set
         useContext: this.config.asrRequestConfig.useContext ?? !!this.config.gameContext,
@@ -5759,6 +5881,12 @@ var RealTimeTwoWayWebSocketRecognitionClient = class _RealTimeTwoWayWebSocketRec
         // Streaming audio metrics opt-in (ms interval). Server only forwards metrics if > 0.
         ...this.config.asrRequestConfig.audioMetricsIntervalMs !== void 0 && {
           audioMetricsIntervalMs: this.config.asrRequestConfig.audioMetricsIntervalMs
+        },
+        // Opt-in: round-trip Deepgram nova-2 search-phrase hits into the
+        // transcript. Only fires server-side when (model = nova-2) AND
+        // (GameContext.gamePhase = 'Solve Puzzle'). See ASRRequestConfig.appendSearch.
+        ...this.config.asrRequestConfig.appendSearch !== void 0 && {
+          appendSearch: this.config.asrRequestConfig.appendSearch
         }
       };
       super.sendMessage(
@@ -5892,7 +6020,7 @@ var RealTimeTwoWayWebSocketRecognitionClient = class _RealTimeTwoWayWebSocketRec
    */
   sendAudioNow(audioData) {
     const byteLength = ArrayBuffer.isView(audioData) ? audioData.byteLength : audioData.byteLength;
-    const encodingId = this.config.asrRequestConfig?.encoding || AudioEncoding.LINEAR16;
+    const encodingId = this.config.asrRequestConfig?.encoding ?? AudioEncoding.LINEAR16;
     const sampleRate = typeof this.config.asrRequestConfig?.sampleRate === "number" ? this.config.asrRequestConfig.sampleRate : SampleRate.RATE_16000;
     super.sendAudio(
       audioData,
@@ -5960,7 +6088,7 @@ var RealTimeTwoWayWebSocketRecognitionClient = class _RealTimeTwoWayWebSocketRec
   sendPrefixAudioNow(audioData) {
     const byteLength = ArrayBuffer.isView(audioData) ? audioData.byteLength : audioData.byteLength;
     if (byteLength === 0) return;
-    const baseEncodingId = this.config.asrRequestConfig?.encoding || AudioEncoding.LINEAR16;
+    const baseEncodingId = this.config.asrRequestConfig?.encoding ?? AudioEncoding.LINEAR16;
     const prefixEncodingId = baseEncodingId + PREFIX_AUDIO_ENCODING_OFFSET;
     const sampleRate = typeof this.config.asrRequestConfig?.sampleRate === "number" ? this.config.asrRequestConfig.sampleRate : SampleRate.RATE_16000;
     this.log("debug", "Sending prefix audio", { bytes: byteLength, encoding: prefixEncodingId });