@omote/core 0.4.3 → 0.4.5

package/dist/index.js

@@ -469,7 +469,14 @@ var AudioScheduler = class {
     source.connect(gainNode);
     const scheduleTime = this.nextPlayTime;
     source.start(scheduleTime);
-    this.scheduledSources.push({ source, gainNode });
+    const entry = { source, gainNode };
+    this.scheduledSources.push(entry);
+    source.onended = () => {
+      const idx = this.scheduledSources.indexOf(entry);
+      if (idx !== -1) {
+        this.scheduledSources.splice(idx, 1);
+      }
+    };
     const duration = audioData.length / ctx.sampleRate;
     this.nextPlayTime = scheduleTime + duration;
     return scheduleTime;
@@ -825,7 +832,7 @@ var LAMPipeline = class {
   }
 };
 
-// src/audio/SyncedAudioPipeline.ts
+// src/audio/audioUtils.ts
 function pcm16ToFloat32(buffer) {
   const byteLen = buffer.byteLength & ~1;
   const int16 = byteLen === buffer.byteLength ? new Int16Array(buffer) : new Int16Array(buffer, 0, byteLen / 2);
@@ -835,6 +842,15 @@ function pcm16ToFloat32(buffer) {
   }
   return float32;
 }
+function int16ToFloat32(int16) {
+  const float32 = new Float32Array(int16.length);
+  for (let i = 0; i < int16.length; i++) {
+    float32[i] = int16[i] / 32768;
+  }
+  return float32;
+}
+
+// src/audio/SyncedAudioPipeline.ts
 var SyncedAudioPipeline = class extends EventEmitter {
   constructor(options) {
     super();
@@ -3536,14 +3552,18 @@ var _Wav2Vec2Inference = class _Wav2Vec2Inference {
         });
         try {
           const startTime = performance.now();
+          let timeoutId;
           const results = await Promise.race([
-            this.session.run(feeds),
-            new Promise(
-              (_, rej) => setTimeout(
+            this.session.run(feeds).then((r) => {
+              clearTimeout(timeoutId);
+              return r;
+            }),
+            new Promise((_, rej) => {
+              timeoutId = setTimeout(
                 () => rej(new Error(`Wav2Vec2 inference timed out after ${_Wav2Vec2Inference.INFERENCE_TIMEOUT_MS}ms`)),
                 _Wav2Vec2Inference.INFERENCE_TIMEOUT_MS
-              )
-            )
+              );
+            })
           ]);
           const inferenceTimeMs = performance.now() - startTime;
           const asrOutput = results["asr_logits"];
@@ -3649,15 +3669,6 @@ var Wav2Vec2Inference = _Wav2Vec2Inference;
 
 // src/audio/FullFacePipeline.ts
 var logger3 = createLogger("FullFacePipeline");
-function pcm16ToFloat322(buffer) {
-  const byteLen = buffer.byteLength & ~1;
-  const int16 = byteLen === buffer.byteLength ? new Int16Array(buffer) : new Int16Array(buffer, 0, byteLen / 2);
-  const float32 = new Float32Array(int16.length);
-  for (let i = 0; i < int16.length; i++) {
-    float32[i] = int16[i] / 32768;
-  }
-  return float32;
-}
 var BLENDSHAPE_INDEX_MAP = /* @__PURE__ */ new Map();
 LAM_BLENDSHAPES.forEach((name, index) => {
   BLENDSHAPE_INDEX_MAP.set(name, index);
@@ -3807,7 +3818,7 @@ var _FullFacePipeline = class _FullFacePipeline extends EventEmitter {
     if (!combined) {
       return;
     }
-    const float32 = pcm16ToFloat322(combined);
+    const float32 = pcm16ToFloat32(combined);
     const scheduleTime = await this.scheduler.schedule(float32);
     if (!this.playbackStarted) {
       this.playbackStarted = true;
@@ -4290,13 +4301,18 @@ function ctcGreedyDecode(logits, seqLen, vocabSize, tokenMap) {
 
 // src/inference/SenseVoiceInference.ts
 var logger4 = createLogger("SenseVoice");
-var SenseVoiceInference = class {
+var _SenseVoiceInference = class _SenseVoiceInference {
   constructor(config) {
     this.session = null;
     this.ort = null;
     this._backend = "wasm";
     this.isLoading = false;
     this.inferenceQueue = Promise.resolve();
+    // Session health: set to true if session.run() times out.
+    // A timed-out session may have a zombie WASM dispatch still running,
+    // so all future transcribe() calls reject immediately to prevent concurrent access.
+    this.poisoned = false;
+    // 10s for SenseVoice (heavier preprocessing)
     // Preprocessing state (loaded once)
     this.tokenMap = null;
     this.negMean = null;
@@ -4444,6 +4460,9 @@ var SenseVoiceInference = class {
     if (!this.session || !this.ort || !this.tokenMap) {
       throw new Error("Model not loaded. Call load() first.");
     }
+    if (this.poisoned) {
+      throw new Error("SenseVoice session timed out \u2014 inference unavailable until page reload");
+    }
     const audio = new Float32Array(audioSamples);
     return this.queueInference(audio);
   }
@@ -4481,7 +4500,19 @@ var SenseVoiceInference = class {
             language: new ort.Tensor("int32", new Int32Array([this.languageId]), [1]),
             text_norm: new ort.Tensor("int32", new Int32Array([this.textNormId]), [1])
           };
-          const results = await this.session.run(feeds);
+          let timeoutId;
+          const results = await Promise.race([
+            this.session.run(feeds).then((r) => {
+              clearTimeout(timeoutId);
+              return r;
+            }),
+            new Promise((_, rej) => {
+              timeoutId = setTimeout(
+                () => rej(new Error(`SenseVoice inference timed out after ${_SenseVoiceInference.INFERENCE_TIMEOUT_MS}ms`)),
+                _SenseVoiceInference.INFERENCE_TIMEOUT_MS
+              );
+            })
+          ]);
           const logitsOutput = results["logits"];
           if (!logitsOutput) {
             throw new Error('Model output missing "logits" tensor');
@@ -4527,6 +4558,32 @@ var SenseVoiceInference = class {
             preprocessTimeMs
           });
         } catch (err) {
+          const errMsg = err instanceof Error ? err.message : String(err);
+          if (errMsg.includes("timed out")) {
+            this.poisoned = true;
+            logger4.error("CRITICAL: Inference session timed out \u2014 SenseVoice is dead. Page reload required.", {
+              backend: this._backend,
+              timeoutMs: _SenseVoiceInference.INFERENCE_TIMEOUT_MS
+            });
+          } else if (typeof err === "number") {
+            const oomError = new Error(
+              `SenseVoice inference failed with raw C++ exception pointer (0x${err.toString(16)}). This is likely an OOM crash in WASM. Try reloading the page.`
+            );
+            logger4.error("ORT WASM OOM \u2014 raw C++ exception pointer", {
+              pointer: `0x${err.toString(16)}`,
+              backend: this._backend
+            });
+            span?.endWithError(oomError);
+            telemetry?.incrementCounter("omote.inference.total", 1, {
+              model: "sensevoice",
+              backend: this._backend,
+              status: "error"
+            });
+            reject(oomError);
+            return;
+          } else {
+            logger4.error("Inference failed", { error: errMsg, backend: this._backend });
+          }
           span?.endWithError(err instanceof Error ? err : new Error(String(err)));
           telemetry?.incrementCounter("omote.inference.total", 1, {
             model: "sensevoice",
@@ -4550,10 +4607,12 @@ var SenseVoiceInference = class {
     this.invStddev = null;
   }
 };
+_SenseVoiceInference.INFERENCE_TIMEOUT_MS = 1e4;
+var SenseVoiceInference = _SenseVoiceInference;
 
 // src/inference/Wav2ArkitCpuInference.ts
 var logger5 = createLogger("Wav2ArkitCpu");
-var Wav2ArkitCpuInference = class {
+var _Wav2ArkitCpuInference = class _Wav2ArkitCpuInference {
   constructor(config) {
     this.modelId = "wav2arkit_cpu";
     this.session = null;
@@ -4562,6 +4621,10 @@ var Wav2ArkitCpuInference = class {
     this.isLoading = false;
     // Inference queue for handling concurrent calls
     this.inferenceQueue = Promise.resolve();
+    // Session health: set to true if session.run() times out.
+    // A timed-out session may have a zombie WASM dispatch still running,
+    // so all future infer() calls reject immediately to prevent concurrent access.
+    this.poisoned = false;
     this.config = config;
   }
   get backend() {
@@ -4734,6 +4797,9 @@ var Wav2ArkitCpuInference = class {
     if (!this.session) {
       throw new Error("Model not loaded. Call load() first.");
     }
+    if (this.poisoned) {
+      throw new Error("Wav2ArkitCpu session timed out \u2014 inference unavailable until page reload");
+    }
     const audioCopy = new Float32Array(audioSamples);
     const feeds = {
       "audio_waveform": new this.ort.Tensor("float32", audioCopy, [1, audioCopy.length])
@@ -4753,7 +4819,19 @@ var Wav2ArkitCpuInference = class {
         });
         try {
           const startTime = performance.now();
-          const results = await this.session.run(feeds);
+          let timeoutId;
+          const results = await Promise.race([
+            this.session.run(feeds).then((r) => {
+              clearTimeout(timeoutId);
+              return r;
+            }),
+            new Promise((_, rej) => {
+              timeoutId = setTimeout(
+                () => rej(new Error(`Wav2ArkitCpu inference timed out after ${_Wav2ArkitCpuInference.INFERENCE_TIMEOUT_MS}ms`)),
+                _Wav2ArkitCpuInference.INFERENCE_TIMEOUT_MS
+              );
+            })
+          ]);
           const inferenceTimeMs = performance.now() - startTime;
           const blendshapeOutput = results["blendshapes"];
           if (!blendshapeOutput) {
@@ -4793,6 +4871,32 @@ var Wav2ArkitCpuInference = class {
             inferenceTimeMs
           });
         } catch (err) {
+          const errMsg = err instanceof Error ? err.message : String(err);
+          if (errMsg.includes("timed out")) {
+            this.poisoned = true;
+            logger5.error("CRITICAL: Inference session timed out \u2014 Wav2ArkitCpu is dead. Page reload required.", {
+              backend: this._backend,
+              timeoutMs: _Wav2ArkitCpuInference.INFERENCE_TIMEOUT_MS
+            });
+          } else if (typeof err === "number") {
+            const oomError = new Error(
+              `Wav2ArkitCpu inference failed with raw C++ exception pointer (0x${err.toString(16)}). This is likely an OOM crash in WASM. Try reloading the page.`
+            );
+            logger5.error("ORT WASM OOM \u2014 raw C++ exception pointer", {
+              pointer: `0x${err.toString(16)}`,
+              backend: this._backend
+            });
+            span?.endWithError(oomError);
+            telemetry?.incrementCounter("omote.inference.total", 1, {
+              model: "wav2arkit_cpu",
+              backend: this._backend,
+              status: "error"
+            });
+            reject(oomError);
+            return;
+          } else {
+            logger5.error("Inference failed", { error: errMsg, backend: this._backend });
+          }
           span?.endWithError(err instanceof Error ? err : new Error(String(err)));
           telemetry?.incrementCounter("omote.inference.total", 1, {
             model: "wav2arkit_cpu",
@@ -4814,6 +4918,8 @@ var Wav2ArkitCpuInference = class {
     }
   }
 };
+_Wav2ArkitCpuInference.INFERENCE_TIMEOUT_MS = 5e3;
+var Wav2ArkitCpuInference = _Wav2ArkitCpuInference;
 
 // src/inference/createLipSync.ts
 var logger6 = createLogger("createLipSync");
@@ -5143,23 +5249,13 @@ var SileroVADInference = class {
     }
     return segments;
   }
-  /**
-   * Calculate RMS energy of audio chunk
-   */
-  calculateRMS(samples) {
-    let sum = 0;
-    for (let i = 0; i < samples.length; i++) {
-      sum += samples[i] * samples[i];
-    }
-    return Math.sqrt(sum / samples.length);
-  }
   /**
    * Queue inference to serialize ONNX session calls
    */
   queueInference(audioChunk) {
     const audioChunkCopy = new Float32Array(audioChunk);
     const MIN_ENERGY_THRESHOLD = 1e-3;
-    const rms = this.calculateRMS(audioChunkCopy);
+    const rms = calculateRMS(audioChunkCopy);
     if (rms < MIN_ENERGY_THRESHOLD) {
       if (!this.wasSpeaking) {
         this.preSpeechBuffer.push(new Float32Array(audioChunkCopy));
@@ -5214,7 +5310,7 @@ var SileroVADInference = class {
               [2, 1, 128]
             );
           }
-          this.context = audioChunk.slice(-this.contextSize);
+          this.context = audioChunkCopy.slice(-this.contextSize);
           const inferenceTimeMs = performance.now() - startTime;
           const isSpeech = probability > this.config.threshold;
           let preSpeechChunks;
@@ -5226,7 +5322,7 @@ var SileroVADInference = class {
               durationMs: Math.round(preSpeechChunks.length * this.getChunkDurationMs())
             });
           } else if (!isSpeech && !this.wasSpeaking) {
-            this.preSpeechBuffer.push(new Float32Array(audioChunk));
+            this.preSpeechBuffer.push(new Float32Array(audioChunkCopy));
             if (this.preSpeechBuffer.length > this.config.preSpeechBufferChunks) {
               this.preSpeechBuffer.shift();
             }
@@ -5261,13 +5357,30 @@ var SileroVADInference = class {
             preSpeechChunks
           });
         } catch (err) {
-          span?.endWithError(err instanceof Error ? err : new Error(String(err)));
-          telemetry?.incrementCounter("omote.inference.total", 1, {
-            model: "silero-vad",
-            backend: this._backend,
-            status: "error"
-          });
-          reject(err);
+          if (typeof err === "number") {
+            const oomError = new Error(
+              `SileroVAD inference failed with raw C++ exception pointer (0x${err.toString(16)}). This is likely an OOM crash in WASM. Try reducing concurrent model sessions or reloading the page.`
+            );
+            logger7.error("ORT WASM OOM \u2014 raw C++ exception pointer", {
+              pointer: `0x${err.toString(16)}`,
+              backend: this._backend
+            });
+            span?.endWithError(oomError);
+            telemetry?.incrementCounter("omote.inference.total", 1, {
+              model: "silero-vad",
+              backend: this._backend,
+              status: "error"
+            });
+            reject(oomError);
+          } else {
+            span?.endWithError(err instanceof Error ? err : new Error(String(err)));
+            telemetry?.incrementCounter("omote.inference.total", 1, {
+              model: "silero-vad",
+              backend: this._backend,
+              status: "error"
+            });
+            reject(err);
+          }
         }
       });
     });
@@ -6494,7 +6607,7 @@ var AgentCoreAdapter = class extends EventEmitter {
         console.error("[AgentCore] VAD error during interruption detection:", error);
       });
     }
-    const float32 = audio instanceof Float32Array ? audio : this.int16ToFloat32(audio);
+    const float32 = audio instanceof Float32Array ? audio : int16ToFloat32(audio);
     this.audioBuffer.push(float32);
     this.scheduleTranscription();
   }
@@ -6826,7 +6939,7 @@ var AgentCoreAdapter = class extends EventEmitter {
    * Falls back to simple RMS if VAD not available
    */
   async detectVoiceActivity(audio) {
-    const float32 = audio instanceof Float32Array ? audio : this.int16ToFloat32(audio);
+    const float32 = audio instanceof Float32Array ? audio : int16ToFloat32(audio);
     if (this.vad) {
       const chunkSize = this.vad.getChunkSize();
       for (let i = 0; i + chunkSize <= float32.length; i += chunkSize) {
@@ -6845,13 +6958,6 @@ var AgentCoreAdapter = class extends EventEmitter {
     const rms = Math.sqrt(sum / float32.length);
     return rms > 0.02;
   }
-  int16ToFloat32(int16) {
-    const float32 = new Float32Array(int16.length);
-    for (let i = 0; i < int16.length; i++) {
-      float32[i] = int16[i] / 32768;
-    }
-    return float32;
-  }
   base64ToArrayBuffer(base64) {
     const binaryString = atob(base64);
     const bytes = new Uint8Array(binaryString.length);