omnius 1.0.426 → 1.0.428

package/dist/index.js

@@ -603384,6 +603384,15 @@ async function findLiveWhisperScript() {
   }
   return null;
 }
+function defaultConsensusModel(primaryModel) {
+  const model = primaryModel.trim().toLowerCase();
+  if (model === "tiny") return "base";
+  if (model === "base") return "small";
+  if (model === "small") return "base";
+  if (model === "medium") return "base";
+  if (model === "large-v3" || model === "large") return "small";
+  return "base";
+}
 async function ensureVenvForTranscribeCli() {
   const bin = process.platform === "win32" ? "Scripts" : "bin";
   const exe = process.platform === "win32" ? "python.exe" : "python3";
@@ -603573,7 +603582,9 @@ var init_listen = __esm({
           lastStatus: `loading whisper ${this.model} model...`
         });
         const consensusEnv = String(process.env["OMNIUS_ASR_CONSENSUS"] ?? "").trim().toLowerCase();
-        const consensusModel = consensusEnv === "0" || consensusEnv === "off" || consensusEnv === "false" ? "off" : consensusEnv || (this.model === "tiny" ? "base" : "tiny");
+        const consensusModel = consensusEnv === "0" || consensusEnv === "off" || consensusEnv === "false" ? "off" : consensusEnv || defaultConsensusModel(this.model);
+        const consensusThreshold = String(process.env["OMNIUS_ASR_CONSENSUS_THRESHOLD"] ?? "0.45");
+        const silenceMs = String(process.env["OMNIUS_ASR_SILENCE_MS"] ?? "800");
         this.process = spawn29(
           pyPath,
           [
@@ -603585,7 +603596,11 @@ var init_listen = __esm({
             "--window-seconds",
             "10",
             "--consensus-model",
-            consensusModel
+            consensusModel,
+            "--consensus-threshold",
+            consensusThreshold,
+            "--silence-ms",
+            silenceMs
           ],
           {
             stdio: ["pipe", "pipe", "pipe"],
@@ -603619,13 +603634,19 @@ var init_listen = __esm({
                 case "transcript":
                   this.emit("transcript", {
                     text: evt.text,
-                    isFinal: evt.isFinal ?? false
+                    isFinal: evt.isFinal ?? false,
+                    consensus: typeof evt.consensus === "boolean" ? evt.consensus : void 0,
+                    doa: typeof evt.doa === "number" ? evt.doa : void 0
                   });
                   break;
                 case "consensus_rejected":
-                  updateListenLiveState({
-                    lastStatus: `consensus rejected (${evt.reason ?? "mismatch"}): ${String(evt.primary ?? "").slice(0, 80)}`
-                  });
+                  {
+                    const message2 = `consensus rejected (${evt.reason ?? "mismatch"}): ${String(evt.primary ?? "").slice(0, 80)}`;
+                    updateListenLiveState({
+                      lastStatus: message2
+                    });
+                    this.emit("status", message2);
+                  }
                   break;
                 case "vad": {
                   lastHardwareVadAt = Date.now();
@@ -604016,7 +604037,7 @@ var init_listen = __esm({
               this.lastTranscriptTime = Date.now();
               this.pendingText = evt.text.trim();
               updateListenLiveState({ lastTranscript: this.pendingText, lastTranscriptAt: this.lastTranscriptTime });
-              this.emit("transcript", this.pendingText, evt.isFinal);
+              this.emit("transcript", this.pendingText, evt.isFinal, { consensus: evt.consensus, doa: evt.doa });
               if (this.config.mode === "auto") this.resetSilenceTimer();
             });
             fallback.on("error", (err) => {
@@ -604031,8 +604052,9 @@ transcribe-cli error: ${transcribeCliError}` : "";
             return `Failed to start live transcription: ${msg}${tcHint}`;
           }
         }
-        this.spawnMicProcess(micCmd);
         this.active = true;
+        this.paused = false;
+        this.spawnMicProcess(micCmd);
         this.blinkState = true;
         this.blinkTimer = setInterval(() => {
           this.blinkState = !this.blinkState;
@@ -617229,6 +617251,25 @@ function rms(arr) {
   return Math.sqrt(sum / arr.length);
 }
 
+// Linear resampler: browsers frequently IGNORE the requested AudioContext
+// sample rate and run at the hardware rate (typically 48kHz). Sending those
+// frames as-if-16kHz made the audio unpack 3x slow/pitch-dropped server-side,
+// wrecking ASR. Always resample the actual context rate down to 16kHz.
+function resampleTo16k(input, srcRate) {
+  if (srcRate === 16000) return input;
+  const ratio = srcRate / 16000;
+  const outLen = Math.floor(input.length / ratio);
+  const out = new Float32Array(outLen);
+  for (let i = 0; i < outLen; i++) {
+    const pos = i * ratio;
+    const i0 = Math.floor(pos);
+    const i1 = Math.min(input.length - 1, i0 + 1);
+    const frac = pos - i0;
+    out[i] = input[i0] * (1 - frac) + input[i1] * frac;
+  }
+  return out;
+}
+
 async function toggleMic() {
   if (micActive) { stopMic(); return; }
   try {
@@ -617237,17 +617278,19 @@ async function toggleMic() {
       audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true, noiseSuppression: true }
     });
     const source = micCtx.createMediaStreamSource(micStream);
+    const actualRate = micCtx.sampleRate; // may be 48000 regardless of the request
     scriptProcessor = micCtx.createScriptProcessor(4096, 1, 1);
     scriptProcessor.onaudioprocess = (e) => {
       if (!micActive || !ws || ws.readyState !== 1) return;
-      const input = e.inputBuffer.getChannelData(0);
+      const raw = e.inputBuffer.getChannelData(0);
+      const input = resampleTo16k(raw, actualRate);
       const int16 = new Int16Array(input.length);
       for (let i = 0; i < input.length; i++) {
         const s = Math.max(-1, Math.min(1, input[i]));
         int16[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
       }
       ws.send(int16.buffer);
-      micLevel = Math.min(1, rms(input) * 5);
+      micLevel = Math.min(1, rms(raw) * 5);
     };
     source.connect(scriptProcessor);
     scriptProcessor.connect(micCtx.destination);
@@ -617256,7 +617299,8 @@ async function toggleMic() {
     micBtn.textContent = 'stop';
     micBtn.classList.add('active');
     hintEl.textContent = 'listening...';
-    ws.send(JSON.stringify({ type: 'mic_start', username: 'web-user' }));
+    // Declare the post-resample rate (and the raw context rate for debugging)
+    ws.send(JSON.stringify({ type: 'mic_start', username: 'web-user', sampleRate: 16000, contextRate: actualRate }));
   } catch (err) {
     hintEl.textContent = 'mic error: ' + err.message;
   }
@@ -617629,6 +617673,23 @@ connectPP();
 </body>
 </html>`;
 }
+function resamplePcm16Mono(pcm, srcRate, dstRate) {
+  if (srcRate === dstRate || srcRate <= 0 || dstRate <= 0) return pcm;
+  const srcSamples = Math.floor(pcm.length / 2);
+  if (srcSamples === 0) return pcm;
+  const ratio = srcRate / dstRate;
+  const outSamples = Math.max(1, Math.floor(srcSamples / ratio));
+  const out = Buffer.allocUnsafe(outSamples * 2);
+  for (let i2 = 0; i2 < outSamples; i2++) {
+    const pos = i2 * ratio;
+    const i0 = Math.min(srcSamples - 1, Math.floor(pos));
+    const i1 = Math.min(srcSamples - 1, i0 + 1);
+    const frac = pos - i0;
+    const sample = pcm.readInt16LE(i0 * 2) * (1 - frac) + pcm.readInt16LE(i1 * 2) * frac;
+    out.writeInt16LE(Math.max(-32768, Math.min(32767, Math.round(sample))), i2 * 2);
+  }
+  return out;
+}
 function renderVoiceSessionStart(tunnelUrl) {
   renderBoxedBlock({
     title: "☁ Live Voice Session",
@@ -617936,17 +617997,26 @@ var init_voice_session = __esm({
             clearInterval(keepaliveInterval);
           }
         }, 1e4);
+        let clientPcmRate = 16e3;
         ws.on("message", (data, isBinary) => {
           if (isBinary) {
             if (!this.ttsSpeaking) {
-              this.onUserAudio?.(Buffer.isBuffer(data) ? data : Buffer.from(data), clientId);
+              let pcm = Buffer.isBuffer(data) ? data : Buffer.from(data);
+              if (clientPcmRate !== 16e3 && clientPcmRate > 0) {
+                pcm = resamplePcm16Mono(pcm, clientPcmRate, 16e3);
+              }
+              this.onUserAudio?.(pcm, clientId);
             }
           } else {
             try {
               const msg = JSON.parse(data.toString());
-              if (msg.type === "mic_start" && msg.username) {
-                const entry = this.state.connectedUsers.get(clientId);
-                if (entry) entry.username = msg.username;
+              if (msg.type === "mic_start") {
+                if (msg.username) {
+                  const entry = this.state.connectedUsers.get(clientId);
+                  if (entry) entry.username = msg.username;
+                }
+                const declared = Number(msg.sampleRate);
+                clientPcmRate = Number.isFinite(declared) && declared >= 8e3 && declared <= 192e3 ? declared : 16e3;
               }
             } catch {
             }
@@ -697514,6 +697584,9 @@ Rules:
       // VAD segment capture
       captureBuffer = "";
       captureStartTime = 0;
+      consensusMetadataSeen = false;
+      consensusText = null;
+      consensusSignalScore = null;
       silenceTimer = null;
       maxSegmentTimer = null;
       lastSignalScore = null;
@@ -697604,18 +697677,25 @@ Rules:
           let isFinal;
           let snr;
           let confidence2;
+          let consensus;
           if (typeof args[0] === "object" && args[0] !== null) {
             const evt = args[0];
             text2 = evt.text ?? "";
             isFinal = evt.isFinal ?? false;
             snr = evt.snr;
             confidence2 = evt.confidence;
+            consensus = typeof evt.consensus === "boolean" ? evt.consensus : void 0;
           } else {
             text2 = String(args[0] ?? "");
             isFinal = Boolean(args[1]);
+            const meta = args[2];
+            if (meta && typeof meta === "object") {
+              const m2 = meta;
+              if (typeof m2.consensus === "boolean") consensus = m2.consensus;
+            }
           }
           if (!text2.trim()) return;
-          this.handleTranscript(text2.trim(), isFinal, snr, confidence2);
+          this.handleTranscript(text2.trim(), isFinal, snr, confidence2, consensus);
         };
         this._onError = (err) => {
           const msg = err instanceof Error ? err.message : String(err);
@@ -697682,7 +697762,7 @@ Rules:
       // ---------------------------------------------------------------------------
       // Transcript handling — VAD-style segment capture (Voryn pattern)
       // ---------------------------------------------------------------------------
-      handleTranscript(text2, isFinal, snr, confidence2) {
+      handleTranscript(text2, isFinal, snr, confidence2, consensus) {
         if (!this.active) return;
         if (this._state !== "LISTENING" && this._state !== "CAPTURING") {
           return;
@@ -697695,7 +697775,7 @@ Rules:
         }
         if (this._state === "LISTENING") {
           this.setState("CAPTURING");
-          this.captureBuffer = "";
+          this.resetCaptureState();
           this.captureStartTime = Date.now();
           this.maxSegmentTimer = setTimeout(() => {
             if (this._state === "CAPTURING") {
@@ -697703,23 +697783,44 @@ Rules:
             }
           }, MAX_SEGMENT_MS);
         }
-        this.captureBuffer = text2;
-        this.lastSignalScore = typeof snr === "number" && !Number.isNaN(snr) ? clamp0116(snr) : computeSignalFromText(text2, confidence2);
+        const signalScore = typeof snr === "number" && !Number.isNaN(snr) ? clamp0116(snr) : computeSignalFromText(text2, confidence2);
+        this.lastSignalScore = signalScore;
+        if (consensus !== void 0) {
+          this.consensusMetadataSeen = true;
+          if (consensus === true) {
+            this.consensusText = text2;
+            this.consensusSignalScore = signalScore;
+            this.captureBuffer = text2;
+          }
+        } else {
+          this.captureBuffer = text2;
+        }
         this.emit("snr", { score: this.lastSignalScore });
         this.onPartialTranscript(text2);
         if (this.silenceTimer) clearTimeout(this.silenceTimer);
-        const waitMs = this._vadSilenceMs ?? VAD_SILENCE_MS;
+        const waitMs = isFinal ? 0 : this._vadSilenceMs ?? VAD_SILENCE_MS;
         this.silenceTimer = setTimeout(() => {
           if (this._state === "CAPTURING") {
             this.finalizeSegment();
           }
         }, waitMs);
       }
+      resetCaptureState() {
+        this.captureBuffer = "";
+        this.captureStartTime = 0;
+        this.lastSignalScore = null;
+        this.consensusMetadataSeen = false;
+        this.consensusText = null;
+        this.consensusSignalScore = null;
+      }
       // ---------------------------------------------------------------------------
       // Segment finalization → Transcribing → Thinking → Speaking
       // ---------------------------------------------------------------------------
       finalizeSegment() {
-        const text2 = this.captureBuffer.trim();
+        const rawText = this.captureBuffer.trim();
+        const requiresConsensus = this.consensusMetadataSeen;
+        const text2 = requiresConsensus ? (this.consensusText ?? "").trim() : rawText;
+        const signalScore = requiresConsensus ? this.consensusSignalScore ?? this.lastSignalScore : this.lastSignalScore;
         if (this.silenceTimer) {
           clearTimeout(this.silenceTimer);
           this.silenceTimer = null;
@@ -697728,18 +697829,20 @@ Rules:
           clearTimeout(this.maxSegmentTimer);
           this.maxSegmentTimer = null;
         }
-        this.captureBuffer = "";
+        this.resetCaptureState();
         if (!text2) {
+          if (requiresConsensus && rawText) {
+            if (this.debugSnr) this.onStatus(`Ignoring utterance without dual-ASR consensus: ${truncateForLog(rawText, 48)}`);
+            this.emit("consensusFiltered", { text: rawText });
+          }
           this.setState("LISTENING");
           return;
         }
-        const score = Math.min(this.lastSignalScore ?? 1, computeSignalFromText(text2));
+        const score = Math.min(signalScore ?? 1, computeSignalFromText(text2));
         if (score < MIN_SIGNAL_SCORE || NOISE_ONLY_RE.test(text2)) {
           if (this.debugSnr) this.onStatus(`Ignoring low-signal utterance (SNR:${score.toFixed(2)}): ${truncateForLog(text2, 48)}`);
           this.emit("snrFiltered", { score, text: text2 });
           this.setState("LISTENING");
-          this.captureBuffer = "";
-          this.lastSignalScore = null;
           return;
         }
         this.setState("TRANSCRIBING");
@@ -697862,35 +697965,35 @@ ${toolOutput}` });
             const reply = extractVoiceModelReply(stripToolJsonLines(response.trim()));
             const finalSpoken = reply.text;
             if (!finalSpoken || reply.action === "silent") {
-              return;
-            }
-            if (typeof this.voice.waitUntilIdle === "function") {
-              try {
-                await this.voice.waitUntilIdle();
-              } catch {
+            } else {
+              if (typeof this.voice.waitUntilIdle === "function") {
+                try {
+                  await this.voice.waitUntilIdle();
+                } catch {
+                }
               }
-            }
-            this.context.push({ role: "assistant", content: finalSpoken });
-            this.trimContext();
-            this.setState("SPEAKING");
-            this.onAgentSpeech(finalSpoken);
-            this.lastAgentSpeech = { text: finalSpoken, startedAt: Date.now() };
-            this.voice.speak(finalSpoken);
-            this.voiceTranscript.push({ role: "assistant", content: finalSpoken, ts: Date.now() });
-            if (this.runner && this.turnCount % SUMMARY_INJECTION_INTERVAL === 0) {
-              this.injectSummary();
-            }
-            if (typeof this.voice.waitUntilIdle === "function") {
-              try {
-                await this.voice.waitUntilIdle();
-              } catch {
+              this.context.push({ role: "assistant", content: finalSpoken });
+              this.trimContext();
+              this.setState("SPEAKING");
+              this.onAgentSpeech(finalSpoken);
+              this.lastAgentSpeech = { text: finalSpoken, startedAt: Date.now() };
+              this.voice.speak(finalSpoken);
+              this.voiceTranscript.push({ role: "assistant", content: finalSpoken, ts: Date.now() });
+              if (this.runner && this.turnCount % SUMMARY_INJECTION_INTERVAL === 0) {
+                this.injectSummary();
+              }
+              if (typeof this.voice.waitUntilIdle === "function") {
+                try {
+                  await this.voice.waitUntilIdle();
+                } catch {
+                }
+              } else {
+                const estimatedMs = Math.max(1500, finalSpoken.length / 5 * (6e4 / 150));
+                await new Promise((r2) => setTimeout(r2, estimatedMs));
+              }
+              if (reply.action === "hangup") {
+                this.active = false;
               }
-            } else {
-              const estimatedMs = Math.max(1500, finalSpoken.length / 5 * (6e4 / 150));
-              await new Promise((r2) => setTimeout(r2, estimatedMs));
-            }
-            if (reply.action === "hangup") {
-              this.active = false;
             }
           }
         } catch (err) {
@@ -737057,6 +737160,11 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
       listenEng.on("level", (evt) => {
         statusBar.setMicActivity(evt.speechActive, evt.levelDb);
       });
+      listenEng.on("info", (msg) => {
+        if (/consensus rejected|loading whisper|whisper model loaded/i.test(msg)) {
+          writeContent(() => renderInfo(`[voicechat/asr] ${msg}`));
+        }
+      });
       const summaryRunner = {
         injectUserMessage(content) {
           if (activeTask?.runner) {

package/dist/scripts/live-whisper.py

@@ -76,10 +76,12 @@ def emit_error(msg: str):
     emit({"type": "error", "message": msg})
 
 
-def emit_transcript(text: str, is_final: bool = False, doa=None):
+def emit_transcript(text: str, is_final: bool = False, doa=None, consensus=None):
     event = {"type": "transcript", "text": text, "isFinal": is_final}
     if doa is not None:
         event["doa"] = doa
+    if consensus is not None:
+        event["consensus"] = bool(consensus)
     emit(event)
 
 # ---------------------------------------------------------------------------
@@ -441,7 +443,7 @@ def main():
             result = run_transcribe(model, amplify(samples))
             text = _segment_filtered_text(result)
             if text:
-                emit_transcript(text, is_final=False)
+                emit_transcript(text, is_final=False, consensus=False)
         except Exception as e:
             emit_error(f"Transcription error: {e}")
 
@@ -481,7 +483,16 @@ def main():
             if text == last_final_text:
                 return
             last_final_text = text
-            emit_transcript(text, is_final=True, doa=tuning.doa() if tuning else None)
+            # consensus=True only when BOTH models ran and converged. When the
+            # second model is off/unavailable, omit consensus metadata entirely
+            # so legacy single-model consumers do not interpret consensus=false
+            # as an explicit dual-ASR rejection.
+            emit_transcript(
+                text,
+                is_final=True,
+                doa=tuning.doa() if tuning else None,
+                consensus=True if consensus_model is not None else None,
+            )
         except Exception as e:
             emit_error(f"Transcription error: {e}")
 

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "omnius",
-  "version": "1.0.426",
+  "version": "1.0.428",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "omnius",
-      "version": "1.0.426",
+      "version": "1.0.428",
       "bundleDependencies": [
         "image-to-ascii"
       ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnius",
-  "version": "1.0.426",
+  "version": "1.0.428",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",