omnius 1.0.419 → 1.0.421

package/dist/index.js

@@ -603509,6 +603509,8 @@ var init_listen = __esm({
           model: this.model,
           lastStatus: `loading whisper ${this.model} model...`
         });
+        const consensusEnv = String(process.env["OMNIUS_ASR_CONSENSUS"] ?? "").trim().toLowerCase();
+        const consensusModel = consensusEnv === "0" || consensusEnv === "off" || consensusEnv === "false" ? "off" : consensusEnv || (this.model === "tiny" ? "base" : "tiny");
         this.process = spawn29(
           pyPath,
           [
@@ -603518,7 +603520,9 @@ var init_listen = __esm({
             "--chunk-seconds",
             "3",
             "--window-seconds",
-            "10"
+            "10",
+            "--consensus-model",
+            consensusModel
           ],
           {
             stdio: ["pipe", "pipe", "pipe"],
@@ -603555,6 +603559,11 @@ var init_listen = __esm({
                     isFinal: evt.isFinal ?? false
                   });
                   break;
+                case "consensus_rejected":
+                  updateListenLiveState({
+                    lastStatus: `consensus rejected (${evt.reason ?? "mismatch"}): ${String(evt.primary ?? "").slice(0, 80)}`
+                  });
+                  break;
                 case "error":
                   this.emit("error", new Error(evt.message));
                   break;
@@ -605469,7 +605478,7 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
   }
   if (snapshot.config.selectedAudioInput) lines.push(`Selected audio input: ${snapshot.config.selectedAudioInput}`);
   if (snapshot.config.selectedAudioOutput) lines.push(`Selected audio output: ${snapshot.config.selectedAudioOutput}`);
-  const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "clip_recognition" || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 18);
+  const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "clip_recognition" || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, toolCapable ? 18 : 6);
   if (activeVideo) {
     lines.push("");
     lines.push("## LIVE CAMERA EVENTS");
@@ -605519,10 +605528,10 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
     if (snapshot.video.framePath) lines.push(`frame: ${snapshot.video.framePath}`);
     const classifications = cameraClassificationsSummary(snapshot.video);
     if (classifications) lines.push(`Live classifications: ${classifications}`);
-    if (snapshot.video.asciiContext) lines.push(snapshot.video.asciiContext);
+    if (toolCapable && snapshot.video.asciiContext) lines.push(snapshot.video.asciiContext);
     if (snapshot.video.inference) {
       lines.push("Video inference:");
-      lines.push(cleanPreview(snapshot.video.inference, 1800));
+      lines.push(cleanPreview(snapshot.video.inference, toolCapable ? 1800 : 600));
     }
   }
   if (cameraSnapshots.length > 1) {
@@ -614228,6 +614237,7 @@ __export(render_exports, {
   SLASH_COMMANDS: () => SLASH_COMMANDS2,
   applyCollapseToBox: () => applyCollapseToBox,
   breakTelegramCoalesce: () => breakTelegramCoalesce,
+  breakVoiceChatCoalesce: () => breakVoiceChatCoalesce,
   c: () => c3,
   charWidth: () => charWidth2,
   fileLink: () => fileLink,
@@ -614264,6 +614274,7 @@ __export(render_exports, {
   renderUserInterrupt: () => renderUserInterrupt,
   renderUserMessage: () => renderUserMessage,
   renderVerbose: () => renderVerbose,
+  renderVoiceChatCoalescedRow: () => renderVoiceChatCoalescedRow,
   renderVoiceText: () => renderVoiceText,
   renderWarning: () => renderWarning,
   setColorsEnabled: () => setColorsEnabled,
@@ -615020,6 +615031,63 @@ function renderTelegramCoalescedRow(row, opts) {
   );
   host.appendDynamicBlock(id2);
 }
+function breakVoiceChatCoalesce() {
+  _voiceChatCoalesce = null;
+}
+function renderVoiceChatCoalescedRow(row, opts) {
+  const colorCode = opts?.colorCode ?? 45;
+  const title = opts?.title ?? "☏ VoiceChat";
+  const redir = _contentWriteHook?.redirect?.();
+  const host = _contentWriteHook?.dynamicBlockHost?.() ?? null;
+  if (redir || !host || typeof host.refreshDynamicBlocks !== "function") {
+    _voiceChatCoalesce = null;
+    const text2 = `${row}
+`;
+    if (redir) redir(text2);
+    else process.stdout.write(text2);
+    return;
+  }
+  if (_voiceChatCoalesce) {
+    _voiceChatCoalesce.rows.push(row);
+    host.refreshDynamicBlocks();
+    return;
+  }
+  const id2 = `voicechat-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  const state = { id: id2, rows: [row], expanded: false, colorCode };
+  _voiceChatCoalesce = state;
+  host.registerDynamicBlock(id2, (width) => {
+    const visible = state.expanded ? state.rows : state.rows.slice(-VOICECHAT_VISIBLE_ROWS);
+    const hidden = state.rows.length - visible.length;
+    const body = [];
+    if (!state.expanded && hidden > 0) {
+      body.push({ text: `▸ ${hidden} ${VOICECHAT_EXPAND_LABEL}`, mode: "truncate", kind: "dim" });
+    } else if (state.expanded && state.rows.length > VOICECHAT_VISIBLE_ROWS) {
+      body.push({ text: `▾ ${VOICECHAT_COLLAPSE_LABEL}`, mode: "truncate", kind: "dim" });
+    }
+    for (const text2 of visible) {
+      body.push({ text: text2, mode: "wrap", kind: "plain" });
+    }
+    return buildToolBoxLines(
+      {
+        title,
+        metrics: state.rows.length > 2 ? `${Math.ceil(state.rows.length / 2)} turns` : "",
+        body,
+        colorCode: state.colorCode
+      },
+      width
+    );
+  });
+  host.registerDynamicBlockClickHandler?.(id2, (event) => {
+    const line = event.lineText ?? "";
+    if (line.includes(VOICECHAT_EXPAND_LABEL) || line.includes(VOICECHAT_COLLAPSE_LABEL)) {
+      state.expanded = !state.expanded;
+      host.refreshDynamicBlocks?.();
+      return true;
+    }
+    return false;
+  });
+  host.appendDynamicBlock(id2);
+}
 function appendContextBoxSection(body, label, value2) {
   const text2 = String(value2 ?? "").trim();
   if (!text2) return;
@@ -615893,7 +615961,7 @@ function formatDuration4(ms) {
   const secs = Math.floor(totalSecs % 60);
   return `${mins}m ${secs}s`;
 }
-var c3, ui, pastel, _emojisEnabled, _colorsEnabled, MD, TOOL_LABELS, TOOL_COLOR_CODES, TOOL_ERROR_COLOR_CODE, BOX_TL2, BOX_TR2, BOX_BL2, BOX_BR2, BOX_H2, BOX_V2, BOX_TJ_L2, BOX_TJ_R2, RESET2, _telegramCoalesce, _pendingToolCall, DIFF_LINE_RE, _contentWriteHook, HINTS, TOOL_NAMES, COMMAND_NAMES, SLASH_COMMANDS2;
+var c3, ui, pastel, _emojisEnabled, _colorsEnabled, MD, TOOL_LABELS, TOOL_COLOR_CODES, TOOL_ERROR_COLOR_CODE, BOX_TL2, BOX_TR2, BOX_BL2, BOX_BR2, BOX_H2, BOX_V2, BOX_TJ_L2, BOX_TJ_R2, RESET2, _telegramCoalesce, _pendingToolCall, _voiceChatCoalesce, VOICECHAT_VISIBLE_ROWS, VOICECHAT_EXPAND_LABEL, VOICECHAT_COLLAPSE_LABEL, DIFF_LINE_RE, _contentWriteHook, HINTS, TOOL_NAMES, COMMAND_NAMES, SLASH_COMMANDS2;
 var init_render = __esm({
   "packages/cli/src/tui/render.ts"() {
     "use strict";
@@ -616054,6 +616122,10 @@ var init_render = __esm({
     RESET2 = "\x1B[0m";
     _telegramCoalesce = null;
     _pendingToolCall = null;
+    _voiceChatCoalesce = null;
+    VOICECHAT_VISIBLE_ROWS = 20;
+    VOICECHAT_EXPAND_LABEL = "earlier turns — click to read full transcript";
+    VOICECHAT_COLLAPSE_LABEL = "full transcript — click to show recent only";
     DIFF_LINE_RE = /^\s*\d+\s([-+])\s/;
     _contentWriteHook = null;
     HINTS = [
@@ -696639,11 +696711,25 @@ function buildRealtimeVoiceMessages(turns, maxMessages = MAX_REALTIME_MODEL_MESS
       dynamic.push(turn);
     }
   }
-  if (latestSnapshot) dynamic.push(latestSnapshot);
+  if (latestSnapshot) {
+    let lastUserIdx = -1;
+    for (let i2 = dynamic.length - 1; i2 >= 0; i2--) {
+      if (dynamic[i2].role === "user") {
+        lastUserIdx = i2;
+        break;
+      }
+    }
+    if (lastUserIdx >= 0) dynamic.splice(lastUserIdx, 0, latestSnapshot);
+    else dynamic.push(latestSnapshot);
+  }
   const available = Math.max(0, maxMessages - pinned.length);
-  return [...pinned, ...dynamic.slice(-available)];
+  let windowed = dynamic.slice(-available);
+  if (latestSnapshot && available > 0 && !windowed.includes(latestSnapshot)) {
+    windowed = [latestSnapshot, ...windowed.slice(-(available - 1))];
+  }
+  return [...pinned, ...windowed];
 }
-var VAD_SILENCE_MS, MAX_SEGMENT_MS, SUMMARY_INJECTION_INTERVAL, MAX_CONTEXT_TURNS, MAX_REALTIME_MODEL_MESSAGES, CONTEXT_SNAPSHOT_PREFIX, MAX_VOICE_REPLY_CHARS, AGENT_ECHO_WINDOW_MS, AGENT_ECHO_SIMILARITY, SYSTEM_PROMPT2, MIN_SIGNAL_SCORE, NOISE_ONLY_RE, VoiceChatSession;
+var VAD_SILENCE_MS, MAX_SEGMENT_MS, SUMMARY_INJECTION_INTERVAL, MAX_CONTEXT_TURNS, MAX_REALTIME_MODEL_MESSAGES, CONTEXT_SNAPSHOT_PREFIX, CONTEXT_SNAPSHOT_SCOPE, MAX_VOICE_REPLY_CHARS, AGENT_ECHO_WINDOW_MS, AGENT_ECHO_SIMILARITY, SYSTEM_PROMPT2, MIN_SIGNAL_SCORE, NOISE_ONLY_RE, VoiceChatSession;
 var init_voicechat = __esm({
   "packages/cli/src/tui/voicechat.ts"() {
     "use strict";
@@ -696652,14 +696738,17 @@ var init_voicechat = __esm({
     SUMMARY_INJECTION_INTERVAL = 4;
     MAX_CONTEXT_TURNS = 20;
     MAX_REALTIME_MODEL_MESSAGES = 16;
-    CONTEXT_SNAPSHOT_PREFIX = "Context snapshot (read-only):";
+    CONTEXT_SNAPSHOT_PREFIX = "[ACTIVE CONTEXT FRAME]";
+    CONTEXT_SNAPSHOT_SCOPE = "Scope: live session + perception state for your next reply. Treat this as the single merged context intake — background state, not a topic. Answer the user's latest message; do not narrate or re-emit this frame unless the user asks about it.";
     MAX_VOICE_REPLY_CHARS = 700;
     AGENT_ECHO_WINDOW_MS = 12e3;
     AGENT_ECHO_SIMILARITY = 0.72;
     SYSTEM_PROMPT2 = `You are a voice assistant having a live spoken conversation. Keep responses extremely brief — 1-2 sentences max. You're speaking aloud, not writing. Be conversational, direct, and helpful. Don't use markdown or formatting — just natural speech.
 
 Rules:
-- Live perception: each turn you may receive a read-only "Context snapshot" system message containing what the camera currently sees (objects, people, classifications, recent visual events) and what is currently heard (sound scene, recent sounds, transcripts). Treat it as your own live sight and hearing. When the user asks what you see, who is there, or what you hear, answer directly from that snapshot. If the snapshot is missing or stale, say you can't see/hear clearly right now — do not invent observations.
+- ALWAYS answer the user's most recent message directly. The live context snapshot is background perception — never the topic. Do not describe cameras, feeds, or surroundings unless the user's latest message asks about them.
+- Never repeat your previous answer. If you already described the scene, don't describe it again unless explicitly asked again — respond to what the user just said.
+- Live perception: each turn you may receive a read-only "[ACTIVE CONTEXT FRAME]" system message containing what the camera currently sees (objects, people, classifications, recent visual events) and what is currently heard (sound scene, recent sounds, transcripts). Treat it as your own live sight and hearing. When the user asks what you SEE, answer from the camera fields; when the user asks what you HEAR, answer from the audio/sound fields. If the snapshot is missing or stale, say you can't see/hear clearly right now — do not invent observations.
 - Never invent environment facts (cwd, OS, specs, repo state). If you need a precise fact from the main agent, request a tool by outputting on a single line EXACTLY one JSON object: {"tool": string, "args": object} and nothing else. Then wait for the tool result before answering.
 - You may also request to relay a user task to the main agent by emitting {"tool":"voice_to_main","args":{"message":"...","start":true}}.
 - Prefer tools for factual queries; otherwise, answer directly with a short reply.
@@ -696948,8 +697037,14 @@ Rules:
               const snap = await Promise.resolve(this.toolRelay.contextSnapshot());
               if (snap && snap.trim()) {
                 this.dropTransientSystemTurns();
-                this.context.push({ role: "system", content: `Context snapshot (read-only):
-${snap.trim()}` });
+                this.context.push({
+                  role: "system",
+                  content: `${CONTEXT_SNAPSHOT_PREFIX}
+turn: ${this.turnCount}
+${CONTEXT_SNAPSHOT_SCOPE}
+
+${snap.trim()}`
+                });
                 this.trimContext();
               }
             } catch {
@@ -697015,6 +697110,19 @@ ${toolOutput}` });
             this.context.push({ role: "system", content: `You have tools. Use them. ${this.toolCatalogNote}` });
             response = await this.streamOllamaInference(this.abortController.signal);
           }
+          if (this.lastAgentSpeech?.text && response.trim()) {
+            const prevTokens = comparableTokens(this.lastAgentSpeech.text);
+            const nextTokens = comparableTokens(stripToolJsonLines(response));
+            const similarity3 = jaccardSimilarity(prevTokens, nextTokens);
+            if (similarity3 >= 0.8 && nextTokens.length >= 8) {
+              this.context.push({
+                role: "system",
+                content: `Your draft reply repeats your previous answer. Do NOT describe the scene or cameras again. Respond directly and briefly to the user's latest message: "${truncateForLog(lastUser ?? "", 200)}"`
+              });
+              const retry = await this.streamOllamaInference(this.abortController.signal);
+              if (retry.trim()) response = retry;
+            }
+          }
           if (response.trim()) {
             const reply = extractVoiceModelReply(stripToolJsonLines(response.trim()));
             const finalSpoken = reply.text;
@@ -736377,14 +736485,16 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
           writeContent(() => renderInfo(`[voicechat] ${msg}`));
         },
         onUserSpeech(text2) {
-          writeContent(() => renderInfo(`\x1B[38;5;45m[you]\x1B[0m ${text2}`));
+          writeContent(
+            () => renderVoiceChatCoalescedRow(`\x1B[38;5;45m[you]\x1B[0m ${text2}`)
+          );
         },
         // Suppressed to keep main loop quiet
         onPartialTranscript(_text) {
         },
         onAgentSpeech(text2) {
           writeContent(
-            () => renderInfo(
+            () => renderVoiceChatCoalescedRow(
               `\x1B[38;5;37m[agent]\x1B[0m ${truncateByLines(text2, 5, 600)}`
             )
           );
@@ -736400,6 +736510,7 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
         await _voiceChatSession2.stop();
         _voiceChatSession2 = null;
       }
+      breakVoiceChatCoalesce();
     },
     isVoiceChatActive() {
       return _voiceChatSession2?.isActive ?? false;

package/dist/scripts/live-whisper.py

@@ -129,6 +129,44 @@ import numpy as np
 # Main transcription loop
 # ---------------------------------------------------------------------------
 
+def _norm_tokens(text):
+    """Normalize a transcript for cross-model comparison: lowercase, strip
+    punctuation, collapse whitespace, drop empty tokens."""
+    import re
+    return [t for t in re.sub(r"[^\w\s]", " ", text.lower()).split() if t]
+
+
+def _texts_agree(a: str, b: str, threshold: float) -> bool:
+    """Consensus check between two model transcripts. Real speech produces
+    similar output from different whisper models; hallucinations on noise are
+    high-variance and virtually never match across models."""
+    ta, tb = _norm_tokens(a), _norm_tokens(b)
+    if not ta or not tb:
+        return False
+    # Containment: short utterances often differ only by dropped filler words
+    ja, jb = " ".join(ta), " ".join(tb)
+    if ja in jb or jb in ja:
+        return True
+    import difflib
+    return difflib.SequenceMatcher(None, ta, tb).ratio() >= threshold
+
+
+def _segment_filtered_text(result) -> str:
+    """Rebuild transcript from segments, dropping low-confidence and
+    likely-non-speech segments (classic hallucination carriers)."""
+    segments = result.get("segments") or []
+    if not segments:
+        return result.get("text", "").strip()
+    kept = []
+    for seg in segments:
+        if float(seg.get("no_speech_prob", 0.0)) > 0.6:
+            continue
+        if float(seg.get("avg_logprob", 0.0)) < -1.2:
+            continue
+        kept.append(str(seg.get("text", "")))
+    return " ".join(part.strip() for part in kept if part.strip()).strip()
+
+
 def main():
     import argparse
     parser = argparse.ArgumentParser(description="Live Whisper transcription worker")
@@ -136,6 +174,10 @@ def main():
     parser.add_argument("--chunk-seconds", type=float, default=CHUNK_SECONDS, help="Transcribe interval")
     parser.add_argument("--window-seconds", type=float, default=WINDOW_SECONDS, help="Sliding window size")
     parser.add_argument("--language", default=None, help="Language code (e.g. en, es, fr). Auto-detect if omitted.")
+    parser.add_argument("--consensus-model", default="tiny",
+                        help="Second whisper model run in parallel on the same audio; transcripts are only emitted when both models agree. 'off' disables.")
+    parser.add_argument("--consensus-threshold", type=float, default=0.55,
+                        help="Token similarity (0-1) required between the two models' transcripts.")
     args = parser.parse_args()
 
     import whisper
@@ -156,6 +198,19 @@ def main():
         emit_error(f"Failed to load model: {e}")
         sys.exit(1)
 
+    # Consensus model: a second (usually smaller) model transcribes the same
+    # window in parallel. Hallucinations (mixed-language garbage on noisy or
+    # quiet audio) don't reproduce across models, so disagreement = reject.
+    consensus_model = None
+    consensus_name = (args.consensus_model or "off").strip().lower()
+    if consensus_name not in ("off", "none", "", args.model):
+        emit_status(f"Loading consensus Whisper {consensus_name} model...")
+        try:
+            consensus_model = whisper.load_model(consensus_name, device=device)
+        except Exception as e:
+            emit_status(f"Consensus model unavailable ({e}); running single-model.")
+            consensus_model = None
+
     emit({"type": "ready"})
 
     # Audio buffer — accumulates PCM16 as float32 @ 16kHz
@@ -235,17 +290,48 @@ def main():
             # Amplify toward full scale before transcription
             window = amplify(window)
 
-            # Transcribe
+            # Transcribe — primary and consensus models in parallel threads
+            # (torch releases the GIL during compute, so this is real
+            # parallelism on multicore ARM).
             try:
                 fp16 = (device == "cuda")
-                result = model.transcribe(
-                    window,
-                    fp16=fp16,
-                    language=args.language,
-                    no_speech_threshold=0.6,
-                    condition_on_previous_text=False,
-                )
-                text = result.get("text", "").strip()
+
+                def run_transcribe(m):
+                    return m.transcribe(
+                        window,
+                        fp16=fp16,
+                        language=args.language,
+                        no_speech_threshold=0.6,
+                        condition_on_previous_text=False,
+                    )
+
+                if consensus_model is not None:
+                    from concurrent.futures import ThreadPoolExecutor
+                    with ThreadPoolExecutor(max_workers=2) as pool:
+                        primary_future = pool.submit(run_transcribe, model)
+                        secondary_future = pool.submit(run_transcribe, consensus_model)
+                        result = primary_future.result()
+                        secondary = secondary_future.result()
+                else:
+                    result = run_transcribe(model)
+                    secondary = None
+
+                text = _segment_filtered_text(result)
+
+                if text and secondary is not None:
+                    secondary_text = _segment_filtered_text(secondary)
+                    lang_a = str(result.get("language", "")).lower()
+                    lang_b = str(secondary.get("language", "")).lower()
+                    lang_disagree = bool(lang_a and lang_b and lang_a != lang_b and not args.language)
+                    if lang_disagree or not _texts_agree(text, secondary_text, args.consensus_threshold):
+                        emit({
+                            "type": "consensus_rejected",
+                            "primary": text[:200],
+                            "secondary": (secondary_text or "")[:200],
+                            "reason": "language_mismatch" if lang_disagree else "low_similarity",
+                        })
+                        continue
+
                 if text and text != last_text:
                     last_text = text
                     emit_transcript(text, is_final=False)
@@ -270,7 +356,7 @@ def main():
                 fp16=fp16,
                 language=args.language,
             )
-            text = result.get("text", "").strip()
+            text = _segment_filtered_text(result)
             if text:
                 emit_transcript(text, is_final=True)
         except Exception:

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "omnius",
-  "version": "1.0.419",
+  "version": "1.0.421",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "omnius",
-      "version": "1.0.419",
+      "version": "1.0.421",
       "bundleDependencies": [
         "image-to-ascii"
       ],
@@ -2505,9 +2505,9 @@
       }
     },
     "node_modules/bare-fs": {
-      "version": "4.7.2",
-      "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.2.tgz",
-      "integrity": "sha512-aTvMFUWkBmjzKtEQMDGGDNF8bkfpD5N1b/FCwt7A3wrU4t1o/e/85Wzkluh6JlODCjqVESYCkQCdTXqZ9G7VFg==",
+      "version": "4.7.3",
+      "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.3.tgz",
+      "integrity": "sha512-xRgplks8SvcKkdlv2M6Z2LZmRsmqd+x0nXXGXeMEjwdibj1HSDrlnqBRLeYdMvsgCox7Bq0e+DHwfczOfsn6IA==",
       "license": "Apache-2.0",
       "optional": true,
       "dependencies": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnius",
-  "version": "1.0.419",
+  "version": "1.0.421",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",