omnius 1.0.418 → 1.0.420

package/dist/index.js

@@ -603059,6 +603059,21 @@ async function resolveMicSource() {
   const best = candidates[0];
   return { pulseSource: best.name, channels: best.channels, label: `pulse:${best.name}` };
 }
+async function ensureSourceCaptureVolume(source) {
+  try {
+    await execFileText4("pactl", ["set-source-mute", source, "0"], { timeout: 3e3 });
+  } catch {
+  }
+  try {
+    const raw = await execFileText4("pactl", ["get-source-volume", source], { timeout: 3e3 });
+    const percents = [...raw.matchAll(/(\d+)%/g)].map((m2) => Number(m2[1])).filter((n2) => Number.isFinite(n2));
+    const current = percents.length ? Math.max(...percents) : null;
+    if (current !== null && current < 100) {
+      await execFileText4("pactl", ["set-source-volume", source, "100%"], { timeout: 3e3 });
+    }
+  } catch {
+  }
+}
 function micFfmpegFilter(channels) {
   const parts = [];
   if ((channels ?? 1) > 2) parts.push("pan=mono|c0=c0");
@@ -603079,6 +603094,7 @@ async function findMicCaptureCommand() {
       };
     }
     if (resolved?.pulseSource && await commandExists2("ffmpeg")) {
+      await ensureSourceCaptureVolume(resolved.pulseSource);
       return {
         cmd: "ffmpeg",
         args: [
@@ -603687,7 +603703,7 @@ var init_listen = __esm({
         else if (levelDb < this.micNoiseFloorDb) this.micNoiseFloorDb = 0.3 * levelDb + 0.7 * this.micNoiseFloorDb;
         else this.micNoiseFloorDb = 0.02 * levelDb + 0.98 * this.micNoiseFloorDb;
         const floor = this.micNoiseFloorDb;
-        const speech = this.micLevelDb > Math.max(-52, floor + 8);
+        const speech = this.micLevelDb > Math.max(-58, floor + 6);
         const changed = speech !== this.micSpeechActive;
         this.micSpeechActive = speech;
         const now2 = Date.now();
@@ -604575,7 +604591,7 @@ function readPcm16WavActivity(filePath, bins = 36) {
       const abs = Math.abs(sample);
       peak = Math.max(peak, abs);
       sumSquares += sample * sample;
-      if (abs > 0.018) active++;
+      if (abs > 8e-3) active++;
       const b = Math.min(bucketSquares.length - 1, Math.floor(i2 / frames * bucketSquares.length));
       bucketSquares[b] += sample * sample;
       bucketCounts[b] += 1;
@@ -605411,16 +605427,22 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
   if (!snapshot) return "";
   const now2 = opts.now ?? Date.now();
   const maxAgeMs = opts.maxAgeMs ?? 9e4;
+  const toolCapable = opts.toolCapable !== false;
   const lines = [];
   const activeVideo = snapshot.config.videoEnabled || Boolean(snapshot.video);
   const activeAudio = snapshot.config.audioEnabled || snapshot.config.audioOutputEnabled || Boolean(snapshot.audio);
   if (!activeVideo && !activeAudio) return "";
   lines.push("<live-sensor-context>");
   lines.push("## LIVE SENSOR STATUS");
-  lines.push("Live sensor context is operator-enabled ambient input. Treat it as current perceptual evidence for this turn; if it is stale, missing, or insufficient, use camera/audio tools to refresh before answering what you see or hear.");
-  lines.push('Live world-model rule: when the user asks what is around, who is present, what changed, what is heard, or asks a deictic question like "what do you see", treat the live camera/audio fields as first-class context. Be curious and tool-capable: refresh stale sensors, inspect frame paths, query vision/CLIP/visual_memory/audio tools, and combine metadata into a concise answer. Do not rely only on summaries when the task requires current perception.');
-  lines.push("Live speech rule: in low-latency voice/live modes, short spoken replies are appropriate for direct address, identity clarification, or important visual/audio triggers. Do not narrate every frame.");
-  lines.push("Identity rule: never invent names for observed people. If a person is unknown and identity matters, ask the user or the person for their name, then store the association with visual_memory(action='enroll', image='<face crop>', name='<name>') when a face crop is available, or multimodal_memory(action='meet', person_name='<name>') when live voice/name context is available.");
+  if (toolCapable) {
+    lines.push("Live sensor context is operator-enabled ambient input. Treat it as current perceptual evidence for this turn; if it is stale, missing, or insufficient, use camera/audio tools to refresh before answering what you see or hear.");
+    lines.push('Live world-model rule: when the user asks what is around, who is present, what changed, what is heard, or asks a deictic question like "what do you see", treat the live camera/audio fields as first-class context. Be curious and tool-capable: refresh stale sensors, inspect frame paths, query vision/CLIP/visual_memory/audio tools, and combine metadata into a concise answer. Do not rely only on summaries when the task requires current perception.');
+    lines.push("Live speech rule: in low-latency voice/live modes, short spoken replies are appropriate for direct address, identity clarification, or important visual/audio triggers. Do not narrate every frame.");
+    lines.push("Identity rule: never invent names for observed people. If a person is unknown and identity matters, ask the user or the person for their name, then store the association with visual_memory(action='enroll', image='<face crop>', name='<name>') when a face crop is available, or multimodal_memory(action='meet', person_name='<name>') when live voice/name context is available.");
+  } else {
+    lines.push("This is your current live perception: what the camera sees and what is heard right now. Answer questions about sight/sound directly from these fields.");
+    lines.push("If the data below is stale or missing, say you cannot see/hear clearly right now. Never invent observations or names for people. To have the main agent investigate further, relay a request with the voice_to_main tool.");
+  }
   lines.push(`Snapshot file: ${liveSnapshotPath(snapshot.repoRoot)}`);
   lines.push(`Streams: video=${snapshot.config.videoEnabled ? "on" : "off"} infer=${snapshot.config.inferEnabled ? "on" : "off"} clip=${snapshot.config.clipEnabled ? "on" : "off"} audio=${snapshot.config.audioEnabled ? "on" : "off"} asr=${snapshot.config.asrEnabled ? "on" : "off"} sounds=${snapshot.config.audioAnalysisEnabled ? "on" : "off"} output-monitor=${snapshot.config.audioOutputEnabled ? "on" : "off"}`);
   if (snapshot.config.selectedCamera) {
@@ -605563,8 +605585,8 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
 ... live sensor context truncated ...
 </live-sensor-context>` : text2;
 }
-function formatLiveSensorContext(repoRoot) {
-  return formatLiveSensorContextFromSnapshot(readLiveSensorSnapshot(repoRoot));
+function formatLiveSensorContext(repoRoot, opts = {}) {
+  return formatLiveSensorContextFromSnapshot(readLiveSensorSnapshot(repoRoot), opts);
 }
 function isLiveSensorSnapshotActive(snapshot) {
   const cfg = snapshot?.config;
@@ -605737,7 +605759,7 @@ async function tryRecordAudioDevice(device, durationSec, outputPath3, channels)
 }
 function hasLiveAudioSignal(activity) {
   if (!activity) return false;
-  return activity.peak >= 0.025 || activity.rmsDb > -48 || activity.activeRatio >= 0.025;
+  return activity.peak >= 0.012 || activity.rmsDb > -55 || activity.activeRatio >= 0.015;
 }
 function audioCaptureMethod(device) {
   return device.startsWith("pulse:") ? "pulse/ffmpeg" : "arecord";
@@ -696617,9 +696639,23 @@ function buildRealtimeVoiceMessages(turns, maxMessages = MAX_REALTIME_MODEL_MESS
       dynamic.push(turn);
     }
   }
-  if (latestSnapshot) dynamic.push(latestSnapshot);
+  if (latestSnapshot) {
+    let lastUserIdx = -1;
+    for (let i2 = dynamic.length - 1; i2 >= 0; i2--) {
+      if (dynamic[i2].role === "user") {
+        lastUserIdx = i2;
+        break;
+      }
+    }
+    if (lastUserIdx >= 0) dynamic.splice(lastUserIdx, 0, latestSnapshot);
+    else dynamic.push(latestSnapshot);
+  }
   const available = Math.max(0, maxMessages - pinned.length);
-  return [...pinned, ...dynamic.slice(-available)];
+  let windowed = dynamic.slice(-available);
+  if (latestSnapshot && available > 0 && !windowed.includes(latestSnapshot)) {
+    windowed = [latestSnapshot, ...windowed.slice(-(available - 1))];
+  }
+  return [...pinned, ...windowed];
 }
 var VAD_SILENCE_MS, MAX_SEGMENT_MS, SUMMARY_INJECTION_INTERVAL, MAX_CONTEXT_TURNS, MAX_REALTIME_MODEL_MESSAGES, CONTEXT_SNAPSHOT_PREFIX, MAX_VOICE_REPLY_CHARS, AGENT_ECHO_WINDOW_MS, AGENT_ECHO_SIMILARITY, SYSTEM_PROMPT2, MIN_SIGNAL_SCORE, NOISE_ONLY_RE, VoiceChatSession;
 var init_voicechat = __esm({
@@ -696637,6 +696673,9 @@ var init_voicechat = __esm({
     SYSTEM_PROMPT2 = `You are a voice assistant having a live spoken conversation. Keep responses extremely brief — 1-2 sentences max. You're speaking aloud, not writing. Be conversational, direct, and helpful. Don't use markdown or formatting — just natural speech.
 
 Rules:
+- ALWAYS answer the user's most recent message directly. The live context snapshot is background perception — never the topic. Do not describe cameras, feeds, or surroundings unless the user's latest message asks about them.
+- Never repeat your previous answer. If you already described the scene, don't describe it again unless explicitly asked again — respond to what the user just said.
+- Live perception: each turn you may receive a read-only "Context snapshot" system message containing what the camera currently sees (objects, people, classifications, recent visual events) and what is currently heard (sound scene, recent sounds, transcripts). Treat it as your own live sight and hearing. When the user asks what you SEE, answer from the camera fields; when the user asks what you HEAR, answer from the audio/sound fields. If the snapshot is missing or stale, say you can't see/hear clearly right now — do not invent observations.
 - Never invent environment facts (cwd, OS, specs, repo state). If you need a precise fact from the main agent, request a tool by outputting on a single line EXACTLY one JSON object: {"tool": string, "args": object} and nothing else. Then wait for the tool result before answering.
 - You may also request to relay a user task to the main agent by emitting {"tool":"voice_to_main","args":{"message":"...","start":true}}.
 - Prefer tools for factual queries; otherwise, answer directly with a short reply.
@@ -696992,6 +697031,19 @@ ${toolOutput}` });
             this.context.push({ role: "system", content: `You have tools. Use them. ${this.toolCatalogNote}` });
             response = await this.streamOllamaInference(this.abortController.signal);
           }
+          if (this.lastAgentSpeech?.text && response.trim()) {
+            const prevTokens = comparableTokens(this.lastAgentSpeech.text);
+            const nextTokens = comparableTokens(stripToolJsonLines(response));
+            const similarity3 = jaccardSimilarity(prevTokens, nextTokens);
+            if (similarity3 >= 0.8 && nextTokens.length >= 8) {
+              this.context.push({
+                role: "system",
+                content: `Your draft reply repeats your previous answer. Do NOT describe the scene or cameras again. Respond directly and briefly to the user's latest message: "${truncateForLog(lastUser ?? "", 200)}"`
+              });
+              const retry = await this.streamOllamaInference(this.abortController.signal);
+              if (retry.trim()) response = retry;
+            }
+          }
           if (response.trim()) {
             const reply = extractVoiceModelReply(stripToolJsonLines(response.trim()));
             const finalSpoken = reply.text;
@@ -736343,7 +736395,7 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
             } else {
               parts.push("active: no");
             }
-            const liveContext = formatLiveSensorContext(repoRoot);
+            const liveContext = formatLiveSensorContext(repoRoot, { toolCapable: false, maxChars: 7e3 });
             if (liveContext) parts.push(liveContext);
             return parts.join("\n");
           }

package/dist/scripts/live-whisper.py

@@ -165,6 +165,24 @@ def main():
     window_samples = int(args.window_seconds * SAMPLE_RATE)
     last_text = ""
     running = True
+    noise_floor = None  # EMA of the quietest 0.5s block RMS — adapts to the capture chain
+
+    def block_energies(samples):
+        """RMS of consecutive 0.5s blocks — lets speech anywhere in the
+        window register even when most of the window is silence."""
+        block = SAMPLE_RATE // 2
+        n = max(1, len(samples) // block)
+        return [float(np.sqrt(np.mean(samples[i * block:(i + 1) * block] ** 2))) for i in range(n)]
+
+    def amplify(samples):
+        """Bring quiet-but-real speech toward full scale. Whisper performs far
+        better near normal levels; quiet capture chains (distant speakers,
+        low source gain, beamformed arrays) otherwise force users to shout
+        into the mic. Gain capped at 40x (~32dB), output clipped to [-1, 1]."""
+        peak = float(np.max(np.abs(samples))) if len(samples) else 0.0
+        if 0 < peak < 0.5:
+            samples = np.clip(samples * min(40.0, 0.9 / max(peak, 1e-4)), -1.0, 1.0)
+        return samples
 
     def read_stdin():
         """Read PCM16 from stdin in a background thread."""
@@ -197,18 +215,25 @@ def main():
                 # Take the last window_seconds of audio
                 window = audio_buf[-window_samples:].copy() if len(audio_buf) > window_samples else audio_buf.copy()
 
-            # Silence gate: whisper hallucinates repetitive garbage on
-            # near-silent input (quiet mics, diluted multichannel downmixes).
-            # Skip windows with no meaningful energy instead of transcribing.
-            rms = float(np.sqrt(np.mean(window ** 2))) if len(window) else 0.0
-            peak = float(np.max(np.abs(window))) if len(window) else 0.0
-            if rms < 3e-4 or peak < 2e-3:  # ~-70dB RMS: definitely silence
+            # Adaptive silence gate: whisper hallucinates repetitive garbage
+            # on silent input, but absolute thresholds reject real speech on
+            # quiet capture chains. Instead track the noise floor (quietest
+            # 0.5s block) and require the loudest block to rise above it.
+            energies = block_energies(window)
+            quiet = min(energies)
+            loud = max(energies)
+            if noise_floor is None:
+                noise_floor = quiet
+            elif quiet < noise_floor:
+                noise_floor = 0.7 * noise_floor + 0.3 * quiet  # drop fast
+            else:
+                noise_floor = 0.995 * noise_floor + 0.005 * quiet  # creep up slowly
+            gate = max(1.5e-4, noise_floor * 2.5)  # ≥ ~8dB above floor, ~-76dB min
+            if loud < gate:
                 continue
 
-            # Normalize quiet-but-real audio: whisper performs much better
-            # near full scale. Cap gain at 20x so noise floors aren't blasted.
-            if peak < 0.3:
-                window = window * min(20.0, 0.9 / max(peak, 1e-4))
+            # Amplify toward full scale before transcription
+            window = amplify(window)
 
             # Transcribe
             try:
@@ -234,12 +259,11 @@ def main():
     with buf_lock:
         full_audio = audio_buf.copy()
 
-    final_rms = float(np.sqrt(np.mean(full_audio ** 2))) if len(full_audio) else 0.0
-    if len(full_audio) >= SAMPLE_RATE and final_rms >= 3e-4:
+    final_loud = max(block_energies(full_audio)) if len(full_audio) >= SAMPLE_RATE else 0.0
+    final_gate = max(1.5e-4, (noise_floor or 0.0) * 2.5)
+    if len(full_audio) >= SAMPLE_RATE and final_loud >= final_gate:
         try:
-            final_peak = float(np.max(np.abs(full_audio)))
-            if 0 < final_peak < 0.3:
-                full_audio = full_audio * min(20.0, 0.9 / max(final_peak, 1e-4))
+            full_audio = amplify(full_audio)
             fp16 = (device == "cuda")
             result = model.transcribe(
                 full_audio,

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "omnius",
-  "version": "1.0.418",
+  "version": "1.0.420",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "omnius",
-      "version": "1.0.418",
+      "version": "1.0.420",
       "bundleDependencies": [
         "image-to-ascii"
       ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnius",
-  "version": "1.0.418",
+  "version": "1.0.420",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",