omnius 1.0.411 → 1.0.412

package/dist/index.js

@@ -41456,17 +41456,30 @@ function isTranscribable(path12) {
   const ext = extname4(path12).toLowerCase();
   return AUDIO_EXTS.has(ext) || VIDEO_EXTS.has(ext);
 }
+async function requireTranscribeCliFrom(packageDir) {
+  const { createRequire: createRequire11 } = await import("node:module");
+  const req3 = createRequire11(import.meta.url);
+  const distPath = join40(packageDir, "dist", "index.js");
+  if (existsSync38(distPath))
+    return req3(distPath);
+  return req3(packageDir);
+}
 async function loadTranscribeCli() {
   if (_tcChecked)
     return _tcModule;
   _tcChecked = true;
+  try {
+    const { createRequire: createRequire11 } = await import("node:module");
+    const req3 = createRequire11(import.meta.url);
+    _tcModule = req3("transcribe-cli");
+    return _tcModule;
+  } catch {
+  }
   try {
     const globalRoot = (await execFileText("npm", ["root", "-g"], { timeout: 5e3 })).trim();
     const tcPath = join40(globalRoot, "transcribe-cli");
     if (existsSync38(join40(tcPath, "dist", "index.js"))) {
-      const { createRequire: createRequire11 } = await import("node:module");
-      const req3 = createRequire11(import.meta.url);
-      _tcModule = req3(join40(tcPath, "dist", "index.js"));
+      _tcModule = await requireTranscribeCliFrom(tcPath);
       return _tcModule;
     }
   } catch {
@@ -41478,9 +41491,7 @@ async function loadTranscribeCli() {
       for (const ver of readdirSync61(nvmBase)) {
         const tcPath = join40(nvmBase, ver, "lib", "node_modules", "transcribe-cli");
         if (existsSync38(join40(tcPath, "dist", "index.js"))) {
-          const { createRequire: createRequire11 } = await import("node:module");
-          const req3 = createRequire11(import.meta.url);
-          _tcModule = req3(join40(tcPath, "dist", "index.js"));
+          _tcModule = await requireTranscribeCliFrom(tcPath);
           return _tcModule;
         }
       }
@@ -41489,6 +41500,58 @@ async function loadTranscribeCli() {
   }
   return null;
 }
+async function ensureManagedTranscribeCliRuntime() {
+  const packageDir = join40(MANAGED_TRANSCRIBE_CLI_DIR, "node_modules", "transcribe-cli");
+  if (existsSync38(join40(packageDir, "dist", "index.js"))) {
+    try {
+      _managedTranscribeCliReady = true;
+      _tcModule = await requireTranscribeCliFrom(packageDir);
+      _tcChecked = true;
+      return _tcModule;
+    } catch {
+    }
+  }
+  if (_managedTranscribeCliChecked && !_managedTranscribeCliReady)
+    return null;
+  _managedTranscribeCliChecked = true;
+  try {
+    mkdirSync22(MANAGED_TRANSCRIBE_CLI_DIR, { recursive: true });
+    if (!existsSync38(join40(MANAGED_TRANSCRIBE_CLI_DIR, "package.json"))) {
+      writeFileSync21(join40(MANAGED_TRANSCRIBE_CLI_DIR, "package.json"), JSON.stringify({ private: true, dependencies: {} }, null, 2), "utf8");
+    }
+    await execFileText("npm", ["install", "--prefix", MANAGED_TRANSCRIBE_CLI_DIR, "transcribe-cli@^2.0.1"], {
+      timeout: 18e4,
+      env: transcriptionPythonEnv()
+    });
+    _managedTranscribeCliReady = true;
+    _tcModule = await requireTranscribeCliFrom(packageDir);
+    _tcChecked = true;
+    return _tcModule;
+  } catch {
+    _managedTranscribeCliReady = false;
+    return null;
+  }
+}
+async function ensureTranscribeCliPythonModule(python) {
+  if (_transcribeCliPythonModuleChecked)
+    return;
+  _transcribeCliPythonModuleChecked = true;
+  try {
+    await execFileText(python, ["-c", "import transcribe_cli"], {
+      timeout: 1e4,
+      env: transcriptionPythonEnv()
+    });
+    return;
+  } catch {
+  }
+  try {
+    await execFileText(python, ["-m", "pip", "install", "-U", "transcribe-cli"], {
+      timeout: 3e5,
+      env: transcriptionPythonEnv()
+    });
+  } catch {
+  }
+}
 function isYouTubeUrl(url) {
   return /(?:youtube\.com\/(?:watch|shorts|live|embed|v\/)|youtu\.be\/)/i.test(url);
 }
@@ -41519,7 +41582,7 @@ function formatTime(seconds) {
   const s2 = Math.floor(seconds % 60);
   return `${String(m2).padStart(2, "0")}:${String(s2).padStart(2, "0")}`;
 }
-var AUDIO_EXTS, VIDEO_EXTS, MAX_TRANSCRIBE_URL_BYTES, MANAGED_ASR_VENV, _tcModule, _tcChecked, _managedAsrReady, TranscribeFileTool, YT_DLP_VENV, TranscribeUrlTool, YouTubeDownloadTool;
+var AUDIO_EXTS, VIDEO_EXTS, MAX_TRANSCRIBE_URL_BYTES, MANAGED_ASR_VENV, MANAGED_TRANSCRIBE_CLI_DIR, _tcModule, _tcChecked, _managedAsrReady, _managedTranscribeCliChecked, _managedTranscribeCliReady, _transcribeCliPythonModuleChecked, TranscribeFileTool, YT_DLP_VENV, TranscribeUrlTool, YouTubeDownloadTool;
 var init_transcribe_tool = __esm({
   "packages/execution/dist/tools/transcribe-tool.js"() {
     "use strict";
@@ -41551,9 +41614,13 @@ var init_transcribe_tool = __esm({
     ]);
     MAX_TRANSCRIBE_URL_BYTES = 100 * 1024 * 1024;
     MANAGED_ASR_VENV = join40(homedir11(), ".omnius", "runtimes", "asr", ".venv-whisper");
+    MANAGED_TRANSCRIBE_CLI_DIR = join40(homedir11(), ".omnius", "runtimes", "asr", "transcribe-cli-node");
     _tcModule = null;
     _tcChecked = false;
     _managedAsrReady = false;
+    _managedTranscribeCliChecked = false;
+    _managedTranscribeCliReady = false;
+    _transcribeCliPythonModuleChecked = false;
     TranscribeFileTool = class {
       name = "transcribe_file";
       description = "Transcribe a local audio or video file to text using Whisper (faster-whisper). Supports MP3, WAV, FLAC, AAC, M4A, OGG, MP4, MKV, AVI, MOV, WebM. Returns the full transcription text with optional speaker diarization. Transcription is 100% local — no API keys needed.";
@@ -41628,7 +41695,15 @@ var init_transcribe_tool = __esm({
         if (effectiveModel !== askedModel)
           model = effectiveModel;
         const failures = [];
-        const tc = await loadTranscribeCli();
+        let managedPython = "";
+        try {
+          managedPython = await this.ensureManagedWhisperRuntime();
+          process.env.TRANSCRIBE_PYTHON = managedPython;
+          await ensureTranscribeCliPythonModule(managedPython);
+        } catch (err) {
+          failures.push(`managed ASR runtime bootstrap: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        const tc = await loadTranscribeCli() ?? await ensureManagedTranscribeCliRuntime();
         if (tc) {
           try {
             const result = await withProcessEnv(transcriptionPythonEnv(), () => tc.transcribe(filePath, {
@@ -41641,6 +41716,8 @@ var init_transcribe_tool = __esm({
           } catch (err) {
             failures.push(`transcribe-cli module: ${err instanceof Error ? err.message : String(err)}`);
           }
+        } else {
+          failures.push("transcribe-cli npm package unavailable after managed auto-install");
         }
         const cli = await this.execViaCli(filePath, model, diarize, start2);
         if (cli.success)
@@ -602864,9 +602941,9 @@ function parseCameraOrientationDecision(value2) {
 }
 function formatCameraRotation(rotation) {
   if (rotation === 0) return "upright/no correction";
-  if (rotation === 90) return "90 degrees clockwise";
-  if (rotation === 180) return "180 degrees";
-  return "90 degrees counter-clockwise";
+  if (rotation === 90) return "90 degrees clockwise correction";
+  if (rotation === 180) return "180 degrees correction";
+  return "90 degrees counter-clockwise correction";
 }
 function sanitizeCameraOrientation(value2) {
   if (typeof value2 !== "object" || value2 === null || Array.isArray(value2)) return {};
@@ -603127,6 +603204,116 @@ function fallbackLiveAudioIntakeDecision(transcript) {
     reason: addressed ? "fallback detected direct address or question shape" : "fallback could not establish that ambient speech was addressed to Omnius"
   };
 }
+function parseLiveSpeechDecision(value2) {
+  const parsed = parseJsonObjectFromText(value2);
+  if (!parsed) return null;
+  const speak = parsed["speak"];
+  const confidence2 = Math.max(0, Math.min(1, Number(parsed["confidence"] ?? 0)));
+  if (typeof speak !== "boolean" || !Number.isFinite(confidence2)) return null;
+  const actionRaw = String(parsed["action"] ?? (speak ? "speak" : "silent")).toLowerCase();
+  const action = actionRaw === "speak" || actionRaw === "silent" || actionRaw === "queue_review" ? actionRaw : speak ? "speak" : "silent";
+  return {
+    speak,
+    text: trimOneLine(parsed["text"] ?? "", 180),
+    confidence: confidence2,
+    action,
+    reason: trimOneLine(parsed["reason"] ?? "", 220) || "live speech arbiter decision"
+  };
+}
+function fallbackLiveSpeechDecision(proposal) {
+  if (proposal.kind === "unknown_person") {
+    return {
+      speak: true,
+      text: proposal.defaultText,
+      confidence: 0.72,
+      action: "speak",
+      reason: "unknown person visible and identity clarification is useful"
+    };
+  }
+  if (proposal.kind === "audio_intent") {
+    return {
+      speak: true,
+      text: proposal.defaultText,
+      confidence: 0.68,
+      action: "speak",
+      reason: "audio intake classified speech as addressed to Omnius"
+    };
+  }
+  if (proposal.urgency === "high") {
+    return {
+      speak: true,
+      text: proposal.defaultText,
+      confidence: 0.6,
+      action: "speak",
+      reason: "high-urgency live trigger"
+    };
+  }
+  return {
+    speak: false,
+    text: "",
+    confidence: 0.55,
+    action: "silent",
+    reason: "ambient event did not justify interrupting with speech"
+  };
+}
+async function classifyLiveSpeech(proposal, config) {
+  const now2 = Date.now();
+  if (config?.backendUrl && config.model) {
+    const url = `${config.backendUrl.replace(/\/+$/, "")}/v1/chat/completions`;
+    const body = {
+      model: config.model,
+      temperature: 0,
+      max_tokens: 180,
+      think: false,
+      messages: [
+        {
+          role: "system",
+          content: [
+            "You are Omnius' live audio/video speech arbiter.",
+            "Decide whether the agent should speak aloud into the physical environment right now.",
+            "Speak rarely. Do not narrate routine detections or every frame.",
+            "Speak when directly addressed, when an unknown person should be politely identified, or when an important visual/audio trigger needs immediate clarification.",
+            "Never invent a person's identity. If asking identity, ask one short natural question.",
+            "Keep spoken text under 16 words.",
+            'Return only JSON: {"speak": boolean, "text": "short utterance", "confidence": 0.0-1.0, "action": "speak"|"silent"|"queue_review", "reason": "brief evidence"}.'
+          ].join("\n")
+        },
+        {
+          role: "user",
+          content: [
+            `kind=${proposal.kind}`,
+            `source=${proposal.source}`,
+            `urgency=${proposal.urgency}`,
+            `summary=${proposal.summary}`,
+            `evidence=${proposal.evidence}`,
+            `default_utterance=${proposal.defaultText}`
+          ].join("\n").slice(0, 1600)
+        }
+      ]
+    };
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), 5e3).unref();
+      const resp = await fetch(url, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          ...config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}
+        },
+        body: JSON.stringify(body),
+        signal: controller.signal
+      });
+      clearTimeout(timer);
+      if (resp.ok) {
+        const json = await resp.json();
+        const parsed = parseLiveSpeechDecision(json.choices?.[0]?.message?.content ?? "");
+        if (parsed) return { ...parsed, source: "live-arbiter", updatedAt: now2 };
+      }
+    } catch {
+    }
+  }
+  return { ...fallbackLiveSpeechDecision(proposal), source: "fallback", updatedAt: now2 };
+}
 async function classifyLiveAudioIntake(transcript, config) {
   const now2 = Date.now();
   const clean5 = transcript.trim();
@@ -603679,6 +603866,8 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
   lines.push("<live-sensor-context>");
   lines.push("## LIVE SENSOR STATUS");
   lines.push("Live sensor context is operator-enabled ambient input. Treat it as current perceptual evidence for this turn; if it is stale, missing, or insufficient, use camera/audio tools to refresh before answering what you see or hear.");
+  lines.push('Live world-model rule: when the user asks what is around, who is present, what changed, what is heard, or asks a deictic question like "what do you see", treat the live camera/audio fields as first-class context. Be curious and tool-capable: refresh stale sensors, inspect frame paths, query vision/CLIP/visual_memory/audio tools, and combine metadata into a concise answer. Do not rely only on summaries when the task requires current perception.');
+  lines.push("Live speech rule: in low-latency voice/live modes, short spoken replies are appropriate for direct address, identity clarification, or important visual/audio triggers. Do not narrate every frame.");
   lines.push("Identity rule: never invent names for observed people. If a person is unknown and identity matters, ask the user or the person for their name, then store the association with visual_memory(action='enroll', image='<face crop>', name='<name>') when a face crop is available, or multimodal_memory(action='meet', person_name='<name>') when live voice/name context is available.");
   lines.push(`Snapshot file: ${liveSnapshotPath(snapshot.repoRoot)}`);
   lines.push(`Streams: video=${snapshot.config.videoEnabled ? "on" : "off"} infer=${snapshot.config.inferEnabled ? "on" : "off"} clip=${snapshot.config.clipEnabled ? "on" : "off"} audio=${snapshot.config.audioEnabled ? "on" : "off"} asr=${snapshot.config.asrEnabled ? "on" : "off"} sounds=${snapshot.config.audioAnalysisEnabled ? "on" : "off"} output-monitor=${snapshot.config.audioOutputEnabled ? "on" : "off"}`);
@@ -603825,6 +604014,16 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
 function formatLiveSensorContext(repoRoot) {
   return formatLiveSensorContextFromSnapshot(readLiveSensorSnapshot(repoRoot));
 }
+function isLiveSensorSnapshotActive(snapshot) {
+  const cfg = snapshot?.config;
+  if (!cfg) return false;
+  return Boolean(
+    cfg.videoEnabled || cfg.audioEnabled || cfg.audioOutputEnabled || cfg.inferEnabled || cfg.clipEnabled || cfg.asrEnabled || cfg.audioAnalysisEnabled
+  );
+}
+function isLiveSensorActiveForRepo(repoRoot) {
+  return isLiveSensorSnapshotActive(readLiveSensorSnapshot(repoRoot));
+}
 async function discoverAudioInputs() {
   const devices = [];
   const errors = [];
@@ -603958,31 +604157,101 @@ print(json.dumps({"ok": True, "scores": scores}))
 async function tryRecordAudioDevice(device, durationSec, outputPath3) {
   await recordAudioSample(device, durationSec, outputPath3);
 }
+function hasLiveAudioSignal(activity) {
+  if (!activity) return false;
+  return activity.peak >= 0.025 || activity.rmsDb > -48 || activity.activeRatio >= 0.025;
+}
+function audioCaptureMethod(device) {
+  return device.startsWith("pulse:") ? "pulse/ffmpeg" : "arecord";
+}
 async function recordAudioSampleWithRecovery(preferred, devices, durationSec, outputPath3) {
   const ordered = [];
   const add3 = (id2) => {
     const clean5 = String(id2 ?? "").trim();
     if (clean5 && !ordered.includes(clean5)) ordered.push(clean5);
   };
-  add3(preferred);
-  for (const device of devices.filter((entry) => entry.source === "pulse" || entry.source === "pipewire")) add3(device.id);
+  const preferredInput = preferredLiveAudioInputId(devices, preferred);
+  add3(preferredInput);
+  if (preferred && !isAudioOutputMonitorId(preferred)) add3(preferred);
+  for (const device of rankLiveAudioInputDevices(devices, preferred).filter((entry) => !isAudioOutputMonitorDevice(entry))) add3(device.id);
   add3("pulse:default");
   add3("default");
-  for (const device of devices.filter((entry) => entry.source !== "pulse" && entry.source !== "pipewire")) add3(device.id);
+  for (const device of rankLiveAudioInputDevices(devices, preferred).filter((entry) => isAudioOutputMonitorDevice(entry))) add3(device.id);
   const errors = [];
+  const attempts = [];
+  let firstQuietCapture = null;
   for (const candidate of ordered) {
+    attempts.push(candidate);
     try {
       await tryRecordAudioDevice(candidate, durationSec, outputPath3);
-      return { ok: true, device: candidate, method: candidate.startsWith("pulse:") ? "pulse/ffmpeg" : "arecord", errors };
+      const activity = readPcm16WavActivity(outputPath3);
+      const method = audioCaptureMethod(candidate);
+      if (hasLiveAudioSignal(activity)) {
+        return { ok: true, device: candidate, method, errors, attempts, activity, signalDetected: true };
+      }
+      if (!firstQuietCapture) {
+        firstQuietCapture = {
+          device: candidate,
+          method,
+          activity,
+          bytes: readFileSync88(outputPath3),
+          errors: [...errors]
+        };
+      }
+      errors.push(`${candidate}: captured but no live input signal detected`);
     } catch (err) {
       errors.push(`${candidate}: ${err instanceof Error ? err.message : String(err)}`.slice(0, 360));
     }
   }
-  return { ok: false, device: preferred || "default", method: "none", errors };
+  if (firstQuietCapture) {
+    writeFileSync56(outputPath3, firstQuietCapture.bytes);
+    return {
+      ok: true,
+      device: firstQuietCapture.device,
+      method: firstQuietCapture.method,
+      errors,
+      attempts,
+      activity: firstQuietCapture.activity,
+      signalDetected: false
+    };
+  }
+  return { ok: false, device: preferredInput || preferred || "default", method: "none", errors, attempts };
 }
 function firstDeviceId(devices) {
   return devices.find((device) => device.id && !/metadata\/non-capture/i.test(device.detail ?? ""))?.id ?? devices[0]?.id;
 }
+function isAudioOutputMonitorId(id2) {
+  return /(?:\.monitor\b|sink\.monitor|alsa_output|output\.monitor)/i.test(String(id2 ?? ""));
+}
+function isAudioOutputMonitorDevice(device) {
+  if (!device) return false;
+  return isAudioOutputMonitorId(device.id) || isAudioOutputMonitorId(device.label) || isAudioOutputMonitorId(device.detail);
+}
+function preferredLiveAudioInputId(devices, configured) {
+  const usable = devices.filter((device) => device.id && !isAudioOutputMonitorDevice(device));
+  if (configured && usable.some((device) => device.id === configured)) return configured;
+  return rankLiveAudioInputDevices(devices, configured)[0]?.id ?? firstDeviceId(devices);
+}
+function liveAudioDeviceRank(device, configured) {
+  let rank = 0;
+  if (device.id === configured) rank -= 40;
+  if (isAudioOutputMonitorDevice(device)) rank += 500;
+  if (device.source === "pulse" || device.source === "pipewire") rank -= 20;
+  if (device.source === "alsa") rank -= 10;
+  if (device.source === "default") rank += 30;
+  if (/metadata\/non-capture/i.test(device.detail ?? "")) rank += 300;
+  return rank;
+}
+function rankLiveAudioInputDevices(devices, configured) {
+  const seen = /* @__PURE__ */ new Set();
+  const ranked = [];
+  for (const device of devices) {
+    if (!device.id || seen.has(device.id)) continue;
+    seen.add(device.id);
+    ranked.push(device);
+  }
+  return ranked.sort((a2, b) => liveAudioDeviceRank(a2, configured) - liveAudioDeviceRank(b, configured));
+}
 function getLiveSensorManager(repoRoot) {
   const key = repoRoot;
   let manager = managers.get(key);
@@ -604076,6 +604345,13 @@ var init_live_sensors = __esm({
       lastAgentReviewAt = 0;
       autoOrientationInFlight = /* @__PURE__ */ new Set();
       lastFeedbackAt = /* @__PURE__ */ new Map();
+      lastClipAt = /* @__PURE__ */ new Map();
+      nextVideoSourceIndex = 0;
+      liveSpeechSink = null;
+      liveSpeechEnabled = false;
+      lastLiveSpeechAt = 0;
+      lastLiveSpeechByKey = /* @__PURE__ */ new Map();
+      liveSpeechInFlight = /* @__PURE__ */ new Set();
       intakeAgentConfig;
       setStatusSink(sink) {
         this.statusSink = sink ?? null;
@@ -604087,6 +604363,12 @@ var init_live_sensors = __esm({
       setAgentActionSink(sink) {
         this.agentActionSink = sink ?? null;
       }
+      setLiveSpeechSink(sink) {
+        this.liveSpeechSink = sink ?? null;
+      }
+      setLiveSpeechEnabled(enabled2) {
+        this.liveSpeechEnabled = enabled2;
+      }
       setAudioIntakeAgentConfig(config) {
         this.intakeAgentConfig = config && config.backendUrl && config.model ? { backendUrl: config.backendUrl, model: config.model, apiKey: config.apiKey } : void 0;
       }
@@ -604182,16 +604464,14 @@ var init_live_sensors = __esm({
           if (!framePath) return `Camera orientation detection captured ${target}, but no frame path was returned.`;
           const cvScores = await scoreCameraOrientationWithOpenCv(framePath);
           const cvDecision = cvScores ? chooseCameraOrientationFromScores(cvScores) : null;
-          if (cvDecision && cvDecision.confidence >= 0.68) {
-            const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
-            return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
-          }
           const prompt = [
             "Determine whether this camera frame is upright for a human viewer.",
             "Return only JSON with this schema:",
             '{"rotation_degrees":0|90|180|270,"confidence":0.0-1.0,"reason":"brief visual evidence"}',
             "rotation_degrees is the clockwise correction to apply to future frames so they are upright.",
-            "Use 90 if the image must rotate clockwise; use 270 if it must rotate counter-clockwise; use 180 if upside down; use 0 if already upright or inconclusive."
+            "Use 90 if the image must rotate clockwise; use 270 if it must rotate counter-clockwise; use 180 if upside down; use 0 if already upright or inconclusive.",
+            "If the visible scene is sideways, choose the correction direction that makes people, faces, screens, and text upright; if the opposite direction would make them sideways, do not choose it.",
+            cvDecision ? `Computer-vision candidate: ${formatCameraRotation(cvDecision.rotation)} confidence=${cvDecision.confidence.toFixed(2)} evidence=${cvDecision.reason}` : "Computer-vision candidate: unavailable."
           ].join("\n");
           const vision = await new VisionTool(this.repoRoot).execute({
             action: "query",
@@ -604214,9 +604494,10 @@ var init_live_sensors = __esm({
             }
             return `Camera orientation detection for ${target} was inconclusive. Vision output did not contain a structured rotation decision.`;
           }
-          const useCv = cvDecision && (cvDecision.confidence > (decision2.confidence ?? 0) + 0.12 || cvDecision.rotation !== decision2.rotation && cvDecision.confidence >= 0.62 && (decision2.confidence ?? 0) < 0.76);
-          const selected = useCv ? cvDecision : decision2;
-          const evidence = useCv ? cvDecision.reason : `${decision2.reason || boundedText(vision.llmContent || vision.output, 320)}${cvDecision ? `; cv=${cvDecision.reason}` : ""}`;
+          const visionConfidence = decision2.confidence ?? 0;
+          const useCv = Boolean(cvDecision) && (visionConfidence < 0.55 || cvDecision.rotation === decision2.rotation && cvDecision.confidence > visionConfidence + 0.18);
+          const selected = useCv && cvDecision ? cvDecision : decision2;
+          const evidence = useCv ? cvDecision?.reason ?? "" : `${decision2.reason || boundedText(vision.llmContent || vision.output, 320)}${cvDecision ? `; cv=${cvDecision.reason}` : ""}`;
           const orientation = this.setCameraRotation(target, selected.rotation, "auto", evidence, selected.confidence);
           return orientation ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation)}` : `Auto orientation for ${target} could not be persisted.`;
         } finally {
@@ -604245,7 +604526,10 @@ var init_live_sensors = __esm({
           errors
         };
         if (!this.config.selectedCamera) this.config.selectedCamera = firstDeviceId(video);
-        if (!this.config.selectedAudioInput) this.config.selectedAudioInput = firstDeviceId(inputs.devices);
+        const preferredInput = preferredLiveAudioInputId(inputs.devices, this.config.selectedAudioInput);
+        if (!this.config.selectedAudioInput || isAudioOutputMonitorId(this.config.selectedAudioInput) && preferredInput && !isAudioOutputMonitorId(preferredInput)) {
+          this.config.selectedAudioInput = preferredInput;
+        }
         if (!this.config.selectedAudioOutput) this.config.selectedAudioOutput = firstDeviceId(outputs.devices);
         this.persist();
         return this.devices;
@@ -604265,6 +604549,7 @@ var init_live_sensors = __esm({
       }
       stopAll() {
         this.agentActionEnabled = false;
+        this.liveSpeechEnabled = false;
         this.configure({
           videoEnabled: false,
           audioEnabled: false,
@@ -604277,6 +604562,7 @@ var init_live_sensors = __esm({
       }
       startRunMode(options2 = {}) {
         this.agentActionEnabled = Boolean(options2.agent);
+        this.liveSpeechEnabled = Boolean(options2.speech ?? options2.agent);
         this.configure({
           videoEnabled: true,
           inferEnabled: true,
@@ -604308,7 +604594,7 @@ var init_live_sensors = __esm({
             }, 6e4);
           });
         }
-        void this.sampleVideoNow(true).catch((err) => {
+        void this.sampleVideoNow(true, { mode: "all" }).catch((err) => {
           this.emitFeedback({
             kind: "error",
             title: "Live video sample failed",
@@ -604503,17 +604789,18 @@ var init_live_sensors = __esm({
         const sampledAt = Date.now();
         if (forceInfer && source) {
           const loop = new LiveMediaLoopTool(this.repoRoot);
+          const inferFromCapturedFrame = preview.ok && Boolean(preview.framePath);
           const result = await loop.execute({
             action: "watch",
-            source_kind: "camera",
-            camera: source,
+            source_kind: inferFromCapturedFrame ? "file" : "camera",
+            ...inferFromCapturedFrame ? { path: preview.framePath } : { camera: source },
             yolo_family: "yoloe-26",
             yolo_scale: "n",
             yoloe_prompt_mode: "prompt_free",
-            duration_sec: 0.75,
-            sample_interval_sec: 0.25,
+            duration_sec: inferFromCapturedFrame ? 0.1 : 0.75,
+            sample_interval_sec: inferFromCapturedFrame ? 0.1 : 0.25,
             max_frames: 1,
-            rotate_degrees: orientation?.rotation ?? 0,
+            rotate_degrees: inferFromCapturedFrame ? 0 : orientation?.rotation ?? 0,
             auto_bootstrap: true,
             audio_mode: "none",
             detect_objects: true,
@@ -604526,6 +604813,7 @@ var init_live_sensors = __esm({
           });
           inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
           const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
+          if (livePayload) livePayload.source = source;
           const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
           const classifications = classificationsFromLiveMediaPayload(livePayload, sampledAt, source);
           this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
@@ -604562,6 +604850,16 @@ var init_live_sensors = __esm({
               `${unknownPeople.length} unknown individual(s) observed on live camera ${source}`,
               unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; ")
             );
+            this.maybeSpeakLive({
+              key: `unknown-person:${source}`,
+              kind: "unknown_person",
+              source,
+              summary: `${unknownPeople.length} unknown individual(s) observed on live camera ${source}`,
+              evidence: unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; "),
+              defaultText: "I see someone here, but I do not know who. Who is there?",
+              urgency: "high",
+              observedAt: sampledAt
+            }, 9e4);
           }
           const triggerEvents = observations.events.filter((entry) => entry.kind === "visual_trigger");
           if (triggerEvents.length > 0) {
@@ -604569,9 +604867,21 @@ var init_live_sensors = __esm({
               `Visual trigger hit in live camera stream ${source}`,
               triggerEvents.map((entry) => entry.summary).join("; ")
             );
+            this.maybeSpeakLive({
+              key: `visual-trigger:${source}:${triggerEvents[0]?.title ?? "trigger"}`,
+              kind: "visual_trigger",
+              source,
+              summary: `Visual trigger hit in live camera stream ${source}`,
+              evidence: triggerEvents.map((entry) => entry.summary).join("; "),
+              defaultText: "I noticed something important in view. Should I inspect it?",
+              urgency: "high",
+              observedAt: sampledAt
+            }, 45e3);
           }
         }
-        if (this.config.clipEnabled && preview.ok && preview.framePath && source) {
+        const lastClip = this.lastClipAt.get(source) ?? 0;
+        if (this.config.clipEnabled && preview.ok && preview.framePath && source && sampledAt - lastClip >= 5e3) {
+          this.lastClipAt.set(source, sampledAt);
           clipSummary = await this.recognizeFrameObjects(preview.framePath, source, sampledAt, { emit: false });
           const current = this.snapshot.cameras?.[source];
           if (current) {
@@ -604590,14 +604900,16 @@ ${inference}` : "", clipSummary ? `
 CLIP visual memory:
 ${clipSummary}` : ""].filter(Boolean).join("\n");
       }
-      async sampleVideoNow(forceInfer = this.config.inferEnabled) {
+      async sampleVideoNow(forceInfer = this.config.inferEnabled, options2 = {}) {
         if (this.videoInFlight) return "Video sampler is already running.";
         this.videoInFlight = true;
         try {
-          const sources = this.activeVideoSources();
+          const allSources = this.activeVideoSources();
+          if (allSources.length === 0) return "No camera selected. Use /live refresh after connecting hardware.";
+          const sources = options2.mode === "round-robin" ? [allSources[this.nextVideoSourceIndex++ % allSources.length]] : allSources;
           if (sources.length === 0) return "No camera selected. Use /live refresh after connecting hardware.";
           const outputs = [];
-          const previewWidth = liveDashboardPreviewWidthForSources(sources.length);
+          const previewWidth = liveDashboardPreviewWidthForSources(allSources.length);
           for (const source of sources) {
             outputs.push(await this.sampleSingleVideoNow(source, forceInfer, previewWidth));
           }
@@ -604612,7 +604924,9 @@ ${output}`).join("\n\n");
         if (this.audioInFlight) return "Audio sampler is already running.";
         this.audioInFlight = true;
         try {
-          const input = this.config.selectedAudioInput || firstDeviceId(this.devices.audioInputs) || "default";
+          const preferredInput = preferredLiveAudioInputId(this.devices.audioInputs, this.config.selectedAudioInput) || "default";
+          const input = this.config.selectedAudioInput && !isAudioOutputMonitorId(this.config.selectedAudioInput) ? this.config.selectedAudioInput : preferredInput;
+          if (input !== this.config.selectedAudioInput) this.config.selectedAudioInput = input;
           const outputDir2 = liveDir(this.repoRoot);
           ensureLiveDir(this.repoRoot);
           const recordingPath = join124(outputDir2, `audio-${Date.now()}.wav`);
@@ -604643,7 +604957,10 @@ ${output}`).join("\n\n");
           if (capture.device !== this.config.selectedAudioInput) {
             this.config.selectedAudioInput = capture.device;
           }
-          const activity = readPcm16WavActivity(recordingPath);
+          const activity = capture.activity ?? readPcm16WavActivity(recordingPath);
+          if (capture.ok && capture.signalDetected === false) {
+            audioErrors.push(`No live input signal detected after probing ${capture.attempts?.length ?? 1} audio source(s); using quiet capture from ${capture.device}`);
+          }
           if (this.config.asrEnabled) {
             try {
               const asr = await new TranscribeFileTool(this.repoRoot).execute({ path: recordingPath, model: "tiny" });
@@ -604720,6 +605037,16 @@ ${output}`).join("\n\n");
               "Live audio intake classified speech as intended for Omnius",
               `transcript=${transcript}; confidence=${intake.confidence.toFixed(2)}; reason=${intake.reason}`
             );
+            this.maybeSpeakLive({
+              key: `audio-intent:${capture.device}:${transcript.slice(0, 48)}`,
+              kind: "audio_intent",
+              source: capture.device,
+              summary: "Live audio intake classified speech as intended for Omnius",
+              evidence: `transcript=${transcript}; confidence=${intake.confidence.toFixed(2)}; reason=${intake.reason}`,
+              defaultText: "I heard you. I am checking the live context now.",
+              urgency: "high",
+              observedAt: this.snapshot.audio.updatedAt
+            }, 2e4);
           }
           this.emitDashboard();
           return [
@@ -604753,7 +605080,7 @@ ${analysis}` : ""
           this.videoTimer = null;
           if (!this.config.videoEnabled) return;
           try {
-            await this.sampleVideoNow(this.config.inferEnabled || this.config.clipEnabled);
+            await this.sampleVideoNow(this.config.inferEnabled || this.config.clipEnabled, { mode: "round-robin" });
           } catch {
           } finally {
             if (this.config.videoEnabled) {
@@ -604828,7 +605155,7 @@ ${analysis}` : ""
       maybeRequestAgentReview(reason, evidence) {
         if (!this.agentActionEnabled || !this.agentActionSink) return;
         const now2 = Date.now();
-        if (now2 - this.lastAgentReviewAt < 2e4) return;
+        if (now2 - this.lastAgentReviewAt < 6e4) return;
         this.lastAgentReviewAt = now2;
         const prompt = [
           "Live environment PFC review.",
@@ -604851,6 +605178,38 @@ ${analysis}` : ""
         } catch {
         }
       }
+      maybeSpeakLive(proposal, perKeyCooldownMs) {
+        if (!this.liveSpeechEnabled || !this.liveSpeechSink) return;
+        const now2 = Date.now();
+        if (now2 - this.lastLiveSpeechAt < 12e3) return;
+        const lastForKey = this.lastLiveSpeechByKey.get(proposal.key) ?? 0;
+        if (now2 - lastForKey < perKeyCooldownMs) return;
+        if (this.liveSpeechInFlight.has(proposal.key)) return;
+        this.liveSpeechInFlight.add(proposal.key);
+        void classifyLiveSpeech(proposal, this.intakeAgentConfig).then((decision2) => {
+          this.lastLiveSpeechByKey.set(proposal.key, Date.now());
+          if (!decision2.speak || !decision2.text.trim()) return;
+          this.lastLiveSpeechAt = Date.now();
+          const text2 = trimOneLine(decision2.text, 180);
+          this.liveSpeechSink?.(text2, decision2);
+          this.emitFeedbackThrottled(`speech:${proposal.key}`, {
+            kind: "speech",
+            title: "Live speech emitted",
+            observedAt: decision2.updatedAt,
+            message: `${text2}
+reason=${decision2.reason} confidence=${decision2.confidence.toFixed(2)} source=${decision2.source}`
+          }, 15e3);
+        }).catch((err) => {
+          this.emitFeedbackThrottled(`speech-error:${proposal.key}`, {
+            kind: "error",
+            title: "Live speech arbiter failed",
+            observedAt: Date.now(),
+            message: err instanceof Error ? err.message : String(err)
+          }, 6e4);
+        }).finally(() => {
+          this.liveSpeechInFlight.delete(proposal.key);
+        });
+      }
       persist() {
         ensureLiveDir(this.repoRoot);
         writeFileSync56(liveConfigPath(this.repoRoot), JSON.stringify(this.config, null, 2), "utf8");
@@ -639208,6 +639567,10 @@ async function showPruneMenu(options2) {
   });
   if (!result.confirmed || !result.key || result.key === "back") return;
   const start2 = (label, o2) => {
+    if (isLiveSensorActiveForRepo(workingDir)) {
+      renderWarning("Live audio/video mode is active; pruning is deferred so sensor, ASR, vision, and reasoning inference stay available. Stop live mode before running prune.");
+      return;
+    }
     renderInfo(`${label} (running in background — summary will follow)…`);
     runPruneWorkerOnce({
       repoRoot: workingDir,
@@ -639243,6 +639606,7 @@ var init_prune_menu = __esm({
     init_render();
     init_kg_prune();
     init_memory_maintenance();
+    init_live_sensors();
     import_chalk2 = __toESM(require_source(), 1);
   }
 });
@@ -648242,6 +648606,7 @@ async function handleSlashCommand(input, ctx3) {
     case "prune": {
       const sub2 = (arg?.trim().split(/\s+/)[0] ?? "").toLowerCase();
       const workingDir = ctx3.repoRoot || process.cwd();
+      const force = /\b--force\b/i.test(arg ?? "");
       const targetMatch = arg?.match(/--target\s+(\d+)/);
       const targetNodes = targetMatch ? parseInt(targetMatch[1], 10) : void 0;
       const { knowledgeGraphStats: knowledgeGraphStats2, formatKgStatsLine: formatKgStatsLine2 } = await Promise.resolve().then(() => (init_kg_prune(), kg_prune_exports));
@@ -648250,6 +648615,10 @@ async function handleSlashCommand(input, ctx3) {
         renderInfo(summary);
       };
       const startBackground = (label, o2) => {
+        if (isLiveSensorActiveForRepo(workingDir) && !force) {
+          renderWarning("Live audio/video mode is active; pruning is deferred so sensor, ASR, vision, and reasoning inference stay available. Stop live mode first, or rerun with --force if you intentionally want pruning to compete with live inference.");
+          return;
+        }
         renderInfo(`${label} (running in background — summary will follow)…`);
         runPruneWorkerOnce2({
           repoRoot: workingDir,
@@ -656573,6 +656942,12 @@ function attachLiveSinks(ctx3, manager, agentEnabled = false) {
     model: ctx3.config.model,
     apiKey: ctx3.config.apiKey
   });
+  const speechEnabled = agentEnabled && Boolean(ctx3.voiceSpeak);
+  manager.setLiveSpeechEnabled(speechEnabled);
+  manager.setLiveSpeechSink(speechEnabled ? (text2) => {
+    if (ctx3.voiceIsEnabled && !ctx3.voiceIsEnabled()) return;
+    ctx3.voiceSpeak?.(text2);
+  } : void 0);
   if (agentEnabled && ctx3.startBackgroundPrompt) {
     manager.setAgentActionSink((prompt) => {
       const id2 = ctx3.startBackgroundPrompt?.(prompt);
@@ -656586,13 +656961,14 @@ async function startLiveVoicechat(ctx3, manager) {
   const agentEnabled = Boolean(ctx3.startBackgroundPrompt);
   attachLiveSinks(ctx3, manager, agentEnabled);
   await ensureLiveInventory(manager);
-  renderInfo(manager.startRunMode({ agent: agentEnabled, lowLatency: true }));
   if (!ctx3.voiceChatStart) {
+    renderInfo(manager.startRunMode({ agent: agentEnabled, lowLatency: true, speech: false }));
     renderWarning("Voice chat is not available in this context; live audio/video feedback remains active.");
     return;
   }
   ctx3.voiceSetMode?.("voicechat");
   if (ctx3.isVoiceChatActive?.()) {
+    renderInfo(manager.startRunMode({ agent: agentEnabled, lowLatency: true, speech: Boolean(ctx3.voiceSpeak) }));
     renderInfo("Voice chat is already active. Live audio/video feedback remains attached.");
     return;
   }
@@ -656603,6 +656979,7 @@ async function startLiveVoicechat(ctx3, manager) {
   try {
     renderInfo("Starting live voicechat with audio/video context...");
     await ctx3.voiceChatStart();
+    renderInfo(manager.startRunMode({ agent: agentEnabled, lowLatency: true, speech: Boolean(ctx3.voiceSpeak) }));
     renderInfo("Live voicechat active: low-latency audio/video context is updating while voice conversation continues.");
   } catch (err) {
     renderError(`Live voicechat failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -656663,7 +657040,7 @@ async function handleLiveCommand(ctx3, arg) {
       renderWarning("Live PFC agent review is unavailable in this context; starting visual/audio feedback only.");
     }
     await ensureLiveInventory(manager);
-    renderInfo(manager.startRunMode({ agent: effectiveAgent, lowLatency: true }));
+    renderInfo(manager.startRunMode({ agent: effectiveAgent, lowLatency: true, speech: effectiveAgent && Boolean(ctx3.voiceSpeak) }));
     return;
   }
   if (sub2 === "chat" || sub2 === "voicechat" || sub2 === "voice") {
@@ -656918,14 +657295,14 @@ async function showLiveMenu(ctx3, manager) {
         continue;
       case "run-live":
         attachLiveSinks(ctx3, manager, false);
-        renderInfo(manager.startRunMode({ agent: false, lowLatency: true }));
+        renderInfo(manager.startRunMode({ agent: false, lowLatency: true, speech: false }));
         continue;
       case "run-agent":
         attachLiveSinks(ctx3, manager, true);
         if (!ctx3.startBackgroundPrompt) {
           renderWarning("Live PFC agent review is unavailable in this context; starting visual/audio feedback only.");
         }
-        renderInfo(manager.startRunMode({ agent: Boolean(ctx3.startBackgroundPrompt), lowLatency: true }));
+        renderInfo(manager.startRunMode({ agent: Boolean(ctx3.startBackgroundPrompt), lowLatency: true, speech: Boolean(ctx3.startBackgroundPrompt && ctx3.voiceSpeak) }));
         continue;
       case "run-chat":
         await startLiveVoicechat(ctx3, manager);
@@ -732153,9 +732530,10 @@ Rationale: ${proposal.rationale}${provenanceNote}${dmnDevDiscipline(proposal.cat
     }, delayMs);
   }
   let activeTask = null;
+  const liveSensorsBusy = () => isLiveSensorActiveForRepo(repoRoot);
   idleMemoryMaintenance = startIdleMemoryMaintenance({
     repoRoot,
-    isBusy: () => Boolean(activeTask) || _interactiveSessionActive,
+    isBusy: () => Boolean(activeTask) || _interactiveSessionActive || liveSensorsBusy(),
     onSummary: (summary) => {
       if (!summary) return;
       writeContent(() => renderInfo(summary));
@@ -734709,6 +735087,7 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
     },
     setLiveMediaStatus(status) {
       statusBar.setLiveMediaStatus(status);
+      if (status.audio || status.video) idleMemoryMaintenance?.notifyActivity();
     },
     liveDynamicBlockHost: {
       registerDynamicBlock: (id2, render2) => statusBar.registerDynamicBlock(id2, render2),

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "omnius",
-  "version": "1.0.411",
+  "version": "1.0.412",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "omnius",
-      "version": "1.0.411",
+      "version": "1.0.412",
       "bundleDependencies": [
         "image-to-ascii"
       ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnius",
-  "version": "1.0.411",
+  "version": "1.0.412",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",