omnius 1.0.407 → 1.0.408

package/dist/index.js

@@ -602166,6 +602166,103 @@ function describeCameraOrientation(orientation) {
   if (!orientation) return "upright/no correction (default)";
   return `${formatCameraRotation(orientation.rotation)} (${orientation.source}${orientation.confidence !== void 0 ? ` confidence=${orientation.confidence.toFixed(2)}` : ""}, updated=${nowIso3(orientation.updatedAt)})`;
 }
+function chooseCameraOrientationFromScores(scores) {
+  const sorted = [...scores].filter((score) => Number.isFinite(score.score)).sort((a2, b) => b.score - a2.score);
+  const best = sorted[0];
+  if (!best || best.score <= 0 || best.faces <= 0) return null;
+  const second3 = sorted[1];
+  const gap = best.score - (second3?.score ?? 0);
+  const total = sorted.reduce((sum, score) => sum + Math.max(0, score.score), 0);
+  const share = total > 0 ? best.score / total : 1;
+  const confidence2 = Math.max(0.55, Math.min(0.98, share * 0.75 + Math.min(0.23, gap / Math.max(1, best.score))));
+  if (confidence2 < 0.58 && gap < 0.4) return null;
+  return {
+    rotation: best.rotation,
+    confidence: confidence2,
+    reason: `cv-face-orientation faces=${best.faces} score=${best.score.toFixed(2)} next=${(second3?.score ?? 0).toFixed(2)} area=${(best.faceAreaRatio ?? 0).toFixed(3)}`
+  };
+}
+function trimOneLine(value2, maxChars) {
+  return String(value2 ?? "").replace(/\r/g, " ").replace(/\n/g, " ").replace(/\s+/g, " ").trim().slice(0, maxChars);
+}
+function compactSourceId(source) {
+  return source.replace(/^\/dev\//, "");
+}
+function truncateCell(value2, width) {
+  if (width <= 1) return value2.slice(0, Math.max(0, width));
+  return value2.length <= width ? value2.padEnd(width, " ") : `${value2.slice(0, Math.max(1, width - 1))}…`;
+}
+function summarizeInference(value2) {
+  if (!value2) return "objects: pending";
+  const lines = value2.split("\n").map((line) => line.trim()).filter(Boolean);
+  const objectsIndex = lines.findIndex((line) => /^##\s+Objects/i.test(line));
+  if (objectsIndex >= 0) {
+    const objects = lines.slice(objectsIndex + 1).filter((line) => line.startsWith("- ")).slice(0, 4).map((line) => line.replace(/^-\s*/, ""));
+    if (objects.length > 0) return `objects: ${objects.join("; ")}`;
+  }
+  const sampled = lines.find((line) => /^objects:/i.test(line));
+  return sampled ? trimOneLine(sampled, 160) : trimOneLine(lines.slice(0, 4).join("; "), 160);
+}
+function cameraSnapshotSummary(camera) {
+  if (camera.error) return `error: ${trimOneLine(camera.error, 160)}`;
+  return camera.summary || summarizeInference(camera.inference);
+}
+function formatLiveDashboardFromSnapshot(snapshot, opts = {}) {
+  const width = Math.max(60, opts.width ?? 100);
+  const now2 = opts.now ?? Date.now();
+  const maxCameras = opts.maxCameras ?? 4;
+  const cameras = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {})).sort((a2, b) => {
+    if (a2.source === snapshot.config.selectedCamera) return -1;
+    if (b.source === snapshot.config.selectedCamera) return 1;
+    return a2.source.localeCompare(b.source);
+  }).slice(0, maxCameras);
+  const audioAge = snapshot.audio?.updatedAt ? `${Math.max(0, Math.round((now2 - snapshot.audio.updatedAt) / 1e3))}s` : "none";
+  const title = `live audio/video  cameras=${cameras.length}/${snapshot.devices.video.length}  audio=${snapshot.config.audioEnabled ? "on" : "off"} age=${audioAge}`;
+  const horizontal = "─".repeat(Math.max(0, width - 2));
+  const lines = [`╭${horizontal}╮`];
+  lines.push(`│${truncateCell(` ${title}`, width - 2)}│`);
+  if (cameras.length === 0) {
+    lines.push(`│${truncateCell(" no camera frames captured yet", width - 2)}│`);
+  }
+  for (const camera of cameras) {
+    const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
+    const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
+    lines.push(`├${horizontal}┤`);
+    lines.push(`│${truncateCell(` ${compactSourceId(camera.source)}  age=${age}s  rotation=${rotation}`, width - 2)}│`);
+    const asciiLines = (camera.ascii ?? "").split("\n").filter((line) => line.length > 0).slice(0, 10);
+    const detailLines = [
+      cameraSnapshotSummary(camera),
+      camera.framePath ? `frame: ${camera.displayPath || camera.framePath}` : "",
+      camera.orientation?.evidence ? `orientation: ${trimOneLine(camera.orientation.evidence, 120)}` : ""
+    ].filter(Boolean);
+    if (asciiLines.length === 0) {
+      for (const detail of detailLines.length ? detailLines : ["preview pending"]) {
+        lines.push(`│${truncateCell(` ${detail}`, width - 2)}│`);
+      }
+      continue;
+    }
+    const imageWidth = Math.max(24, Math.min(52, Math.floor((width - 5) * 0.58)));
+    const detailWidth = Math.max(18, width - imageWidth - 5);
+    const rowCount = Math.max(asciiLines.length, detailLines.length);
+    for (let i2 = 0; i2 < rowCount; i2++) {
+      const image = truncateCell(asciiLines[i2] ?? "", imageWidth);
+      const detail = truncateCell(detailLines[i2] ?? "", detailWidth);
+      lines.push(`│ ${image} │ ${detail}│`);
+    }
+  }
+  if (snapshot.audio) {
+    lines.push(`├${horizontal}┤`);
+    const audioBits = [
+      `audio input=${snapshot.audio.input || snapshot.config.selectedAudioInput || "none"}`,
+      snapshot.audio.error ? `error=${trimOneLine(snapshot.audio.error, 120)}` : "",
+      snapshot.audio.transcript ? `asr=${trimOneLine(snapshot.audio.transcript, 120)}` : "",
+      snapshot.audio.analysis ? `sound=${trimOneLine(snapshot.audio.analysis, 120)}` : ""
+    ].filter(Boolean);
+    for (const item of audioBits.slice(0, 4)) lines.push(`│${truncateCell(` ${item}`, width - 2)}│`);
+  }
+  lines.push(`╰${horizontal}╯`);
+  return lines;
+}
 function parseLiveMediaPayload(value2) {
   const parsed = parseJsonObject3(value2);
   if (!parsed) return null;
@@ -602425,6 +602522,21 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
     lines.push(`Selected camera: ${snapshot.config.selectedCamera}`);
     lines.push(`Camera orientation correction: ${describeCameraOrientation(orientation)}`);
   }
+  const cameraSnapshots = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {})).sort((a2, b) => {
+    if (a2.source === snapshot.config.selectedCamera) return -1;
+    if (b.source === snapshot.config.selectedCamera) return 1;
+    return b.updatedAt - a2.updatedAt;
+  });
+  if (cameraSnapshots.length > 0) {
+    lines.push("Camera snapshot set:");
+    for (const camera of cameraSnapshots.slice(0, 6)) {
+      const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
+      const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
+      lines.push(`- ${camera.source} age=${age}s rotation=${rotation}${camera.framePath ? ` frame=${camera.framePath}` : ""}${camera.error ? ` error=${trimOneLine(camera.error, 240)}` : ""}`);
+      const summary = cameraSnapshotSummary(camera);
+      if (summary) lines.push(`  summary: ${summary}`);
+    }
+  }
   if (snapshot.config.selectedAudioInput) lines.push(`Selected audio input: ${snapshot.config.selectedAudioInput}`);
   if (snapshot.config.selectedAudioOutput) lines.push(`Selected audio output: ${snapshot.config.selectedAudioOutput}`);
   const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "clip_recognition" || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 18);
@@ -602481,6 +602593,19 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
       lines.push(cleanPreview(snapshot.video.inference, 1800));
     }
   }
+  if (cameraSnapshots.length > 1) {
+    lines.push("");
+    lines.push("## LIVE CAMERA LATEST FRAMES");
+    for (const camera of cameraSnapshots.slice(0, 6)) {
+      const age = now2 - camera.updatedAt;
+      const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
+      lines.push(`### ${camera.source}`);
+      lines.push(`timestamp: ${nowIso3(camera.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s rotation=${rotation}${age > maxAgeMs ? " STALE" : ""}`);
+      if (camera.error) lines.push(`error: ${camera.error}`);
+      if (camera.framePath) lines.push(`frame: ${camera.framePath}`);
+      lines.push(`summary: ${cameraSnapshotSummary(camera)}`);
+    }
+  }
   if (snapshot.audio) {
     const audioEvents = (snapshot.events ?? []).filter((event) => event.kind === "audio_transcript" || event.kind === "audio_sound").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 8);
     if (audioEvents.length > 0) {
@@ -602587,6 +602712,93 @@ async function recordAudioSample(device, durationSec, outputPath3) {
     outputPath3
   ], { timeout: (Number(seconds) + 10) * 1e3 });
 }
+async function scoreCameraOrientationWithOpenCv(framePath) {
+  if (!await commandExists2("python3", 1200)) return null;
+  const script = String.raw`
+import json, sys
+try:
+    import cv2
+except Exception as exc:
+    print(json.dumps({"ok": False, "error": str(exc)}))
+    raise SystemExit(0)
+
+path = sys.argv[1]
+img = cv2.imread(path)
+if img is None:
+    print(json.dumps({"ok": False, "error": "could not read image"}))
+    raise SystemExit(0)
+
+face = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml")
+eye = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_eye.xml")
+
+def rotate(mat, deg):
+    if deg == 90:
+        return cv2.rotate(mat, cv2.ROTATE_90_CLOCKWISE)
+    if deg == 180:
+        return cv2.rotate(mat, cv2.ROTATE_180)
+    if deg == 270:
+        return cv2.rotate(mat, cv2.ROTATE_90_COUNTERCLOCKWISE)
+    return mat
+
+scores = []
+for deg in [0, 90, 180, 270]:
+    frame = rotate(img, deg)
+    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+    h, w = gray.shape[:2]
+    faces = face.detectMultiScale(gray, scaleFactor=1.08, minNeighbors=4, minSize=(32, 32))
+    face_area = 0.0
+    eye_count = 0
+    for (x, y, fw, fh) in faces[:8]:
+        face_area += float(fw * fh) / max(1.0, float(w * h))
+        roi = gray[y:y+fh, x:x+fw]
+        try:
+            eye_count += len(eye.detectMultiScale(roi, scaleFactor=1.1, minNeighbors=3, minSize=(8, 8)))
+        except Exception:
+            pass
+    score = len(faces) * 2.5 + min(4.0, face_area * 35.0) + min(2.0, eye_count * 0.35)
+    scores.append({"rotation": deg, "score": score, "faces": int(len(faces)), "faceAreaRatio": face_area})
+print(json.dumps({"ok": True, "scores": scores}))
+`;
+  try {
+    const raw = await execFileText4("python3", ["-c", script, framePath], { timeout: 15e3, maxBuffer: 1024 * 1024 });
+    const parsed = parseJsonObjectFromText(raw);
+    const scores = Array.isArray(parsed?.["scores"]) ? parsed["scores"] : [];
+    if (scores.length === 0) return null;
+    return scores.map((score) => ({
+      rotation: normalizeLiveCameraRotation(score["rotation"]),
+      score: Math.max(0, Number(score["score"] ?? 0)),
+      faces: Math.max(0, Number(score["faces"] ?? 0)),
+      faceAreaRatio: Number.isFinite(Number(score["faceAreaRatio"])) ? Number(score["faceAreaRatio"]) : void 0
+    }));
+  } catch {
+    return null;
+  }
+}
+async function tryRecordAudioDevice(device, durationSec, outputPath3) {
+  await recordAudioSample(device, durationSec, outputPath3);
+}
+async function recordAudioSampleWithRecovery(preferred, devices, durationSec, outputPath3) {
+  const ordered = [];
+  const add3 = (id2) => {
+    const clean5 = String(id2 ?? "").trim();
+    if (clean5 && !ordered.includes(clean5)) ordered.push(clean5);
+  };
+  add3(preferred);
+  for (const device of devices.filter((entry) => entry.source === "pulse" || entry.source === "pipewire")) add3(device.id);
+  add3("pulse:default");
+  add3("default");
+  for (const device of devices.filter((entry) => entry.source !== "pulse" && entry.source !== "pipewire")) add3(device.id);
+  const errors = [];
+  for (const candidate of ordered) {
+    try {
+      await tryRecordAudioDevice(candidate, durationSec, outputPath3);
+      return { ok: true, device: candidate, method: candidate.startsWith("pulse:") ? "pulse/ffmpeg" : "arecord", errors };
+    } catch (err) {
+      errors.push(`${candidate}: ${err instanceof Error ? err.message : String(err)}`.slice(0, 360));
+    }
+  }
+  return { ok: false, device: preferred || "default", method: "none", errors };
+}
 function firstDeviceId(devices) {
   return devices.find((device) => device.id && !/metadata\/non-capture/i.test(device.detail ?? ""))?.id ?? devices[0]?.id;
 }
@@ -602677,6 +602889,7 @@ var init_live_sensors = __esm({
       agentActionEnabled = false;
       lastAgentReviewAt = 0;
       autoOrientationInFlight = /* @__PURE__ */ new Set();
+      lastFeedbackAt = /* @__PURE__ */ new Map();
       setStatusSink(sink) {
         this.statusSink = sink ?? null;
         this.emitStatus();
@@ -602693,6 +602906,20 @@ var init_live_sensors = __esm({
       getSnapshot() {
         return this.snapshot;
       }
+      activeVideoSources() {
+        const ordered = [];
+        const add3 = (id2) => {
+          const clean5 = String(id2 ?? "").trim();
+          if (!clean5 || ordered.includes(clean5)) return;
+          const device = this.devices.video.find((entry) => entry.id === clean5);
+          if (device && /metadata\/non-capture/i.test(device.detail ?? "")) return;
+          ordered.push(clean5);
+        };
+        add3(this.config.selectedCamera);
+        for (const device of this.devices.video) add3(device.id);
+        if (ordered.length === 0) add3(firstDeviceId(this.devices.video));
+        return ordered.slice(0, 4);
+      }
       getCameraOrientation(device = this.config.selectedCamera) {
         return device ? this.config.cameraOrientation?.[device] : void 0;
       }
@@ -602763,6 +602990,12 @@ var init_live_sensors = __esm({
           }
           const framePath = extractSavedImagePath(captured.output, this.repoRoot);
           if (!framePath) return `Camera orientation detection captured ${target}, but no frame path was returned.`;
+          const cvScores = await scoreCameraOrientationWithOpenCv(framePath);
+          const cvDecision = cvScores ? chooseCameraOrientationFromScores(cvScores) : null;
+          if (cvDecision && cvDecision.confidence >= 0.68) {
+            const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
+            return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
+          }
           const prompt = [
             "Determine whether this camera frame is upright for a human viewer.",
             "Return only JSON with this schema:",
@@ -602777,14 +603010,24 @@ var init_live_sensors = __esm({
             model: "moondream3-preview"
           });
           if (!vision.success) {
+            if (cvDecision) {
+              const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
+              return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
+            }
             return `Camera orientation detection captured ${target}, but vision inference was unavailable: ${vision.error || vision.output || "unknown error"}`;
           }
           const decision2 = parseCameraOrientationDecision(vision.llmContent || vision.output);
           if (!decision2) {
+            if (cvDecision) {
+              const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
+              return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
+            }
             return `Camera orientation detection for ${target} was inconclusive. Vision output did not contain a structured rotation decision.`;
           }
-          const evidence = decision2.reason || boundedText(vision.llmContent || vision.output, 400);
-          const orientation = this.setCameraRotation(target, decision2.rotation, "auto", evidence, decision2.confidence);
+          const useCv = cvDecision && (cvDecision.confidence > (decision2.confidence ?? 0) + 0.12 || cvDecision.rotation !== decision2.rotation && cvDecision.confidence >= 0.62 && (decision2.confidence ?? 0) < 0.76);
+          const selected = useCv ? cvDecision : decision2;
+          const evidence = useCv ? cvDecision.reason : `${decision2.reason || boundedText(vision.llmContent || vision.output, 320)}${cvDecision ? `; cv=${cvDecision.reason}` : ""}`;
+          const orientation = this.setCameraRotation(target, selected.rotation, "auto", evidence, selected.confidence);
           return orientation ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation)}` : `Auto orientation for ${target} could not be persisted.`;
         } finally {
           this.autoOrientationInFlight.delete(target);
@@ -602853,22 +603096,25 @@ var init_live_sensors = __esm({
           videoIntervalMs: options2.lowLatency === false ? this.config.videoIntervalMs : Math.min(this.config.videoIntervalMs, 1500),
           audioIntervalMs: options2.lowLatency === false ? this.config.audioIntervalMs : Math.min(this.config.audioIntervalMs, 3e3)
         });
-        const source = this.config.selectedCamera || firstDeviceId(this.devices.video);
-        if (source && !this.getCameraOrientation(source)) {
-          void this.autoOrientCamera(source, { onlyIfUnset: true }).then((message2) => {
-            this.emitFeedback({
+        for (const source of this.activeVideoSources()) {
+          const current = this.getCameraOrientation(source);
+          if (current?.source === "manual") continue;
+          if (current && current.confidence !== void 0 && current.confidence >= 0.75 && Date.now() - current.updatedAt < 15 * 6e4) continue;
+          void this.autoOrientCamera(source).then((message2) => {
+            this.emitFeedbackThrottled(`orient:${source}`, {
               kind: "orientation",
               title: "Camera auto-orientation",
               observedAt: Date.now(),
               message: message2
-            });
+            }, 3e4);
+            this.emitDashboard();
           }).catch((err) => {
-            this.emitFeedback({
+            this.emitFeedbackThrottled(`orient-error:${source}`, {
               kind: "error",
               title: "Camera auto-orientation failed",
               observedAt: Date.now(),
               message: err instanceof Error ? err.message : String(err)
-            });
+            }, 6e4);
           });
         }
         void this.sampleVideoNow(true).catch((err) => {
@@ -602895,7 +603141,7 @@ var init_live_sensors = __esm({
         });
         return options2.agent ? "Live PFC run enabled: visual/audio feedback is active and significant events can launch background agent review." : "Live run enabled: visual/audio feedback is active and live context will update each turn.";
       }
-      async previewCamera(device = this.config.selectedCamera) {
+      async previewCamera(device = this.config.selectedCamera, options2 = {}) {
         const source = device || firstDeviceId(this.devices.video);
         if (!source) return { ok: false, message: "No camera selected. Use /live to select a camera after device discovery." };
         const orientation = this.getCameraOrientation(source);
@@ -602910,10 +603156,22 @@ var init_live_sensors = __esm({
         if (!result.success) {
           const error = result.error || result.output || "camera capture failed";
           const observedAt2 = Date.now();
-          this.snapshot.video = {
+          const errorView = {
             updatedAt: observedAt2,
             source,
-            error
+            error,
+            rotation,
+            orientation
+          };
+          if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = errorView;
+          this.snapshot.cameras = {
+            ...this.snapshot.cameras ?? {},
+            [source]: {
+              ...this.snapshot.cameras?.[source] ?? { source, updatedAt: observedAt2 },
+              ...errorView,
+              updatedAt: observedAt2,
+              source
+            }
           };
           this.snapshot.events = mergeRecentById(this.snapshot.events, [{
             id: stableEventId("camera_error", source, observedAt2, error.slice(0, 80)),
@@ -602924,12 +603182,14 @@ var init_live_sensors = __esm({
             summary: error
           }], 50);
           this.persist();
-          this.emitFeedback({
-            kind: "error",
-            title: "Camera capture failed",
-            observedAt: observedAt2,
-            message: error
-          });
+          if (options2.emit !== false) {
+            this.emitFeedbackThrottled(`camera-capture:${source}:${error.slice(0, 80)}`, {
+              kind: "error",
+              title: "Camera capture failed",
+              observedAt: observedAt2,
+              message: error
+            }, 3e4);
+          }
           return { ok: false, message: error };
         }
         const framePath = extractSavedImagePath(result.output, this.repoRoot);
@@ -602938,15 +603198,26 @@ var init_live_sensors = __esm({
         const preview = await buildImageAsciiPreview(framePath, { width: 72, height: 22 });
         const asciiContext = preview ? formatImageAsciiContext(preview, displayPath) : void 0;
         const observedAt = Date.now();
-        this.snapshot.video = {
+        const cameraView = {
           updatedAt: observedAt,
           source,
           framePath,
           displayPath,
+          ascii: preview?.ascii,
+          renderer: preview?.renderer,
           asciiContext,
           rotation,
           orientation
         };
+        if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = cameraView;
+        this.snapshot.cameras = {
+          ...this.snapshot.cameras ?? {},
+          [source]: {
+            ...cameraView,
+            summary: this.snapshot.cameras?.[source]?.summary,
+            inference: this.snapshot.cameras?.[source]?.inference
+          }
+        };
         this.snapshot.events = mergeRecentById(this.snapshot.events, [{
           id: stableEventId("camera_frame", source, observedAt, framePath),
           kind: "camera_frame",
@@ -602957,16 +603228,18 @@ var init_live_sensors = __esm({
           evidencePath: framePath
         }], 50);
         this.persist();
-        this.emitFeedback({
-          kind: "camera-frame",
-          title: "Live camera frame",
-          observedAt,
-          message: `Captured from ${source}${orientation ? ` (${formatCameraRotation(orientation.rotation)})` : ""}: ${framePath}`,
-          imagePath: framePath,
-          displayPath,
-          ascii: preview?.ascii,
-          renderer: preview?.renderer
-        });
+        if (options2.emit !== false) {
+          this.emitFeedback({
+            kind: "camera-frame",
+            title: "Live camera frame",
+            observedAt,
+            message: `Captured from ${source}${orientation ? ` (${formatCameraRotation(orientation.rotation)})` : ""}: ${framePath}`,
+            imagePath: framePath,
+            displayPath,
+            ascii: preview?.ascii,
+            renderer: preview?.renderer
+          });
+        }
         return {
           ok: true,
           message: `Camera frame captured from ${source}: ${framePath}`,
@@ -602976,7 +603249,7 @@ var init_live_sensors = __esm({
           renderer: preview?.renderer
         };
       }
-      async recognizeFrameObjects(framePath, source, observedAt = Date.now()) {
+      async recognizeFrameObjects(framePath, source, observedAt = Date.now(), options2 = {}) {
         try {
           const result = await new VisualMemoryTool().execute({
             action: "recognize",
@@ -602996,109 +603269,124 @@ var init_live_sensors = __esm({
             evidencePath: framePath
           }], 50);
           this.persist();
-          this.emitFeedback({
-            kind: "clip",
-            title: result.success ? "CLIP visual-memory recognition" : "CLIP visual-memory unavailable",
-            observedAt,
-            message: summary
-          });
+          if (options2.emit !== false) {
+            this.emitFeedback({
+              kind: "clip",
+              title: result.success ? "CLIP visual-memory recognition" : "CLIP visual-memory unavailable",
+              observedAt,
+              message: summary
+            });
+          }
           return summary;
         } catch (err) {
           const message2 = `CLIP/visual-memory failed: ${err instanceof Error ? err.message : String(err)}`;
-          this.emitFeedback({
-            kind: "error",
-            title: "CLIP visual-memory failed",
-            observedAt,
-            message: message2
-          });
+          if (options2.emit !== false) {
+            this.emitFeedbackThrottled(`clip:${source}`, {
+              kind: "error",
+              title: "CLIP visual-memory failed",
+              observedAt,
+              message: message2
+            }, 6e4);
+          }
           return message2;
         }
       }
-      async sampleVideoNow(forceInfer = this.config.inferEnabled) {
-        if (this.videoInFlight) return "Video sampler is already running.";
-        this.videoInFlight = true;
-        try {
-          const preview = await this.previewCamera(this.config.selectedCamera);
-          let inference = "";
-          let clipSummary = "";
-          const source = this.config.selectedCamera || firstDeviceId(this.devices.video);
-          const orientation = source ? this.getCameraOrientation(source) : void 0;
-          const sampledAt = Date.now();
-          if (forceInfer && source) {
-            const loop = new LiveMediaLoopTool(this.repoRoot);
-            const result = await loop.execute({
-              action: "watch",
-              source_kind: "camera",
-              camera: source,
-              duration_sec: 4,
-              sample_interval_sec: 1,
-              max_frames: 4,
-              rotate_degrees: orientation?.rotation ?? 0,
-              auto_bootstrap: true,
-              audio_mode: "none",
-              detect_objects: true,
-              track_objects: true,
-              crop_faces: true,
-              recognize_faces: true
-            });
-            inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
-            const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
-            const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
-            this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
-            this.snapshot.people = mergeRecentById(this.snapshot.people, observations.people, 24);
-            this.snapshot.video = {
-              ...this.snapshot.video ?? { updatedAt: Date.now(), source },
-              updatedAt: sampledAt,
-              source,
-              inference,
-              rotation: orientation?.rotation ?? 0,
-              orientation,
-              ...preview.ok ? {} : { error: preview.message }
-            };
-            this.persist();
-            this.emitFeedback({
-              kind: result.success ? "video-inference" : "error",
-              title: result.success ? "Live video inference" : "Live video inference failed",
+      async sampleSingleVideoNow(source, forceInfer) {
+        const preview = await this.previewCamera(source, { emit: false });
+        let inference = "";
+        let clipSummary = "";
+        const orientation = this.getCameraOrientation(source);
+        const sampledAt = Date.now();
+        if (forceInfer && source) {
+          const loop = new LiveMediaLoopTool(this.repoRoot);
+          const result = await loop.execute({
+            action: "watch",
+            source_kind: "camera",
+            camera: source,
+            duration_sec: 2.5,
+            sample_interval_sec: 1,
+            max_frames: 2,
+            rotate_degrees: orientation?.rotation ?? 0,
+            auto_bootstrap: true,
+            audio_mode: "none",
+            detect_objects: true,
+            track_objects: true,
+            crop_faces: true,
+            recognize_faces: true
+          });
+          inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
+          const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
+          const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
+          this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
+          this.snapshot.people = mergeRecentById(this.snapshot.people, observations.people, 24);
+          const existing = this.snapshot.cameras?.[source] ?? this.snapshot.video ?? { updatedAt: Date.now(), source };
+          const camera = {
+            ...existing,
+            updatedAt: sampledAt,
+            source,
+            inference,
+            summary: summarizeInference(inference),
+            rotation: orientation?.rotation ?? 0,
+            orientation,
+            ...preview.ok ? { error: void 0 } : { error: preview.message }
+          };
+          this.snapshot.cameras = {
+            ...this.snapshot.cameras ?? {},
+            [source]: camera
+          };
+          if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = camera;
+          this.persist();
+          if (!result.success) {
+            this.emitFeedbackThrottled(`video-infer:${source}`, {
+              kind: "error",
+              title: "Live video inference failed",
               observedAt: sampledAt,
-              message: boundedText(inference, 1800)
-            });
-            for (const person of observations.people.filter((entry) => entry.cropPath).slice(0, 4)) {
-              const cropPath = person.cropPath;
-              const displayPath = relative11(this.repoRoot, cropPath).startsWith("..") ? cropPath : relative11(this.repoRoot, cropPath);
-              const facePreview = await buildImageAsciiPreview(cropPath, { width: 38, height: 14 });
-              this.emitFeedback({
-                kind: "face-crop",
-                title: person.status === "known" && person.name ? `Known face: ${person.name}` : "Unknown face awaiting identification",
-                observedAt: person.observedAt,
-                message: person.status === "known" && person.name ? `Identified ${person.name}${person.confidence !== void 0 ? ` confidence=${person.confidence.toFixed(2)}` : ""}` : `Face crop saved and awaiting identification${person.confidence !== void 0 ? ` confidence=${person.confidence.toFixed(2)}` : ""}`,
-                imagePath: cropPath,
-                displayPath,
-                ascii: facePreview?.ascii,
-                renderer: facePreview?.renderer
-              });
-            }
-            const unknownPeople = observations.people.filter((entry) => entry.status === "unknown");
-            if (unknownPeople.length > 0) {
-              this.maybeRequestAgentReview(
-                `${unknownPeople.length} unknown individual(s) observed on live camera`,
-                unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; ")
-              );
-            }
-            const triggerEvents = observations.events.filter((entry) => entry.kind === "visual_trigger");
-            if (triggerEvents.length > 0) {
-              this.maybeRequestAgentReview(
-                "Visual trigger hit in live camera stream",
-                triggerEvents.map((entry) => entry.summary).join("; ")
-              );
-            }
+              message: boundedText(inference, 1200)
+            }, 45e3);
+          }
+          const unknownPeople = observations.people.filter((entry) => entry.status === "unknown");
+          if (unknownPeople.length > 0) {
+            this.maybeRequestAgentReview(
+              `${unknownPeople.length} unknown individual(s) observed on live camera ${source}`,
+              unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; ")
+            );
+          }
+          const triggerEvents = observations.events.filter((entry) => entry.kind === "visual_trigger");
+          if (triggerEvents.length > 0) {
+            this.maybeRequestAgentReview(
+              `Visual trigger hit in live camera stream ${source}`,
+              triggerEvents.map((entry) => entry.summary).join("; ")
+            );
           }
-          if (this.config.clipEnabled && preview.ok && preview.framePath && source) {
-            clipSummary = await this.recognizeFrameObjects(preview.framePath, source, sampledAt);
+        }
+        if (this.config.clipEnabled && preview.ok && preview.framePath && source) {
+          clipSummary = await this.recognizeFrameObjects(preview.framePath, source, sampledAt, { emit: false });
+          const current = this.snapshot.cameras?.[source];
+          if (current) {
+            current.summary = [current.summary, clipSummary ? `clip: ${trimOneLine(clipSummary, 160)}` : ""].filter(Boolean).join(" | ");
+            this.snapshot.cameras = { ...this.snapshot.cameras ?? {}, [source]: current };
+            if (this.snapshot.video?.source === source) this.snapshot.video = current;
+            this.persist();
           }
-          return [preview.message, inference ? `
+        }
+        return [preview.message, inference ? `
 ${inference}` : "", clipSummary ? `
 CLIP visual memory:
 ${clipSummary}` : ""].filter(Boolean).join("\n");
+      }
+      async sampleVideoNow(forceInfer = this.config.inferEnabled) {
+        if (this.videoInFlight) return "Video sampler is already running.";
+        this.videoInFlight = true;
+        try {
+          const sources = this.activeVideoSources();
+          if (sources.length === 0) return "No camera selected. Use /live refresh after connecting hardware.";
+          const outputs = [];
+          for (const source of sources) {
+            outputs.push(await this.sampleSingleVideoNow(source, forceInfer));
+          }
+          this.emitDashboard();
+          return outputs.map((output, index) => `## Camera ${sources[index]}
+${output}`).join("\n\n");
         } finally {
           this.videoInFlight = false;
         }
@@ -603113,10 +603401,9 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
           const recordingPath = join124(outputDir2, `audio-${Date.now()}.wav`);
           let transcript = "";
           let analysis = "";
-          try {
-            await recordAudioSample(input, 3, recordingPath);
-          } catch (err) {
-            const message2 = `Audio capture failed from ${input}: ${err instanceof Error ? err.message : String(err)}`;
+          const capture = await recordAudioSampleWithRecovery(input, this.devices.audioInputs, 3, recordingPath);
+          if (!capture.ok) {
+            const message2 = `Audio capture failed from ${input}; tried fallbacks: ${capture.errors.slice(0, 5).join(" | ")}`;
             this.snapshot.audio = {
               updatedAt: Date.now(),
               input,
@@ -603124,14 +603411,18 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
               error: message2
             };
             this.persist();
-            this.emitFeedback({
+            this.emitFeedbackThrottled(`audio-capture:${input}`, {
               kind: "error",
               title: "Live audio capture failed",
               observedAt: this.snapshot.audio.updatedAt,
               message: message2
-            });
+            }, 6e4);
+            this.emitDashboard();
             return message2;
           }
+          if (capture.device !== this.config.selectedAudioInput) {
+            this.config.selectedAudioInput = capture.device;
+          }
           if (this.config.asrEnabled) {
             try {
               const asr = await new TranscribeFileTool(this.repoRoot).execute({ path: recordingPath, model: "tiny" });
@@ -603159,7 +603450,7 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
           }
           this.snapshot.audio = {
             updatedAt: Date.now(),
-            input,
+            input: capture.device,
             output: this.config.selectedAudioOutput,
             recordingPath,
             transcript,
@@ -603168,10 +603459,10 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
           const audioEvents = [];
           if (transcript) {
             audioEvents.push({
-              id: stableEventId("audio_transcript", input, this.snapshot.audio.updatedAt, recordingPath),
+              id: stableEventId("audio_transcript", capture.device, this.snapshot.audio.updatedAt, recordingPath),
               kind: "audio_transcript",
               observedAt: this.snapshot.audio.updatedAt,
-              source: input,
+              source: capture.device,
               title: "Live ASR transcript",
               summary: transcript,
               evidencePath: recordingPath
@@ -603179,10 +603470,10 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
           }
           if (analysis) {
             audioEvents.push({
-              id: stableEventId("audio_sound", input, this.snapshot.audio.updatedAt, recordingPath),
+              id: stableEventId("audio_sound", capture.device, this.snapshot.audio.updatedAt, recordingPath),
               kind: "audio_sound",
               observedAt: this.snapshot.audio.updatedAt,
-              source: input,
+              source: capture.device,
               title: "Live sound analysis",
               summary: analysis,
               evidencePath: recordingPath
@@ -603190,21 +603481,9 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
           }
           this.snapshot.events = mergeRecentById(this.snapshot.events, audioEvents, 50);
           this.persist();
-          if (transcript || analysis) {
-            this.emitFeedback({
-              kind: "audio",
-              title: "Live audio sample",
-              observedAt: this.snapshot.audio.updatedAt,
-              message: [
-                transcript ? `ASR:
-${boundedText(transcript, 700)}` : "",
-                analysis ? `Sounds:
-${boundedText(analysis, 900)}` : ""
-              ].filter(Boolean).join("\n\n")
-            });
-          }
+          this.emitDashboard();
           return [
-            `Audio sample captured from ${input}: ${recordingPath}`,
+            `Audio sample captured from ${capture.device}${capture.device !== input ? ` (recovered from ${input})` : ""}: ${recordingPath}`,
             transcript ? `
 ASR:
 ${transcript}` : "",
@@ -603256,6 +603535,21 @@ ${analysis}` : ""
         } catch {
         }
       }
+      emitFeedbackThrottled(key, feedback, minIntervalMs) {
+        const now2 = Date.now();
+        const last2 = this.lastFeedbackAt.get(key) ?? 0;
+        if (now2 - last2 < minIntervalMs) return;
+        this.lastFeedbackAt.set(key, now2);
+        this.emitFeedback(feedback);
+      }
+      emitDashboard() {
+        this.emitFeedback({
+          kind: "dashboard",
+          title: "Live audio/video monitor",
+          observedAt: Date.now(),
+          message: "live dashboard refresh"
+        });
+      }
       maybeRequestAgentReview(reason, evidence) {
         if (!this.agentActionEnabled || !this.agentActionSink) return;
         const now2 = Date.now();
@@ -654970,7 +655264,33 @@ function renderLivePreview(preview) {
     renderWarning("Camera preview was captured, but ASCII rendering was unavailable.");
   }
 }
-function renderLiveFeedback(feedback) {
+function renderLiveDashboard(ctx3, manager) {
+  const host = ctx3.liveDynamicBlockHost;
+  if (!host) {
+    renderInfo(formatLiveDashboardFromSnapshot(manager.getSnapshot(), { width: 100 }).join("\n"));
+    return;
+  }
+  const key = ctx3.repoRoot;
+  let state = liveDashboardBlocks.get(key);
+  if (!state || state.host !== host) {
+    const id2 = `live-dashboard-${Math.random().toString(36).slice(2, 9)}`;
+    state = { host, id: id2, manager };
+    liveDashboardBlocks.set(key, state);
+    host.registerDynamicBlock(
+      id2,
+      (width) => formatLiveDashboardFromSnapshot(manager.getSnapshot(), { width })
+    );
+    host.appendDynamicBlock(id2);
+    return;
+  }
+  state.manager = manager;
+  host.refreshDynamicBlocks?.();
+}
+function renderLiveFeedback(ctx3, manager, feedback) {
+  if (feedback.kind === "dashboard") {
+    renderLiveDashboard(ctx3, manager);
+    return;
+  }
   const stamp = new Date(feedback.observedAt).toISOString();
   if (feedback.kind === "error") {
     renderWarning(`[live ${stamp}] ${feedback.title}
@@ -654987,7 +655307,7 @@ ${feedback.message}`);
 }
 function attachLiveSinks(ctx3, manager, agentEnabled = false) {
   manager.setStatusSink(ctx3.setLiveMediaStatus);
-  manager.setFeedbackSink((feedback) => renderLiveFeedback(feedback));
+  manager.setFeedbackSink((feedback) => renderLiveFeedback(ctx3, manager, feedback));
   if (agentEnabled && ctx3.startBackgroundPrompt) {
     manager.setAgentActionSink((prompt) => {
       const id2 = ctx3.startBackgroundPrompt?.(prompt);
@@ -660485,7 +660805,7 @@ async function showExposeDashboard(gateway, rl, ctx3) {
     renderInfo("Expose gateway stopped.");
   }
 }
-var OMNIUS_PINNED_DEPENDENCY_SPECS, NEXUS_DIRECTORY_ORIGIN, NEXUS_SPONSORS_URL, _sponsorHeartbeatTimer, _lastRegisteredSponsorPayload, __COMMAND_REGISTRY, DASH_INTERNAL, localGpuMetricsCache, localGpuMetricsProbeAt, localGpuMetricsProbeInFlight;
+var OMNIUS_PINNED_DEPENDENCY_SPECS, NEXUS_DIRECTORY_ORIGIN, NEXUS_SPONSORS_URL, _sponsorHeartbeatTimer, _lastRegisteredSponsorPayload, __COMMAND_REGISTRY, liveDashboardBlocks, DASH_INTERNAL, localGpuMetricsCache, localGpuMetricsProbeAt, localGpuMetricsProbeInFlight;
 var init_commands = __esm({
   "packages/cli/src/tui/commands.ts"() {
     "use strict";
@@ -660803,6 +661123,7 @@ var init_commands = __esm({
         }
       });
     })();
+    liveDashboardBlocks = /* @__PURE__ */ new Map();
     DASH_INTERNAL = /* @__PURE__ */ new Set(["system_metrics", "__list_capabilities"]);
     localGpuMetricsCache = null;
     localGpuMetricsProbeAt = 0;
@@ -733124,6 +733445,11 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
     setLiveMediaStatus(status) {
       statusBar.setLiveMediaStatus(status);
     },
+    liveDynamicBlockHost: {
+      registerDynamicBlock: (id2, render2) => statusBar.registerDynamicBlock(id2, render2),
+      appendDynamicBlock: (id2) => statusBar.appendDynamicBlock(id2),
+      refreshDynamicBlocks: () => statusBar.refreshDynamicBlocks()
+    },
     // Voice call session — standalone cloudflared tunnel for /call
     // Each connecting WebSocket client gets a dedicated CallSubAgent.
     // Admin callers (matching session key) get full tool access; public callers get read-only.

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "omnius",
-  "version": "1.0.407",
+  "version": "1.0.408",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "omnius",
-      "version": "1.0.407",
+      "version": "1.0.408",
       "bundleDependencies": [
         "image-to-ascii"
       ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omnius",
-  "version": "1.0.407",
+  "version": "1.0.408",
   "description": "AI coding agent powered by open-source models (Ollama/vLLM) — interactive TUI with agentic tool-calling loop",
   "type": "module",
   "main": "./dist/index.js",