omnius 1.0.407 → 1.0.409

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -41519,7 +41519,7 @@ function formatTime(seconds) {
41519
41519
  const s2 = Math.floor(seconds % 60);
41520
41520
  return `${String(m2).padStart(2, "0")}:${String(s2).padStart(2, "0")}`;
41521
41521
  }
41522
- var AUDIO_EXTS, VIDEO_EXTS, MAX_TRANSCRIBE_URL_BYTES, _tcModule, _tcChecked, TranscribeFileTool, YT_DLP_VENV, TranscribeUrlTool, YouTubeDownloadTool;
41522
+ var AUDIO_EXTS, VIDEO_EXTS, MAX_TRANSCRIBE_URL_BYTES, MANAGED_ASR_VENV, _tcModule, _tcChecked, _managedAsrReady, TranscribeFileTool, YT_DLP_VENV, TranscribeUrlTool, YouTubeDownloadTool;
41523
41523
  var init_transcribe_tool = __esm({
41524
41524
  "packages/execution/dist/tools/transcribe-tool.js"() {
41525
41525
  "use strict";
@@ -41550,8 +41550,10 @@ var init_transcribe_tool = __esm({
41550
41550
  ".ts"
41551
41551
  ]);
41552
41552
  MAX_TRANSCRIBE_URL_BYTES = 100 * 1024 * 1024;
41553
+ MANAGED_ASR_VENV = join40(homedir11(), ".omnius", "runtimes", "asr", ".venv-whisper");
41553
41554
  _tcModule = null;
41554
41555
  _tcChecked = false;
41556
+ _managedAsrReady = false;
41555
41557
  TranscribeFileTool = class {
41556
41558
  name = "transcribe_file";
41557
41559
  description = "Transcribe a local audio or video file to text using Whisper (faster-whisper). Supports MP3, WAV, FLAC, AAC, M4A, OGG, MP4, MKV, AVI, MOV, WebM. Returns the full transcription text with optional speaker diarization. Transcription is 100% local — no API keys needed.";
@@ -41625,102 +41627,131 @@ var init_transcribe_tool = __esm({
41625
41627
  }
41626
41628
  if (effectiveModel !== askedModel)
41627
41629
  model = effectiveModel;
41630
+ const failures = [];
41628
41631
  const tc = await loadTranscribeCli();
41629
- if (!tc) {
41630
- return this.execViaCli(filePath, model, diarize, start2);
41631
- }
41632
+ if (tc) {
41633
+ try {
41634
+ const result = await withProcessEnv(transcriptionPythonEnv(), () => tc.transcribe(filePath, {
41635
+ model,
41636
+ format: "json",
41637
+ diarize,
41638
+ wordTimestamps: true
41639
+ }));
41640
+ return this.formatTranscriptionResult(filePath, model, result, start2, "transcribe-cli");
41641
+ } catch (err) {
41642
+ failures.push(`transcribe-cli module: ${err instanceof Error ? err.message : String(err)}`);
41643
+ }
41644
+ }
41645
+ const cli = await this.execViaCli(filePath, model, diarize, start2);
41646
+ if (cli.success)
41647
+ return cli;
41648
+ if (cli.error)
41649
+ failures.push(cli.error);
41650
+ const managed = await this.execViaManagedWhisper(filePath, model, start2);
41651
+ if (managed.success)
41652
+ return managed;
41653
+ if (managed.error)
41654
+ failures.push(managed.error);
41655
+ return {
41656
+ success: false,
41657
+ output: "",
41658
+ error: `Transcription failed after all ASR backends. ${failures.join(" | ").slice(0, 1200)}`,
41659
+ durationMs: performance.now() - start2
41660
+ };
41661
+ }
41662
+ formatTranscriptionResult(filePath, model, result, start2, backend) {
41663
+ const segments = Array.isArray(result.segments) ? result.segments : [];
41664
+ const text2 = String(result.text ?? result.fullText ?? "").trim();
41665
+ const speakers = Array.isArray(result.speakers) ? result.speakers : [];
41666
+ const wordCount2 = Number(result.wordCount ?? (text2 ? text2.split(/\s+/).filter(Boolean).length : 0));
41667
+ const transcriptDir = join40(this.workingDir, ".omnius", "transcripts");
41668
+ mkdirSync22(transcriptDir, { recursive: true });
41669
+ const fileBase = basename7(filePath).replace(/\.[^.]+$/, "");
41670
+ const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
41671
+ const txtFile = join40(transcriptDir, `${fileBase}-${timestamp}.txt`);
41672
+ const jsonFile = join40(transcriptDir, `${fileBase}-${timestamp}.json`);
41673
+ writeFileSync21(txtFile, text2, "utf-8");
41674
+ const structured = {
41675
+ source: filePath,
41676
+ model,
41677
+ backend,
41678
+ language: result.language,
41679
+ duration: result.duration,
41680
+ wordCount: wordCount2,
41681
+ speakers,
41682
+ transcribedAt: (/* @__PURE__ */ new Date()).toISOString(),
41683
+ segments: segments.map((seg) => ({
41684
+ start: Number(seg.start ?? 0),
41685
+ end: Number(seg.end ?? seg.start ?? 0),
41686
+ text: String(seg.text ?? ""),
41687
+ speaker: seg.speaker || void 0
41688
+ })),
41689
+ fullText: text2
41690
+ };
41691
+ writeFileSync21(jsonFile, JSON.stringify(structured, null, 2), "utf-8");
41632
41692
  try {
41633
- const result = await withProcessEnv(transcriptionPythonEnv(), () => tc.transcribe(filePath, {
41634
- model,
41635
- format: "json",
41636
- diarize,
41637
- wordTimestamps: true
41638
- // Always get timestamps for structured output
41639
- }));
41640
- const transcriptDir = join40(this.workingDir, ".omnius", "transcripts");
41641
- mkdirSync22(transcriptDir, { recursive: true });
41642
- const fileBase = basename7(filePath).replace(/\.[^.]+$/, "");
41643
- const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
41644
- const txtFile = join40(transcriptDir, `${fileBase}-${timestamp}.txt`);
41645
- writeFileSync21(txtFile, result.text, "utf-8");
41646
- const jsonFile = join40(transcriptDir, `${fileBase}-${timestamp}.json`);
41647
- const structured = {
41648
- source: filePath,
41649
- model,
41650
- language: result.language,
41651
- duration: result.duration,
41652
- wordCount: result.wordCount,
41653
- speakers: result.speakers,
41654
- transcribedAt: (/* @__PURE__ */ new Date()).toISOString(),
41655
- segments: result.segments.map((seg) => ({
41656
- start: seg.start,
41657
- end: seg.end,
41658
- text: seg.text,
41659
- speaker: seg.speaker || void 0
41660
- })),
41661
- fullText: result.text
41662
- };
41663
- writeFileSync21(jsonFile, JSON.stringify(structured, null, 2), "utf-8");
41693
+ const memDir = join40(this.workingDir, ".omnius", "memory");
41694
+ mkdirSync22(memDir, { recursive: true });
41695
+ const memFile = join40(memDir, "transcripts.json");
41696
+ let memEntries = [];
41664
41697
  try {
41665
- const memDir = join40(this.workingDir, ".omnius", "memory");
41666
- mkdirSync22(memDir, { recursive: true });
41667
- const memFile = join40(memDir, "transcripts.json");
41668
- let memEntries = [];
41669
- try {
41670
- memEntries = JSON.parse(readFileSync30(memFile, "utf-8"));
41671
- } catch {
41672
- }
41673
- memEntries.push({
41674
- source: basename7(filePath),
41675
- timestamp: (/* @__PURE__ */ new Date()).toISOString(),
41676
- duration: result.duration,
41677
- wordCount: result.wordCount,
41678
- language: result.language,
41679
- summary: result.text.slice(0, 200),
41680
- jsonPath: jsonFile,
41681
- txtPath: txtFile
41682
- });
41683
- if (memEntries.length > 50)
41684
- memEntries = memEntries.slice(-50);
41685
- writeFileSync21(memFile, JSON.stringify(memEntries, null, 2), "utf-8");
41698
+ memEntries = JSON.parse(readFileSync30(memFile, "utf-8"));
41686
41699
  } catch {
41687
41700
  }
41688
- const lines = [
41689
- `Transcription of: ${basename7(filePath)}`,
41690
- `Model: ${model} | Language: ${result.language} | Duration: ${result.duration ? `${result.duration.toFixed(1)}s` : "unknown"}`,
41691
- `Words: ${result.wordCount} | Segments: ${result.segments.length}`,
41692
- `Saved: ${txtFile} (text) + ${jsonFile} (structured with timestamps)`,
41693
- ""
41694
- ];
41695
- if (result.speakers.length > 1) {
41696
- lines.push(`Speakers: ${result.speakers.join(", ")}`);
41697
- lines.push("");
41698
- }
41699
- if (result.segments.length > 0) {
41700
- for (const seg of result.segments) {
41701
- const ts = `[${formatTime(seg.start)} → ${formatTime(seg.end)}]`;
41702
- const speaker = seg.speaker ? `${seg.speaker}: ` : "";
41703
- lines.push(`${ts} ${speaker}${seg.text}`);
41704
- }
41705
- } else {
41706
- lines.push(result.text);
41701
+ memEntries.push({
41702
+ source: basename7(filePath),
41703
+ timestamp: (/* @__PURE__ */ new Date()).toISOString(),
41704
+ duration: result.duration,
41705
+ wordCount: wordCount2,
41706
+ language: result.language,
41707
+ backend,
41708
+ summary: text2.slice(0, 200),
41709
+ jsonPath: jsonFile,
41710
+ txtPath: txtFile
41711
+ });
41712
+ if (memEntries.length > 50)
41713
+ memEntries = memEntries.slice(-50);
41714
+ writeFileSync21(memFile, JSON.stringify(memEntries, null, 2), "utf-8");
41715
+ } catch {
41716
+ }
41717
+ const lines = [
41718
+ `Transcription of: ${basename7(filePath)}`,
41719
+ `Backend: ${backend}`,
41720
+ `Model: ${model} | Language: ${result.language ?? "unknown"} | Duration: ${result.duration ? `${Number(result.duration).toFixed(1)}s` : "unknown"}`,
41721
+ `Words: ${wordCount2} | Segments: ${segments.length}`,
41722
+ `Saved: ${txtFile} (text) + ${jsonFile} (structured with timestamps)`,
41723
+ ""
41724
+ ];
41725
+ if (speakers.length > 1) {
41726
+ lines.push(`Speakers: ${speakers.join(", ")}`);
41727
+ lines.push("");
41728
+ }
41729
+ if (segments.length > 0) {
41730
+ for (const seg of segments) {
41731
+ const ts = `[${formatTime(Number(seg.start ?? 0))} → ${formatTime(Number(seg.end ?? seg.start ?? 0))}]`;
41732
+ const speaker = seg.speaker ? `${seg.speaker}: ` : "";
41733
+ lines.push(`${ts} ${speaker}${String(seg.text ?? "").trim()}`);
41707
41734
  }
41708
- return {
41709
- success: true,
41710
- output: lines.join("\n"),
41711
- durationMs: performance.now() - start2
41712
- };
41713
- } catch (err) {
41735
+ } else {
41736
+ lines.push(text2 || "[silent — no speech detected]");
41737
+ }
41738
+ return {
41739
+ success: true,
41740
+ output: lines.join("\n"),
41741
+ llmContent: text2,
41742
+ durationMs: performance.now() - start2
41743
+ };
41744
+ }
41745
+ /** Fallback: invoke transcribe-cli as a subprocess. */
41746
+ async execViaCli(filePath, model, diarize, start2) {
41747
+ if (!await commandExists("transcribe-cli", 1500)) {
41714
41748
  return {
41715
41749
  success: false,
41716
41750
  output: "",
41717
- error: `Transcription failed: ${err instanceof Error ? err.message : String(err)}`,
41751
+ error: "transcribe-cli command unavailable",
41718
41752
  durationMs: performance.now() - start2
41719
41753
  };
41720
41754
  }
41721
- }
41722
- /** Fallback: invoke transcribe-cli as a subprocess. */
41723
- async execViaCli(filePath, model, diarize, start2) {
41724
41755
  try {
41725
41756
  const args = [filePath, "-m", model, "-f", "txt"];
41726
41757
  if (diarize)
@@ -41734,6 +41765,7 @@ var init_transcribe_tool = __esm({
41734
41765
  return {
41735
41766
  success: true,
41736
41767
  output: output.trim() || "(empty transcription)",
41768
+ llmContent: output.trim() || "",
41737
41769
  durationMs: performance.now() - start2
41738
41770
  };
41739
41771
  } catch (err) {
@@ -41747,6 +41779,112 @@ var init_transcribe_tool = __esm({
41747
41779
  };
41748
41780
  }
41749
41781
  }
41782
+ async ensureManagedWhisperRuntime() {
41783
+ const python = venvPython(MANAGED_ASR_VENV);
41784
+ if (!existsSync38(python)) {
41785
+ mkdirSync22(join40(homedir11(), ".omnius", "runtimes", "asr"), { recursive: true });
41786
+ await execFileText("python3", ["-m", "venv", MANAGED_ASR_VENV], {
41787
+ timeout: 6e4,
41788
+ env: transcriptionPythonEnv()
41789
+ });
41790
+ }
41791
+ if (_managedAsrReady)
41792
+ return python;
41793
+ try {
41794
+ await execFileText(python, ["-c", "import whisper, numpy"], {
41795
+ timeout: 1e4,
41796
+ env: transcriptionPythonEnv()
41797
+ });
41798
+ _managedAsrReady = true;
41799
+ return python;
41800
+ } catch {
41801
+ await execFileText(python, [
41802
+ "-m",
41803
+ "pip",
41804
+ "install",
41805
+ "-U",
41806
+ "pip",
41807
+ "wheel",
41808
+ "setuptools",
41809
+ "numpy",
41810
+ "openai-whisper"
41811
+ ], {
41812
+ timeout: 9e5,
41813
+ env: transcriptionPythonEnv()
41814
+ });
41815
+ _managedAsrReady = true;
41816
+ return python;
41817
+ }
41818
+ }
41819
+ async execViaManagedWhisper(filePath, model, start2) {
41820
+ const scriptPath2 = join40(this.workingDir, ".omnius", "tmp", `managed-whisper-${Date.now()}.py`);
41821
+ mkdirSync22(join40(this.workingDir, ".omnius", "tmp"), { recursive: true });
41822
+ const script = `
41823
+ import json, os, warnings
41824
+ warnings.filterwarnings("ignore")
41825
+ os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3")
41826
+ audio_file = ${JSON.stringify(filePath)}
41827
+ model_name = ${JSON.stringify(model)}
41828
+ try:
41829
+ import whisper
41830
+ device = "cpu"
41831
+ try:
41832
+ import torch
41833
+ if torch.cuda.is_available():
41834
+ device = "cuda"
41835
+ except Exception:
41836
+ pass
41837
+ m = whisper.load_model(model_name, device=device)
41838
+ result = m.transcribe(audio_file, fp16=(device == "cuda"), condition_on_previous_text=False)
41839
+ segments = []
41840
+ for seg in result.get("segments", []) or []:
41841
+ segments.append({"start": float(seg.get("start", 0)), "end": float(seg.get("end", seg.get("start", 0))), "text": str(seg.get("text", "")).strip()})
41842
+ text = str(result.get("text", "")).strip()
41843
+ print(json.dumps({
41844
+ "ok": True,
41845
+ "text": text,
41846
+ "language": result.get("language"),
41847
+ "duration": segments[-1]["end"] if segments else None,
41848
+ "wordCount": len([w for w in text.split() if w]),
41849
+ "speakers": [],
41850
+ "segments": segments,
41851
+ }))
41852
+ except Exception as exc:
41853
+ print(json.dumps({"ok": False, "error": str(exc)}))
41854
+ `;
41855
+ try {
41856
+ writeFileSync21(scriptPath2, script, "utf-8");
41857
+ const python = await this.ensureManagedWhisperRuntime();
41858
+ const raw = await execFileText(python, [scriptPath2], {
41859
+ timeout: 6e5,
41860
+ cwd: this.workingDir,
41861
+ env: transcriptionPythonEnv({ PYTHONUNBUFFERED: "1" }),
41862
+ maxBuffer: 16 * 1024 * 1024
41863
+ });
41864
+ const parsed = JSON.parse(raw.trim().split("\n").reverse().find((line) => line.trim().startsWith("{")) ?? "{}");
41865
+ if (!parsed.ok) {
41866
+ return {
41867
+ success: false,
41868
+ output: "",
41869
+ error: `managed openai-whisper failed: ${parsed.error || raw.slice(-500)}`,
41870
+ durationMs: performance.now() - start2
41871
+ };
41872
+ }
41873
+ return this.formatTranscriptionResult(filePath, model, parsed, start2, "managed openai-whisper");
41874
+ } catch (err) {
41875
+ return {
41876
+ success: false,
41877
+ output: "",
41878
+ error: `managed openai-whisper failed: ${err instanceof Error ? err.message : String(err)}`,
41879
+ durationMs: performance.now() - start2
41880
+ };
41881
+ } finally {
41882
+ try {
41883
+ unlinkSync5(scriptPath2);
41884
+ } catch {
41885
+ }
41886
+ }
41887
+ }
41750
41888
  };
41751
41889
  YT_DLP_VENV = join40(homedir11(), ".omnius", "runtimes", "media-tools", ".venv-ytdlp");
41752
41890
  TranscribeUrlTool = class {
@@ -547804,17 +547942,23 @@ except Exception as e:
547804
547942
  const audioFile = await this.ensureAudioFile(args);
547805
547943
  if (!audioFile)
547806
547944
  return { success: false, output: "", error: "No audio available.", durationMs: performance.now() - start2 };
547807
- await this.ensureVenv(["torch", "torchaudio", "numpy", "soundfile"]);
547945
+ await this.ensureVenv(["torch", "numpy", "soundfile", "resampy"]);
547808
547946
  const audioLiteral = JSON.stringify(audioFile);
547809
547947
  const script = `
547810
547948
  import sys, json, numpy as np
547811
547949
  try:
547812
- import torch, torchaudio, soundfile as sf
547950
+ import torch, soundfile as sf, resampy
547813
547951
 
547814
547952
  model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad', force_reload=False, trust_repo=True)
547815
- (get_speech_timestamps, _, read_audio, *_) = utils
547953
+ (get_speech_timestamps, *_) = utils
547816
547954
 
547817
- wav = read_audio(${audioLiteral}, sampling_rate=16000)
547955
+ audio, sr = sf.read(${audioLiteral})
547956
+ if len(getattr(audio, "shape", [])) > 1:
547957
+ audio = audio.mean(axis=1)
547958
+ if sr != 16000:
547959
+ audio = resampy.resample(audio, sr, 16000)
547960
+ audio = audio.astype(np.float32)
547961
+ wav = torch.from_numpy(audio)
547818
547962
  speech_timestamps = get_speech_timestamps(wav, model, sampling_rate=16000, threshold=0.5)
547819
547963
 
547820
547964
  total_samples = len(wav)
@@ -547825,6 +547969,7 @@ try:
547825
547969
 
547826
547970
  print(json.dumps({
547827
547971
  "success": True,
547972
+ "backend": "silero-vad",
547828
547973
  "voice_detected": len(speech_timestamps) > 0,
547829
547974
  "speech_percentage": round(speech_pct, 1),
547830
547975
  "speech_segments": len(speech_timestamps),
@@ -547832,7 +547977,57 @@ try:
547832
547977
  "segments": segments[:10]
547833
547978
  }))
547834
547979
  except Exception as e:
547835
- print(json.dumps({"success": False, "error": str(e)}))
547980
+ try:
547981
+ import soundfile as sf
547982
+ import resampy
547983
+ audio, sr = sf.read(${audioLiteral})
547984
+ if len(getattr(audio, "shape", [])) > 1:
547985
+ audio = audio.mean(axis=1)
547986
+ if sr != 16000:
547987
+ audio = resampy.resample(audio, sr, 16000)
547988
+ audio = audio.astype(np.float32)
547989
+ frame = max(1, int(0.03 * 16000))
547990
+ hop = max(1, int(0.01 * 16000))
547991
+ rms = []
547992
+ for i in range(0, max(1, len(audio) - frame + 1), hop):
547993
+ chunk = audio[i:i+frame]
547994
+ if len(chunk) == 0:
547995
+ continue
547996
+ rms.append(float(np.sqrt(np.mean(chunk * chunk))))
547997
+ arr = np.array(rms, dtype=np.float32)
547998
+ if len(arr) == 0:
547999
+ active = np.array([], dtype=bool)
548000
+ else:
548001
+ floor = float(np.percentile(arr, 35))
548002
+ peak = float(np.max(arr))
548003
+ threshold = max(0.008, floor * 2.5, peak * 0.12)
548004
+ active = arr > threshold
548005
+ segments = []
548006
+ start_idx = None
548007
+ for idx, flag in enumerate(active.tolist()):
548008
+ if flag and start_idx is None:
548009
+ start_idx = idx
548010
+ if (not flag or idx == len(active) - 1) and start_idx is not None:
548011
+ end_idx = idx if not flag else idx + 1
548012
+ start_s = start_idx * hop / 16000.0
548013
+ end_s = min(len(audio) / 16000.0, (end_idx * hop + frame) / 16000.0)
548014
+ if end_s - start_s >= 0.12:
548015
+ segments.append({"start_s": start_s, "end_s": end_s, "duration_s": end_s - start_s})
548016
+ start_idx = None
548017
+ speech_duration = sum(seg["duration_s"] for seg in segments)
548018
+ total_duration = len(audio) / 16000.0 if len(audio) else 0
548019
+ print(json.dumps({
548020
+ "success": True,
548021
+ "backend": "energy-vad-fallback",
548022
+ "warning": "silero unavailable: " + str(e),
548023
+ "voice_detected": len(segments) > 0,
548024
+ "speech_percentage": round((speech_duration / total_duration * 100) if total_duration else 0, 1),
548025
+ "speech_segments": len(segments),
548026
+ "total_duration_s": round(total_duration, 2),
548027
+ "segments": segments[:10]
548028
+ }))
548029
+ except Exception as fallback:
548030
+ print(json.dumps({"success": False, "error": str(e) + "; fallback failed: " + str(fallback)}))
547836
548031
  `;
547837
548032
  return await this.runPythonScript(script, start2, "Voice activity detection");
547838
548033
  }
@@ -555516,6 +555711,9 @@ function summarizeObjects(objects) {
555516
555711
  }
555517
555712
  return [...counts.entries()].sort((a2, b) => b[1].count - a2[1].count).slice(0, 12).map(([label, value2]) => `${label} x${value2.count} mean_conf=${(value2.confidence / Math.max(1, value2.count)).toFixed(2)}`);
555518
555713
  }
555714
+ function isDiagnosticAudioEvidence(text2) {
555715
+ return /^(?:ASR|VAD|Audio comprehension)\s+(?:failed|unavailable|completed)|No module named ['"]?transcribe_cli|torchcodec|torchaudio version|no speech detected|empty transcription/i.test(text2.trim());
555716
+ }
555519
555717
  function observationsFromResult(result, extraText) {
555520
555718
  const observations = [];
555521
555719
  for (const obj of result.objects) {
@@ -555541,6 +555739,8 @@ function observationsFromResult(result, extraText) {
555541
555739
  });
555542
555740
  }
555543
555741
  for (const text2 of extraText) {
555742
+ if (isDiagnosticAudioEvidence(text2))
555743
+ continue;
555544
555744
  observations.push({
555545
555745
  type: "transcript",
555546
555746
  confidence: 0.75,
@@ -555958,7 +556158,12 @@ var init_live_media_loop = __esm({
555958
556158
  try {
555959
556159
  const transcribe = new TranscribeFileTool(this.workingDir);
555960
556160
  const asr = await transcribe.execute({ path: result.media_path, model: "tiny" });
555961
- audioEvidence.push(asr.success ? String(asr.llmContent || asr.output || "").slice(0, 3e3) : `ASR unavailable: ${asr.error}`);
556161
+ if (asr.success) {
556162
+ const transcript = typeof asr.llmContent === "string" ? asr.llmContent.trim() : "";
556163
+ audioEvidence.push(transcript ? transcript.slice(0, 3e3) : "ASR completed: no speech detected");
556164
+ } else {
556165
+ audioEvidence.push(`ASR unavailable: ${asr.error}`);
556166
+ }
555962
556167
  } catch (err) {
555963
556168
  audioEvidence.push(`ASR failed: ${err instanceof Error ? err.message : String(err)}`);
555964
556169
  }
@@ -601694,10 +601899,13 @@ function defaultAsciiPreviewSize(dimensions) {
601694
601899
  const columns = process.stdout.columns || 100;
601695
601900
  const rows = process.stdout.rows || 32;
601696
601901
  const width = clamp7(columns - 14, 42, 96);
601697
- const imageAspect = dimensions && dimensions.width > 0 && dimensions.height > 0 ? dimensions.height / dimensions.width : 1;
601698
- const height = clamp7(Math.min(Math.round(width * imageAspect * TERMINAL_CELL_ASPECT), rows - 10), 8, 42);
601902
+ const height = defaultAsciiPreviewHeight(width, dimensions, rows);
601699
601903
  return { width, height };
601700
601904
  }
601905
+ function defaultAsciiPreviewHeight(width, dimensions, rows = process.stdout.rows || 32) {
601906
+ const imageAspect = dimensions && dimensions.width > 0 && dimensions.height > 0 ? dimensions.height / dimensions.width : 1;
601907
+ return clamp7(Math.min(Math.round(width * imageAspect * TERMINAL_CELL_ASPECT), rows - 10), 8, 42);
601908
+ }
601701
601909
  function readImageDimensions2(imagePath) {
601702
601910
  try {
601703
601911
  const buf = readFileSync87(imagePath);
@@ -601959,7 +602167,7 @@ async function buildImageAsciiPreview(inputPath, options2 = {}) {
601959
602167
  const dimensions = readImageDimensions2(imagePath);
601960
602168
  const defaults3 = defaultAsciiPreviewSize(dimensions);
601961
602169
  const width = clamp7(options2.width ?? defaults3.width, 24, 140);
601962
- const height = clamp7(options2.height ?? defaults3.height, 6, 60);
602170
+ const height = clamp7(options2.height ?? defaultAsciiPreviewHeight(width, dimensions), 6, 60);
601963
602171
  const timeoutMs = clamp7(options2.timeoutMs ?? 5e3, 500, 3e4);
601964
602172
  if (options2.preferPackage !== false) {
601965
602173
  const result = await convertWithImageToAscii(imagePath, width, height, timeoutMs);
@@ -602166,6 +602374,176 @@ function describeCameraOrientation(orientation) {
602166
602374
  if (!orientation) return "upright/no correction (default)";
602167
602375
  return `${formatCameraRotation(orientation.rotation)} (${orientation.source}${orientation.confidence !== void 0 ? ` confidence=${orientation.confidence.toFixed(2)}` : ""}, updated=${nowIso3(orientation.updatedAt)})`;
602168
602376
  }
602377
+ function chooseCameraOrientationFromScores(scores) {
602378
+ const sorted = [...scores].filter((score) => Number.isFinite(score.score)).sort((a2, b) => b.score - a2.score);
602379
+ const best = sorted[0];
602380
+ if (!best || best.score <= 0 || best.faces <= 0) return null;
602381
+ const second3 = sorted[1];
602382
+ const gap = best.score - (second3?.score ?? 0);
602383
+ const total = sorted.reduce((sum, score) => sum + Math.max(0, score.score), 0);
602384
+ const share = total > 0 ? best.score / total : 1;
602385
+ const confidence2 = Math.max(0.55, Math.min(0.98, share * 0.75 + Math.min(0.23, gap / Math.max(1, best.score))));
602386
+ if (confidence2 < 0.58 && gap < 0.4) return null;
602387
+ return {
602388
+ rotation: best.rotation,
602389
+ confidence: confidence2,
602390
+ reason: `cv-face-orientation faces=${best.faces} score=${best.score.toFixed(2)} next=${(second3?.score ?? 0).toFixed(2)} area=${(best.faceAreaRatio ?? 0).toFixed(3)}`
602391
+ };
602392
+ }
602393
+ function trimOneLine(value2, maxChars) {
602394
+ return String(value2 ?? "").replace(/\r/g, " ").replace(/\n/g, " ").replace(/\s+/g, " ").trim().slice(0, maxChars);
602395
+ }
602396
+ function compactSourceId(source) {
602397
+ return source.replace(/^\/dev\//, "");
602398
+ }
602399
+ function truncateCell(value2, width) {
602400
+ value2 = stripDashboardAnsi(String(value2 ?? "")).replace(/\r?\n/g, " ");
602401
+ if (width <= 1) return value2.slice(0, Math.max(0, width));
602402
+ return value2.length <= width ? value2.padEnd(width, " ") : `${value2.slice(0, Math.max(1, width - 1))}…`;
602403
+ }
602404
+ function stripDashboardAnsi(value2) {
602405
+ return value2.replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, "");
602406
+ }
602407
+ function dashboardPreviewLines(camera) {
602408
+ const raw = camera.plainAscii || camera.ascii || "";
602409
+ return raw.replace(/\r/g, "").split("\n").map((line) => stripDashboardAnsi(line).replace(/\s+$/g, "")).filter((line) => line.trim().length > 0).slice(0, 10);
602410
+ }
602411
+ function centerDashboardCell(value2, width) {
602412
+ const text2 = stripDashboardAnsi(String(value2 ?? "")).replace(/\r?\n/g, " ");
602413
+ if (width <= 1) return text2.slice(0, Math.max(0, width));
602414
+ const clipped = text2.length > width ? `${text2.slice(0, Math.max(1, width - 1))}…` : text2;
602415
+ const left = Math.floor((width - clipped.length) / 2);
602416
+ return `${" ".repeat(Math.max(0, left))}${clipped}${" ".repeat(Math.max(0, width - clipped.length - left))}`;
602417
+ }
602418
+ function dashboardPreviewAspect(lines) {
602419
+ const widths = lines.map((line) => stripDashboardAnsi(line).length).filter((n2) => n2 > 0);
602420
+ const maxWidth = Math.max(1, ...widths);
602421
+ return Math.max(0.12, Math.min(1.6, lines.length / maxWidth));
602422
+ }
602423
+ function fitDashboardPane(lines, paneWidth, paneHeight) {
602424
+ const source = lines.length > 0 ? lines : ["preview pending"];
602425
+ const normalized = [];
602426
+ for (let i2 = 0; i2 < paneHeight; i2++) {
602427
+ const sourceIndex = source.length <= paneHeight ? i2 : Math.min(source.length - 1, Math.floor(i2 * source.length / paneHeight));
602428
+ normalized.push(truncateCell(source[sourceIndex] ?? "", paneWidth));
602429
+ }
602430
+ while (normalized.length < paneHeight) normalized.push(" ".repeat(paneWidth));
602431
+ return normalized;
602432
+ }
602433
+ function pushDashboardWrapped(lines, value2, width) {
602434
+ const contentWidth = Math.max(10, width - 2);
602435
+ const words = stripDashboardAnsi(value2).replace(/\s+/g, " ").trim().split(" ").filter(Boolean);
602436
+ if (words.length === 0) {
602437
+ lines.push(`│${" ".repeat(contentWidth)}│`);
602438
+ return;
602439
+ }
602440
+ let current = "";
602441
+ for (const word2 of words) {
602442
+ if (!current) {
602443
+ current = word2.length > contentWidth ? word2.slice(0, contentWidth) : word2;
602444
+ continue;
602445
+ }
602446
+ if (current.length + 1 + word2.length <= contentWidth) {
602447
+ current += ` ${word2}`;
602448
+ } else {
602449
+ lines.push(`│${truncateCell(` ${current}`, contentWidth)}│`);
602450
+ current = word2.length > contentWidth ? word2.slice(0, contentWidth) : word2;
602451
+ }
602452
+ }
602453
+ if (current) lines.push(`│${truncateCell(` ${current}`, contentWidth)}│`);
602454
+ }
602455
+ function isAudioFailureText(value2) {
602456
+ const text2 = String(value2 ?? "").trim();
602457
+ if (!text2) return false;
602458
+ return /^(?:ASR|VAD|Transcription)\s+(?:failed|unavailable)|No module named ['"]?transcribe_cli|torchcodec|torchaudio version/i.test(text2);
602459
+ }
602460
+ function summarizeInference(value2) {
602461
+ if (!value2) return "objects: pending";
602462
+ const lines = value2.split("\n").map((line) => line.trim()).filter(Boolean);
602463
+ const objectsIndex = lines.findIndex((line) => /^##\s+Objects/i.test(line));
602464
+ if (objectsIndex >= 0) {
602465
+ const objects = lines.slice(objectsIndex + 1).filter((line) => line.startsWith("- ")).slice(0, 4).map((line) => line.replace(/^-\s*/, ""));
602466
+ if (objects.length > 0) return `objects: ${objects.join("; ")}`;
602467
+ }
602468
+ const sampled = lines.find((line) => /^objects:/i.test(line));
602469
+ return sampled ? trimOneLine(sampled, 160) : trimOneLine(lines.slice(0, 4).join("; "), 160);
602470
+ }
602471
+ function cameraSnapshotSummary(camera) {
602472
+ if (camera.error) return `error: ${trimOneLine(camera.error, 160)}`;
602473
+ return camera.summary || summarizeInference(camera.inference);
602474
+ }
602475
+ function formatLiveDashboardFromSnapshot(snapshot, opts = {}) {
602476
+ const width = Math.max(60, opts.width ?? 100);
602477
+ const now2 = opts.now ?? Date.now();
602478
+ const maxCameras = opts.maxCameras ?? 4;
602479
+ const cameras = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {})).sort((a2, b) => {
602480
+ if (a2.source === snapshot.config.selectedCamera) return -1;
602481
+ if (b.source === snapshot.config.selectedCamera) return 1;
602482
+ return a2.source.localeCompare(b.source);
602483
+ }).slice(0, maxCameras);
602484
+ const audioAge = snapshot.audio?.updatedAt ? `${Math.max(0, Math.round((now2 - snapshot.audio.updatedAt) / 1e3))}s` : "none";
602485
+ const title = `live audio/video cameras=${cameras.length}/${snapshot.devices.video.length} audio=${snapshot.config.audioEnabled ? "on" : "off"} age=${audioAge}`;
602486
+ const horizontal = "─".repeat(Math.max(0, width - 2));
602487
+ const lines = [`╭${horizontal}╮`];
602488
+ lines.push(`│${truncateCell(` ${title}`, width - 2)}│`);
602489
+ if (cameras.length === 0) {
602490
+ lines.push(`│${truncateCell(" no camera frames captured yet", width - 2)}│`);
602491
+ }
602492
+ if (cameras.length > 0) {
602493
+ lines.push(`├${horizontal}┤`);
602494
+ const paneCount = Math.min(cameras.length, width >= 116 ? 3 : width >= 78 ? 2 : 1);
602495
+ const innerWidth = width - 2;
602496
+ const gutter = paneCount - 1;
602497
+ const paneWidth = Math.max(24, Math.floor((innerWidth - gutter) / paneCount));
602498
+ for (let start2 = 0; start2 < cameras.length; start2 += paneCount) {
602499
+ const group = cameras.slice(start2, start2 + paneCount);
602500
+ const sourceLines = group.map((camera) => dashboardPreviewLines(camera));
602501
+ const paneHeight = Math.max(7, Math.min(22, Math.round(paneWidth * Math.max(...sourceLines.map(dashboardPreviewAspect)))));
602502
+ const panes = sourceLines.map((previewLines) => fitDashboardPane(previewLines, paneWidth, paneHeight));
602503
+ for (let row = 0; row < paneHeight; row++) {
602504
+ const body = panes.map((pane) => pane[row] ?? " ".repeat(paneWidth)).join(" ");
602505
+ lines.push(`│${truncateCell(body, innerWidth)}│`);
602506
+ }
602507
+ const labels = group.map((camera) => {
602508
+ const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
602509
+ return centerDashboardCell(`${compactSourceId(camera.source)} age=${age}s`, paneWidth);
602510
+ }).join(" ");
602511
+ lines.push(`│${truncateCell(labels, innerWidth)}│`);
602512
+ }
602513
+ }
602514
+ if (cameras.length > 0) {
602515
+ lines.push(`├${horizontal}┤`);
602516
+ lines.push(`│${truncateCell(" camera observations", width - 2)}│`);
602517
+ }
602518
+ for (const camera of cameras) {
602519
+ const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
602520
+ const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
602521
+ const detailLines = [
602522
+ `${compactSourceId(camera.source)} age=${age}s rotation=${rotation}`,
602523
+ cameraSnapshotSummary(camera),
602524
+ camera.framePath ? `frame: ${camera.displayPath || camera.framePath}` : "",
602525
+ camera.orientation?.evidence ? `orientation: ${trimOneLine(camera.orientation.evidence, 120)}` : ""
602526
+ ].filter(Boolean);
602527
+ for (const detail of detailLines.length ? detailLines : ["preview pending"]) {
602528
+ pushDashboardWrapped(lines, detail, width);
602529
+ }
602530
+ }
602531
+ if (snapshot.audio) {
602532
+ lines.push(`├${horizontal}┤`);
602533
+ const transcriptFailure = isAudioFailureText(snapshot.audio.transcript) ? snapshot.audio.transcript : "";
602534
+ const transcript = transcriptFailure ? "" : snapshot.audio.transcript;
602535
+ const audioError = [snapshot.audio.error, transcriptFailure].filter(Boolean).join(" | ");
602536
+ const audioBits = [
602537
+ `audio input=${snapshot.audio.input || snapshot.config.selectedAudioInput || "none"}`,
602538
+ audioError ? `error=${trimOneLine(audioError, 120)}` : "",
602539
+ transcript ? `asr=${trimOneLine(transcript, 120)}` : "",
602540
+ snapshot.audio.analysis ? `sound=${trimOneLine(snapshot.audio.analysis, 120)}` : ""
602541
+ ].filter(Boolean);
602542
+ for (const item of audioBits.slice(0, 4)) lines.push(`│${truncateCell(` ${item}`, width - 2)}│`);
602543
+ }
602544
+ lines.push(`╰${horizontal}╯`);
602545
+ return lines;
602546
+ }
602169
602547
  function parseLiveMediaPayload(value2) {
602170
602548
  const parsed = parseJsonObject3(value2);
602171
602549
  if (!parsed) return null;
@@ -602425,6 +602803,21 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
602425
602803
  lines.push(`Selected camera: ${snapshot.config.selectedCamera}`);
602426
602804
  lines.push(`Camera orientation correction: ${describeCameraOrientation(orientation)}`);
602427
602805
  }
602806
+ const cameraSnapshots = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {})).sort((a2, b) => {
602807
+ if (a2.source === snapshot.config.selectedCamera) return -1;
602808
+ if (b.source === snapshot.config.selectedCamera) return 1;
602809
+ return b.updatedAt - a2.updatedAt;
602810
+ });
602811
+ if (cameraSnapshots.length > 0) {
602812
+ lines.push("Camera snapshot set:");
602813
+ for (const camera of cameraSnapshots.slice(0, 6)) {
602814
+ const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
602815
+ const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
602816
+ lines.push(`- ${camera.source} age=${age}s rotation=${rotation}${camera.framePath ? ` frame=${camera.framePath}` : ""}${camera.error ? ` error=${trimOneLine(camera.error, 240)}` : ""}`);
602817
+ const summary = cameraSnapshotSummary(camera);
602818
+ if (summary) lines.push(` summary: ${summary}`);
602819
+ }
602820
+ }
602428
602821
  if (snapshot.config.selectedAudioInput) lines.push(`Selected audio input: ${snapshot.config.selectedAudioInput}`);
602429
602822
  if (snapshot.config.selectedAudioOutput) lines.push(`Selected audio output: ${snapshot.config.selectedAudioOutput}`);
602430
602823
  const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "clip_recognition" || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 18);
@@ -602481,7 +602874,23 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
602481
602874
  lines.push(cleanPreview(snapshot.video.inference, 1800));
602482
602875
  }
602483
602876
  }
602877
+ if (cameraSnapshots.length > 1) {
602878
+ lines.push("");
602879
+ lines.push("## LIVE CAMERA LATEST FRAMES");
602880
+ for (const camera of cameraSnapshots.slice(0, 6)) {
602881
+ const age = now2 - camera.updatedAt;
602882
+ const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
602883
+ lines.push(`### ${camera.source}`);
602884
+ lines.push(`timestamp: ${nowIso3(camera.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s rotation=${rotation}${age > maxAgeMs ? " STALE" : ""}`);
602885
+ if (camera.error) lines.push(`error: ${camera.error}`);
602886
+ if (camera.framePath) lines.push(`frame: ${camera.framePath}`);
602887
+ lines.push(`summary: ${cameraSnapshotSummary(camera)}`);
602888
+ }
602889
+ }
602484
602890
  if (snapshot.audio) {
602891
+ const transcriptFailure = isAudioFailureText(snapshot.audio.transcript) ? snapshot.audio.transcript : "";
602892
+ const transcript = transcriptFailure ? "" : snapshot.audio.transcript;
602893
+ const audioError = [snapshot.audio.error, transcriptFailure].filter(Boolean).join(" | ");
602485
602894
  const audioEvents = (snapshot.events ?? []).filter((event) => event.kind === "audio_transcript" || event.kind === "audio_sound").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 8);
602486
602895
  if (audioEvents.length > 0) {
602487
602896
  lines.push("");
@@ -602498,11 +602907,11 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
602498
602907
  lines.push("");
602499
602908
  lines.push("## LIVE AUDIO LATEST SAMPLE");
602500
602909
  lines.push(`timestamp: ${nowIso3(snapshot.audio.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s${age > maxAgeMs ? " STALE" : ""}`);
602501
- if (snapshot.audio.error) lines.push(`error: ${snapshot.audio.error}`);
602910
+ if (audioError) lines.push(`error: ${audioError}`);
602502
602911
  if (snapshot.audio.recordingPath) lines.push(`recording: ${snapshot.audio.recordingPath}`);
602503
- if (snapshot.audio.transcript) {
602912
+ if (transcript) {
602504
602913
  lines.push("Live ASR:");
602505
- lines.push(cleanPreview(snapshot.audio.transcript, 1200));
602914
+ lines.push(cleanPreview(transcript, 1200));
602506
602915
  }
602507
602916
  if (snapshot.audio.analysis) {
602508
602917
  lines.push("Live sound analysis:");
@@ -602587,6 +602996,93 @@ async function recordAudioSample(device, durationSec, outputPath3) {
602587
602996
  outputPath3
602588
602997
  ], { timeout: (Number(seconds) + 10) * 1e3 });
602589
602998
  }
602999
+ async function scoreCameraOrientationWithOpenCv(framePath) {
603000
+ if (!await commandExists2("python3", 1200)) return null;
603001
+ const script = String.raw`
603002
+ import json, sys
603003
+ try:
603004
+ import cv2
603005
+ except Exception as exc:
603006
+ print(json.dumps({"ok": False, "error": str(exc)}))
603007
+ raise SystemExit(0)
603008
+
603009
+ path = sys.argv[1]
603010
+ img = cv2.imread(path)
603011
+ if img is None:
603012
+ print(json.dumps({"ok": False, "error": "could not read image"}))
603013
+ raise SystemExit(0)
603014
+
603015
+ face = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml")
603016
+ eye = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_eye.xml")
603017
+
603018
+ def rotate(mat, deg):
603019
+ if deg == 90:
603020
+ return cv2.rotate(mat, cv2.ROTATE_90_CLOCKWISE)
603021
+ if deg == 180:
603022
+ return cv2.rotate(mat, cv2.ROTATE_180)
603023
+ if deg == 270:
603024
+ return cv2.rotate(mat, cv2.ROTATE_90_COUNTERCLOCKWISE)
603025
+ return mat
603026
+
603027
+ scores = []
603028
+ for deg in [0, 90, 180, 270]:
603029
+ frame = rotate(img, deg)
603030
+ gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
603031
+ h, w = gray.shape[:2]
603032
+ faces = face.detectMultiScale(gray, scaleFactor=1.08, minNeighbors=4, minSize=(32, 32))
603033
+ face_area = 0.0
603034
+ eye_count = 0
603035
+ for (x, y, fw, fh) in faces[:8]:
603036
+ face_area += float(fw * fh) / max(1.0, float(w * h))
603037
+ roi = gray[y:y+fh, x:x+fw]
603038
+ try:
603039
+ eye_count += len(eye.detectMultiScale(roi, scaleFactor=1.1, minNeighbors=3, minSize=(8, 8)))
603040
+ except Exception:
603041
+ pass
603042
+ score = len(faces) * 2.5 + min(4.0, face_area * 35.0) + min(2.0, eye_count * 0.35)
603043
+ scores.append({"rotation": deg, "score": score, "faces": int(len(faces)), "faceAreaRatio": face_area})
603044
+ print(json.dumps({"ok": True, "scores": scores}))
603045
+ `;
603046
+ try {
603047
+ const raw = await execFileText4("python3", ["-c", script, framePath], { timeout: 15e3, maxBuffer: 1024 * 1024 });
603048
+ const parsed = parseJsonObjectFromText(raw);
603049
+ const scores = Array.isArray(parsed?.["scores"]) ? parsed["scores"] : [];
603050
+ if (scores.length === 0) return null;
603051
+ return scores.map((score) => ({
603052
+ rotation: normalizeLiveCameraRotation(score["rotation"]),
603053
+ score: Math.max(0, Number(score["score"] ?? 0)),
603054
+ faces: Math.max(0, Number(score["faces"] ?? 0)),
603055
+ faceAreaRatio: Number.isFinite(Number(score["faceAreaRatio"])) ? Number(score["faceAreaRatio"]) : void 0
603056
+ }));
603057
+ } catch {
603058
+ return null;
603059
+ }
603060
+ }
603061
+ async function tryRecordAudioDevice(device, durationSec, outputPath3) {
603062
+ await recordAudioSample(device, durationSec, outputPath3);
603063
+ }
603064
+ async function recordAudioSampleWithRecovery(preferred, devices, durationSec, outputPath3) {
603065
+ const ordered = [];
603066
+ const add3 = (id2) => {
603067
+ const clean5 = String(id2 ?? "").trim();
603068
+ if (clean5 && !ordered.includes(clean5)) ordered.push(clean5);
603069
+ };
603070
+ add3(preferred);
603071
+ for (const device of devices.filter((entry) => entry.source === "pulse" || entry.source === "pipewire")) add3(device.id);
603072
+ add3("pulse:default");
603073
+ add3("default");
603074
+ for (const device of devices.filter((entry) => entry.source !== "pulse" && entry.source !== "pipewire")) add3(device.id);
603075
+ const errors = [];
603076
+ for (const candidate of ordered) {
603077
+ try {
603078
+ await tryRecordAudioDevice(candidate, durationSec, outputPath3);
603079
+ return { ok: true, device: candidate, method: candidate.startsWith("pulse:") ? "pulse/ffmpeg" : "arecord", errors };
603080
+ } catch (err) {
603081
+ errors.push(`${candidate}: ${err instanceof Error ? err.message : String(err)}`.slice(0, 360));
603082
+ }
603083
+ }
603084
+ return { ok: false, device: preferred || "default", method: "none", errors };
603085
+ }
602590
603086
  function firstDeviceId(devices) {
602591
603087
  return devices.find((device) => device.id && !/metadata\/non-capture/i.test(device.detail ?? ""))?.id ?? devices[0]?.id;
602592
603088
  }
@@ -602677,6 +603173,7 @@ var init_live_sensors = __esm({
602677
603173
  agentActionEnabled = false;
602678
603174
  lastAgentReviewAt = 0;
602679
603175
  autoOrientationInFlight = /* @__PURE__ */ new Set();
603176
+ lastFeedbackAt = /* @__PURE__ */ new Map();
602680
603177
  setStatusSink(sink) {
602681
603178
  this.statusSink = sink ?? null;
602682
603179
  this.emitStatus();
@@ -602693,6 +603190,20 @@ var init_live_sensors = __esm({
602693
603190
  getSnapshot() {
602694
603191
  return this.snapshot;
602695
603192
  }
603193
+ activeVideoSources() {
603194
+ const ordered = [];
603195
+ const add3 = (id2) => {
603196
+ const clean5 = String(id2 ?? "").trim();
603197
+ if (!clean5 || ordered.includes(clean5)) return;
603198
+ const device = this.devices.video.find((entry) => entry.id === clean5);
603199
+ if (device && /metadata\/non-capture/i.test(device.detail ?? "")) return;
603200
+ ordered.push(clean5);
603201
+ };
603202
+ add3(this.config.selectedCamera);
603203
+ for (const device of this.devices.video) add3(device.id);
603204
+ if (ordered.length === 0) add3(firstDeviceId(this.devices.video));
603205
+ return ordered.slice(0, 4);
603206
+ }
602696
603207
  getCameraOrientation(device = this.config.selectedCamera) {
602697
603208
  return device ? this.config.cameraOrientation?.[device] : void 0;
602698
603209
  }
@@ -602763,6 +603274,12 @@ var init_live_sensors = __esm({
602763
603274
  }
602764
603275
  const framePath = extractSavedImagePath(captured.output, this.repoRoot);
602765
603276
  if (!framePath) return `Camera orientation detection captured ${target}, but no frame path was returned.`;
603277
+ const cvScores = await scoreCameraOrientationWithOpenCv(framePath);
603278
+ const cvDecision = cvScores ? chooseCameraOrientationFromScores(cvScores) : null;
603279
+ if (cvDecision && cvDecision.confidence >= 0.68) {
603280
+ const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
603281
+ return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
603282
+ }
602766
603283
  const prompt = [
602767
603284
  "Determine whether this camera frame is upright for a human viewer.",
602768
603285
  "Return only JSON with this schema:",
@@ -602777,14 +603294,24 @@ var init_live_sensors = __esm({
602777
603294
  model: "moondream3-preview"
602778
603295
  });
602779
603296
  if (!vision.success) {
603297
+ if (cvDecision) {
603298
+ const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
603299
+ return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
603300
+ }
602780
603301
  return `Camera orientation detection captured ${target}, but vision inference was unavailable: ${vision.error || vision.output || "unknown error"}`;
602781
603302
  }
602782
603303
  const decision2 = parseCameraOrientationDecision(vision.llmContent || vision.output);
602783
603304
  if (!decision2) {
603305
+ if (cvDecision) {
603306
+ const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
603307
+ return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
603308
+ }
602784
603309
  return `Camera orientation detection for ${target} was inconclusive. Vision output did not contain a structured rotation decision.`;
602785
603310
  }
602786
- const evidence = decision2.reason || boundedText(vision.llmContent || vision.output, 400);
602787
- const orientation = this.setCameraRotation(target, decision2.rotation, "auto", evidence, decision2.confidence);
603311
+ const useCv = cvDecision && (cvDecision.confidence > (decision2.confidence ?? 0) + 0.12 || cvDecision.rotation !== decision2.rotation && cvDecision.confidence >= 0.62 && (decision2.confidence ?? 0) < 0.76);
603312
+ const selected = useCv ? cvDecision : decision2;
603313
+ const evidence = useCv ? cvDecision.reason : `${decision2.reason || boundedText(vision.llmContent || vision.output, 320)}${cvDecision ? `; cv=${cvDecision.reason}` : ""}`;
603314
+ const orientation = this.setCameraRotation(target, selected.rotation, "auto", evidence, selected.confidence);
602788
603315
  return orientation ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation)}` : `Auto orientation for ${target} could not be persisted.`;
602789
603316
  } finally {
602790
603317
  this.autoOrientationInFlight.delete(target);
@@ -602853,22 +603380,25 @@ var init_live_sensors = __esm({
602853
603380
  videoIntervalMs: options2.lowLatency === false ? this.config.videoIntervalMs : Math.min(this.config.videoIntervalMs, 1500),
602854
603381
  audioIntervalMs: options2.lowLatency === false ? this.config.audioIntervalMs : Math.min(this.config.audioIntervalMs, 3e3)
602855
603382
  });
602856
- const source = this.config.selectedCamera || firstDeviceId(this.devices.video);
602857
- if (source && !this.getCameraOrientation(source)) {
602858
- void this.autoOrientCamera(source, { onlyIfUnset: true }).then((message2) => {
602859
- this.emitFeedback({
603383
+ for (const source of this.activeVideoSources()) {
603384
+ const current = this.getCameraOrientation(source);
603385
+ if (current?.source === "manual") continue;
603386
+ if (current && current.confidence !== void 0 && current.confidence >= 0.75 && Date.now() - current.updatedAt < 15 * 6e4) continue;
603387
+ void this.autoOrientCamera(source).then((message2) => {
603388
+ this.emitFeedbackThrottled(`orient:${source}`, {
602860
603389
  kind: "orientation",
602861
603390
  title: "Camera auto-orientation",
602862
603391
  observedAt: Date.now(),
602863
603392
  message: message2
602864
- });
603393
+ }, 3e4);
603394
+ this.emitDashboard();
602865
603395
  }).catch((err) => {
602866
- this.emitFeedback({
603396
+ this.emitFeedbackThrottled(`orient-error:${source}`, {
602867
603397
  kind: "error",
602868
603398
  title: "Camera auto-orientation failed",
602869
603399
  observedAt: Date.now(),
602870
603400
  message: err instanceof Error ? err.message : String(err)
602871
- });
603401
+ }, 6e4);
602872
603402
  });
602873
603403
  }
602874
603404
  void this.sampleVideoNow(true).catch((err) => {
@@ -602895,7 +603425,7 @@ var init_live_sensors = __esm({
602895
603425
  });
602896
603426
  return options2.agent ? "Live PFC run enabled: visual/audio feedback is active and significant events can launch background agent review." : "Live run enabled: visual/audio feedback is active and live context will update each turn.";
602897
603427
  }
602898
- async previewCamera(device = this.config.selectedCamera) {
603428
+ async previewCamera(device = this.config.selectedCamera, options2 = {}) {
602899
603429
  const source = device || firstDeviceId(this.devices.video);
602900
603430
  if (!source) return { ok: false, message: "No camera selected. Use /live to select a camera after device discovery." };
602901
603431
  const orientation = this.getCameraOrientation(source);
@@ -602910,10 +603440,22 @@ var init_live_sensors = __esm({
602910
603440
  if (!result.success) {
602911
603441
  const error = result.error || result.output || "camera capture failed";
602912
603442
  const observedAt2 = Date.now();
602913
- this.snapshot.video = {
603443
+ const errorView = {
602914
603444
  updatedAt: observedAt2,
602915
603445
  source,
602916
- error
603446
+ error,
603447
+ rotation,
603448
+ orientation
603449
+ };
603450
+ if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = errorView;
603451
+ this.snapshot.cameras = {
603452
+ ...this.snapshot.cameras ?? {},
603453
+ [source]: {
603454
+ ...this.snapshot.cameras?.[source] ?? { source, updatedAt: observedAt2 },
603455
+ ...errorView,
603456
+ updatedAt: observedAt2,
603457
+ source
603458
+ }
602917
603459
  };
602918
603460
  this.snapshot.events = mergeRecentById(this.snapshot.events, [{
602919
603461
  id: stableEventId("camera_error", source, observedAt2, error.slice(0, 80)),
@@ -602924,29 +603466,44 @@ var init_live_sensors = __esm({
602924
603466
  summary: error
602925
603467
  }], 50);
602926
603468
  this.persist();
602927
- this.emitFeedback({
602928
- kind: "error",
602929
- title: "Camera capture failed",
602930
- observedAt: observedAt2,
602931
- message: error
602932
- });
603469
+ if (options2.emit !== false) {
603470
+ this.emitFeedbackThrottled(`camera-capture:${source}:${error.slice(0, 80)}`, {
603471
+ kind: "error",
603472
+ title: "Camera capture failed",
603473
+ observedAt: observedAt2,
603474
+ message: error
603475
+ }, 3e4);
603476
+ }
602933
603477
  return { ok: false, message: error };
602934
603478
  }
602935
603479
  const framePath = extractSavedImagePath(result.output, this.repoRoot);
602936
603480
  if (!framePath) return { ok: false, message: "Camera capture succeeded but no saved image path was returned." };
602937
603481
  const displayPath = relative11(this.repoRoot, framePath).startsWith("..") ? framePath : relative11(this.repoRoot, framePath);
602938
- const preview = await buildImageAsciiPreview(framePath, { width: 72, height: 22 });
603482
+ const previewWidth = Math.max(48, Math.min(96, (process.stdout.columns || 100) - 14));
603483
+ const preview = await buildImageAsciiPreview(framePath, { width: previewWidth });
602939
603484
  const asciiContext = preview ? formatImageAsciiContext(preview, displayPath) : void 0;
602940
603485
  const observedAt = Date.now();
602941
- this.snapshot.video = {
603486
+ const cameraView = {
602942
603487
  updatedAt: observedAt,
602943
603488
  source,
602944
603489
  framePath,
602945
603490
  displayPath,
603491
+ ascii: preview?.ascii,
603492
+ plainAscii: preview?.plainAscii,
603493
+ renderer: preview?.renderer,
602946
603494
  asciiContext,
602947
603495
  rotation,
602948
603496
  orientation
602949
603497
  };
603498
+ if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = cameraView;
603499
+ this.snapshot.cameras = {
603500
+ ...this.snapshot.cameras ?? {},
603501
+ [source]: {
603502
+ ...cameraView,
603503
+ summary: this.snapshot.cameras?.[source]?.summary,
603504
+ inference: this.snapshot.cameras?.[source]?.inference
603505
+ }
603506
+ };
602950
603507
  this.snapshot.events = mergeRecentById(this.snapshot.events, [{
602951
603508
  id: stableEventId("camera_frame", source, observedAt, framePath),
602952
603509
  kind: "camera_frame",
@@ -602957,16 +603514,18 @@ var init_live_sensors = __esm({
602957
603514
  evidencePath: framePath
602958
603515
  }], 50);
602959
603516
  this.persist();
602960
- this.emitFeedback({
602961
- kind: "camera-frame",
602962
- title: "Live camera frame",
602963
- observedAt,
602964
- message: `Captured from ${source}${orientation ? ` (${formatCameraRotation(orientation.rotation)})` : ""}: ${framePath}`,
602965
- imagePath: framePath,
602966
- displayPath,
602967
- ascii: preview?.ascii,
602968
- renderer: preview?.renderer
602969
- });
603517
+ if (options2.emit !== false) {
603518
+ this.emitFeedback({
603519
+ kind: "camera-frame",
603520
+ title: "Live camera frame",
603521
+ observedAt,
603522
+ message: `Captured from ${source}${orientation ? ` (${formatCameraRotation(orientation.rotation)})` : ""}: ${framePath}`,
603523
+ imagePath: framePath,
603524
+ displayPath,
603525
+ ascii: preview?.ascii,
603526
+ renderer: preview?.renderer
603527
+ });
603528
+ }
602970
603529
  return {
602971
603530
  ok: true,
602972
603531
  message: `Camera frame captured from ${source}: ${framePath}`,
@@ -602976,7 +603535,7 @@ var init_live_sensors = __esm({
602976
603535
  renderer: preview?.renderer
602977
603536
  };
602978
603537
  }
602979
- async recognizeFrameObjects(framePath, source, observedAt = Date.now()) {
603538
+ async recognizeFrameObjects(framePath, source, observedAt = Date.now(), options2 = {}) {
602980
603539
  try {
602981
603540
  const result = await new VisualMemoryTool().execute({
602982
603541
  action: "recognize",
@@ -602996,109 +603555,124 @@ var init_live_sensors = __esm({
602996
603555
  evidencePath: framePath
602997
603556
  }], 50);
602998
603557
  this.persist();
602999
- this.emitFeedback({
603000
- kind: "clip",
603001
- title: result.success ? "CLIP visual-memory recognition" : "CLIP visual-memory unavailable",
603002
- observedAt,
603003
- message: summary
603004
- });
603558
+ if (options2.emit !== false) {
603559
+ this.emitFeedback({
603560
+ kind: "clip",
603561
+ title: result.success ? "CLIP visual-memory recognition" : "CLIP visual-memory unavailable",
603562
+ observedAt,
603563
+ message: summary
603564
+ });
603565
+ }
603005
603566
  return summary;
603006
603567
  } catch (err) {
603007
603568
  const message2 = `CLIP/visual-memory failed: ${err instanceof Error ? err.message : String(err)}`;
603008
- this.emitFeedback({
603009
- kind: "error",
603010
- title: "CLIP visual-memory failed",
603011
- observedAt,
603012
- message: message2
603013
- });
603569
+ if (options2.emit !== false) {
603570
+ this.emitFeedbackThrottled(`clip:${source}`, {
603571
+ kind: "error",
603572
+ title: "CLIP visual-memory failed",
603573
+ observedAt,
603574
+ message: message2
603575
+ }, 6e4);
603576
+ }
603014
603577
  return message2;
603015
603578
  }
603016
603579
  }
603017
- async sampleVideoNow(forceInfer = this.config.inferEnabled) {
603018
- if (this.videoInFlight) return "Video sampler is already running.";
603019
- this.videoInFlight = true;
603020
- try {
603021
- const preview = await this.previewCamera(this.config.selectedCamera);
603022
- let inference = "";
603023
- let clipSummary = "";
603024
- const source = this.config.selectedCamera || firstDeviceId(this.devices.video);
603025
- const orientation = source ? this.getCameraOrientation(source) : void 0;
603026
- const sampledAt = Date.now();
603027
- if (forceInfer && source) {
603028
- const loop = new LiveMediaLoopTool(this.repoRoot);
603029
- const result = await loop.execute({
603030
- action: "watch",
603031
- source_kind: "camera",
603032
- camera: source,
603033
- duration_sec: 4,
603034
- sample_interval_sec: 1,
603035
- max_frames: 4,
603036
- rotate_degrees: orientation?.rotation ?? 0,
603037
- auto_bootstrap: true,
603038
- audio_mode: "none",
603039
- detect_objects: true,
603040
- track_objects: true,
603041
- crop_faces: true,
603042
- recognize_faces: true
603043
- });
603044
- inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
603045
- const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
603046
- const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
603047
- this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
603048
- this.snapshot.people = mergeRecentById(this.snapshot.people, observations.people, 24);
603049
- this.snapshot.video = {
603050
- ...this.snapshot.video ?? { updatedAt: Date.now(), source },
603051
- updatedAt: sampledAt,
603052
- source,
603053
- inference,
603054
- rotation: orientation?.rotation ?? 0,
603055
- orientation,
603056
- ...preview.ok ? {} : { error: preview.message }
603057
- };
603058
- this.persist();
603059
- this.emitFeedback({
603060
- kind: result.success ? "video-inference" : "error",
603061
- title: result.success ? "Live video inference" : "Live video inference failed",
603580
+ async sampleSingleVideoNow(source, forceInfer) {
603581
+ const preview = await this.previewCamera(source, { emit: false });
603582
+ let inference = "";
603583
+ let clipSummary = "";
603584
+ const orientation = this.getCameraOrientation(source);
603585
+ const sampledAt = Date.now();
603586
+ if (forceInfer && source) {
603587
+ const loop = new LiveMediaLoopTool(this.repoRoot);
603588
+ const result = await loop.execute({
603589
+ action: "watch",
603590
+ source_kind: "camera",
603591
+ camera: source,
603592
+ duration_sec: 2.5,
603593
+ sample_interval_sec: 1,
603594
+ max_frames: 2,
603595
+ rotate_degrees: orientation?.rotation ?? 0,
603596
+ auto_bootstrap: true,
603597
+ audio_mode: "none",
603598
+ detect_objects: true,
603599
+ track_objects: true,
603600
+ crop_faces: true,
603601
+ recognize_faces: true
603602
+ });
603603
+ inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
603604
+ const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
603605
+ const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
603606
+ this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
603607
+ this.snapshot.people = mergeRecentById(this.snapshot.people, observations.people, 24);
603608
+ const existing = this.snapshot.cameras?.[source] ?? this.snapshot.video ?? { updatedAt: Date.now(), source };
603609
+ const camera = {
603610
+ ...existing,
603611
+ updatedAt: sampledAt,
603612
+ source,
603613
+ inference,
603614
+ summary: summarizeInference(inference),
603615
+ rotation: orientation?.rotation ?? 0,
603616
+ orientation,
603617
+ ...preview.ok ? { error: void 0 } : { error: preview.message }
603618
+ };
603619
+ this.snapshot.cameras = {
603620
+ ...this.snapshot.cameras ?? {},
603621
+ [source]: camera
603622
+ };
603623
+ if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = camera;
603624
+ this.persist();
603625
+ if (!result.success) {
603626
+ this.emitFeedbackThrottled(`video-infer:${source}`, {
603627
+ kind: "error",
603628
+ title: "Live video inference failed",
603062
603629
  observedAt: sampledAt,
603063
- message: boundedText(inference, 1800)
603064
- });
603065
- for (const person of observations.people.filter((entry) => entry.cropPath).slice(0, 4)) {
603066
- const cropPath = person.cropPath;
603067
- const displayPath = relative11(this.repoRoot, cropPath).startsWith("..") ? cropPath : relative11(this.repoRoot, cropPath);
603068
- const facePreview = await buildImageAsciiPreview(cropPath, { width: 38, height: 14 });
603069
- this.emitFeedback({
603070
- kind: "face-crop",
603071
- title: person.status === "known" && person.name ? `Known face: ${person.name}` : "Unknown face awaiting identification",
603072
- observedAt: person.observedAt,
603073
- message: person.status === "known" && person.name ? `Identified ${person.name}${person.confidence !== void 0 ? ` confidence=${person.confidence.toFixed(2)}` : ""}` : `Face crop saved and awaiting identification${person.confidence !== void 0 ? ` confidence=${person.confidence.toFixed(2)}` : ""}`,
603074
- imagePath: cropPath,
603075
- displayPath,
603076
- ascii: facePreview?.ascii,
603077
- renderer: facePreview?.renderer
603078
- });
603079
- }
603080
- const unknownPeople = observations.people.filter((entry) => entry.status === "unknown");
603081
- if (unknownPeople.length > 0) {
603082
- this.maybeRequestAgentReview(
603083
- `${unknownPeople.length} unknown individual(s) observed on live camera`,
603084
- unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; ")
603085
- );
603086
- }
603087
- const triggerEvents = observations.events.filter((entry) => entry.kind === "visual_trigger");
603088
- if (triggerEvents.length > 0) {
603089
- this.maybeRequestAgentReview(
603090
- "Visual trigger hit in live camera stream",
603091
- triggerEvents.map((entry) => entry.summary).join("; ")
603092
- );
603093
- }
603630
+ message: boundedText(inference, 1200)
603631
+ }, 45e3);
603632
+ }
603633
+ const unknownPeople = observations.people.filter((entry) => entry.status === "unknown");
603634
+ if (unknownPeople.length > 0) {
603635
+ this.maybeRequestAgentReview(
603636
+ `${unknownPeople.length} unknown individual(s) observed on live camera ${source}`,
603637
+ unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; ")
603638
+ );
603639
+ }
603640
+ const triggerEvents = observations.events.filter((entry) => entry.kind === "visual_trigger");
603641
+ if (triggerEvents.length > 0) {
603642
+ this.maybeRequestAgentReview(
603643
+ `Visual trigger hit in live camera stream ${source}`,
603644
+ triggerEvents.map((entry) => entry.summary).join("; ")
603645
+ );
603094
603646
  }
603095
- if (this.config.clipEnabled && preview.ok && preview.framePath && source) {
603096
- clipSummary = await this.recognizeFrameObjects(preview.framePath, source, sampledAt);
603647
+ }
603648
+ if (this.config.clipEnabled && preview.ok && preview.framePath && source) {
603649
+ clipSummary = await this.recognizeFrameObjects(preview.framePath, source, sampledAt, { emit: false });
603650
+ const current = this.snapshot.cameras?.[source];
603651
+ if (current) {
603652
+ current.summary = [current.summary, clipSummary ? `clip: ${trimOneLine(clipSummary, 160)}` : ""].filter(Boolean).join(" | ");
603653
+ this.snapshot.cameras = { ...this.snapshot.cameras ?? {}, [source]: current };
603654
+ if (this.snapshot.video?.source === source) this.snapshot.video = current;
603655
+ this.persist();
603097
603656
  }
603098
- return [preview.message, inference ? `
603657
+ }
603658
+ return [preview.message, inference ? `
603099
603659
  ${inference}` : "", clipSummary ? `
603100
603660
  CLIP visual memory:
603101
603661
  ${clipSummary}` : ""].filter(Boolean).join("\n");
603662
+ }
603663
+ async sampleVideoNow(forceInfer = this.config.inferEnabled) {
603664
+ if (this.videoInFlight) return "Video sampler is already running.";
603665
+ this.videoInFlight = true;
603666
+ try {
603667
+ const sources = this.activeVideoSources();
603668
+ if (sources.length === 0) return "No camera selected. Use /live refresh after connecting hardware.";
603669
+ const outputs = [];
603670
+ for (const source of sources) {
603671
+ outputs.push(await this.sampleSingleVideoNow(source, forceInfer));
603672
+ }
603673
+ this.emitDashboard();
603674
+ return outputs.map((output, index) => `## Camera ${sources[index]}
603675
+ ${output}`).join("\n\n");
603102
603676
  } finally {
603103
603677
  this.videoInFlight = false;
603104
603678
  }
@@ -603113,10 +603687,10 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
603113
603687
  const recordingPath = join124(outputDir2, `audio-${Date.now()}.wav`);
603114
603688
  let transcript = "";
603115
603689
  let analysis = "";
603116
- try {
603117
- await recordAudioSample(input, 3, recordingPath);
603118
- } catch (err) {
603119
- const message2 = `Audio capture failed from ${input}: ${err instanceof Error ? err.message : String(err)}`;
603690
+ const audioErrors = [];
603691
+ const capture = await recordAudioSampleWithRecovery(input, this.devices.audioInputs, 3, recordingPath);
603692
+ if (!capture.ok) {
603693
+ const message2 = `Audio capture failed from ${input}; tried fallbacks: ${capture.errors.slice(0, 5).join(" | ")}`;
603120
603694
  this.snapshot.audio = {
603121
603695
  updatedAt: Date.now(),
603122
603696
  input,
@@ -603124,20 +603698,28 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
603124
603698
  error: message2
603125
603699
  };
603126
603700
  this.persist();
603127
- this.emitFeedback({
603701
+ this.emitFeedbackThrottled(`audio-capture:${input}`, {
603128
603702
  kind: "error",
603129
603703
  title: "Live audio capture failed",
603130
603704
  observedAt: this.snapshot.audio.updatedAt,
603131
603705
  message: message2
603132
- });
603706
+ }, 6e4);
603707
+ this.emitDashboard();
603133
603708
  return message2;
603134
603709
  }
603710
+ if (capture.device !== this.config.selectedAudioInput) {
603711
+ this.config.selectedAudioInput = capture.device;
603712
+ }
603135
603713
  if (this.config.asrEnabled) {
603136
603714
  try {
603137
603715
  const asr = await new TranscribeFileTool(this.repoRoot).execute({ path: recordingPath, model: "tiny" });
603138
- transcript = asr.success ? cleanPreview(asr.llmContent || asr.output, 1600) : `ASR failed: ${asr.error || asr.output || "unknown error"}`;
603716
+ if (asr.success) {
603717
+ transcript = cleanPreview(typeof asr.llmContent === "string" ? asr.llmContent : "", 1600);
603718
+ } else {
603719
+ audioErrors.push(`ASR unavailable: ${asr.error || asr.output || "unknown error"}`);
603720
+ }
603139
603721
  } catch (err) {
603140
- transcript = `ASR failed: ${err instanceof Error ? err.message : String(err)}`;
603722
+ audioErrors.push(`ASR unavailable: ${err instanceof Error ? err.message : String(err)}`);
603141
603723
  }
603142
603724
  }
603143
603725
  if (this.config.audioAnalysisEnabled) {
@@ -603151,27 +603733,29 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
603151
603733
  }
603152
603734
  try {
603153
603735
  const vad = await analyzer.execute({ action: "vad", file: recordingPath });
603154
- parts.push(vad.success ? cleanPreview(vad.output, 700) : `vad failed: ${vad.error}`);
603736
+ if (vad.success) parts.push(cleanPreview(vad.output, 700));
603737
+ else audioErrors.push(`VAD unavailable: ${vad.error || vad.output || "unknown error"}`);
603155
603738
  } catch (err) {
603156
- parts.push(`vad failed: ${err instanceof Error ? err.message : String(err)}`);
603739
+ audioErrors.push(`VAD unavailable: ${err instanceof Error ? err.message : String(err)}`);
603157
603740
  }
603158
603741
  analysis = parts.filter(Boolean).join("\n");
603159
603742
  }
603160
603743
  this.snapshot.audio = {
603161
603744
  updatedAt: Date.now(),
603162
- input,
603745
+ input: capture.device,
603163
603746
  output: this.config.selectedAudioOutput,
603164
603747
  recordingPath,
603165
603748
  transcript,
603166
- analysis
603749
+ analysis,
603750
+ error: audioErrors.length > 0 ? audioErrors.join("\n") : void 0
603167
603751
  };
603168
603752
  const audioEvents = [];
603169
603753
  if (transcript) {
603170
603754
  audioEvents.push({
603171
- id: stableEventId("audio_transcript", input, this.snapshot.audio.updatedAt, recordingPath),
603755
+ id: stableEventId("audio_transcript", capture.device, this.snapshot.audio.updatedAt, recordingPath),
603172
603756
  kind: "audio_transcript",
603173
603757
  observedAt: this.snapshot.audio.updatedAt,
603174
- source: input,
603758
+ source: capture.device,
603175
603759
  title: "Live ASR transcript",
603176
603760
  summary: transcript,
603177
603761
  evidencePath: recordingPath
@@ -603179,10 +603763,10 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
603179
603763
  }
603180
603764
  if (analysis) {
603181
603765
  audioEvents.push({
603182
- id: stableEventId("audio_sound", input, this.snapshot.audio.updatedAt, recordingPath),
603766
+ id: stableEventId("audio_sound", capture.device, this.snapshot.audio.updatedAt, recordingPath),
603183
603767
  kind: "audio_sound",
603184
603768
  observedAt: this.snapshot.audio.updatedAt,
603185
- source: input,
603769
+ source: capture.device,
603186
603770
  title: "Live sound analysis",
603187
603771
  summary: analysis,
603188
603772
  evidencePath: recordingPath
@@ -603190,21 +603774,9 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
603190
603774
  }
603191
603775
  this.snapshot.events = mergeRecentById(this.snapshot.events, audioEvents, 50);
603192
603776
  this.persist();
603193
- if (transcript || analysis) {
603194
- this.emitFeedback({
603195
- kind: "audio",
603196
- title: "Live audio sample",
603197
- observedAt: this.snapshot.audio.updatedAt,
603198
- message: [
603199
- transcript ? `ASR:
603200
- ${boundedText(transcript, 700)}` : "",
603201
- analysis ? `Sounds:
603202
- ${boundedText(analysis, 900)}` : ""
603203
- ].filter(Boolean).join("\n\n")
603204
- });
603205
- }
603777
+ this.emitDashboard();
603206
603778
  return [
603207
- `Audio sample captured from ${input}: ${recordingPath}`,
603779
+ `Audio sample captured from ${capture.device}${capture.device !== input ? ` (recovered from ${input})` : ""}: ${recordingPath}`,
603208
603780
  transcript ? `
603209
603781
  ASR:
603210
603782
  ${transcript}` : "",
@@ -603256,6 +603828,21 @@ ${analysis}` : ""
603256
603828
  } catch {
603257
603829
  }
603258
603830
  }
603831
+ emitFeedbackThrottled(key, feedback, minIntervalMs) {
603832
+ const now2 = Date.now();
603833
+ const last2 = this.lastFeedbackAt.get(key) ?? 0;
603834
+ if (now2 - last2 < minIntervalMs) return;
603835
+ this.lastFeedbackAt.set(key, now2);
603836
+ this.emitFeedback(feedback);
603837
+ }
603838
+ emitDashboard() {
603839
+ this.emitFeedback({
603840
+ kind: "dashboard",
603841
+ title: "Live audio/video monitor",
603842
+ observedAt: Date.now(),
603843
+ message: "live dashboard refresh"
603844
+ });
603845
+ }
603259
603846
  maybeRequestAgentReview(reason, evidence) {
603260
603847
  if (!this.agentActionEnabled || !this.agentActionSink) return;
603261
603848
  const now2 = Date.now();
@@ -654970,14 +655557,45 @@ function renderLivePreview(preview) {
654970
655557
  renderWarning("Camera preview was captured, but ASCII rendering was unavailable.");
654971
655558
  }
654972
655559
  }
654973
- function renderLiveFeedback(feedback) {
655560
+ function renderLiveDashboard(ctx3, manager) {
655561
+ const host = ctx3.liveDynamicBlockHost;
655562
+ if (!host) {
655563
+ renderInfo(formatLiveDashboardFromSnapshot(manager.getSnapshot(), { width: 100 }).join("\n"));
655564
+ return;
655565
+ }
655566
+ const key = ctx3.repoRoot;
655567
+ let state = liveDashboardBlocks.get(key);
655568
+ if (!state || state.host !== host) {
655569
+ const id2 = `live-dashboard-${Math.random().toString(36).slice(2, 9)}`;
655570
+ state = { host, id: id2, manager };
655571
+ liveDashboardBlocks.set(key, state);
655572
+ host.registerDynamicBlock(
655573
+ id2,
655574
+ (width) => formatLiveDashboardFromSnapshot(manager.getSnapshot(), { width })
655575
+ );
655576
+ host.appendDynamicBlock(id2);
655577
+ return;
655578
+ }
655579
+ state.manager = manager;
655580
+ host.refreshDynamicBlocks?.();
655581
+ }
655582
+ function renderLiveFeedback(ctx3, manager, feedback) {
655583
+ if (feedback.kind === "dashboard") {
655584
+ renderLiveDashboard(ctx3, manager);
655585
+ return;
655586
+ }
654974
655587
  const stamp = new Date(feedback.observedAt).toISOString();
655588
+ const explicitImagePreview = (feedback.kind === "camera-frame" || feedback.kind === "face-crop") && feedback.ascii && feedback.displayPath && feedback.renderer;
655589
+ if (ctx3.liveDynamicBlockHost && !explicitImagePreview) {
655590
+ renderLiveDashboard(ctx3, manager);
655591
+ return;
655592
+ }
654975
655593
  if (feedback.kind === "error") {
654976
655594
  renderWarning(`[live ${stamp}] ${feedback.title}
654977
655595
  ${feedback.message}`);
654978
655596
  return;
654979
655597
  }
654980
- if ((feedback.kind === "camera-frame" || feedback.kind === "face-crop") && feedback.ascii && feedback.displayPath && feedback.renderer) {
655598
+ if (explicitImagePreview && feedback.displayPath && feedback.ascii && feedback.renderer) {
654981
655599
  renderInfo(`[live ${stamp}] ${feedback.message}`);
654982
655600
  renderImageAsciiPreview(feedback.title, feedback.displayPath, feedback.ascii, feedback.renderer);
654983
655601
  return;
@@ -654987,7 +655605,7 @@ ${feedback.message}`);
654987
655605
  }
654988
655606
  function attachLiveSinks(ctx3, manager, agentEnabled = false) {
654989
655607
  manager.setStatusSink(ctx3.setLiveMediaStatus);
654990
- manager.setFeedbackSink((feedback) => renderLiveFeedback(feedback));
655608
+ manager.setFeedbackSink((feedback) => renderLiveFeedback(ctx3, manager, feedback));
654991
655609
  if (agentEnabled && ctx3.startBackgroundPrompt) {
654992
655610
  manager.setAgentActionSink((prompt) => {
654993
655611
  const id2 = ctx3.startBackgroundPrompt?.(prompt);
@@ -660485,7 +661103,7 @@ async function showExposeDashboard(gateway, rl, ctx3) {
660485
661103
  renderInfo("Expose gateway stopped.");
660486
661104
  }
660487
661105
  }
660488
- var OMNIUS_PINNED_DEPENDENCY_SPECS, NEXUS_DIRECTORY_ORIGIN, NEXUS_SPONSORS_URL, _sponsorHeartbeatTimer, _lastRegisteredSponsorPayload, __COMMAND_REGISTRY, DASH_INTERNAL, localGpuMetricsCache, localGpuMetricsProbeAt, localGpuMetricsProbeInFlight;
661106
+ var OMNIUS_PINNED_DEPENDENCY_SPECS, NEXUS_DIRECTORY_ORIGIN, NEXUS_SPONSORS_URL, _sponsorHeartbeatTimer, _lastRegisteredSponsorPayload, __COMMAND_REGISTRY, liveDashboardBlocks, DASH_INTERNAL, localGpuMetricsCache, localGpuMetricsProbeAt, localGpuMetricsProbeInFlight;
660489
661107
  var init_commands = __esm({
660490
661108
  "packages/cli/src/tui/commands.ts"() {
660491
661109
  "use strict";
@@ -660803,6 +661421,7 @@ var init_commands = __esm({
660803
661421
  }
660804
661422
  });
660805
661423
  })();
661424
+ liveDashboardBlocks = /* @__PURE__ */ new Map();
660806
661425
  DASH_INTERNAL = /* @__PURE__ */ new Set(["system_metrics", "__list_capabilities"]);
660807
661426
  localGpuMetricsCache = null;
660808
661427
  localGpuMetricsProbeAt = 0;
@@ -733124,6 +733743,11 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
733124
733743
  setLiveMediaStatus(status) {
733125
733744
  statusBar.setLiveMediaStatus(status);
733126
733745
  },
733746
+ liveDynamicBlockHost: {
733747
+ registerDynamicBlock: (id2, render2) => statusBar.registerDynamicBlock(id2, render2),
733748
+ appendDynamicBlock: (id2) => statusBar.appendDynamicBlock(id2),
733749
+ refreshDynamicBlocks: () => statusBar.refreshDynamicBlocks()
733750
+ },
733127
733751
  // Voice call session — standalone cloudflared tunnel for /call
733128
733752
  // Each connecting WebSocket client gets a dedicated CallSubAgent.
733129
733753
  // Admin callers (matching session key) get full tool access; public callers get read-only.
@@ -733510,6 +734134,8 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
733510
734134
  } else {
733511
734135
  parts.push("active: no");
733512
734136
  }
734137
+ const liveContext = formatLiveSensorContext(repoRoot);
734138
+ if (liveContext) parts.push(liveContext);
733513
734139
  return parts.join("\n");
733514
734140
  }
733515
734141
  },