omnius 1.0.407 → 1.0.409
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +879 -253
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -41519,7 +41519,7 @@ function formatTime(seconds) {
|
|
|
41519
41519
|
const s2 = Math.floor(seconds % 60);
|
|
41520
41520
|
return `${String(m2).padStart(2, "0")}:${String(s2).padStart(2, "0")}`;
|
|
41521
41521
|
}
|
|
41522
|
-
var AUDIO_EXTS, VIDEO_EXTS, MAX_TRANSCRIBE_URL_BYTES, _tcModule, _tcChecked, TranscribeFileTool, YT_DLP_VENV, TranscribeUrlTool, YouTubeDownloadTool;
|
|
41522
|
+
var AUDIO_EXTS, VIDEO_EXTS, MAX_TRANSCRIBE_URL_BYTES, MANAGED_ASR_VENV, _tcModule, _tcChecked, _managedAsrReady, TranscribeFileTool, YT_DLP_VENV, TranscribeUrlTool, YouTubeDownloadTool;
|
|
41523
41523
|
var init_transcribe_tool = __esm({
|
|
41524
41524
|
"packages/execution/dist/tools/transcribe-tool.js"() {
|
|
41525
41525
|
"use strict";
|
|
@@ -41550,8 +41550,10 @@ var init_transcribe_tool = __esm({
|
|
|
41550
41550
|
".ts"
|
|
41551
41551
|
]);
|
|
41552
41552
|
MAX_TRANSCRIBE_URL_BYTES = 100 * 1024 * 1024;
|
|
41553
|
+
MANAGED_ASR_VENV = join40(homedir11(), ".omnius", "runtimes", "asr", ".venv-whisper");
|
|
41553
41554
|
_tcModule = null;
|
|
41554
41555
|
_tcChecked = false;
|
|
41556
|
+
_managedAsrReady = false;
|
|
41555
41557
|
TranscribeFileTool = class {
|
|
41556
41558
|
name = "transcribe_file";
|
|
41557
41559
|
description = "Transcribe a local audio or video file to text using Whisper (faster-whisper). Supports MP3, WAV, FLAC, AAC, M4A, OGG, MP4, MKV, AVI, MOV, WebM. Returns the full transcription text with optional speaker diarization. Transcription is 100% local — no API keys needed.";
|
|
@@ -41625,102 +41627,131 @@ var init_transcribe_tool = __esm({
|
|
|
41625
41627
|
}
|
|
41626
41628
|
if (effectiveModel !== askedModel)
|
|
41627
41629
|
model = effectiveModel;
|
|
41630
|
+
const failures = [];
|
|
41628
41631
|
const tc = await loadTranscribeCli();
|
|
41629
|
-
if (
|
|
41630
|
-
|
|
41631
|
-
|
|
41632
|
+
if (tc) {
|
|
41633
|
+
try {
|
|
41634
|
+
const result = await withProcessEnv(transcriptionPythonEnv(), () => tc.transcribe(filePath, {
|
|
41635
|
+
model,
|
|
41636
|
+
format: "json",
|
|
41637
|
+
diarize,
|
|
41638
|
+
wordTimestamps: true
|
|
41639
|
+
}));
|
|
41640
|
+
return this.formatTranscriptionResult(filePath, model, result, start2, "transcribe-cli");
|
|
41641
|
+
} catch (err) {
|
|
41642
|
+
failures.push(`transcribe-cli module: ${err instanceof Error ? err.message : String(err)}`);
|
|
41643
|
+
}
|
|
41644
|
+
}
|
|
41645
|
+
const cli = await this.execViaCli(filePath, model, diarize, start2);
|
|
41646
|
+
if (cli.success)
|
|
41647
|
+
return cli;
|
|
41648
|
+
if (cli.error)
|
|
41649
|
+
failures.push(cli.error);
|
|
41650
|
+
const managed = await this.execViaManagedWhisper(filePath, model, start2);
|
|
41651
|
+
if (managed.success)
|
|
41652
|
+
return managed;
|
|
41653
|
+
if (managed.error)
|
|
41654
|
+
failures.push(managed.error);
|
|
41655
|
+
return {
|
|
41656
|
+
success: false,
|
|
41657
|
+
output: "",
|
|
41658
|
+
error: `Transcription failed after all ASR backends. ${failures.join(" | ").slice(0, 1200)}`,
|
|
41659
|
+
durationMs: performance.now() - start2
|
|
41660
|
+
};
|
|
41661
|
+
}
|
|
41662
|
+
formatTranscriptionResult(filePath, model, result, start2, backend) {
|
|
41663
|
+
const segments = Array.isArray(result.segments) ? result.segments : [];
|
|
41664
|
+
const text2 = String(result.text ?? result.fullText ?? "").trim();
|
|
41665
|
+
const speakers = Array.isArray(result.speakers) ? result.speakers : [];
|
|
41666
|
+
const wordCount2 = Number(result.wordCount ?? (text2 ? text2.split(/\s+/).filter(Boolean).length : 0));
|
|
41667
|
+
const transcriptDir = join40(this.workingDir, ".omnius", "transcripts");
|
|
41668
|
+
mkdirSync22(transcriptDir, { recursive: true });
|
|
41669
|
+
const fileBase = basename7(filePath).replace(/\.[^.]+$/, "");
|
|
41670
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
41671
|
+
const txtFile = join40(transcriptDir, `${fileBase}-${timestamp}.txt`);
|
|
41672
|
+
const jsonFile = join40(transcriptDir, `${fileBase}-${timestamp}.json`);
|
|
41673
|
+
writeFileSync21(txtFile, text2, "utf-8");
|
|
41674
|
+
const structured = {
|
|
41675
|
+
source: filePath,
|
|
41676
|
+
model,
|
|
41677
|
+
backend,
|
|
41678
|
+
language: result.language,
|
|
41679
|
+
duration: result.duration,
|
|
41680
|
+
wordCount: wordCount2,
|
|
41681
|
+
speakers,
|
|
41682
|
+
transcribedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
41683
|
+
segments: segments.map((seg) => ({
|
|
41684
|
+
start: Number(seg.start ?? 0),
|
|
41685
|
+
end: Number(seg.end ?? seg.start ?? 0),
|
|
41686
|
+
text: String(seg.text ?? ""),
|
|
41687
|
+
speaker: seg.speaker || void 0
|
|
41688
|
+
})),
|
|
41689
|
+
fullText: text2
|
|
41690
|
+
};
|
|
41691
|
+
writeFileSync21(jsonFile, JSON.stringify(structured, null, 2), "utf-8");
|
|
41632
41692
|
try {
|
|
41633
|
-
const
|
|
41634
|
-
|
|
41635
|
-
|
|
41636
|
-
|
|
41637
|
-
wordTimestamps: true
|
|
41638
|
-
// Always get timestamps for structured output
|
|
41639
|
-
}));
|
|
41640
|
-
const transcriptDir = join40(this.workingDir, ".omnius", "transcripts");
|
|
41641
|
-
mkdirSync22(transcriptDir, { recursive: true });
|
|
41642
|
-
const fileBase = basename7(filePath).replace(/\.[^.]+$/, "");
|
|
41643
|
-
const timestamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
41644
|
-
const txtFile = join40(transcriptDir, `${fileBase}-${timestamp}.txt`);
|
|
41645
|
-
writeFileSync21(txtFile, result.text, "utf-8");
|
|
41646
|
-
const jsonFile = join40(transcriptDir, `${fileBase}-${timestamp}.json`);
|
|
41647
|
-
const structured = {
|
|
41648
|
-
source: filePath,
|
|
41649
|
-
model,
|
|
41650
|
-
language: result.language,
|
|
41651
|
-
duration: result.duration,
|
|
41652
|
-
wordCount: result.wordCount,
|
|
41653
|
-
speakers: result.speakers,
|
|
41654
|
-
transcribedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
41655
|
-
segments: result.segments.map((seg) => ({
|
|
41656
|
-
start: seg.start,
|
|
41657
|
-
end: seg.end,
|
|
41658
|
-
text: seg.text,
|
|
41659
|
-
speaker: seg.speaker || void 0
|
|
41660
|
-
})),
|
|
41661
|
-
fullText: result.text
|
|
41662
|
-
};
|
|
41663
|
-
writeFileSync21(jsonFile, JSON.stringify(structured, null, 2), "utf-8");
|
|
41693
|
+
const memDir = join40(this.workingDir, ".omnius", "memory");
|
|
41694
|
+
mkdirSync22(memDir, { recursive: true });
|
|
41695
|
+
const memFile = join40(memDir, "transcripts.json");
|
|
41696
|
+
let memEntries = [];
|
|
41664
41697
|
try {
|
|
41665
|
-
|
|
41666
|
-
mkdirSync22(memDir, { recursive: true });
|
|
41667
|
-
const memFile = join40(memDir, "transcripts.json");
|
|
41668
|
-
let memEntries = [];
|
|
41669
|
-
try {
|
|
41670
|
-
memEntries = JSON.parse(readFileSync30(memFile, "utf-8"));
|
|
41671
|
-
} catch {
|
|
41672
|
-
}
|
|
41673
|
-
memEntries.push({
|
|
41674
|
-
source: basename7(filePath),
|
|
41675
|
-
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
41676
|
-
duration: result.duration,
|
|
41677
|
-
wordCount: result.wordCount,
|
|
41678
|
-
language: result.language,
|
|
41679
|
-
summary: result.text.slice(0, 200),
|
|
41680
|
-
jsonPath: jsonFile,
|
|
41681
|
-
txtPath: txtFile
|
|
41682
|
-
});
|
|
41683
|
-
if (memEntries.length > 50)
|
|
41684
|
-
memEntries = memEntries.slice(-50);
|
|
41685
|
-
writeFileSync21(memFile, JSON.stringify(memEntries, null, 2), "utf-8");
|
|
41698
|
+
memEntries = JSON.parse(readFileSync30(memFile, "utf-8"));
|
|
41686
41699
|
} catch {
|
|
41687
41700
|
}
|
|
41688
|
-
|
|
41689
|
-
|
|
41690
|
-
|
|
41691
|
-
|
|
41692
|
-
|
|
41693
|
-
|
|
41694
|
-
|
|
41695
|
-
|
|
41696
|
-
|
|
41697
|
-
|
|
41698
|
-
}
|
|
41699
|
-
if (
|
|
41700
|
-
|
|
41701
|
-
|
|
41702
|
-
|
|
41703
|
-
|
|
41704
|
-
|
|
41705
|
-
|
|
41706
|
-
|
|
41701
|
+
memEntries.push({
|
|
41702
|
+
source: basename7(filePath),
|
|
41703
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
41704
|
+
duration: result.duration,
|
|
41705
|
+
wordCount: wordCount2,
|
|
41706
|
+
language: result.language,
|
|
41707
|
+
backend,
|
|
41708
|
+
summary: text2.slice(0, 200),
|
|
41709
|
+
jsonPath: jsonFile,
|
|
41710
|
+
txtPath: txtFile
|
|
41711
|
+
});
|
|
41712
|
+
if (memEntries.length > 50)
|
|
41713
|
+
memEntries = memEntries.slice(-50);
|
|
41714
|
+
writeFileSync21(memFile, JSON.stringify(memEntries, null, 2), "utf-8");
|
|
41715
|
+
} catch {
|
|
41716
|
+
}
|
|
41717
|
+
const lines = [
|
|
41718
|
+
`Transcription of: ${basename7(filePath)}`,
|
|
41719
|
+
`Backend: ${backend}`,
|
|
41720
|
+
`Model: ${model} | Language: ${result.language ?? "unknown"} | Duration: ${result.duration ? `${Number(result.duration).toFixed(1)}s` : "unknown"}`,
|
|
41721
|
+
`Words: ${wordCount2} | Segments: ${segments.length}`,
|
|
41722
|
+
`Saved: ${txtFile} (text) + ${jsonFile} (structured with timestamps)`,
|
|
41723
|
+
""
|
|
41724
|
+
];
|
|
41725
|
+
if (speakers.length > 1) {
|
|
41726
|
+
lines.push(`Speakers: ${speakers.join(", ")}`);
|
|
41727
|
+
lines.push("");
|
|
41728
|
+
}
|
|
41729
|
+
if (segments.length > 0) {
|
|
41730
|
+
for (const seg of segments) {
|
|
41731
|
+
const ts = `[${formatTime(Number(seg.start ?? 0))} → ${formatTime(Number(seg.end ?? seg.start ?? 0))}]`;
|
|
41732
|
+
const speaker = seg.speaker ? `${seg.speaker}: ` : "";
|
|
41733
|
+
lines.push(`${ts} ${speaker}${String(seg.text ?? "").trim()}`);
|
|
41707
41734
|
}
|
|
41708
|
-
|
|
41709
|
-
|
|
41710
|
-
|
|
41711
|
-
|
|
41712
|
-
|
|
41713
|
-
|
|
41735
|
+
} else {
|
|
41736
|
+
lines.push(text2 || "[silent — no speech detected]");
|
|
41737
|
+
}
|
|
41738
|
+
return {
|
|
41739
|
+
success: true,
|
|
41740
|
+
output: lines.join("\n"),
|
|
41741
|
+
llmContent: text2,
|
|
41742
|
+
durationMs: performance.now() - start2
|
|
41743
|
+
};
|
|
41744
|
+
}
|
|
41745
|
+
/** Fallback: invoke transcribe-cli as a subprocess. */
|
|
41746
|
+
async execViaCli(filePath, model, diarize, start2) {
|
|
41747
|
+
if (!await commandExists("transcribe-cli", 1500)) {
|
|
41714
41748
|
return {
|
|
41715
41749
|
success: false,
|
|
41716
41750
|
output: "",
|
|
41717
|
-
error:
|
|
41751
|
+
error: "transcribe-cli command unavailable",
|
|
41718
41752
|
durationMs: performance.now() - start2
|
|
41719
41753
|
};
|
|
41720
41754
|
}
|
|
41721
|
-
}
|
|
41722
|
-
/** Fallback: invoke transcribe-cli as a subprocess. */
|
|
41723
|
-
async execViaCli(filePath, model, diarize, start2) {
|
|
41724
41755
|
try {
|
|
41725
41756
|
const args = [filePath, "-m", model, "-f", "txt"];
|
|
41726
41757
|
if (diarize)
|
|
@@ -41734,6 +41765,7 @@ var init_transcribe_tool = __esm({
|
|
|
41734
41765
|
return {
|
|
41735
41766
|
success: true,
|
|
41736
41767
|
output: output.trim() || "(empty transcription)",
|
|
41768
|
+
llmContent: output.trim() || "",
|
|
41737
41769
|
durationMs: performance.now() - start2
|
|
41738
41770
|
};
|
|
41739
41771
|
} catch (err) {
|
|
@@ -41747,6 +41779,112 @@ var init_transcribe_tool = __esm({
|
|
|
41747
41779
|
};
|
|
41748
41780
|
}
|
|
41749
41781
|
}
|
|
41782
|
+
async ensureManagedWhisperRuntime() {
|
|
41783
|
+
const python = venvPython(MANAGED_ASR_VENV);
|
|
41784
|
+
if (!existsSync38(python)) {
|
|
41785
|
+
mkdirSync22(join40(homedir11(), ".omnius", "runtimes", "asr"), { recursive: true });
|
|
41786
|
+
await execFileText("python3", ["-m", "venv", MANAGED_ASR_VENV], {
|
|
41787
|
+
timeout: 6e4,
|
|
41788
|
+
env: transcriptionPythonEnv()
|
|
41789
|
+
});
|
|
41790
|
+
}
|
|
41791
|
+
if (_managedAsrReady)
|
|
41792
|
+
return python;
|
|
41793
|
+
try {
|
|
41794
|
+
await execFileText(python, ["-c", "import whisper, numpy"], {
|
|
41795
|
+
timeout: 1e4,
|
|
41796
|
+
env: transcriptionPythonEnv()
|
|
41797
|
+
});
|
|
41798
|
+
_managedAsrReady = true;
|
|
41799
|
+
return python;
|
|
41800
|
+
} catch {
|
|
41801
|
+
await execFileText(python, [
|
|
41802
|
+
"-m",
|
|
41803
|
+
"pip",
|
|
41804
|
+
"install",
|
|
41805
|
+
"-U",
|
|
41806
|
+
"pip",
|
|
41807
|
+
"wheel",
|
|
41808
|
+
"setuptools",
|
|
41809
|
+
"numpy",
|
|
41810
|
+
"openai-whisper"
|
|
41811
|
+
], {
|
|
41812
|
+
timeout: 9e5,
|
|
41813
|
+
env: transcriptionPythonEnv()
|
|
41814
|
+
});
|
|
41815
|
+
_managedAsrReady = true;
|
|
41816
|
+
return python;
|
|
41817
|
+
}
|
|
41818
|
+
}
|
|
41819
|
+
async execViaManagedWhisper(filePath, model, start2) {
|
|
41820
|
+
const scriptPath2 = join40(this.workingDir, ".omnius", "tmp", `managed-whisper-${Date.now()}.py`);
|
|
41821
|
+
mkdirSync22(join40(this.workingDir, ".omnius", "tmp"), { recursive: true });
|
|
41822
|
+
const script = `
|
|
41823
|
+
import json, os, warnings
|
|
41824
|
+
warnings.filterwarnings("ignore")
|
|
41825
|
+
os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3")
|
|
41826
|
+
audio_file = ${JSON.stringify(filePath)}
|
|
41827
|
+
model_name = ${JSON.stringify(model)}
|
|
41828
|
+
try:
|
|
41829
|
+
import whisper
|
|
41830
|
+
device = "cpu"
|
|
41831
|
+
try:
|
|
41832
|
+
import torch
|
|
41833
|
+
if torch.cuda.is_available():
|
|
41834
|
+
device = "cuda"
|
|
41835
|
+
except Exception:
|
|
41836
|
+
pass
|
|
41837
|
+
m = whisper.load_model(model_name, device=device)
|
|
41838
|
+
result = m.transcribe(audio_file, fp16=(device == "cuda"), condition_on_previous_text=False)
|
|
41839
|
+
segments = []
|
|
41840
|
+
for seg in result.get("segments", []) or []:
|
|
41841
|
+
segments.append({"start": float(seg.get("start", 0)), "end": float(seg.get("end", seg.get("start", 0))), "text": str(seg.get("text", "")).strip()})
|
|
41842
|
+
text = str(result.get("text", "")).strip()
|
|
41843
|
+
print(json.dumps({
|
|
41844
|
+
"ok": True,
|
|
41845
|
+
"text": text,
|
|
41846
|
+
"language": result.get("language"),
|
|
41847
|
+
"duration": segments[-1]["end"] if segments else None,
|
|
41848
|
+
"wordCount": len([w for w in text.split() if w]),
|
|
41849
|
+
"speakers": [],
|
|
41850
|
+
"segments": segments,
|
|
41851
|
+
}))
|
|
41852
|
+
except Exception as exc:
|
|
41853
|
+
print(json.dumps({"ok": False, "error": str(exc)}))
|
|
41854
|
+
`;
|
|
41855
|
+
try {
|
|
41856
|
+
writeFileSync21(scriptPath2, script, "utf-8");
|
|
41857
|
+
const python = await this.ensureManagedWhisperRuntime();
|
|
41858
|
+
const raw = await execFileText(python, [scriptPath2], {
|
|
41859
|
+
timeout: 6e5,
|
|
41860
|
+
cwd: this.workingDir,
|
|
41861
|
+
env: transcriptionPythonEnv({ PYTHONUNBUFFERED: "1" }),
|
|
41862
|
+
maxBuffer: 16 * 1024 * 1024
|
|
41863
|
+
});
|
|
41864
|
+
const parsed = JSON.parse(raw.trim().split("\n").reverse().find((line) => line.trim().startsWith("{")) ?? "{}");
|
|
41865
|
+
if (!parsed.ok) {
|
|
41866
|
+
return {
|
|
41867
|
+
success: false,
|
|
41868
|
+
output: "",
|
|
41869
|
+
error: `managed openai-whisper failed: ${parsed.error || raw.slice(-500)}`,
|
|
41870
|
+
durationMs: performance.now() - start2
|
|
41871
|
+
};
|
|
41872
|
+
}
|
|
41873
|
+
return this.formatTranscriptionResult(filePath, model, parsed, start2, "managed openai-whisper");
|
|
41874
|
+
} catch (err) {
|
|
41875
|
+
return {
|
|
41876
|
+
success: false,
|
|
41877
|
+
output: "",
|
|
41878
|
+
error: `managed openai-whisper failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
41879
|
+
durationMs: performance.now() - start2
|
|
41880
|
+
};
|
|
41881
|
+
} finally {
|
|
41882
|
+
try {
|
|
41883
|
+
unlinkSync5(scriptPath2);
|
|
41884
|
+
} catch {
|
|
41885
|
+
}
|
|
41886
|
+
}
|
|
41887
|
+
}
|
|
41750
41888
|
};
|
|
41751
41889
|
YT_DLP_VENV = join40(homedir11(), ".omnius", "runtimes", "media-tools", ".venv-ytdlp");
|
|
41752
41890
|
TranscribeUrlTool = class {
|
|
@@ -547804,17 +547942,23 @@ except Exception as e:
|
|
|
547804
547942
|
const audioFile = await this.ensureAudioFile(args);
|
|
547805
547943
|
if (!audioFile)
|
|
547806
547944
|
return { success: false, output: "", error: "No audio available.", durationMs: performance.now() - start2 };
|
|
547807
|
-
await this.ensureVenv(["torch", "
|
|
547945
|
+
await this.ensureVenv(["torch", "numpy", "soundfile", "resampy"]);
|
|
547808
547946
|
const audioLiteral = JSON.stringify(audioFile);
|
|
547809
547947
|
const script = `
|
|
547810
547948
|
import sys, json, numpy as np
|
|
547811
547949
|
try:
|
|
547812
|
-
import torch,
|
|
547950
|
+
import torch, soundfile as sf, resampy
|
|
547813
547951
|
|
|
547814
547952
|
model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', model='silero_vad', force_reload=False, trust_repo=True)
|
|
547815
|
-
(get_speech_timestamps,
|
|
547953
|
+
(get_speech_timestamps, *_) = utils
|
|
547816
547954
|
|
|
547817
|
-
|
|
547955
|
+
audio, sr = sf.read(${audioLiteral})
|
|
547956
|
+
if len(getattr(audio, "shape", [])) > 1:
|
|
547957
|
+
audio = audio.mean(axis=1)
|
|
547958
|
+
if sr != 16000:
|
|
547959
|
+
audio = resampy.resample(audio, sr, 16000)
|
|
547960
|
+
audio = audio.astype(np.float32)
|
|
547961
|
+
wav = torch.from_numpy(audio)
|
|
547818
547962
|
speech_timestamps = get_speech_timestamps(wav, model, sampling_rate=16000, threshold=0.5)
|
|
547819
547963
|
|
|
547820
547964
|
total_samples = len(wav)
|
|
@@ -547825,6 +547969,7 @@ try:
|
|
|
547825
547969
|
|
|
547826
547970
|
print(json.dumps({
|
|
547827
547971
|
"success": True,
|
|
547972
|
+
"backend": "silero-vad",
|
|
547828
547973
|
"voice_detected": len(speech_timestamps) > 0,
|
|
547829
547974
|
"speech_percentage": round(speech_pct, 1),
|
|
547830
547975
|
"speech_segments": len(speech_timestamps),
|
|
@@ -547832,7 +547977,57 @@ try:
|
|
|
547832
547977
|
"segments": segments[:10]
|
|
547833
547978
|
}))
|
|
547834
547979
|
except Exception as e:
|
|
547835
|
-
|
|
547980
|
+
try:
|
|
547981
|
+
import soundfile as sf
|
|
547982
|
+
import resampy
|
|
547983
|
+
audio, sr = sf.read(${audioLiteral})
|
|
547984
|
+
if len(getattr(audio, "shape", [])) > 1:
|
|
547985
|
+
audio = audio.mean(axis=1)
|
|
547986
|
+
if sr != 16000:
|
|
547987
|
+
audio = resampy.resample(audio, sr, 16000)
|
|
547988
|
+
audio = audio.astype(np.float32)
|
|
547989
|
+
frame = max(1, int(0.03 * 16000))
|
|
547990
|
+
hop = max(1, int(0.01 * 16000))
|
|
547991
|
+
rms = []
|
|
547992
|
+
for i in range(0, max(1, len(audio) - frame + 1), hop):
|
|
547993
|
+
chunk = audio[i:i+frame]
|
|
547994
|
+
if len(chunk) == 0:
|
|
547995
|
+
continue
|
|
547996
|
+
rms.append(float(np.sqrt(np.mean(chunk * chunk))))
|
|
547997
|
+
arr = np.array(rms, dtype=np.float32)
|
|
547998
|
+
if len(arr) == 0:
|
|
547999
|
+
active = np.array([], dtype=bool)
|
|
548000
|
+
else:
|
|
548001
|
+
floor = float(np.percentile(arr, 35))
|
|
548002
|
+
peak = float(np.max(arr))
|
|
548003
|
+
threshold = max(0.008, floor * 2.5, peak * 0.12)
|
|
548004
|
+
active = arr > threshold
|
|
548005
|
+
segments = []
|
|
548006
|
+
start_idx = None
|
|
548007
|
+
for idx, flag in enumerate(active.tolist()):
|
|
548008
|
+
if flag and start_idx is None:
|
|
548009
|
+
start_idx = idx
|
|
548010
|
+
if (not flag or idx == len(active) - 1) and start_idx is not None:
|
|
548011
|
+
end_idx = idx if not flag else idx + 1
|
|
548012
|
+
start_s = start_idx * hop / 16000.0
|
|
548013
|
+
end_s = min(len(audio) / 16000.0, (end_idx * hop + frame) / 16000.0)
|
|
548014
|
+
if end_s - start_s >= 0.12:
|
|
548015
|
+
segments.append({"start_s": start_s, "end_s": end_s, "duration_s": end_s - start_s})
|
|
548016
|
+
start_idx = None
|
|
548017
|
+
speech_duration = sum(seg["duration_s"] for seg in segments)
|
|
548018
|
+
total_duration = len(audio) / 16000.0 if len(audio) else 0
|
|
548019
|
+
print(json.dumps({
|
|
548020
|
+
"success": True,
|
|
548021
|
+
"backend": "energy-vad-fallback",
|
|
548022
|
+
"warning": "silero unavailable: " + str(e),
|
|
548023
|
+
"voice_detected": len(segments) > 0,
|
|
548024
|
+
"speech_percentage": round((speech_duration / total_duration * 100) if total_duration else 0, 1),
|
|
548025
|
+
"speech_segments": len(segments),
|
|
548026
|
+
"total_duration_s": round(total_duration, 2),
|
|
548027
|
+
"segments": segments[:10]
|
|
548028
|
+
}))
|
|
548029
|
+
except Exception as fallback:
|
|
548030
|
+
print(json.dumps({"success": False, "error": str(e) + "; fallback failed: " + str(fallback)}))
|
|
547836
548031
|
`;
|
|
547837
548032
|
return await this.runPythonScript(script, start2, "Voice activity detection");
|
|
547838
548033
|
}
|
|
@@ -555516,6 +555711,9 @@ function summarizeObjects(objects) {
|
|
|
555516
555711
|
}
|
|
555517
555712
|
return [...counts.entries()].sort((a2, b) => b[1].count - a2[1].count).slice(0, 12).map(([label, value2]) => `${label} x${value2.count} mean_conf=${(value2.confidence / Math.max(1, value2.count)).toFixed(2)}`);
|
|
555518
555713
|
}
|
|
555714
|
+
function isDiagnosticAudioEvidence(text2) {
|
|
555715
|
+
return /^(?:ASR|VAD|Audio comprehension)\s+(?:failed|unavailable|completed)|No module named ['"]?transcribe_cli|torchcodec|torchaudio version|no speech detected|empty transcription/i.test(text2.trim());
|
|
555716
|
+
}
|
|
555519
555717
|
function observationsFromResult(result, extraText) {
|
|
555520
555718
|
const observations = [];
|
|
555521
555719
|
for (const obj of result.objects) {
|
|
@@ -555541,6 +555739,8 @@ function observationsFromResult(result, extraText) {
|
|
|
555541
555739
|
});
|
|
555542
555740
|
}
|
|
555543
555741
|
for (const text2 of extraText) {
|
|
555742
|
+
if (isDiagnosticAudioEvidence(text2))
|
|
555743
|
+
continue;
|
|
555544
555744
|
observations.push({
|
|
555545
555745
|
type: "transcript",
|
|
555546
555746
|
confidence: 0.75,
|
|
@@ -555958,7 +556158,12 @@ var init_live_media_loop = __esm({
|
|
|
555958
556158
|
try {
|
|
555959
556159
|
const transcribe = new TranscribeFileTool(this.workingDir);
|
|
555960
556160
|
const asr = await transcribe.execute({ path: result.media_path, model: "tiny" });
|
|
555961
|
-
|
|
556161
|
+
if (asr.success) {
|
|
556162
|
+
const transcript = typeof asr.llmContent === "string" ? asr.llmContent.trim() : "";
|
|
556163
|
+
audioEvidence.push(transcript ? transcript.slice(0, 3e3) : "ASR completed: no speech detected");
|
|
556164
|
+
} else {
|
|
556165
|
+
audioEvidence.push(`ASR unavailable: ${asr.error}`);
|
|
556166
|
+
}
|
|
555962
556167
|
} catch (err) {
|
|
555963
556168
|
audioEvidence.push(`ASR failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
555964
556169
|
}
|
|
@@ -601694,10 +601899,13 @@ function defaultAsciiPreviewSize(dimensions) {
|
|
|
601694
601899
|
const columns = process.stdout.columns || 100;
|
|
601695
601900
|
const rows = process.stdout.rows || 32;
|
|
601696
601901
|
const width = clamp7(columns - 14, 42, 96);
|
|
601697
|
-
const
|
|
601698
|
-
const height = clamp7(Math.min(Math.round(width * imageAspect * TERMINAL_CELL_ASPECT), rows - 10), 8, 42);
|
|
601902
|
+
const height = defaultAsciiPreviewHeight(width, dimensions, rows);
|
|
601699
601903
|
return { width, height };
|
|
601700
601904
|
}
|
|
601905
|
+
function defaultAsciiPreviewHeight(width, dimensions, rows = process.stdout.rows || 32) {
|
|
601906
|
+
const imageAspect = dimensions && dimensions.width > 0 && dimensions.height > 0 ? dimensions.height / dimensions.width : 1;
|
|
601907
|
+
return clamp7(Math.min(Math.round(width * imageAspect * TERMINAL_CELL_ASPECT), rows - 10), 8, 42);
|
|
601908
|
+
}
|
|
601701
601909
|
function readImageDimensions2(imagePath) {
|
|
601702
601910
|
try {
|
|
601703
601911
|
const buf = readFileSync87(imagePath);
|
|
@@ -601959,7 +602167,7 @@ async function buildImageAsciiPreview(inputPath, options2 = {}) {
|
|
|
601959
602167
|
const dimensions = readImageDimensions2(imagePath);
|
|
601960
602168
|
const defaults3 = defaultAsciiPreviewSize(dimensions);
|
|
601961
602169
|
const width = clamp7(options2.width ?? defaults3.width, 24, 140);
|
|
601962
|
-
const height = clamp7(options2.height ??
|
|
602170
|
+
const height = clamp7(options2.height ?? defaultAsciiPreviewHeight(width, dimensions), 6, 60);
|
|
601963
602171
|
const timeoutMs = clamp7(options2.timeoutMs ?? 5e3, 500, 3e4);
|
|
601964
602172
|
if (options2.preferPackage !== false) {
|
|
601965
602173
|
const result = await convertWithImageToAscii(imagePath, width, height, timeoutMs);
|
|
@@ -602166,6 +602374,176 @@ function describeCameraOrientation(orientation) {
|
|
|
602166
602374
|
if (!orientation) return "upright/no correction (default)";
|
|
602167
602375
|
return `${formatCameraRotation(orientation.rotation)} (${orientation.source}${orientation.confidence !== void 0 ? ` confidence=${orientation.confidence.toFixed(2)}` : ""}, updated=${nowIso3(orientation.updatedAt)})`;
|
|
602168
602376
|
}
|
|
602377
|
+
function chooseCameraOrientationFromScores(scores) {
|
|
602378
|
+
const sorted = [...scores].filter((score) => Number.isFinite(score.score)).sort((a2, b) => b.score - a2.score);
|
|
602379
|
+
const best = sorted[0];
|
|
602380
|
+
if (!best || best.score <= 0 || best.faces <= 0) return null;
|
|
602381
|
+
const second3 = sorted[1];
|
|
602382
|
+
const gap = best.score - (second3?.score ?? 0);
|
|
602383
|
+
const total = sorted.reduce((sum, score) => sum + Math.max(0, score.score), 0);
|
|
602384
|
+
const share = total > 0 ? best.score / total : 1;
|
|
602385
|
+
const confidence2 = Math.max(0.55, Math.min(0.98, share * 0.75 + Math.min(0.23, gap / Math.max(1, best.score))));
|
|
602386
|
+
if (confidence2 < 0.58 && gap < 0.4) return null;
|
|
602387
|
+
return {
|
|
602388
|
+
rotation: best.rotation,
|
|
602389
|
+
confidence: confidence2,
|
|
602390
|
+
reason: `cv-face-orientation faces=${best.faces} score=${best.score.toFixed(2)} next=${(second3?.score ?? 0).toFixed(2)} area=${(best.faceAreaRatio ?? 0).toFixed(3)}`
|
|
602391
|
+
};
|
|
602392
|
+
}
|
|
602393
|
+
function trimOneLine(value2, maxChars) {
|
|
602394
|
+
return String(value2 ?? "").replace(/\r/g, " ").replace(/\n/g, " ").replace(/\s+/g, " ").trim().slice(0, maxChars);
|
|
602395
|
+
}
|
|
602396
|
+
function compactSourceId(source) {
|
|
602397
|
+
return source.replace(/^\/dev\//, "");
|
|
602398
|
+
}
|
|
602399
|
+
function truncateCell(value2, width) {
|
|
602400
|
+
value2 = stripDashboardAnsi(String(value2 ?? "")).replace(/\r?\n/g, " ");
|
|
602401
|
+
if (width <= 1) return value2.slice(0, Math.max(0, width));
|
|
602402
|
+
return value2.length <= width ? value2.padEnd(width, " ") : `${value2.slice(0, Math.max(1, width - 1))}…`;
|
|
602403
|
+
}
|
|
602404
|
+
function stripDashboardAnsi(value2) {
|
|
602405
|
+
return value2.replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, "");
|
|
602406
|
+
}
|
|
602407
|
+
function dashboardPreviewLines(camera) {
|
|
602408
|
+
const raw = camera.plainAscii || camera.ascii || "";
|
|
602409
|
+
return raw.replace(/\r/g, "").split("\n").map((line) => stripDashboardAnsi(line).replace(/\s+$/g, "")).filter((line) => line.trim().length > 0).slice(0, 10);
|
|
602410
|
+
}
|
|
602411
|
+
function centerDashboardCell(value2, width) {
|
|
602412
|
+
const text2 = stripDashboardAnsi(String(value2 ?? "")).replace(/\r?\n/g, " ");
|
|
602413
|
+
if (width <= 1) return text2.slice(0, Math.max(0, width));
|
|
602414
|
+
const clipped = text2.length > width ? `${text2.slice(0, Math.max(1, width - 1))}…` : text2;
|
|
602415
|
+
const left = Math.floor((width - clipped.length) / 2);
|
|
602416
|
+
return `${" ".repeat(Math.max(0, left))}${clipped}${" ".repeat(Math.max(0, width - clipped.length - left))}`;
|
|
602417
|
+
}
|
|
602418
|
+
function dashboardPreviewAspect(lines) {
|
|
602419
|
+
const widths = lines.map((line) => stripDashboardAnsi(line).length).filter((n2) => n2 > 0);
|
|
602420
|
+
const maxWidth = Math.max(1, ...widths);
|
|
602421
|
+
return Math.max(0.12, Math.min(1.6, lines.length / maxWidth));
|
|
602422
|
+
}
|
|
602423
|
+
function fitDashboardPane(lines, paneWidth, paneHeight) {
|
|
602424
|
+
const source = lines.length > 0 ? lines : ["preview pending"];
|
|
602425
|
+
const normalized = [];
|
|
602426
|
+
for (let i2 = 0; i2 < paneHeight; i2++) {
|
|
602427
|
+
const sourceIndex = source.length <= paneHeight ? i2 : Math.min(source.length - 1, Math.floor(i2 * source.length / paneHeight));
|
|
602428
|
+
normalized.push(truncateCell(source[sourceIndex] ?? "", paneWidth));
|
|
602429
|
+
}
|
|
602430
|
+
while (normalized.length < paneHeight) normalized.push(" ".repeat(paneWidth));
|
|
602431
|
+
return normalized;
|
|
602432
|
+
}
|
|
602433
|
+
function pushDashboardWrapped(lines, value2, width) {
|
|
602434
|
+
const contentWidth = Math.max(10, width - 2);
|
|
602435
|
+
const words = stripDashboardAnsi(value2).replace(/\s+/g, " ").trim().split(" ").filter(Boolean);
|
|
602436
|
+
if (words.length === 0) {
|
|
602437
|
+
lines.push(`│${" ".repeat(contentWidth)}│`);
|
|
602438
|
+
return;
|
|
602439
|
+
}
|
|
602440
|
+
let current = "";
|
|
602441
|
+
for (const word2 of words) {
|
|
602442
|
+
if (!current) {
|
|
602443
|
+
current = word2.length > contentWidth ? word2.slice(0, contentWidth) : word2;
|
|
602444
|
+
continue;
|
|
602445
|
+
}
|
|
602446
|
+
if (current.length + 1 + word2.length <= contentWidth) {
|
|
602447
|
+
current += ` ${word2}`;
|
|
602448
|
+
} else {
|
|
602449
|
+
lines.push(`│${truncateCell(` ${current}`, contentWidth)}│`);
|
|
602450
|
+
current = word2.length > contentWidth ? word2.slice(0, contentWidth) : word2;
|
|
602451
|
+
}
|
|
602452
|
+
}
|
|
602453
|
+
if (current) lines.push(`│${truncateCell(` ${current}`, contentWidth)}│`);
|
|
602454
|
+
}
|
|
602455
|
+
function isAudioFailureText(value2) {
|
|
602456
|
+
const text2 = String(value2 ?? "").trim();
|
|
602457
|
+
if (!text2) return false;
|
|
602458
|
+
return /^(?:ASR|VAD|Transcription)\s+(?:failed|unavailable)|No module named ['"]?transcribe_cli|torchcodec|torchaudio version/i.test(text2);
|
|
602459
|
+
}
|
|
602460
|
+
function summarizeInference(value2) {
|
|
602461
|
+
if (!value2) return "objects: pending";
|
|
602462
|
+
const lines = value2.split("\n").map((line) => line.trim()).filter(Boolean);
|
|
602463
|
+
const objectsIndex = lines.findIndex((line) => /^##\s+Objects/i.test(line));
|
|
602464
|
+
if (objectsIndex >= 0) {
|
|
602465
|
+
const objects = lines.slice(objectsIndex + 1).filter((line) => line.startsWith("- ")).slice(0, 4).map((line) => line.replace(/^-\s*/, ""));
|
|
602466
|
+
if (objects.length > 0) return `objects: ${objects.join("; ")}`;
|
|
602467
|
+
}
|
|
602468
|
+
const sampled = lines.find((line) => /^objects:/i.test(line));
|
|
602469
|
+
return sampled ? trimOneLine(sampled, 160) : trimOneLine(lines.slice(0, 4).join("; "), 160);
|
|
602470
|
+
}
|
|
602471
|
+
function cameraSnapshotSummary(camera) {
|
|
602472
|
+
if (camera.error) return `error: ${trimOneLine(camera.error, 160)}`;
|
|
602473
|
+
return camera.summary || summarizeInference(camera.inference);
|
|
602474
|
+
}
|
|
602475
|
+
function formatLiveDashboardFromSnapshot(snapshot, opts = {}) {
|
|
602476
|
+
const width = Math.max(60, opts.width ?? 100);
|
|
602477
|
+
const now2 = opts.now ?? Date.now();
|
|
602478
|
+
const maxCameras = opts.maxCameras ?? 4;
|
|
602479
|
+
const cameras = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {})).sort((a2, b) => {
|
|
602480
|
+
if (a2.source === snapshot.config.selectedCamera) return -1;
|
|
602481
|
+
if (b.source === snapshot.config.selectedCamera) return 1;
|
|
602482
|
+
return a2.source.localeCompare(b.source);
|
|
602483
|
+
}).slice(0, maxCameras);
|
|
602484
|
+
const audioAge = snapshot.audio?.updatedAt ? `${Math.max(0, Math.round((now2 - snapshot.audio.updatedAt) / 1e3))}s` : "none";
|
|
602485
|
+
const title = `live audio/video cameras=${cameras.length}/${snapshot.devices.video.length} audio=${snapshot.config.audioEnabled ? "on" : "off"} age=${audioAge}`;
|
|
602486
|
+
const horizontal = "─".repeat(Math.max(0, width - 2));
|
|
602487
|
+
const lines = [`╭${horizontal}╮`];
|
|
602488
|
+
lines.push(`│${truncateCell(` ${title}`, width - 2)}│`);
|
|
602489
|
+
if (cameras.length === 0) {
|
|
602490
|
+
lines.push(`│${truncateCell(" no camera frames captured yet", width - 2)}│`);
|
|
602491
|
+
}
|
|
602492
|
+
if (cameras.length > 0) {
|
|
602493
|
+
lines.push(`├${horizontal}┤`);
|
|
602494
|
+
const paneCount = Math.min(cameras.length, width >= 116 ? 3 : width >= 78 ? 2 : 1);
|
|
602495
|
+
const innerWidth = width - 2;
|
|
602496
|
+
const gutter = paneCount - 1;
|
|
602497
|
+
const paneWidth = Math.max(24, Math.floor((innerWidth - gutter) / paneCount));
|
|
602498
|
+
for (let start2 = 0; start2 < cameras.length; start2 += paneCount) {
|
|
602499
|
+
const group = cameras.slice(start2, start2 + paneCount);
|
|
602500
|
+
const sourceLines = group.map((camera) => dashboardPreviewLines(camera));
|
|
602501
|
+
const paneHeight = Math.max(7, Math.min(22, Math.round(paneWidth * Math.max(...sourceLines.map(dashboardPreviewAspect)))));
|
|
602502
|
+
const panes = sourceLines.map((previewLines) => fitDashboardPane(previewLines, paneWidth, paneHeight));
|
|
602503
|
+
for (let row = 0; row < paneHeight; row++) {
|
|
602504
|
+
const body = panes.map((pane) => pane[row] ?? " ".repeat(paneWidth)).join(" ");
|
|
602505
|
+
lines.push(`│${truncateCell(body, innerWidth)}│`);
|
|
602506
|
+
}
|
|
602507
|
+
const labels = group.map((camera) => {
|
|
602508
|
+
const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
|
|
602509
|
+
return centerDashboardCell(`${compactSourceId(camera.source)} age=${age}s`, paneWidth);
|
|
602510
|
+
}).join(" ");
|
|
602511
|
+
lines.push(`│${truncateCell(labels, innerWidth)}│`);
|
|
602512
|
+
}
|
|
602513
|
+
}
|
|
602514
|
+
if (cameras.length > 0) {
|
|
602515
|
+
lines.push(`├${horizontal}┤`);
|
|
602516
|
+
lines.push(`│${truncateCell(" camera observations", width - 2)}│`);
|
|
602517
|
+
}
|
|
602518
|
+
for (const camera of cameras) {
|
|
602519
|
+
const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
|
|
602520
|
+
const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
|
|
602521
|
+
const detailLines = [
|
|
602522
|
+
`${compactSourceId(camera.source)} age=${age}s rotation=${rotation}`,
|
|
602523
|
+
cameraSnapshotSummary(camera),
|
|
602524
|
+
camera.framePath ? `frame: ${camera.displayPath || camera.framePath}` : "",
|
|
602525
|
+
camera.orientation?.evidence ? `orientation: ${trimOneLine(camera.orientation.evidence, 120)}` : ""
|
|
602526
|
+
].filter(Boolean);
|
|
602527
|
+
for (const detail of detailLines.length ? detailLines : ["preview pending"]) {
|
|
602528
|
+
pushDashboardWrapped(lines, detail, width);
|
|
602529
|
+
}
|
|
602530
|
+
}
|
|
602531
|
+
if (snapshot.audio) {
|
|
602532
|
+
lines.push(`├${horizontal}┤`);
|
|
602533
|
+
const transcriptFailure = isAudioFailureText(snapshot.audio.transcript) ? snapshot.audio.transcript : "";
|
|
602534
|
+
const transcript = transcriptFailure ? "" : snapshot.audio.transcript;
|
|
602535
|
+
const audioError = [snapshot.audio.error, transcriptFailure].filter(Boolean).join(" | ");
|
|
602536
|
+
const audioBits = [
|
|
602537
|
+
`audio input=${snapshot.audio.input || snapshot.config.selectedAudioInput || "none"}`,
|
|
602538
|
+
audioError ? `error=${trimOneLine(audioError, 120)}` : "",
|
|
602539
|
+
transcript ? `asr=${trimOneLine(transcript, 120)}` : "",
|
|
602540
|
+
snapshot.audio.analysis ? `sound=${trimOneLine(snapshot.audio.analysis, 120)}` : ""
|
|
602541
|
+
].filter(Boolean);
|
|
602542
|
+
for (const item of audioBits.slice(0, 4)) lines.push(`│${truncateCell(` ${item}`, width - 2)}│`);
|
|
602543
|
+
}
|
|
602544
|
+
lines.push(`╰${horizontal}╯`);
|
|
602545
|
+
return lines;
|
|
602546
|
+
}
|
|
602169
602547
|
function parseLiveMediaPayload(value2) {
|
|
602170
602548
|
const parsed = parseJsonObject3(value2);
|
|
602171
602549
|
if (!parsed) return null;
|
|
@@ -602425,6 +602803,21 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
602425
602803
|
lines.push(`Selected camera: ${snapshot.config.selectedCamera}`);
|
|
602426
602804
|
lines.push(`Camera orientation correction: ${describeCameraOrientation(orientation)}`);
|
|
602427
602805
|
}
|
|
602806
|
+
const cameraSnapshots = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {})).sort((a2, b) => {
|
|
602807
|
+
if (a2.source === snapshot.config.selectedCamera) return -1;
|
|
602808
|
+
if (b.source === snapshot.config.selectedCamera) return 1;
|
|
602809
|
+
return b.updatedAt - a2.updatedAt;
|
|
602810
|
+
});
|
|
602811
|
+
if (cameraSnapshots.length > 0) {
|
|
602812
|
+
lines.push("Camera snapshot set:");
|
|
602813
|
+
for (const camera of cameraSnapshots.slice(0, 6)) {
|
|
602814
|
+
const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
|
|
602815
|
+
const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
|
|
602816
|
+
lines.push(`- ${camera.source} age=${age}s rotation=${rotation}${camera.framePath ? ` frame=${camera.framePath}` : ""}${camera.error ? ` error=${trimOneLine(camera.error, 240)}` : ""}`);
|
|
602817
|
+
const summary = cameraSnapshotSummary(camera);
|
|
602818
|
+
if (summary) lines.push(` summary: ${summary}`);
|
|
602819
|
+
}
|
|
602820
|
+
}
|
|
602428
602821
|
if (snapshot.config.selectedAudioInput) lines.push(`Selected audio input: ${snapshot.config.selectedAudioInput}`);
|
|
602429
602822
|
if (snapshot.config.selectedAudioOutput) lines.push(`Selected audio output: ${snapshot.config.selectedAudioOutput}`);
|
|
602430
602823
|
const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "clip_recognition" || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 18);
|
|
@@ -602481,7 +602874,23 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
602481
602874
|
lines.push(cleanPreview(snapshot.video.inference, 1800));
|
|
602482
602875
|
}
|
|
602483
602876
|
}
|
|
602877
|
+
if (cameraSnapshots.length > 1) {
|
|
602878
|
+
lines.push("");
|
|
602879
|
+
lines.push("## LIVE CAMERA LATEST FRAMES");
|
|
602880
|
+
for (const camera of cameraSnapshots.slice(0, 6)) {
|
|
602881
|
+
const age = now2 - camera.updatedAt;
|
|
602882
|
+
const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
|
|
602883
|
+
lines.push(`### ${camera.source}`);
|
|
602884
|
+
lines.push(`timestamp: ${nowIso3(camera.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s rotation=${rotation}${age > maxAgeMs ? " STALE" : ""}`);
|
|
602885
|
+
if (camera.error) lines.push(`error: ${camera.error}`);
|
|
602886
|
+
if (camera.framePath) lines.push(`frame: ${camera.framePath}`);
|
|
602887
|
+
lines.push(`summary: ${cameraSnapshotSummary(camera)}`);
|
|
602888
|
+
}
|
|
602889
|
+
}
|
|
602484
602890
|
if (snapshot.audio) {
|
|
602891
|
+
const transcriptFailure = isAudioFailureText(snapshot.audio.transcript) ? snapshot.audio.transcript : "";
|
|
602892
|
+
const transcript = transcriptFailure ? "" : snapshot.audio.transcript;
|
|
602893
|
+
const audioError = [snapshot.audio.error, transcriptFailure].filter(Boolean).join(" | ");
|
|
602485
602894
|
const audioEvents = (snapshot.events ?? []).filter((event) => event.kind === "audio_transcript" || event.kind === "audio_sound").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 8);
|
|
602486
602895
|
if (audioEvents.length > 0) {
|
|
602487
602896
|
lines.push("");
|
|
@@ -602498,11 +602907,11 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
602498
602907
|
lines.push("");
|
|
602499
602908
|
lines.push("## LIVE AUDIO LATEST SAMPLE");
|
|
602500
602909
|
lines.push(`timestamp: ${nowIso3(snapshot.audio.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s${age > maxAgeMs ? " STALE" : ""}`);
|
|
602501
|
-
if (
|
|
602910
|
+
if (audioError) lines.push(`error: ${audioError}`);
|
|
602502
602911
|
if (snapshot.audio.recordingPath) lines.push(`recording: ${snapshot.audio.recordingPath}`);
|
|
602503
|
-
if (
|
|
602912
|
+
if (transcript) {
|
|
602504
602913
|
lines.push("Live ASR:");
|
|
602505
|
-
lines.push(cleanPreview(
|
|
602914
|
+
lines.push(cleanPreview(transcript, 1200));
|
|
602506
602915
|
}
|
|
602507
602916
|
if (snapshot.audio.analysis) {
|
|
602508
602917
|
lines.push("Live sound analysis:");
|
|
@@ -602587,6 +602996,93 @@ async function recordAudioSample(device, durationSec, outputPath3) {
|
|
|
602587
602996
|
outputPath3
|
|
602588
602997
|
], { timeout: (Number(seconds) + 10) * 1e3 });
|
|
602589
602998
|
}
|
|
602999
|
+
async function scoreCameraOrientationWithOpenCv(framePath) {
|
|
603000
|
+
if (!await commandExists2("python3", 1200)) return null;
|
|
603001
|
+
const script = String.raw`
|
|
603002
|
+
import json, sys
|
|
603003
|
+
try:
|
|
603004
|
+
import cv2
|
|
603005
|
+
except Exception as exc:
|
|
603006
|
+
print(json.dumps({"ok": False, "error": str(exc)}))
|
|
603007
|
+
raise SystemExit(0)
|
|
603008
|
+
|
|
603009
|
+
path = sys.argv[1]
|
|
603010
|
+
img = cv2.imread(path)
|
|
603011
|
+
if img is None:
|
|
603012
|
+
print(json.dumps({"ok": False, "error": "could not read image"}))
|
|
603013
|
+
raise SystemExit(0)
|
|
603014
|
+
|
|
603015
|
+
face = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml")
|
|
603016
|
+
eye = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_eye.xml")
|
|
603017
|
+
|
|
603018
|
+
def rotate(mat, deg):
|
|
603019
|
+
if deg == 90:
|
|
603020
|
+
return cv2.rotate(mat, cv2.ROTATE_90_CLOCKWISE)
|
|
603021
|
+
if deg == 180:
|
|
603022
|
+
return cv2.rotate(mat, cv2.ROTATE_180)
|
|
603023
|
+
if deg == 270:
|
|
603024
|
+
return cv2.rotate(mat, cv2.ROTATE_90_COUNTERCLOCKWISE)
|
|
603025
|
+
return mat
|
|
603026
|
+
|
|
603027
|
+
scores = []
|
|
603028
|
+
for deg in [0, 90, 180, 270]:
|
|
603029
|
+
frame = rotate(img, deg)
|
|
603030
|
+
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
|
603031
|
+
h, w = gray.shape[:2]
|
|
603032
|
+
faces = face.detectMultiScale(gray, scaleFactor=1.08, minNeighbors=4, minSize=(32, 32))
|
|
603033
|
+
face_area = 0.0
|
|
603034
|
+
eye_count = 0
|
|
603035
|
+
for (x, y, fw, fh) in faces[:8]:
|
|
603036
|
+
face_area += float(fw * fh) / max(1.0, float(w * h))
|
|
603037
|
+
roi = gray[y:y+fh, x:x+fw]
|
|
603038
|
+
try:
|
|
603039
|
+
eye_count += len(eye.detectMultiScale(roi, scaleFactor=1.1, minNeighbors=3, minSize=(8, 8)))
|
|
603040
|
+
except Exception:
|
|
603041
|
+
pass
|
|
603042
|
+
score = len(faces) * 2.5 + min(4.0, face_area * 35.0) + min(2.0, eye_count * 0.35)
|
|
603043
|
+
scores.append({"rotation": deg, "score": score, "faces": int(len(faces)), "faceAreaRatio": face_area})
|
|
603044
|
+
print(json.dumps({"ok": True, "scores": scores}))
|
|
603045
|
+
`;
|
|
603046
|
+
try {
|
|
603047
|
+
const raw = await execFileText4("python3", ["-c", script, framePath], { timeout: 15e3, maxBuffer: 1024 * 1024 });
|
|
603048
|
+
const parsed = parseJsonObjectFromText(raw);
|
|
603049
|
+
const scores = Array.isArray(parsed?.["scores"]) ? parsed["scores"] : [];
|
|
603050
|
+
if (scores.length === 0) return null;
|
|
603051
|
+
return scores.map((score) => ({
|
|
603052
|
+
rotation: normalizeLiveCameraRotation(score["rotation"]),
|
|
603053
|
+
score: Math.max(0, Number(score["score"] ?? 0)),
|
|
603054
|
+
faces: Math.max(0, Number(score["faces"] ?? 0)),
|
|
603055
|
+
faceAreaRatio: Number.isFinite(Number(score["faceAreaRatio"])) ? Number(score["faceAreaRatio"]) : void 0
|
|
603056
|
+
}));
|
|
603057
|
+
} catch {
|
|
603058
|
+
return null;
|
|
603059
|
+
}
|
|
603060
|
+
}
|
|
603061
|
+
async function tryRecordAudioDevice(device, durationSec, outputPath3) {
|
|
603062
|
+
await recordAudioSample(device, durationSec, outputPath3);
|
|
603063
|
+
}
|
|
603064
|
+
async function recordAudioSampleWithRecovery(preferred, devices, durationSec, outputPath3) {
|
|
603065
|
+
const ordered = [];
|
|
603066
|
+
const add3 = (id2) => {
|
|
603067
|
+
const clean5 = String(id2 ?? "").trim();
|
|
603068
|
+
if (clean5 && !ordered.includes(clean5)) ordered.push(clean5);
|
|
603069
|
+
};
|
|
603070
|
+
add3(preferred);
|
|
603071
|
+
for (const device of devices.filter((entry) => entry.source === "pulse" || entry.source === "pipewire")) add3(device.id);
|
|
603072
|
+
add3("pulse:default");
|
|
603073
|
+
add3("default");
|
|
603074
|
+
for (const device of devices.filter((entry) => entry.source !== "pulse" && entry.source !== "pipewire")) add3(device.id);
|
|
603075
|
+
const errors = [];
|
|
603076
|
+
for (const candidate of ordered) {
|
|
603077
|
+
try {
|
|
603078
|
+
await tryRecordAudioDevice(candidate, durationSec, outputPath3);
|
|
603079
|
+
return { ok: true, device: candidate, method: candidate.startsWith("pulse:") ? "pulse/ffmpeg" : "arecord", errors };
|
|
603080
|
+
} catch (err) {
|
|
603081
|
+
errors.push(`${candidate}: ${err instanceof Error ? err.message : String(err)}`.slice(0, 360));
|
|
603082
|
+
}
|
|
603083
|
+
}
|
|
603084
|
+
return { ok: false, device: preferred || "default", method: "none", errors };
|
|
603085
|
+
}
|
|
602590
603086
|
function firstDeviceId(devices) {
|
|
602591
603087
|
return devices.find((device) => device.id && !/metadata\/non-capture/i.test(device.detail ?? ""))?.id ?? devices[0]?.id;
|
|
602592
603088
|
}
|
|
@@ -602677,6 +603173,7 @@ var init_live_sensors = __esm({
|
|
|
602677
603173
|
agentActionEnabled = false;
|
|
602678
603174
|
lastAgentReviewAt = 0;
|
|
602679
603175
|
autoOrientationInFlight = /* @__PURE__ */ new Set();
|
|
603176
|
+
lastFeedbackAt = /* @__PURE__ */ new Map();
|
|
602680
603177
|
setStatusSink(sink) {
|
|
602681
603178
|
this.statusSink = sink ?? null;
|
|
602682
603179
|
this.emitStatus();
|
|
@@ -602693,6 +603190,20 @@ var init_live_sensors = __esm({
|
|
|
602693
603190
|
getSnapshot() {
|
|
602694
603191
|
return this.snapshot;
|
|
602695
603192
|
}
|
|
603193
|
+
activeVideoSources() {
|
|
603194
|
+
const ordered = [];
|
|
603195
|
+
const add3 = (id2) => {
|
|
603196
|
+
const clean5 = String(id2 ?? "").trim();
|
|
603197
|
+
if (!clean5 || ordered.includes(clean5)) return;
|
|
603198
|
+
const device = this.devices.video.find((entry) => entry.id === clean5);
|
|
603199
|
+
if (device && /metadata\/non-capture/i.test(device.detail ?? "")) return;
|
|
603200
|
+
ordered.push(clean5);
|
|
603201
|
+
};
|
|
603202
|
+
add3(this.config.selectedCamera);
|
|
603203
|
+
for (const device of this.devices.video) add3(device.id);
|
|
603204
|
+
if (ordered.length === 0) add3(firstDeviceId(this.devices.video));
|
|
603205
|
+
return ordered.slice(0, 4);
|
|
603206
|
+
}
|
|
602696
603207
|
getCameraOrientation(device = this.config.selectedCamera) {
|
|
602697
603208
|
return device ? this.config.cameraOrientation?.[device] : void 0;
|
|
602698
603209
|
}
|
|
@@ -602763,6 +603274,12 @@ var init_live_sensors = __esm({
|
|
|
602763
603274
|
}
|
|
602764
603275
|
const framePath = extractSavedImagePath(captured.output, this.repoRoot);
|
|
602765
603276
|
if (!framePath) return `Camera orientation detection captured ${target}, but no frame path was returned.`;
|
|
603277
|
+
const cvScores = await scoreCameraOrientationWithOpenCv(framePath);
|
|
603278
|
+
const cvDecision = cvScores ? chooseCameraOrientationFromScores(cvScores) : null;
|
|
603279
|
+
if (cvDecision && cvDecision.confidence >= 0.68) {
|
|
603280
|
+
const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
|
|
603281
|
+
return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
|
|
603282
|
+
}
|
|
602766
603283
|
const prompt = [
|
|
602767
603284
|
"Determine whether this camera frame is upright for a human viewer.",
|
|
602768
603285
|
"Return only JSON with this schema:",
|
|
@@ -602777,14 +603294,24 @@ var init_live_sensors = __esm({
|
|
|
602777
603294
|
model: "moondream3-preview"
|
|
602778
603295
|
});
|
|
602779
603296
|
if (!vision.success) {
|
|
603297
|
+
if (cvDecision) {
|
|
603298
|
+
const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
|
|
603299
|
+
return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
|
|
603300
|
+
}
|
|
602780
603301
|
return `Camera orientation detection captured ${target}, but vision inference was unavailable: ${vision.error || vision.output || "unknown error"}`;
|
|
602781
603302
|
}
|
|
602782
603303
|
const decision2 = parseCameraOrientationDecision(vision.llmContent || vision.output);
|
|
602783
603304
|
if (!decision2) {
|
|
603305
|
+
if (cvDecision) {
|
|
603306
|
+
const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
|
|
603307
|
+
return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
|
|
603308
|
+
}
|
|
602784
603309
|
return `Camera orientation detection for ${target} was inconclusive. Vision output did not contain a structured rotation decision.`;
|
|
602785
603310
|
}
|
|
602786
|
-
const
|
|
602787
|
-
const
|
|
603311
|
+
const useCv = cvDecision && (cvDecision.confidence > (decision2.confidence ?? 0) + 0.12 || cvDecision.rotation !== decision2.rotation && cvDecision.confidence >= 0.62 && (decision2.confidence ?? 0) < 0.76);
|
|
603312
|
+
const selected = useCv ? cvDecision : decision2;
|
|
603313
|
+
const evidence = useCv ? cvDecision.reason : `${decision2.reason || boundedText(vision.llmContent || vision.output, 320)}${cvDecision ? `; cv=${cvDecision.reason}` : ""}`;
|
|
603314
|
+
const orientation = this.setCameraRotation(target, selected.rotation, "auto", evidence, selected.confidence);
|
|
602788
603315
|
return orientation ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation)}` : `Auto orientation for ${target} could not be persisted.`;
|
|
602789
603316
|
} finally {
|
|
602790
603317
|
this.autoOrientationInFlight.delete(target);
|
|
@@ -602853,22 +603380,25 @@ var init_live_sensors = __esm({
|
|
|
602853
603380
|
videoIntervalMs: options2.lowLatency === false ? this.config.videoIntervalMs : Math.min(this.config.videoIntervalMs, 1500),
|
|
602854
603381
|
audioIntervalMs: options2.lowLatency === false ? this.config.audioIntervalMs : Math.min(this.config.audioIntervalMs, 3e3)
|
|
602855
603382
|
});
|
|
602856
|
-
const source
|
|
602857
|
-
|
|
602858
|
-
|
|
602859
|
-
|
|
603383
|
+
for (const source of this.activeVideoSources()) {
|
|
603384
|
+
const current = this.getCameraOrientation(source);
|
|
603385
|
+
if (current?.source === "manual") continue;
|
|
603386
|
+
if (current && current.confidence !== void 0 && current.confidence >= 0.75 && Date.now() - current.updatedAt < 15 * 6e4) continue;
|
|
603387
|
+
void this.autoOrientCamera(source).then((message2) => {
|
|
603388
|
+
this.emitFeedbackThrottled(`orient:${source}`, {
|
|
602860
603389
|
kind: "orientation",
|
|
602861
603390
|
title: "Camera auto-orientation",
|
|
602862
603391
|
observedAt: Date.now(),
|
|
602863
603392
|
message: message2
|
|
602864
|
-
});
|
|
603393
|
+
}, 3e4);
|
|
603394
|
+
this.emitDashboard();
|
|
602865
603395
|
}).catch((err) => {
|
|
602866
|
-
this.
|
|
603396
|
+
this.emitFeedbackThrottled(`orient-error:${source}`, {
|
|
602867
603397
|
kind: "error",
|
|
602868
603398
|
title: "Camera auto-orientation failed",
|
|
602869
603399
|
observedAt: Date.now(),
|
|
602870
603400
|
message: err instanceof Error ? err.message : String(err)
|
|
602871
|
-
});
|
|
603401
|
+
}, 6e4);
|
|
602872
603402
|
});
|
|
602873
603403
|
}
|
|
602874
603404
|
void this.sampleVideoNow(true).catch((err) => {
|
|
@@ -602895,7 +603425,7 @@ var init_live_sensors = __esm({
|
|
|
602895
603425
|
});
|
|
602896
603426
|
return options2.agent ? "Live PFC run enabled: visual/audio feedback is active and significant events can launch background agent review." : "Live run enabled: visual/audio feedback is active and live context will update each turn.";
|
|
602897
603427
|
}
|
|
602898
|
-
async previewCamera(device = this.config.selectedCamera) {
|
|
603428
|
+
async previewCamera(device = this.config.selectedCamera, options2 = {}) {
|
|
602899
603429
|
const source = device || firstDeviceId(this.devices.video);
|
|
602900
603430
|
if (!source) return { ok: false, message: "No camera selected. Use /live to select a camera after device discovery." };
|
|
602901
603431
|
const orientation = this.getCameraOrientation(source);
|
|
@@ -602910,10 +603440,22 @@ var init_live_sensors = __esm({
|
|
|
602910
603440
|
if (!result.success) {
|
|
602911
603441
|
const error = result.error || result.output || "camera capture failed";
|
|
602912
603442
|
const observedAt2 = Date.now();
|
|
602913
|
-
|
|
603443
|
+
const errorView = {
|
|
602914
603444
|
updatedAt: observedAt2,
|
|
602915
603445
|
source,
|
|
602916
|
-
error
|
|
603446
|
+
error,
|
|
603447
|
+
rotation,
|
|
603448
|
+
orientation
|
|
603449
|
+
};
|
|
603450
|
+
if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = errorView;
|
|
603451
|
+
this.snapshot.cameras = {
|
|
603452
|
+
...this.snapshot.cameras ?? {},
|
|
603453
|
+
[source]: {
|
|
603454
|
+
...this.snapshot.cameras?.[source] ?? { source, updatedAt: observedAt2 },
|
|
603455
|
+
...errorView,
|
|
603456
|
+
updatedAt: observedAt2,
|
|
603457
|
+
source
|
|
603458
|
+
}
|
|
602917
603459
|
};
|
|
602918
603460
|
this.snapshot.events = mergeRecentById(this.snapshot.events, [{
|
|
602919
603461
|
id: stableEventId("camera_error", source, observedAt2, error.slice(0, 80)),
|
|
@@ -602924,29 +603466,44 @@ var init_live_sensors = __esm({
|
|
|
602924
603466
|
summary: error
|
|
602925
603467
|
}], 50);
|
|
602926
603468
|
this.persist();
|
|
602927
|
-
|
|
602928
|
-
|
|
602929
|
-
|
|
602930
|
-
|
|
602931
|
-
|
|
602932
|
-
|
|
603469
|
+
if (options2.emit !== false) {
|
|
603470
|
+
this.emitFeedbackThrottled(`camera-capture:${source}:${error.slice(0, 80)}`, {
|
|
603471
|
+
kind: "error",
|
|
603472
|
+
title: "Camera capture failed",
|
|
603473
|
+
observedAt: observedAt2,
|
|
603474
|
+
message: error
|
|
603475
|
+
}, 3e4);
|
|
603476
|
+
}
|
|
602933
603477
|
return { ok: false, message: error };
|
|
602934
603478
|
}
|
|
602935
603479
|
const framePath = extractSavedImagePath(result.output, this.repoRoot);
|
|
602936
603480
|
if (!framePath) return { ok: false, message: "Camera capture succeeded but no saved image path was returned." };
|
|
602937
603481
|
const displayPath = relative11(this.repoRoot, framePath).startsWith("..") ? framePath : relative11(this.repoRoot, framePath);
|
|
602938
|
-
const
|
|
603482
|
+
const previewWidth = Math.max(48, Math.min(96, (process.stdout.columns || 100) - 14));
|
|
603483
|
+
const preview = await buildImageAsciiPreview(framePath, { width: previewWidth });
|
|
602939
603484
|
const asciiContext = preview ? formatImageAsciiContext(preview, displayPath) : void 0;
|
|
602940
603485
|
const observedAt = Date.now();
|
|
602941
|
-
|
|
603486
|
+
const cameraView = {
|
|
602942
603487
|
updatedAt: observedAt,
|
|
602943
603488
|
source,
|
|
602944
603489
|
framePath,
|
|
602945
603490
|
displayPath,
|
|
603491
|
+
ascii: preview?.ascii,
|
|
603492
|
+
plainAscii: preview?.plainAscii,
|
|
603493
|
+
renderer: preview?.renderer,
|
|
602946
603494
|
asciiContext,
|
|
602947
603495
|
rotation,
|
|
602948
603496
|
orientation
|
|
602949
603497
|
};
|
|
603498
|
+
if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = cameraView;
|
|
603499
|
+
this.snapshot.cameras = {
|
|
603500
|
+
...this.snapshot.cameras ?? {},
|
|
603501
|
+
[source]: {
|
|
603502
|
+
...cameraView,
|
|
603503
|
+
summary: this.snapshot.cameras?.[source]?.summary,
|
|
603504
|
+
inference: this.snapshot.cameras?.[source]?.inference
|
|
603505
|
+
}
|
|
603506
|
+
};
|
|
602950
603507
|
this.snapshot.events = mergeRecentById(this.snapshot.events, [{
|
|
602951
603508
|
id: stableEventId("camera_frame", source, observedAt, framePath),
|
|
602952
603509
|
kind: "camera_frame",
|
|
@@ -602957,16 +603514,18 @@ var init_live_sensors = __esm({
|
|
|
602957
603514
|
evidencePath: framePath
|
|
602958
603515
|
}], 50);
|
|
602959
603516
|
this.persist();
|
|
602960
|
-
|
|
602961
|
-
|
|
602962
|
-
|
|
602963
|
-
|
|
602964
|
-
|
|
602965
|
-
|
|
602966
|
-
|
|
602967
|
-
|
|
602968
|
-
|
|
602969
|
-
|
|
603517
|
+
if (options2.emit !== false) {
|
|
603518
|
+
this.emitFeedback({
|
|
603519
|
+
kind: "camera-frame",
|
|
603520
|
+
title: "Live camera frame",
|
|
603521
|
+
observedAt,
|
|
603522
|
+
message: `Captured from ${source}${orientation ? ` (${formatCameraRotation(orientation.rotation)})` : ""}: ${framePath}`,
|
|
603523
|
+
imagePath: framePath,
|
|
603524
|
+
displayPath,
|
|
603525
|
+
ascii: preview?.ascii,
|
|
603526
|
+
renderer: preview?.renderer
|
|
603527
|
+
});
|
|
603528
|
+
}
|
|
602970
603529
|
return {
|
|
602971
603530
|
ok: true,
|
|
602972
603531
|
message: `Camera frame captured from ${source}: ${framePath}`,
|
|
@@ -602976,7 +603535,7 @@ var init_live_sensors = __esm({
|
|
|
602976
603535
|
renderer: preview?.renderer
|
|
602977
603536
|
};
|
|
602978
603537
|
}
|
|
602979
|
-
async recognizeFrameObjects(framePath, source, observedAt = Date.now()) {
|
|
603538
|
+
async recognizeFrameObjects(framePath, source, observedAt = Date.now(), options2 = {}) {
|
|
602980
603539
|
try {
|
|
602981
603540
|
const result = await new VisualMemoryTool().execute({
|
|
602982
603541
|
action: "recognize",
|
|
@@ -602996,109 +603555,124 @@ var init_live_sensors = __esm({
|
|
|
602996
603555
|
evidencePath: framePath
|
|
602997
603556
|
}], 50);
|
|
602998
603557
|
this.persist();
|
|
602999
|
-
|
|
603000
|
-
|
|
603001
|
-
|
|
603002
|
-
|
|
603003
|
-
|
|
603004
|
-
|
|
603558
|
+
if (options2.emit !== false) {
|
|
603559
|
+
this.emitFeedback({
|
|
603560
|
+
kind: "clip",
|
|
603561
|
+
title: result.success ? "CLIP visual-memory recognition" : "CLIP visual-memory unavailable",
|
|
603562
|
+
observedAt,
|
|
603563
|
+
message: summary
|
|
603564
|
+
});
|
|
603565
|
+
}
|
|
603005
603566
|
return summary;
|
|
603006
603567
|
} catch (err) {
|
|
603007
603568
|
const message2 = `CLIP/visual-memory failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
603008
|
-
|
|
603009
|
-
|
|
603010
|
-
|
|
603011
|
-
|
|
603012
|
-
|
|
603013
|
-
|
|
603569
|
+
if (options2.emit !== false) {
|
|
603570
|
+
this.emitFeedbackThrottled(`clip:${source}`, {
|
|
603571
|
+
kind: "error",
|
|
603572
|
+
title: "CLIP visual-memory failed",
|
|
603573
|
+
observedAt,
|
|
603574
|
+
message: message2
|
|
603575
|
+
}, 6e4);
|
|
603576
|
+
}
|
|
603014
603577
|
return message2;
|
|
603015
603578
|
}
|
|
603016
603579
|
}
|
|
603017
|
-
async
|
|
603018
|
-
|
|
603019
|
-
|
|
603020
|
-
|
|
603021
|
-
|
|
603022
|
-
|
|
603023
|
-
|
|
603024
|
-
const
|
|
603025
|
-
const
|
|
603026
|
-
|
|
603027
|
-
|
|
603028
|
-
|
|
603029
|
-
|
|
603030
|
-
|
|
603031
|
-
|
|
603032
|
-
|
|
603033
|
-
|
|
603034
|
-
|
|
603035
|
-
|
|
603036
|
-
|
|
603037
|
-
|
|
603038
|
-
|
|
603039
|
-
|
|
603040
|
-
|
|
603041
|
-
|
|
603042
|
-
|
|
603043
|
-
|
|
603044
|
-
|
|
603045
|
-
|
|
603046
|
-
|
|
603047
|
-
|
|
603048
|
-
|
|
603049
|
-
|
|
603050
|
-
|
|
603051
|
-
|
|
603052
|
-
|
|
603053
|
-
|
|
603054
|
-
|
|
603055
|
-
|
|
603056
|
-
|
|
603057
|
-
}
|
|
603058
|
-
|
|
603059
|
-
|
|
603060
|
-
|
|
603061
|
-
|
|
603580
|
+
async sampleSingleVideoNow(source, forceInfer) {
|
|
603581
|
+
const preview = await this.previewCamera(source, { emit: false });
|
|
603582
|
+
let inference = "";
|
|
603583
|
+
let clipSummary = "";
|
|
603584
|
+
const orientation = this.getCameraOrientation(source);
|
|
603585
|
+
const sampledAt = Date.now();
|
|
603586
|
+
if (forceInfer && source) {
|
|
603587
|
+
const loop = new LiveMediaLoopTool(this.repoRoot);
|
|
603588
|
+
const result = await loop.execute({
|
|
603589
|
+
action: "watch",
|
|
603590
|
+
source_kind: "camera",
|
|
603591
|
+
camera: source,
|
|
603592
|
+
duration_sec: 2.5,
|
|
603593
|
+
sample_interval_sec: 1,
|
|
603594
|
+
max_frames: 2,
|
|
603595
|
+
rotate_degrees: orientation?.rotation ?? 0,
|
|
603596
|
+
auto_bootstrap: true,
|
|
603597
|
+
audio_mode: "none",
|
|
603598
|
+
detect_objects: true,
|
|
603599
|
+
track_objects: true,
|
|
603600
|
+
crop_faces: true,
|
|
603601
|
+
recognize_faces: true
|
|
603602
|
+
});
|
|
603603
|
+
inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
|
|
603604
|
+
const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
|
|
603605
|
+
const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
|
|
603606
|
+
this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
|
|
603607
|
+
this.snapshot.people = mergeRecentById(this.snapshot.people, observations.people, 24);
|
|
603608
|
+
const existing = this.snapshot.cameras?.[source] ?? this.snapshot.video ?? { updatedAt: Date.now(), source };
|
|
603609
|
+
const camera = {
|
|
603610
|
+
...existing,
|
|
603611
|
+
updatedAt: sampledAt,
|
|
603612
|
+
source,
|
|
603613
|
+
inference,
|
|
603614
|
+
summary: summarizeInference(inference),
|
|
603615
|
+
rotation: orientation?.rotation ?? 0,
|
|
603616
|
+
orientation,
|
|
603617
|
+
...preview.ok ? { error: void 0 } : { error: preview.message }
|
|
603618
|
+
};
|
|
603619
|
+
this.snapshot.cameras = {
|
|
603620
|
+
...this.snapshot.cameras ?? {},
|
|
603621
|
+
[source]: camera
|
|
603622
|
+
};
|
|
603623
|
+
if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = camera;
|
|
603624
|
+
this.persist();
|
|
603625
|
+
if (!result.success) {
|
|
603626
|
+
this.emitFeedbackThrottled(`video-infer:${source}`, {
|
|
603627
|
+
kind: "error",
|
|
603628
|
+
title: "Live video inference failed",
|
|
603062
603629
|
observedAt: sampledAt,
|
|
603063
|
-
message: boundedText(inference,
|
|
603064
|
-
});
|
|
603065
|
-
|
|
603066
|
-
|
|
603067
|
-
|
|
603068
|
-
|
|
603069
|
-
|
|
603070
|
-
|
|
603071
|
-
|
|
603072
|
-
|
|
603073
|
-
|
|
603074
|
-
|
|
603075
|
-
|
|
603076
|
-
|
|
603077
|
-
|
|
603078
|
-
|
|
603079
|
-
}
|
|
603080
|
-
const unknownPeople = observations.people.filter((entry) => entry.status === "unknown");
|
|
603081
|
-
if (unknownPeople.length > 0) {
|
|
603082
|
-
this.maybeRequestAgentReview(
|
|
603083
|
-
`${unknownPeople.length} unknown individual(s) observed on live camera`,
|
|
603084
|
-
unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; ")
|
|
603085
|
-
);
|
|
603086
|
-
}
|
|
603087
|
-
const triggerEvents = observations.events.filter((entry) => entry.kind === "visual_trigger");
|
|
603088
|
-
if (triggerEvents.length > 0) {
|
|
603089
|
-
this.maybeRequestAgentReview(
|
|
603090
|
-
"Visual trigger hit in live camera stream",
|
|
603091
|
-
triggerEvents.map((entry) => entry.summary).join("; ")
|
|
603092
|
-
);
|
|
603093
|
-
}
|
|
603630
|
+
message: boundedText(inference, 1200)
|
|
603631
|
+
}, 45e3);
|
|
603632
|
+
}
|
|
603633
|
+
const unknownPeople = observations.people.filter((entry) => entry.status === "unknown");
|
|
603634
|
+
if (unknownPeople.length > 0) {
|
|
603635
|
+
this.maybeRequestAgentReview(
|
|
603636
|
+
`${unknownPeople.length} unknown individual(s) observed on live camera ${source}`,
|
|
603637
|
+
unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; ")
|
|
603638
|
+
);
|
|
603639
|
+
}
|
|
603640
|
+
const triggerEvents = observations.events.filter((entry) => entry.kind === "visual_trigger");
|
|
603641
|
+
if (triggerEvents.length > 0) {
|
|
603642
|
+
this.maybeRequestAgentReview(
|
|
603643
|
+
`Visual trigger hit in live camera stream ${source}`,
|
|
603644
|
+
triggerEvents.map((entry) => entry.summary).join("; ")
|
|
603645
|
+
);
|
|
603094
603646
|
}
|
|
603095
|
-
|
|
603096
|
-
|
|
603647
|
+
}
|
|
603648
|
+
if (this.config.clipEnabled && preview.ok && preview.framePath && source) {
|
|
603649
|
+
clipSummary = await this.recognizeFrameObjects(preview.framePath, source, sampledAt, { emit: false });
|
|
603650
|
+
const current = this.snapshot.cameras?.[source];
|
|
603651
|
+
if (current) {
|
|
603652
|
+
current.summary = [current.summary, clipSummary ? `clip: ${trimOneLine(clipSummary, 160)}` : ""].filter(Boolean).join(" | ");
|
|
603653
|
+
this.snapshot.cameras = { ...this.snapshot.cameras ?? {}, [source]: current };
|
|
603654
|
+
if (this.snapshot.video?.source === source) this.snapshot.video = current;
|
|
603655
|
+
this.persist();
|
|
603097
603656
|
}
|
|
603098
|
-
|
|
603657
|
+
}
|
|
603658
|
+
return [preview.message, inference ? `
|
|
603099
603659
|
${inference}` : "", clipSummary ? `
|
|
603100
603660
|
CLIP visual memory:
|
|
603101
603661
|
${clipSummary}` : ""].filter(Boolean).join("\n");
|
|
603662
|
+
}
|
|
603663
|
+
async sampleVideoNow(forceInfer = this.config.inferEnabled) {
|
|
603664
|
+
if (this.videoInFlight) return "Video sampler is already running.";
|
|
603665
|
+
this.videoInFlight = true;
|
|
603666
|
+
try {
|
|
603667
|
+
const sources = this.activeVideoSources();
|
|
603668
|
+
if (sources.length === 0) return "No camera selected. Use /live refresh after connecting hardware.";
|
|
603669
|
+
const outputs = [];
|
|
603670
|
+
for (const source of sources) {
|
|
603671
|
+
outputs.push(await this.sampleSingleVideoNow(source, forceInfer));
|
|
603672
|
+
}
|
|
603673
|
+
this.emitDashboard();
|
|
603674
|
+
return outputs.map((output, index) => `## Camera ${sources[index]}
|
|
603675
|
+
${output}`).join("\n\n");
|
|
603102
603676
|
} finally {
|
|
603103
603677
|
this.videoInFlight = false;
|
|
603104
603678
|
}
|
|
@@ -603113,10 +603687,10 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
|
|
|
603113
603687
|
const recordingPath = join124(outputDir2, `audio-${Date.now()}.wav`);
|
|
603114
603688
|
let transcript = "";
|
|
603115
603689
|
let analysis = "";
|
|
603116
|
-
|
|
603117
|
-
|
|
603118
|
-
|
|
603119
|
-
const message2 = `Audio capture failed from ${input}: ${
|
|
603690
|
+
const audioErrors = [];
|
|
603691
|
+
const capture = await recordAudioSampleWithRecovery(input, this.devices.audioInputs, 3, recordingPath);
|
|
603692
|
+
if (!capture.ok) {
|
|
603693
|
+
const message2 = `Audio capture failed from ${input}; tried fallbacks: ${capture.errors.slice(0, 5).join(" | ")}`;
|
|
603120
603694
|
this.snapshot.audio = {
|
|
603121
603695
|
updatedAt: Date.now(),
|
|
603122
603696
|
input,
|
|
@@ -603124,20 +603698,28 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
|
|
|
603124
603698
|
error: message2
|
|
603125
603699
|
};
|
|
603126
603700
|
this.persist();
|
|
603127
|
-
this.
|
|
603701
|
+
this.emitFeedbackThrottled(`audio-capture:${input}`, {
|
|
603128
603702
|
kind: "error",
|
|
603129
603703
|
title: "Live audio capture failed",
|
|
603130
603704
|
observedAt: this.snapshot.audio.updatedAt,
|
|
603131
603705
|
message: message2
|
|
603132
|
-
});
|
|
603706
|
+
}, 6e4);
|
|
603707
|
+
this.emitDashboard();
|
|
603133
603708
|
return message2;
|
|
603134
603709
|
}
|
|
603710
|
+
if (capture.device !== this.config.selectedAudioInput) {
|
|
603711
|
+
this.config.selectedAudioInput = capture.device;
|
|
603712
|
+
}
|
|
603135
603713
|
if (this.config.asrEnabled) {
|
|
603136
603714
|
try {
|
|
603137
603715
|
const asr = await new TranscribeFileTool(this.repoRoot).execute({ path: recordingPath, model: "tiny" });
|
|
603138
|
-
|
|
603716
|
+
if (asr.success) {
|
|
603717
|
+
transcript = cleanPreview(typeof asr.llmContent === "string" ? asr.llmContent : "", 1600);
|
|
603718
|
+
} else {
|
|
603719
|
+
audioErrors.push(`ASR unavailable: ${asr.error || asr.output || "unknown error"}`);
|
|
603720
|
+
}
|
|
603139
603721
|
} catch (err) {
|
|
603140
|
-
|
|
603722
|
+
audioErrors.push(`ASR unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
603141
603723
|
}
|
|
603142
603724
|
}
|
|
603143
603725
|
if (this.config.audioAnalysisEnabled) {
|
|
@@ -603151,27 +603733,29 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
|
|
|
603151
603733
|
}
|
|
603152
603734
|
try {
|
|
603153
603735
|
const vad = await analyzer.execute({ action: "vad", file: recordingPath });
|
|
603154
|
-
|
|
603736
|
+
if (vad.success) parts.push(cleanPreview(vad.output, 700));
|
|
603737
|
+
else audioErrors.push(`VAD unavailable: ${vad.error || vad.output || "unknown error"}`);
|
|
603155
603738
|
} catch (err) {
|
|
603156
|
-
|
|
603739
|
+
audioErrors.push(`VAD unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
603157
603740
|
}
|
|
603158
603741
|
analysis = parts.filter(Boolean).join("\n");
|
|
603159
603742
|
}
|
|
603160
603743
|
this.snapshot.audio = {
|
|
603161
603744
|
updatedAt: Date.now(),
|
|
603162
|
-
input,
|
|
603745
|
+
input: capture.device,
|
|
603163
603746
|
output: this.config.selectedAudioOutput,
|
|
603164
603747
|
recordingPath,
|
|
603165
603748
|
transcript,
|
|
603166
|
-
analysis
|
|
603749
|
+
analysis,
|
|
603750
|
+
error: audioErrors.length > 0 ? audioErrors.join("\n") : void 0
|
|
603167
603751
|
};
|
|
603168
603752
|
const audioEvents = [];
|
|
603169
603753
|
if (transcript) {
|
|
603170
603754
|
audioEvents.push({
|
|
603171
|
-
id: stableEventId("audio_transcript",
|
|
603755
|
+
id: stableEventId("audio_transcript", capture.device, this.snapshot.audio.updatedAt, recordingPath),
|
|
603172
603756
|
kind: "audio_transcript",
|
|
603173
603757
|
observedAt: this.snapshot.audio.updatedAt,
|
|
603174
|
-
source:
|
|
603758
|
+
source: capture.device,
|
|
603175
603759
|
title: "Live ASR transcript",
|
|
603176
603760
|
summary: transcript,
|
|
603177
603761
|
evidencePath: recordingPath
|
|
@@ -603179,10 +603763,10 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
|
|
|
603179
603763
|
}
|
|
603180
603764
|
if (analysis) {
|
|
603181
603765
|
audioEvents.push({
|
|
603182
|
-
id: stableEventId("audio_sound",
|
|
603766
|
+
id: stableEventId("audio_sound", capture.device, this.snapshot.audio.updatedAt, recordingPath),
|
|
603183
603767
|
kind: "audio_sound",
|
|
603184
603768
|
observedAt: this.snapshot.audio.updatedAt,
|
|
603185
|
-
source:
|
|
603769
|
+
source: capture.device,
|
|
603186
603770
|
title: "Live sound analysis",
|
|
603187
603771
|
summary: analysis,
|
|
603188
603772
|
evidencePath: recordingPath
|
|
@@ -603190,21 +603774,9 @@ ${clipSummary}` : ""].filter(Boolean).join("\n");
|
|
|
603190
603774
|
}
|
|
603191
603775
|
this.snapshot.events = mergeRecentById(this.snapshot.events, audioEvents, 50);
|
|
603192
603776
|
this.persist();
|
|
603193
|
-
|
|
603194
|
-
this.emitFeedback({
|
|
603195
|
-
kind: "audio",
|
|
603196
|
-
title: "Live audio sample",
|
|
603197
|
-
observedAt: this.snapshot.audio.updatedAt,
|
|
603198
|
-
message: [
|
|
603199
|
-
transcript ? `ASR:
|
|
603200
|
-
${boundedText(transcript, 700)}` : "",
|
|
603201
|
-
analysis ? `Sounds:
|
|
603202
|
-
${boundedText(analysis, 900)}` : ""
|
|
603203
|
-
].filter(Boolean).join("\n\n")
|
|
603204
|
-
});
|
|
603205
|
-
}
|
|
603777
|
+
this.emitDashboard();
|
|
603206
603778
|
return [
|
|
603207
|
-
`Audio sample captured from ${input}: ${recordingPath}`,
|
|
603779
|
+
`Audio sample captured from ${capture.device}${capture.device !== input ? ` (recovered from ${input})` : ""}: ${recordingPath}`,
|
|
603208
603780
|
transcript ? `
|
|
603209
603781
|
ASR:
|
|
603210
603782
|
${transcript}` : "",
|
|
@@ -603256,6 +603828,21 @@ ${analysis}` : ""
|
|
|
603256
603828
|
} catch {
|
|
603257
603829
|
}
|
|
603258
603830
|
}
|
|
603831
|
+
emitFeedbackThrottled(key, feedback, minIntervalMs) {
|
|
603832
|
+
const now2 = Date.now();
|
|
603833
|
+
const last2 = this.lastFeedbackAt.get(key) ?? 0;
|
|
603834
|
+
if (now2 - last2 < minIntervalMs) return;
|
|
603835
|
+
this.lastFeedbackAt.set(key, now2);
|
|
603836
|
+
this.emitFeedback(feedback);
|
|
603837
|
+
}
|
|
603838
|
+
emitDashboard() {
|
|
603839
|
+
this.emitFeedback({
|
|
603840
|
+
kind: "dashboard",
|
|
603841
|
+
title: "Live audio/video monitor",
|
|
603842
|
+
observedAt: Date.now(),
|
|
603843
|
+
message: "live dashboard refresh"
|
|
603844
|
+
});
|
|
603845
|
+
}
|
|
603259
603846
|
maybeRequestAgentReview(reason, evidence) {
|
|
603260
603847
|
if (!this.agentActionEnabled || !this.agentActionSink) return;
|
|
603261
603848
|
const now2 = Date.now();
|
|
@@ -654970,14 +655557,45 @@ function renderLivePreview(preview) {
|
|
|
654970
655557
|
renderWarning("Camera preview was captured, but ASCII rendering was unavailable.");
|
|
654971
655558
|
}
|
|
654972
655559
|
}
|
|
654973
|
-
function
|
|
655560
|
+
function renderLiveDashboard(ctx3, manager) {
|
|
655561
|
+
const host = ctx3.liveDynamicBlockHost;
|
|
655562
|
+
if (!host) {
|
|
655563
|
+
renderInfo(formatLiveDashboardFromSnapshot(manager.getSnapshot(), { width: 100 }).join("\n"));
|
|
655564
|
+
return;
|
|
655565
|
+
}
|
|
655566
|
+
const key = ctx3.repoRoot;
|
|
655567
|
+
let state = liveDashboardBlocks.get(key);
|
|
655568
|
+
if (!state || state.host !== host) {
|
|
655569
|
+
const id2 = `live-dashboard-${Math.random().toString(36).slice(2, 9)}`;
|
|
655570
|
+
state = { host, id: id2, manager };
|
|
655571
|
+
liveDashboardBlocks.set(key, state);
|
|
655572
|
+
host.registerDynamicBlock(
|
|
655573
|
+
id2,
|
|
655574
|
+
(width) => formatLiveDashboardFromSnapshot(manager.getSnapshot(), { width })
|
|
655575
|
+
);
|
|
655576
|
+
host.appendDynamicBlock(id2);
|
|
655577
|
+
return;
|
|
655578
|
+
}
|
|
655579
|
+
state.manager = manager;
|
|
655580
|
+
host.refreshDynamicBlocks?.();
|
|
655581
|
+
}
|
|
655582
|
+
function renderLiveFeedback(ctx3, manager, feedback) {
|
|
655583
|
+
if (feedback.kind === "dashboard") {
|
|
655584
|
+
renderLiveDashboard(ctx3, manager);
|
|
655585
|
+
return;
|
|
655586
|
+
}
|
|
654974
655587
|
const stamp = new Date(feedback.observedAt).toISOString();
|
|
655588
|
+
const explicitImagePreview = (feedback.kind === "camera-frame" || feedback.kind === "face-crop") && feedback.ascii && feedback.displayPath && feedback.renderer;
|
|
655589
|
+
if (ctx3.liveDynamicBlockHost && !explicitImagePreview) {
|
|
655590
|
+
renderLiveDashboard(ctx3, manager);
|
|
655591
|
+
return;
|
|
655592
|
+
}
|
|
654975
655593
|
if (feedback.kind === "error") {
|
|
654976
655594
|
renderWarning(`[live ${stamp}] ${feedback.title}
|
|
654977
655595
|
${feedback.message}`);
|
|
654978
655596
|
return;
|
|
654979
655597
|
}
|
|
654980
|
-
if (
|
|
655598
|
+
if (explicitImagePreview && feedback.displayPath && feedback.ascii && feedback.renderer) {
|
|
654981
655599
|
renderInfo(`[live ${stamp}] ${feedback.message}`);
|
|
654982
655600
|
renderImageAsciiPreview(feedback.title, feedback.displayPath, feedback.ascii, feedback.renderer);
|
|
654983
655601
|
return;
|
|
@@ -654987,7 +655605,7 @@ ${feedback.message}`);
|
|
|
654987
655605
|
}
|
|
654988
655606
|
function attachLiveSinks(ctx3, manager, agentEnabled = false) {
|
|
654989
655607
|
manager.setStatusSink(ctx3.setLiveMediaStatus);
|
|
654990
|
-
manager.setFeedbackSink((feedback) => renderLiveFeedback(feedback));
|
|
655608
|
+
manager.setFeedbackSink((feedback) => renderLiveFeedback(ctx3, manager, feedback));
|
|
654991
655609
|
if (agentEnabled && ctx3.startBackgroundPrompt) {
|
|
654992
655610
|
manager.setAgentActionSink((prompt) => {
|
|
654993
655611
|
const id2 = ctx3.startBackgroundPrompt?.(prompt);
|
|
@@ -660485,7 +661103,7 @@ async function showExposeDashboard(gateway, rl, ctx3) {
|
|
|
660485
661103
|
renderInfo("Expose gateway stopped.");
|
|
660486
661104
|
}
|
|
660487
661105
|
}
|
|
660488
|
-
var OMNIUS_PINNED_DEPENDENCY_SPECS, NEXUS_DIRECTORY_ORIGIN, NEXUS_SPONSORS_URL, _sponsorHeartbeatTimer, _lastRegisteredSponsorPayload, __COMMAND_REGISTRY, DASH_INTERNAL, localGpuMetricsCache, localGpuMetricsProbeAt, localGpuMetricsProbeInFlight;
|
|
661106
|
+
var OMNIUS_PINNED_DEPENDENCY_SPECS, NEXUS_DIRECTORY_ORIGIN, NEXUS_SPONSORS_URL, _sponsorHeartbeatTimer, _lastRegisteredSponsorPayload, __COMMAND_REGISTRY, liveDashboardBlocks, DASH_INTERNAL, localGpuMetricsCache, localGpuMetricsProbeAt, localGpuMetricsProbeInFlight;
|
|
660489
661107
|
var init_commands = __esm({
|
|
660490
661108
|
"packages/cli/src/tui/commands.ts"() {
|
|
660491
661109
|
"use strict";
|
|
@@ -660803,6 +661421,7 @@ var init_commands = __esm({
|
|
|
660803
661421
|
}
|
|
660804
661422
|
});
|
|
660805
661423
|
})();
|
|
661424
|
+
liveDashboardBlocks = /* @__PURE__ */ new Map();
|
|
660806
661425
|
DASH_INTERNAL = /* @__PURE__ */ new Set(["system_metrics", "__list_capabilities"]);
|
|
660807
661426
|
localGpuMetricsCache = null;
|
|
660808
661427
|
localGpuMetricsProbeAt = 0;
|
|
@@ -733124,6 +733743,11 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
733124
733743
|
setLiveMediaStatus(status) {
|
|
733125
733744
|
statusBar.setLiveMediaStatus(status);
|
|
733126
733745
|
},
|
|
733746
|
+
liveDynamicBlockHost: {
|
|
733747
|
+
registerDynamicBlock: (id2, render2) => statusBar.registerDynamicBlock(id2, render2),
|
|
733748
|
+
appendDynamicBlock: (id2) => statusBar.appendDynamicBlock(id2),
|
|
733749
|
+
refreshDynamicBlocks: () => statusBar.refreshDynamicBlocks()
|
|
733750
|
+
},
|
|
733127
733751
|
// Voice call session — standalone cloudflared tunnel for /call
|
|
733128
733752
|
// Each connecting WebSocket client gets a dedicated CallSubAgent.
|
|
733129
733753
|
// Admin callers (matching session key) get full tool access; public callers get read-only.
|
|
@@ -733510,6 +734134,8 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
733510
734134
|
} else {
|
|
733511
734135
|
parts.push("active: no");
|
|
733512
734136
|
}
|
|
734137
|
+
const liveContext = formatLiveSensorContext(repoRoot);
|
|
734138
|
+
if (liveContext) parts.push(liveContext);
|
|
733513
734139
|
return parts.join("\n");
|
|
733514
734140
|
}
|
|
733515
734141
|
},
|