omnius 1.0.434 → 1.0.436
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +916 -125
- package/dist/scripts/live-whisper.py +101 -46
- package/dist/scripts/transcribe-file.py +18 -8
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -906,9 +906,9 @@ var init_model_broker = __esm({
|
|
|
906
906
|
// VRAM total/free comes from system-metrics; broker computes its own snapshot
|
|
907
907
|
]);
|
|
908
908
|
const snapshot = this.buildSnapshot();
|
|
909
|
+
await this.checkPressure(snapshot);
|
|
909
910
|
this._lastSnapshot = snapshot;
|
|
910
911
|
this.emit("snapshot", snapshot);
|
|
911
|
-
this.checkPressure(snapshot);
|
|
912
912
|
return snapshot;
|
|
913
913
|
}
|
|
914
914
|
/** Best-known current snapshot. */
|
|
@@ -9286,6 +9286,74 @@ function uniqueStrings(values) {
|
|
|
9286
9286
|
}
|
|
9287
9287
|
return out;
|
|
9288
9288
|
}
|
|
9289
|
+
function optionalPositiveInteger(value2) {
|
|
9290
|
+
if (value2 === void 0 || value2 === null || value2 === "")
|
|
9291
|
+
return void 0;
|
|
9292
|
+
const parsed = typeof value2 === "number" ? value2 : Number(String(value2).trim());
|
|
9293
|
+
if (!Number.isFinite(parsed) || parsed <= 0)
|
|
9294
|
+
return void 0;
|
|
9295
|
+
return Math.round(parsed);
|
|
9296
|
+
}
|
|
9297
|
+
function normalizeCameraPresetFps(value2) {
|
|
9298
|
+
const parsed = optionalPositiveInteger(typeof value2 === "string" ? value2.replace(/\s*fps$/i, "") : value2);
|
|
9299
|
+
return parsed === void 0 ? void 0 : Math.max(1, Math.min(30, parsed));
|
|
9300
|
+
}
|
|
9301
|
+
function parseCameraPresetResolution(value2) {
|
|
9302
|
+
const raw = String(value2 ?? "").trim().toLowerCase();
|
|
9303
|
+
if (!raw || raw === "native" || raw === "auto")
|
|
9304
|
+
return null;
|
|
9305
|
+
const alias = {
|
|
9306
|
+
"480": "640x480",
|
|
9307
|
+
"480p": "640x480",
|
|
9308
|
+
"720": "1280x720",
|
|
9309
|
+
"720p": "1280x720",
|
|
9310
|
+
"1080": "1920x1080",
|
|
9311
|
+
"1080p": "1920x1080",
|
|
9312
|
+
"1440": "2560x1440",
|
|
9313
|
+
"1440p": "2560x1440",
|
|
9314
|
+
"2k": "2560x1440",
|
|
9315
|
+
"2160": "3840x2160",
|
|
9316
|
+
"2160p": "3840x2160",
|
|
9317
|
+
"4k": "3840x2160",
|
|
9318
|
+
"uhd": "3840x2160"
|
|
9319
|
+
};
|
|
9320
|
+
const normalized = alias[raw] ?? raw;
|
|
9321
|
+
const match = normalized.match(/^(\d{3,5})x(\d{3,5})$/);
|
|
9322
|
+
if (!match)
|
|
9323
|
+
return null;
|
|
9324
|
+
const width = Number(match[1]);
|
|
9325
|
+
const height = Number(match[2]);
|
|
9326
|
+
if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0)
|
|
9327
|
+
return null;
|
|
9328
|
+
return { width, height, label: `${width}x${height}` };
|
|
9329
|
+
}
|
|
9330
|
+
function loadLiveCameraPreset(requestedDevice) {
|
|
9331
|
+
const repoRoot = process.env["OMNIUS_REPO_ROOT"] || process.cwd();
|
|
9332
|
+
const configPath2 = join13(repoRoot, ".omnius", "live", "config.json");
|
|
9333
|
+
try {
|
|
9334
|
+
const parsed = JSON.parse(readFileSync13(configPath2, "utf8"));
|
|
9335
|
+
const streams = parsed["cameraStreams"] && typeof parsed["cameraStreams"] === "object" ? parsed["cameraStreams"] : {};
|
|
9336
|
+
const orientations = parsed["cameraOrientation"] && typeof parsed["cameraOrientation"] === "object" ? parsed["cameraOrientation"] : {};
|
|
9337
|
+
const selected = typeof parsed["selectedCamera"] === "string" ? parsed["selectedCamera"] : void 0;
|
|
9338
|
+
const device = requestedDevice || selected;
|
|
9339
|
+
if (!device)
|
|
9340
|
+
return {};
|
|
9341
|
+
const stream = streams[device] ?? {};
|
|
9342
|
+
const orientation = orientations[device] ?? {};
|
|
9343
|
+
const resolution = parseCameraPresetResolution(stream["resolution"]);
|
|
9344
|
+
const rotation = orientation["rotation"] !== void 0 ? normalizeCameraRotationDegrees(orientation["rotation"]) : void 0;
|
|
9345
|
+
return {
|
|
9346
|
+
device,
|
|
9347
|
+
width: resolution?.width,
|
|
9348
|
+
height: resolution?.height,
|
|
9349
|
+
resolution: resolution?.label ?? String(stream["resolution"] ?? "native"),
|
|
9350
|
+
rotation,
|
|
9351
|
+
fps: normalizeCameraPresetFps(stream["fps"] ?? stream["framerate"])
|
|
9352
|
+
};
|
|
9353
|
+
} catch {
|
|
9354
|
+
return {};
|
|
9355
|
+
}
|
|
9356
|
+
}
|
|
9289
9357
|
function buildV4l2CaptureAttempts(width, height, formats) {
|
|
9290
9358
|
const requested = width > 0 && height > 0 ? `${width}x${height}` : void 0;
|
|
9291
9359
|
const sizeArea = (size) => {
|
|
@@ -9465,6 +9533,14 @@ var init_camera_capture = __esm({
|
|
|
9465
9533
|
type: "number",
|
|
9466
9534
|
description: "Capture height in pixels (default: camera native format)"
|
|
9467
9535
|
},
|
|
9536
|
+
fps: {
|
|
9537
|
+
type: "number",
|
|
9538
|
+
description: "Requested capture frame rate for v4l2 devices. Defaults to the shared /live or /camera preset when available."
|
|
9539
|
+
},
|
|
9540
|
+
framerate: {
|
|
9541
|
+
type: "number",
|
|
9542
|
+
description: "Alias for fps."
|
|
9543
|
+
},
|
|
9468
9544
|
rotate_degrees: {
|
|
9469
9545
|
type: "number",
|
|
9470
9546
|
enum: [0, 90, 180, 270],
|
|
@@ -9564,20 +9640,33 @@ ${formatRouterPlan(plan)}`,
|
|
|
9564
9640
|
// Capture frame — auto-selects v4l2 or QooCam
|
|
9565
9641
|
// =========================================================================
|
|
9566
9642
|
async captureFrame(args, start2) {
|
|
9567
|
-
const
|
|
9643
|
+
const requestedDevice = args["device"] || "";
|
|
9644
|
+
const preset = loadLiveCameraPreset(requestedDevice);
|
|
9645
|
+
const device = requestedDevice || preset.device || "";
|
|
9646
|
+
const effectiveArgs = { ...args, device };
|
|
9647
|
+
if (effectiveArgs["width"] === void 0 && preset.width !== void 0)
|
|
9648
|
+
effectiveArgs["width"] = preset.width;
|
|
9649
|
+
if (effectiveArgs["height"] === void 0 && preset.height !== void 0)
|
|
9650
|
+
effectiveArgs["height"] = preset.height;
|
|
9651
|
+
if (effectiveArgs["rotate_degrees"] === void 0 && effectiveArgs["rotation_degrees"] === void 0 && effectiveArgs["rotate"] === void 0 && preset.rotation !== void 0) {
|
|
9652
|
+
effectiveArgs["rotate_degrees"] = preset.rotation;
|
|
9653
|
+
}
|
|
9654
|
+
if (effectiveArgs["fps"] === void 0 && effectiveArgs["framerate"] === void 0 && preset.fps !== void 0) {
|
|
9655
|
+
effectiveArgs["framerate"] = preset.fps;
|
|
9656
|
+
}
|
|
9568
9657
|
const outputPath3 = args["output_path"];
|
|
9569
9658
|
if (device === "qoocam" || device.startsWith("qoocam")) {
|
|
9570
|
-
return this.captureQooCam(
|
|
9659
|
+
return this.captureQooCam(effectiveArgs, start2);
|
|
9571
9660
|
}
|
|
9572
9661
|
const v4l2Device = await this.resolveV4l2Device(device);
|
|
9573
9662
|
if (!v4l2Device) {
|
|
9574
9663
|
const qoocam = await this.discoverQooCam();
|
|
9575
9664
|
if (qoocam.found) {
|
|
9576
|
-
return this.captureQooCam(
|
|
9665
|
+
return this.captureQooCam(effectiveArgs, start2);
|
|
9577
9666
|
}
|
|
9578
9667
|
return { success: false, output: "", error: "No camera found. Connect a USB camera or enable QooCam WiFi hotspot.", durationMs: performance.now() - start2 };
|
|
9579
9668
|
}
|
|
9580
|
-
return this.captureV4l2(v4l2Device,
|
|
9669
|
+
return this.captureV4l2(v4l2Device, effectiveArgs, start2);
|
|
9581
9670
|
}
|
|
9582
9671
|
// =========================================================================
|
|
9583
9672
|
// Camera info
|
|
@@ -9687,10 +9776,11 @@ ${formatRouterPlan(plan)}`,
|
|
|
9687
9776
|
return rawDevice;
|
|
9688
9777
|
}
|
|
9689
9778
|
async captureV4l2(device, args, start2) {
|
|
9690
|
-
const width = args["width"]
|
|
9691
|
-
const height = args["height"]
|
|
9779
|
+
const width = optionalPositiveInteger(args["width"]) ?? 0;
|
|
9780
|
+
const height = optionalPositiveInteger(args["height"]) ?? 0;
|
|
9692
9781
|
const outputPath3 = args["output_path"];
|
|
9693
9782
|
const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
|
|
9783
|
+
const fps = normalizeCameraPresetFps(args["framerate"] ?? args["fps"]);
|
|
9694
9784
|
const captureDir = join13(tmpdir2(), "omnius-camera");
|
|
9695
9785
|
if (!existsSync15(captureDir))
|
|
9696
9786
|
mkdirSync9(captureDir, { recursive: true });
|
|
@@ -9722,7 +9812,7 @@ ${formatRouterPlan(plan)}`,
|
|
|
9722
9812
|
unlinkSync2(tempFile);
|
|
9723
9813
|
} catch {
|
|
9724
9814
|
}
|
|
9725
|
-
const argv = this.ffmpegCaptureArgs(device, tempFile, attempt);
|
|
9815
|
+
const argv = this.ffmpegCaptureArgs(device, tempFile, attempt, fps);
|
|
9726
9816
|
try {
|
|
9727
9817
|
await runProcessBuffer(ffmpegBin(), argv, { timeout: 15e3 });
|
|
9728
9818
|
if (existsSync15(tempFile) && readFileSync13(tempFile).length > 0) {
|
|
@@ -9765,8 +9855,10 @@ ${formatRouterPlan(plan)}`,
|
|
|
9765
9855
|
Recovered with capture profile: ${captureState.successfulAttempt?.label ?? "unknown"}. Last failed attempt: ${errors[errors.length - 1].slice(0, 260)}` : "";
|
|
9766
9856
|
const rotationNote = rotation !== 0 ? `
|
|
9767
9857
|
Applied camera rotation correction: ${rotation} degrees clockwise.` : "";
|
|
9858
|
+
const fpsNote = fps ? `
|
|
9859
|
+
Applied camera capture FPS preset: ${fps} fps.` : "";
|
|
9768
9860
|
const result = this.returnCapturedFile(tempFile, captureState.successfulAttempt?.videoSize ?? (width && height ? `${width}x${height}` : "native"), device, outputPath3, start2);
|
|
9769
|
-
const notes = `${firstSuccess}${rotationNote}`;
|
|
9861
|
+
const notes = `${firstSuccess}${rotationNote}${fpsNote}`;
|
|
9770
9862
|
return result.success && notes ? { ...result, output: result.output + notes, llmContent: result.llmContent ? result.llmContent + notes : result.llmContent } : result;
|
|
9771
9863
|
}
|
|
9772
9864
|
const allErrors = errors.join("\n\n");
|
|
@@ -9841,8 +9933,10 @@ ${allErrors.slice(0, 900)}`, durationMs: performance.now() - start2 };
|
|
|
9841
9933
|
return [];
|
|
9842
9934
|
}
|
|
9843
9935
|
}
|
|
9844
|
-
ffmpegCaptureArgs(device, outputPath3, attempt) {
|
|
9936
|
+
ffmpegCaptureArgs(device, outputPath3, attempt, fps) {
|
|
9845
9937
|
const argv = ["-hide_banner", "-loglevel", "error", "-f", "v4l2", "-thread_queue_size", "4"];
|
|
9938
|
+
if (fps)
|
|
9939
|
+
argv.push("-framerate", String(fps));
|
|
9846
9940
|
if (attempt.inputFormat)
|
|
9847
9941
|
argv.push("-input_format", attempt.inputFormat);
|
|
9848
9942
|
if (attempt.videoSize)
|
|
@@ -42773,6 +42867,8 @@ import { join as join41, basename as basename7, extname as extname4, resolve as
|
|
|
42773
42867
|
import { homedir as homedir11, tmpdir as tmpdir6 } from "node:os";
|
|
42774
42868
|
function transcriptionPythonEnv(extra = {}) {
|
|
42775
42869
|
const env2 = { ...process.env, ...extra };
|
|
42870
|
+
if (!env2["OMNIUS_ASR_DEVICE"])
|
|
42871
|
+
env2["OMNIUS_ASR_DEVICE"] = "cuda";
|
|
42776
42872
|
applyMediaCudaDeviceFilterToEnv(env2, "asr");
|
|
42777
42873
|
return env2;
|
|
42778
42874
|
}
|
|
@@ -43285,17 +43381,18 @@ audio_file = ${JSON.stringify(filePath)}
|
|
|
43285
43381
|
model_name = ${JSON.stringify(model)}
|
|
43286
43382
|
try:
|
|
43287
43383
|
import whisper
|
|
43288
|
-
device = os.environ.get("OMNIUS_ASR_DEVICE", "
|
|
43384
|
+
device = os.environ.get("OMNIUS_ASR_DEVICE", "cuda").strip().lower() or "cuda"
|
|
43385
|
+
allow_cpu = os.environ.get("OMNIUS_ASR_ALLOW_CPU", "").strip().lower() in ("1", "true", "yes", "on")
|
|
43289
43386
|
if device == "auto":
|
|
43290
|
-
device = "
|
|
43291
|
-
|
|
43292
|
-
|
|
43293
|
-
|
|
43294
|
-
|
|
43295
|
-
|
|
43296
|
-
|
|
43387
|
+
device = "cuda"
|
|
43388
|
+
if device.startswith("cuda"):
|
|
43389
|
+
import torch
|
|
43390
|
+
if not torch.cuda.is_available() or torch.cuda.device_count() <= 0:
|
|
43391
|
+
raise RuntimeError(f"CUDA-only ASR requested but torch cannot use CUDA (torch.version.cuda={getattr(torch.version, 'cuda', None)}, device_count={torch.cuda.device_count()}). Install a CUDA-enabled PyTorch build for this device, or set OMNIUS_ASR_ALLOW_CPU=1 only for emergency fallback.")
|
|
43392
|
+
elif device == "cpu" and not allow_cpu:
|
|
43393
|
+
raise RuntimeError("CPU ASR refused. Set OMNIUS_ASR_ALLOW_CPU=1 only for explicit emergency fallback.")
|
|
43297
43394
|
m = whisper.load_model(model_name, device=device)
|
|
43298
|
-
result = m.transcribe(audio_file, fp16=(
|
|
43395
|
+
result = m.transcribe(audio_file, fp16=device.startswith("cuda"), condition_on_previous_text=False)
|
|
43299
43396
|
segments = []
|
|
43300
43397
|
for seg in result.get("segments", []) or []:
|
|
43301
43398
|
segments.append({"start": float(seg.get("start", 0)), "end": float(seg.get("end", seg.get("start", 0))), "text": str(seg.get("text", "")).strip()})
|
|
@@ -603722,6 +603819,10 @@ function asrConsensusEnabled() {
|
|
|
603722
603819
|
const consensusEnv = String(process.env["OMNIUS_ASR_CONSENSUS"] ?? "").trim().toLowerCase();
|
|
603723
603820
|
return !(consensusEnv === "0" || consensusEnv === "off" || consensusEnv === "false");
|
|
603724
603821
|
}
|
|
603822
|
+
function liveAsrUseTranscribeCli() {
|
|
603823
|
+
const value2 = String(process.env["OMNIUS_ASR_USE_TRANSCRIBE_CLI"] ?? "").trim().toLowerCase();
|
|
603824
|
+
return value2 === "1" || value2 === "true" || value2 === "yes" || value2 === "on";
|
|
603825
|
+
}
|
|
603725
603826
|
async function ensureVenvForTranscribeCli() {
|
|
603726
603827
|
const bin = process.platform === "win32" ? "Scripts" : "bin";
|
|
603727
603828
|
const exe = process.platform === "win32" ? "python.exe" : "python3";
|
|
@@ -603836,6 +603937,7 @@ var init_listen = __esm({
|
|
|
603836
603937
|
"use strict";
|
|
603837
603938
|
init_typed_node_events();
|
|
603838
603939
|
init_async_process();
|
|
603940
|
+
init_dist5();
|
|
603839
603941
|
MANAGED_TRANSCRIBE_CLI_DIR2 = join126(homedir40(), ".omnius", "runtimes", "asr", "transcribe-cli-node");
|
|
603840
603942
|
AUDIO_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
603841
603943
|
".mp3",
|
|
@@ -603869,6 +603971,8 @@ var init_listen = __esm({
|
|
|
603869
603971
|
doa: null,
|
|
603870
603972
|
lastTranscript: "",
|
|
603871
603973
|
lastTranscriptAt: 0,
|
|
603974
|
+
lastPartialTranscript: "",
|
|
603975
|
+
lastPartialTranscriptAt: 0,
|
|
603872
603976
|
lastStatus: "",
|
|
603873
603977
|
updatedAt: 0
|
|
603874
603978
|
};
|
|
@@ -603883,6 +603987,7 @@ var init_listen = __esm({
|
|
|
603883
603987
|
scriptPath;
|
|
603884
603988
|
process = null;
|
|
603885
603989
|
_ready = false;
|
|
603990
|
+
registeredBrokerName = null;
|
|
603886
603991
|
get ready() {
|
|
603887
603992
|
return this._ready;
|
|
603888
603993
|
}
|
|
@@ -603957,6 +604062,7 @@ var init_listen = __esm({
|
|
|
603957
604062
|
this._ready = true;
|
|
603958
604063
|
clearTimeout(timeout2);
|
|
603959
604064
|
updateListenLiveState({ phase: "ready", lastStatus: "whisper model loaded" });
|
|
604065
|
+
this.registerBrokerModel(evt);
|
|
603960
604066
|
this.emit("ready");
|
|
603961
604067
|
resolve76();
|
|
603962
604068
|
break;
|
|
@@ -604003,6 +604109,7 @@ var init_listen = __esm({
|
|
|
604003
604109
|
reject(err);
|
|
604004
604110
|
});
|
|
604005
604111
|
onChildClose(this.process, (code8) => {
|
|
604112
|
+
this.unregisterBrokerModel();
|
|
604006
604113
|
if (!this._ready) {
|
|
604007
604114
|
clearTimeout(timeout2);
|
|
604008
604115
|
reject(
|
|
@@ -604020,6 +604127,7 @@ var init_listen = __esm({
|
|
|
604020
604127
|
}
|
|
604021
604128
|
/** Stop the worker. */
|
|
604022
604129
|
stop() {
|
|
604130
|
+
this.unregisterBrokerModel();
|
|
604023
604131
|
if (this.process) {
|
|
604024
604132
|
try {
|
|
604025
604133
|
this.process.stdin?.end();
|
|
@@ -604030,6 +604138,40 @@ var init_listen = __esm({
|
|
|
604030
604138
|
}
|
|
604031
604139
|
this._ready = false;
|
|
604032
604140
|
}
|
|
604141
|
+
registerBrokerModel(evt) {
|
|
604142
|
+
const device = String(evt["device"] ?? "");
|
|
604143
|
+
const cuda = evt["cuda"] === true || device.startsWith("cuda");
|
|
604144
|
+
const gpuIndexMatch = device.match(/^cuda:(\d+)/);
|
|
604145
|
+
const gpuIndex = gpuIndexMatch ? Number(gpuIndexMatch[1]) : null;
|
|
604146
|
+
const consensus = typeof evt["consensusModel"] === "string" && String(evt["consensusModel"]).trim() ? String(evt["consensusModel"]).trim() : "";
|
|
604147
|
+
const name10 = `whisper:${this.model}${consensus ? `+${consensus}` : ""}`;
|
|
604148
|
+
const vramMB = Math.max(0, Math.round(Number(evt["vramUsedMB"] ?? 0)));
|
|
604149
|
+
try {
|
|
604150
|
+
getModelBroker().registerLoaded({
|
|
604151
|
+
key: `whisper-cli:${name10}`,
|
|
604152
|
+
name: name10,
|
|
604153
|
+
domain: "asr",
|
|
604154
|
+
host: "whisper-cli",
|
|
604155
|
+
owner: "live-asr",
|
|
604156
|
+
vramMB: cuda ? vramMB : 0,
|
|
604157
|
+
ramMB: cuda ? 0 : vramMB,
|
|
604158
|
+
priority: 88,
|
|
604159
|
+
gpuIndex,
|
|
604160
|
+
gpuUuid: null
|
|
604161
|
+
});
|
|
604162
|
+
this.registeredBrokerName = name10;
|
|
604163
|
+
} catch {
|
|
604164
|
+
this.registeredBrokerName = null;
|
|
604165
|
+
}
|
|
604166
|
+
}
|
|
604167
|
+
unregisterBrokerModel() {
|
|
604168
|
+
if (!this.registeredBrokerName) return;
|
|
604169
|
+
try {
|
|
604170
|
+
getModelBroker().unregisterLoaded("whisper-cli", this.registeredBrokerName, "live-asr stopped");
|
|
604171
|
+
} catch {
|
|
604172
|
+
}
|
|
604173
|
+
this.registeredBrokerName = null;
|
|
604174
|
+
}
|
|
604033
604175
|
};
|
|
604034
604176
|
ListenEngine = class extends EventEmitter6 {
|
|
604035
604177
|
config;
|
|
@@ -604046,6 +604188,7 @@ var init_listen = __esm({
|
|
|
604046
604188
|
blinkTimer = null;
|
|
604047
604189
|
blinkState = false;
|
|
604048
604190
|
transcribeCliAvailable = null;
|
|
604191
|
+
owners = /* @__PURE__ */ new Set();
|
|
604049
604192
|
constructor(config) {
|
|
604050
604193
|
super();
|
|
604051
604194
|
this.config = {
|
|
@@ -604074,6 +604217,29 @@ var init_listen = __esm({
|
|
|
604074
604217
|
get pendingTranscript() {
|
|
604075
604218
|
return this.pendingText;
|
|
604076
604219
|
}
|
|
604220
|
+
hasOwner(owner) {
|
|
604221
|
+
return this.owners.has(owner);
|
|
604222
|
+
}
|
|
604223
|
+
ownerCount() {
|
|
604224
|
+
return this.owners.size;
|
|
604225
|
+
}
|
|
604226
|
+
async acquire(owner) {
|
|
604227
|
+
const clean5 = owner.trim() || "shared";
|
|
604228
|
+
this.owners.add(clean5);
|
|
604229
|
+
if (this.active) return `Live ASR already active (${[...this.owners].join(", ")}).`;
|
|
604230
|
+
const result = await this.start();
|
|
604231
|
+
if (!this.active) this.owners.delete(clean5);
|
|
604232
|
+
return result;
|
|
604233
|
+
}
|
|
604234
|
+
async release(owner) {
|
|
604235
|
+
const clean5 = owner.trim() || "shared";
|
|
604236
|
+
this.owners.delete(clean5);
|
|
604237
|
+
if (this.owners.size > 0) {
|
|
604238
|
+
return `Live ASR remains active (${[...this.owners].join(", ")}).`;
|
|
604239
|
+
}
|
|
604240
|
+
if (!this.active) return "Live ASR is not active.";
|
|
604241
|
+
return this.stop();
|
|
604242
|
+
}
|
|
604077
604243
|
setModel(model) {
|
|
604078
604244
|
this.config.model = model;
|
|
604079
604245
|
}
|
|
@@ -604224,19 +604390,21 @@ var init_listen = __esm({
|
|
|
604224
604390
|
updateListenLiveState({ phase: "error", lastStatus: "no mic capture tool (arecord/sox/ffmpeg)" });
|
|
604225
604391
|
return "No microphone capture tool found. Install arecord (Linux), sox (macOS), or ffmpeg.";
|
|
604226
604392
|
}
|
|
604227
|
-
const
|
|
604393
|
+
const consensusEnabled = asrConsensusEnabled();
|
|
604394
|
+
const useTranscribeCli = liveAsrUseTranscribeCli() && !consensusEnabled;
|
|
604395
|
+
const preferWhisperFallback = !useTranscribeCli;
|
|
604228
604396
|
if (preferWhisperFallback) {
|
|
604229
604397
|
this.emit(
|
|
604230
604398
|
"info",
|
|
604231
|
-
"ASR consensus enabled — using live-whisper.py
|
|
604399
|
+
consensusEnabled ? "ASR consensus enabled — using live-whisper.py with CUDA-only Whisper guard." : "Using live-whisper.py with CUDA-only Whisper guard."
|
|
604232
604400
|
);
|
|
604233
604401
|
updateListenLiveState({
|
|
604234
604402
|
backend: "openai-whisper",
|
|
604235
|
-
lastStatus: "starting consensus Whisper backend..."
|
|
604403
|
+
lastStatus: consensusEnabled ? "starting consensus Whisper backend..." : "starting CUDA Whisper backend..."
|
|
604236
604404
|
});
|
|
604237
604405
|
}
|
|
604238
|
-
let tc =
|
|
604239
|
-
if (
|
|
604406
|
+
let tc = useTranscribeCli ? await this.loadTranscribeCli() : null;
|
|
604407
|
+
if (useTranscribeCli && !tc) {
|
|
604240
604408
|
if (_bgInstallPromise) {
|
|
604241
604409
|
this.emit("info", "Waiting for transcribe-cli install...");
|
|
604242
604410
|
await _bgInstallPromise;
|
|
@@ -604319,7 +604487,7 @@ var init_listen = __esm({
|
|
|
604319
604487
|
if (!evt.text.trim()) return;
|
|
604320
604488
|
this.lastTranscriptTime = Date.now();
|
|
604321
604489
|
this.pendingText = evt.text.trim();
|
|
604322
|
-
updateListenLiveState({ lastTranscript: this.pendingText, lastTranscriptAt: this.lastTranscriptTime });
|
|
604490
|
+
updateListenLiveState(evt.isFinal ? { lastTranscript: this.pendingText, lastTranscriptAt: this.lastTranscriptTime } : { lastPartialTranscript: this.pendingText, lastPartialTranscriptAt: this.lastTranscriptTime });
|
|
604323
604491
|
this.emit("transcript", this.pendingText, evt.isFinal);
|
|
604324
604492
|
if (this.config.mode === "auto") this.resetSilenceTimer();
|
|
604325
604493
|
}
|
|
@@ -604377,7 +604545,7 @@ var init_listen = __esm({
|
|
|
604377
604545
|
if (!evt.text.trim()) return;
|
|
604378
604546
|
this.lastTranscriptTime = Date.now();
|
|
604379
604547
|
this.pendingText = evt.text.trim();
|
|
604380
|
-
updateListenLiveState({ lastTranscript: this.pendingText, lastTranscriptAt: this.lastTranscriptTime });
|
|
604548
|
+
updateListenLiveState(evt.isFinal ? { lastTranscript: this.pendingText, lastTranscriptAt: this.lastTranscriptTime } : { lastPartialTranscript: this.pendingText, lastPartialTranscriptAt: this.lastTranscriptTime });
|
|
604381
604549
|
this.emit("transcript", this.pendingText, evt.isFinal, { consensus: evt.consensus, doa: evt.doa });
|
|
604382
604550
|
if (this.config.mode === "auto") this.resetSilenceTimer();
|
|
604383
604551
|
});
|
|
@@ -604420,6 +604588,7 @@ transcribe-cli error: ${transcribeCliError}` : "";
|
|
|
604420
604588
|
async stop() {
|
|
604421
604589
|
if (!this.active) return "Not listening.";
|
|
604422
604590
|
this.active = false;
|
|
604591
|
+
this.owners.clear();
|
|
604423
604592
|
this.blinkState = false;
|
|
604424
604593
|
updateListenLiveState({ phase: "idle", speechActive: false, micLevelDb: null, lastStatus: "stopped" });
|
|
604425
604594
|
if (this.blinkTimer) {
|
|
@@ -604515,13 +604684,14 @@ transcribe-cli error: ${transcribeCliError}` : "";
|
|
|
604515
604684
|
*/
|
|
604516
604685
|
async createCallTranscriber() {
|
|
604517
604686
|
const requireConsensus = asrConsensusEnabled();
|
|
604518
|
-
|
|
604519
|
-
|
|
604687
|
+
const useTranscribeCli = liveAsrUseTranscribeCli() && !requireConsensus;
|
|
604688
|
+
let tc = useTranscribeCli ? await this.loadTranscribeCli() : null;
|
|
604689
|
+
if (useTranscribeCli && !tc && _bgInstallPromise) {
|
|
604520
604690
|
await _bgInstallPromise;
|
|
604521
604691
|
this.transcribeCliAvailable = null;
|
|
604522
604692
|
tc = await this.loadTranscribeCli();
|
|
604523
604693
|
}
|
|
604524
|
-
if (
|
|
604694
|
+
if (useTranscribeCli && !tc) {
|
|
604525
604695
|
try {
|
|
604526
604696
|
await ensureManagedTranscribeCliNode();
|
|
604527
604697
|
this.transcribeCliAvailable = null;
|
|
@@ -604529,7 +604699,7 @@ transcribe-cli error: ${transcribeCliError}` : "";
|
|
|
604529
604699
|
} catch {
|
|
604530
604700
|
}
|
|
604531
604701
|
}
|
|
604532
|
-
if (
|
|
604702
|
+
if (useTranscribeCli && tc?.TranscribeLive && await ensureVenvForTranscribeCli()) {
|
|
604533
604703
|
try {
|
|
604534
604704
|
const transcriber = new tc.TranscribeLive({
|
|
604535
604705
|
model: this.config.model,
|
|
@@ -604669,8 +604839,8 @@ transcribe-cli error: ${transcribeCliError}` : "";
|
|
|
604669
604839
|
import { spawn as spawn30 } from "node:child_process";
|
|
604670
604840
|
import { existsSync as existsSync112, mkdirSync as mkdirSync69, statSync as statSync43, unlinkSync as unlinkSync20 } from "node:fs";
|
|
604671
604841
|
import { join as join127 } from "node:path";
|
|
604672
|
-
function streamFps() {
|
|
604673
|
-
const raw = Number(process.env["OMNIUS_LIVE_STREAM_FPS"] ?? "");
|
|
604842
|
+
function streamFps(configured) {
|
|
604843
|
+
const raw = configured ?? Number(process.env["OMNIUS_LIVE_STREAM_FPS"] ?? "");
|
|
604674
604844
|
if (Number.isFinite(raw) && raw > 0) return Math.max(1, Math.min(30, Math.round(raw)));
|
|
604675
604845
|
return 8;
|
|
604676
604846
|
}
|
|
@@ -604751,7 +604921,7 @@ var init_camera_streamer = __esm({
|
|
|
604751
604921
|
* Only v4l2 device paths are streamable; other backends (OSC/WiFi cams)
|
|
604752
604922
|
* keep the one-shot capture path.
|
|
604753
604923
|
*/
|
|
604754
|
-
sync(sources, rotationFor, resolutionFor) {
|
|
604924
|
+
sync(sources, rotationFor, resolutionFor, fpsFor) {
|
|
604755
604925
|
const wanted = new Set(sources.filter((source) => source.startsWith("/dev/video")));
|
|
604756
604926
|
for (const [source, stream] of this.streams) {
|
|
604757
604927
|
if (!wanted.has(source)) {
|
|
@@ -604762,20 +604932,21 @@ var init_camera_streamer = __esm({
|
|
|
604762
604932
|
for (const source of wanted) {
|
|
604763
604933
|
const rotation = rotationFor(source);
|
|
604764
604934
|
const resolution = resolutionFor(source);
|
|
604935
|
+
const fps = streamFps(fpsFor(source));
|
|
604765
604936
|
const existing = this.streams.get(source);
|
|
604766
|
-
if (existing && existing.rotation === rotation && existing.resolution === resolution && !existing.stopped) continue;
|
|
604937
|
+
if (existing && existing.rotation === rotation && existing.resolution === resolution && existing.fps === fps && !existing.stopped) continue;
|
|
604767
604938
|
if (existing) {
|
|
604768
604939
|
this.stopStream(existing);
|
|
604769
604940
|
this.streams.delete(source);
|
|
604770
604941
|
}
|
|
604771
|
-
this.startStream(source, rotation, resolution);
|
|
604942
|
+
this.startStream(source, rotation, resolution, fps);
|
|
604772
604943
|
}
|
|
604773
604944
|
}
|
|
604774
604945
|
stopAll() {
|
|
604775
604946
|
for (const stream of this.streams.values()) this.stopStream(stream);
|
|
604776
604947
|
this.streams.clear();
|
|
604777
604948
|
}
|
|
604778
|
-
startStream(source, rotation, resolution) {
|
|
604949
|
+
startStream(source, rotation, resolution, fps) {
|
|
604779
604950
|
try {
|
|
604780
604951
|
mkdirSync69(this.streamDir, { recursive: true });
|
|
604781
604952
|
} catch {
|
|
@@ -604785,6 +604956,7 @@ var init_camera_streamer = __esm({
|
|
|
604785
604956
|
framePath: join127(this.streamDir, `${slugForSource(source)}.jpg`),
|
|
604786
604957
|
rotation,
|
|
604787
604958
|
resolution,
|
|
604959
|
+
fps,
|
|
604788
604960
|
process: null,
|
|
604789
604961
|
attempt: 0,
|
|
604790
604962
|
consecutiveFailures: 0,
|
|
@@ -604798,7 +604970,7 @@ var init_camera_streamer = __esm({
|
|
|
604798
604970
|
}
|
|
604799
604971
|
spawnStream(stream) {
|
|
604800
604972
|
if (stream.stopped) return;
|
|
604801
|
-
const fps = streamFps();
|
|
604973
|
+
const fps = streamFps(stream.fps);
|
|
604802
604974
|
const resolution = parseResolution(stream.resolution);
|
|
604803
604975
|
const filters = [`fps=${fps}`];
|
|
604804
604976
|
const scale = scaleFilter(resolution);
|
|
@@ -605109,6 +605281,135 @@ function formatLiveResourceBudgetLines(width = 100) {
|
|
|
605109
605281
|
`└─ leases: ${truncateAnsi(leaseText, inner - 12)}`
|
|
605110
605282
|
];
|
|
605111
605283
|
}
|
|
605284
|
+
async function refreshLiveResourceModelSnapshot(minIntervalMs = 2500) {
|
|
605285
|
+
const now2 = Date.now();
|
|
605286
|
+
if (now2 - lastModelSnapshotRefreshAt < minIntervalMs) return;
|
|
605287
|
+
lastModelSnapshotRefreshAt = now2;
|
|
605288
|
+
await getModelBroker().pollOnce();
|
|
605289
|
+
}
|
|
605290
|
+
function formatLiveMenuResourceSummary() {
|
|
605291
|
+
const snap = getLiveResourceArbiter().snapshot();
|
|
605292
|
+
const broker = getModelBroker().snapshot();
|
|
605293
|
+
const totalGB = snap.totalMemMB / 1024;
|
|
605294
|
+
const usedGB = snap.usedMemMB / 1024;
|
|
605295
|
+
const vram = broker.vramMB;
|
|
605296
|
+
const vramText = vram ? `GPU/UMA ${mbToGb(vram.used)}/${mbToGb(vram.total)}GB` : "GPU/UMA unknown";
|
|
605297
|
+
const cuda = broker.vramPerDevice.length > 0 ? "CUDA on" : "CUDA off/unknown";
|
|
605298
|
+
return `RAM ${usedGB.toFixed(1)}/${totalGB.toFixed(1)}GB · ${vramText} · ${cuda} · models ${broker.loaded.length} loaded/${broker.inflight.length} loading`;
|
|
605299
|
+
}
|
|
605300
|
+
function formatLiveModelRuntimeLines(width = 100, opts = {}) {
|
|
605301
|
+
const broker = getModelBroker().snapshot();
|
|
605302
|
+
const inner = Math.max(24, width - 2);
|
|
605303
|
+
const loaded = broker.loaded ?? [];
|
|
605304
|
+
const inflight = broker.inflight ?? [];
|
|
605305
|
+
const modelsShown = Math.max(0, opts.maxModels ?? 4);
|
|
605306
|
+
const cudaAvailable = broker.vramPerDevice.length > 0;
|
|
605307
|
+
const gpuLine = formatGpuRuntimeLine(broker);
|
|
605308
|
+
const subsystemLine = formatSubsystemMemoryLine(loaded, inner);
|
|
605309
|
+
const unattributedLine = formatUnattributedMemoryLine(loaded, broker);
|
|
605310
|
+
const pollAge = broker.lastPollAt > 0 ? Math.max(0, Math.round((Date.now() - broker.lastPollAt) / 1e3)) : null;
|
|
605311
|
+
const lines = [
|
|
605312
|
+
colorText(" model runtime", 81),
|
|
605313
|
+
gpuLine,
|
|
605314
|
+
subsystemLine,
|
|
605315
|
+
unattributedLine,
|
|
605316
|
+
`├─ state: loaded=${loaded.length} loading=${inflight.length} slots=${broker.slots.inUse}/${broker.slots.capacity} queue=${broker.slots.queueDepth}/${broker.slots.queueCapacity}${pollAge !== null ? ` poll=${pollAge}s` : ""}`
|
|
605317
|
+
];
|
|
605318
|
+
if (loaded.length === 0 && inflight.length === 0) {
|
|
605319
|
+
lines.push(`└─ models: ${colorText("none tracked yet", 244)}${cudaAvailable ? "" : colorText(" (CUDA state unknown until broker/JTOP poll succeeds)", 244)}`);
|
|
605320
|
+
return lines;
|
|
605321
|
+
}
|
|
605322
|
+
const modelLines = loaded.sort((a2, b) => b.vramMB + b.ramMB - (a2.vramMB + a2.ramMB)).slice(0, modelsShown).map((model, index, shown) => {
|
|
605323
|
+
const last2 = index === shown.length - 1 && inflight.length === 0 && loaded.length <= modelsShown;
|
|
605324
|
+
return `${last2 ? "└" : "├"}─ ${formatLoadedModelRuntime(model, broker)}`;
|
|
605325
|
+
});
|
|
605326
|
+
lines.push(...modelLines.map((line) => truncateAnsi(line, inner)));
|
|
605327
|
+
if (loaded.length > modelsShown) {
|
|
605328
|
+
lines.push(`├─ ${colorText(`+${loaded.length - modelsShown} more loaded model(s)`, 244)}`);
|
|
605329
|
+
}
|
|
605330
|
+
if (inflight.length > 0) {
|
|
605331
|
+
const loading = inflight.slice(0, 3).map((entry) => {
|
|
605332
|
+
const age = Math.max(0, Math.round((Date.now() - entry.startedMs) / 1e3));
|
|
605333
|
+
return `${entry.key} owner=${entry.owner} ${age}s`;
|
|
605334
|
+
}).join("; ");
|
|
605335
|
+
lines.push(`└─ loading: ${truncateAnsi(loading, inner - 12)}`);
|
|
605336
|
+
}
|
|
605337
|
+
return lines;
|
|
605338
|
+
}
|
|
605339
|
+
function formatLiveResourceModelReport(width = 100) {
|
|
605340
|
+
return [
|
|
605341
|
+
...formatLiveResourceBudgetLines(width),
|
|
605342
|
+
...formatLiveModelRuntimeLines(width, { maxModels: 20 })
|
|
605343
|
+
].join("\n");
|
|
605344
|
+
}
|
|
605345
|
+
function formatGpuRuntimeLine(broker) {
|
|
605346
|
+
const devices = broker.vramPerDevice ?? [];
|
|
605347
|
+
if (devices.length === 0) {
|
|
605348
|
+
return `├─ accelerator: ${colorText("CUDA off/unknown", 208)} vram=unavailable`;
|
|
605349
|
+
}
|
|
605350
|
+
const parts = devices.map((device) => {
|
|
605351
|
+
const label = device.uuid === "jetson-uma" ? "jetson-uma" : `gpu${device.index}`;
|
|
605352
|
+
const pct = device.total > 0 ? device.used / device.total : 0;
|
|
605353
|
+
return `${label} ${pressureColor(pct, 0.78)(`${mbToGb(device.used)}/${mbToGb(device.total)}GB`)}`;
|
|
605354
|
+
});
|
|
605355
|
+
return `├─ accelerator: ${colorText("CUDA on", 82)} ${parts.join(" ")}`;
|
|
605356
|
+
}
|
|
605357
|
+
function formatSubsystemMemoryLine(loaded, width) {
|
|
605358
|
+
const totals = /* @__PURE__ */ new Map();
|
|
605359
|
+
for (const model of loaded) {
|
|
605360
|
+
const key = subsystemForModel(model);
|
|
605361
|
+
const current = totals.get(key) ?? { vram: 0, ram: 0, count: 0 };
|
|
605362
|
+
current.vram += Math.max(0, Number(model.vramMB ?? 0));
|
|
605363
|
+
current.ram += Math.max(0, Number(model.ramMB ?? 0));
|
|
605364
|
+
current.count += 1;
|
|
605365
|
+
totals.set(key, current);
|
|
605366
|
+
}
|
|
605367
|
+
if (totals.size === 0) return `├─ subsystems(tracked): ${colorText("no tracked model memory", 244)}`;
|
|
605368
|
+
const preferred = ["llm", "vision", "asr", "embedding", "image", "video", "audio", "other"];
|
|
605369
|
+
const parts = [...totals.entries()].sort((a2, b) => {
|
|
605370
|
+
const ai = preferred.indexOf(a2[0]);
|
|
605371
|
+
const bi = preferred.indexOf(b[0]);
|
|
605372
|
+
return (ai === -1 ? 99 : ai) - (bi === -1 ? 99 : bi);
|
|
605373
|
+
}).map(([name10, total]) => `${name10}=${mbToGb(total.vram)}Gv/${mbToGb(total.ram)}Gr`);
|
|
605374
|
+
return `├─ subsystems(tracked): ${truncateAnsi(parts.join(" "), width - 25)}`;
|
|
605375
|
+
}
|
|
605376
|
+
function formatUnattributedMemoryLine(loaded, broker) {
|
|
605377
|
+
const resource = getLiveResourceArbiter().snapshot();
|
|
605378
|
+
const trackedRam = loaded.reduce((sum, model) => sum + Math.max(0, Number(model.ramMB ?? 0)), 0);
|
|
605379
|
+
const trackedVram = loaded.reduce((sum, model) => sum + Math.max(0, Number(model.vramMB ?? 0)), 0);
|
|
605380
|
+
const ramGap = Math.max(0, resource.usedMemMB - trackedRam);
|
|
605381
|
+
const vramGap = broker.vramMB ? Math.max(0, broker.vramMB.used - trackedVram) : null;
|
|
605382
|
+
return `├─ unattributed/system: ram=${mbToGb(ramGap)}G${vramGap !== null ? ` vram/uma=${mbToGb(vramGap)}G` : ""} ${colorText("(OS, camera ffmpeg, CV libs, CUDA overhead, untracked subprocesses)", 244)}`;
|
|
605383
|
+
}
|
|
605384
|
+
function formatLoadedModelRuntime(model, broker) {
|
|
605385
|
+
const slots = broker.slots.byModel[model.name] ?? broker.slots.byModel[model.key] ?? null;
|
|
605386
|
+
const active = slots && slots.inUse > 0;
|
|
605387
|
+
const state = active ? colorText(`running x${slots.inUse}`, 82) : colorText("loaded idle", 244);
|
|
605388
|
+
const cuda = cudaStateForModel(model, broker.vramPerDevice.length > 0);
|
|
605389
|
+
const idle = Math.max(0, Math.round((Date.now() - model.lastUsedAt) / 1e3));
|
|
605390
|
+
const tps = slots && slots.samples > 0 ? ` ${slots.tokensPerSec.toFixed(1)}tok/s` : "";
|
|
605391
|
+
const ctx3 = model.numCtx ? ` ctx=${model.numCtx}` : "";
|
|
605392
|
+
return `${subsystemForModel(model)} ${model.host}:${model.name} ${state} ${cuda} vram=${model.vramMB}MB ram=${model.ramMB}MB idle=${idle}s${tps}${ctx3}`;
|
|
605393
|
+
}
|
|
605394
|
+
function cudaStateForModel(model, cudaAvailable) {
|
|
605395
|
+
if (model.gpuIndex !== null && model.gpuIndex !== void 0) return colorText(`cuda:on gpu${model.gpuIndex}`, 82);
|
|
605396
|
+
if (model.vramMB > 0) return colorText(cudaAvailable ? "cuda:on" : "gpu:on", 82);
|
|
605397
|
+
if (model.host === "external") return colorText("cuda:external", 244);
|
|
605398
|
+
return colorText(cudaAvailable ? "cuda:off/cpu" : "cuda:off", 208);
|
|
605399
|
+
}
|
|
605400
|
+
function subsystemForModel(model) {
|
|
605401
|
+
if (model.domain === "chat" || model.domain === "subagent") return "llm";
|
|
605402
|
+
if (model.domain === "vision" || model.host.includes("moondream")) return "vision";
|
|
605403
|
+
if (model.domain === "asr" || model.host.includes("whisper")) return "asr";
|
|
605404
|
+
if (model.domain === "embedding") return "embedding";
|
|
605405
|
+
if (model.domain === "image-gen") return "image";
|
|
605406
|
+
if (model.domain === "video-gen") return "video";
|
|
605407
|
+
if (model.domain === "tts" || model.domain === "music" || model.domain === "sound") return "audio";
|
|
605408
|
+
return "other";
|
|
605409
|
+
}
|
|
605410
|
+
function mbToGb(mb) {
|
|
605411
|
+
return (Math.max(0, Number(mb) || 0) / 1024).toFixed(1);
|
|
605412
|
+
}
|
|
605112
605413
|
function readBudget() {
|
|
605113
605414
|
const raw = String(process.env["OMNIUS_LIVE_RESOURCE_BUDGET"] ?? "").trim().toLowerCase();
|
|
605114
605415
|
if (raw === "low" || raw === "conserve" || raw === "conservative" || raw === "battery") return "conserve";
|
|
@@ -605144,10 +605445,11 @@ function truncateAnsi(text2, max) {
|
|
|
605144
605445
|
if (plain.length <= max) return text2;
|
|
605145
605446
|
return `${plain.slice(0, Math.max(0, max - 3))}...`;
|
|
605146
605447
|
}
|
|
605147
|
-
var LiveResourceArbiter, singleton;
|
|
605448
|
+
var LiveResourceArbiter, singleton, lastModelSnapshotRefreshAt;
|
|
605148
605449
|
var init_live_resource_arbiter = __esm({
|
|
605149
605450
|
"packages/cli/src/tui/live-resource-arbiter.ts"() {
|
|
605150
605451
|
"use strict";
|
|
605452
|
+
init_dist5();
|
|
605151
605453
|
LiveResourceArbiter = class {
|
|
605152
605454
|
leases = /* @__PURE__ */ new Map();
|
|
605153
605455
|
seq = 0;
|
|
@@ -605304,6 +605606,7 @@ var init_live_resource_arbiter = __esm({
|
|
|
605304
605606
|
}
|
|
605305
605607
|
};
|
|
605306
605608
|
singleton = null;
|
|
605609
|
+
lastModelSnapshotRefreshAt = 0;
|
|
605307
605610
|
}
|
|
605308
605611
|
});
|
|
605309
605612
|
|
|
@@ -605482,6 +605785,17 @@ function formatLiveCameraResolution(value2) {
|
|
|
605482
605785
|
if (res === "3840x2160") return "4K (3840x2160)";
|
|
605483
605786
|
return res;
|
|
605484
605787
|
}
|
|
605788
|
+
function normalizeLiveCameraFps(value2) {
|
|
605789
|
+
if (value2 === void 0 || value2 === null || value2 === "") return DEFAULT_CAMERA_FPS;
|
|
605790
|
+
const parsed = typeof value2 === "number" ? value2 : Number(String(value2).trim().toLowerCase().replace(/\s*fps$/, ""));
|
|
605791
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
605792
|
+
throw new Error(`Unsupported live camera FPS: ${String(value2)}. Use a number from 1 to 30.`);
|
|
605793
|
+
}
|
|
605794
|
+
return Math.max(1, Math.min(30, Math.round(parsed)));
|
|
605795
|
+
}
|
|
605796
|
+
function formatLiveCameraFps(value2) {
|
|
605797
|
+
return `${normalizeLiveCameraFps(value2)} fps`;
|
|
605798
|
+
}
|
|
605485
605799
|
function liveCameraResolutionSize(value2) {
|
|
605486
605800
|
const res = value2 ?? DEFAULT_CAMERA_RESOLUTION;
|
|
605487
605801
|
if (res === "native") return null;
|
|
@@ -605499,10 +605813,11 @@ function sanitizeCameraStreams(value2) {
|
|
|
605499
605813
|
try {
|
|
605500
605814
|
out[device] = {
|
|
605501
605815
|
enabled: entry["enabled"] !== false,
|
|
605502
|
-
resolution: normalizeLiveCameraResolution(entry["resolution"] ?? DEFAULT_CAMERA_RESOLUTION)
|
|
605816
|
+
resolution: normalizeLiveCameraResolution(entry["resolution"] ?? DEFAULT_CAMERA_RESOLUTION),
|
|
605817
|
+
fps: normalizeLiveCameraFps(entry["fps"] ?? entry["framerate"] ?? DEFAULT_CAMERA_FPS)
|
|
605503
605818
|
};
|
|
605504
605819
|
} catch {
|
|
605505
|
-
out[device] = { enabled: entry["enabled"] !== false, resolution: DEFAULT_CAMERA_RESOLUTION };
|
|
605820
|
+
out[device] = { enabled: entry["enabled"] !== false, resolution: DEFAULT_CAMERA_RESOLUTION, fps: DEFAULT_CAMERA_FPS };
|
|
605506
605821
|
}
|
|
605507
605822
|
}
|
|
605508
605823
|
return out;
|
|
@@ -605537,6 +605852,18 @@ function truncateCell(value2, width) {
|
|
|
605537
605852
|
function stripDashboardAnsi(value2) {
|
|
605538
605853
|
return value2.replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, "");
|
|
605539
605854
|
}
|
|
605855
|
+
function dashboardAnsi(code8, text2) {
|
|
605856
|
+
return `\x1B[${code8}m${text2}\x1B[0m`;
|
|
605857
|
+
}
|
|
605858
|
+
function dashboardDim(text2) {
|
|
605859
|
+
return dashboardAnsi("38;5;244", text2);
|
|
605860
|
+
}
|
|
605861
|
+
function dashboardSoundColor(score) {
|
|
605862
|
+
if (score === void 0) return "38;5;81";
|
|
605863
|
+
if (score >= 0.75) return "38;5;82";
|
|
605864
|
+
if (score >= 0.5) return "38;5;220";
|
|
605865
|
+
return "38;5;208";
|
|
605866
|
+
}
|
|
605540
605867
|
function dashboardVisibleWidth(value2) {
|
|
605541
605868
|
return stripDashboardAnsi(value2).length;
|
|
605542
605869
|
}
|
|
@@ -605785,13 +606112,6 @@ function readPcm16WavActivity(filePath, bins = 36) {
|
|
|
605785
606112
|
return void 0;
|
|
605786
606113
|
}
|
|
605787
606114
|
}
|
|
605788
|
-
function extractAsrBackend(output) {
|
|
605789
|
-
const match = String(output ?? "").match(/^Backend:\s*(.+)$/m);
|
|
605790
|
-
if (!match?.[1]) return void 0;
|
|
605791
|
-
const backend = match[1].trim();
|
|
605792
|
-
if (/managed openai-whisper|transcribe-cli|whisper/i.test(backend)) return backend;
|
|
605793
|
-
return backend.slice(0, 80);
|
|
605794
|
-
}
|
|
605795
606115
|
function parseLiveAudioIntakeDecision(value2) {
|
|
605796
606116
|
const parsed = parseJsonObjectFromText(value2);
|
|
605797
606117
|
if (!parsed) return null;
|
|
@@ -606185,6 +606505,23 @@ function formatAudioAnalysisDashboardSummary(value2) {
|
|
|
606185
606505
|
}
|
|
606186
606506
|
return `sound: ${trimOneLine(text2.replace(/\{[\s\S]*$/, ""), 140) || trimOneLine(text2, 140)}`;
|
|
606187
606507
|
}
|
|
606508
|
+
function formatAudioAnalysisDashboardSummaryColored(value2) {
|
|
606509
|
+
const text2 = String(value2 ?? "").trim();
|
|
606510
|
+
if (!text2) return "";
|
|
606511
|
+
const parsed = parseJsonObjectFromText(text2);
|
|
606512
|
+
const classifications = Array.isArray(parsed?.["classifications"]) ? parsed["classifications"] : [];
|
|
606513
|
+
if (classifications.length > 0) {
|
|
606514
|
+
const parts = classifications.slice(0, 4).map((entry) => {
|
|
606515
|
+
const label = trimOneLine(entry["class"] ?? entry["label"] ?? entry["name"] ?? "sound", 44);
|
|
606516
|
+
const score = boundedConfidence(entry["score"] ?? entry["confidence"]);
|
|
606517
|
+
const coloredLabel = dashboardAnsi(dashboardSoundColor(score), label || "sound");
|
|
606518
|
+
return `${coloredLabel}${score !== void 0 ? ` ${dashboardDim(score.toFixed(2))}` : ""}`;
|
|
606519
|
+
});
|
|
606520
|
+
return `${dashboardAnsi("38;5;45", "sounds heard")}: ${parts.join(dashboardDim("; "))}`;
|
|
606521
|
+
}
|
|
606522
|
+
const summary = trimOneLine(text2.replace(/\{[\s\S]*$/, ""), 140) || trimOneLine(text2, 140);
|
|
606523
|
+
return `${dashboardAnsi("38;5;45", "sound")}: ${dashboardAnsi("38;5;250", summary)}`;
|
|
606524
|
+
}
|
|
606188
606525
|
function formatCameraRotationDashboard(camera) {
|
|
606189
606526
|
const orientation = camera.orientation;
|
|
606190
606527
|
const rotation = orientation?.rotation ?? camera.rotation ?? 0;
|
|
@@ -606270,6 +606607,9 @@ function formatLiveDashboardFromSnapshot(snapshot, opts = {}) {
|
|
|
606270
606607
|
for (const budgetLine of formatLiveResourceBudgetLines(width - 2)) {
|
|
606271
606608
|
lines.push(`│${truncateAnsiCell(budgetLine, width - 2)}│`);
|
|
606272
606609
|
}
|
|
606610
|
+
for (const modelLine of formatLiveModelRuntimeLines(width - 2, { maxModels: 4 })) {
|
|
606611
|
+
lines.push(`│${truncateAnsiCell(modelLine, width - 2)}│`);
|
|
606612
|
+
}
|
|
606273
606613
|
const enabledStreams = Object.entries(snapshot.config.cameraStreams ?? {}).filter(([, stream]) => stream.enabled !== false);
|
|
606274
606614
|
const selectedResolution = snapshot.config.selectedCamera ? formatLiveCameraResolution(snapshot.config.cameraStreams?.[snapshot.config.selectedCamera]?.resolution) : "none";
|
|
606275
606615
|
lines.push(`│${truncateCell(` systems: camera-streams=${enabledStreams.length}/${snapshot.devices.video.length} selected-res=${selectedResolution} yolo=${snapshot.config.inferEnabled ? "on" : "off"} clip=${snapshot.config.clipEnabled ? "on" : "off"} asr=${snapshot.config.asrEnabled ? "on" : "off"} sounds=${snapshot.config.audioAnalysisEnabled ? "on" : "off"}`, width - 2)}│`);
|
|
@@ -606315,13 +606655,16 @@ function formatLiveDashboardFromSnapshot(snapshot, opts = {}) {
|
|
|
606315
606655
|
pushDashboardWrapped(lines, detail, width);
|
|
606316
606656
|
}
|
|
606317
606657
|
});
|
|
606318
|
-
|
|
606658
|
+
const dashboardAsr = getListenLiveState();
|
|
606659
|
+
const showDashboardAudio = Boolean(snapshot.audio) || dashboardAsr.phase !== "idle";
|
|
606660
|
+
if (showDashboardAudio) {
|
|
606661
|
+
const audioSnapshot = snapshot.audio;
|
|
606319
606662
|
lines.push(`├${horizontal}┤`);
|
|
606320
606663
|
lines.push(`│${truncateCell(" audio monitor", width - 2)}│`);
|
|
606321
|
-
const transcriptFailure = isAudioFailureText(
|
|
606322
|
-
const transcript = transcriptFailure ?
|
|
606323
|
-
const audioError = sanitizeLiveAudioError([
|
|
606324
|
-
const asr =
|
|
606664
|
+
const transcriptFailure = audioSnapshot && isAudioFailureText(audioSnapshot.transcript) ? audioSnapshot.transcript : "";
|
|
606665
|
+
const transcript = audioSnapshot && !transcriptFailure ? audioSnapshot.transcript : "";
|
|
606666
|
+
const audioError = sanitizeLiveAudioError([audioSnapshot?.error, transcriptFailure].filter(Boolean).join(" | "));
|
|
606667
|
+
const asr = dashboardAsr;
|
|
606325
606668
|
const asrPhaseLabel = {
|
|
606326
606669
|
bootstrapping: "bootstrapping deps…",
|
|
606327
606670
|
"loading-model": "loading model…",
|
|
@@ -606332,21 +606675,26 @@ function formatLiveDashboardFromSnapshot(snapshot, opts = {}) {
|
|
|
606332
606675
|
};
|
|
606333
606676
|
const asrMicBits = asr.micLevelDb !== null ? ` mic=${asr.micLevelDb.toFixed(1)}dB${asr.speechActive ? " ●speech" : ""}${asr.doa !== null ? ` dir=${asr.doa}°` : ""}` : "";
|
|
606334
606677
|
const asrLine = asr.phase !== "idle" ? `│ ├─ whisper: ${[asr.backend, asr.model].filter(Boolean).join(" ") || "starting"} — ${asrPhaseLabel[asr.phase] ?? asr.phase}${asrMicBits}` : "";
|
|
606335
|
-
const
|
|
606336
|
-
const
|
|
606678
|
+
const asrFinalAge = asr.lastTranscriptAt > 0 ? Math.max(0, Math.round((now2 - asr.lastTranscriptAt) / 1e3)) : null;
|
|
606679
|
+
const asrPartialAge = asr.lastPartialTranscriptAt > 0 ? Math.max(0, Math.round((now2 - asr.lastPartialTranscriptAt) / 1e3)) : null;
|
|
606680
|
+
const showPartial = asr.lastPartialTranscript && (!asr.lastTranscriptAt || asr.lastPartialTranscriptAt >= asr.lastTranscriptAt);
|
|
606681
|
+
const asrHeardLine = asr.phase !== "idle" && (showPartial || asr.lastTranscript) ? showPartial ? `│ ├─ hearing${asrPartialAge !== null ? ` ${asrPartialAge}s ago` : ""}: ${trimOneLine(asr.lastPartialTranscript, 120)}` : `│ ├─ heard${asrFinalAge !== null ? ` ${asrFinalAge}s ago` : ""}: ${trimOneLine(asr.lastTranscript, 120)}` : "";
|
|
606337
606682
|
const asrStatusLine = (asr.phase === "bootstrapping" || asr.phase === "loading-model" || asr.phase === "error") && asr.lastStatus ? `│ ├─ whisper status: ${trimOneLine(asr.lastStatus, 120)}` : "";
|
|
606683
|
+
const coloredSoundSummary = audioSnapshot?.analysis ? formatAudioAnalysisDashboardSummaryColored(audioSnapshot.analysis) : "";
|
|
606338
606684
|
const audioBits = [
|
|
606339
|
-
`├─ input: ${
|
|
606685
|
+
`├─ input: ${audioSnapshot?.input || snapshot.config.selectedAudioInput || "none"}`,
|
|
606340
606686
|
asrLine,
|
|
606341
606687
|
asrStatusLine,
|
|
606342
606688
|
asrHeardLine,
|
|
606343
|
-
|
|
606689
|
+
audioSnapshot?.activity ? `│ ├─ activity: ${audioSnapshot.activity.waveform} rms=${audioSnapshot.activity.rmsDb.toFixed(1)}dB active=${Math.round(audioSnapshot.activity.activeRatio * 100)}%` : "",
|
|
606344
606690
|
audioError ? `│ ├─ error: ${trimOneLine(audioError, 140)}` : "",
|
|
606345
|
-
transcript ? `│ ├─ asr${
|
|
606346
|
-
|
|
606347
|
-
snapshot.audio.analysis ? `│ └─ ${formatAudioAnalysisDashboardSummary(snapshot.audio.analysis)}` : ""
|
|
606691
|
+
transcript ? `│ ├─ asr${audioSnapshot?.asrBackend ? ` (${audioSnapshot.asrBackend})` : ""}: ${trimOneLine(transcript, 140)}` : "",
|
|
606692
|
+
audioSnapshot?.intake ? `│ ├─ intake: ${audioSnapshot.intake.intendedForAgent ? "for-agent" : "ambient"} conf=${audioSnapshot.intake.confidence.toFixed(2)} action=${audioSnapshot.intake.action}` : ""
|
|
606348
606693
|
].filter(Boolean);
|
|
606349
606694
|
for (const item of audioBits.slice(0, 9)) pushDashboardWrapped(lines, item, width);
|
|
606695
|
+
if (coloredSoundSummary) {
|
|
606696
|
+
lines.push(`│${truncateAnsiCell(` └─ ${coloredSoundSummary}`, width - 2)}│`);
|
|
606697
|
+
}
|
|
606350
606698
|
}
|
|
606351
606699
|
lines.push(`╰${horizontal}╯`);
|
|
606352
606700
|
return lines;
|
|
@@ -606641,7 +606989,7 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
606641
606989
|
const orientation = snapshot.config.cameraOrientation?.[snapshot.config.selectedCamera];
|
|
606642
606990
|
const stream = snapshot.config.cameraStreams?.[snapshot.config.selectedCamera];
|
|
606643
606991
|
lines.push(`Selected camera: ${snapshot.config.selectedCamera}`);
|
|
606644
|
-
lines.push(`Selected camera stream: ${stream?.enabled !== false ? "enabled" : "disabled"} ${formatLiveCameraResolution(stream?.resolution)}`);
|
|
606992
|
+
lines.push(`Selected camera stream: ${stream?.enabled !== false ? "enabled" : "disabled"} ${formatLiveCameraResolution(stream?.resolution)} ${formatLiveCameraFps(stream?.fps)}`);
|
|
606645
606993
|
lines.push(`Camera orientation correction: ${describeCameraOrientation(orientation)}`);
|
|
606646
606994
|
}
|
|
606647
606995
|
const cameraSnapshots = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {})).sort((a2, b) => {
|
|
@@ -606739,6 +607087,17 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
606739
607087
|
lines.push(`summary: ${cameraSnapshotSummary(camera)}`);
|
|
606740
607088
|
}
|
|
606741
607089
|
}
|
|
607090
|
+
const liveAsr = getListenLiveState();
|
|
607091
|
+
const liveAsrFinalAge = liveAsr.lastTranscriptAt > 0 ? now2 - liveAsr.lastTranscriptAt : Number.POSITIVE_INFINITY;
|
|
607092
|
+
const liveAsrPartialAge = liveAsr.lastPartialTranscriptAt > 0 ? now2 - liveAsr.lastPartialTranscriptAt : Number.POSITIVE_INFINITY;
|
|
607093
|
+
const liveAsrText = liveAsr.lastTranscript && liveAsrFinalAge < maxAgeMs ? liveAsr.lastTranscript : liveAsr.lastPartialTranscript && liveAsrPartialAge < Math.min(maxAgeMs, 3e4) ? liveAsr.lastPartialTranscript : "";
|
|
607094
|
+
if (liveAsrText) {
|
|
607095
|
+
const label = liveAsr.lastTranscript && liveAsrFinalAge < maxAgeMs ? "Live streaming ASR" : "Live streaming ASR partial";
|
|
607096
|
+
lines.push("");
|
|
607097
|
+
lines.push("## LIVE STREAMING ASR");
|
|
607098
|
+
lines.push(`${label}${liveAsr.backend ? ` (${liveAsr.backend} ${liveAsr.model})` : ""}:`);
|
|
607099
|
+
lines.push(cleanPreview(liveAsrText, 1200));
|
|
607100
|
+
}
|
|
606742
607101
|
if (snapshot.audio) {
|
|
606743
607102
|
const transcriptFailure = isAudioFailureText(snapshot.audio.transcript) ? snapshot.audio.transcript : "";
|
|
606744
607103
|
const transcript = transcriptFailure ? "" : snapshot.audio.transcript;
|
|
@@ -606773,6 +607132,7 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
606773
607132
|
lines.push(`intake_reason: ${snapshot.audio.intake.reason}`);
|
|
606774
607133
|
}
|
|
606775
607134
|
if (snapshot.audio.analysis) {
|
|
607135
|
+
lines.push(`Live sound summary: ${formatAudioAnalysisDashboardSummary(snapshot.audio.analysis)}`);
|
|
606776
607136
|
lines.push("Live sound analysis:");
|
|
606777
607137
|
lines.push(cleanPreview(snapshot.audio.analysis, 1600));
|
|
606778
607138
|
}
|
|
@@ -607098,10 +607458,11 @@ function formatLiveStatus(snapshot) {
|
|
|
607098
607458
|
const lines = [
|
|
607099
607459
|
`Live streams: video=${cfg.videoEnabled ? "on" : "off"} infer=${cfg.inferEnabled ? "on" : "off"} clip=${cfg.clipEnabled ? "on" : "off"} audio=${cfg.audioEnabled ? "on" : "off"} asr=${cfg.asrEnabled ? "on" : "off"} sounds=${cfg.audioAnalysisEnabled ? "on" : "off"} output=${cfg.audioOutputEnabled ? "on" : "off"}`,
|
|
607100
607460
|
`Camera: ${cfg.selectedCamera || "none"}`,
|
|
607101
|
-
`Camera stream: ${cfg.selectedCamera ? `${cfg.cameraStreams?.[cfg.selectedCamera]?.enabled !== false ? "enabled" : "disabled"} ${formatLiveCameraResolution(cfg.cameraStreams?.[cfg.selectedCamera]?.resolution)}` : "none"}`,
|
|
607461
|
+
`Camera stream: ${cfg.selectedCamera ? `${cfg.cameraStreams?.[cfg.selectedCamera]?.enabled !== false ? "enabled" : "disabled"} ${formatLiveCameraResolution(cfg.cameraStreams?.[cfg.selectedCamera]?.resolution)} ${formatLiveCameraFps(cfg.cameraStreams?.[cfg.selectedCamera]?.fps)}` : "none"}`,
|
|
607102
607462
|
`Camera orientation: ${cfg.selectedCamera ? describeCameraOrientation(cfg.cameraOrientation?.[cfg.selectedCamera]) : "none"}`,
|
|
607103
607463
|
`Audio input: ${cfg.selectedAudioInput || "none"}`,
|
|
607104
607464
|
`Audio output: ${cfg.selectedAudioOutput || "none"}`,
|
|
607465
|
+
`Resources: ${formatLiveMenuResourceSummary()}`,
|
|
607105
607466
|
`Snapshot: ${liveSnapshotPath(snapshot.repoRoot)}`
|
|
607106
607467
|
];
|
|
607107
607468
|
if (snapshot.video?.updatedAt) {
|
|
@@ -607112,7 +607473,7 @@ function formatLiveStatus(snapshot) {
|
|
|
607112
607473
|
}
|
|
607113
607474
|
return lines.join("\n");
|
|
607114
607475
|
}
|
|
607115
|
-
var MIN_VIDEO_INTERVAL_MS, LOW_LATENCY_VIDEO_INTERVAL_MS, MIN_AUDIO_INTERVAL_MS, LOW_LATENCY_AUDIO_INTERVAL_MS, AUDIO_ACTIVITY_INTERVAL_MS, AUDIO_ACTIVITY_SAMPLE_SEC, DEFAULT_CAMERA_RESOLUTION, DEFAULT_CONFIG7, managers, DASHBOARD_ANSI_STICKY_PATTERN, _liveDashboardMaximizedCamera, LiveSensorManager;
|
|
607476
|
+
var MIN_VIDEO_INTERVAL_MS, LOW_LATENCY_VIDEO_INTERVAL_MS, MIN_AUDIO_INTERVAL_MS, LOW_LATENCY_AUDIO_INTERVAL_MS, AUDIO_ACTIVITY_INTERVAL_MS, AUDIO_ACTIVITY_SAMPLE_SEC, DEFAULT_CAMERA_RESOLUTION, DEFAULT_CAMERA_FPS, DEFAULT_CONFIG7, managers, DASHBOARD_ANSI_STICKY_PATTERN, _liveDashboardMaximizedCamera, LiveSensorManager;
|
|
607116
607477
|
var init_live_sensors = __esm({
|
|
607117
607478
|
"packages/cli/src/tui/live-sensors.ts"() {
|
|
607118
607479
|
"use strict";
|
|
@@ -607130,6 +607491,7 @@ var init_live_sensors = __esm({
|
|
|
607130
607491
|
AUDIO_ACTIVITY_INTERVAL_MS = 200;
|
|
607131
607492
|
AUDIO_ACTIVITY_SAMPLE_SEC = 0.2;
|
|
607132
607493
|
DEFAULT_CAMERA_RESOLUTION = "1920x1080";
|
|
607494
|
+
DEFAULT_CAMERA_FPS = 8;
|
|
607133
607495
|
DEFAULT_CONFIG7 = {
|
|
607134
607496
|
videoEnabled: false,
|
|
607135
607497
|
audioEnabled: false,
|
|
@@ -607156,6 +607518,10 @@ var init_live_sensors = __esm({
|
|
|
607156
607518
|
config: this.config,
|
|
607157
607519
|
devices: this.devices
|
|
607158
607520
|
};
|
|
607521
|
+
try {
|
|
607522
|
+
getModelBroker().startPolling(2500);
|
|
607523
|
+
} catch {
|
|
607524
|
+
}
|
|
607159
607525
|
this.ensureLoops();
|
|
607160
607526
|
}
|
|
607161
607527
|
repoRoot;
|
|
@@ -607339,7 +607705,8 @@ var init_live_sensors = __esm({
|
|
|
607339
607705
|
...this.config.cameraStreams ?? {},
|
|
607340
607706
|
[fallback]: {
|
|
607341
607707
|
enabled: true,
|
|
607342
|
-
resolution: this.config.cameraStreams?.[fallback]?.resolution ?? DEFAULT_CAMERA_RESOLUTION
|
|
607708
|
+
resolution: this.config.cameraStreams?.[fallback]?.resolution ?? DEFAULT_CAMERA_RESOLUTION,
|
|
607709
|
+
fps: this.config.cameraStreams?.[fallback]?.fps ?? DEFAULT_CAMERA_FPS
|
|
607343
607710
|
}
|
|
607344
607711
|
}
|
|
607345
607712
|
};
|
|
@@ -607360,13 +607727,18 @@ var init_live_sensors = __esm({
|
|
|
607360
607727
|
const target = device || this.config.selectedCamera;
|
|
607361
607728
|
return target ? this.config.cameraStreams?.[target]?.resolution ?? DEFAULT_CAMERA_RESOLUTION : DEFAULT_CAMERA_RESOLUTION;
|
|
607362
607729
|
}
|
|
607730
|
+
getCameraStreamFps(device = this.config.selectedCamera) {
|
|
607731
|
+
const target = device || this.config.selectedCamera;
|
|
607732
|
+
return target ? normalizeLiveCameraFps(this.config.cameraStreams?.[target]?.fps ?? DEFAULT_CAMERA_FPS) : DEFAULT_CAMERA_FPS;
|
|
607733
|
+
}
|
|
607363
607734
|
setCameraStreamEnabled(device, enabled2) {
|
|
607364
607735
|
const target = device || this.config.selectedCamera;
|
|
607365
607736
|
if (!target) return null;
|
|
607366
607737
|
const current = this.config.cameraStreams?.[target];
|
|
607367
607738
|
const next = {
|
|
607368
607739
|
enabled: enabled2,
|
|
607369
|
-
resolution: current?.resolution ?? DEFAULT_CAMERA_RESOLUTION
|
|
607740
|
+
resolution: current?.resolution ?? DEFAULT_CAMERA_RESOLUTION,
|
|
607741
|
+
fps: current?.fps ?? DEFAULT_CAMERA_FPS
|
|
607370
607742
|
};
|
|
607371
607743
|
this.config = {
|
|
607372
607744
|
...this.config,
|
|
@@ -607388,7 +607760,30 @@ var init_live_sensors = __esm({
|
|
|
607388
607760
|
const current = this.config.cameraStreams?.[target];
|
|
607389
607761
|
const next = {
|
|
607390
607762
|
enabled: current?.enabled ?? target === this.config.selectedCamera,
|
|
607391
|
-
resolution: normalizeLiveCameraResolution(resolution)
|
|
607763
|
+
resolution: normalizeLiveCameraResolution(resolution),
|
|
607764
|
+
fps: current?.fps ?? DEFAULT_CAMERA_FPS
|
|
607765
|
+
};
|
|
607766
|
+
this.config = {
|
|
607767
|
+
...this.config,
|
|
607768
|
+
cameraStreams: {
|
|
607769
|
+
...this.config.cameraStreams ?? {},
|
|
607770
|
+
[target]: next
|
|
607771
|
+
}
|
|
607772
|
+
};
|
|
607773
|
+
this.videoGeneration++;
|
|
607774
|
+
this.lastStreamFrameMtime.delete(target);
|
|
607775
|
+
this.persist();
|
|
607776
|
+
this.ensureLoops();
|
|
607777
|
+
return next;
|
|
607778
|
+
}
|
|
607779
|
+
setCameraStreamFps(device, fps) {
|
|
607780
|
+
const target = device || this.config.selectedCamera;
|
|
607781
|
+
if (!target) return null;
|
|
607782
|
+
const current = this.config.cameraStreams?.[target];
|
|
607783
|
+
const next = {
|
|
607784
|
+
enabled: current?.enabled ?? target === this.config.selectedCamera,
|
|
607785
|
+
resolution: current?.resolution ?? DEFAULT_CAMERA_RESOLUTION,
|
|
607786
|
+
fps: normalizeLiveCameraFps(fps)
|
|
607392
607787
|
};
|
|
607393
607788
|
this.config = {
|
|
607394
607789
|
...this.config,
|
|
@@ -607642,7 +608037,8 @@ var init_live_sensors = __esm({
|
|
|
607642
608037
|
if (!device.id || /metadata\/non-capture/i.test(device.detail ?? "")) continue;
|
|
607643
608038
|
next[device.id] = {
|
|
607644
608039
|
enabled: current[device.id]?.enabled ?? device.id === selected,
|
|
607645
|
-
resolution: current[device.id]?.resolution ?? DEFAULT_CAMERA_RESOLUTION
|
|
608040
|
+
resolution: current[device.id]?.resolution ?? DEFAULT_CAMERA_RESOLUTION,
|
|
608041
|
+
fps: current[device.id]?.fps ?? DEFAULT_CAMERA_FPS
|
|
607646
608042
|
};
|
|
607647
608043
|
}
|
|
607648
608044
|
return next;
|
|
@@ -607662,7 +608058,8 @@ var init_live_sensors = __esm({
|
|
|
607662
608058
|
...this.config.cameraStreams ?? {},
|
|
607663
608059
|
[selected]: {
|
|
607664
608060
|
enabled: true,
|
|
607665
|
-
resolution: this.config.cameraStreams?.[selected]?.resolution ?? DEFAULT_CAMERA_RESOLUTION
|
|
608061
|
+
resolution: this.config.cameraStreams?.[selected]?.resolution ?? DEFAULT_CAMERA_RESOLUTION,
|
|
608062
|
+
fps: this.config.cameraStreams?.[selected]?.fps ?? DEFAULT_CAMERA_FPS
|
|
607666
608063
|
}
|
|
607667
608064
|
};
|
|
607668
608065
|
}
|
|
@@ -607882,7 +608279,8 @@ var init_live_sensors = __esm({
|
|
|
607882
608279
|
action: "capture",
|
|
607883
608280
|
device: source,
|
|
607884
608281
|
rotate_degrees: rotation,
|
|
607885
|
-
...captureSize ? { width: captureSize.width, height: captureSize.height } : {}
|
|
608282
|
+
...captureSize ? { width: captureSize.width, height: captureSize.height } : {},
|
|
608283
|
+
framerate: this.getCameraStreamFps(source)
|
|
607886
608284
|
});
|
|
607887
608285
|
if (!this.isVideoWorkCurrent(generation)) return { ok: false, message: "Video context is disabled." };
|
|
607888
608286
|
if (!result.success) {
|
|
@@ -608345,27 +608743,20 @@ ${output}`).join("\n\n");
|
|
|
608345
608743
|
audioErrors.push(`No live input signal detected from selected input ${capture.device}; keeping the explicit selection.`);
|
|
608346
608744
|
}
|
|
608347
608745
|
if (this.config.asrEnabled) {
|
|
608348
|
-
|
|
608349
|
-
|
|
608350
|
-
|
|
608351
|
-
|
|
608352
|
-
|
|
608353
|
-
|
|
608354
|
-
|
|
608355
|
-
|
|
608356
|
-
|
|
608357
|
-
|
|
608358
|
-
|
|
608359
|
-
|
|
608360
|
-
|
|
608361
|
-
|
|
608362
|
-
transcript = cleanPreview(typeof asr.llmContent === "string" ? asr.llmContent : "", 1600);
|
|
608363
|
-
asrBackend = extractAsrBackend(asr.output) ?? "whisper";
|
|
608364
|
-
} else {
|
|
608365
|
-
audioErrors.push(`ASR unavailable: ${asr.error || asr.output || "unknown error"}`);
|
|
608366
|
-
}
|
|
608367
|
-
} catch (err) {
|
|
608368
|
-
audioErrors.push(`ASR unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
608746
|
+
const liveAsr = getListenLiveState();
|
|
608747
|
+
const now2 = Date.now();
|
|
608748
|
+
const finalAge = liveAsr.lastTranscriptAt > 0 ? now2 - liveAsr.lastTranscriptAt : Number.POSITIVE_INFINITY;
|
|
608749
|
+
const partialAge = liveAsr.lastPartialTranscriptAt > 0 ? now2 - liveAsr.lastPartialTranscriptAt : Number.POSITIVE_INFINITY;
|
|
608750
|
+
if (liveAsr.lastTranscript && finalAge < 45e3) {
|
|
608751
|
+
transcript = cleanPreview(liveAsr.lastTranscript, 1600);
|
|
608752
|
+
asrBackend = [liveAsr.backend, liveAsr.model, "streaming"].filter(Boolean).join(" ");
|
|
608753
|
+
} else if (liveAsr.lastPartialTranscript && partialAge < 2e4) {
|
|
608754
|
+
transcript = cleanPreview(liveAsr.lastPartialTranscript, 1600);
|
|
608755
|
+
asrBackend = [liveAsr.backend, liveAsr.model, "streaming-partial"].filter(Boolean).join(" ");
|
|
608756
|
+
} else if (liveAsr.phase === "idle") {
|
|
608757
|
+
audioErrors.push("Streaming ASR is not running. Start /live run or /listen to enable CUDA Whisper streaming.");
|
|
608758
|
+
} else if (liveAsr.lastStatus) {
|
|
608759
|
+
audioErrors.push(`Streaming ASR status: ${liveAsr.lastStatus}`);
|
|
608369
608760
|
}
|
|
608370
608761
|
}
|
|
608371
608762
|
if (this.config.audioAnalysisEnabled) {
|
|
@@ -608387,6 +608778,13 @@ ${output}`).join("\n\n");
|
|
|
608387
608778
|
analysis = parts.filter(Boolean).join("\n");
|
|
608388
608779
|
}
|
|
608389
608780
|
if (transcript) {
|
|
608781
|
+
asrLease = this.resourceArbiter.claim({
|
|
608782
|
+
owner: "live-asr",
|
|
608783
|
+
priority: 88,
|
|
608784
|
+
ttlMs: 18e3,
|
|
608785
|
+
reason: "live streaming ASR transcript accepted",
|
|
608786
|
+
suppress: ["vision"]
|
|
608787
|
+
});
|
|
608390
608788
|
intake = await classifyLiveAudioIntake(transcript, this.intakeAgentConfig);
|
|
608391
608789
|
keepAsrLease = true;
|
|
608392
608790
|
asrLease?.extend(
|
|
@@ -608502,7 +608900,8 @@ ${analysis}` : ""
|
|
|
608502
608900
|
this.cameraStreamers.sync(
|
|
608503
608901
|
this.activeVideoSources(),
|
|
608504
608902
|
(source) => this.getCameraOrientation(source)?.rotation ?? 0,
|
|
608505
|
-
(source) => this.getCameraStreamResolution(source)
|
|
608903
|
+
(source) => this.getCameraStreamResolution(source),
|
|
608904
|
+
(source) => this.getCameraStreamFps(source)
|
|
608506
608905
|
);
|
|
608507
608906
|
}
|
|
608508
608907
|
previewTickIntervalMs() {
|
|
@@ -608593,17 +608992,17 @@ ${analysis}` : ""
|
|
|
608593
608992
|
this.videoTimer.unref?.();
|
|
608594
608993
|
}
|
|
608595
608994
|
scheduleAudioLoop(delayMs) {
|
|
608596
|
-
const wantsAudio = this.config.audioEnabled &&
|
|
608995
|
+
const wantsAudio = this.config.audioEnabled && this.config.audioAnalysisEnabled;
|
|
608597
608996
|
if (!wantsAudio || this.audioTimer) return;
|
|
608598
608997
|
this.audioTimer = setTimeout(async () => {
|
|
608599
608998
|
this.audioTimer = null;
|
|
608600
|
-
const stillWantsAudio = this.config.audioEnabled &&
|
|
608999
|
+
const stillWantsAudio = this.config.audioEnabled && this.config.audioAnalysisEnabled;
|
|
608601
609000
|
if (!stillWantsAudio) return;
|
|
608602
609001
|
try {
|
|
608603
609002
|
await this.sampleAudioNow();
|
|
608604
609003
|
} catch {
|
|
608605
609004
|
} finally {
|
|
608606
|
-
const wantsNext = this.config.audioEnabled &&
|
|
609005
|
+
const wantsNext = this.config.audioEnabled && this.config.audioAnalysisEnabled;
|
|
608607
609006
|
if (wantsNext) {
|
|
608608
609007
|
this.scheduleAudioLoop(this.config.audioIntervalMs);
|
|
608609
609008
|
}
|
|
@@ -608641,7 +609040,7 @@ ${analysis}` : ""
|
|
|
608641
609040
|
clearTimeout(this.previewTickTimer);
|
|
608642
609041
|
this.previewTickTimer = null;
|
|
608643
609042
|
}
|
|
608644
|
-
const wantsAudio = this.config.audioEnabled &&
|
|
609043
|
+
const wantsAudio = this.config.audioEnabled && this.config.audioAnalysisEnabled;
|
|
608645
609044
|
if (wantsAudio && !this.audioTimer) {
|
|
608646
609045
|
this.scheduleAudioLoop(0);
|
|
608647
609046
|
} else if (!wantsAudio && this.audioTimer) {
|
|
@@ -613305,12 +613704,16 @@ var init_command_registry = __esm({
|
|
|
613305
613704
|
["/livechat", "Alias for /live chat, used by the top live button"],
|
|
613306
613705
|
["/live camera <device>", "Select a camera and render an ASCII preview"],
|
|
613307
613706
|
["/live resolution [camera] 720p|1080p|1440p|4k|native", "Set per-camera live stream resolution (default 1080p)"],
|
|
613707
|
+
["/live fps [camera] 1|2|4|8|12|15|24|30", "Set per-camera live stream FPS (default 8)"],
|
|
613308
613708
|
["/live camera-on|camera-off [camera]", "Enable or disable an individual live camera stream"],
|
|
613309
613709
|
["/live rotate auto|cw|ccw|180|none [camera]", "Detect or set persistent camera orientation correction"],
|
|
613310
613710
|
["/live infer [now|on|off]", "Toggle or run camera object/location inference"],
|
|
613311
613711
|
["/live clip [on|off]", "Toggle CLIP visual-memory recognition on live frames"],
|
|
613312
613712
|
["/live audio|asr|sounds on|off", "Toggle live audio context streams"],
|
|
613313
613713
|
["/live stop", "Disable all live sensor streams"],
|
|
613714
|
+
["/camera", "Open compact camera preset menu shared with /live"],
|
|
613715
|
+
["/camera status", "Show selected camera and per-device presets"],
|
|
613716
|
+
["/camera select|preview|resolution|fps|rotate", "Manage camera source, preview, resolution, FPS, and orientation"],
|
|
613314
613717
|
["/paste", "Attach clipboard image content to the current or next prompt"],
|
|
613315
613718
|
["/image", "Open image-generation model/setup menu"],
|
|
613316
613719
|
[
|
|
@@ -616803,7 +617206,13 @@ function renderVoiceChatCoalescedRow(row, opts) {
|
|
|
616803
617206
|
return;
|
|
616804
617207
|
}
|
|
616805
617208
|
if (_voiceChatCoalesce) {
|
|
616806
|
-
|
|
617209
|
+
const replacePrefix = opts?.replaceLastPrefix;
|
|
617210
|
+
const lastIndex = _voiceChatCoalesce.rows.length - 1;
|
|
617211
|
+
if (replacePrefix && lastIndex >= 0 && _voiceChatCoalesce.rows[lastIndex]?.startsWith(replacePrefix)) {
|
|
617212
|
+
_voiceChatCoalesce.rows[lastIndex] = row;
|
|
617213
|
+
} else {
|
|
617214
|
+
_voiceChatCoalesce.rows.push(row);
|
|
617215
|
+
}
|
|
616807
617216
|
host.refreshDynamicBlocks();
|
|
616808
617217
|
return;
|
|
616809
617218
|
}
|
|
@@ -653830,6 +654239,10 @@ sleep 1
|
|
|
653830
654239
|
await handleLiveCommand(ctx3, arg);
|
|
653831
654240
|
return "handled";
|
|
653832
654241
|
}
|
|
654242
|
+
case "camera": {
|
|
654243
|
+
await handleCameraCommand(ctx3, arg);
|
|
654244
|
+
return "handled";
|
|
654245
|
+
}
|
|
653833
654246
|
case "livechat": {
|
|
653834
654247
|
await handleLiveCommand(ctx3, "chat");
|
|
653835
654248
|
return "handled";
|
|
@@ -659539,6 +659952,19 @@ function attachLiveSinks(ctx3, manager, agentEnabled = false, speechEnabledOverr
|
|
|
659539
659952
|
manager.setAgentActionSink(void 0);
|
|
659540
659953
|
}
|
|
659541
659954
|
}
|
|
659955
|
+
async function ensureLiveStreamingAsr(ctx3, manager) {
|
|
659956
|
+
if (!manager.getConfig().asrEnabled) return;
|
|
659957
|
+
if (!ctx3.listenStart) return;
|
|
659958
|
+
ctx3.listenSetMode?.("auto");
|
|
659959
|
+
const msg = await ctx3.listenStart("live");
|
|
659960
|
+
if (!/already active/i.test(msg)) renderInfo(`Live ASR: ${msg}`);
|
|
659961
|
+
}
|
|
659962
|
+
async function releaseLiveStreamingAsr(ctx3) {
|
|
659963
|
+
try {
|
|
659964
|
+
await ctx3.listenRelease?.("live");
|
|
659965
|
+
} catch {
|
|
659966
|
+
}
|
|
659967
|
+
}
|
|
659542
659968
|
async function startLiveVoicechat(ctx3, manager) {
|
|
659543
659969
|
const speechEnabled = Boolean(ctx3.voiceSpeak);
|
|
659544
659970
|
attachLiveSinks(ctx3, manager, false, speechEnabled);
|
|
@@ -659588,7 +660014,7 @@ async function chooseLiveDevice(ctx3, title, devices, activeKey) {
|
|
|
659588
660014
|
return devices.find((device) => device.id === result.key) ?? null;
|
|
659589
660015
|
}
|
|
659590
660016
|
function liveCameraStreamSummary(manager, deviceId) {
|
|
659591
|
-
return `${manager.isCameraStreamEnabled(deviceId) ? liveEnabled(true) : liveEnabled(false)} · ${formatLiveCameraResolution(manager.getCameraStreamResolution(deviceId))} · ${formatCameraRotation(manager.getCameraRotation(deviceId))}`;
|
|
660017
|
+
return `${manager.isCameraStreamEnabled(deviceId) ? liveEnabled(true) : liveEnabled(false)} · ${formatLiveCameraResolution(manager.getCameraStreamResolution(deviceId))} · ${formatLiveCameraFps(manager.getCameraStreamFps(deviceId))} · ${formatCameraRotation(manager.getCameraRotation(deviceId))}`;
|
|
659592
660018
|
}
|
|
659593
660019
|
async function chooseLiveCameraResolution(ctx3, manager, deviceId) {
|
|
659594
660020
|
const current = manager.getCameraStreamResolution(deviceId);
|
|
@@ -659607,11 +660033,29 @@ async function chooseLiveCameraResolution(ctx3, manager, deviceId) {
|
|
|
659607
660033
|
if (!result.confirmed || !result.key) return null;
|
|
659608
660034
|
return normalizeLiveCameraResolution(result.key);
|
|
659609
660035
|
}
|
|
660036
|
+
async function chooseLiveCameraFps(ctx3, manager, deviceId) {
|
|
660037
|
+
const current = manager.getCameraStreamFps(deviceId);
|
|
660038
|
+
const choices = [1, 2, 4, 8, 12, 15, 24, 30];
|
|
660039
|
+
const result = await tuiSelect({
|
|
660040
|
+
items: choices.map((fps) => ({
|
|
660041
|
+
key: String(fps),
|
|
660042
|
+
label: formatLiveCameraFps(fps),
|
|
660043
|
+
detail: fps <= 4 ? "lowest hardware burden" : fps === 8 ? "default balanced live stream rate" : "smoother preview, higher camera/CPU burden"
|
|
660044
|
+
})),
|
|
660045
|
+
title: `Camera FPS — ${deviceId}`,
|
|
660046
|
+
activeKey: String(current),
|
|
660047
|
+
rl: ctx3.rl,
|
|
660048
|
+
availableRows: ctx3.availableContentRows?.()
|
|
660049
|
+
});
|
|
660050
|
+
if (!result.confirmed || !result.key) return null;
|
|
660051
|
+
return normalizeLiveCameraFps(result.key);
|
|
660052
|
+
}
|
|
659610
660053
|
async function showLiveCameraMenu(ctx3, manager, device) {
|
|
659611
660054
|
while (true) {
|
|
659612
660055
|
const selected = manager.getConfig().selectedCamera === device.id;
|
|
659613
660056
|
const enabled2 = manager.isCameraStreamEnabled(device.id);
|
|
659614
660057
|
const resolution = manager.getCameraStreamResolution(device.id);
|
|
660058
|
+
const fps = manager.getCameraStreamFps(device.id);
|
|
659615
660059
|
const items = [
|
|
659616
660060
|
{ key: "hdr:camera", label: selectColors.dim(`─── Camera ${device.id} ───`) },
|
|
659617
660061
|
{
|
|
@@ -659633,6 +660077,11 @@ async function showLiveCameraMenu(ctx3, manager, device) {
|
|
|
659633
660077
|
label: "Set Stream Resolution",
|
|
659634
660078
|
detail: formatLiveCameraResolution(resolution)
|
|
659635
660079
|
},
|
|
660080
|
+
{
|
|
660081
|
+
key: "fps",
|
|
660082
|
+
label: "Set Stream FPS",
|
|
660083
|
+
detail: formatLiveCameraFps(fps)
|
|
660084
|
+
},
|
|
659636
660085
|
{
|
|
659637
660086
|
key: "preview",
|
|
659638
660087
|
label: "Preview Camera",
|
|
@@ -659685,6 +660134,14 @@ async function showLiveCameraMenu(ctx3, manager, device) {
|
|
|
659685
660134
|
}
|
|
659686
660135
|
continue;
|
|
659687
660136
|
}
|
|
660137
|
+
case "fps": {
|
|
660138
|
+
const next = await chooseLiveCameraFps(ctx3, manager, device.id);
|
|
660139
|
+
if (next) {
|
|
660140
|
+
manager.setCameraStreamFps(device.id, next);
|
|
660141
|
+
renderInfo(`Camera ${device.id} stream FPS set to ${formatLiveCameraFps(next)}.`);
|
|
660142
|
+
}
|
|
660143
|
+
continue;
|
|
660144
|
+
}
|
|
659688
660145
|
case "preview":
|
|
659689
660146
|
renderLivePreview(await manager.previewCamera(device.id));
|
|
659690
660147
|
continue;
|
|
@@ -659739,6 +660196,7 @@ async function handleLiveCommand(ctx3, arg) {
|
|
|
659739
660196
|
if (sub2 === "stop" || sub2 === "off" || sub2 === "disable") {
|
|
659740
660197
|
manager.stopAll();
|
|
659741
660198
|
manager.setAgentActionSink(void 0);
|
|
660199
|
+
await releaseLiveStreamingAsr(ctx3);
|
|
659742
660200
|
if (ctx3.isVoiceChatActive?.()) {
|
|
659743
660201
|
try {
|
|
659744
660202
|
await ctx3.voiceChatStop?.();
|
|
@@ -659757,6 +660215,7 @@ async function handleLiveCommand(ctx3, arg) {
|
|
|
659757
660215
|
}
|
|
659758
660216
|
await ensureLiveInventory(manager);
|
|
659759
660217
|
renderInfo(manager.startRunMode({ agent: effectiveAgent, lowLatency: true, speech: effectiveAgent && Boolean(ctx3.voiceSpeak) }));
|
|
660218
|
+
await ensureLiveStreamingAsr(ctx3, manager);
|
|
659760
660219
|
return;
|
|
659761
660220
|
}
|
|
659762
660221
|
if (sub2 === "chat" || sub2 === "voicechat" || sub2 === "voice") {
|
|
@@ -659808,6 +660267,28 @@ async function handleLiveCommand(ctx3, arg) {
|
|
|
659808
660267
|
}
|
|
659809
660268
|
return;
|
|
659810
660269
|
}
|
|
660270
|
+
if (sub2 === "fps" || sub2 === "framerate" || sub2 === "rate") {
|
|
660271
|
+
await ensureLiveInventory(manager);
|
|
660272
|
+
const tokens = parts.slice(1);
|
|
660273
|
+
const rawFps = tokens.pop();
|
|
660274
|
+
if (!rawFps) {
|
|
660275
|
+
renderWarning("Usage: /live fps [camera] 1|2|4|8|12|15|24|30");
|
|
660276
|
+
return;
|
|
660277
|
+
}
|
|
660278
|
+
const device = tokens.join(" ") || manager.getConfig().selectedCamera;
|
|
660279
|
+
if (!device) {
|
|
660280
|
+
renderWarning("No camera selected. Use /live camera <device> first.");
|
|
660281
|
+
return;
|
|
660282
|
+
}
|
|
660283
|
+
try {
|
|
660284
|
+
const fps = normalizeLiveCameraFps(rawFps);
|
|
660285
|
+
manager.setCameraStreamFps(device, fps);
|
|
660286
|
+
renderInfo(`Camera ${device} stream FPS set to ${formatLiveCameraFps(fps)}.`);
|
|
660287
|
+
} catch (err) {
|
|
660288
|
+
renderWarning(err instanceof Error ? err.message : String(err));
|
|
660289
|
+
}
|
|
660290
|
+
return;
|
|
660291
|
+
}
|
|
659811
660292
|
if (sub2 === "camera-on" || sub2 === "camera-off" || sub2 === "camera-enable" || sub2 === "camera-disable") {
|
|
659812
660293
|
await ensureLiveInventory(manager);
|
|
659813
660294
|
const device = value2 || manager.getConfig().selectedCamera;
|
|
@@ -659834,10 +660315,13 @@ async function handleLiveCommand(ctx3, arg) {
|
|
|
659834
660315
|
}
|
|
659835
660316
|
if (sub2 === "asr") {
|
|
659836
660317
|
const cfg = manager.getConfig();
|
|
660318
|
+
const enabled2 = setBool(value2, !cfg.asrEnabled);
|
|
659837
660319
|
manager.configure({
|
|
659838
660320
|
audioEnabled: true,
|
|
659839
|
-
asrEnabled:
|
|
660321
|
+
asrEnabled: enabled2
|
|
659840
660322
|
});
|
|
660323
|
+
if (enabled2) await ensureLiveStreamingAsr(ctx3, manager);
|
|
660324
|
+
else await releaseLiveStreamingAsr(ctx3);
|
|
659841
660325
|
renderInfo(formatLiveStatus(manager.getSnapshot()));
|
|
659842
660326
|
return;
|
|
659843
660327
|
}
|
|
@@ -659906,9 +660390,226 @@ async function handleLiveCommand(ctx3, arg) {
|
|
|
659906
660390
|
}
|
|
659907
660391
|
await showLiveMenu(ctx3, manager);
|
|
659908
660392
|
}
|
|
660393
|
+
function formatCameraStatus(manager) {
|
|
660394
|
+
const snapshot = manager.getSnapshot();
|
|
660395
|
+
const cfg = manager.getConfig();
|
|
660396
|
+
const lines = [
|
|
660397
|
+
`Selected camera: ${cfg.selectedCamera || "none"}`,
|
|
660398
|
+
`Video context: ${cfg.videoEnabled ? "enabled" : "disabled"}`,
|
|
660399
|
+
`Camera devices: ${snapshot.devices.video.length}`
|
|
660400
|
+
];
|
|
660401
|
+
for (const device of snapshot.devices.video) {
|
|
660402
|
+
lines.push(`- ${device.id}: ${liveCameraStreamSummary(manager, device.id)} — ${device.label}${device.detail ? ` — ${device.detail}` : ""}`);
|
|
660403
|
+
}
|
|
660404
|
+
return lines.join("\n");
|
|
660405
|
+
}
|
|
660406
|
+
async function showCameraMenu(ctx3, manager) {
|
|
660407
|
+
await ensureLiveInventory(manager);
|
|
660408
|
+
while (true) {
|
|
660409
|
+
const snapshot = manager.getSnapshot();
|
|
660410
|
+
const cfg = manager.getConfig();
|
|
660411
|
+
const devices = snapshot.devices.video;
|
|
660412
|
+
const items = [
|
|
660413
|
+
{ key: "hdr:camera", label: selectColors.dim("─── Camera Presets ───") },
|
|
660414
|
+
{
|
|
660415
|
+
key: "info:camera",
|
|
660416
|
+
label: selectColors.dim(` selected ${cfg.selectedCamera || "none"} · video ${liveEnabled(cfg.videoEnabled)} · ${devices.length} device(s)`)
|
|
660417
|
+
},
|
|
660418
|
+
{
|
|
660419
|
+
key: "select",
|
|
660420
|
+
label: "Select Primary Camera",
|
|
660421
|
+
detail: liveDeviceSummary(devices, cfg.selectedCamera)
|
|
660422
|
+
},
|
|
660423
|
+
{
|
|
660424
|
+
key: "preview",
|
|
660425
|
+
label: "Preview Selected Camera",
|
|
660426
|
+
detail: "uses selected resolution, fps, and orientation preset"
|
|
660427
|
+
},
|
|
660428
|
+
{
|
|
660429
|
+
key: "toggle-video",
|
|
660430
|
+
label: cfg.videoEnabled ? "Disable Video Context" : "Enable Video Context",
|
|
660431
|
+
detail: "does not delete camera presets"
|
|
660432
|
+
},
|
|
660433
|
+
...devices.map((device, index) => ({
|
|
660434
|
+
key: `camera-device:${index}`,
|
|
660435
|
+
label: `Camera: ${device.id}`,
|
|
660436
|
+
detail: `${liveCameraStreamSummary(manager, device.id)} — ${device.label}${device.detail ? ` — ${device.detail}` : ""}`
|
|
660437
|
+
})),
|
|
660438
|
+
{ key: "refresh", label: "Refresh Devices", detail: "rescan /dev/video* and audio/video inventory" },
|
|
660439
|
+
{ key: "status", label: "Show Camera Status", detail: "selected camera and per-device presets" }
|
|
660440
|
+
];
|
|
660441
|
+
const result = await tuiSelect({
|
|
660442
|
+
items,
|
|
660443
|
+
title: "Camera Presets",
|
|
660444
|
+
activeKey: cfg.selectedCamera ? "preview" : "select",
|
|
660445
|
+
rl: ctx3.rl,
|
|
660446
|
+
availableRows: ctx3.availableContentRows?.(),
|
|
660447
|
+
skipKeys: liveSkipKeys(items),
|
|
660448
|
+
customKeyHint: " r refresh",
|
|
660449
|
+
onCustomKey(_item, key, helpers) {
|
|
660450
|
+
if (key === "r" || key === "R") {
|
|
660451
|
+
void manager.refreshDevices().then(() => helpers.done()).catch(() => helpers.done());
|
|
660452
|
+
return true;
|
|
660453
|
+
}
|
|
660454
|
+
return false;
|
|
660455
|
+
}
|
|
660456
|
+
});
|
|
660457
|
+
if (!result.confirmed || !result.key) return;
|
|
660458
|
+
if (result.key.startsWith("camera-device:")) {
|
|
660459
|
+
const index = Number(result.key.slice("camera-device:".length));
|
|
660460
|
+
const device = devices[index];
|
|
660461
|
+
if (device) await showLiveCameraMenu(ctx3, manager, device);
|
|
660462
|
+
continue;
|
|
660463
|
+
}
|
|
660464
|
+
switch (result.key) {
|
|
660465
|
+
case "select": {
|
|
660466
|
+
const device = await chooseLiveDevice(ctx3, "Select Camera", devices, cfg.selectedCamera);
|
|
660467
|
+
if (device) {
|
|
660468
|
+
manager.configure({ selectedCamera: device.id, videoEnabled: true });
|
|
660469
|
+
manager.setCameraStreamEnabled(device.id, true);
|
|
660470
|
+
renderInfo(`Selected camera ${device.id}.`);
|
|
660471
|
+
}
|
|
660472
|
+
continue;
|
|
660473
|
+
}
|
|
660474
|
+
case "preview":
|
|
660475
|
+
renderLivePreview(await manager.previewCamera(cfg.selectedCamera));
|
|
660476
|
+
continue;
|
|
660477
|
+
case "toggle-video":
|
|
660478
|
+
manager.configure({ videoEnabled: !cfg.videoEnabled });
|
|
660479
|
+
continue;
|
|
660480
|
+
case "refresh":
|
|
660481
|
+
renderInfo(formatLiveInventory(await manager.refreshDevices()));
|
|
660482
|
+
continue;
|
|
660483
|
+
case "status":
|
|
660484
|
+
renderInfo(formatCameraStatus(manager));
|
|
660485
|
+
continue;
|
|
660486
|
+
default:
|
|
660487
|
+
return;
|
|
660488
|
+
}
|
|
660489
|
+
}
|
|
660490
|
+
}
|
|
660491
|
+
async function handleCameraCommand(ctx3, arg) {
|
|
660492
|
+
const manager = getLiveSensorManager(ctx3.repoRoot);
|
|
660493
|
+
attachLiveSinks(ctx3, manager);
|
|
660494
|
+
const parts = arg.trim().split(/\s+/).filter(Boolean);
|
|
660495
|
+
const sub2 = (parts[0] || "").toLowerCase();
|
|
660496
|
+
const value2 = parts.slice(1).join(" ");
|
|
660497
|
+
if (sub2 === "status") {
|
|
660498
|
+
await ensureLiveInventory(manager);
|
|
660499
|
+
renderInfo(formatCameraStatus(manager));
|
|
660500
|
+
return;
|
|
660501
|
+
}
|
|
660502
|
+
if (sub2 === "refresh" || sub2 === "devices" || sub2 === "list") {
|
|
660503
|
+
renderInfo(formatLiveInventory(await manager.refreshDevices()));
|
|
660504
|
+
return;
|
|
660505
|
+
}
|
|
660506
|
+
if (sub2 === "preview" || sub2 === "view" || sub2 === "capture") {
|
|
660507
|
+
await ensureLiveInventory(manager);
|
|
660508
|
+
renderLivePreview(await manager.previewCamera(value2 || manager.getConfig().selectedCamera));
|
|
660509
|
+
return;
|
|
660510
|
+
}
|
|
660511
|
+
if (sub2 === "select" || sub2 === "use") {
|
|
660512
|
+
await ensureLiveInventory(manager);
|
|
660513
|
+
const device = value2;
|
|
660514
|
+
if (!device) {
|
|
660515
|
+
const selected = await chooseLiveDevice(ctx3, "Select Camera", manager.getSnapshot().devices.video, manager.getConfig().selectedCamera);
|
|
660516
|
+
if (selected) {
|
|
660517
|
+
manager.configure({ selectedCamera: selected.id, videoEnabled: true });
|
|
660518
|
+
manager.setCameraStreamEnabled(selected.id, true);
|
|
660519
|
+
renderInfo(`Selected camera ${selected.id}.`);
|
|
660520
|
+
}
|
|
660521
|
+
return;
|
|
660522
|
+
}
|
|
660523
|
+
manager.configure({ selectedCamera: device, videoEnabled: true });
|
|
660524
|
+
manager.setCameraStreamEnabled(device, true);
|
|
660525
|
+
renderInfo(`Selected camera ${device}.`);
|
|
660526
|
+
return;
|
|
660527
|
+
}
|
|
660528
|
+
if (sub2 === "on" || sub2 === "enable" || sub2 === "off" || sub2 === "disable") {
|
|
660529
|
+
await ensureLiveInventory(manager);
|
|
660530
|
+
const device = value2 || manager.getConfig().selectedCamera;
|
|
660531
|
+
if (!device) {
|
|
660532
|
+
renderWarning("No camera selected. Use /camera select <device> first.");
|
|
660533
|
+
return;
|
|
660534
|
+
}
|
|
660535
|
+
const enabled2 = sub2 === "on" || sub2 === "enable";
|
|
660536
|
+
manager.setCameraStreamEnabled(device, enabled2);
|
|
660537
|
+
renderInfo(`${enabled2 ? "Enabled" : "Disabled"} camera stream ${device}.`);
|
|
660538
|
+
return;
|
|
660539
|
+
}
|
|
660540
|
+
if (sub2 === "resolution" || sub2 === "res" || sub2 === "size") {
|
|
660541
|
+
await ensureLiveInventory(manager);
|
|
660542
|
+
const tokens = parts.slice(1);
|
|
660543
|
+
const rawResolution = tokens.pop();
|
|
660544
|
+
if (!rawResolution) {
|
|
660545
|
+
renderWarning("Usage: /camera resolution [camera] 720p|1080p|1440p|4k|native");
|
|
660546
|
+
return;
|
|
660547
|
+
}
|
|
660548
|
+
const device = tokens.join(" ") || manager.getConfig().selectedCamera;
|
|
660549
|
+
if (!device) {
|
|
660550
|
+
renderWarning("No camera selected. Use /camera select <device> first.");
|
|
660551
|
+
return;
|
|
660552
|
+
}
|
|
660553
|
+
try {
|
|
660554
|
+
const resolution = normalizeLiveCameraResolution(rawResolution);
|
|
660555
|
+
manager.setCameraStreamResolution(device, resolution);
|
|
660556
|
+
renderInfo(`Camera ${device} stream resolution set to ${formatLiveCameraResolution(resolution)}.`);
|
|
660557
|
+
} catch (err) {
|
|
660558
|
+
renderWarning(err instanceof Error ? err.message : String(err));
|
|
660559
|
+
}
|
|
660560
|
+
return;
|
|
660561
|
+
}
|
|
660562
|
+
if (sub2 === "fps" || sub2 === "framerate" || sub2 === "rate") {
|
|
660563
|
+
await ensureLiveInventory(manager);
|
|
660564
|
+
const tokens = parts.slice(1);
|
|
660565
|
+
const rawFps = tokens.pop();
|
|
660566
|
+
if (!rawFps) {
|
|
660567
|
+
renderWarning("Usage: /camera fps [camera] 1|2|4|8|12|15|24|30");
|
|
660568
|
+
return;
|
|
660569
|
+
}
|
|
660570
|
+
const device = tokens.join(" ") || manager.getConfig().selectedCamera;
|
|
660571
|
+
if (!device) {
|
|
660572
|
+
renderWarning("No camera selected. Use /camera select <device> first.");
|
|
660573
|
+
return;
|
|
660574
|
+
}
|
|
660575
|
+
try {
|
|
660576
|
+
const fps = normalizeLiveCameraFps(rawFps);
|
|
660577
|
+
manager.setCameraStreamFps(device, fps);
|
|
660578
|
+
renderInfo(`Camera ${device} stream FPS set to ${formatLiveCameraFps(fps)}.`);
|
|
660579
|
+
} catch (err) {
|
|
660580
|
+
renderWarning(err instanceof Error ? err.message : String(err));
|
|
660581
|
+
}
|
|
660582
|
+
return;
|
|
660583
|
+
}
|
|
660584
|
+
if (sub2 === "rotate" || sub2 === "rotation" || sub2 === "orient" || sub2 === "orientation") {
|
|
660585
|
+
await ensureLiveInventory(manager);
|
|
660586
|
+
const mode = (parts[1] || "cycle").toLowerCase();
|
|
660587
|
+
const deviceArg = parts.slice(2).join(" ") || manager.getConfig().selectedCamera;
|
|
660588
|
+
if (mode === "auto" || mode === "detect") {
|
|
660589
|
+
renderInfo("Detecting camera orientation...");
|
|
660590
|
+
renderInfo(await manager.autoOrientCamera(deviceArg));
|
|
660591
|
+
return;
|
|
660592
|
+
}
|
|
660593
|
+
if (mode === "cycle") {
|
|
660594
|
+
const orientation = manager.cycleCameraRotation(deviceArg);
|
|
660595
|
+
renderInfo(orientation ? `Camera rotation set to ${formatCameraRotation(orientation.rotation)} for ${deviceArg || manager.getConfig().selectedCamera || "selected camera"}.` : "No camera selected. Use /camera select <device> first.");
|
|
660596
|
+
return;
|
|
660597
|
+
}
|
|
660598
|
+
try {
|
|
660599
|
+
const rotation = normalizeLiveCameraRotation(mode);
|
|
660600
|
+
const orientation = manager.setCameraRotation(deviceArg, rotation, "manual", `/camera rotate ${mode}`);
|
|
660601
|
+
renderInfo(orientation ? `Camera rotation set to ${formatCameraRotation(orientation.rotation)} for ${deviceArg || manager.getConfig().selectedCamera || "selected camera"}.` : "No camera selected. Use /camera select <device> first.");
|
|
660602
|
+
} catch (err) {
|
|
660603
|
+
renderWarning(err instanceof Error ? err.message : String(err));
|
|
660604
|
+
}
|
|
660605
|
+
return;
|
|
660606
|
+
}
|
|
660607
|
+
await showCameraMenu(ctx3, manager);
|
|
660608
|
+
}
|
|
659909
660609
|
async function showLiveMenu(ctx3, manager) {
|
|
659910
660610
|
await ensureLiveInventory(manager);
|
|
659911
660611
|
while (true) {
|
|
660612
|
+
await refreshLiveResourceModelSnapshot().catch(() => void 0);
|
|
659912
660613
|
const snapshot = manager.getSnapshot();
|
|
659913
660614
|
const cfg = manager.getConfig();
|
|
659914
660615
|
const devices = snapshot.devices;
|
|
@@ -659920,6 +660621,15 @@ async function showLiveMenu(ctx3, manager) {
|
|
|
659920
660621
|
` video ${liveEnabled(cfg.videoEnabled)} · infer ${liveEnabled(cfg.inferEnabled)} · clip ${liveEnabled(cfg.clipEnabled)} · audio ${liveEnabled(cfg.audioEnabled)} · asr ${liveEnabled(cfg.asrEnabled)} · sounds ${liveEnabled(cfg.audioAnalysisEnabled)}`
|
|
659921
660622
|
)
|
|
659922
660623
|
},
|
|
660624
|
+
{
|
|
660625
|
+
key: "info:resources",
|
|
660626
|
+
label: selectColors.dim(` ${formatLiveMenuResourceSummary()}`)
|
|
660627
|
+
},
|
|
660628
|
+
{
|
|
660629
|
+
key: "resources",
|
|
660630
|
+
label: "Show Resource + Model State",
|
|
660631
|
+
detail: "RAM/VRAM by subsystem, CUDA per loaded model, active slots, inflight loads"
|
|
660632
|
+
},
|
|
659923
660633
|
{
|
|
659924
660634
|
key: "run-live",
|
|
659925
660635
|
label: "Start Live Run",
|
|
@@ -660057,6 +660767,7 @@ async function showLiveMenu(ctx3, manager) {
|
|
|
660057
660767
|
case "run-live":
|
|
660058
660768
|
attachLiveSinks(ctx3, manager, false);
|
|
660059
660769
|
renderInfo(manager.startRunMode({ agent: false, lowLatency: true, speech: false }));
|
|
660770
|
+
await ensureLiveStreamingAsr(ctx3, manager);
|
|
660060
660771
|
continue;
|
|
660061
660772
|
case "run-agent":
|
|
660062
660773
|
attachLiveSinks(ctx3, manager, true);
|
|
@@ -660064,6 +660775,7 @@ async function showLiveMenu(ctx3, manager) {
|
|
|
660064
660775
|
renderWarning("Live PFC agent review is unavailable in this context; starting visual/audio feedback only.");
|
|
660065
660776
|
}
|
|
660066
660777
|
renderInfo(manager.startRunMode({ agent: Boolean(ctx3.startBackgroundPrompt), lowLatency: true, speech: Boolean(ctx3.startBackgroundPrompt && ctx3.voiceSpeak) }));
|
|
660778
|
+
await ensureLiveStreamingAsr(ctx3, manager);
|
|
660067
660779
|
continue;
|
|
660068
660780
|
case "run-chat":
|
|
660069
660781
|
await startLiveVoicechat(ctx3, manager);
|
|
@@ -660082,6 +660794,8 @@ async function showLiveMenu(ctx3, manager) {
|
|
|
660082
660794
|
continue;
|
|
660083
660795
|
case "toggle-asr":
|
|
660084
660796
|
manager.configure({ audioEnabled: true, asrEnabled: !cfg.asrEnabled });
|
|
660797
|
+
if (!cfg.asrEnabled) await ensureLiveStreamingAsr(ctx3, manager);
|
|
660798
|
+
else await releaseLiveStreamingAsr(ctx3);
|
|
660085
660799
|
continue;
|
|
660086
660800
|
case "toggle-sounds":
|
|
660087
660801
|
manager.configure({ audioEnabled: true, audioAnalysisEnabled: !cfg.audioAnalysisEnabled });
|
|
@@ -660142,6 +660856,10 @@ Previewing microphone activity...`);
|
|
|
660142
660856
|
renderWarning("Live transcript UI is unavailable in this context.");
|
|
660143
660857
|
}
|
|
660144
660858
|
continue;
|
|
660859
|
+
case "resources":
|
|
660860
|
+
await refreshLiveResourceModelSnapshot(0).catch(() => void 0);
|
|
660861
|
+
renderInfo(formatLiveResourceModelReport(110));
|
|
660862
|
+
continue;
|
|
660145
660863
|
case "refresh":
|
|
660146
660864
|
renderInfo(formatLiveInventory(await manager.refreshDevices()));
|
|
660147
660865
|
continue;
|
|
@@ -660151,6 +660869,7 @@ Previewing microphone activity...`);
|
|
|
660151
660869
|
case "stop":
|
|
660152
660870
|
manager.stopAll();
|
|
660153
660871
|
manager.setAgentActionSink(void 0);
|
|
660872
|
+
await releaseLiveStreamingAsr(ctx3);
|
|
660154
660873
|
if (ctx3.isVoiceChatActive?.()) {
|
|
660155
660874
|
try {
|
|
660156
660875
|
await ctx3.voiceChatStop?.();
|
|
@@ -665225,6 +665944,7 @@ var init_commands = __esm({
|
|
|
665225
665944
|
init_generative_progress();
|
|
665226
665945
|
init_cad_model_viewer();
|
|
665227
665946
|
init_live_sensors();
|
|
665947
|
+
init_live_resource_arbiter();
|
|
665228
665948
|
init_command_registry();
|
|
665229
665949
|
init_hf_token_prompt();
|
|
665230
665950
|
init_dist5();
|
|
@@ -699164,9 +699884,15 @@ Rules:
|
|
|
699164
699884
|
this._retryMicTimer = null;
|
|
699165
699885
|
if (!this.active) return;
|
|
699166
699886
|
try {
|
|
699167
|
-
|
|
699168
|
-
|
|
699169
|
-
|
|
699887
|
+
if (typeof this.listen.release === "function" && typeof this.listen.acquire === "function") {
|
|
699888
|
+
await this.listen.release("voicechat").catch(() => {
|
|
699889
|
+
});
|
|
699890
|
+
await this.listen.acquire("voicechat");
|
|
699891
|
+
} else {
|
|
699892
|
+
await this.listen.stop().catch(() => {
|
|
699893
|
+
});
|
|
699894
|
+
await this.listen.start();
|
|
699895
|
+
}
|
|
699170
699896
|
if (this.verbose) this.onStatus("Mic auto-recovered — LISTENING");
|
|
699171
699897
|
} catch {
|
|
699172
699898
|
}
|
|
@@ -699176,7 +699902,11 @@ Rules:
|
|
|
699176
699902
|
this.listen.on("transcript", this._onTranscript);
|
|
699177
699903
|
this.listen.on("error", this._onError);
|
|
699178
699904
|
try {
|
|
699179
|
-
|
|
699905
|
+
if (typeof this.listen.acquire === "function") {
|
|
699906
|
+
await this.listen.acquire("voicechat");
|
|
699907
|
+
} else {
|
|
699908
|
+
await this.listen.start();
|
|
699909
|
+
}
|
|
699180
699910
|
this.setState("LISTENING");
|
|
699181
699911
|
if (this.verbose) this.onStatus("Mic active — LISTENING for speech...");
|
|
699182
699912
|
} catch (err) {
|
|
@@ -699203,15 +699933,21 @@ Rules:
|
|
|
699203
699933
|
this.finalizeSegment();
|
|
699204
699934
|
}
|
|
699205
699935
|
if (this._onTranscript) {
|
|
699206
|
-
this.listen.
|
|
699936
|
+
if (typeof this.listen.off === "function") this.listen.off("transcript", this._onTranscript);
|
|
699937
|
+
else this.listen.removeAllListeners("transcript");
|
|
699207
699938
|
this._onTranscript = null;
|
|
699208
699939
|
}
|
|
699209
699940
|
if (this._onError) {
|
|
699210
|
-
this.listen.
|
|
699941
|
+
if (typeof this.listen.off === "function") this.listen.off("error", this._onError);
|
|
699942
|
+
else this.listen.removeAllListeners("error");
|
|
699211
699943
|
this._onError = null;
|
|
699212
699944
|
}
|
|
699213
699945
|
try {
|
|
699214
|
-
|
|
699946
|
+
if (typeof this.listen.release === "function") {
|
|
699947
|
+
await this.listen.release("voicechat");
|
|
699948
|
+
} else {
|
|
699949
|
+
await this.listen.stop();
|
|
699950
|
+
}
|
|
699215
699951
|
} catch {
|
|
699216
699952
|
}
|
|
699217
699953
|
this.releaseLiveResources();
|
|
@@ -738311,8 +739047,8 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
738311
739047
|
// Listen mode (transcribe-cli integration + live voice session)
|
|
738312
739048
|
async listenToggle() {
|
|
738313
739049
|
const engine = getListenEngine();
|
|
738314
|
-
if (engine.isActive) {
|
|
738315
|
-
const text2 = await engine.
|
|
739050
|
+
if (engine.isActive && engine.hasOwner("listen")) {
|
|
739051
|
+
const text2 = await engine.release("listen");
|
|
738316
739052
|
statusBar.setRecording(false);
|
|
738317
739053
|
if (voiceSession?.isActive) {
|
|
738318
739054
|
const sessionMsg = await voiceSession.stop();
|
|
@@ -738355,7 +739091,7 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
738355
739091
|
engine.on("info", (msg2) => {
|
|
738356
739092
|
writeContent(() => renderInfo(msg2));
|
|
738357
739093
|
});
|
|
738358
|
-
const msg = await engine.
|
|
739094
|
+
const msg = await engine.acquire("listen");
|
|
738359
739095
|
if (voiceEngine.enabled && voiceEngine.ready) {
|
|
738360
739096
|
try {
|
|
738361
739097
|
voiceSession = new VoiceSession();
|
|
@@ -738384,6 +739120,39 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
738384
739120
|
}
|
|
738385
739121
|
return msg;
|
|
738386
739122
|
},
|
|
739123
|
+
async listenStart(owner = "live") {
|
|
739124
|
+
const engine = getListenEngine();
|
|
739125
|
+
const available = await engine.isAvailable();
|
|
739126
|
+
if (!available) {
|
|
739127
|
+
return "ASR runtime unavailable. Omnius attempted managed Whisper setup; check ~/.omnius/runtimes/asr and CUDA PyTorch.";
|
|
739128
|
+
}
|
|
739129
|
+
const uiEngine = engine;
|
|
739130
|
+
if (!uiEngine.__omniusLiveAsrUiWired) {
|
|
739131
|
+
uiEngine.__omniusLiveAsrUiWired = true;
|
|
739132
|
+
engine.on("recording", (active) => {
|
|
739133
|
+
statusBar.setRecording(active);
|
|
739134
|
+
});
|
|
739135
|
+
engine.on("level", (evt) => {
|
|
739136
|
+
statusBar.setMicActivity(evt.speechActive, evt.levelDb);
|
|
739137
|
+
});
|
|
739138
|
+
engine.on("error", (err) => {
|
|
739139
|
+
writeContent(() => renderWarning(`Live ASR error: ${err.message}`));
|
|
739140
|
+
});
|
|
739141
|
+
engine.on("info", (msg) => {
|
|
739142
|
+
if (/CUDA|Whisper|consensus|ASR|VAD|torch/i.test(msg)) {
|
|
739143
|
+
writeContent(() => renderInfo(`[asr] ${msg}`));
|
|
739144
|
+
}
|
|
739145
|
+
});
|
|
739146
|
+
}
|
|
739147
|
+
return engine.acquire(owner);
|
|
739148
|
+
},
|
|
739149
|
+
async listenRelease(owner = "live") {
|
|
739150
|
+
const engine = getListenEngine();
|
|
739151
|
+
return engine.release(owner);
|
|
739152
|
+
},
|
|
739153
|
+
listenIsActive() {
|
|
739154
|
+
return getListenEngine().isActive;
|
|
739155
|
+
},
|
|
738387
739156
|
async listenSetModel(model) {
|
|
738388
739157
|
const engine = getListenEngine();
|
|
738389
739158
|
engine.setModel(model);
|
|
@@ -738396,8 +739165,8 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
738396
739165
|
},
|
|
738397
739166
|
async listenStop() {
|
|
738398
739167
|
const engine = getListenEngine();
|
|
738399
|
-
const text2 = await engine.stop();
|
|
738400
|
-
statusBar.setRecording(false);
|
|
739168
|
+
const text2 = engine.hasOwner("listen") ? await engine.release("listen") : engine.ownerCount() > 0 ? "Shared ASR is still running for /live, /voicechat, or /call." : await engine.stop();
|
|
739169
|
+
if (engine.ownerCount() === 0) statusBar.setRecording(false);
|
|
738401
739170
|
if (voiceSession?.isActive) {
|
|
738402
739171
|
const runtime = voiceSession.runtime;
|
|
738403
739172
|
await voiceSession.stop();
|
|
@@ -738709,19 +739478,22 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
738709
739478
|
writeContent(() => renderInfo(voiceMsg));
|
|
738710
739479
|
}
|
|
738711
739480
|
const { VoiceChatSession: VoiceChatSession2 } = await Promise.resolve().then(() => (init_voicechat(), voicechat_exports));
|
|
738712
|
-
const
|
|
738713
|
-
const
|
|
738714
|
-
|
|
738715
|
-
|
|
738716
|
-
|
|
738717
|
-
|
|
738718
|
-
|
|
738719
|
-
|
|
738720
|
-
|
|
738721
|
-
|
|
738722
|
-
|
|
738723
|
-
|
|
738724
|
-
|
|
739481
|
+
const listenEng = getListenEngine();
|
|
739482
|
+
const voicechatUiEngine = listenEng;
|
|
739483
|
+
if (!voicechatUiEngine.__omniusVoiceChatAsrUiWired) {
|
|
739484
|
+
voicechatUiEngine.__omniusVoiceChatAsrUiWired = true;
|
|
739485
|
+
listenEng.on("recording", (active) => {
|
|
739486
|
+
statusBar.setRecording(active);
|
|
739487
|
+
});
|
|
739488
|
+
listenEng.on("level", (evt) => {
|
|
739489
|
+
statusBar.setMicActivity(evt.speechActive, evt.levelDb);
|
|
739490
|
+
});
|
|
739491
|
+
listenEng.on("info", (msg) => {
|
|
739492
|
+
if (/consensus rejected|loading whisper|whisper model loaded|Whisper VAD|no accepted speech text|very short utterance/i.test(msg)) {
|
|
739493
|
+
writeContent(() => renderInfo(`[voicechat/asr] ${msg}`));
|
|
739494
|
+
}
|
|
739495
|
+
});
|
|
739496
|
+
}
|
|
738725
739497
|
const summaryRunner = {
|
|
738726
739498
|
injectUserMessage(content) {
|
|
738727
739499
|
if (activeTask?.runner) {
|
|
@@ -738729,6 +739501,7 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
738729
739501
|
}
|
|
738730
739502
|
}
|
|
738731
739503
|
};
|
|
739504
|
+
const voicechatPartialPrefix = `\x1B[38;5;244m[hearing]\x1B[0m`;
|
|
738732
739505
|
_voiceChatSession2 = new VoiceChatSession2({
|
|
738733
739506
|
voice: voiceEngine,
|
|
738734
739507
|
listen: listenEng,
|
|
@@ -738886,11 +739659,20 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
738886
739659
|
},
|
|
738887
739660
|
onUserSpeech(text2) {
|
|
738888
739661
|
writeContent(
|
|
738889
|
-
() => renderVoiceChatCoalescedRow(`\x1B[38;5;45m[you]\x1B[0m ${text2}
|
|
739662
|
+
() => renderVoiceChatCoalescedRow(`\x1B[38;5;45m[you]\x1B[0m ${text2}`, {
|
|
739663
|
+
replaceLastPrefix: voicechatPartialPrefix
|
|
739664
|
+
})
|
|
738890
739665
|
);
|
|
738891
739666
|
},
|
|
738892
|
-
|
|
738893
|
-
|
|
739667
|
+
onPartialTranscript(text2) {
|
|
739668
|
+
const cleaned = text2.trim();
|
|
739669
|
+
if (!cleaned) return;
|
|
739670
|
+
writeContent(
|
|
739671
|
+
() => renderVoiceChatCoalescedRow(
|
|
739672
|
+
`${voicechatPartialPrefix} ${truncateByLines(cleaned, 2, 240)}`,
|
|
739673
|
+
{ replaceLastPrefix: voicechatPartialPrefix }
|
|
739674
|
+
)
|
|
739675
|
+
);
|
|
738894
739676
|
},
|
|
738895
739677
|
onAgentSpeech(text2) {
|
|
738896
739678
|
writeContent(
|
|
@@ -738903,6 +739685,15 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
738903
739685
|
onStateChange(_state4) {
|
|
738904
739686
|
}
|
|
738905
739687
|
});
|
|
739688
|
+
_voiceChatSession2.on("consensusFiltered", (evt) => {
|
|
739689
|
+
const text2 = String(evt?.text ?? "").trim();
|
|
739690
|
+
writeContent(
|
|
739691
|
+
() => renderVoiceChatCoalescedRow(
|
|
739692
|
+
`\x1B[38;5;244m[filtered]\x1B[0m no ASR consensus${text2 ? `: ${truncateByLines(text2, 1, 180)}` : ""}`,
|
|
739693
|
+
{ replaceLastPrefix: voicechatPartialPrefix }
|
|
739694
|
+
)
|
|
739695
|
+
);
|
|
739696
|
+
});
|
|
738906
739697
|
await _voiceChatSession2.start();
|
|
738907
739698
|
},
|
|
738908
739699
|
async voiceChatStop() {
|