omnius 1.0.411 → 1.0.413
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +663 -146
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -41412,7 +41412,7 @@ ${content}`
|
|
|
41412
41412
|
// packages/execution/dist/tools/transcribe-tool.js
|
|
41413
41413
|
import { existsSync as existsSync38, mkdirSync as mkdirSync22, writeFileSync as writeFileSync21, readFileSync as readFileSync30, unlinkSync as unlinkSync5, readdirSync as readdirSync14 } from "node:fs";
|
|
41414
41414
|
import { join as join40, basename as basename7, extname as extname4, resolve as resolve21 } from "node:path";
|
|
41415
|
-
import { homedir as homedir11 } from "node:os";
|
|
41415
|
+
import { homedir as homedir11, tmpdir as tmpdir5 } from "node:os";
|
|
41416
41416
|
function transcriptionPythonEnv(extra = {}) {
|
|
41417
41417
|
const env2 = { ...process.env, ...extra };
|
|
41418
41418
|
applyMediaCudaDeviceFilterToEnv(env2, "asr");
|
|
@@ -41456,17 +41456,30 @@ function isTranscribable(path12) {
|
|
|
41456
41456
|
const ext = extname4(path12).toLowerCase();
|
|
41457
41457
|
return AUDIO_EXTS.has(ext) || VIDEO_EXTS.has(ext);
|
|
41458
41458
|
}
|
|
41459
|
+
async function requireTranscribeCliFrom(packageDir) {
|
|
41460
|
+
const { createRequire: createRequire11 } = await import("node:module");
|
|
41461
|
+
const req3 = createRequire11(import.meta.url);
|
|
41462
|
+
const distPath = join40(packageDir, "dist", "index.js");
|
|
41463
|
+
if (existsSync38(distPath))
|
|
41464
|
+
return req3(distPath);
|
|
41465
|
+
return req3(packageDir);
|
|
41466
|
+
}
|
|
41459
41467
|
async function loadTranscribeCli() {
|
|
41460
41468
|
if (_tcChecked)
|
|
41461
41469
|
return _tcModule;
|
|
41462
41470
|
_tcChecked = true;
|
|
41471
|
+
try {
|
|
41472
|
+
const { createRequire: createRequire11 } = await import("node:module");
|
|
41473
|
+
const req3 = createRequire11(import.meta.url);
|
|
41474
|
+
_tcModule = req3("transcribe-cli");
|
|
41475
|
+
return _tcModule;
|
|
41476
|
+
} catch {
|
|
41477
|
+
}
|
|
41463
41478
|
try {
|
|
41464
41479
|
const globalRoot = (await execFileText("npm", ["root", "-g"], { timeout: 5e3 })).trim();
|
|
41465
41480
|
const tcPath = join40(globalRoot, "transcribe-cli");
|
|
41466
41481
|
if (existsSync38(join40(tcPath, "dist", "index.js"))) {
|
|
41467
|
-
|
|
41468
|
-
const req3 = createRequire11(import.meta.url);
|
|
41469
|
-
_tcModule = req3(join40(tcPath, "dist", "index.js"));
|
|
41482
|
+
_tcModule = await requireTranscribeCliFrom(tcPath);
|
|
41470
41483
|
return _tcModule;
|
|
41471
41484
|
}
|
|
41472
41485
|
} catch {
|
|
@@ -41478,9 +41491,7 @@ async function loadTranscribeCli() {
|
|
|
41478
41491
|
for (const ver of readdirSync61(nvmBase)) {
|
|
41479
41492
|
const tcPath = join40(nvmBase, ver, "lib", "node_modules", "transcribe-cli");
|
|
41480
41493
|
if (existsSync38(join40(tcPath, "dist", "index.js"))) {
|
|
41481
|
-
|
|
41482
|
-
const req3 = createRequire11(import.meta.url);
|
|
41483
|
-
_tcModule = req3(join40(tcPath, "dist", "index.js"));
|
|
41494
|
+
_tcModule = await requireTranscribeCliFrom(tcPath);
|
|
41484
41495
|
return _tcModule;
|
|
41485
41496
|
}
|
|
41486
41497
|
}
|
|
@@ -41489,6 +41500,58 @@ async function loadTranscribeCli() {
|
|
|
41489
41500
|
}
|
|
41490
41501
|
return null;
|
|
41491
41502
|
}
|
|
41503
|
+
async function ensureManagedTranscribeCliRuntime() {
|
|
41504
|
+
const packageDir = join40(MANAGED_TRANSCRIBE_CLI_DIR, "node_modules", "transcribe-cli");
|
|
41505
|
+
if (existsSync38(join40(packageDir, "dist", "index.js"))) {
|
|
41506
|
+
try {
|
|
41507
|
+
_managedTranscribeCliReady = true;
|
|
41508
|
+
_tcModule = await requireTranscribeCliFrom(packageDir);
|
|
41509
|
+
_tcChecked = true;
|
|
41510
|
+
return _tcModule;
|
|
41511
|
+
} catch {
|
|
41512
|
+
}
|
|
41513
|
+
}
|
|
41514
|
+
if (_managedTranscribeCliChecked && !_managedTranscribeCliReady)
|
|
41515
|
+
return null;
|
|
41516
|
+
_managedTranscribeCliChecked = true;
|
|
41517
|
+
try {
|
|
41518
|
+
mkdirSync22(MANAGED_TRANSCRIBE_CLI_DIR, { recursive: true });
|
|
41519
|
+
if (!existsSync38(join40(MANAGED_TRANSCRIBE_CLI_DIR, "package.json"))) {
|
|
41520
|
+
writeFileSync21(join40(MANAGED_TRANSCRIBE_CLI_DIR, "package.json"), JSON.stringify({ private: true, dependencies: {} }, null, 2), "utf8");
|
|
41521
|
+
}
|
|
41522
|
+
await execFileText("npm", ["install", "--prefix", MANAGED_TRANSCRIBE_CLI_DIR, "transcribe-cli@^2.0.1"], {
|
|
41523
|
+
timeout: 18e4,
|
|
41524
|
+
env: transcriptionPythonEnv()
|
|
41525
|
+
});
|
|
41526
|
+
_managedTranscribeCliReady = true;
|
|
41527
|
+
_tcModule = await requireTranscribeCliFrom(packageDir);
|
|
41528
|
+
_tcChecked = true;
|
|
41529
|
+
return _tcModule;
|
|
41530
|
+
} catch {
|
|
41531
|
+
_managedTranscribeCliReady = false;
|
|
41532
|
+
return null;
|
|
41533
|
+
}
|
|
41534
|
+
}
|
|
41535
|
+
async function ensureTranscribeCliPythonModule(python) {
|
|
41536
|
+
if (_transcribeCliPythonModuleChecked)
|
|
41537
|
+
return;
|
|
41538
|
+
_transcribeCliPythonModuleChecked = true;
|
|
41539
|
+
try {
|
|
41540
|
+
await execFileText(python, ["-c", "import transcribe_cli"], {
|
|
41541
|
+
timeout: 1e4,
|
|
41542
|
+
env: transcriptionPythonEnv()
|
|
41543
|
+
});
|
|
41544
|
+
return;
|
|
41545
|
+
} catch {
|
|
41546
|
+
}
|
|
41547
|
+
try {
|
|
41548
|
+
await execFileText(python, ["-m", "pip", "install", "-U", "transcribe-cli"], {
|
|
41549
|
+
timeout: 3e5,
|
|
41550
|
+
env: transcriptionPythonEnv()
|
|
41551
|
+
});
|
|
41552
|
+
} catch {
|
|
41553
|
+
}
|
|
41554
|
+
}
|
|
41492
41555
|
function isYouTubeUrl(url) {
|
|
41493
41556
|
return /(?:youtube\.com\/(?:watch|shorts|live|embed|v\/)|youtu\.be\/)/i.test(url);
|
|
41494
41557
|
}
|
|
@@ -41519,7 +41582,7 @@ function formatTime(seconds) {
|
|
|
41519
41582
|
const s2 = Math.floor(seconds % 60);
|
|
41520
41583
|
return `${String(m2).padStart(2, "0")}:${String(s2).padStart(2, "0")}`;
|
|
41521
41584
|
}
|
|
41522
|
-
var AUDIO_EXTS, VIDEO_EXTS, MAX_TRANSCRIBE_URL_BYTES, MANAGED_ASR_VENV, _tcModule, _tcChecked, _managedAsrReady, TranscribeFileTool, YT_DLP_VENV, TranscribeUrlTool, YouTubeDownloadTool;
|
|
41585
|
+
var AUDIO_EXTS, VIDEO_EXTS, MAX_TRANSCRIBE_URL_BYTES, MANAGED_ASR_VENV, MANAGED_TRANSCRIBE_CLI_DIR, _tcModule, _tcChecked, _managedAsrReady, _managedTranscribeCliChecked, _managedTranscribeCliReady, _transcribeCliPythonModuleChecked, TranscribeFileTool, YT_DLP_VENV, TranscribeUrlTool, YouTubeDownloadTool;
|
|
41523
41586
|
var init_transcribe_tool = __esm({
|
|
41524
41587
|
"packages/execution/dist/tools/transcribe-tool.js"() {
|
|
41525
41588
|
"use strict";
|
|
@@ -41551,9 +41614,13 @@ var init_transcribe_tool = __esm({
|
|
|
41551
41614
|
]);
|
|
41552
41615
|
MAX_TRANSCRIBE_URL_BYTES = 100 * 1024 * 1024;
|
|
41553
41616
|
MANAGED_ASR_VENV = join40(homedir11(), ".omnius", "runtimes", "asr", ".venv-whisper");
|
|
41617
|
+
MANAGED_TRANSCRIBE_CLI_DIR = join40(homedir11(), ".omnius", "runtimes", "asr", "transcribe-cli-node");
|
|
41554
41618
|
_tcModule = null;
|
|
41555
41619
|
_tcChecked = false;
|
|
41556
41620
|
_managedAsrReady = false;
|
|
41621
|
+
_managedTranscribeCliChecked = false;
|
|
41622
|
+
_managedTranscribeCliReady = false;
|
|
41623
|
+
_transcribeCliPythonModuleChecked = false;
|
|
41557
41624
|
TranscribeFileTool = class {
|
|
41558
41625
|
name = "transcribe_file";
|
|
41559
41626
|
description = "Transcribe a local audio or video file to text using Whisper (faster-whisper). Supports MP3, WAV, FLAC, AAC, M4A, OGG, MP4, MKV, AVI, MOV, WebM. Returns the full transcription text with optional speaker diarization. Transcription is 100% local — no API keys needed.";
|
|
@@ -41572,6 +41639,11 @@ var init_transcribe_tool = __esm({
|
|
|
41572
41639
|
diarize: {
|
|
41573
41640
|
type: "boolean",
|
|
41574
41641
|
description: "Enable speaker diarization (identify who said what). Default: false"
|
|
41642
|
+
},
|
|
41643
|
+
backend: {
|
|
41644
|
+
type: "string",
|
|
41645
|
+
description: "ASR backend preference. auto tries fast transcribe-cli first; managed-whisper uses Omnius' isolated Whisper venv directly.",
|
|
41646
|
+
enum: ["auto", "managed-whisper", "transcribe-cli"]
|
|
41575
41647
|
}
|
|
41576
41648
|
},
|
|
41577
41649
|
required: ["path"]
|
|
@@ -41585,6 +41657,7 @@ var init_transcribe_tool = __esm({
|
|
|
41585
41657
|
const filePath = resolve21(this.workingDir, String(args["path"] ?? ""));
|
|
41586
41658
|
let model = String(args["model"] ?? "base");
|
|
41587
41659
|
const diarize = Boolean(args["diarize"] ?? false);
|
|
41660
|
+
const backend = String(args["backend"] ?? "auto").toLowerCase();
|
|
41588
41661
|
if (!existsSync38(filePath)) {
|
|
41589
41662
|
return {
|
|
41590
41663
|
success: false,
|
|
@@ -41628,8 +41701,29 @@ var init_transcribe_tool = __esm({
|
|
|
41628
41701
|
if (effectiveModel !== askedModel)
|
|
41629
41702
|
model = effectiveModel;
|
|
41630
41703
|
const failures = [];
|
|
41631
|
-
|
|
41632
|
-
|
|
41704
|
+
let managedPython = "";
|
|
41705
|
+
try {
|
|
41706
|
+
managedPython = await this.ensureManagedWhisperRuntime();
|
|
41707
|
+
process.env.TRANSCRIBE_PYTHON = managedPython;
|
|
41708
|
+
await ensureTranscribeCliPythonModule(managedPython);
|
|
41709
|
+
} catch (err) {
|
|
41710
|
+
failures.push(`managed ASR runtime bootstrap: ${err instanceof Error ? err.message : String(err)}`);
|
|
41711
|
+
}
|
|
41712
|
+
if (backend === "managed-whisper") {
|
|
41713
|
+
const managed2 = await this.execViaManagedWhisper(filePath, model, start2);
|
|
41714
|
+
if (managed2.success)
|
|
41715
|
+
return managed2;
|
|
41716
|
+
if (managed2.error)
|
|
41717
|
+
failures.push(managed2.error);
|
|
41718
|
+
return {
|
|
41719
|
+
success: false,
|
|
41720
|
+
output: "",
|
|
41721
|
+
error: `Managed Whisper ASR failed. ${failures.join(" | ").slice(0, 1200)}`,
|
|
41722
|
+
durationMs: performance.now() - start2
|
|
41723
|
+
};
|
|
41724
|
+
}
|
|
41725
|
+
const tc = await loadTranscribeCli() ?? await ensureManagedTranscribeCliRuntime();
|
|
41726
|
+
if (tc && backend !== "managed-whisper") {
|
|
41633
41727
|
try {
|
|
41634
41728
|
const result = await withProcessEnv(transcriptionPythonEnv(), () => tc.transcribe(filePath, {
|
|
41635
41729
|
model,
|
|
@@ -41641,6 +41735,8 @@ var init_transcribe_tool = __esm({
|
|
|
41641
41735
|
} catch (err) {
|
|
41642
41736
|
failures.push(`transcribe-cli module: ${err instanceof Error ? err.message : String(err)}`);
|
|
41643
41737
|
}
|
|
41738
|
+
} else {
|
|
41739
|
+
failures.push("transcribe-cli npm package unavailable after managed auto-install");
|
|
41644
41740
|
}
|
|
41645
41741
|
const cli = await this.execViaCli(filePath, model, diarize, start2);
|
|
41646
41742
|
if (cli.success)
|
|
@@ -41817,8 +41913,14 @@ var init_transcribe_tool = __esm({
|
|
|
41817
41913
|
}
|
|
41818
41914
|
}
|
|
41819
41915
|
async execViaManagedWhisper(filePath, model, start2) {
|
|
41820
|
-
|
|
41821
|
-
|
|
41916
|
+
let scriptDir = join40(this.workingDir, ".omnius", "tmp");
|
|
41917
|
+
try {
|
|
41918
|
+
mkdirSync22(scriptDir, { recursive: true });
|
|
41919
|
+
} catch {
|
|
41920
|
+
scriptDir = join40(tmpdir5(), "omnius-asr");
|
|
41921
|
+
mkdirSync22(scriptDir, { recursive: true });
|
|
41922
|
+
}
|
|
41923
|
+
const scriptPath2 = join40(scriptDir, `managed-whisper-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.py`);
|
|
41822
41924
|
const script = `
|
|
41823
41925
|
import json, os, warnings
|
|
41824
41926
|
warnings.filterwarnings("ignore")
|
|
@@ -41827,13 +41929,15 @@ audio_file = ${JSON.stringify(filePath)}
|
|
|
41827
41929
|
model_name = ${JSON.stringify(model)}
|
|
41828
41930
|
try:
|
|
41829
41931
|
import whisper
|
|
41830
|
-
device = "cpu"
|
|
41831
|
-
|
|
41832
|
-
|
|
41833
|
-
|
|
41834
|
-
|
|
41835
|
-
|
|
41836
|
-
|
|
41932
|
+
device = os.environ.get("OMNIUS_ASR_DEVICE", "cpu").strip().lower() or "cpu"
|
|
41933
|
+
if device == "auto":
|
|
41934
|
+
device = "cpu"
|
|
41935
|
+
try:
|
|
41936
|
+
import torch
|
|
41937
|
+
if torch.cuda.is_available():
|
|
41938
|
+
device = "cuda"
|
|
41939
|
+
except Exception:
|
|
41940
|
+
pass
|
|
41837
41941
|
m = whisper.load_model(model_name, device=device)
|
|
41838
41942
|
result = m.transcribe(audio_file, fp16=(device == "cuda"), condition_on_previous_text=False)
|
|
41839
41943
|
segments = []
|
|
@@ -42414,7 +42518,7 @@ var init_structured_file = __esm({
|
|
|
42414
42518
|
import { spawn as spawn7 } from "node:child_process";
|
|
42415
42519
|
import { writeFile as writeFile9, mkdtemp, rm as rm2, readdir as readdir4, stat as stat4 } from "node:fs/promises";
|
|
42416
42520
|
import { join as join41 } from "node:path";
|
|
42417
|
-
import { tmpdir as
|
|
42521
|
+
import { tmpdir as tmpdir6 } from "node:os";
|
|
42418
42522
|
function runProcess(cmd, args, options2) {
|
|
42419
42523
|
return new Promise((resolve76) => {
|
|
42420
42524
|
const proc = spawn7(cmd, args, {
|
|
@@ -42612,7 +42716,7 @@ ${result.filesCreated.join("\n")}`);
|
|
|
42612
42716
|
// Subprocess mode — temp directory + separate process
|
|
42613
42717
|
// -------------------------------------------------------------------------
|
|
42614
42718
|
async #runSubprocess(code8, langConfig, timeoutMs, stdin) {
|
|
42615
|
-
const sandboxDir = await mkdtemp(join41(
|
|
42719
|
+
const sandboxDir = await mkdtemp(join41(tmpdir6(), "omnius-sandbox-"));
|
|
42616
42720
|
try {
|
|
42617
42721
|
const scriptFile = join41(sandboxDir, `_sandbox_script${langConfig.ext}`);
|
|
42618
42722
|
await writeFile9(scriptFile, code8, "utf-8");
|
|
@@ -42639,7 +42743,7 @@ ${result.filesCreated.join("\n")}`);
|
|
|
42639
42743
|
bash: "bash:5"
|
|
42640
42744
|
};
|
|
42641
42745
|
const image = images[language] ?? "node:22-slim";
|
|
42642
|
-
const sandboxDir = await mkdtemp(join41(
|
|
42746
|
+
const sandboxDir = await mkdtemp(join41(tmpdir6(), "omnius-docker-sandbox-"));
|
|
42643
42747
|
try {
|
|
42644
42748
|
const scriptFile = `_sandbox_script${langConfig.ext}`;
|
|
42645
42749
|
await writeFile9(join41(sandboxDir, scriptFile), code8, "utf-8");
|
|
@@ -274164,7 +274268,7 @@ New: ${newNarrative.slice(0, 200)}...`,
|
|
|
274164
274268
|
import { spawn as spawn9 } from "node:child_process";
|
|
274165
274269
|
import { createServer as createServer2 } from "node:net";
|
|
274166
274270
|
import { join as join44 } from "node:path";
|
|
274167
|
-
import { tmpdir as
|
|
274271
|
+
import { tmpdir as tmpdir7 } from "node:os";
|
|
274168
274272
|
import { randomBytes as randomBytes11 } from "node:crypto";
|
|
274169
274273
|
import { unlinkSync as unlinkSync6 } from "node:fs";
|
|
274170
274274
|
var ReplTool;
|
|
@@ -274236,7 +274340,7 @@ var init_repl = __esm({
|
|
|
274236
274340
|
if (!this.proc || this.proc.killed || this.proc.exitCode !== null) {
|
|
274237
274341
|
await this.startProcess();
|
|
274238
274342
|
}
|
|
274239
|
-
const tempFile = join44(
|
|
274343
|
+
const tempFile = join44(tmpdir7(), `omnius-repl-ctx-${randomBytes11(6).toString("hex")}.txt`);
|
|
274240
274344
|
const { writeFileSync: writeFs, unlinkSync: unlinkFs } = await import("node:fs");
|
|
274241
274345
|
writeFs(tempFile, content, "utf8");
|
|
274242
274346
|
const result = await this.executeCode(`with open(${JSON.stringify(tempFile)}, "r") as _f:
|
|
@@ -274463,7 +274567,7 @@ print("__OMNIUS_REPL_READY__")
|
|
|
274463
274567
|
if (this.ipcServer)
|
|
274464
274568
|
return;
|
|
274465
274569
|
const sockId = randomBytes11(8).toString("hex");
|
|
274466
|
-
this.ipcPath = join44(
|
|
274570
|
+
this.ipcPath = join44(tmpdir7(), `omnius-repl-ipc-${sockId}.sock`);
|
|
274467
274571
|
return new Promise((resolve76, reject) => {
|
|
274468
274572
|
this.ipcServer = createServer2((conn) => {
|
|
274469
274573
|
let buffer2 = new Uint8Array(0);
|
|
@@ -287642,7 +287746,7 @@ ${parts.join("\n\n")}`,
|
|
|
287642
287746
|
// packages/execution/dist/tools/desktop-click.js
|
|
287643
287747
|
import { readFileSync as readFileSync35 } from "node:fs";
|
|
287644
287748
|
import { execSync as execSync16 } from "node:child_process";
|
|
287645
|
-
import { tmpdir as
|
|
287749
|
+
import { tmpdir as tmpdir8 } from "node:os";
|
|
287646
287750
|
import { join as join57 } from "node:path";
|
|
287647
287751
|
function getImageDimensions2(filePath) {
|
|
287648
287752
|
try {
|
|
@@ -287859,7 +287963,7 @@ Button: ${button}, Type: ${clickType}`,
|
|
|
287859
287963
|
durationMs: performance.now() - start2
|
|
287860
287964
|
};
|
|
287861
287965
|
}
|
|
287862
|
-
const screenshotPath = join57(
|
|
287966
|
+
const screenshotPath = join57(tmpdir8(), `omnius-desktop-click-${Date.now()}.png`);
|
|
287863
287967
|
const screenshotBackend = captureDesktopScreenshot(screenshotPath);
|
|
287864
287968
|
const dims = getImageDimensions2(screenshotPath);
|
|
287865
287969
|
if (!dims) {
|
|
@@ -288026,7 +288130,7 @@ Screenshot: ${screenshotPath}`,
|
|
|
288026
288130
|
if (delayMs > 0) {
|
|
288027
288131
|
await new Promise((r2) => setTimeout(r2, delayMs));
|
|
288028
288132
|
}
|
|
288029
|
-
const screenshotPath = join57(
|
|
288133
|
+
const screenshotPath = join57(tmpdir8(), `omnius-desktop-describe-${Date.now()}.png`);
|
|
288030
288134
|
const screenshotBackend = captureDesktopScreenshot(screenshotPath);
|
|
288031
288135
|
const dims = getImageDimensions2(screenshotPath);
|
|
288032
288136
|
const imageBuffer = readFileSync35(screenshotPath);
|
|
@@ -289230,7 +289334,7 @@ Language: ${language}
|
|
|
289230
289334
|
import { existsSync as existsSync50, statSync as statSync21, readFileSync as readFileSync37, unlinkSync as unlinkSync8 } from "node:fs";
|
|
289231
289335
|
import { resolve as resolve29, basename as basename10, join as join59 } from "node:path";
|
|
289232
289336
|
import { execSync as execSync18 } from "node:child_process";
|
|
289233
|
-
import { tmpdir as
|
|
289337
|
+
import { tmpdir as tmpdir9 } from "node:os";
|
|
289234
289338
|
var PdfToTextTool;
|
|
289235
289339
|
var init_pdf_to_text = __esm({
|
|
289236
289340
|
"packages/execution/dist/tools/pdf-to-text.js"() {
|
|
@@ -289382,7 +289486,7 @@ ${text2}`,
|
|
|
289382
289486
|
if (!ocrCheck.available || !tesCheck.available || !gsCheck.available) {
|
|
289383
289487
|
return null;
|
|
289384
289488
|
}
|
|
289385
|
-
const tmpPdf = join59(
|
|
289489
|
+
const tmpPdf = join59(tmpdir9(), `omnius-ocr-${Date.now()}.pdf`);
|
|
289386
289490
|
try {
|
|
289387
289491
|
const ocrCmd = `ocrmypdf -l ${language} --skip-text ${JSON.stringify(inputPath)} ${JSON.stringify(tmpPdf)}`;
|
|
289388
289492
|
execSync18(ocrCmd, { stdio: "pipe", timeout: 6e5 });
|
|
@@ -289416,7 +289520,7 @@ import { existsSync as existsSync51, mkdirSync as mkdirSync28, statSync as statS
|
|
|
289416
289520
|
import { resolve as resolve30, basename as basename11, dirname as dirname16, join as join60 } from "node:path";
|
|
289417
289521
|
import { execSync as execSync19 } from "node:child_process";
|
|
289418
289522
|
import { fileURLToPath as fileURLToPath6 } from "node:url";
|
|
289419
|
-
import { homedir as homedir13, tmpdir as
|
|
289523
|
+
import { homedir as homedir13, tmpdir as tmpdir10 } from "node:os";
|
|
289420
289524
|
function findOcrScript() {
|
|
289421
289525
|
const thisDir = dirname16(fileURLToPath6(import.meta.url));
|
|
289422
289526
|
const devPath3 = resolve30(thisDir, "../../scripts/ocr-advanced.py");
|
|
@@ -289649,7 +289753,7 @@ var init_ocr_image_advanced = __esm({
|
|
|
289649
289753
|
cmdParts.push("--output-dir", JSON.stringify(resolve30(this.workingDir, outputDir2)));
|
|
289650
289754
|
let debugDir;
|
|
289651
289755
|
if (debug) {
|
|
289652
|
-
debugDir = join60(
|
|
289756
|
+
debugDir = join60(tmpdir10(), `omnius-ocr-debug-${Date.now()}`);
|
|
289653
289757
|
cmdParts.push("--debug-dir", debugDir);
|
|
289654
289758
|
}
|
|
289655
289759
|
try {
|
|
@@ -543118,7 +543222,7 @@ var init_process_health = __esm({
|
|
|
543118
543222
|
// packages/execution/dist/tools/camera-capture.js
|
|
543119
543223
|
import { accessSync, constants, readFileSync as readFileSync44, writeFileSync as writeFileSync28, unlinkSync as unlinkSync11, existsSync as existsSync60, mkdirSync as mkdirSync34, readdirSync as readdirSync21, renameSync as renameSync7 } from "node:fs";
|
|
543120
543224
|
import { join as join75 } from "node:path";
|
|
543121
|
-
import { tmpdir as
|
|
543225
|
+
import { tmpdir as tmpdir11 } from "node:os";
|
|
543122
543226
|
function normalizeCameraRotationDegrees(value2) {
|
|
543123
543227
|
if (value2 === void 0 || value2 === null || value2 === "")
|
|
543124
543228
|
return 0;
|
|
@@ -543616,7 +543720,7 @@ ${formatRouterPlan(plan)}`,
|
|
|
543616
543720
|
const height = args["height"] || 720;
|
|
543617
543721
|
const outputPath3 = args["output_path"];
|
|
543618
543722
|
const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
|
|
543619
|
-
const captureDir = join75(
|
|
543723
|
+
const captureDir = join75(tmpdir11(), "omnius-camera");
|
|
543620
543724
|
if (!existsSync60(captureDir))
|
|
543621
543725
|
mkdirSync34(captureDir, { recursive: true });
|
|
543622
543726
|
const tempFile = outputPath3 || join75(captureDir, `capture-${Date.now()}.jpg`);
|
|
@@ -543998,7 +544102,7 @@ Auto-installed camera dependencies: ${[
|
|
|
543998
544102
|
} catch (err) {
|
|
543999
544103
|
return { success: false, output: "", error: `QooCam OSC error: ${err instanceof Error ? err.message : String(err)}`, durationMs: performance.now() - start2 };
|
|
544000
544104
|
}
|
|
544001
|
-
const captureDir = join75(
|
|
544105
|
+
const captureDir = join75(tmpdir11(), "omnius-camera");
|
|
544002
544106
|
if (!existsSync60(captureDir))
|
|
544003
544107
|
mkdirSync34(captureDir, { recursive: true });
|
|
544004
544108
|
const tempFile = outputPath3 || join75(captureDir, `qoocam-${Date.now()}.jpg`);
|
|
@@ -544148,7 +544252,7 @@ Saved to: ${outputPath3}`;
|
|
|
544148
544252
|
import { execSync as execSync26 } from "node:child_process";
|
|
544149
544253
|
import { readFileSync as readFileSync45, unlinkSync as unlinkSync12, existsSync as existsSync61, mkdirSync as mkdirSync35, statSync as statSync25 } from "node:fs";
|
|
544150
544254
|
import { join as join76 } from "node:path";
|
|
544151
|
-
import { tmpdir as
|
|
544255
|
+
import { tmpdir as tmpdir12 } from "node:os";
|
|
544152
544256
|
var AudioCaptureTool;
|
|
544153
544257
|
var init_audio_capture = __esm({
|
|
544154
544258
|
"packages/execution/dist/tools/audio-capture.js"() {
|
|
@@ -544254,7 +544358,7 @@ ${devices.join("\n")}`,
|
|
|
544254
544358
|
const channels = args["channels"] || 1;
|
|
544255
544359
|
const format3 = args["format"] || "wav";
|
|
544256
544360
|
const outputPath3 = args["output_path"];
|
|
544257
|
-
const captureDir = join76(
|
|
544361
|
+
const captureDir = join76(tmpdir12(), "omnius-audio");
|
|
544258
544362
|
if (!existsSync61(captureDir))
|
|
544259
544363
|
mkdirSync35(captureDir, { recursive: true });
|
|
544260
544364
|
const ext = format3 === "mp3" ? "mp3" : "wav";
|
|
@@ -544341,7 +544445,7 @@ Saved to: ${tempFile}`,
|
|
|
544341
544445
|
}
|
|
544342
544446
|
checkLevel(args, start2) {
|
|
544343
544447
|
const device = args["device"] || "default";
|
|
544344
|
-
const tempFile = join76(
|
|
544448
|
+
const tempFile = join76(tmpdir12(), `omnius-level-${Date.now()}.raw`);
|
|
544345
544449
|
try {
|
|
544346
544450
|
execSync26(`arecord -D ${device} -f S16_LE -r 16000 -c 1 -d 1 -t raw -q ${tempFile}`, { timeout: 5e3, stdio: "pipe" });
|
|
544347
544451
|
if (!existsSync61(tempFile)) {
|
|
@@ -544399,7 +544503,7 @@ Saved to: ${tempFile}`,
|
|
|
544399
544503
|
import { execFileSync as execFileSync2, execSync as execSync27, spawn as spawn18 } from "node:child_process";
|
|
544400
544504
|
import { copyFileSync as copyFileSync4, existsSync as existsSync62, statSync as statSync26, writeFileSync as writeFileSync29, mkdirSync as mkdirSync36, readdirSync as readdirSync22, writeSync, rmSync as rmSync7 } from "node:fs";
|
|
544401
544505
|
import { basename as basename15, dirname as dirname21, extname as extname12, isAbsolute as isAbsolute4, join as join77, resolve as resolve39 } from "node:path";
|
|
544402
|
-
import { homedir as homedir17, tmpdir as
|
|
544506
|
+
import { homedir as homedir17, tmpdir as tmpdir13 } from "node:os";
|
|
544403
544507
|
function ttsPythonEnv(extra = {}) {
|
|
544404
544508
|
const env2 = { ...process.env, ...extra };
|
|
544405
544509
|
applyMediaCudaDeviceFilterToEnv(env2, "tts");
|
|
@@ -545280,7 +545384,7 @@ function ensureLuxttsDaemon() {
|
|
|
545280
545384
|
}, 12e4);
|
|
545281
545385
|
const daemon = spawn18(venvPy, [inferScript], {
|
|
545282
545386
|
stdio: ["pipe", "pipe", "pipe"],
|
|
545283
|
-
cwd:
|
|
545387
|
+
cwd: tmpdir13(),
|
|
545284
545388
|
env: ttsPythonEnv({ LUXTTS_REPO_PATH: repoDir })
|
|
545285
545389
|
});
|
|
545286
545390
|
_luxttsDaemon = daemon;
|
|
@@ -545337,7 +545441,7 @@ function luxttsSynthesize(text2, cloneRef, outputPath3, speed = 1) {
|
|
|
545337
545441
|
return;
|
|
545338
545442
|
}
|
|
545339
545443
|
const id2 = `tts-${++_luxttsRequestId}`;
|
|
545340
|
-
const outPath = outputPath3 || join77(
|
|
545444
|
+
const outPath = outputPath3 || join77(tmpdir13(), `omnius-luxtts-${Date.now()}.wav`);
|
|
545341
545445
|
const timeout2 = setTimeout(() => {
|
|
545342
545446
|
_luxttsPending.delete(id2);
|
|
545343
545447
|
reject(new Error("LuxTTS synthesis timeout"));
|
|
@@ -545480,7 +545584,7 @@ function ensureMisottsDaemon() {
|
|
|
545480
545584
|
}, 12e4);
|
|
545481
545585
|
const daemon = spawn18(venvPy, [inferScript], {
|
|
545482
545586
|
stdio: ["pipe", "pipe", "pipe"],
|
|
545483
|
-
cwd:
|
|
545587
|
+
cwd: tmpdir13(),
|
|
545484
545588
|
env: ttsPythonEnv({ MISO_TTS_REPO_PATH: repoDir, NO_TORCH_COMPILE: "1" })
|
|
545485
545589
|
});
|
|
545486
545590
|
_misottsDaemon = daemon;
|
|
@@ -545537,7 +545641,7 @@ function misottsSynthesize(text2, cloneRef, outputPath3, maxAudioLengthMs = 3e4)
|
|
|
545537
545641
|
return;
|
|
545538
545642
|
}
|
|
545539
545643
|
const id2 = `tts-${++_misottsRequestId}`;
|
|
545540
|
-
const outPath = outputPath3 || join77(
|
|
545644
|
+
const outPath = outputPath3 || join77(tmpdir13(), `omnius-misotts-${Date.now()}.wav`);
|
|
545541
545645
|
const timeout2 = setTimeout(() => {
|
|
545542
545646
|
_misottsPending.delete(id2);
|
|
545543
545647
|
reject(new Error("MisoTTS synthesis timeout"));
|
|
@@ -546195,7 +546299,7 @@ ${tried.map((line) => `- ${line}`).join("\n")}`,
|
|
|
546195
546299
|
], {
|
|
546196
546300
|
stdio: "pipe",
|
|
546197
546301
|
timeout: 18e4,
|
|
546198
|
-
cwd:
|
|
546302
|
+
cwd: tmpdir13()
|
|
546199
546303
|
});
|
|
546200
546304
|
return `${voice} (${model})`;
|
|
546201
546305
|
}
|
|
@@ -546992,7 +547096,7 @@ ${info}`, durationMs: performance.now() - start2 };
|
|
|
546992
547096
|
import { execSync as execSync30 } from "node:child_process";
|
|
546993
547097
|
import { readFileSync as readFileSync46, unlinkSync as unlinkSync13, existsSync as existsSync63, mkdirSync as mkdirSync37, statSync as statSync27 } from "node:fs";
|
|
546994
547098
|
import { join as join78 } from "node:path";
|
|
546995
|
-
import { tmpdir as
|
|
547099
|
+
import { tmpdir as tmpdir14 } from "node:os";
|
|
546996
547100
|
var SdrScanTool;
|
|
546997
547101
|
var init_sdr_scan = __esm({
|
|
546998
547102
|
"packages/execution/dist/tools/sdr-scan.js"() {
|
|
@@ -547163,7 +547267,7 @@ Tools installed but rtl_test failed — device may be in use by another process.
|
|
|
547163
547267
|
if (!await this.ensureSdrTools()) {
|
|
547164
547268
|
return { success: false, output: "", error: "rtl-sdr tools not installed. Connect an RTL-SDR dongle and retry (will prompt for install).", durationMs: performance.now() - start2 };
|
|
547165
547269
|
}
|
|
547166
|
-
const captureDir = join78(
|
|
547270
|
+
const captureDir = join78(tmpdir14(), "omnius-sdr");
|
|
547167
547271
|
if (!existsSync63(captureDir))
|
|
547168
547272
|
mkdirSync37(captureDir, { recursive: true });
|
|
547169
547273
|
const outFile = join78(captureDir, `scan-${Date.now()}.csv`);
|
|
@@ -547247,7 +547351,7 @@ ${output.slice(0, 2e3)}`,
|
|
|
547247
547351
|
} catch {
|
|
547248
547352
|
return { success: false, output: "", error: "rtl_fm not installed. Run: sudo apt install rtl-sdr", durationMs: performance.now() - start2 };
|
|
547249
547353
|
}
|
|
547250
|
-
const captureDir = join78(
|
|
547354
|
+
const captureDir = join78(tmpdir14(), "omnius-sdr");
|
|
547251
547355
|
if (!existsSync63(captureDir))
|
|
547252
547356
|
mkdirSync37(captureDir, { recursive: true });
|
|
547253
547357
|
const outFile = join78(captureDir, `fm-${Date.now()}.wav`);
|
|
@@ -547704,7 +547808,7 @@ ${result.trim()}`, durationMs: performance.now() - start2 };
|
|
|
547704
547808
|
// packages/execution/dist/tools/audio-analyze.js
|
|
547705
547809
|
import { existsSync as existsSync65, mkdirSync as mkdirSync38, writeFileSync as writeFileSync30 } from "node:fs";
|
|
547706
547810
|
import { basename as basename16, isAbsolute as isAbsolute5, join as join79, resolve as resolve40 } from "node:path";
|
|
547707
|
-
import { homedir as homedir18, tmpdir as
|
|
547811
|
+
import { homedir as homedir18, tmpdir as tmpdir15 } from "node:os";
|
|
547708
547812
|
import { fileURLToPath as fileURLToPath9 } from "node:url";
|
|
547709
547813
|
function audioAnalysisPythonEnv(extra = {}) {
|
|
547710
547814
|
const env2 = { ...process.env, ...extra };
|
|
@@ -548135,7 +548239,7 @@ except Exception as e:
|
|
|
548135
548239
|
const contextDir = join79(homedir18(), ".omnius", "audio-context");
|
|
548136
548240
|
if (!existsSync65(contextDir))
|
|
548137
548241
|
mkdirSync38(contextDir, { recursive: true });
|
|
548138
|
-
const audioFile = join79(
|
|
548242
|
+
const audioFile = join79(tmpdir15(), `omnius-listen-${Date.now()}.wav`);
|
|
548139
548243
|
try {
|
|
548140
548244
|
await execFileText("arecord", ["-D", "default", "-f", "S16_LE", "-r", "16000", "-c", "1", "-d", String(Math.min(duration, 60)), "-q", audioFile], {
|
|
548141
548245
|
timeout: (duration + 10) * 1e3
|
|
@@ -548276,7 +548380,7 @@ ${e2.stderr ? String(e2.stderr) : ""}`.trim();
|
|
|
548276
548380
|
return file;
|
|
548277
548381
|
}
|
|
548278
548382
|
const duration = args["duration"] || 5;
|
|
548279
|
-
const outFile = join79(
|
|
548383
|
+
const outFile = join79(tmpdir15(), `omnius-analyze-${Date.now()}.wav`);
|
|
548280
548384
|
try {
|
|
548281
548385
|
await execFileText("arecord", ["-D", "default", "-f", "S16_LE", "-r", "16000", "-c", "1", "-d", String(duration), "-q", outFile], {
|
|
548282
548386
|
timeout: (duration + 10) * 1e3
|
|
@@ -548305,7 +548409,7 @@ ${e2.stderr ? String(e2.stderr) : ""}`.trim();
|
|
|
548305
548409
|
}
|
|
548306
548410
|
/** Run a Python script in the venv and parse JSON output */
|
|
548307
548411
|
async runPythonScript(script, start2, label) {
|
|
548308
|
-
const scriptFile = join79(
|
|
548412
|
+
const scriptFile = join79(tmpdir15(), `omnius-audio-${Date.now()}.py`);
|
|
548309
548413
|
writeFileSync30(scriptFile, script);
|
|
548310
548414
|
try {
|
|
548311
548415
|
const output = await execFileText(VENV_PYTHON, [scriptFile], {
|
|
@@ -548341,7 +548445,7 @@ ${output}`, durationMs: performance.now() - start2 };
|
|
|
548341
548445
|
import { execSync as execSync33, spawnSync as spawnSync6 } from "node:child_process";
|
|
548342
548446
|
import { existsSync as existsSync66, readFileSync as readFileSync47, writeFileSync as writeFileSync31, mkdirSync as mkdirSync39 } from "node:fs";
|
|
548343
548447
|
import { join as join80 } from "node:path";
|
|
548344
|
-
import { tmpdir as
|
|
548448
|
+
import { tmpdir as tmpdir16, homedir as homedir19 } from "node:os";
|
|
548345
548449
|
var GPS_USB_IDS, GpsLocationTool;
|
|
548346
548450
|
var init_gps_location = __esm({
|
|
548347
548451
|
"packages/execution/dist/tools/gps-location.js"() {
|
|
@@ -548466,7 +548570,7 @@ var init_gps_location = __esm({
|
|
|
548466
548570
|
/** Run a Python GPS script using pyserial+pynmea2 and return JSON result */
|
|
548467
548571
|
async runGpsPython(script, timeoutMs = 3e4) {
|
|
548468
548572
|
await this.ensureGpsVenv();
|
|
548469
|
-
const scriptFile = join80(
|
|
548573
|
+
const scriptFile = join80(tmpdir16(), `omnius-gps-${Date.now()}.py`);
|
|
548470
548574
|
writeFileSync31(scriptFile, script);
|
|
548471
548575
|
try {
|
|
548472
548576
|
const output = execSync33(`${this.GPS_PYTHON} ${scriptFile}`, {
|
|
@@ -548938,7 +549042,7 @@ Drift: ${drift != null ? drift + "ms" : "unknown"}`,
|
|
|
548938
549042
|
// =========================================================================
|
|
548939
549043
|
async recordTrack(args, start2) {
|
|
548940
549044
|
const duration = args["duration"] || 60;
|
|
548941
|
-
const outputPath3 = args["output_path"] || join80(
|
|
549045
|
+
const outputPath3 = args["output_path"] || join80(tmpdir16(), `omnius-gps-track-${Date.now()}.gpx`);
|
|
548942
549046
|
if (!await this.ensureGpsd(args)) {
|
|
548943
549047
|
return { success: false, output: "", error: "gpsd required for track recording. Connect a GPS device.", durationMs: performance.now() - start2 };
|
|
548944
549048
|
}
|
|
@@ -549171,7 +549275,7 @@ def _omnius_normalized_features(features):
|
|
|
549171
549275
|
import { execFile as execFile6 } from "node:child_process";
|
|
549172
549276
|
import { existsSync as existsSync67, mkdirSync as mkdirSync40, writeFileSync as writeFileSync32, readFileSync as readFileSync48 } from "node:fs";
|
|
549173
549277
|
import { join as join81 } from "node:path";
|
|
549174
|
-
import { homedir as homedir20, tmpdir as
|
|
549278
|
+
import { homedir as homedir20, tmpdir as tmpdir17 } from "node:os";
|
|
549175
549279
|
function visualMemoryPythonEnv(extra = {}) {
|
|
549176
549280
|
const env2 = { ...process.env, ...extra };
|
|
549177
549281
|
applyMediaCudaDeviceFilterToEnv(env2, "vision");
|
|
@@ -549943,7 +550047,7 @@ ${objects.join("\n") || " (none taught)"}`,
|
|
|
549943
550047
|
}
|
|
549944
550048
|
}
|
|
549945
550049
|
async runVisionPython(script, timeoutMs = 6e4) {
|
|
549946
|
-
const scriptFile = join81(
|
|
550050
|
+
const scriptFile = join81(tmpdir17(), `omnius-vmem-${Date.now()}.py`);
|
|
549947
550051
|
writeFileSync32(scriptFile, script);
|
|
549948
550052
|
try {
|
|
549949
550053
|
const { stdout: output } = await execFileText2(VENV_PY, [scriptFile], {
|
|
@@ -549976,7 +550080,7 @@ ${objects.join("\n") || " (none taught)"}`,
|
|
|
549976
550080
|
import { execSync as execSync34 } from "node:child_process";
|
|
549977
550081
|
import { appendFileSync as appendFileSync5, existsSync as existsSync68, mkdirSync as mkdirSync41, writeFileSync as writeFileSync33, readFileSync as readFileSync49, readdirSync as readdirSync23 } from "node:fs";
|
|
549978
550082
|
import { join as join82 } from "node:path";
|
|
549979
|
-
import { homedir as homedir21, tmpdir as
|
|
550083
|
+
import { homedir as homedir21, tmpdir as tmpdir18 } from "node:os";
|
|
549980
550084
|
import { randomUUID as randomUUID15 } from "node:crypto";
|
|
549981
550085
|
var MM_DIR, MM_INDEX, MultimodalMemoryTool;
|
|
549982
550086
|
var init_multimodal_memory = __esm({
|
|
@@ -550095,7 +550199,7 @@ with torch.no_grad():
|
|
|
550095
550199
|
features = _omnius_normalized_features(model.get_image_features(**inputs))
|
|
550096
550200
|
print(json.dumps(features[0].cpu().numpy().tolist()))
|
|
550097
550201
|
`;
|
|
550098
|
-
const scriptFile = join82(
|
|
550202
|
+
const scriptFile = join82(tmpdir18(), `mm-clip-${Date.now()}.py`);
|
|
550099
550203
|
writeFileSync33(scriptFile, clipScript);
|
|
550100
550204
|
const clipOutput = execSync34(`${venvPy} ${scriptFile}`, { encoding: "utf8", timeout: 12e4, env: { ...process.env, PYTHONUNBUFFERED: "1" } });
|
|
550101
550205
|
const embedding = JSON.parse(clipOutput.trim().split("\n").pop());
|
|
@@ -550117,7 +550221,7 @@ faces = app.get(img) if img is not None else []
|
|
|
550117
550221
|
result = [{"confidence": float(f.det_score), "age": int(f.age) if hasattr(f, 'age') else None} for f in faces]
|
|
550118
550222
|
print(json.dumps(result))
|
|
550119
550223
|
`;
|
|
550120
|
-
const scriptFile = join82(
|
|
550224
|
+
const scriptFile = join82(tmpdir18(), `mm-face-${Date.now()}.py`);
|
|
550121
550225
|
writeFileSync33(scriptFile, faceScript);
|
|
550122
550226
|
const faceOutput = execSync34(`${venvPy} ${scriptFile}`, { encoding: "utf8", timeout: 6e4, env: { ...process.env, PYTHONUNBUFFERED: "1" } });
|
|
550123
550227
|
const faces = JSON.parse(faceOutput.trim().split("\n").pop());
|
|
@@ -550169,7 +550273,7 @@ with open(model.class_map_path().numpy().decode()) as f:
|
|
|
550169
550273
|
top=scores.numpy().mean(axis=0).argsort()[-1]
|
|
550170
550274
|
print(classes[top])
|
|
550171
550275
|
`;
|
|
550172
|
-
const scriptFile = join82(
|
|
550276
|
+
const scriptFile = join82(tmpdir18(), `mm-yamnet-${Date.now()}.py`);
|
|
550173
550277
|
writeFileSync33(scriptFile, classifyScript);
|
|
550174
550278
|
const soundClass = execSync34(`${mlVenvPy} ${scriptFile}`, { encoding: "utf8", timeout: 12e4 }).trim().split("\n").pop();
|
|
550175
550279
|
episode.audio.soundClass = soundClass;
|
|
@@ -550265,7 +550369,7 @@ if faces:
|
|
|
550265
550369
|
else:
|
|
550266
550370
|
print(json.dumps({"enrolled": False, "reason": "no face detected"}))
|
|
550267
550371
|
`;
|
|
550268
|
-
const scriptFile = join82(
|
|
550372
|
+
const scriptFile = join82(tmpdir18(), `mm-enroll-${Date.now()}.py`);
|
|
550269
550373
|
writeFileSync33(scriptFile, enrollScript);
|
|
550270
550374
|
const enrollOutput = execSync34(`${venvPy} ${scriptFile}`, { encoding: "utf8", timeout: 6e4, env: { ...process.env, PYTHONUNBUFFERED: "1" } });
|
|
550271
550375
|
const enrollResult = JSON.parse(enrollOutput.trim().split("\n").pop());
|
|
@@ -550320,7 +550424,7 @@ with torch.no_grad():
|
|
|
550320
550424
|
features = _omnius_normalized_features(model.get_text_features(**inputs))
|
|
550321
550425
|
print(json.dumps(features[0].cpu().numpy().tolist()))
|
|
550322
550426
|
`;
|
|
550323
|
-
const scriptFile = join82(
|
|
550427
|
+
const scriptFile = join82(tmpdir18(), `mm-clipq-${Date.now()}.py`);
|
|
550324
550428
|
writeFileSync33(scriptFile, clipTextScript);
|
|
550325
550429
|
const output = execSync34(`${venvPy} ${scriptFile}`, { encoding: "utf8", timeout: 6e4, env: { ...process.env, PYTHONUNBUFFERED: "1" } });
|
|
550326
550430
|
queryClipEmbedding = JSON.parse(output.trim().split("\n").pop());
|
|
@@ -550602,7 +550706,7 @@ ${lines.join("\n")}`,
|
|
|
550602
550706
|
import { execSync as execSync35, spawnSync as spawnSync7 } from "node:child_process";
|
|
550603
550707
|
import { existsSync as existsSync69, mkdirSync as mkdirSync42, writeFileSync as writeFileSync34, readFileSync as readFileSync50, unlinkSync as unlinkSync14 } from "node:fs";
|
|
550604
550708
|
import { dirname as dirname22, join as join83, resolve as resolve41 } from "node:path";
|
|
550605
|
-
import { tmpdir as
|
|
550709
|
+
import { tmpdir as tmpdir19, homedir as homedir22 } from "node:os";
|
|
550606
550710
|
import { fileURLToPath as fileURLToPath10 } from "node:url";
|
|
550607
550711
|
function asrPythonEnv(extra = {}) {
|
|
550608
550712
|
const env2 = { ...process.env, ...extra };
|
|
@@ -550717,7 +550821,7 @@ var init_asr_listen = __esm({
|
|
|
550717
550821
|
listenAndTranscribe(args, start2) {
|
|
550718
550822
|
const duration = args["duration"] || 8;
|
|
550719
550823
|
const device = args["device"] || "default";
|
|
550720
|
-
const captureDir = join83(
|
|
550824
|
+
const captureDir = join83(tmpdir19(), "omnius-asr");
|
|
550721
550825
|
if (!existsSync69(captureDir))
|
|
550722
550826
|
mkdirSync42(captureDir, { recursive: true });
|
|
550723
550827
|
const audioFile = join83(captureDir, `listen-${Date.now()}.wav`);
|
|
@@ -550864,7 +550968,7 @@ except Exception:
|
|
|
550864
550968
|
|
|
550865
550969
|
print(json.dumps({"ok": False, "error": "No whisper backend available"}))
|
|
550866
550970
|
`;
|
|
550867
|
-
const scriptFile = join83(
|
|
550971
|
+
const scriptFile = join83(tmpdir19(), `omnius-asr-whisper-${Date.now()}.py`);
|
|
550868
550972
|
writeFileSync34(scriptFile, whisperScript);
|
|
550869
550973
|
const pyPaths = [
|
|
550870
550974
|
join83(homedir22(), ".omnius", "venv", "bin", "python3"),
|
|
@@ -555642,7 +555746,7 @@ var init_visual_trigger = __esm({
|
|
|
555642
555746
|
|
|
555643
555747
|
// packages/execution/dist/tools/live-media-loop.js
|
|
555644
555748
|
import { existsSync as existsSync78, mkdirSync as mkdirSync47, readFileSync as readFileSync57, rmSync as rmSync10, writeFileSync as writeFileSync39 } from "node:fs";
|
|
555645
|
-
import { homedir as homedir28, tmpdir as
|
|
555749
|
+
import { homedir as homedir28, tmpdir as tmpdir20 } from "node:os";
|
|
555646
555750
|
import { basename as basename19, isAbsolute as isAbsolute8, join as join92, resolve as resolve46 } from "node:path";
|
|
555647
555751
|
function buildYolo26BootstrapPlan(model = DEFAULT_YOLO26_MODEL) {
|
|
555648
555752
|
return {
|
|
@@ -555961,8 +556065,8 @@ async function ensureRuntime(autoBootstrap) {
|
|
|
555961
556065
|
}
|
|
555962
556066
|
}
|
|
555963
556067
|
async function exportYolo26Model(python, model, options2, yoloeClasses = []) {
|
|
555964
|
-
const cfgPath = join92(
|
|
555965
|
-
const scriptPath2 = join92(
|
|
556068
|
+
const cfgPath = join92(tmpdir20(), `omnius-yolo26-export-${Date.now()}.json`);
|
|
556069
|
+
const scriptPath2 = join92(tmpdir20(), `omnius-yolo26-export-${Date.now()}.py`);
|
|
555966
556070
|
writeFileSync39(cfgPath, JSON.stringify({ model, options: options2, yoloeClasses }, null, 2), "utf8");
|
|
555967
556071
|
writeFileSync39(scriptPath2, String.raw`
|
|
555968
556072
|
import json, sys
|
|
@@ -556538,8 +556642,8 @@ var init_live_media_loop = __esm({
|
|
|
556538
556642
|
return { success: false, output: "", error: runtime.error, durationMs: performance.now() - start2 };
|
|
556539
556643
|
const outDir = join92(this.workingDir, ".omnius", "live-media", `run-${Date.now().toString(36)}`);
|
|
556540
556644
|
mkdirSync47(outDir, { recursive: true });
|
|
556541
|
-
const cfgPath = join92(
|
|
556542
|
-
const scriptPath2 = join92(
|
|
556645
|
+
const cfgPath = join92(tmpdir20(), `omnius-live-media-${Date.now()}.json`);
|
|
556646
|
+
const scriptPath2 = join92(tmpdir20(), `omnius-live-media-${Date.now()}.py`);
|
|
556543
556647
|
writeFileSync39(cfgPath, JSON.stringify({
|
|
556544
556648
|
source_kind: sourceKind(args),
|
|
556545
556649
|
source: src2,
|
|
@@ -593670,9 +593774,9 @@ ${result}`
|
|
|
593670
593774
|
try {
|
|
593671
593775
|
const { writeFileSync: writeFileSync95, readFileSync: readFileSync137, unlinkSync: unlinkSync37 } = await import("node:fs");
|
|
593672
593776
|
const { join: join186 } = await import("node:path");
|
|
593673
|
-
const { tmpdir:
|
|
593674
|
-
const tmpIn = join186(
|
|
593675
|
-
const tmpOut = join186(
|
|
593777
|
+
const { tmpdir: tmpdir26 } = await import("node:os");
|
|
593778
|
+
const tmpIn = join186(tmpdir26(), `omnius_img_in_${Date.now()}.png`);
|
|
593779
|
+
const tmpOut = join186(tmpdir26(), `omnius_img_out_${Date.now()}.jpg`);
|
|
593676
593780
|
writeFileSync95(tmpIn, buffer2);
|
|
593677
593781
|
const pyBin = process.platform === "win32" ? "python" : "python3";
|
|
593678
593782
|
const resizeScript = [
|
|
@@ -595207,7 +595311,7 @@ var init_constraint_learner = __esm({
|
|
|
595207
595311
|
import { existsSync as existsSync102, statSync as statSync39, openSync, readSync, closeSync, unlinkSync as unlinkSync18, writeFileSync as writeFileSync49 } from "node:fs";
|
|
595208
595312
|
import { watch as fsWatch } from "node:fs";
|
|
595209
595313
|
import { join as join113 } from "node:path";
|
|
595210
|
-
import { tmpdir as
|
|
595314
|
+
import { tmpdir as tmpdir21 } from "node:os";
|
|
595211
595315
|
import { randomBytes as randomBytes21 } from "node:crypto";
|
|
595212
595316
|
var NexusAgenticBackend;
|
|
595213
595317
|
var init_nexusBackend = __esm({
|
|
@@ -595409,7 +595513,7 @@ ${suffix}` } : m2);
|
|
|
595409
595513
|
* Falls back to unary + word-split if streaming setup fails.
|
|
595410
595514
|
*/
|
|
595411
595515
|
async *chatCompletionStream(request) {
|
|
595412
|
-
const streamFile = join113(
|
|
595516
|
+
const streamFile = join113(tmpdir21(), `nexus-stream-${randomBytes21(6).toString("hex")}.jsonl`);
|
|
595413
595517
|
writeFileSync49(streamFile, "", "utf8");
|
|
595414
595518
|
const effectiveThink = this.effectiveThink(request);
|
|
595415
595519
|
const daemonArgs = {
|
|
@@ -602864,9 +602968,9 @@ function parseCameraOrientationDecision(value2) {
|
|
|
602864
602968
|
}
|
|
602865
602969
|
function formatCameraRotation(rotation) {
|
|
602866
602970
|
if (rotation === 0) return "upright/no correction";
|
|
602867
|
-
if (rotation === 90) return "90 degrees clockwise";
|
|
602868
|
-
if (rotation === 180) return "180 degrees";
|
|
602869
|
-
return "90 degrees counter-clockwise";
|
|
602971
|
+
if (rotation === 90) return "90 degrees clockwise correction";
|
|
602972
|
+
if (rotation === 180) return "180 degrees correction";
|
|
602973
|
+
return "90 degrees counter-clockwise correction";
|
|
602870
602974
|
}
|
|
602871
602975
|
function sanitizeCameraOrientation(value2) {
|
|
602872
602976
|
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2)) return {};
|
|
@@ -602992,7 +603096,7 @@ function liveDashboardPreviewWidthForSources(sourceCount, termWidth = process.st
|
|
|
602992
603096
|
function formatCameraPane(camera, paneWidth, paneHeight, now2) {
|
|
602993
603097
|
const contentWidth = Math.max(10, paneWidth - 2);
|
|
602994
603098
|
const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
|
|
602995
|
-
const title =
|
|
603099
|
+
const title = `↺ ${compactSourceId(camera.source)} ${age}s${cameraPaneBadge(camera)} ↻`;
|
|
602996
603100
|
const topTitle = ` ${title} `;
|
|
602997
603101
|
const left = Math.max(0, Math.floor((contentWidth - topTitle.length) / 2));
|
|
602998
603102
|
const right = Math.max(0, contentWidth - topTitle.length - left);
|
|
@@ -603003,6 +603107,30 @@ function formatCameraPane(camera, paneWidth, paneHeight, now2) {
|
|
|
603003
603107
|
const body = fitDashboardPane(previewLines, contentWidth, paneHeight, fallback);
|
|
603004
603108
|
return [top, ...body.map((line) => `│${truncateAnsiCell(line, contentWidth)}│`), bottom];
|
|
603005
603109
|
}
|
|
603110
|
+
function liveDashboardRotationActionFromClick(snapshot, lineText, col) {
|
|
603111
|
+
const plain = stripDashboardAnsi(lineText);
|
|
603112
|
+
const idx = Math.max(0, col - 1);
|
|
603113
|
+
const clicked = plain[idx];
|
|
603114
|
+
if (clicked !== "↺" && clicked !== "↻") return null;
|
|
603115
|
+
const direction = clicked === "↺" ? "ccw" : "cw";
|
|
603116
|
+
const cameras = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {}));
|
|
603117
|
+
if (cameras.length === 0) return null;
|
|
603118
|
+
const leftPositions = [];
|
|
603119
|
+
for (let i2 = 0; i2 < plain.length; i2++) {
|
|
603120
|
+
if (plain[i2] === "↺") leftPositions.push(i2);
|
|
603121
|
+
}
|
|
603122
|
+
for (let i2 = 0; i2 < leftPositions.length; i2++) {
|
|
603123
|
+
const left = leftPositions[i2];
|
|
603124
|
+
const nextLeft = leftPositions[i2 + 1] ?? plain.length;
|
|
603125
|
+
const right = plain.indexOf("↻", left + 1);
|
|
603126
|
+
if (right < 0 || right >= nextLeft) continue;
|
|
603127
|
+
if (idx !== left && idx !== right) continue;
|
|
603128
|
+
const segment = plain.slice(left, right + 1);
|
|
603129
|
+
const camera = cameras.find((entry) => segment.includes(compactSourceId(entry.source)));
|
|
603130
|
+
if (camera) return { source: camera.source, direction };
|
|
603131
|
+
}
|
|
603132
|
+
return null;
|
|
603133
|
+
}
|
|
603006
603134
|
function pushDashboardWrapped(lines, value2, width) {
|
|
603007
603135
|
const contentWidth = Math.max(10, width - 2);
|
|
603008
603136
|
const words = stripDashboardAnsi(value2).replace(/\s+/g, " ").trim().split(" ").filter(Boolean);
|
|
@@ -603127,6 +603255,116 @@ function fallbackLiveAudioIntakeDecision(transcript) {
|
|
|
603127
603255
|
reason: addressed ? "fallback detected direct address or question shape" : "fallback could not establish that ambient speech was addressed to Omnius"
|
|
603128
603256
|
};
|
|
603129
603257
|
}
|
|
603258
|
+
function parseLiveSpeechDecision(value2) {
|
|
603259
|
+
const parsed = parseJsonObjectFromText(value2);
|
|
603260
|
+
if (!parsed) return null;
|
|
603261
|
+
const speak = parsed["speak"];
|
|
603262
|
+
const confidence2 = Math.max(0, Math.min(1, Number(parsed["confidence"] ?? 0)));
|
|
603263
|
+
if (typeof speak !== "boolean" || !Number.isFinite(confidence2)) return null;
|
|
603264
|
+
const actionRaw = String(parsed["action"] ?? (speak ? "speak" : "silent")).toLowerCase();
|
|
603265
|
+
const action = actionRaw === "speak" || actionRaw === "silent" || actionRaw === "queue_review" ? actionRaw : speak ? "speak" : "silent";
|
|
603266
|
+
return {
|
|
603267
|
+
speak,
|
|
603268
|
+
text: trimOneLine(parsed["text"] ?? "", 180),
|
|
603269
|
+
confidence: confidence2,
|
|
603270
|
+
action,
|
|
603271
|
+
reason: trimOneLine(parsed["reason"] ?? "", 220) || "live speech arbiter decision"
|
|
603272
|
+
};
|
|
603273
|
+
}
|
|
603274
|
+
function fallbackLiveSpeechDecision(proposal) {
|
|
603275
|
+
if (proposal.kind === "unknown_person") {
|
|
603276
|
+
return {
|
|
603277
|
+
speak: true,
|
|
603278
|
+
text: proposal.defaultText,
|
|
603279
|
+
confidence: 0.72,
|
|
603280
|
+
action: "speak",
|
|
603281
|
+
reason: "unknown person visible and identity clarification is useful"
|
|
603282
|
+
};
|
|
603283
|
+
}
|
|
603284
|
+
if (proposal.kind === "audio_intent") {
|
|
603285
|
+
return {
|
|
603286
|
+
speak: true,
|
|
603287
|
+
text: proposal.defaultText,
|
|
603288
|
+
confidence: 0.68,
|
|
603289
|
+
action: "speak",
|
|
603290
|
+
reason: "audio intake classified speech as addressed to Omnius"
|
|
603291
|
+
};
|
|
603292
|
+
}
|
|
603293
|
+
if (proposal.urgency === "high") {
|
|
603294
|
+
return {
|
|
603295
|
+
speak: true,
|
|
603296
|
+
text: proposal.defaultText,
|
|
603297
|
+
confidence: 0.6,
|
|
603298
|
+
action: "speak",
|
|
603299
|
+
reason: "high-urgency live trigger"
|
|
603300
|
+
};
|
|
603301
|
+
}
|
|
603302
|
+
return {
|
|
603303
|
+
speak: false,
|
|
603304
|
+
text: "",
|
|
603305
|
+
confidence: 0.55,
|
|
603306
|
+
action: "silent",
|
|
603307
|
+
reason: "ambient event did not justify interrupting with speech"
|
|
603308
|
+
};
|
|
603309
|
+
}
|
|
603310
|
+
async function classifyLiveSpeech(proposal, config) {
|
|
603311
|
+
const now2 = Date.now();
|
|
603312
|
+
if (config?.backendUrl && config.model) {
|
|
603313
|
+
const url = `${config.backendUrl.replace(/\/+$/, "")}/v1/chat/completions`;
|
|
603314
|
+
const body = {
|
|
603315
|
+
model: config.model,
|
|
603316
|
+
temperature: 0,
|
|
603317
|
+
max_tokens: 180,
|
|
603318
|
+
think: false,
|
|
603319
|
+
messages: [
|
|
603320
|
+
{
|
|
603321
|
+
role: "system",
|
|
603322
|
+
content: [
|
|
603323
|
+
"You are Omnius' live audio/video speech arbiter.",
|
|
603324
|
+
"Decide whether the agent should speak aloud into the physical environment right now.",
|
|
603325
|
+
"Speak rarely. Do not narrate routine detections or every frame.",
|
|
603326
|
+
"Speak when directly addressed, when an unknown person should be politely identified, or when an important visual/audio trigger needs immediate clarification.",
|
|
603327
|
+
"Never invent a person's identity. If asking identity, ask one short natural question.",
|
|
603328
|
+
"Keep spoken text under 16 words.",
|
|
603329
|
+
'Return only JSON: {"speak": boolean, "text": "short utterance", "confidence": 0.0-1.0, "action": "speak"|"silent"|"queue_review", "reason": "brief evidence"}.'
|
|
603330
|
+
].join("\n")
|
|
603331
|
+
},
|
|
603332
|
+
{
|
|
603333
|
+
role: "user",
|
|
603334
|
+
content: [
|
|
603335
|
+
`kind=${proposal.kind}`,
|
|
603336
|
+
`source=${proposal.source}`,
|
|
603337
|
+
`urgency=${proposal.urgency}`,
|
|
603338
|
+
`summary=${proposal.summary}`,
|
|
603339
|
+
`evidence=${proposal.evidence}`,
|
|
603340
|
+
`default_utterance=${proposal.defaultText}`
|
|
603341
|
+
].join("\n").slice(0, 1600)
|
|
603342
|
+
}
|
|
603343
|
+
]
|
|
603344
|
+
};
|
|
603345
|
+
try {
|
|
603346
|
+
const controller = new AbortController();
|
|
603347
|
+
const timer = setTimeout(() => controller.abort(), 5e3).unref();
|
|
603348
|
+
const resp = await fetch(url, {
|
|
603349
|
+
method: "POST",
|
|
603350
|
+
headers: {
|
|
603351
|
+
"Content-Type": "application/json",
|
|
603352
|
+
...config.apiKey ? { Authorization: `Bearer ${config.apiKey}` } : {}
|
|
603353
|
+
},
|
|
603354
|
+
body: JSON.stringify(body),
|
|
603355
|
+
signal: controller.signal
|
|
603356
|
+
});
|
|
603357
|
+
clearTimeout(timer);
|
|
603358
|
+
if (resp.ok) {
|
|
603359
|
+
const json = await resp.json();
|
|
603360
|
+
const parsed = parseLiveSpeechDecision(json.choices?.[0]?.message?.content ?? "");
|
|
603361
|
+
if (parsed) return { ...parsed, source: "live-arbiter", updatedAt: now2 };
|
|
603362
|
+
}
|
|
603363
|
+
} catch {
|
|
603364
|
+
}
|
|
603365
|
+
}
|
|
603366
|
+
return { ...fallbackLiveSpeechDecision(proposal), source: "fallback", updatedAt: now2 };
|
|
603367
|
+
}
|
|
603130
603368
|
async function classifyLiveAudioIntake(transcript, config) {
|
|
603131
603369
|
const now2 = Date.now();
|
|
603132
603370
|
const clean5 = transcript.trim();
|
|
@@ -603679,6 +603917,8 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
603679
603917
|
lines.push("<live-sensor-context>");
|
|
603680
603918
|
lines.push("## LIVE SENSOR STATUS");
|
|
603681
603919
|
lines.push("Live sensor context is operator-enabled ambient input. Treat it as current perceptual evidence for this turn; if it is stale, missing, or insufficient, use camera/audio tools to refresh before answering what you see or hear.");
|
|
603920
|
+
lines.push('Live world-model rule: when the user asks what is around, who is present, what changed, what is heard, or asks a deictic question like "what do you see", treat the live camera/audio fields as first-class context. Be curious and tool-capable: refresh stale sensors, inspect frame paths, query vision/CLIP/visual_memory/audio tools, and combine metadata into a concise answer. Do not rely only on summaries when the task requires current perception.');
|
|
603921
|
+
lines.push("Live speech rule: in low-latency voice/live modes, short spoken replies are appropriate for direct address, identity clarification, or important visual/audio triggers. Do not narrate every frame.");
|
|
603682
603922
|
lines.push("Identity rule: never invent names for observed people. If a person is unknown and identity matters, ask the user or the person for their name, then store the association with visual_memory(action='enroll', image='<face crop>', name='<name>') when a face crop is available, or multimodal_memory(action='meet', person_name='<name>') when live voice/name context is available.");
|
|
603683
603923
|
lines.push(`Snapshot file: ${liveSnapshotPath(snapshot.repoRoot)}`);
|
|
603684
603924
|
lines.push(`Streams: video=${snapshot.config.videoEnabled ? "on" : "off"} infer=${snapshot.config.inferEnabled ? "on" : "off"} clip=${snapshot.config.clipEnabled ? "on" : "off"} audio=${snapshot.config.audioEnabled ? "on" : "off"} asr=${snapshot.config.asrEnabled ? "on" : "off"} sounds=${snapshot.config.audioAnalysisEnabled ? "on" : "off"} output-monitor=${snapshot.config.audioOutputEnabled ? "on" : "off"}`);
|
|
@@ -603825,6 +604065,16 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
603825
604065
|
function formatLiveSensorContext(repoRoot) {
|
|
603826
604066
|
return formatLiveSensorContextFromSnapshot(readLiveSensorSnapshot(repoRoot));
|
|
603827
604067
|
}
|
|
604068
|
+
function isLiveSensorSnapshotActive(snapshot) {
|
|
604069
|
+
const cfg = snapshot?.config;
|
|
604070
|
+
if (!cfg) return false;
|
|
604071
|
+
return Boolean(
|
|
604072
|
+
cfg.videoEnabled || cfg.audioEnabled || cfg.audioOutputEnabled || cfg.inferEnabled || cfg.clipEnabled || cfg.asrEnabled || cfg.audioAnalysisEnabled
|
|
604073
|
+
);
|
|
604074
|
+
}
|
|
604075
|
+
function isLiveSensorActiveForRepo(repoRoot) {
|
|
604076
|
+
return isLiveSensorSnapshotActive(readLiveSensorSnapshot(repoRoot));
|
|
604077
|
+
}
|
|
603828
604078
|
async function discoverAudioInputs() {
|
|
603829
604079
|
const devices = [];
|
|
603830
604080
|
const errors = [];
|
|
@@ -603958,31 +604208,101 @@ print(json.dumps({"ok": True, "scores": scores}))
|
|
|
603958
604208
|
async function tryRecordAudioDevice(device, durationSec, outputPath3) {
|
|
603959
604209
|
await recordAudioSample(device, durationSec, outputPath3);
|
|
603960
604210
|
}
|
|
604211
|
+
function hasLiveAudioSignal(activity) {
|
|
604212
|
+
if (!activity) return false;
|
|
604213
|
+
return activity.peak >= 0.025 || activity.rmsDb > -48 || activity.activeRatio >= 0.025;
|
|
604214
|
+
}
|
|
604215
|
+
function audioCaptureMethod(device) {
|
|
604216
|
+
return device.startsWith("pulse:") ? "pulse/ffmpeg" : "arecord";
|
|
604217
|
+
}
|
|
603961
604218
|
async function recordAudioSampleWithRecovery(preferred, devices, durationSec, outputPath3) {
|
|
603962
604219
|
const ordered = [];
|
|
603963
604220
|
const add3 = (id2) => {
|
|
603964
604221
|
const clean5 = String(id2 ?? "").trim();
|
|
603965
604222
|
if (clean5 && !ordered.includes(clean5)) ordered.push(clean5);
|
|
603966
604223
|
};
|
|
603967
|
-
|
|
603968
|
-
|
|
604224
|
+
const preferredInput = preferredLiveAudioInputId(devices, preferred);
|
|
604225
|
+
add3(preferredInput);
|
|
604226
|
+
if (preferred && !isAudioOutputMonitorId(preferred)) add3(preferred);
|
|
604227
|
+
for (const device of rankLiveAudioInputDevices(devices, preferred).filter((entry) => !isAudioOutputMonitorDevice(entry))) add3(device.id);
|
|
603969
604228
|
add3("pulse:default");
|
|
603970
604229
|
add3("default");
|
|
603971
|
-
for (const device of devices.filter((entry) => entry
|
|
604230
|
+
for (const device of rankLiveAudioInputDevices(devices, preferred).filter((entry) => isAudioOutputMonitorDevice(entry))) add3(device.id);
|
|
603972
604231
|
const errors = [];
|
|
604232
|
+
const attempts = [];
|
|
604233
|
+
let firstQuietCapture = null;
|
|
603973
604234
|
for (const candidate of ordered) {
|
|
604235
|
+
attempts.push(candidate);
|
|
603974
604236
|
try {
|
|
603975
604237
|
await tryRecordAudioDevice(candidate, durationSec, outputPath3);
|
|
603976
|
-
|
|
604238
|
+
const activity = readPcm16WavActivity(outputPath3);
|
|
604239
|
+
const method = audioCaptureMethod(candidate);
|
|
604240
|
+
if (hasLiveAudioSignal(activity)) {
|
|
604241
|
+
return { ok: true, device: candidate, method, errors, attempts, activity, signalDetected: true };
|
|
604242
|
+
}
|
|
604243
|
+
if (!firstQuietCapture) {
|
|
604244
|
+
firstQuietCapture = {
|
|
604245
|
+
device: candidate,
|
|
604246
|
+
method,
|
|
604247
|
+
activity,
|
|
604248
|
+
bytes: readFileSync88(outputPath3),
|
|
604249
|
+
errors: [...errors]
|
|
604250
|
+
};
|
|
604251
|
+
}
|
|
604252
|
+
errors.push(`${candidate}: captured but no live input signal detected`);
|
|
603977
604253
|
} catch (err) {
|
|
603978
604254
|
errors.push(`${candidate}: ${err instanceof Error ? err.message : String(err)}`.slice(0, 360));
|
|
603979
604255
|
}
|
|
603980
604256
|
}
|
|
603981
|
-
|
|
604257
|
+
if (firstQuietCapture) {
|
|
604258
|
+
writeFileSync56(outputPath3, firstQuietCapture.bytes);
|
|
604259
|
+
return {
|
|
604260
|
+
ok: true,
|
|
604261
|
+
device: firstQuietCapture.device,
|
|
604262
|
+
method: firstQuietCapture.method,
|
|
604263
|
+
errors,
|
|
604264
|
+
attempts,
|
|
604265
|
+
activity: firstQuietCapture.activity,
|
|
604266
|
+
signalDetected: false
|
|
604267
|
+
};
|
|
604268
|
+
}
|
|
604269
|
+
return { ok: false, device: preferredInput || preferred || "default", method: "none", errors, attempts };
|
|
603982
604270
|
}
|
|
603983
604271
|
function firstDeviceId(devices) {
|
|
603984
604272
|
return devices.find((device) => device.id && !/metadata\/non-capture/i.test(device.detail ?? ""))?.id ?? devices[0]?.id;
|
|
603985
604273
|
}
|
|
604274
|
+
function isAudioOutputMonitorId(id2) {
|
|
604275
|
+
return /(?:\.monitor\b|sink\.monitor|alsa_output|output\.monitor)/i.test(String(id2 ?? ""));
|
|
604276
|
+
}
|
|
604277
|
+
function isAudioOutputMonitorDevice(device) {
|
|
604278
|
+
if (!device) return false;
|
|
604279
|
+
return isAudioOutputMonitorId(device.id) || isAudioOutputMonitorId(device.label) || isAudioOutputMonitorId(device.detail);
|
|
604280
|
+
}
|
|
604281
|
+
function preferredLiveAudioInputId(devices, configured) {
|
|
604282
|
+
const usable = devices.filter((device) => device.id && !isAudioOutputMonitorDevice(device));
|
|
604283
|
+
if (configured && usable.some((device) => device.id === configured)) return configured;
|
|
604284
|
+
return rankLiveAudioInputDevices(devices, configured)[0]?.id ?? firstDeviceId(devices);
|
|
604285
|
+
}
|
|
604286
|
+
function liveAudioDeviceRank(device, configured) {
|
|
604287
|
+
let rank = 0;
|
|
604288
|
+
if (device.id === configured) rank -= 40;
|
|
604289
|
+
if (isAudioOutputMonitorDevice(device)) rank += 500;
|
|
604290
|
+
if (device.source === "pulse" || device.source === "pipewire") rank -= 20;
|
|
604291
|
+
if (device.source === "alsa") rank -= 10;
|
|
604292
|
+
if (device.source === "default") rank += 30;
|
|
604293
|
+
if (/metadata\/non-capture/i.test(device.detail ?? "")) rank += 300;
|
|
604294
|
+
return rank;
|
|
604295
|
+
}
|
|
604296
|
+
function rankLiveAudioInputDevices(devices, configured) {
|
|
604297
|
+
const seen = /* @__PURE__ */ new Set();
|
|
604298
|
+
const ranked = [];
|
|
604299
|
+
for (const device of devices) {
|
|
604300
|
+
if (!device.id || seen.has(device.id)) continue;
|
|
604301
|
+
seen.add(device.id);
|
|
604302
|
+
ranked.push(device);
|
|
604303
|
+
}
|
|
604304
|
+
return ranked.sort((a2, b) => liveAudioDeviceRank(a2, configured) - liveAudioDeviceRank(b, configured));
|
|
604305
|
+
}
|
|
603986
604306
|
function getLiveSensorManager(repoRoot) {
|
|
603987
604307
|
const key = repoRoot;
|
|
603988
604308
|
let manager = managers.get(key);
|
|
@@ -604076,6 +604396,16 @@ var init_live_sensors = __esm({
|
|
|
604076
604396
|
lastAgentReviewAt = 0;
|
|
604077
604397
|
autoOrientationInFlight = /* @__PURE__ */ new Set();
|
|
604078
604398
|
lastFeedbackAt = /* @__PURE__ */ new Map();
|
|
604399
|
+
lastClipAt = /* @__PURE__ */ new Map();
|
|
604400
|
+
lastInferenceAt = /* @__PURE__ */ new Map();
|
|
604401
|
+
nextVideoSourceIndex = 0;
|
|
604402
|
+
liveSpeechSink = null;
|
|
604403
|
+
liveSpeechEnabled = false;
|
|
604404
|
+
lastLiveSpeechAt = 0;
|
|
604405
|
+
lastLiveSpeechByKey = /* @__PURE__ */ new Map();
|
|
604406
|
+
liveSpeechInFlight = /* @__PURE__ */ new Set();
|
|
604407
|
+
liveSpeechMutedUntil = 0;
|
|
604408
|
+
liveTtsSpeaking = null;
|
|
604079
604409
|
intakeAgentConfig;
|
|
604080
604410
|
setStatusSink(sink) {
|
|
604081
604411
|
this.statusSink = sink ?? null;
|
|
@@ -604087,6 +604417,21 @@ var init_live_sensors = __esm({
|
|
|
604087
604417
|
setAgentActionSink(sink) {
|
|
604088
604418
|
this.agentActionSink = sink ?? null;
|
|
604089
604419
|
}
|
|
604420
|
+
setLiveSpeechSink(sink) {
|
|
604421
|
+
this.liveSpeechSink = sink ?? null;
|
|
604422
|
+
}
|
|
604423
|
+
setLiveSpeechEnabled(enabled2) {
|
|
604424
|
+
this.liveSpeechEnabled = enabled2;
|
|
604425
|
+
}
|
|
604426
|
+
setLiveTtsSpeakingSource(source) {
|
|
604427
|
+
this.liveTtsSpeaking = source ?? null;
|
|
604428
|
+
}
|
|
604429
|
+
muteLiveAudioFor(ms) {
|
|
604430
|
+
this.liveSpeechMutedUntil = Math.max(this.liveSpeechMutedUntil, Date.now() + Math.max(0, ms));
|
|
604431
|
+
}
|
|
604432
|
+
isLiveAudioMutedForTts() {
|
|
604433
|
+
return Date.now() < this.liveSpeechMutedUntil || Boolean(this.liveTtsSpeaking?.());
|
|
604434
|
+
}
|
|
604090
604435
|
setAudioIntakeAgentConfig(config) {
|
|
604091
604436
|
this.intakeAgentConfig = config && config.backendUrl && config.model ? { backendUrl: config.backendUrl, model: config.model, apiKey: config.apiKey } : void 0;
|
|
604092
604437
|
}
|
|
@@ -604157,6 +604502,12 @@ var init_live_sensors = __esm({
|
|
|
604157
604502
|
const next = current === 0 ? 90 : current === 90 ? 180 : current === 180 ? 270 : 0;
|
|
604158
604503
|
return this.setCameraRotation(device, next, "manual", "manual cycle from /live menu");
|
|
604159
604504
|
}
|
|
604505
|
+
rotateCameraRelative(device, direction) {
|
|
604506
|
+
const current = this.getCameraRotation(device);
|
|
604507
|
+
const delta = direction === "cw" ? 90 : 270;
|
|
604508
|
+
const next = (current + delta) % 360;
|
|
604509
|
+
return this.setCameraRotation(device, next, "manual", `manual ${direction === "cw" ? "clockwise" : "counter-clockwise"} dashboard override`);
|
|
604510
|
+
}
|
|
604160
604511
|
async autoOrientCamera(device = this.config.selectedCamera, options2 = {}) {
|
|
604161
604512
|
const target = device || this.config.selectedCamera || firstDeviceId(this.devices.video);
|
|
604162
604513
|
if (!target) return "No camera selected. Use /live camera <device> or select a camera in /live.";
|
|
@@ -604182,16 +604533,14 @@ var init_live_sensors = __esm({
|
|
|
604182
604533
|
if (!framePath) return `Camera orientation detection captured ${target}, but no frame path was returned.`;
|
|
604183
604534
|
const cvScores = await scoreCameraOrientationWithOpenCv(framePath);
|
|
604184
604535
|
const cvDecision = cvScores ? chooseCameraOrientationFromScores(cvScores) : null;
|
|
604185
|
-
if (cvDecision && cvDecision.confidence >= 0.68) {
|
|
604186
|
-
const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
|
|
604187
|
-
return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
|
|
604188
|
-
}
|
|
604189
604536
|
const prompt = [
|
|
604190
604537
|
"Determine whether this camera frame is upright for a human viewer.",
|
|
604191
604538
|
"Return only JSON with this schema:",
|
|
604192
604539
|
'{"rotation_degrees":0|90|180|270,"confidence":0.0-1.0,"reason":"brief visual evidence"}',
|
|
604193
604540
|
"rotation_degrees is the clockwise correction to apply to future frames so they are upright.",
|
|
604194
|
-
"Use 90 if the image must rotate clockwise; use 270 if it must rotate counter-clockwise; use 180 if upside down; use 0 if already upright or inconclusive."
|
|
604541
|
+
"Use 90 if the image must rotate clockwise; use 270 if it must rotate counter-clockwise; use 180 if upside down; use 0 if already upright or inconclusive.",
|
|
604542
|
+
"If the visible scene is sideways, choose the correction direction that makes people, faces, screens, and text upright; if the opposite direction would make them sideways, do not choose it.",
|
|
604543
|
+
cvDecision ? `Computer-vision candidate: ${formatCameraRotation(cvDecision.rotation)} confidence=${cvDecision.confidence.toFixed(2)} evidence=${cvDecision.reason}` : "Computer-vision candidate: unavailable."
|
|
604195
604544
|
].join("\n");
|
|
604196
604545
|
const vision = await new VisionTool(this.repoRoot).execute({
|
|
604197
604546
|
action: "query",
|
|
@@ -604214,9 +604563,10 @@ var init_live_sensors = __esm({
|
|
|
604214
604563
|
}
|
|
604215
604564
|
return `Camera orientation detection for ${target} was inconclusive. Vision output did not contain a structured rotation decision.`;
|
|
604216
604565
|
}
|
|
604217
|
-
const
|
|
604218
|
-
const
|
|
604219
|
-
const
|
|
604566
|
+
const visionConfidence = decision2.confidence ?? 0;
|
|
604567
|
+
const useCv = Boolean(cvDecision) && (visionConfidence < 0.55 || cvDecision.rotation === decision2.rotation && cvDecision.confidence > visionConfidence + 0.18);
|
|
604568
|
+
const selected = useCv && cvDecision ? cvDecision : decision2;
|
|
604569
|
+
const evidence = useCv ? cvDecision?.reason ?? "" : `${decision2.reason || boundedText(vision.llmContent || vision.output, 320)}${cvDecision ? `; cv=${cvDecision.reason}` : ""}`;
|
|
604220
604570
|
const orientation = this.setCameraRotation(target, selected.rotation, "auto", evidence, selected.confidence);
|
|
604221
604571
|
return orientation ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation)}` : `Auto orientation for ${target} could not be persisted.`;
|
|
604222
604572
|
} finally {
|
|
@@ -604245,7 +604595,10 @@ var init_live_sensors = __esm({
|
|
|
604245
604595
|
errors
|
|
604246
604596
|
};
|
|
604247
604597
|
if (!this.config.selectedCamera) this.config.selectedCamera = firstDeviceId(video);
|
|
604248
|
-
|
|
604598
|
+
const preferredInput = preferredLiveAudioInputId(inputs.devices, this.config.selectedAudioInput);
|
|
604599
|
+
if (!this.config.selectedAudioInput || isAudioOutputMonitorId(this.config.selectedAudioInput) && preferredInput && !isAudioOutputMonitorId(preferredInput)) {
|
|
604600
|
+
this.config.selectedAudioInput = preferredInput;
|
|
604601
|
+
}
|
|
604249
604602
|
if (!this.config.selectedAudioOutput) this.config.selectedAudioOutput = firstDeviceId(outputs.devices);
|
|
604250
604603
|
this.persist();
|
|
604251
604604
|
return this.devices;
|
|
@@ -604265,6 +604618,7 @@ var init_live_sensors = __esm({
|
|
|
604265
604618
|
}
|
|
604266
604619
|
stopAll() {
|
|
604267
604620
|
this.agentActionEnabled = false;
|
|
604621
|
+
this.liveSpeechEnabled = false;
|
|
604268
604622
|
this.configure({
|
|
604269
604623
|
videoEnabled: false,
|
|
604270
604624
|
audioEnabled: false,
|
|
@@ -604277,6 +604631,7 @@ var init_live_sensors = __esm({
|
|
|
604277
604631
|
}
|
|
604278
604632
|
startRunMode(options2 = {}) {
|
|
604279
604633
|
this.agentActionEnabled = Boolean(options2.agent);
|
|
604634
|
+
this.liveSpeechEnabled = Boolean(options2.speech ?? options2.agent);
|
|
604280
604635
|
this.configure({
|
|
604281
604636
|
videoEnabled: true,
|
|
604282
604637
|
inferEnabled: true,
|
|
@@ -604308,7 +604663,7 @@ var init_live_sensors = __esm({
|
|
|
604308
604663
|
}, 6e4);
|
|
604309
604664
|
});
|
|
604310
604665
|
}
|
|
604311
|
-
void this.sampleVideoNow(true).catch((err) => {
|
|
604666
|
+
void this.sampleVideoNow(true, { mode: "all", forceInferenceNow: true }).catch((err) => {
|
|
604312
604667
|
this.emitFeedback({
|
|
604313
604668
|
kind: "error",
|
|
604314
604669
|
title: "Live video sample failed",
|
|
@@ -604495,25 +604850,30 @@ var init_live_sensors = __esm({
|
|
|
604495
604850
|
return message2;
|
|
604496
604851
|
}
|
|
604497
604852
|
}
|
|
604498
|
-
async sampleSingleVideoNow(source, forceInfer, previewWidth) {
|
|
604853
|
+
async sampleSingleVideoNow(source, forceInfer, previewWidth, options2 = {}) {
|
|
604499
604854
|
const preview = await this.previewCamera(source, { emit: false, previewWidth });
|
|
604500
604855
|
let inference = "";
|
|
604501
604856
|
let clipSummary = "";
|
|
604502
604857
|
const orientation = this.getCameraOrientation(source);
|
|
604503
604858
|
const sampledAt = Date.now();
|
|
604504
|
-
|
|
604859
|
+
const inferenceThrottleMs = Math.max(0, Number(options2.inferThrottleMs ?? 2500));
|
|
604860
|
+
const lastInferenceAt = this.lastInferenceAt.get(source) ?? 0;
|
|
604861
|
+
const shouldInfer = Boolean(forceInfer && source && (options2.forceInferenceNow || sampledAt - lastInferenceAt >= inferenceThrottleMs));
|
|
604862
|
+
if (shouldInfer) {
|
|
604863
|
+
this.lastInferenceAt.set(source, sampledAt);
|
|
604505
604864
|
const loop = new LiveMediaLoopTool(this.repoRoot);
|
|
604865
|
+
const inferFromCapturedFrame = preview.ok && Boolean(preview.framePath);
|
|
604506
604866
|
const result = await loop.execute({
|
|
604507
604867
|
action: "watch",
|
|
604508
|
-
source_kind: "camera",
|
|
604509
|
-
camera: source,
|
|
604868
|
+
source_kind: inferFromCapturedFrame ? "file" : "camera",
|
|
604869
|
+
...inferFromCapturedFrame ? { path: preview.framePath } : { camera: source },
|
|
604510
604870
|
yolo_family: "yoloe-26",
|
|
604511
604871
|
yolo_scale: "n",
|
|
604512
604872
|
yoloe_prompt_mode: "prompt_free",
|
|
604513
|
-
duration_sec: 0.75,
|
|
604514
|
-
sample_interval_sec: 0.25,
|
|
604873
|
+
duration_sec: inferFromCapturedFrame ? 0.1 : 0.75,
|
|
604874
|
+
sample_interval_sec: inferFromCapturedFrame ? 0.1 : 0.25,
|
|
604515
604875
|
max_frames: 1,
|
|
604516
|
-
rotate_degrees: orientation?.rotation ?? 0,
|
|
604876
|
+
rotate_degrees: inferFromCapturedFrame ? 0 : orientation?.rotation ?? 0,
|
|
604517
604877
|
auto_bootstrap: true,
|
|
604518
604878
|
audio_mode: "none",
|
|
604519
604879
|
detect_objects: true,
|
|
@@ -604526,6 +604886,7 @@ var init_live_sensors = __esm({
|
|
|
604526
604886
|
});
|
|
604527
604887
|
inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
|
|
604528
604888
|
const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
|
|
604889
|
+
if (livePayload) livePayload.source = source;
|
|
604529
604890
|
const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
|
|
604530
604891
|
const classifications = classificationsFromLiveMediaPayload(livePayload, sampledAt, source);
|
|
604531
604892
|
this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
|
|
@@ -604562,6 +604923,16 @@ var init_live_sensors = __esm({
|
|
|
604562
604923
|
`${unknownPeople.length} unknown individual(s) observed on live camera ${source}`,
|
|
604563
604924
|
unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; ")
|
|
604564
604925
|
);
|
|
604926
|
+
this.maybeSpeakLive({
|
|
604927
|
+
key: `unknown-person:${source}`,
|
|
604928
|
+
kind: "unknown_person",
|
|
604929
|
+
source,
|
|
604930
|
+
summary: `${unknownPeople.length} unknown individual(s) observed on live camera ${source}`,
|
|
604931
|
+
evidence: unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; "),
|
|
604932
|
+
defaultText: "I see someone here, but I do not know who. Who is there?",
|
|
604933
|
+
urgency: "high",
|
|
604934
|
+
observedAt: sampledAt
|
|
604935
|
+
}, 9e4);
|
|
604565
604936
|
}
|
|
604566
604937
|
const triggerEvents = observations.events.filter((entry) => entry.kind === "visual_trigger");
|
|
604567
604938
|
if (triggerEvents.length > 0) {
|
|
@@ -604569,9 +604940,21 @@ var init_live_sensors = __esm({
|
|
|
604569
604940
|
`Visual trigger hit in live camera stream ${source}`,
|
|
604570
604941
|
triggerEvents.map((entry) => entry.summary).join("; ")
|
|
604571
604942
|
);
|
|
604943
|
+
this.maybeSpeakLive({
|
|
604944
|
+
key: `visual-trigger:${source}:${triggerEvents[0]?.title ?? "trigger"}`,
|
|
604945
|
+
kind: "visual_trigger",
|
|
604946
|
+
source,
|
|
604947
|
+
summary: `Visual trigger hit in live camera stream ${source}`,
|
|
604948
|
+
evidence: triggerEvents.map((entry) => entry.summary).join("; "),
|
|
604949
|
+
defaultText: "I noticed something important in view. Should I inspect it?",
|
|
604950
|
+
urgency: "high",
|
|
604951
|
+
observedAt: sampledAt
|
|
604952
|
+
}, 45e3);
|
|
604572
604953
|
}
|
|
604573
604954
|
}
|
|
604574
|
-
|
|
604955
|
+
const lastClip = this.lastClipAt.get(source) ?? 0;
|
|
604956
|
+
if (this.config.clipEnabled && preview.ok && preview.framePath && source && sampledAt - lastClip >= 5e3) {
|
|
604957
|
+
this.lastClipAt.set(source, sampledAt);
|
|
604575
604958
|
clipSummary = await this.recognizeFrameObjects(preview.framePath, source, sampledAt, { emit: false });
|
|
604576
604959
|
const current = this.snapshot.cameras?.[source];
|
|
604577
604960
|
if (current) {
|
|
@@ -604590,16 +604973,27 @@ ${inference}` : "", clipSummary ? `
|
|
|
604590
604973
|
CLIP visual memory:
|
|
604591
604974
|
${clipSummary}` : ""].filter(Boolean).join("\n");
|
|
604592
604975
|
}
|
|
604593
|
-
async
|
|
604976
|
+
async sampleCameraNow(source, forceInfer = this.config.inferEnabled || this.config.clipEnabled, options2 = {}) {
|
|
604977
|
+
const previewWidth = liveDashboardPreviewWidthForSources(this.activeVideoSources().length);
|
|
604978
|
+
const result = await this.sampleSingleVideoNow(source, forceInfer, previewWidth, {
|
|
604979
|
+
forceInferenceNow: options2.forceInferenceNow,
|
|
604980
|
+
inferThrottleMs: options2.forceInferenceNow ? 0 : 2500
|
|
604981
|
+
});
|
|
604982
|
+
this.emitDashboard();
|
|
604983
|
+
return result;
|
|
604984
|
+
}
|
|
604985
|
+
async sampleVideoNow(forceInfer = this.config.inferEnabled, options2 = {}) {
|
|
604594
604986
|
if (this.videoInFlight) return "Video sampler is already running.";
|
|
604595
604987
|
this.videoInFlight = true;
|
|
604596
604988
|
try {
|
|
604597
|
-
const
|
|
604989
|
+
const allSources = this.activeVideoSources();
|
|
604990
|
+
if (allSources.length === 0) return "No camera selected. Use /live refresh after connecting hardware.";
|
|
604991
|
+
const sources = options2.mode === "round-robin" ? [allSources[this.nextVideoSourceIndex++ % allSources.length]] : allSources;
|
|
604598
604992
|
if (sources.length === 0) return "No camera selected. Use /live refresh after connecting hardware.";
|
|
604599
604993
|
const outputs = [];
|
|
604600
|
-
const previewWidth = liveDashboardPreviewWidthForSources(
|
|
604994
|
+
const previewWidth = liveDashboardPreviewWidthForSources(allSources.length);
|
|
604601
604995
|
for (const source of sources) {
|
|
604602
|
-
outputs.push(await this.sampleSingleVideoNow(source, forceInfer, previewWidth));
|
|
604996
|
+
outputs.push(await this.sampleSingleVideoNow(source, forceInfer, previewWidth, options2));
|
|
604603
604997
|
}
|
|
604604
604998
|
this.emitDashboard();
|
|
604605
604999
|
return outputs.map((output, index) => `## Camera ${sources[index]}
|
|
@@ -604612,7 +605006,23 @@ ${output}`).join("\n\n");
|
|
|
604612
605006
|
if (this.audioInFlight) return "Audio sampler is already running.";
|
|
604613
605007
|
this.audioInFlight = true;
|
|
604614
605008
|
try {
|
|
604615
|
-
const
|
|
605009
|
+
const preferredInput = preferredLiveAudioInputId(this.devices.audioInputs, this.config.selectedAudioInput) || "default";
|
|
605010
|
+
const input = this.config.selectedAudioInput && !isAudioOutputMonitorId(this.config.selectedAudioInput) ? this.config.selectedAudioInput : preferredInput;
|
|
605011
|
+
if (input !== this.config.selectedAudioInput) this.config.selectedAudioInput = input;
|
|
605012
|
+
if (this.isLiveAudioMutedForTts()) {
|
|
605013
|
+
const now2 = Date.now();
|
|
605014
|
+
this.snapshot.audio = {
|
|
605015
|
+
updatedAt: now2,
|
|
605016
|
+
input,
|
|
605017
|
+
output: this.config.selectedAudioOutput,
|
|
605018
|
+
transcript: "",
|
|
605019
|
+
analysis: "",
|
|
605020
|
+
error: "Live audio capture muted during Omnius TTS playback to prevent echo."
|
|
605021
|
+
};
|
|
605022
|
+
this.persist();
|
|
605023
|
+
this.emitDashboard();
|
|
605024
|
+
return "Live audio capture muted during Omnius TTS playback to prevent echo.";
|
|
605025
|
+
}
|
|
604616
605026
|
const outputDir2 = liveDir(this.repoRoot);
|
|
604617
605027
|
ensureLiveDir(this.repoRoot);
|
|
604618
605028
|
const recordingPath = join124(outputDir2, `audio-${Date.now()}.wav`);
|
|
@@ -604643,10 +605053,17 @@ ${output}`).join("\n\n");
|
|
|
604643
605053
|
if (capture.device !== this.config.selectedAudioInput) {
|
|
604644
605054
|
this.config.selectedAudioInput = capture.device;
|
|
604645
605055
|
}
|
|
604646
|
-
const activity = readPcm16WavActivity(recordingPath);
|
|
605056
|
+
const activity = capture.activity ?? readPcm16WavActivity(recordingPath);
|
|
605057
|
+
if (capture.ok && capture.signalDetected === false) {
|
|
605058
|
+
audioErrors.push(`No live input signal detected after probing ${capture.attempts?.length ?? 1} audio source(s); using quiet capture from ${capture.device}`);
|
|
605059
|
+
}
|
|
604647
605060
|
if (this.config.asrEnabled) {
|
|
604648
605061
|
try {
|
|
604649
|
-
const asr = await new TranscribeFileTool(this.repoRoot).execute({
|
|
605062
|
+
const asr = await new TranscribeFileTool(this.repoRoot).execute({
|
|
605063
|
+
path: recordingPath,
|
|
605064
|
+
model: "tiny",
|
|
605065
|
+
backend: "managed-whisper"
|
|
605066
|
+
});
|
|
604650
605067
|
if (asr.success) {
|
|
604651
605068
|
transcript = cleanPreview(typeof asr.llmContent === "string" ? asr.llmContent : "", 1600);
|
|
604652
605069
|
asrBackend = extractAsrBackend(asr.output) ?? "whisper";
|
|
@@ -604720,6 +605137,16 @@ ${output}`).join("\n\n");
|
|
|
604720
605137
|
"Live audio intake classified speech as intended for Omnius",
|
|
604721
605138
|
`transcript=${transcript}; confidence=${intake.confidence.toFixed(2)}; reason=${intake.reason}`
|
|
604722
605139
|
);
|
|
605140
|
+
this.maybeSpeakLive({
|
|
605141
|
+
key: `audio-intent:${capture.device}:${transcript.slice(0, 48)}`,
|
|
605142
|
+
kind: "audio_intent",
|
|
605143
|
+
source: capture.device,
|
|
605144
|
+
summary: "Live audio intake classified speech as intended for Omnius",
|
|
605145
|
+
evidence: `transcript=${transcript}; confidence=${intake.confidence.toFixed(2)}; reason=${intake.reason}`,
|
|
605146
|
+
defaultText: "I heard you. I am checking the live context now.",
|
|
605147
|
+
urgency: "high",
|
|
605148
|
+
observedAt: this.snapshot.audio.updatedAt
|
|
605149
|
+
}, 2e4);
|
|
604723
605150
|
}
|
|
604724
605151
|
this.emitDashboard();
|
|
604725
605152
|
return [
|
|
@@ -604753,7 +605180,7 @@ ${analysis}` : ""
|
|
|
604753
605180
|
this.videoTimer = null;
|
|
604754
605181
|
if (!this.config.videoEnabled) return;
|
|
604755
605182
|
try {
|
|
604756
|
-
await this.sampleVideoNow(this.config.inferEnabled || this.config.clipEnabled);
|
|
605183
|
+
await this.sampleVideoNow(this.config.inferEnabled || this.config.clipEnabled, { mode: "round-robin" });
|
|
604757
605184
|
} catch {
|
|
604758
605185
|
} finally {
|
|
604759
605186
|
if (this.config.videoEnabled) {
|
|
@@ -604828,7 +605255,7 @@ ${analysis}` : ""
|
|
|
604828
605255
|
maybeRequestAgentReview(reason, evidence) {
|
|
604829
605256
|
if (!this.agentActionEnabled || !this.agentActionSink) return;
|
|
604830
605257
|
const now2 = Date.now();
|
|
604831
|
-
if (now2 - this.lastAgentReviewAt <
|
|
605258
|
+
if (now2 - this.lastAgentReviewAt < 6e4) return;
|
|
604832
605259
|
this.lastAgentReviewAt = now2;
|
|
604833
605260
|
const prompt = [
|
|
604834
605261
|
"Live environment PFC review.",
|
|
@@ -604851,6 +605278,38 @@ ${analysis}` : ""
|
|
|
604851
605278
|
} catch {
|
|
604852
605279
|
}
|
|
604853
605280
|
}
|
|
605281
|
+
maybeSpeakLive(proposal, perKeyCooldownMs) {
|
|
605282
|
+
if (!this.liveSpeechEnabled || !this.liveSpeechSink) return;
|
|
605283
|
+
const now2 = Date.now();
|
|
605284
|
+
if (now2 - this.lastLiveSpeechAt < 12e3) return;
|
|
605285
|
+
const lastForKey = this.lastLiveSpeechByKey.get(proposal.key) ?? 0;
|
|
605286
|
+
if (now2 - lastForKey < perKeyCooldownMs) return;
|
|
605287
|
+
if (this.liveSpeechInFlight.has(proposal.key)) return;
|
|
605288
|
+
this.liveSpeechInFlight.add(proposal.key);
|
|
605289
|
+
void classifyLiveSpeech(proposal, this.intakeAgentConfig).then((decision2) => {
|
|
605290
|
+
this.lastLiveSpeechByKey.set(proposal.key, Date.now());
|
|
605291
|
+
if (!decision2.speak || !decision2.text.trim()) return;
|
|
605292
|
+
this.lastLiveSpeechAt = Date.now();
|
|
605293
|
+
const text2 = trimOneLine(decision2.text, 180);
|
|
605294
|
+
this.liveSpeechSink?.(text2, decision2);
|
|
605295
|
+
this.emitFeedbackThrottled(`speech:${proposal.key}`, {
|
|
605296
|
+
kind: "speech",
|
|
605297
|
+
title: "Live speech emitted",
|
|
605298
|
+
observedAt: decision2.updatedAt,
|
|
605299
|
+
message: `${text2}
|
|
605300
|
+
reason=${decision2.reason} confidence=${decision2.confidence.toFixed(2)} source=${decision2.source}`
|
|
605301
|
+
}, 15e3);
|
|
605302
|
+
}).catch((err) => {
|
|
605303
|
+
this.emitFeedbackThrottled(`speech-error:${proposal.key}`, {
|
|
605304
|
+
kind: "error",
|
|
605305
|
+
title: "Live speech arbiter failed",
|
|
605306
|
+
observedAt: Date.now(),
|
|
605307
|
+
message: err instanceof Error ? err.message : String(err)
|
|
605308
|
+
}, 6e4);
|
|
605309
|
+
}).finally(() => {
|
|
605310
|
+
this.liveSpeechInFlight.delete(proposal.key);
|
|
605311
|
+
});
|
|
605312
|
+
}
|
|
604854
605313
|
persist() {
|
|
604855
605314
|
ensureLiveDir(this.repoRoot);
|
|
604856
605315
|
writeFileSync56(liveConfigPath(this.repoRoot), JSON.stringify(this.config, null, 2), "utf8");
|
|
@@ -605957,7 +606416,7 @@ var init_listen = __esm({
|
|
|
605957
606416
|
let caughtUncaught = null;
|
|
605958
606417
|
const uncaughtHandler = (err) => {
|
|
605959
606418
|
const msg = err?.message || String(err);
|
|
605960
|
-
if (msg.includes("live_worker") || msg.includes("numpy") || msg.includes("Traceback")) {
|
|
606419
|
+
if (msg.includes("live_worker") || msg.includes("numpy") || msg.includes("transcribe_cli") || msg.includes("ModuleNotFoundError") || msg.includes("Traceback")) {
|
|
605961
606420
|
caughtUncaught = err;
|
|
605962
606421
|
} else {
|
|
605963
606422
|
throw err;
|
|
@@ -624909,6 +625368,7 @@ var init_status_bar = __esm({
|
|
|
624909
625368
|
* repaint, including selection updates and scroll events.
|
|
624910
625369
|
*/
|
|
624911
625370
|
_dynamicBlocks = /* @__PURE__ */ new Map();
|
|
625371
|
+
_dynamicBlockClickHandlers = /* @__PURE__ */ new Map();
|
|
624912
625372
|
/** Sentinel marker — used both for scrollback storage and reflow detection. */
|
|
624913
625373
|
DYNAMIC_BLOCK_MARK_PREFIX = "DYNBLOCK:";
|
|
624914
625374
|
DYNAMIC_BLOCK_MARK_SUFFIX = "";
|
|
@@ -625055,9 +625515,13 @@ var init_status_bar = __esm({
|
|
|
625055
625515
|
this.invalidateRowCountCache();
|
|
625056
625516
|
return `${this.DYNAMIC_BLOCK_MARK_PREFIX}${id2}${this.DYNAMIC_BLOCK_MARK_SUFFIX}`;
|
|
625057
625517
|
}
|
|
625518
|
+
registerDynamicBlockClickHandler(id2, handler) {
|
|
625519
|
+
this._dynamicBlockClickHandlers.set(id2, handler);
|
|
625520
|
+
}
|
|
625058
625521
|
/** Unregister a dynamic block. Existing sentinels in scrollback become inert (rendered as empty). */
|
|
625059
625522
|
unregisterDynamicBlock(id2) {
|
|
625060
625523
|
this._dynamicBlocks.delete(id2);
|
|
625524
|
+
this._dynamicBlockClickHandlers.delete(id2);
|
|
625061
625525
|
this._dynamicBlockVersion++;
|
|
625062
625526
|
this.invalidateRowCountCache();
|
|
625063
625527
|
}
|
|
@@ -626760,9 +627224,23 @@ var init_status_bar = __esm({
|
|
|
626760
627224
|
* other body rows are left alone so text selection still works. Returns true
|
|
626761
627225
|
* when the click was consumed.
|
|
626762
627226
|
*/
|
|
626763
|
-
handleContentBlockClick(screenRow) {
|
|
627227
|
+
handleContentBlockClick(screenRow, col) {
|
|
626764
627228
|
const hit = this.hitTestContentBlock(screenRow);
|
|
626765
|
-
if (!hit
|
|
627229
|
+
if (!hit) return false;
|
|
627230
|
+
const handler = this._dynamicBlockClickHandlers.get(hit.id);
|
|
627231
|
+
if (handler?.({
|
|
627232
|
+
id: hit.id,
|
|
627233
|
+
offset: hit.offset,
|
|
627234
|
+
col,
|
|
627235
|
+
row: screenRow,
|
|
627236
|
+
lineText: hit.lineText
|
|
627237
|
+
})) {
|
|
627238
|
+
if (this.active && !isOverlayActive() && !this._suspendContentLayer) {
|
|
627239
|
+
this.repaintContent();
|
|
627240
|
+
}
|
|
627241
|
+
return true;
|
|
627242
|
+
}
|
|
627243
|
+
if (!isCollapsibleBlock(hit.id)) return false;
|
|
626766
627244
|
const plain = stripAnsi(hit.lineText);
|
|
626767
627245
|
if (plain.includes(READ_MORE_LABEL) || plain.includes(READ_LESS_LABEL)) {
|
|
626768
627246
|
toggleMore(hit.id);
|
|
@@ -626786,7 +627264,7 @@ var init_status_bar = __esm({
|
|
|
626786
627264
|
if (!this.active) return;
|
|
626787
627265
|
const w = termCols();
|
|
626788
627266
|
if (type === "press" && row >= this.scrollRegionTop) {
|
|
626789
|
-
if (this.handleContentBlockClick(row)) return;
|
|
627267
|
+
if (this.handleContentBlockClick(row, col)) return;
|
|
626790
627268
|
}
|
|
626791
627269
|
if (type === "press" && this._suggestions.length > 0) {
|
|
626792
627270
|
if (this.suggestClickAt(row)) return;
|
|
@@ -637997,7 +638475,7 @@ var init_audio_waveform = __esm({
|
|
|
637997
638475
|
|
|
637998
638476
|
// packages/cli/src/tui/neovim-mode.ts
|
|
637999
638477
|
import { existsSync as existsSync129, unlinkSync as unlinkSync24 } from "node:fs";
|
|
638000
|
-
import { tmpdir as
|
|
638478
|
+
import { tmpdir as tmpdir22 } from "node:os";
|
|
638001
638479
|
import { join as join143 } from "node:path";
|
|
638002
638480
|
function isNeovimActive() {
|
|
638003
638481
|
return _state2 !== null && !_state2.cleanedUp;
|
|
@@ -638043,7 +638521,7 @@ async function startNeovimMode(opts) {
|
|
|
638043
638521
|
);
|
|
638044
638522
|
} catch {
|
|
638045
638523
|
}
|
|
638046
|
-
const socketPath = join143(
|
|
638524
|
+
const socketPath = join143(tmpdir22(), `omnius-nvim-${process.pid}-${Date.now()}.sock`);
|
|
638047
638525
|
try {
|
|
638048
638526
|
if (existsSync129(socketPath)) unlinkSync24(socketPath);
|
|
638049
638527
|
} catch {
|
|
@@ -639208,6 +639686,10 @@ async function showPruneMenu(options2) {
|
|
|
639208
639686
|
});
|
|
639209
639687
|
if (!result.confirmed || !result.key || result.key === "back") return;
|
|
639210
639688
|
const start2 = (label, o2) => {
|
|
639689
|
+
if (isLiveSensorActiveForRepo(workingDir)) {
|
|
639690
|
+
renderWarning("Live audio/video mode is active; pruning is deferred so sensor, ASR, vision, and reasoning inference stay available. Stop live mode before running prune.");
|
|
639691
|
+
return;
|
|
639692
|
+
}
|
|
639211
639693
|
renderInfo(`${label} (running in background — summary will follow)…`);
|
|
639212
639694
|
runPruneWorkerOnce({
|
|
639213
639695
|
repoRoot: workingDir,
|
|
@@ -639243,6 +639725,7 @@ var init_prune_menu = __esm({
|
|
|
639243
639725
|
init_render();
|
|
639244
639726
|
init_kg_prune();
|
|
639245
639727
|
init_memory_maintenance();
|
|
639728
|
+
init_live_sensors();
|
|
639246
639729
|
import_chalk2 = __toESM(require_source(), 1);
|
|
639247
639730
|
}
|
|
639248
639731
|
});
|
|
@@ -641579,7 +642062,7 @@ import {
|
|
|
641579
642062
|
rmSync as rmSync11
|
|
641580
642063
|
} from "node:fs";
|
|
641581
642064
|
import { join as join150, dirname as dirname45, resolve as resolve61 } from "node:path";
|
|
641582
|
-
import { homedir as homedir50, tmpdir as
|
|
642065
|
+
import { homedir as homedir50, tmpdir as tmpdir23, platform as platform6 } from "node:os";
|
|
641583
642066
|
import {
|
|
641584
642067
|
spawn as nodeSpawn
|
|
641585
642068
|
} from "node:child_process";
|
|
@@ -643275,6 +643758,9 @@ except Exception as exc:
|
|
|
643275
643758
|
await this.sleep(100);
|
|
643276
643759
|
}
|
|
643277
643760
|
}
|
|
643761
|
+
isSpeaking() {
|
|
643762
|
+
return this.speaking || this.speakQueue.length > 0 || Boolean(this.currentPlayback);
|
|
643763
|
+
}
|
|
643278
643764
|
enqueueSpeech(text2, volume, pitchFactor, speedFactor = 1, stereoDelayMs = 0.6, emotion) {
|
|
643279
643765
|
if (!this.enabled || !this.ready) return;
|
|
643280
643766
|
text2 = sanitizeForTTS(text2);
|
|
@@ -643672,7 +644158,7 @@ except Exception as exc:
|
|
|
643672
644158
|
}
|
|
643673
644159
|
this.onPCMOutput(Buffer.from(int16.buffer), sampleRate);
|
|
643674
644160
|
}
|
|
643675
|
-
const wavPath = join150(
|
|
644161
|
+
const wavPath = join150(tmpdir23(), `omnius-voice-${Date.now()}.wav`);
|
|
643676
644162
|
this.writeStereoWav(stereo.left, stereo.right, sampleRate, wavPath);
|
|
643677
644163
|
await this.playWav(wavPath);
|
|
643678
644164
|
try {
|
|
@@ -644185,7 +644671,7 @@ except Exception as exc:
|
|
|
644185
644671
|
const cleaned = injectExpressionTags ? applySupertonicExpressionTags(baseText, settings.expression, emotion) : baseText;
|
|
644186
644672
|
if (!cleaned) return null;
|
|
644187
644673
|
const wavPath = join150(
|
|
644188
|
-
|
|
644674
|
+
tmpdir23(),
|
|
644189
644675
|
`omnius-supertonic3-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.wav`
|
|
644190
644676
|
);
|
|
644191
644677
|
try {
|
|
@@ -644256,7 +644742,7 @@ except Exception as exc:
|
|
|
644256
644742
|
return new Promise((resolve76, reject) => {
|
|
644257
644743
|
const proc = nodeSpawn("sh", ["-c", command], {
|
|
644258
644744
|
stdio: ["ignore", "pipe", "pipe"],
|
|
644259
|
-
cwd:
|
|
644745
|
+
cwd: tmpdir23(),
|
|
644260
644746
|
env: voicePythonEnv()
|
|
644261
644747
|
});
|
|
644262
644748
|
let stdout = "";
|
|
@@ -644343,7 +644829,7 @@ Error: ${err instanceof Error ? err.message : String(err)}`
|
|
|
644343
644829
|
const mlxModelId = model.mlxModelId ?? "mlx-community/Kokoro-82M-bf16";
|
|
644344
644830
|
const mlxVoice = model.mlxVoice ?? "af_heart";
|
|
644345
644831
|
const mlxLangCode = model.mlxLangCode ?? "a";
|
|
644346
|
-
const wavPath = join150(
|
|
644832
|
+
const wavPath = join150(tmpdir23(), `omnius-mlx-${Date.now()}.wav`);
|
|
644347
644833
|
const pyScript = [
|
|
644348
644834
|
"import sys, json",
|
|
644349
644835
|
"from mlx_audio.tts import generate as tts_gen",
|
|
@@ -644424,7 +644910,7 @@ Error: ${err instanceof Error ? err.message : String(err)}`
|
|
|
644424
644910
|
const mlxModelId = model.mlxModelId ?? "mlx-community/Kokoro-82M-bf16";
|
|
644425
644911
|
const mlxVoice = model.mlxVoice ?? "af_heart";
|
|
644426
644912
|
const mlxLangCode = model.mlxLangCode ?? "a";
|
|
644427
|
-
const wavPath = join150(
|
|
644913
|
+
const wavPath = join150(tmpdir23(), `omnius-mlx-buf-${Date.now()}.wav`);
|
|
644428
644914
|
const pyScript = [
|
|
644429
644915
|
"import sys, json",
|
|
644430
644916
|
"from mlx_audio.tts import generate as tts_gen",
|
|
@@ -645200,7 +645686,7 @@ if __name__ == '__main__':
|
|
|
645200
645686
|
const env2 = voicePythonEnv({ LUXTTS_REPO_PATH: luxttsRepoDir2() });
|
|
645201
645687
|
const daemon = nodeSpawn(venvPy, [luxttsInferScript2()], {
|
|
645202
645688
|
stdio: ["pipe", "pipe", "pipe"],
|
|
645203
|
-
cwd:
|
|
645689
|
+
cwd: tmpdir23(),
|
|
645204
645690
|
env: env2
|
|
645205
645691
|
});
|
|
645206
645692
|
this._luxttsDaemon = daemon;
|
|
@@ -645290,7 +645776,7 @@ if __name__ == '__main__':
|
|
|
645290
645776
|
const ready = await this.ensureLuxttsDaemon();
|
|
645291
645777
|
if (!ready) return null;
|
|
645292
645778
|
const wavPath = join150(
|
|
645293
|
-
|
|
645779
|
+
tmpdir23(),
|
|
645294
645780
|
`omnius-luxtts-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.wav`
|
|
645295
645781
|
);
|
|
645296
645782
|
try {
|
|
@@ -645431,7 +645917,7 @@ if __name__ == '__main__':
|
|
|
645431
645917
|
if (!cleaned) return null;
|
|
645432
645918
|
const ready = await this.ensureLuxttsDaemon();
|
|
645433
645919
|
if (!ready) return null;
|
|
645434
|
-
const wavPath = join150(
|
|
645920
|
+
const wavPath = join150(tmpdir23(), `omnius-luxtts-buf-${Date.now()}.wav`);
|
|
645435
645921
|
try {
|
|
645436
645922
|
await this.luxttsRequest({
|
|
645437
645923
|
action: "synthesize",
|
|
@@ -645565,7 +646051,7 @@ if __name__ == "__main__":
|
|
|
645565
646051
|
});
|
|
645566
646052
|
const daemon = nodeSpawn(venvPy, [misottsInferScript2()], {
|
|
645567
646053
|
stdio: ["pipe", "pipe", "pipe"],
|
|
645568
|
-
cwd:
|
|
646054
|
+
cwd: tmpdir23(),
|
|
645569
646055
|
env: env2
|
|
645570
646056
|
});
|
|
645571
646057
|
this._misottsDaemon = daemon;
|
|
@@ -645649,7 +646135,7 @@ if __name__ == "__main__":
|
|
|
645649
646135
|
const ready = await this.ensureMisottsDaemon();
|
|
645650
646136
|
if (!ready) return null;
|
|
645651
646137
|
const wavPath = join150(
|
|
645652
|
-
|
|
646138
|
+
tmpdir23(),
|
|
645653
646139
|
`omnius-misotts-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.wav`
|
|
645654
646140
|
);
|
|
645655
646141
|
try {
|
|
@@ -645799,7 +646285,7 @@ if __name__ == "__main__":
|
|
|
645799
646285
|
if (!cleaned) return null;
|
|
645800
646286
|
const ready = await this.ensureMisottsDaemon();
|
|
645801
646287
|
if (!ready) return null;
|
|
645802
|
-
const wavPath = join150(
|
|
646288
|
+
const wavPath = join150(tmpdir23(), `omnius-misotts-buf-${Date.now()}.wav`);
|
|
645803
646289
|
try {
|
|
645804
646290
|
const settings = this.getMisottsSettings();
|
|
645805
646291
|
const cloneName = this.misottsCloneRef.split("/").pop() ?? "";
|
|
@@ -648242,6 +648728,7 @@ async function handleSlashCommand(input, ctx3) {
|
|
|
648242
648728
|
case "prune": {
|
|
648243
648729
|
const sub2 = (arg?.trim().split(/\s+/)[0] ?? "").toLowerCase();
|
|
648244
648730
|
const workingDir = ctx3.repoRoot || process.cwd();
|
|
648731
|
+
const force = /\b--force\b/i.test(arg ?? "");
|
|
648245
648732
|
const targetMatch = arg?.match(/--target\s+(\d+)/);
|
|
648246
648733
|
const targetNodes = targetMatch ? parseInt(targetMatch[1], 10) : void 0;
|
|
648247
648734
|
const { knowledgeGraphStats: knowledgeGraphStats2, formatKgStatsLine: formatKgStatsLine2 } = await Promise.resolve().then(() => (init_kg_prune(), kg_prune_exports));
|
|
@@ -648250,6 +648737,10 @@ async function handleSlashCommand(input, ctx3) {
|
|
|
648250
648737
|
renderInfo(summary);
|
|
648251
648738
|
};
|
|
648252
648739
|
const startBackground = (label, o2) => {
|
|
648740
|
+
if (isLiveSensorActiveForRepo(workingDir) && !force) {
|
|
648741
|
+
renderWarning("Live audio/video mode is active; pruning is deferred so sensor, ASR, vision, and reasoning inference stay available. Stop live mode first, or rerun with --force if you intentionally want pruning to compete with live inference.");
|
|
648742
|
+
return;
|
|
648743
|
+
}
|
|
648253
648744
|
renderInfo(`${label} (running in background — summary will follow)…`);
|
|
648254
648745
|
runPruneWorkerOnce2({
|
|
648255
648746
|
repoRoot: workingDir,
|
|
@@ -656535,6 +657026,15 @@ function renderLiveDashboard(ctx3, manager) {
|
|
|
656535
657026
|
id2,
|
|
656536
657027
|
(width) => formatLiveDashboardFromSnapshot(manager.getSnapshot(), { width })
|
|
656537
657028
|
);
|
|
657029
|
+
host.registerDynamicBlockClickHandler?.(id2, ({ col, lineText }) => {
|
|
657030
|
+
const action = liveDashboardRotationActionFromClick(manager.getSnapshot(), lineText, col);
|
|
657031
|
+
if (!action) return false;
|
|
657032
|
+
manager.rotateCameraRelative(action.source, action.direction);
|
|
657033
|
+
void manager.sampleCameraNow(action.source, true, { forceInferenceNow: true }).catch((err) => {
|
|
657034
|
+
renderWarning(`Camera refresh failed after manual rotation: ${err instanceof Error ? err.message : String(err)}`);
|
|
657035
|
+
});
|
|
657036
|
+
return true;
|
|
657037
|
+
});
|
|
656538
657038
|
host.appendDynamicBlock(id2);
|
|
656539
657039
|
return;
|
|
656540
657040
|
}
|
|
@@ -656565,7 +657065,7 @@ ${feedback.message}`);
|
|
|
656565
657065
|
renderInfo(`[live ${stamp}] ${feedback.title}
|
|
656566
657066
|
${feedback.message}`);
|
|
656567
657067
|
}
|
|
656568
|
-
function attachLiveSinks(ctx3, manager, agentEnabled = false) {
|
|
657068
|
+
function attachLiveSinks(ctx3, manager, agentEnabled = false, speechEnabledOverride) {
|
|
656569
657069
|
manager.setStatusSink(ctx3.setLiveMediaStatus);
|
|
656570
657070
|
manager.setFeedbackSink((feedback) => renderLiveFeedback(ctx3, manager, feedback));
|
|
656571
657071
|
manager.setAudioIntakeAgentConfig({
|
|
@@ -656573,6 +657073,15 @@ function attachLiveSinks(ctx3, manager, agentEnabled = false) {
|
|
|
656573
657073
|
model: ctx3.config.model,
|
|
656574
657074
|
apiKey: ctx3.config.apiKey
|
|
656575
657075
|
});
|
|
657076
|
+
const speechEnabled = Boolean(speechEnabledOverride ?? agentEnabled) && Boolean(ctx3.voiceSpeak);
|
|
657077
|
+
manager.setLiveSpeechEnabled(speechEnabled);
|
|
657078
|
+
manager.setLiveTtsSpeakingSource(ctx3.voiceIsSpeaking);
|
|
657079
|
+
manager.setLiveSpeechSink(speechEnabled ? (text2) => {
|
|
657080
|
+
if (ctx3.voiceIsEnabled && !ctx3.voiceIsEnabled()) return;
|
|
657081
|
+
const estimatedSpeechMs = Math.max(1500, Math.min(15e3, text2.length * 85));
|
|
657082
|
+
manager.muteLiveAudioFor(estimatedSpeechMs + 1e3);
|
|
657083
|
+
ctx3.voiceSpeak?.(text2);
|
|
657084
|
+
} : void 0);
|
|
656576
657085
|
if (agentEnabled && ctx3.startBackgroundPrompt) {
|
|
656577
657086
|
manager.setAgentActionSink((prompt) => {
|
|
656578
657087
|
const id2 = ctx3.startBackgroundPrompt?.(prompt);
|
|
@@ -656583,16 +657092,17 @@ function attachLiveSinks(ctx3, manager, agentEnabled = false) {
|
|
|
656583
657092
|
}
|
|
656584
657093
|
}
|
|
656585
657094
|
async function startLiveVoicechat(ctx3, manager) {
|
|
656586
|
-
const
|
|
656587
|
-
attachLiveSinks(ctx3, manager,
|
|
657095
|
+
const speechEnabled = Boolean(ctx3.voiceSpeak);
|
|
657096
|
+
attachLiveSinks(ctx3, manager, false, speechEnabled);
|
|
656588
657097
|
await ensureLiveInventory(manager);
|
|
656589
|
-
renderInfo(manager.startRunMode({ agent: agentEnabled, lowLatency: true }));
|
|
656590
657098
|
if (!ctx3.voiceChatStart) {
|
|
657099
|
+
renderInfo(manager.startRunMode({ agent: false, lowLatency: true, speech: false }));
|
|
656591
657100
|
renderWarning("Voice chat is not available in this context; live audio/video feedback remains active.");
|
|
656592
657101
|
return;
|
|
656593
657102
|
}
|
|
656594
657103
|
ctx3.voiceSetMode?.("voicechat");
|
|
656595
657104
|
if (ctx3.isVoiceChatActive?.()) {
|
|
657105
|
+
renderInfo(manager.startRunMode({ agent: false, lowLatency: true, speech: speechEnabled }));
|
|
656596
657106
|
renderInfo("Voice chat is already active. Live audio/video feedback remains attached.");
|
|
656597
657107
|
return;
|
|
656598
657108
|
}
|
|
@@ -656603,6 +657113,7 @@ async function startLiveVoicechat(ctx3, manager) {
|
|
|
656603
657113
|
try {
|
|
656604
657114
|
renderInfo("Starting live voicechat with audio/video context...");
|
|
656605
657115
|
await ctx3.voiceChatStart();
|
|
657116
|
+
renderInfo(manager.startRunMode({ agent: false, lowLatency: true, speech: speechEnabled }));
|
|
656606
657117
|
renderInfo("Live voicechat active: low-latency audio/video context is updating while voice conversation continues.");
|
|
656607
657118
|
} catch (err) {
|
|
656608
657119
|
renderError(`Live voicechat failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -656663,7 +657174,7 @@ async function handleLiveCommand(ctx3, arg) {
|
|
|
656663
657174
|
renderWarning("Live PFC agent review is unavailable in this context; starting visual/audio feedback only.");
|
|
656664
657175
|
}
|
|
656665
657176
|
await ensureLiveInventory(manager);
|
|
656666
|
-
renderInfo(manager.startRunMode({ agent: effectiveAgent, lowLatency: true }));
|
|
657177
|
+
renderInfo(manager.startRunMode({ agent: effectiveAgent, lowLatency: true, speech: effectiveAgent && Boolean(ctx3.voiceSpeak) }));
|
|
656667
657178
|
return;
|
|
656668
657179
|
}
|
|
656669
657180
|
if (sub2 === "chat" || sub2 === "voicechat" || sub2 === "voice") {
|
|
@@ -656918,14 +657429,14 @@ async function showLiveMenu(ctx3, manager) {
|
|
|
656918
657429
|
continue;
|
|
656919
657430
|
case "run-live":
|
|
656920
657431
|
attachLiveSinks(ctx3, manager, false);
|
|
656921
|
-
renderInfo(manager.startRunMode({ agent: false, lowLatency: true }));
|
|
657432
|
+
renderInfo(manager.startRunMode({ agent: false, lowLatency: true, speech: false }));
|
|
656922
657433
|
continue;
|
|
656923
657434
|
case "run-agent":
|
|
656924
657435
|
attachLiveSinks(ctx3, manager, true);
|
|
656925
657436
|
if (!ctx3.startBackgroundPrompt) {
|
|
656926
657437
|
renderWarning("Live PFC agent review is unavailable in this context; starting visual/audio feedback only.");
|
|
656927
657438
|
}
|
|
656928
|
-
renderInfo(manager.startRunMode({ agent: Boolean(ctx3.startBackgroundPrompt), lowLatency: true }));
|
|
657439
|
+
renderInfo(manager.startRunMode({ agent: Boolean(ctx3.startBackgroundPrompt), lowLatency: true, speech: Boolean(ctx3.startBackgroundPrompt && ctx3.voiceSpeak) }));
|
|
656929
657440
|
continue;
|
|
656930
657441
|
case "run-chat":
|
|
656931
657442
|
await startLiveVoicechat(ctx3, manager);
|
|
@@ -694965,15 +695476,15 @@ Content-Type: ${mimeForPath(pathOrFileId, field === "photo" ? "image" : "video")
|
|
|
694965
695476
|
let filename = "response.wav";
|
|
694966
695477
|
let mimeType = "audio/wav";
|
|
694967
695478
|
try {
|
|
694968
|
-
const { tmpdir:
|
|
695479
|
+
const { tmpdir: tmpdir26 } = await import("node:os");
|
|
694969
695480
|
const { join: joinPath } = await import("node:path");
|
|
694970
695481
|
const {
|
|
694971
695482
|
writeFileSync: writeFs,
|
|
694972
695483
|
readFileSync: readFs,
|
|
694973
695484
|
unlinkSync: unlinkFs
|
|
694974
695485
|
} = await import("node:fs");
|
|
694975
|
-
const tmpWav = joinPath(
|
|
694976
|
-
const tmpOgg = joinPath(
|
|
695486
|
+
const tmpWav = joinPath(tmpdir26(), `omnius-tg-voice-${Date.now()}.wav`);
|
|
695487
|
+
const tmpOgg = joinPath(tmpdir26(), `omnius-tg-voice-${Date.now()}.ogg`);
|
|
694977
695488
|
writeFs(tmpWav, wavBuffer);
|
|
694978
695489
|
await execFileText4(
|
|
694979
695490
|
"ffmpeg",
|
|
@@ -699993,7 +700504,7 @@ __export(graphical_sudo_exports, {
|
|
|
699993
700504
|
import { spawn as spawn35 } from "node:child_process";
|
|
699994
700505
|
import { existsSync as existsSync161, mkdirSync as mkdirSync102, writeFileSync as writeFileSync87, chmodSync as chmodSync6 } from "node:fs";
|
|
699995
700506
|
import { join as join174 } from "node:path";
|
|
699996
|
-
import { tmpdir as
|
|
700507
|
+
import { tmpdir as tmpdir24 } from "node:os";
|
|
699997
700508
|
function detectSudoHelper() {
|
|
699998
700509
|
if (process.platform === "win32") return null;
|
|
699999
700510
|
if (process.platform === "darwin") return "osascript";
|
|
@@ -700014,7 +700525,7 @@ function which2(cmd) {
|
|
|
700014
700525
|
return null;
|
|
700015
700526
|
}
|
|
700016
700527
|
function ensureAskpassShim(helper, description) {
|
|
700017
|
-
const shimDir = join174(
|
|
700528
|
+
const shimDir = join174(tmpdir24(), "omnius-askpass");
|
|
700018
700529
|
mkdirSync102(shimDir, { recursive: true });
|
|
700019
700530
|
const shim = join174(shimDir, `${helper}.sh`);
|
|
700020
700531
|
let body;
|
|
@@ -722357,11 +722868,11 @@ async function handleRequest(req3, res, ollamaUrl, verbose, runtimeDefaults = {}
|
|
|
722357
722868
|
});
|
|
722358
722869
|
return;
|
|
722359
722870
|
}
|
|
722360
|
-
const { tmpdir:
|
|
722871
|
+
const { tmpdir: tmpdir26 } = await import("node:os");
|
|
722361
722872
|
const { writeFileSync: writeFileSync95, unlinkSync: unlinkSync37 } = await import("node:fs");
|
|
722362
722873
|
const { join: pjoin } = await import("node:path");
|
|
722363
722874
|
const tmpPath = pjoin(
|
|
722364
|
-
|
|
722875
|
+
tmpdir26(),
|
|
722365
722876
|
`omnius-clone-upload-${Date.now()}-${safeName3}`
|
|
722366
722877
|
);
|
|
722367
722878
|
writeFileSync95(tmpPath, buf);
|
|
@@ -732153,9 +732664,10 @@ Rationale: ${proposal.rationale}${provenanceNote}${dmnDevDiscipline(proposal.cat
|
|
|
732153
732664
|
}, delayMs);
|
|
732154
732665
|
}
|
|
732155
732666
|
let activeTask = null;
|
|
732667
|
+
const liveSensorsBusy = () => isLiveSensorActiveForRepo(repoRoot);
|
|
732156
732668
|
idleMemoryMaintenance = startIdleMemoryMaintenance({
|
|
732157
732669
|
repoRoot,
|
|
732158
|
-
isBusy: () => Boolean(activeTask) || _interactiveSessionActive,
|
|
732670
|
+
isBusy: () => Boolean(activeTask) || _interactiveSessionActive || liveSensorsBusy(),
|
|
732159
732671
|
onSummary: (summary) => {
|
|
732160
732672
|
if (!summary) return;
|
|
732161
732673
|
writeContent(() => renderInfo(summary));
|
|
@@ -733835,6 +734347,9 @@ The user pasted a clipboard image saved at ${relPath}. Use the OCR, vision analy
|
|
|
733835
734347
|
voiceIsEnabled() {
|
|
733836
734348
|
return voiceEngine.enabled;
|
|
733837
734349
|
},
|
|
734350
|
+
voiceIsSpeaking() {
|
|
734351
|
+
return voiceEngine.isSpeaking();
|
|
734352
|
+
},
|
|
733838
734353
|
voiceGetModel() {
|
|
733839
734354
|
return voiceEngine.modelId ?? "unknown";
|
|
733840
734355
|
},
|
|
@@ -734709,10 +735224,12 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
734709
735224
|
},
|
|
734710
735225
|
setLiveMediaStatus(status) {
|
|
734711
735226
|
statusBar.setLiveMediaStatus(status);
|
|
735227
|
+
if (status.audio || status.video) idleMemoryMaintenance?.notifyActivity();
|
|
734712
735228
|
},
|
|
734713
735229
|
liveDynamicBlockHost: {
|
|
734714
735230
|
registerDynamicBlock: (id2, render2) => statusBar.registerDynamicBlock(id2, render2),
|
|
734715
735231
|
appendDynamicBlock: (id2) => statusBar.appendDynamicBlock(id2),
|
|
735232
|
+
registerDynamicBlockClickHandler: (id2, handler) => statusBar.registerDynamicBlockClickHandler(id2, handler),
|
|
734716
735233
|
refreshDynamicBlocks: () => statusBar.refreshDynamicBlocks()
|
|
734717
735234
|
},
|
|
734718
735235
|
// Voice call session — standalone cloudflared tunnel for /call
|
|
@@ -738395,7 +738912,7 @@ __export(eval_exports, {
|
|
|
738395
738912
|
evalCommand: () => evalCommand,
|
|
738396
738913
|
expectedStatusesForEvalTask: () => expectedStatusesForEvalTask
|
|
738397
738914
|
});
|
|
738398
|
-
import { tmpdir as
|
|
738915
|
+
import { tmpdir as tmpdir25 } from "node:os";
|
|
738399
738916
|
import { mkdirSync as mkdirSync110, writeFileSync as writeFileSync94 } from "node:fs";
|
|
738400
738917
|
import { join as join184 } from "node:path";
|
|
738401
738918
|
function expectedStatusesForEvalTask(task, live) {
|
|
@@ -738539,7 +739056,7 @@ async function evalCommand(opts, config) {
|
|
|
738539
739056
|
process.exit(failed > 0 ? 1 : 0);
|
|
738540
739057
|
}
|
|
738541
739058
|
function createTempEvalRepo() {
|
|
738542
|
-
const dir = join184(
|
|
739059
|
+
const dir = join184(tmpdir25(), `omnius-eval-${Date.now()}`);
|
|
738543
739060
|
mkdirSync110(dir, { recursive: true });
|
|
738544
739061
|
mkdirSync110(join184(dir, "src"), { recursive: true });
|
|
738545
739062
|
mkdirSync110(join184(dir, "tests"), { recursive: true });
|