omnius 1.0.406 → 1.0.408
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1132 -96
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -542978,9 +542978,45 @@ var init_process_health = __esm({
|
|
|
542978
542978
|
});
|
|
542979
542979
|
|
|
542980
542980
|
// packages/execution/dist/tools/camera-capture.js
|
|
542981
|
-
import { accessSync, constants, readFileSync as readFileSync44, writeFileSync as writeFileSync28, unlinkSync as unlinkSync11, existsSync as existsSync60, mkdirSync as mkdirSync34, readdirSync as readdirSync21 } from "node:fs";
|
|
542981
|
+
import { accessSync, constants, readFileSync as readFileSync44, writeFileSync as writeFileSync28, unlinkSync as unlinkSync11, existsSync as existsSync60, mkdirSync as mkdirSync34, readdirSync as readdirSync21, renameSync as renameSync7 } from "node:fs";
|
|
542982
542982
|
import { join as join75 } from "node:path";
|
|
542983
542983
|
import { tmpdir as tmpdir10 } from "node:os";
|
|
542984
|
+
function normalizeCameraRotationDegrees(value2) {
|
|
542985
|
+
if (value2 === void 0 || value2 === null || value2 === "")
|
|
542986
|
+
return 0;
|
|
542987
|
+
if (typeof value2 === "number" && Number.isFinite(value2)) {
|
|
542988
|
+
const n2 = (Math.round(value2) % 360 + 360) % 360;
|
|
542989
|
+
if (n2 === 0 || n2 === 90 || n2 === 180 || n2 === 270)
|
|
542990
|
+
return n2;
|
|
542991
|
+
}
|
|
542992
|
+
const raw = String(value2).trim().toLowerCase();
|
|
542993
|
+
if (!raw)
|
|
542994
|
+
return 0;
|
|
542995
|
+
const numeric = Number(raw.replace(/deg(?:rees)?$/i, ""));
|
|
542996
|
+
if (Number.isFinite(numeric))
|
|
542997
|
+
return normalizeCameraRotationDegrees(numeric);
|
|
542998
|
+
if (["none", "off", "upright", "normal", "0"].includes(raw))
|
|
542999
|
+
return 0;
|
|
543000
|
+
if (["cw", "clockwise", "right", "rotate-cw", "rotate_cw", "90"].includes(raw))
|
|
543001
|
+
return 90;
|
|
543002
|
+
if (["180", "upside-down", "upside_down", "flip", "inverted"].includes(raw))
|
|
543003
|
+
return 180;
|
|
543004
|
+
if (["ccw", "counterclockwise", "counter-clockwise", "anticlockwise", "left", "rotate-ccw", "rotate_ccw", "270"].includes(raw))
|
|
543005
|
+
return 270;
|
|
543006
|
+
throw new Error(`Unsupported camera rotation: ${String(value2)}. Use 0, 90, 180, 270, cw, ccw, or none.`);
|
|
543007
|
+
}
|
|
543008
|
+
function ffmpegRotateFilterForDegrees(rotation) {
|
|
543009
|
+
switch (rotation) {
|
|
543010
|
+
case 0:
|
|
543011
|
+
return null;
|
|
543012
|
+
case 90:
|
|
543013
|
+
return "transpose=1";
|
|
543014
|
+
case 180:
|
|
543015
|
+
return "hflip,vflip";
|
|
543016
|
+
case 270:
|
|
543017
|
+
return "transpose=2";
|
|
543018
|
+
}
|
|
543019
|
+
}
|
|
542984
543020
|
function sleep3(ms) {
|
|
542985
543021
|
return new Promise((resolve76) => setTimeout(resolve76, ms));
|
|
542986
543022
|
}
|
|
@@ -543216,6 +543252,21 @@ var init_camera_capture = __esm({
|
|
|
543216
543252
|
type: "number",
|
|
543217
543253
|
description: "Capture height in pixels (default: 720 for USB, native for QooCam)"
|
|
543218
543254
|
},
|
|
543255
|
+
rotate_degrees: {
|
|
543256
|
+
type: "number",
|
|
543257
|
+
enum: [0, 90, 180, 270],
|
|
543258
|
+
description: "Optional clockwise correction applied to the captured frame before saving/returning. Use 90 for clockwise, 270 for counter-clockwise."
|
|
543259
|
+
},
|
|
543260
|
+
rotation_degrees: {
|
|
543261
|
+
type: "number",
|
|
543262
|
+
enum: [0, 90, 180, 270],
|
|
543263
|
+
description: "Alias for rotate_degrees."
|
|
543264
|
+
},
|
|
543265
|
+
rotate: {
|
|
543266
|
+
type: "string",
|
|
543267
|
+
enum: ["none", "cw", "ccw", "180", "clockwise", "counterclockwise"],
|
|
543268
|
+
description: "Optional human-readable rotation alias. Equivalent to rotate_degrees."
|
|
543269
|
+
},
|
|
543219
543270
|
output_path: {
|
|
543220
543271
|
type: "string",
|
|
543221
543272
|
description: "Save captured image to this path instead of returning base64"
|
|
@@ -543426,6 +543477,7 @@ ${formatRouterPlan(plan)}`,
|
|
|
543426
543477
|
const width = args["width"] || 1280;
|
|
543427
543478
|
const height = args["height"] || 720;
|
|
543428
543479
|
const outputPath3 = args["output_path"];
|
|
543480
|
+
const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
|
|
543429
543481
|
const captureDir = join75(tmpdir10(), "omnius-camera");
|
|
543430
543482
|
if (!existsSync60(captureDir))
|
|
543431
543483
|
mkdirSync34(captureDir, { recursive: true });
|
|
@@ -543484,10 +543536,25 @@ ${formatRouterPlan(plan)}`,
|
|
|
543484
543536
|
return false;
|
|
543485
543537
|
};
|
|
543486
543538
|
if (await runAttempts()) {
|
|
543539
|
+
if (rotation !== 0) {
|
|
543540
|
+
try {
|
|
543541
|
+
await this.rotateCapturedImage(tempFile, rotation);
|
|
543542
|
+
} catch (err) {
|
|
543543
|
+
return {
|
|
543544
|
+
success: false,
|
|
543545
|
+
output: "",
|
|
543546
|
+
error: `Captured from ${device}, but failed to apply ${rotation} degree rotation: ${err instanceof Error ? err.message : String(err)}`,
|
|
543547
|
+
durationMs: performance.now() - start2
|
|
543548
|
+
};
|
|
543549
|
+
}
|
|
543550
|
+
}
|
|
543487
543551
|
const firstSuccess = errors.length > 0 ? `
|
|
543488
543552
|
Recovered with capture profile: ${captureState.successfulAttempt?.label ?? "unknown"}. Last failed attempt: ${errors[errors.length - 1].slice(0, 260)}` : "";
|
|
543553
|
+
const rotationNote = rotation !== 0 ? `
|
|
543554
|
+
Applied camera rotation correction: ${rotation} degrees clockwise.` : "";
|
|
543489
543555
|
const result = this.returnCapturedFile(tempFile, captureState.successfulAttempt?.videoSize ?? `${width}x${height}`, device, outputPath3, start2);
|
|
543490
|
-
|
|
543556
|
+
const notes = `${firstSuccess}${rotationNote}`;
|
|
543557
|
+
return result.success && notes ? { ...result, output: result.output + notes, llmContent: result.llmContent ? result.llmContent + notes : result.llmContent } : result;
|
|
543491
543558
|
}
|
|
543492
543559
|
const allErrors = errors.join("\n\n");
|
|
543493
543560
|
if (permissionBlocked || cameraFfmpegErrorIsPermissionDenied(allErrors)) {
|
|
@@ -543576,6 +543643,37 @@ ${allErrors.slice(0, 900)}`, durationMs: performance.now() - start2 };
|
|
|
543576
543643
|
argv.push("-y", outputPath3);
|
|
543577
543644
|
return argv;
|
|
543578
543645
|
}
|
|
543646
|
+
async rotateCapturedImage(filePath, rotation) {
|
|
543647
|
+
const vf = ffmpegRotateFilterForDegrees(rotation);
|
|
543648
|
+
if (!vf)
|
|
543649
|
+
return;
|
|
543650
|
+
const rotatedPath = `${filePath}.rotated-${Date.now()}.jpg`;
|
|
543651
|
+
try {
|
|
543652
|
+
await runProcessBuffer(ffmpegBin2(), [
|
|
543653
|
+
"-hide_banner",
|
|
543654
|
+
"-loglevel",
|
|
543655
|
+
"error",
|
|
543656
|
+
"-y",
|
|
543657
|
+
"-i",
|
|
543658
|
+
filePath,
|
|
543659
|
+
"-vf",
|
|
543660
|
+
vf,
|
|
543661
|
+
"-q:v",
|
|
543662
|
+
"2",
|
|
543663
|
+
rotatedPath
|
|
543664
|
+
], { timeout: 15e3 });
|
|
543665
|
+
if (!existsSync60(rotatedPath) || readFileSync44(rotatedPath).length === 0) {
|
|
543666
|
+
throw new Error("ffmpeg produced no rotated image bytes");
|
|
543667
|
+
}
|
|
543668
|
+
renameSync7(rotatedPath, filePath);
|
|
543669
|
+
} finally {
|
|
543670
|
+
try {
|
|
543671
|
+
if (existsSync60(rotatedPath))
|
|
543672
|
+
unlinkSync11(rotatedPath);
|
|
543673
|
+
} catch {
|
|
543674
|
+
}
|
|
543675
|
+
}
|
|
543676
|
+
}
|
|
543579
543677
|
async v4l2Info(device, start2) {
|
|
543580
543678
|
let info = `Camera: ${device} [v4l2]
|
|
543581
543679
|
`;
|
|
@@ -543726,6 +543824,7 @@ Auto-installed camera dependencies: ${[
|
|
|
543726
543824
|
/** Capture a frame from QooCam via OSC API */
|
|
543727
543825
|
async captureQooCam(args, start2) {
|
|
543728
543826
|
const outputPath3 = args["output_path"];
|
|
543827
|
+
const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
|
|
543729
543828
|
const conn = await this.connectQooCamWiFi();
|
|
543730
543829
|
if (!conn.connected) {
|
|
543731
543830
|
return { success: false, output: "", error: conn.error || "Could not connect to QooCam", durationMs: performance.now() - start2 };
|
|
@@ -543745,13 +543844,13 @@ Auto-installed camera dependencies: ${[
|
|
|
543745
543844
|
let width = 0;
|
|
543746
543845
|
let height = 0;
|
|
543747
543846
|
try {
|
|
543748
|
-
const
|
|
543847
|
+
const result2 = await this.oscFetch("POST", "/osc/commands/execute", {
|
|
543749
543848
|
name: "camera.takePicture"
|
|
543750
543849
|
});
|
|
543751
|
-
if (
|
|
543752
|
-
return { success: false, output: "", error: `QooCam capture failed: ${
|
|
543850
|
+
if (result2?.state === "error") {
|
|
543851
|
+
return { success: false, output: "", error: `QooCam capture failed: ${result2.error?.message || JSON.stringify(result2.error)}`, durationMs: performance.now() - start2 };
|
|
543753
543852
|
}
|
|
543754
|
-
const entry =
|
|
543853
|
+
const entry = result2?.results?.entries?.[0];
|
|
543755
543854
|
if (!entry?.fileUrl) {
|
|
543756
543855
|
return { success: false, output: "", error: "QooCam took a picture but returned no file URL", durationMs: performance.now() - start2 };
|
|
543757
543856
|
}
|
|
@@ -543775,8 +543874,27 @@ Auto-installed camera dependencies: ${[
|
|
|
543775
543874
|
} catch (err) {
|
|
543776
543875
|
return { success: false, output: "", error: `Failed to download frame from QooCam: ${err instanceof Error ? err.message : String(err)}`, durationMs: performance.now() - start2 };
|
|
543777
543876
|
}
|
|
543877
|
+
if (rotation !== 0) {
|
|
543878
|
+
try {
|
|
543879
|
+
await this.rotateCapturedImage(tempFile, rotation);
|
|
543880
|
+
} catch (err) {
|
|
543881
|
+
return {
|
|
543882
|
+
success: false,
|
|
543883
|
+
output: "",
|
|
543884
|
+
error: `Downloaded QooCam frame, but failed to apply ${rotation} degree rotation: ${err instanceof Error ? err.message : String(err)}`,
|
|
543885
|
+
durationMs: performance.now() - start2
|
|
543886
|
+
};
|
|
543887
|
+
}
|
|
543888
|
+
}
|
|
543778
543889
|
const res = width && height ? `${width}x${height}` : "native";
|
|
543779
|
-
|
|
543890
|
+
const result = this.returnCapturedFile(tempFile, res, "QooCam 8K (OSC/WiFi)", outputPath3, start2);
|
|
543891
|
+
return result.success && rotation !== 0 ? {
|
|
543892
|
+
...result,
|
|
543893
|
+
output: `${result.output}
|
|
543894
|
+
Applied camera rotation correction: ${rotation} degrees clockwise.`,
|
|
543895
|
+
llmContent: result.llmContent ? `${result.llmContent}
|
|
543896
|
+
Applied camera rotation correction: ${rotation} degrees clockwise.` : result.llmContent
|
|
543897
|
+
} : result;
|
|
543780
543898
|
}
|
|
543781
543899
|
/** Get QooCam info via OSC */
|
|
543782
543900
|
async qoocamInfo(start2) {
|
|
@@ -553723,7 +553841,7 @@ var init_plugin_system = __esm({
|
|
|
553723
553841
|
});
|
|
553724
553842
|
|
|
553725
553843
|
// packages/execution/dist/tools/notebook-edit.js
|
|
553726
|
-
import { readFileSync as readFileSync54, writeFileSync as writeFileSync37, existsSync as existsSync74, renameSync as
|
|
553844
|
+
import { readFileSync as readFileSync54, writeFileSync as writeFileSync37, existsSync as existsSync74, renameSync as renameSync8, unlinkSync as unlinkSync15 } from "node:fs";
|
|
553727
553845
|
import { resolve as resolve43 } from "node:path";
|
|
553728
553846
|
function readNotebook(path12) {
|
|
553729
553847
|
if (!existsSync74(path12))
|
|
@@ -553744,7 +553862,7 @@ function writeNotebook(path12, nb) {
|
|
|
553744
553862
|
const tempPath = `${path12}.tmp-${process.pid}-${Date.now()}`;
|
|
553745
553863
|
writeFileSync37(tempPath, JSON.stringify(nb, null, 1) + "\n", "utf8");
|
|
553746
553864
|
try {
|
|
553747
|
-
|
|
553865
|
+
renameSync8(tempPath, path12);
|
|
553748
553866
|
} catch (err) {
|
|
553749
553867
|
try {
|
|
553750
553868
|
unlinkSync15(tempPath);
|
|
@@ -555483,6 +555601,15 @@ function makePythonScript() {
|
|
|
555483
555601
|
return String.raw`
|
|
555484
555602
|
import json, os, sys, time
|
|
555485
555603
|
|
|
555604
|
+
def apply_rotation(frame, rotate_degrees, cv2):
|
|
555605
|
+
if rotate_degrees == 90:
|
|
555606
|
+
return cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
|
|
555607
|
+
if rotate_degrees == 180:
|
|
555608
|
+
return cv2.rotate(frame, cv2.ROTATE_180)
|
|
555609
|
+
if rotate_degrees == 270:
|
|
555610
|
+
return cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE)
|
|
555611
|
+
return frame
|
|
555612
|
+
|
|
555486
555613
|
def main():
|
|
555487
555614
|
cfg_path = sys.argv[1]
|
|
555488
555615
|
with open(cfg_path, "r", encoding="utf-8") as f:
|
|
@@ -555494,6 +555621,7 @@ def main():
|
|
|
555494
555621
|
source_kind = cfg["source_kind"]
|
|
555495
555622
|
source = cfg["source"]
|
|
555496
555623
|
media_path = None
|
|
555624
|
+
rotate_degrees = int(cfg.get("rotate_degrees") or 0)
|
|
555497
555625
|
|
|
555498
555626
|
try:
|
|
555499
555627
|
import cv2
|
|
@@ -555567,6 +555695,7 @@ def main():
|
|
|
555567
555695
|
ret, frame = cap.read()
|
|
555568
555696
|
if not ret:
|
|
555569
555697
|
break
|
|
555698
|
+
frame = apply_rotation(frame, rotate_degrees, cv2)
|
|
555570
555699
|
ts = cap.get(cv2.CAP_PROP_POS_MSEC) / 1000.0
|
|
555571
555700
|
if not ts or ts < 0:
|
|
555572
555701
|
ts = time.time() - start_wall
|
|
@@ -555649,6 +555778,7 @@ def main():
|
|
|
555649
555778
|
"output_dir": out_dir,
|
|
555650
555779
|
"yolo_model": yolo_model,
|
|
555651
555780
|
"yolo_live": True,
|
|
555781
|
+
"rotate_degrees": rotate_degrees,
|
|
555652
555782
|
"frames_sampled": frame_index,
|
|
555653
555783
|
"duration_observed_sec": round(float(duration if source_kind != "camera" else time.time() - start_wall), 3),
|
|
555654
555784
|
"objects": objects,
|
|
@@ -555662,6 +555792,7 @@ if __name__ == "__main__":
|
|
|
555662
555792
|
}
|
|
555663
555793
|
function mockResult(args, workingDir) {
|
|
555664
555794
|
const kind = sourceKind(args);
|
|
555795
|
+
const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
|
|
555665
555796
|
const outDir = join92(workingDir, ".omnius", "live-media", `mock-${Date.now().toString(36)}`);
|
|
555666
555797
|
mkdirSync47(join92(outDir, "crops"), { recursive: true });
|
|
555667
555798
|
const facePath = join92(outDir, "crops", "face_f0000_00.jpg");
|
|
@@ -555673,6 +555804,7 @@ function mockResult(args, workingDir) {
|
|
|
555673
555804
|
output_dir: outDir,
|
|
555674
555805
|
yolo_model: String(args["yolo_model"] ?? DEFAULT_YOLO26_MODEL),
|
|
555675
555806
|
yolo_live: false,
|
|
555807
|
+
rotate_degrees: rotation,
|
|
555676
555808
|
frames_sampled: 3,
|
|
555677
555809
|
duration_observed_sec: Number(args["duration_sec"] ?? 3),
|
|
555678
555810
|
objects: [
|
|
@@ -555694,6 +555826,7 @@ var init_live_media_loop = __esm({
|
|
|
555694
555826
|
init_process_async();
|
|
555695
555827
|
init_venv_paths();
|
|
555696
555828
|
init_audio_analyze();
|
|
555829
|
+
init_camera_capture();
|
|
555697
555830
|
init_transcribe_tool();
|
|
555698
555831
|
init_visual_memory();
|
|
555699
555832
|
init_visual_trigger();
|
|
@@ -555713,6 +555846,9 @@ var init_live_media_loop = __esm({
|
|
|
555713
555846
|
path: { type: "string", description: "Local media path" },
|
|
555714
555847
|
file: { type: "string", description: "Alias for path" },
|
|
555715
555848
|
camera: { type: ["string", "number"], description: "Camera device index/path" },
|
|
555849
|
+
rotate_degrees: { type: "number", enum: [0, 90, 180, 270], description: "Clockwise correction applied to sampled frames before object tracking and face crops." },
|
|
555850
|
+
rotation_degrees: { type: "number", enum: [0, 90, 180, 270], description: "Alias for rotate_degrees." },
|
|
555851
|
+
rotate: { type: "string", enum: ["none", "cw", "ccw", "180", "clockwise", "counterclockwise"], description: "Optional human-readable rotation alias." },
|
|
555716
555852
|
duration_sec: { type: "number", description: "Bounded watch duration, default 8" },
|
|
555717
555853
|
sample_interval_sec: { type: "number", description: "Seconds between sampled frames, default 1" },
|
|
555718
555854
|
max_frames: { type: "number", description: "Max sampled frames, default 8" },
|
|
@@ -555736,6 +555872,7 @@ var init_live_media_loop = __esm({
|
|
|
555736
555872
|
const start2 = performance.now();
|
|
555737
555873
|
const action = String(args["action"] ?? "watch").toLowerCase();
|
|
555738
555874
|
const yoloModel = String(args["yolo_model"] ?? DEFAULT_YOLO26_MODEL);
|
|
555875
|
+
const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
|
|
555739
555876
|
const plan = buildYolo26BootstrapPlan(yoloModel);
|
|
555740
555877
|
if (action === "status" || action === "ensure") {
|
|
555741
555878
|
const runtime = action === "ensure" ? await ensureRuntime(args["auto_bootstrap"] !== false) : await ensureRuntime(false);
|
|
@@ -555780,6 +555917,7 @@ var init_live_media_loop = __esm({
|
|
|
555780
555917
|
duration_sec: Number(args["duration_sec"] ?? 8),
|
|
555781
555918
|
sample_interval_sec: Number(args["sample_interval_sec"] ?? 1),
|
|
555782
555919
|
max_frames: Number(args["max_frames"] ?? 8),
|
|
555920
|
+
rotate_degrees: rotation,
|
|
555783
555921
|
detect_objects: args["detect_objects"] !== false,
|
|
555784
555922
|
track_objects: args["track_objects"] !== false,
|
|
555785
555923
|
crop_faces: args["crop_faces"] !== false,
|
|
@@ -555872,6 +556010,8 @@ var init_live_media_loop = __esm({
|
|
|
555872
556010
|
if (result.media_path)
|
|
555873
556011
|
lines.push(`media: ${result.media_path}`);
|
|
555874
556012
|
lines.push(`YOLO model: ${result.yolo_model} (${result.yolo_live ? "live" : "mock/harness"})`);
|
|
556013
|
+
if (result.rotate_degrees)
|
|
556014
|
+
lines.push(`frame rotation correction: ${result.rotate_degrees} degrees clockwise`);
|
|
555875
556015
|
lines.push(`sampled: ${result.frames_sampled} frame(s) over ${result.duration_observed_sec.toFixed(1)}s`);
|
|
555876
556016
|
lines.push(`objects: ${result.objects.length} detection(s), tracks: ${result.tracks.length}, face crops: ${result.faces.length}`);
|
|
555877
556017
|
const objectSummary = summarizeObjects(result.objects);
|
|
@@ -555918,6 +556058,7 @@ var init_live_media_loop = __esm({
|
|
|
555918
556058
|
media_path: result.media_path,
|
|
555919
556059
|
yolo_model: result.yolo_model,
|
|
555920
556060
|
yolo_live: result.yolo_live,
|
|
556061
|
+
rotate_degrees: result.rotate_degrees ?? 0,
|
|
555921
556062
|
frames_sampled: result.frames_sampled,
|
|
555922
556063
|
objects: result.objects.slice(0, 80),
|
|
555923
556064
|
tracks: result.tracks.slice(0, 40),
|
|
@@ -569468,7 +569609,7 @@ var init_memory_consolidation = __esm({
|
|
|
569468
569609
|
});
|
|
569469
569610
|
|
|
569470
569611
|
// packages/orchestrator/dist/consolidation-runtime.js
|
|
569471
|
-
import { existsSync as existsSync97, mkdirSync as mkdirSync56, readFileSync as readFileSync74, writeFileSync as writeFileSync46, renameSync as
|
|
569612
|
+
import { existsSync as existsSync97, mkdirSync as mkdirSync56, readFileSync as readFileSync74, writeFileSync as writeFileSync46, renameSync as renameSync9 } from "node:fs";
|
|
569472
569613
|
import { homedir as homedir33 } from "node:os";
|
|
569473
569614
|
import { join as join108, dirname as dirname35 } from "node:path";
|
|
569474
569615
|
function storePath2() {
|
|
@@ -569490,7 +569631,7 @@ function save2(s2) {
|
|
|
569490
569631
|
mkdirSync56(dirname35(p2), { recursive: true });
|
|
569491
569632
|
const tmp = `${p2}.tmp`;
|
|
569492
569633
|
writeFileSync46(tmp, JSON.stringify(s2), "utf8");
|
|
569493
|
-
|
|
569634
|
+
renameSync9(tmp, p2);
|
|
569494
569635
|
} catch {
|
|
569495
569636
|
}
|
|
569496
569637
|
}
|
|
@@ -597786,7 +597927,7 @@ var init_agent_task = __esm({
|
|
|
597786
597927
|
});
|
|
597787
597928
|
|
|
597788
597929
|
// packages/orchestrator/dist/task-recovery.js
|
|
597789
|
-
import { existsSync as existsSync105, readFileSync as readFileSync81, writeFileSync as writeFileSync51, mkdirSync as mkdirSync61, readdirSync as readdirSync33, renameSync as
|
|
597930
|
+
import { existsSync as existsSync105, readFileSync as readFileSync81, writeFileSync as writeFileSync51, mkdirSync as mkdirSync61, readdirSync as readdirSync33, renameSync as renameSync10, unlinkSync as unlinkSync19 } from "node:fs";
|
|
597790
597931
|
import { join as join117 } from "node:path";
|
|
597791
597932
|
import { homedir as homedir36 } from "node:os";
|
|
597792
597933
|
function sidecarDir() {
|
|
@@ -597803,7 +597944,7 @@ function persistAgentTaskSidecar(task) {
|
|
|
597803
597944
|
const tmpPath = `${finalPath}.tmp.${process.pid}.${Date.now()}`;
|
|
597804
597945
|
const { abortController: _ignore, ...serializable } = task;
|
|
597805
597946
|
writeFileSync51(tmpPath, JSON.stringify(serializable, null, 2), "utf-8");
|
|
597806
|
-
|
|
597947
|
+
renameSync10(tmpPath, finalPath);
|
|
597807
597948
|
} catch {
|
|
597808
597949
|
}
|
|
597809
597950
|
}
|
|
@@ -601956,6 +602097,172 @@ function parseJsonObject3(value2) {
|
|
|
601956
602097
|
return null;
|
|
601957
602098
|
}
|
|
601958
602099
|
}
|
|
602100
|
+
function parseJsonObjectFromText(value2) {
|
|
602101
|
+
const direct = parseJsonObject3(value2);
|
|
602102
|
+
if (direct) return direct;
|
|
602103
|
+
const text2 = String(value2 ?? "");
|
|
602104
|
+
const start2 = text2.indexOf("{");
|
|
602105
|
+
const end = text2.lastIndexOf("}");
|
|
602106
|
+
if (start2 < 0 || end <= start2) return null;
|
|
602107
|
+
return parseJsonObject3(text2.slice(start2, end + 1));
|
|
602108
|
+
}
|
|
602109
|
+
function normalizeLiveCameraRotation(value2) {
|
|
602110
|
+
if (value2 === void 0 || value2 === null || value2 === "") return 0;
|
|
602111
|
+
if (typeof value2 === "number" && Number.isFinite(value2)) {
|
|
602112
|
+
const n2 = (Math.round(value2) % 360 + 360) % 360;
|
|
602113
|
+
if (n2 === 0 || n2 === 90 || n2 === 180 || n2 === 270) return n2;
|
|
602114
|
+
}
|
|
602115
|
+
const raw = String(value2).trim().toLowerCase();
|
|
602116
|
+
const numeric = Number(raw.replace(/deg(?:rees)?$/i, ""));
|
|
602117
|
+
if (Number.isFinite(numeric)) return normalizeLiveCameraRotation(numeric);
|
|
602118
|
+
if (["none", "off", "upright", "normal", "0"].includes(raw)) return 0;
|
|
602119
|
+
if (["cw", "clockwise", "right", "rotate-cw", "rotate_cw", "90"].includes(raw)) return 90;
|
|
602120
|
+
if (["180", "upside-down", "upside_down", "flip", "inverted"].includes(raw)) return 180;
|
|
602121
|
+
if (["ccw", "counterclockwise", "counter-clockwise", "counter_clockwise", "anticlockwise", "left", "rotate-ccw", "rotate_ccw", "270"].includes(raw)) return 270;
|
|
602122
|
+
throw new Error(`Unsupported camera rotation: ${String(value2)}. Use 0, 90, 180, 270, cw, ccw, or none.`);
|
|
602123
|
+
}
|
|
602124
|
+
function parseCameraOrientationDecision(value2) {
|
|
602125
|
+
const parsed = parseJsonObjectFromText(value2);
|
|
602126
|
+
if (!parsed) return null;
|
|
602127
|
+
const rawRotation = parsed["rotation_degrees"] ?? parsed["correction_degrees"] ?? parsed["rotate_degrees"] ?? parsed["rotation"] ?? parsed["correction"];
|
|
602128
|
+
if (rawRotation === void 0 || rawRotation === null || rawRotation === "") return null;
|
|
602129
|
+
let rotation;
|
|
602130
|
+
try {
|
|
602131
|
+
rotation = normalizeLiveCameraRotation(rawRotation);
|
|
602132
|
+
} catch {
|
|
602133
|
+
return null;
|
|
602134
|
+
}
|
|
602135
|
+
const confidenceNumber = Number(parsed["confidence"]);
|
|
602136
|
+
const confidence2 = Number.isFinite(confidenceNumber) ? Math.max(0, Math.min(1, confidenceNumber)) : void 0;
|
|
602137
|
+
const reason = typeof parsed["reason"] === "string" ? boundedText(parsed["reason"], 240) : typeof parsed["evidence"] === "string" ? boundedText(parsed["evidence"], 240) : void 0;
|
|
602138
|
+
return { rotation, confidence: confidence2, reason };
|
|
602139
|
+
}
|
|
602140
|
+
function formatCameraRotation(rotation) {
|
|
602141
|
+
if (rotation === 0) return "upright/no correction";
|
|
602142
|
+
if (rotation === 90) return "90 degrees clockwise";
|
|
602143
|
+
if (rotation === 180) return "180 degrees";
|
|
602144
|
+
return "90 degrees counter-clockwise";
|
|
602145
|
+
}
|
|
602146
|
+
function sanitizeCameraOrientation(value2) {
|
|
602147
|
+
if (typeof value2 !== "object" || value2 === null || Array.isArray(value2)) return {};
|
|
602148
|
+
const out = {};
|
|
602149
|
+
for (const [device, raw] of Object.entries(value2)) {
|
|
602150
|
+
if (!device || typeof raw !== "object" || raw === null || Array.isArray(raw)) continue;
|
|
602151
|
+
const entry = raw;
|
|
602152
|
+
try {
|
|
602153
|
+
out[device] = {
|
|
602154
|
+
rotation: normalizeLiveCameraRotation(entry["rotation"]),
|
|
602155
|
+
source: entry["source"] === "manual" ? "manual" : "auto",
|
|
602156
|
+
updatedAt: Math.max(0, Number(entry["updatedAt"] ?? Date.now())),
|
|
602157
|
+
confidence: typeof entry["confidence"] === "number" ? Math.max(0, Math.min(1, entry["confidence"])) : void 0,
|
|
602158
|
+
evidence: typeof entry["evidence"] === "string" ? boundedText(entry["evidence"], 800) : void 0
|
|
602159
|
+
};
|
|
602160
|
+
} catch {
|
|
602161
|
+
}
|
|
602162
|
+
}
|
|
602163
|
+
return out;
|
|
602164
|
+
}
|
|
602165
|
+
function describeCameraOrientation(orientation) {
|
|
602166
|
+
if (!orientation) return "upright/no correction (default)";
|
|
602167
|
+
return `${formatCameraRotation(orientation.rotation)} (${orientation.source}${orientation.confidence !== void 0 ? ` confidence=${orientation.confidence.toFixed(2)}` : ""}, updated=${nowIso3(orientation.updatedAt)})`;
|
|
602168
|
+
}
|
|
602169
|
+
function chooseCameraOrientationFromScores(scores) {
|
|
602170
|
+
const sorted = [...scores].filter((score) => Number.isFinite(score.score)).sort((a2, b) => b.score - a2.score);
|
|
602171
|
+
const best = sorted[0];
|
|
602172
|
+
if (!best || best.score <= 0 || best.faces <= 0) return null;
|
|
602173
|
+
const second3 = sorted[1];
|
|
602174
|
+
const gap = best.score - (second3?.score ?? 0);
|
|
602175
|
+
const total = sorted.reduce((sum, score) => sum + Math.max(0, score.score), 0);
|
|
602176
|
+
const share = total > 0 ? best.score / total : 1;
|
|
602177
|
+
const confidence2 = Math.max(0.55, Math.min(0.98, share * 0.75 + Math.min(0.23, gap / Math.max(1, best.score))));
|
|
602178
|
+
if (confidence2 < 0.58 && gap < 0.4) return null;
|
|
602179
|
+
return {
|
|
602180
|
+
rotation: best.rotation,
|
|
602181
|
+
confidence: confidence2,
|
|
602182
|
+
reason: `cv-face-orientation faces=${best.faces} score=${best.score.toFixed(2)} next=${(second3?.score ?? 0).toFixed(2)} area=${(best.faceAreaRatio ?? 0).toFixed(3)}`
|
|
602183
|
+
};
|
|
602184
|
+
}
|
|
602185
|
+
function trimOneLine(value2, maxChars) {
|
|
602186
|
+
return String(value2 ?? "").replace(/\r/g, " ").replace(/\n/g, " ").replace(/\s+/g, " ").trim().slice(0, maxChars);
|
|
602187
|
+
}
|
|
602188
|
+
function compactSourceId(source) {
|
|
602189
|
+
return source.replace(/^\/dev\//, "");
|
|
602190
|
+
}
|
|
602191
|
+
function truncateCell(value2, width) {
|
|
602192
|
+
if (width <= 1) return value2.slice(0, Math.max(0, width));
|
|
602193
|
+
return value2.length <= width ? value2.padEnd(width, " ") : `${value2.slice(0, Math.max(1, width - 1))}…`;
|
|
602194
|
+
}
|
|
602195
|
+
function summarizeInference(value2) {
|
|
602196
|
+
if (!value2) return "objects: pending";
|
|
602197
|
+
const lines = value2.split("\n").map((line) => line.trim()).filter(Boolean);
|
|
602198
|
+
const objectsIndex = lines.findIndex((line) => /^##\s+Objects/i.test(line));
|
|
602199
|
+
if (objectsIndex >= 0) {
|
|
602200
|
+
const objects = lines.slice(objectsIndex + 1).filter((line) => line.startsWith("- ")).slice(0, 4).map((line) => line.replace(/^-\s*/, ""));
|
|
602201
|
+
if (objects.length > 0) return `objects: ${objects.join("; ")}`;
|
|
602202
|
+
}
|
|
602203
|
+
const sampled = lines.find((line) => /^objects:/i.test(line));
|
|
602204
|
+
return sampled ? trimOneLine(sampled, 160) : trimOneLine(lines.slice(0, 4).join("; "), 160);
|
|
602205
|
+
}
|
|
602206
|
+
function cameraSnapshotSummary(camera) {
|
|
602207
|
+
if (camera.error) return `error: ${trimOneLine(camera.error, 160)}`;
|
|
602208
|
+
return camera.summary || summarizeInference(camera.inference);
|
|
602209
|
+
}
|
|
602210
|
+
function formatLiveDashboardFromSnapshot(snapshot, opts = {}) {
|
|
602211
|
+
const width = Math.max(60, opts.width ?? 100);
|
|
602212
|
+
const now2 = opts.now ?? Date.now();
|
|
602213
|
+
const maxCameras = opts.maxCameras ?? 4;
|
|
602214
|
+
const cameras = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {})).sort((a2, b) => {
|
|
602215
|
+
if (a2.source === snapshot.config.selectedCamera) return -1;
|
|
602216
|
+
if (b.source === snapshot.config.selectedCamera) return 1;
|
|
602217
|
+
return a2.source.localeCompare(b.source);
|
|
602218
|
+
}).slice(0, maxCameras);
|
|
602219
|
+
const audioAge = snapshot.audio?.updatedAt ? `${Math.max(0, Math.round((now2 - snapshot.audio.updatedAt) / 1e3))}s` : "none";
|
|
602220
|
+
const title = `live audio/video cameras=${cameras.length}/${snapshot.devices.video.length} audio=${snapshot.config.audioEnabled ? "on" : "off"} age=${audioAge}`;
|
|
602221
|
+
const horizontal = "─".repeat(Math.max(0, width - 2));
|
|
602222
|
+
const lines = [`╭${horizontal}╮`];
|
|
602223
|
+
lines.push(`│${truncateCell(` ${title}`, width - 2)}│`);
|
|
602224
|
+
if (cameras.length === 0) {
|
|
602225
|
+
lines.push(`│${truncateCell(" no camera frames captured yet", width - 2)}│`);
|
|
602226
|
+
}
|
|
602227
|
+
for (const camera of cameras) {
|
|
602228
|
+
const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
|
|
602229
|
+
const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
|
|
602230
|
+
lines.push(`├${horizontal}┤`);
|
|
602231
|
+
lines.push(`│${truncateCell(` ${compactSourceId(camera.source)} age=${age}s rotation=${rotation}`, width - 2)}│`);
|
|
602232
|
+
const asciiLines = (camera.ascii ?? "").split("\n").filter((line) => line.length > 0).slice(0, 10);
|
|
602233
|
+
const detailLines = [
|
|
602234
|
+
cameraSnapshotSummary(camera),
|
|
602235
|
+
camera.framePath ? `frame: ${camera.displayPath || camera.framePath}` : "",
|
|
602236
|
+
camera.orientation?.evidence ? `orientation: ${trimOneLine(camera.orientation.evidence, 120)}` : ""
|
|
602237
|
+
].filter(Boolean);
|
|
602238
|
+
if (asciiLines.length === 0) {
|
|
602239
|
+
for (const detail of detailLines.length ? detailLines : ["preview pending"]) {
|
|
602240
|
+
lines.push(`│${truncateCell(` ${detail}`, width - 2)}│`);
|
|
602241
|
+
}
|
|
602242
|
+
continue;
|
|
602243
|
+
}
|
|
602244
|
+
const imageWidth = Math.max(24, Math.min(52, Math.floor((width - 5) * 0.58)));
|
|
602245
|
+
const detailWidth = Math.max(18, width - imageWidth - 5);
|
|
602246
|
+
const rowCount = Math.max(asciiLines.length, detailLines.length);
|
|
602247
|
+
for (let i2 = 0; i2 < rowCount; i2++) {
|
|
602248
|
+
const image = truncateCell(asciiLines[i2] ?? "", imageWidth);
|
|
602249
|
+
const detail = truncateCell(detailLines[i2] ?? "", detailWidth);
|
|
602250
|
+
lines.push(`│ ${image} │ ${detail}│`);
|
|
602251
|
+
}
|
|
602252
|
+
}
|
|
602253
|
+
if (snapshot.audio) {
|
|
602254
|
+
lines.push(`├${horizontal}┤`);
|
|
602255
|
+
const audioBits = [
|
|
602256
|
+
`audio input=${snapshot.audio.input || snapshot.config.selectedAudioInput || "none"}`,
|
|
602257
|
+
snapshot.audio.error ? `error=${trimOneLine(snapshot.audio.error, 120)}` : "",
|
|
602258
|
+
snapshot.audio.transcript ? `asr=${trimOneLine(snapshot.audio.transcript, 120)}` : "",
|
|
602259
|
+
snapshot.audio.analysis ? `sound=${trimOneLine(snapshot.audio.analysis, 120)}` : ""
|
|
602260
|
+
].filter(Boolean);
|
|
602261
|
+
for (const item of audioBits.slice(0, 4)) lines.push(`│${truncateCell(` ${item}`, width - 2)}│`);
|
|
602262
|
+
}
|
|
602263
|
+
lines.push(`╰${horizontal}╯`);
|
|
602264
|
+
return lines;
|
|
602265
|
+
}
|
|
601959
602266
|
function parseLiveMediaPayload(value2) {
|
|
601960
602267
|
const parsed = parseJsonObject3(value2);
|
|
601961
602268
|
if (!parsed) return null;
|
|
@@ -602180,6 +602487,7 @@ function loadLiveSensorConfig(repoRoot) {
|
|
|
602180
602487
|
return {
|
|
602181
602488
|
...DEFAULT_CONFIG7,
|
|
602182
602489
|
...parsed,
|
|
602490
|
+
cameraOrientation: sanitizeCameraOrientation(parsed.cameraOrientation),
|
|
602183
602491
|
videoIntervalMs: Math.max(1500, Number(parsed.videoIntervalMs ?? DEFAULT_CONFIG7.videoIntervalMs)),
|
|
602184
602492
|
audioIntervalMs: Math.max(3e3, Number(parsed.audioIntervalMs ?? DEFAULT_CONFIG7.audioIntervalMs))
|
|
602185
602493
|
};
|
|
@@ -602208,11 +602516,30 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
602208
602516
|
lines.push("Live sensor context is operator-enabled ambient input. Treat it as current perceptual evidence for this turn; if it is stale, missing, or insufficient, use camera/audio tools to refresh before answering what you see or hear.");
|
|
602209
602517
|
lines.push("Identity rule: never invent names for observed people. If a person is unknown and identity matters, ask the user or the person for their name, then store the association with visual_memory(action='enroll', image='<face crop>', name='<name>') when a face crop is available, or multimodal_memory(action='meet', person_name='<name>') when live voice/name context is available.");
|
|
602210
602518
|
lines.push(`Snapshot file: ${liveSnapshotPath(snapshot.repoRoot)}`);
|
|
602211
|
-
lines.push(`Streams: video=${snapshot.config.videoEnabled ? "on" : "off"} infer=${snapshot.config.inferEnabled ? "on" : "off"} audio=${snapshot.config.audioEnabled ? "on" : "off"} asr=${snapshot.config.asrEnabled ? "on" : "off"} sounds=${snapshot.config.audioAnalysisEnabled ? "on" : "off"} output-monitor=${snapshot.config.audioOutputEnabled ? "on" : "off"}`);
|
|
602212
|
-
if (snapshot.config.selectedCamera)
|
|
602519
|
+
lines.push(`Streams: video=${snapshot.config.videoEnabled ? "on" : "off"} infer=${snapshot.config.inferEnabled ? "on" : "off"} clip=${snapshot.config.clipEnabled ? "on" : "off"} audio=${snapshot.config.audioEnabled ? "on" : "off"} asr=${snapshot.config.asrEnabled ? "on" : "off"} sounds=${snapshot.config.audioAnalysisEnabled ? "on" : "off"} output-monitor=${snapshot.config.audioOutputEnabled ? "on" : "off"}`);
|
|
602520
|
+
if (snapshot.config.selectedCamera) {
|
|
602521
|
+
const orientation = snapshot.config.cameraOrientation?.[snapshot.config.selectedCamera];
|
|
602522
|
+
lines.push(`Selected camera: ${snapshot.config.selectedCamera}`);
|
|
602523
|
+
lines.push(`Camera orientation correction: ${describeCameraOrientation(orientation)}`);
|
|
602524
|
+
}
|
|
602525
|
+
const cameraSnapshots = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {})).sort((a2, b) => {
|
|
602526
|
+
if (a2.source === snapshot.config.selectedCamera) return -1;
|
|
602527
|
+
if (b.source === snapshot.config.selectedCamera) return 1;
|
|
602528
|
+
return b.updatedAt - a2.updatedAt;
|
|
602529
|
+
});
|
|
602530
|
+
if (cameraSnapshots.length > 0) {
|
|
602531
|
+
lines.push("Camera snapshot set:");
|
|
602532
|
+
for (const camera of cameraSnapshots.slice(0, 6)) {
|
|
602533
|
+
const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
|
|
602534
|
+
const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
|
|
602535
|
+
lines.push(`- ${camera.source} age=${age}s rotation=${rotation}${camera.framePath ? ` frame=${camera.framePath}` : ""}${camera.error ? ` error=${trimOneLine(camera.error, 240)}` : ""}`);
|
|
602536
|
+
const summary = cameraSnapshotSummary(camera);
|
|
602537
|
+
if (summary) lines.push(` summary: ${summary}`);
|
|
602538
|
+
}
|
|
602539
|
+
}
|
|
602213
602540
|
if (snapshot.config.selectedAudioInput) lines.push(`Selected audio input: ${snapshot.config.selectedAudioInput}`);
|
|
602214
602541
|
if (snapshot.config.selectedAudioOutput) lines.push(`Selected audio output: ${snapshot.config.selectedAudioOutput}`);
|
|
602215
|
-
const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 18);
|
|
602542
|
+
const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "clip_recognition" || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 18);
|
|
602216
602543
|
if (activeVideo) {
|
|
602217
602544
|
lines.push("");
|
|
602218
602545
|
lines.push("## LIVE CAMERA EVENTS");
|
|
@@ -602257,7 +602584,7 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
602257
602584
|
const age = now2 - snapshot.video.updatedAt;
|
|
602258
602585
|
lines.push("");
|
|
602259
602586
|
lines.push("## LIVE CAMERA LATEST FRAME");
|
|
602260
|
-
lines.push(`timestamp: ${nowIso3(snapshot.video.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s source=${snapshot.video.source}${age > maxAgeMs ? " STALE" : ""}`);
|
|
602587
|
+
lines.push(`timestamp: ${nowIso3(snapshot.video.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s source=${snapshot.video.source}${snapshot.video.orientation ? ` rotation=${formatCameraRotation(snapshot.video.orientation.rotation)}` : ""}${age > maxAgeMs ? " STALE" : ""}`);
|
|
602261
602588
|
if (snapshot.video.error) lines.push(`error: ${snapshot.video.error}`);
|
|
602262
602589
|
if (snapshot.video.framePath) lines.push(`frame: ${snapshot.video.framePath}`);
|
|
602263
602590
|
if (snapshot.video.asciiContext) lines.push(snapshot.video.asciiContext);
|
|
@@ -602266,6 +602593,19 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
|
|
|
602266
602593
|
lines.push(cleanPreview(snapshot.video.inference, 1800));
|
|
602267
602594
|
}
|
|
602268
602595
|
}
|
|
602596
|
+
if (cameraSnapshots.length > 1) {
|
|
602597
|
+
lines.push("");
|
|
602598
|
+
lines.push("## LIVE CAMERA LATEST FRAMES");
|
|
602599
|
+
for (const camera of cameraSnapshots.slice(0, 6)) {
|
|
602600
|
+
const age = now2 - camera.updatedAt;
|
|
602601
|
+
const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
|
|
602602
|
+
lines.push(`### ${camera.source}`);
|
|
602603
|
+
lines.push(`timestamp: ${nowIso3(camera.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s rotation=${rotation}${age > maxAgeMs ? " STALE" : ""}`);
|
|
602604
|
+
if (camera.error) lines.push(`error: ${camera.error}`);
|
|
602605
|
+
if (camera.framePath) lines.push(`frame: ${camera.framePath}`);
|
|
602606
|
+
lines.push(`summary: ${cameraSnapshotSummary(camera)}`);
|
|
602607
|
+
}
|
|
602608
|
+
}
|
|
602269
602609
|
if (snapshot.audio) {
|
|
602270
602610
|
const audioEvents = (snapshot.events ?? []).filter((event) => event.kind === "audio_transcript" || event.kind === "audio_sound").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 8);
|
|
602271
602611
|
if (audioEvents.length > 0) {
|
|
@@ -602372,6 +602712,93 @@ async function recordAudioSample(device, durationSec, outputPath3) {
|
|
|
602372
602712
|
outputPath3
|
|
602373
602713
|
], { timeout: (Number(seconds) + 10) * 1e3 });
|
|
602374
602714
|
}
|
|
602715
|
+
async function scoreCameraOrientationWithOpenCv(framePath) {
|
|
602716
|
+
if (!await commandExists2("python3", 1200)) return null;
|
|
602717
|
+
const script = String.raw`
|
|
602718
|
+
import json, sys
|
|
602719
|
+
try:
|
|
602720
|
+
import cv2
|
|
602721
|
+
except Exception as exc:
|
|
602722
|
+
print(json.dumps({"ok": False, "error": str(exc)}))
|
|
602723
|
+
raise SystemExit(0)
|
|
602724
|
+
|
|
602725
|
+
path = sys.argv[1]
|
|
602726
|
+
img = cv2.imread(path)
|
|
602727
|
+
if img is None:
|
|
602728
|
+
print(json.dumps({"ok": False, "error": "could not read image"}))
|
|
602729
|
+
raise SystemExit(0)
|
|
602730
|
+
|
|
602731
|
+
face = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml")
|
|
602732
|
+
eye = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_eye.xml")
|
|
602733
|
+
|
|
602734
|
+
def rotate(mat, deg):
|
|
602735
|
+
if deg == 90:
|
|
602736
|
+
return cv2.rotate(mat, cv2.ROTATE_90_CLOCKWISE)
|
|
602737
|
+
if deg == 180:
|
|
602738
|
+
return cv2.rotate(mat, cv2.ROTATE_180)
|
|
602739
|
+
if deg == 270:
|
|
602740
|
+
return cv2.rotate(mat, cv2.ROTATE_90_COUNTERCLOCKWISE)
|
|
602741
|
+
return mat
|
|
602742
|
+
|
|
602743
|
+
scores = []
|
|
602744
|
+
for deg in [0, 90, 180, 270]:
|
|
602745
|
+
frame = rotate(img, deg)
|
|
602746
|
+
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
|
602747
|
+
h, w = gray.shape[:2]
|
|
602748
|
+
faces = face.detectMultiScale(gray, scaleFactor=1.08, minNeighbors=4, minSize=(32, 32))
|
|
602749
|
+
face_area = 0.0
|
|
602750
|
+
eye_count = 0
|
|
602751
|
+
for (x, y, fw, fh) in faces[:8]:
|
|
602752
|
+
face_area += float(fw * fh) / max(1.0, float(w * h))
|
|
602753
|
+
roi = gray[y:y+fh, x:x+fw]
|
|
602754
|
+
try:
|
|
602755
|
+
eye_count += len(eye.detectMultiScale(roi, scaleFactor=1.1, minNeighbors=3, minSize=(8, 8)))
|
|
602756
|
+
except Exception:
|
|
602757
|
+
pass
|
|
602758
|
+
score = len(faces) * 2.5 + min(4.0, face_area * 35.0) + min(2.0, eye_count * 0.35)
|
|
602759
|
+
scores.append({"rotation": deg, "score": score, "faces": int(len(faces)), "faceAreaRatio": face_area})
|
|
602760
|
+
print(json.dumps({"ok": True, "scores": scores}))
|
|
602761
|
+
`;
|
|
602762
|
+
try {
|
|
602763
|
+
const raw = await execFileText4("python3", ["-c", script, framePath], { timeout: 15e3, maxBuffer: 1024 * 1024 });
|
|
602764
|
+
const parsed = parseJsonObjectFromText(raw);
|
|
602765
|
+
const scores = Array.isArray(parsed?.["scores"]) ? parsed["scores"] : [];
|
|
602766
|
+
if (scores.length === 0) return null;
|
|
602767
|
+
return scores.map((score) => ({
|
|
602768
|
+
rotation: normalizeLiveCameraRotation(score["rotation"]),
|
|
602769
|
+
score: Math.max(0, Number(score["score"] ?? 0)),
|
|
602770
|
+
faces: Math.max(0, Number(score["faces"] ?? 0)),
|
|
602771
|
+
faceAreaRatio: Number.isFinite(Number(score["faceAreaRatio"])) ? Number(score["faceAreaRatio"]) : void 0
|
|
602772
|
+
}));
|
|
602773
|
+
} catch {
|
|
602774
|
+
return null;
|
|
602775
|
+
}
|
|
602776
|
+
}
|
|
602777
|
+
async function tryRecordAudioDevice(device, durationSec, outputPath3) {
|
|
602778
|
+
await recordAudioSample(device, durationSec, outputPath3);
|
|
602779
|
+
}
|
|
602780
|
+
async function recordAudioSampleWithRecovery(preferred, devices, durationSec, outputPath3) {
|
|
602781
|
+
const ordered = [];
|
|
602782
|
+
const add3 = (id2) => {
|
|
602783
|
+
const clean5 = String(id2 ?? "").trim();
|
|
602784
|
+
if (clean5 && !ordered.includes(clean5)) ordered.push(clean5);
|
|
602785
|
+
};
|
|
602786
|
+
add3(preferred);
|
|
602787
|
+
for (const device of devices.filter((entry) => entry.source === "pulse" || entry.source === "pipewire")) add3(device.id);
|
|
602788
|
+
add3("pulse:default");
|
|
602789
|
+
add3("default");
|
|
602790
|
+
for (const device of devices.filter((entry) => entry.source !== "pulse" && entry.source !== "pipewire")) add3(device.id);
|
|
602791
|
+
const errors = [];
|
|
602792
|
+
for (const candidate of ordered) {
|
|
602793
|
+
try {
|
|
602794
|
+
await tryRecordAudioDevice(candidate, durationSec, outputPath3);
|
|
602795
|
+
return { ok: true, device: candidate, method: candidate.startsWith("pulse:") ? "pulse/ffmpeg" : "arecord", errors };
|
|
602796
|
+
} catch (err) {
|
|
602797
|
+
errors.push(`${candidate}: ${err instanceof Error ? err.message : String(err)}`.slice(0, 360));
|
|
602798
|
+
}
|
|
602799
|
+
}
|
|
602800
|
+
return { ok: false, device: preferred || "default", method: "none", errors };
|
|
602801
|
+
}
|
|
602375
602802
|
function firstDeviceId(devices) {
|
|
602376
602803
|
return devices.find((device) => device.id && !/metadata\/non-capture/i.test(device.detail ?? ""))?.id ?? devices[0]?.id;
|
|
602377
602804
|
}
|
|
@@ -602401,8 +602828,9 @@ function formatLiveInventory(inventory) {
|
|
|
602401
602828
|
function formatLiveStatus(snapshot) {
|
|
602402
602829
|
const cfg = snapshot.config;
|
|
602403
602830
|
const lines = [
|
|
602404
|
-
`Live streams: video=${cfg.videoEnabled ? "on" : "off"} infer=${cfg.inferEnabled ? "on" : "off"} audio=${cfg.audioEnabled ? "on" : "off"} asr=${cfg.asrEnabled ? "on" : "off"} sounds=${cfg.audioAnalysisEnabled ? "on" : "off"} output=${cfg.audioOutputEnabled ? "on" : "off"}`,
|
|
602831
|
+
`Live streams: video=${cfg.videoEnabled ? "on" : "off"} infer=${cfg.inferEnabled ? "on" : "off"} clip=${cfg.clipEnabled ? "on" : "off"} audio=${cfg.audioEnabled ? "on" : "off"} asr=${cfg.asrEnabled ? "on" : "off"} sounds=${cfg.audioAnalysisEnabled ? "on" : "off"} output=${cfg.audioOutputEnabled ? "on" : "off"}`,
|
|
602405
602832
|
`Camera: ${cfg.selectedCamera || "none"}`,
|
|
602833
|
+
`Camera orientation: ${cfg.selectedCamera ? describeCameraOrientation(cfg.cameraOrientation?.[cfg.selectedCamera]) : "none"}`,
|
|
602406
602834
|
`Audio input: ${cfg.selectedAudioInput || "none"}`,
|
|
602407
602835
|
`Audio output: ${cfg.selectedAudioOutput || "none"}`,
|
|
602408
602836
|
`Snapshot: ${liveSnapshotPath(snapshot.repoRoot)}`
|
|
@@ -602427,8 +602855,10 @@ var init_live_sensors = __esm({
|
|
|
602427
602855
|
audioEnabled: false,
|
|
602428
602856
|
audioOutputEnabled: false,
|
|
602429
602857
|
inferEnabled: false,
|
|
602858
|
+
clipEnabled: false,
|
|
602430
602859
|
asrEnabled: false,
|
|
602431
602860
|
audioAnalysisEnabled: false,
|
|
602861
|
+
cameraOrientation: {},
|
|
602432
602862
|
videoIntervalMs: 5e3,
|
|
602433
602863
|
audioIntervalMs: 9e3
|
|
602434
602864
|
};
|
|
@@ -602454,16 +602884,155 @@ var init_live_sensors = __esm({
|
|
|
602454
602884
|
videoInFlight = false;
|
|
602455
602885
|
audioInFlight = false;
|
|
602456
602886
|
statusSink = null;
|
|
602887
|
+
feedbackSink = null;
|
|
602888
|
+
agentActionSink = null;
|
|
602889
|
+
agentActionEnabled = false;
|
|
602890
|
+
lastAgentReviewAt = 0;
|
|
602891
|
+
autoOrientationInFlight = /* @__PURE__ */ new Set();
|
|
602892
|
+
lastFeedbackAt = /* @__PURE__ */ new Map();
|
|
602457
602893
|
setStatusSink(sink) {
|
|
602458
602894
|
this.statusSink = sink ?? null;
|
|
602459
602895
|
this.emitStatus();
|
|
602460
602896
|
}
|
|
602897
|
+
setFeedbackSink(sink) {
|
|
602898
|
+
this.feedbackSink = sink ?? null;
|
|
602899
|
+
}
|
|
602900
|
+
setAgentActionSink(sink) {
|
|
602901
|
+
this.agentActionSink = sink ?? null;
|
|
602902
|
+
}
|
|
602461
602903
|
getConfig() {
|
|
602462
602904
|
return { ...this.config };
|
|
602463
602905
|
}
|
|
602464
602906
|
getSnapshot() {
|
|
602465
602907
|
return this.snapshot;
|
|
602466
602908
|
}
|
|
602909
|
+
activeVideoSources() {
|
|
602910
|
+
const ordered = [];
|
|
602911
|
+
const add3 = (id2) => {
|
|
602912
|
+
const clean5 = String(id2 ?? "").trim();
|
|
602913
|
+
if (!clean5 || ordered.includes(clean5)) return;
|
|
602914
|
+
const device = this.devices.video.find((entry) => entry.id === clean5);
|
|
602915
|
+
if (device && /metadata\/non-capture/i.test(device.detail ?? "")) return;
|
|
602916
|
+
ordered.push(clean5);
|
|
602917
|
+
};
|
|
602918
|
+
add3(this.config.selectedCamera);
|
|
602919
|
+
for (const device of this.devices.video) add3(device.id);
|
|
602920
|
+
if (ordered.length === 0) add3(firstDeviceId(this.devices.video));
|
|
602921
|
+
return ordered.slice(0, 4);
|
|
602922
|
+
}
|
|
602923
|
+
getCameraOrientation(device = this.config.selectedCamera) {
|
|
602924
|
+
return device ? this.config.cameraOrientation?.[device] : void 0;
|
|
602925
|
+
}
|
|
602926
|
+
getCameraRotation(device = this.config.selectedCamera) {
|
|
602927
|
+
return this.getCameraOrientation(device)?.rotation ?? 0;
|
|
602928
|
+
}
|
|
602929
|
+
setCameraRotation(device, rotation, source, evidence, confidence2) {
|
|
602930
|
+
const target = device || this.config.selectedCamera || firstDeviceId(this.devices.video);
|
|
602931
|
+
if (!target) return null;
|
|
602932
|
+
const orientation = {
|
|
602933
|
+
rotation,
|
|
602934
|
+
source,
|
|
602935
|
+
updatedAt: Date.now(),
|
|
602936
|
+
confidence: confidence2,
|
|
602937
|
+
evidence: evidence ? boundedText(evidence, 800) : void 0
|
|
602938
|
+
};
|
|
602939
|
+
this.config = {
|
|
602940
|
+
...this.config,
|
|
602941
|
+
selectedCamera: target,
|
|
602942
|
+
cameraOrientation: {
|
|
602943
|
+
...this.config.cameraOrientation ?? {},
|
|
602944
|
+
[target]: orientation
|
|
602945
|
+
}
|
|
602946
|
+
};
|
|
602947
|
+
this.snapshot.events = mergeRecentById(this.snapshot.events, [{
|
|
602948
|
+
id: stableEventId("camera_orientation", target, orientation.updatedAt, `${source}:${rotation}`),
|
|
602949
|
+
kind: "camera_orientation",
|
|
602950
|
+
observedAt: orientation.updatedAt,
|
|
602951
|
+
source: target,
|
|
602952
|
+
title: "Camera orientation correction updated",
|
|
602953
|
+
summary: `${target}: ${formatCameraRotation(rotation)} via ${source}${confidence2 !== void 0 ? ` confidence=${confidence2.toFixed(2)}` : ""}${evidence ? ` evidence=${boundedText(evidence, 300)}` : ""}`,
|
|
602954
|
+
confidence: confidence2
|
|
602955
|
+
}], 50);
|
|
602956
|
+
this.persist();
|
|
602957
|
+
this.emitFeedback({
|
|
602958
|
+
kind: "orientation",
|
|
602959
|
+
title: "Camera orientation correction updated",
|
|
602960
|
+
observedAt: orientation.updatedAt,
|
|
602961
|
+
message: `${target}: ${describeCameraOrientation(orientation)}`
|
|
602962
|
+
});
|
|
602963
|
+
return orientation;
|
|
602964
|
+
}
|
|
602965
|
+
cycleCameraRotation(device = this.config.selectedCamera) {
|
|
602966
|
+
const current = this.getCameraRotation(device);
|
|
602967
|
+
const next = current === 0 ? 90 : current === 90 ? 180 : current === 180 ? 270 : 0;
|
|
602968
|
+
return this.setCameraRotation(device, next, "manual", "manual cycle from /live menu");
|
|
602969
|
+
}
|
|
602970
|
+
async autoOrientCamera(device = this.config.selectedCamera, options2 = {}) {
|
|
602971
|
+
const target = device || this.config.selectedCamera || firstDeviceId(this.devices.video);
|
|
602972
|
+
if (!target) return "No camera selected. Use /live camera <device> or select a camera in /live.";
|
|
602973
|
+
if (options2.onlyIfUnset && this.getCameraOrientation(target)) {
|
|
602974
|
+
return `Camera orientation already set for ${target}: ${describeCameraOrientation(this.getCameraOrientation(target))}`;
|
|
602975
|
+
}
|
|
602976
|
+
if (this.autoOrientationInFlight.has(target)) {
|
|
602977
|
+
return `Camera orientation detection is already running for ${target}.`;
|
|
602978
|
+
}
|
|
602979
|
+
this.autoOrientationInFlight.add(target);
|
|
602980
|
+
try {
|
|
602981
|
+
const captured = await new CameraCaptureTool().execute({
|
|
602982
|
+
action: "capture",
|
|
602983
|
+
device: target,
|
|
602984
|
+
width: 640,
|
|
602985
|
+
height: 480,
|
|
602986
|
+
rotate_degrees: 0
|
|
602987
|
+
});
|
|
602988
|
+
if (!captured.success) {
|
|
602989
|
+
return `Camera orientation detection could not capture ${target}: ${captured.error || captured.output || "unknown error"}`;
|
|
602990
|
+
}
|
|
602991
|
+
const framePath = extractSavedImagePath(captured.output, this.repoRoot);
|
|
602992
|
+
if (!framePath) return `Camera orientation detection captured ${target}, but no frame path was returned.`;
|
|
602993
|
+
const cvScores = await scoreCameraOrientationWithOpenCv(framePath);
|
|
602994
|
+
const cvDecision = cvScores ? chooseCameraOrientationFromScores(cvScores) : null;
|
|
602995
|
+
if (cvDecision && cvDecision.confidence >= 0.68) {
|
|
602996
|
+
const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
|
|
602997
|
+
return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
|
|
602998
|
+
}
|
|
602999
|
+
const prompt = [
|
|
603000
|
+
"Determine whether this camera frame is upright for a human viewer.",
|
|
603001
|
+
"Return only JSON with this schema:",
|
|
603002
|
+
'{"rotation_degrees":0|90|180|270,"confidence":0.0-1.0,"reason":"brief visual evidence"}',
|
|
603003
|
+
"rotation_degrees is the clockwise correction to apply to future frames so they are upright.",
|
|
603004
|
+
"Use 90 if the image must rotate clockwise; use 270 if it must rotate counter-clockwise; use 180 if upside down; use 0 if already upright or inconclusive."
|
|
603005
|
+
].join("\n");
|
|
603006
|
+
const vision = await new VisionTool(this.repoRoot).execute({
|
|
603007
|
+
action: "query",
|
|
603008
|
+
image: framePath,
|
|
603009
|
+
prompt,
|
|
603010
|
+
model: "moondream3-preview"
|
|
603011
|
+
});
|
|
603012
|
+
if (!vision.success) {
|
|
603013
|
+
if (cvDecision) {
|
|
603014
|
+
const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
|
|
603015
|
+
return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
|
|
603016
|
+
}
|
|
603017
|
+
return `Camera orientation detection captured ${target}, but vision inference was unavailable: ${vision.error || vision.output || "unknown error"}`;
|
|
603018
|
+
}
|
|
603019
|
+
const decision2 = parseCameraOrientationDecision(vision.llmContent || vision.output);
|
|
603020
|
+
if (!decision2) {
|
|
603021
|
+
if (cvDecision) {
|
|
603022
|
+
const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
|
|
603023
|
+
return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
|
|
603024
|
+
}
|
|
603025
|
+
return `Camera orientation detection for ${target} was inconclusive. Vision output did not contain a structured rotation decision.`;
|
|
603026
|
+
}
|
|
603027
|
+
const useCv = cvDecision && (cvDecision.confidence > (decision2.confidence ?? 0) + 0.12 || cvDecision.rotation !== decision2.rotation && cvDecision.confidence >= 0.62 && (decision2.confidence ?? 0) < 0.76);
|
|
603028
|
+
const selected = useCv ? cvDecision : decision2;
|
|
603029
|
+
const evidence = useCv ? cvDecision.reason : `${decision2.reason || boundedText(vision.llmContent || vision.output, 320)}${cvDecision ? `; cv=${cvDecision.reason}` : ""}`;
|
|
603030
|
+
const orientation = this.setCameraRotation(target, selected.rotation, "auto", evidence, selected.confidence);
|
|
603031
|
+
return orientation ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation)}` : `Auto orientation for ${target} could not be persisted.`;
|
|
603032
|
+
} finally {
|
|
603033
|
+
this.autoOrientationInFlight.delete(target);
|
|
603034
|
+
}
|
|
603035
|
+
}
|
|
602467
603036
|
async refreshDevices() {
|
|
602468
603037
|
const errors = [];
|
|
602469
603038
|
let video = [];
|
|
@@ -602504,31 +603073,105 @@ var init_live_sensors = __esm({
|
|
|
602504
603073
|
return this.getConfig();
|
|
602505
603074
|
}
|
|
602506
603075
|
stopAll() {
|
|
603076
|
+
this.agentActionEnabled = false;
|
|
602507
603077
|
this.configure({
|
|
602508
603078
|
videoEnabled: false,
|
|
602509
603079
|
audioEnabled: false,
|
|
602510
603080
|
audioOutputEnabled: false,
|
|
602511
603081
|
inferEnabled: false,
|
|
603082
|
+
clipEnabled: false,
|
|
602512
603083
|
asrEnabled: false,
|
|
602513
603084
|
audioAnalysisEnabled: false
|
|
602514
603085
|
});
|
|
602515
603086
|
}
|
|
602516
|
-
|
|
603087
|
+
startRunMode(options2 = {}) {
|
|
603088
|
+
this.agentActionEnabled = Boolean(options2.agent);
|
|
603089
|
+
this.configure({
|
|
603090
|
+
videoEnabled: true,
|
|
603091
|
+
inferEnabled: true,
|
|
603092
|
+
clipEnabled: true,
|
|
603093
|
+
audioEnabled: true,
|
|
603094
|
+
asrEnabled: true,
|
|
603095
|
+
audioAnalysisEnabled: true,
|
|
603096
|
+
videoIntervalMs: options2.lowLatency === false ? this.config.videoIntervalMs : Math.min(this.config.videoIntervalMs, 1500),
|
|
603097
|
+
audioIntervalMs: options2.lowLatency === false ? this.config.audioIntervalMs : Math.min(this.config.audioIntervalMs, 3e3)
|
|
603098
|
+
});
|
|
603099
|
+
for (const source of this.activeVideoSources()) {
|
|
603100
|
+
const current = this.getCameraOrientation(source);
|
|
603101
|
+
if (current?.source === "manual") continue;
|
|
603102
|
+
if (current && current.confidence !== void 0 && current.confidence >= 0.75 && Date.now() - current.updatedAt < 15 * 6e4) continue;
|
|
603103
|
+
void this.autoOrientCamera(source).then((message2) => {
|
|
603104
|
+
this.emitFeedbackThrottled(`orient:${source}`, {
|
|
603105
|
+
kind: "orientation",
|
|
603106
|
+
title: "Camera auto-orientation",
|
|
603107
|
+
observedAt: Date.now(),
|
|
603108
|
+
message: message2
|
|
603109
|
+
}, 3e4);
|
|
603110
|
+
this.emitDashboard();
|
|
603111
|
+
}).catch((err) => {
|
|
603112
|
+
this.emitFeedbackThrottled(`orient-error:${source}`, {
|
|
603113
|
+
kind: "error",
|
|
603114
|
+
title: "Camera auto-orientation failed",
|
|
603115
|
+
observedAt: Date.now(),
|
|
603116
|
+
message: err instanceof Error ? err.message : String(err)
|
|
603117
|
+
}, 6e4);
|
|
603118
|
+
});
|
|
603119
|
+
}
|
|
603120
|
+
void this.sampleVideoNow(true).catch((err) => {
|
|
603121
|
+
this.emitFeedback({
|
|
603122
|
+
kind: "error",
|
|
603123
|
+
title: "Live video sample failed",
|
|
603124
|
+
observedAt: Date.now(),
|
|
603125
|
+
message: err instanceof Error ? err.message : String(err)
|
|
603126
|
+
});
|
|
603127
|
+
});
|
|
603128
|
+
void this.sampleAudioNow().catch((err) => {
|
|
603129
|
+
this.emitFeedback({
|
|
603130
|
+
kind: "error",
|
|
603131
|
+
title: "Live audio sample failed",
|
|
603132
|
+
observedAt: Date.now(),
|
|
603133
|
+
message: err instanceof Error ? err.message : String(err)
|
|
603134
|
+
});
|
|
603135
|
+
});
|
|
603136
|
+
this.emitFeedback({
|
|
603137
|
+
kind: "status",
|
|
603138
|
+
title: options2.agent ? "Live PFC run started" : "Live run started",
|
|
603139
|
+
observedAt: Date.now(),
|
|
603140
|
+
message: "Low-latency video, face, CLIP, ASR, and sound loops are active. Latest observations are injected into each agent turn."
|
|
603141
|
+
});
|
|
603142
|
+
return options2.agent ? "Live PFC run enabled: visual/audio feedback is active and significant events can launch background agent review." : "Live run enabled: visual/audio feedback is active and live context will update each turn.";
|
|
603143
|
+
}
|
|
603144
|
+
async previewCamera(device = this.config.selectedCamera, options2 = {}) {
|
|
602517
603145
|
const source = device || firstDeviceId(this.devices.video);
|
|
602518
603146
|
if (!source) return { ok: false, message: "No camera selected. Use /live to select a camera after device discovery." };
|
|
603147
|
+
const orientation = this.getCameraOrientation(source);
|
|
603148
|
+
const rotation = orientation?.rotation ?? 0;
|
|
602519
603149
|
const result = await new CameraCaptureTool().execute({
|
|
602520
603150
|
action: "capture",
|
|
602521
603151
|
device: source,
|
|
602522
603152
|
width: 640,
|
|
602523
|
-
height: 480
|
|
603153
|
+
height: 480,
|
|
603154
|
+
rotate_degrees: rotation
|
|
602524
603155
|
});
|
|
602525
603156
|
if (!result.success) {
|
|
602526
603157
|
const error = result.error || result.output || "camera capture failed";
|
|
602527
603158
|
const observedAt2 = Date.now();
|
|
602528
|
-
|
|
603159
|
+
const errorView = {
|
|
602529
603160
|
updatedAt: observedAt2,
|
|
602530
603161
|
source,
|
|
602531
|
-
error
|
|
603162
|
+
error,
|
|
603163
|
+
rotation,
|
|
603164
|
+
orientation
|
|
603165
|
+
};
|
|
603166
|
+
if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = errorView;
|
|
603167
|
+
this.snapshot.cameras = {
|
|
603168
|
+
...this.snapshot.cameras ?? {},
|
|
603169
|
+
[source]: {
|
|
603170
|
+
...this.snapshot.cameras?.[source] ?? { source, updatedAt: observedAt2 },
|
|
603171
|
+
...errorView,
|
|
603172
|
+
updatedAt: observedAt2,
|
|
603173
|
+
source
|
|
603174
|
+
}
|
|
602532
603175
|
};
|
|
602533
603176
|
this.snapshot.events = mergeRecentById(this.snapshot.events, [{
|
|
602534
603177
|
id: stableEventId("camera_error", source, observedAt2, error.slice(0, 80)),
|
|
@@ -602539,6 +603182,14 @@ var init_live_sensors = __esm({
|
|
|
602539
603182
|
summary: error
|
|
602540
603183
|
}], 50);
|
|
602541
603184
|
this.persist();
|
|
603185
|
+
if (options2.emit !== false) {
|
|
603186
|
+
this.emitFeedbackThrottled(`camera-capture:${source}:${error.slice(0, 80)}`, {
|
|
603187
|
+
kind: "error",
|
|
603188
|
+
title: "Camera capture failed",
|
|
603189
|
+
observedAt: observedAt2,
|
|
603190
|
+
message: error
|
|
603191
|
+
}, 3e4);
|
|
603192
|
+
}
|
|
602542
603193
|
return { ok: false, message: error };
|
|
602543
603194
|
}
|
|
602544
603195
|
const framePath = extractSavedImagePath(result.output, this.repoRoot);
|
|
@@ -602547,12 +603198,25 @@ var init_live_sensors = __esm({
|
|
|
602547
603198
|
const preview = await buildImageAsciiPreview(framePath, { width: 72, height: 22 });
|
|
602548
603199
|
const asciiContext = preview ? formatImageAsciiContext(preview, displayPath) : void 0;
|
|
602549
603200
|
const observedAt = Date.now();
|
|
602550
|
-
|
|
603201
|
+
const cameraView = {
|
|
602551
603202
|
updatedAt: observedAt,
|
|
602552
603203
|
source,
|
|
602553
603204
|
framePath,
|
|
602554
603205
|
displayPath,
|
|
602555
|
-
|
|
603206
|
+
ascii: preview?.ascii,
|
|
603207
|
+
renderer: preview?.renderer,
|
|
603208
|
+
asciiContext,
|
|
603209
|
+
rotation,
|
|
603210
|
+
orientation
|
|
603211
|
+
};
|
|
603212
|
+
if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = cameraView;
|
|
603213
|
+
this.snapshot.cameras = {
|
|
603214
|
+
...this.snapshot.cameras ?? {},
|
|
603215
|
+
[source]: {
|
|
603216
|
+
...cameraView,
|
|
603217
|
+
summary: this.snapshot.cameras?.[source]?.summary,
|
|
603218
|
+
inference: this.snapshot.cameras?.[source]?.inference
|
|
603219
|
+
}
|
|
602556
603220
|
};
|
|
602557
603221
|
this.snapshot.events = mergeRecentById(this.snapshot.events, [{
|
|
602558
603222
|
id: stableEventId("camera_frame", source, observedAt, framePath),
|
|
@@ -602560,10 +603224,22 @@ var init_live_sensors = __esm({
|
|
|
602560
603224
|
observedAt,
|
|
602561
603225
|
source,
|
|
602562
603226
|
title: "Captured camera frame",
|
|
602563
|
-
summary: `frame=${framePath}${displayPath ? ` display=${displayPath}` : ""}`,
|
|
603227
|
+
summary: `frame=${framePath}${displayPath ? ` display=${displayPath}` : ""}${orientation ? ` rotation=${formatCameraRotation(orientation.rotation)}` : ""}`,
|
|
602564
603228
|
evidencePath: framePath
|
|
602565
603229
|
}], 50);
|
|
602566
603230
|
this.persist();
|
|
603231
|
+
if (options2.emit !== false) {
|
|
603232
|
+
this.emitFeedback({
|
|
603233
|
+
kind: "camera-frame",
|
|
603234
|
+
title: "Live camera frame",
|
|
603235
|
+
observedAt,
|
|
603236
|
+
message: `Captured from ${source}${orientation ? ` (${formatCameraRotation(orientation.rotation)})` : ""}: ${framePath}`,
|
|
603237
|
+
imagePath: framePath,
|
|
603238
|
+
displayPath,
|
|
603239
|
+
ascii: preview?.ascii,
|
|
603240
|
+
renderer: preview?.renderer
|
|
603241
|
+
});
|
|
603242
|
+
}
|
|
602567
603243
|
return {
|
|
602568
603244
|
ok: true,
|
|
602569
603245
|
message: `Camera frame captured from ${source}: ${framePath}`,
|
|
@@ -602573,46 +603249,144 @@ var init_live_sensors = __esm({
|
|
|
602573
603249
|
renderer: preview?.renderer
|
|
602574
603250
|
};
|
|
602575
603251
|
}
|
|
603252
|
+
async recognizeFrameObjects(framePath, source, observedAt = Date.now(), options2 = {}) {
|
|
603253
|
+
try {
|
|
603254
|
+
const result = await new VisualMemoryTool().execute({
|
|
603255
|
+
action: "recognize",
|
|
603256
|
+
image: framePath,
|
|
603257
|
+
format: "json",
|
|
603258
|
+
auto_setup: true,
|
|
603259
|
+
timeout_ms: 45e3
|
|
603260
|
+
});
|
|
603261
|
+
const summary = result.success ? boundedText(result.llmContent || result.output || "No visual-memory object matches returned.", 1400) : `CLIP/visual-memory unavailable: ${result.error || result.output || "unknown error"}`;
|
|
603262
|
+
this.snapshot.events = mergeRecentById(this.snapshot.events, [{
|
|
603263
|
+
id: stableEventId("clip_recognition", source, observedAt, framePath),
|
|
603264
|
+
kind: "clip_recognition",
|
|
603265
|
+
observedAt,
|
|
603266
|
+
source,
|
|
603267
|
+
title: result.success ? "CLIP visual-memory recognition" : "CLIP visual-memory unavailable",
|
|
603268
|
+
summary,
|
|
603269
|
+
evidencePath: framePath
|
|
603270
|
+
}], 50);
|
|
603271
|
+
this.persist();
|
|
603272
|
+
if (options2.emit !== false) {
|
|
603273
|
+
this.emitFeedback({
|
|
603274
|
+
kind: "clip",
|
|
603275
|
+
title: result.success ? "CLIP visual-memory recognition" : "CLIP visual-memory unavailable",
|
|
603276
|
+
observedAt,
|
|
603277
|
+
message: summary
|
|
603278
|
+
});
|
|
603279
|
+
}
|
|
603280
|
+
return summary;
|
|
603281
|
+
} catch (err) {
|
|
603282
|
+
const message2 = `CLIP/visual-memory failed: ${err instanceof Error ? err.message : String(err)}`;
|
|
603283
|
+
if (options2.emit !== false) {
|
|
603284
|
+
this.emitFeedbackThrottled(`clip:${source}`, {
|
|
603285
|
+
kind: "error",
|
|
603286
|
+
title: "CLIP visual-memory failed",
|
|
603287
|
+
observedAt,
|
|
603288
|
+
message: message2
|
|
603289
|
+
}, 6e4);
|
|
603290
|
+
}
|
|
603291
|
+
return message2;
|
|
603292
|
+
}
|
|
603293
|
+
}
|
|
603294
|
+
async sampleSingleVideoNow(source, forceInfer) {
|
|
603295
|
+
const preview = await this.previewCamera(source, { emit: false });
|
|
603296
|
+
let inference = "";
|
|
603297
|
+
let clipSummary = "";
|
|
603298
|
+
const orientation = this.getCameraOrientation(source);
|
|
603299
|
+
const sampledAt = Date.now();
|
|
603300
|
+
if (forceInfer && source) {
|
|
603301
|
+
const loop = new LiveMediaLoopTool(this.repoRoot);
|
|
603302
|
+
const result = await loop.execute({
|
|
603303
|
+
action: "watch",
|
|
603304
|
+
source_kind: "camera",
|
|
603305
|
+
camera: source,
|
|
603306
|
+
duration_sec: 2.5,
|
|
603307
|
+
sample_interval_sec: 1,
|
|
603308
|
+
max_frames: 2,
|
|
603309
|
+
rotate_degrees: orientation?.rotation ?? 0,
|
|
603310
|
+
auto_bootstrap: true,
|
|
603311
|
+
audio_mode: "none",
|
|
603312
|
+
detect_objects: true,
|
|
603313
|
+
track_objects: true,
|
|
603314
|
+
crop_faces: true,
|
|
603315
|
+
recognize_faces: true
|
|
603316
|
+
});
|
|
603317
|
+
inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
|
|
603318
|
+
const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
|
|
603319
|
+
const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
|
|
603320
|
+
this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
|
|
603321
|
+
this.snapshot.people = mergeRecentById(this.snapshot.people, observations.people, 24);
|
|
603322
|
+
const existing = this.snapshot.cameras?.[source] ?? this.snapshot.video ?? { updatedAt: Date.now(), source };
|
|
603323
|
+
const camera = {
|
|
603324
|
+
...existing,
|
|
603325
|
+
updatedAt: sampledAt,
|
|
603326
|
+
source,
|
|
603327
|
+
inference,
|
|
603328
|
+
summary: summarizeInference(inference),
|
|
603329
|
+
rotation: orientation?.rotation ?? 0,
|
|
603330
|
+
orientation,
|
|
603331
|
+
...preview.ok ? { error: void 0 } : { error: preview.message }
|
|
603332
|
+
};
|
|
603333
|
+
this.snapshot.cameras = {
|
|
603334
|
+
...this.snapshot.cameras ?? {},
|
|
603335
|
+
[source]: camera
|
|
603336
|
+
};
|
|
603337
|
+
if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = camera;
|
|
603338
|
+
this.persist();
|
|
603339
|
+
if (!result.success) {
|
|
603340
|
+
this.emitFeedbackThrottled(`video-infer:${source}`, {
|
|
603341
|
+
kind: "error",
|
|
603342
|
+
title: "Live video inference failed",
|
|
603343
|
+
observedAt: sampledAt,
|
|
603344
|
+
message: boundedText(inference, 1200)
|
|
603345
|
+
}, 45e3);
|
|
603346
|
+
}
|
|
603347
|
+
const unknownPeople = observations.people.filter((entry) => entry.status === "unknown");
|
|
603348
|
+
if (unknownPeople.length > 0) {
|
|
603349
|
+
this.maybeRequestAgentReview(
|
|
603350
|
+
`${unknownPeople.length} unknown individual(s) observed on live camera ${source}`,
|
|
603351
|
+
unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; ")
|
|
603352
|
+
);
|
|
603353
|
+
}
|
|
603354
|
+
const triggerEvents = observations.events.filter((entry) => entry.kind === "visual_trigger");
|
|
603355
|
+
if (triggerEvents.length > 0) {
|
|
603356
|
+
this.maybeRequestAgentReview(
|
|
603357
|
+
`Visual trigger hit in live camera stream ${source}`,
|
|
603358
|
+
triggerEvents.map((entry) => entry.summary).join("; ")
|
|
603359
|
+
);
|
|
603360
|
+
}
|
|
603361
|
+
}
|
|
603362
|
+
if (this.config.clipEnabled && preview.ok && preview.framePath && source) {
|
|
603363
|
+
clipSummary = await this.recognizeFrameObjects(preview.framePath, source, sampledAt, { emit: false });
|
|
603364
|
+
const current = this.snapshot.cameras?.[source];
|
|
603365
|
+
if (current) {
|
|
603366
|
+
current.summary = [current.summary, clipSummary ? `clip: ${trimOneLine(clipSummary, 160)}` : ""].filter(Boolean).join(" | ");
|
|
603367
|
+
this.snapshot.cameras = { ...this.snapshot.cameras ?? {}, [source]: current };
|
|
603368
|
+
if (this.snapshot.video?.source === source) this.snapshot.video = current;
|
|
603369
|
+
this.persist();
|
|
603370
|
+
}
|
|
603371
|
+
}
|
|
603372
|
+
return [preview.message, inference ? `
|
|
603373
|
+
${inference}` : "", clipSummary ? `
|
|
603374
|
+
CLIP visual memory:
|
|
603375
|
+
${clipSummary}` : ""].filter(Boolean).join("\n");
|
|
603376
|
+
}
|
|
602576
603377
|
async sampleVideoNow(forceInfer = this.config.inferEnabled) {
|
|
602577
603378
|
if (this.videoInFlight) return "Video sampler is already running.";
|
|
602578
603379
|
this.videoInFlight = true;
|
|
602579
603380
|
try {
|
|
602580
|
-
const
|
|
602581
|
-
|
|
602582
|
-
const
|
|
602583
|
-
|
|
602584
|
-
|
|
602585
|
-
const result = await loop.execute({
|
|
602586
|
-
action: "watch",
|
|
602587
|
-
source_kind: "camera",
|
|
602588
|
-
camera: source,
|
|
602589
|
-
duration_sec: 4,
|
|
602590
|
-
sample_interval_sec: 1,
|
|
602591
|
-
max_frames: 4,
|
|
602592
|
-
auto_bootstrap: true,
|
|
602593
|
-
audio_mode: "none",
|
|
602594
|
-
detect_objects: true,
|
|
602595
|
-
track_objects: true,
|
|
602596
|
-
crop_faces: true,
|
|
602597
|
-
recognize_faces: true
|
|
602598
|
-
});
|
|
602599
|
-
const sampledAt = Date.now();
|
|
602600
|
-
inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
|
|
602601
|
-
const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
|
|
602602
|
-
const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
|
|
602603
|
-
this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
|
|
602604
|
-
this.snapshot.people = mergeRecentById(this.snapshot.people, observations.people, 24);
|
|
602605
|
-
this.snapshot.video = {
|
|
602606
|
-
...this.snapshot.video ?? { updatedAt: Date.now(), source },
|
|
602607
|
-
updatedAt: sampledAt,
|
|
602608
|
-
source,
|
|
602609
|
-
inference,
|
|
602610
|
-
...preview.ok ? {} : { error: preview.message }
|
|
602611
|
-
};
|
|
602612
|
-
this.persist();
|
|
603381
|
+
const sources = this.activeVideoSources();
|
|
603382
|
+
if (sources.length === 0) return "No camera selected. Use /live refresh after connecting hardware.";
|
|
603383
|
+
const outputs = [];
|
|
603384
|
+
for (const source of sources) {
|
|
603385
|
+
outputs.push(await this.sampleSingleVideoNow(source, forceInfer));
|
|
602613
603386
|
}
|
|
602614
|
-
|
|
602615
|
-
|
|
603387
|
+
this.emitDashboard();
|
|
603388
|
+
return outputs.map((output, index) => `## Camera ${sources[index]}
|
|
603389
|
+
${output}`).join("\n\n");
|
|
602616
603390
|
} finally {
|
|
602617
603391
|
this.videoInFlight = false;
|
|
602618
603392
|
}
|
|
@@ -602627,10 +603401,9 @@ ${inference}` : ""].filter(Boolean).join("\n");
|
|
|
602627
603401
|
const recordingPath = join124(outputDir2, `audio-${Date.now()}.wav`);
|
|
602628
603402
|
let transcript = "";
|
|
602629
603403
|
let analysis = "";
|
|
602630
|
-
|
|
602631
|
-
|
|
602632
|
-
|
|
602633
|
-
const message2 = `Audio capture failed from ${input}: ${err instanceof Error ? err.message : String(err)}`;
|
|
603404
|
+
const capture = await recordAudioSampleWithRecovery(input, this.devices.audioInputs, 3, recordingPath);
|
|
603405
|
+
if (!capture.ok) {
|
|
603406
|
+
const message2 = `Audio capture failed from ${input}; tried fallbacks: ${capture.errors.slice(0, 5).join(" | ")}`;
|
|
602634
603407
|
this.snapshot.audio = {
|
|
602635
603408
|
updatedAt: Date.now(),
|
|
602636
603409
|
input,
|
|
@@ -602638,8 +603411,18 @@ ${inference}` : ""].filter(Boolean).join("\n");
|
|
|
602638
603411
|
error: message2
|
|
602639
603412
|
};
|
|
602640
603413
|
this.persist();
|
|
603414
|
+
this.emitFeedbackThrottled(`audio-capture:${input}`, {
|
|
603415
|
+
kind: "error",
|
|
603416
|
+
title: "Live audio capture failed",
|
|
603417
|
+
observedAt: this.snapshot.audio.updatedAt,
|
|
603418
|
+
message: message2
|
|
603419
|
+
}, 6e4);
|
|
603420
|
+
this.emitDashboard();
|
|
602641
603421
|
return message2;
|
|
602642
603422
|
}
|
|
603423
|
+
if (capture.device !== this.config.selectedAudioInput) {
|
|
603424
|
+
this.config.selectedAudioInput = capture.device;
|
|
603425
|
+
}
|
|
602643
603426
|
if (this.config.asrEnabled) {
|
|
602644
603427
|
try {
|
|
602645
603428
|
const asr = await new TranscribeFileTool(this.repoRoot).execute({ path: recordingPath, model: "tiny" });
|
|
@@ -602667,7 +603450,7 @@ ${inference}` : ""].filter(Boolean).join("\n");
|
|
|
602667
603450
|
}
|
|
602668
603451
|
this.snapshot.audio = {
|
|
602669
603452
|
updatedAt: Date.now(),
|
|
602670
|
-
input,
|
|
603453
|
+
input: capture.device,
|
|
602671
603454
|
output: this.config.selectedAudioOutput,
|
|
602672
603455
|
recordingPath,
|
|
602673
603456
|
transcript,
|
|
@@ -602676,10 +603459,10 @@ ${inference}` : ""].filter(Boolean).join("\n");
|
|
|
602676
603459
|
const audioEvents = [];
|
|
602677
603460
|
if (transcript) {
|
|
602678
603461
|
audioEvents.push({
|
|
602679
|
-
id: stableEventId("audio_transcript",
|
|
603462
|
+
id: stableEventId("audio_transcript", capture.device, this.snapshot.audio.updatedAt, recordingPath),
|
|
602680
603463
|
kind: "audio_transcript",
|
|
602681
603464
|
observedAt: this.snapshot.audio.updatedAt,
|
|
602682
|
-
source:
|
|
603465
|
+
source: capture.device,
|
|
602683
603466
|
title: "Live ASR transcript",
|
|
602684
603467
|
summary: transcript,
|
|
602685
603468
|
evidencePath: recordingPath
|
|
@@ -602687,10 +603470,10 @@ ${inference}` : ""].filter(Boolean).join("\n");
|
|
|
602687
603470
|
}
|
|
602688
603471
|
if (analysis) {
|
|
602689
603472
|
audioEvents.push({
|
|
602690
|
-
id: stableEventId("audio_sound",
|
|
603473
|
+
id: stableEventId("audio_sound", capture.device, this.snapshot.audio.updatedAt, recordingPath),
|
|
602691
603474
|
kind: "audio_sound",
|
|
602692
603475
|
observedAt: this.snapshot.audio.updatedAt,
|
|
602693
|
-
source:
|
|
603476
|
+
source: capture.device,
|
|
602694
603477
|
title: "Live sound analysis",
|
|
602695
603478
|
summary: analysis,
|
|
602696
603479
|
evidencePath: recordingPath
|
|
@@ -602698,8 +603481,9 @@ ${inference}` : ""].filter(Boolean).join("\n");
|
|
|
602698
603481
|
}
|
|
602699
603482
|
this.snapshot.events = mergeRecentById(this.snapshot.events, audioEvents, 50);
|
|
602700
603483
|
this.persist();
|
|
603484
|
+
this.emitDashboard();
|
|
602701
603485
|
return [
|
|
602702
|
-
`Audio sample captured from ${input}: ${recordingPath}`,
|
|
603486
|
+
`Audio sample captured from ${capture.device}${capture.device !== input ? ` (recovered from ${input})` : ""}: ${recordingPath}`,
|
|
602703
603487
|
transcript ? `
|
|
602704
603488
|
ASR:
|
|
602705
603489
|
${transcript}` : "",
|
|
@@ -602713,10 +603497,10 @@ ${analysis}` : ""
|
|
|
602713
603497
|
}
|
|
602714
603498
|
ensureLoops() {
|
|
602715
603499
|
if (this.config.videoEnabled && !this.videoTimer) {
|
|
602716
|
-
void this.sampleVideoNow(
|
|
603500
|
+
void this.sampleVideoNow(this.config.inferEnabled || this.config.clipEnabled).catch(() => {
|
|
602717
603501
|
});
|
|
602718
603502
|
this.videoTimer = setInterval(() => {
|
|
602719
|
-
void this.sampleVideoNow(this.config.inferEnabled).catch(() => {
|
|
603503
|
+
void this.sampleVideoNow(this.config.inferEnabled || this.config.clipEnabled).catch(() => {
|
|
602720
603504
|
});
|
|
602721
603505
|
}, this.config.videoIntervalMs);
|
|
602722
603506
|
this.videoTimer.unref?.();
|
|
@@ -602745,6 +603529,53 @@ ${analysis}` : ""
|
|
|
602745
603529
|
label: "live"
|
|
602746
603530
|
});
|
|
602747
603531
|
}
|
|
603532
|
+
emitFeedback(feedback) {
|
|
603533
|
+
try {
|
|
603534
|
+
this.feedbackSink?.(feedback);
|
|
603535
|
+
} catch {
|
|
603536
|
+
}
|
|
603537
|
+
}
|
|
603538
|
+
emitFeedbackThrottled(key, feedback, minIntervalMs) {
|
|
603539
|
+
const now2 = Date.now();
|
|
603540
|
+
const last2 = this.lastFeedbackAt.get(key) ?? 0;
|
|
603541
|
+
if (now2 - last2 < minIntervalMs) return;
|
|
603542
|
+
this.lastFeedbackAt.set(key, now2);
|
|
603543
|
+
this.emitFeedback(feedback);
|
|
603544
|
+
}
|
|
603545
|
+
emitDashboard() {
|
|
603546
|
+
this.emitFeedback({
|
|
603547
|
+
kind: "dashboard",
|
|
603548
|
+
title: "Live audio/video monitor",
|
|
603549
|
+
observedAt: Date.now(),
|
|
603550
|
+
message: "live dashboard refresh"
|
|
603551
|
+
});
|
|
603552
|
+
}
|
|
603553
|
+
maybeRequestAgentReview(reason, evidence) {
|
|
603554
|
+
if (!this.agentActionEnabled || !this.agentActionSink) return;
|
|
603555
|
+
const now2 = Date.now();
|
|
603556
|
+
if (now2 - this.lastAgentReviewAt < 2e4) return;
|
|
603557
|
+
this.lastAgentReviewAt = now2;
|
|
603558
|
+
const prompt = [
|
|
603559
|
+
"Live environment PFC review.",
|
|
603560
|
+
"Inspect the current live sensor evidence and decide whether any immediate action is needed.",
|
|
603561
|
+
"Do not invent person names. If a person is unknown and identity matters, ask the user or person for their name and then store the association with visual_memory or multimodal_memory.",
|
|
603562
|
+
"Prefer minimal, reversible actions. If no action is needed, summarize the observation and stand by.",
|
|
603563
|
+
"",
|
|
603564
|
+
`Reason: ${reason}`,
|
|
603565
|
+
`Evidence: ${evidence}`,
|
|
603566
|
+
`Live snapshot: ${liveSnapshotPath(this.repoRoot)}`
|
|
603567
|
+
].join("\n");
|
|
603568
|
+
this.emitFeedback({
|
|
603569
|
+
kind: "agent-review",
|
|
603570
|
+
title: "Live PFC review queued",
|
|
603571
|
+
observedAt: now2,
|
|
603572
|
+
message: reason
|
|
603573
|
+
});
|
|
603574
|
+
try {
|
|
603575
|
+
this.agentActionSink(prompt);
|
|
603576
|
+
} catch {
|
|
603577
|
+
}
|
|
603578
|
+
}
|
|
602748
603579
|
persist() {
|
|
602749
603580
|
ensureLiveDir(this.repoRoot);
|
|
602750
603581
|
writeFileSync56(liveConfigPath(this.repoRoot), JSON.stringify(this.config, null, 2), "utf8");
|
|
@@ -608474,8 +609305,13 @@ var init_command_registry = __esm({
|
|
|
608474
609305
|
["/listen stop", "Stop listening"],
|
|
608475
609306
|
["/live", "Open live sensor stream menu for video/audio context"],
|
|
608476
609307
|
["/live status", "Show live sensor devices, selected streams, and latest snapshot"],
|
|
609308
|
+
["/live run [agent]", "Start low-latency video/audio feedback; agent enables background PFC review"],
|
|
609309
|
+
["/live chat", "Start low-latency voicechat with live audio/video context and async agent review"],
|
|
609310
|
+
["/livechat", "Alias for /live chat, used by the top live button"],
|
|
608477
609311
|
["/live camera <device>", "Select a camera and render an ASCII preview"],
|
|
609312
|
+
["/live rotate auto|cw|ccw|180|none [camera]", "Detect or set persistent camera orientation correction"],
|
|
608478
609313
|
["/live infer [now|on|off]", "Toggle or run camera object/location inference"],
|
|
609314
|
+
["/live clip [on|off]", "Toggle CLIP visual-memory recognition on live frames"],
|
|
608479
609315
|
["/live audio|asr|sounds on|off", "Toggle live audio context streams"],
|
|
608480
609316
|
["/live stop", "Disable all live sensor streams"],
|
|
608481
609317
|
["/paste", "Attach clipboard image content to the current or next prompt"],
|
|
@@ -618863,7 +619699,7 @@ __export(omnius_directory_exports, {
|
|
|
618863
619699
|
writeIndexMeta: () => writeIndexMeta,
|
|
618864
619700
|
writeTaskHandoff: () => writeTaskHandoff2
|
|
618865
619701
|
});
|
|
618866
|
-
import { appendFileSync as appendFileSync13, cpSync as cpSync2, existsSync as existsSync117, mkdirSync as mkdirSync74, readFileSync as readFileSync95, writeFileSync as writeFileSync63, readdirSync as readdirSync40, statSync as statSync44, unlinkSync as unlinkSync22, openSync as openSync2, closeSync as closeSync2, renameSync as
|
|
619702
|
+
import { appendFileSync as appendFileSync13, cpSync as cpSync2, existsSync as existsSync117, mkdirSync as mkdirSync74, readFileSync as readFileSync95, writeFileSync as writeFileSync63, readdirSync as readdirSync40, statSync as statSync44, unlinkSync as unlinkSync22, openSync as openSync2, closeSync as closeSync2, renameSync as renameSync11, watch as fsWatch2 } from "node:fs";
|
|
618867
619703
|
import { join as join133, relative as relative13, basename as basename25, dirname as dirname40, resolve as resolve57 } from "node:path";
|
|
618868
619704
|
import { homedir as homedir41 } from "node:os";
|
|
618869
619705
|
import { createHash as createHash37 } from "node:crypto";
|
|
@@ -619269,7 +620105,7 @@ function writeTaskHandoff2(repoRoot, handoff) {
|
|
|
619269
620105
|
const tempPath = filePath + ".tmp";
|
|
619270
620106
|
writeFileSync63(tempPath, JSON.stringify(handoff, null, 2) + "\n", "utf-8");
|
|
619271
620107
|
try {
|
|
619272
|
-
|
|
620108
|
+
renameSync11(tempPath, filePath);
|
|
619273
620109
|
} catch {
|
|
619274
620110
|
writeFileSync63(filePath, JSON.stringify(handoff, null, 2) + "\n", "utf-8");
|
|
619275
620111
|
try {
|
|
@@ -619579,7 +620415,7 @@ function saveSessionContext(repoRoot, entry) {
|
|
|
619579
620415
|
const tempFilePath = filePath + ".tmp";
|
|
619580
620416
|
writeFileSync63(tempFilePath, JSON.stringify(ctx3, null, 2) + "\n", "utf-8");
|
|
619581
620417
|
try {
|
|
619582
|
-
|
|
620418
|
+
renameSync11(tempFilePath, filePath);
|
|
619583
620419
|
} catch {
|
|
619584
620420
|
writeFileSync63(filePath, JSON.stringify(ctx3, null, 2) + "\n", "utf-8");
|
|
619585
620421
|
try {
|
|
@@ -622522,7 +623358,7 @@ function setTerminalTitle(task, version5) {
|
|
|
622522
623358
|
process.stdout.write(data);
|
|
622523
623359
|
}
|
|
622524
623360
|
}
|
|
622525
|
-
var EXPERT_TOOL_BASELINES, CONTEXT_SWITCH_OVERHEAD, TURN_PLANNING_OVERHEAD, DEFAULT_TOOL_BASELINE, CODE_READ_CHARS_PER_SEC, PROSE_READ_CHARS_PER_SEC, MIN_CONTENT_FOR_READING, CODE_CONTENT_TOOLS, PROSE_CONTENT_TOOLS, HumanSpeedTracker, PANEL_BG_SEQ, CONTENT_BG_SEQ, BOX_FG, TEXT_PRIMARY, TEXT_DIM, NO_SUB_AGENTS_HEADER_LABEL, HEADER_BUTTON_GLYPH_FG, HEADER_BUTTON_BG, HEADER_BUTTON_FG, HEADER_ACCENT_BOLD_FG, HEADER_TELEGRAM_FG, BOX_TL3, BOX_TR3, BOX_BL3, BOX_BR3, BOX_H3, BOX_V3, BOX_BJ, _globalFooterLock, RESET4, CURSOR_BLINK_BLOCK, _isWindows, SPONSOR_HEADER_LABEL_MAX, _termTitleWriter, StatusBar;
|
|
623361
|
+
var EXPERT_TOOL_BASELINES, CONTEXT_SWITCH_OVERHEAD, TURN_PLANNING_OVERHEAD, DEFAULT_TOOL_BASELINE, CODE_READ_CHARS_PER_SEC, PROSE_READ_CHARS_PER_SEC, MIN_CONTENT_FOR_READING, CODE_CONTENT_TOOLS, PROSE_CONTENT_TOOLS, HumanSpeedTracker, PANEL_BG_SEQ, CONTENT_BG_SEQ, BOX_FG, TEXT_PRIMARY, TEXT_DIM, NO_SUB_AGENTS_HEADER_LABEL, HEADER_BUTTON_GLYPH_FG, HEADER_BUTTON_BG, HEADER_BUTTON_FG, HEADER_ACCENT_BOLD_FG, HEADER_TELEGRAM_FG, BOX_TL3, BOX_TR3, BOX_BL3, BOX_BR3, BOX_H3, BOX_V3, BOX_BJ, _globalFooterLock, RESET4, CURSOR_BLINK_BLOCK, _isWindows, HEADER_BUTTON_LEFT, HEADER_BUTTON_RIGHT, SPONSOR_HEADER_LABEL_MAX, _termTitleWriter, StatusBar;
|
|
622526
623362
|
var init_status_bar = __esm({
|
|
622527
623363
|
"packages/cli/src/tui/status-bar.ts"() {
|
|
622528
623364
|
"use strict";
|
|
@@ -622719,6 +623555,8 @@ var init_status_bar = __esm({
|
|
|
622719
623555
|
RESET4 = "\x1B[0m";
|
|
622720
623556
|
CURSOR_BLINK_BLOCK = "\x1B[1 q";
|
|
622721
623557
|
_isWindows = process.platform === "win32";
|
|
623558
|
+
HEADER_BUTTON_LEFT = _isWindows ? "[" : "🭁";
|
|
623559
|
+
HEADER_BUTTON_RIGHT = _isWindows ? "]" : "🭝";
|
|
622722
623560
|
SPONSOR_HEADER_LABEL_MAX = 48;
|
|
622723
623561
|
_termTitleWriter = null;
|
|
622724
623562
|
StatusBar = class _StatusBar {
|
|
@@ -623267,27 +624105,30 @@ var init_status_bar = __esm({
|
|
|
623267
624105
|
const buttonFg = (cmd) => {
|
|
623268
624106
|
let fg2 = TEXT_DIM;
|
|
623269
624107
|
if (cmd === "voice" && this._voiceActive) fg2 = 82;
|
|
623270
|
-
if (cmd
|
|
624108
|
+
if (cmd.startsWith("live") && (this._liveMediaStatus.audio || this._liveMediaStatus.video)) fg2 = 82;
|
|
623271
624109
|
if (cmd === "nexus") {
|
|
623272
624110
|
fg2 = this._nexusStatus === "connected" ? 82 : this._nexusStatus === "connecting" ? 208 : 196;
|
|
623273
624111
|
}
|
|
623274
624112
|
return fg2;
|
|
623275
624113
|
};
|
|
623276
624114
|
const decorateMenuButton = (cmd, label) => {
|
|
623277
|
-
return `${HEADER_BUTTON_GLYPH_FG}
|
|
624115
|
+
return `${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_LEFT}\x1B[0m${HEADER_BUTTON_BG}${HEADER_BUTTON_FG}${label}\x1B[0m${PANEL_BG_SEQ}${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_RIGHT}\x1B[0m${PANEL_BG_SEQ}`;
|
|
623278
624116
|
};
|
|
623279
624117
|
const decorateAgentButton = (content, color, active) => {
|
|
623280
624118
|
const bg = `\x1B[48;5;${color}m`;
|
|
623281
624119
|
const fg2 = `\x1B[38;5;${contrastTextColor(color)}m`;
|
|
623282
624120
|
const weight = active ? "\x1B[1m" : "";
|
|
623283
|
-
return `\x1B[38;5;${color}m
|
|
624121
|
+
return `\x1B[38;5;${color}m${HEADER_BUTTON_LEFT}\x1B[0m${weight}${fg2}${bg}${content}\x1B[0m${PANEL_BG_SEQ}\x1B[38;5;${color}m${HEADER_BUTTON_RIGHT}\x1B[0m${PANEL_BG_SEQ}`;
|
|
623284
624122
|
};
|
|
623285
624123
|
const renderBtn = (cmd, label) => {
|
|
623286
624124
|
const fg2 = buttonFg(cmd);
|
|
623287
|
-
const btnW = label.length;
|
|
624125
|
+
const btnW = label.length + 2;
|
|
623288
624126
|
const availW2 = getTermWidth() - identity3.width - 1;
|
|
623289
624127
|
if (btnW > availW2) return linkify(cmd, `\x1B[38;5;${fg2}m${label}`);
|
|
623290
|
-
return linkify(
|
|
624128
|
+
return linkify(
|
|
624129
|
+
cmd,
|
|
624130
|
+
`${PANEL_BG_SEQ}${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_LEFT}\x1B[0m${PANEL_BG_SEQ}\x1B[38;5;${fg2}m${label}\x1B[0m${PANEL_BG_SEQ}${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_RIGHT}\x1B[0m${PANEL_BG_SEQ}`
|
|
624131
|
+
);
|
|
623291
624132
|
};
|
|
623292
624133
|
const identity3 = this.buildHeaderIdentityRender();
|
|
623293
624134
|
const modelLabel = this.summarizeHeaderModelName() || "model";
|
|
@@ -623301,6 +624142,8 @@ var init_status_bar = __esm({
|
|
|
623301
624142
|
w: (this._voiceActive ? this._voiceModelId || "voice" : "voice").length + 2
|
|
623302
624143
|
},
|
|
623303
624144
|
// +2 for spaces
|
|
624145
|
+
{ cmd: "livechat", label: "live", w: "live".length + 2 },
|
|
624146
|
+
// +2 for spaces
|
|
623304
624147
|
{ cmd: "model", label: modelLabel, w: modelLabel.length + 2 },
|
|
623305
624148
|
// +2 for spaces
|
|
623306
624149
|
{ cmd: "endpoint", label: endpointLabel, w: endpointLabel.length + 2 }
|
|
@@ -648807,6 +649650,10 @@ sleep 1
|
|
|
648807
649650
|
await handleLiveCommand(ctx3, arg);
|
|
648808
649651
|
return "handled";
|
|
648809
649652
|
}
|
|
649653
|
+
case "livechat": {
|
|
649654
|
+
await handleLiveCommand(ctx3, "chat");
|
|
649655
|
+
return "handled";
|
|
649656
|
+
}
|
|
648810
649657
|
case "call": {
|
|
648811
649658
|
if (!ctx3.callStart) {
|
|
648812
649659
|
renderWarning("Call mode not available in this context.");
|
|
@@ -654417,6 +655264,85 @@ function renderLivePreview(preview) {
|
|
|
654417
655264
|
renderWarning("Camera preview was captured, but ASCII rendering was unavailable.");
|
|
654418
655265
|
}
|
|
654419
655266
|
}
|
|
655267
|
+
function renderLiveDashboard(ctx3, manager) {
|
|
655268
|
+
const host = ctx3.liveDynamicBlockHost;
|
|
655269
|
+
if (!host) {
|
|
655270
|
+
renderInfo(formatLiveDashboardFromSnapshot(manager.getSnapshot(), { width: 100 }).join("\n"));
|
|
655271
|
+
return;
|
|
655272
|
+
}
|
|
655273
|
+
const key = ctx3.repoRoot;
|
|
655274
|
+
let state = liveDashboardBlocks.get(key);
|
|
655275
|
+
if (!state || state.host !== host) {
|
|
655276
|
+
const id2 = `live-dashboard-${Math.random().toString(36).slice(2, 9)}`;
|
|
655277
|
+
state = { host, id: id2, manager };
|
|
655278
|
+
liveDashboardBlocks.set(key, state);
|
|
655279
|
+
host.registerDynamicBlock(
|
|
655280
|
+
id2,
|
|
655281
|
+
(width) => formatLiveDashboardFromSnapshot(manager.getSnapshot(), { width })
|
|
655282
|
+
);
|
|
655283
|
+
host.appendDynamicBlock(id2);
|
|
655284
|
+
return;
|
|
655285
|
+
}
|
|
655286
|
+
state.manager = manager;
|
|
655287
|
+
host.refreshDynamicBlocks?.();
|
|
655288
|
+
}
|
|
655289
|
+
function renderLiveFeedback(ctx3, manager, feedback) {
|
|
655290
|
+
if (feedback.kind === "dashboard") {
|
|
655291
|
+
renderLiveDashboard(ctx3, manager);
|
|
655292
|
+
return;
|
|
655293
|
+
}
|
|
655294
|
+
const stamp = new Date(feedback.observedAt).toISOString();
|
|
655295
|
+
if (feedback.kind === "error") {
|
|
655296
|
+
renderWarning(`[live ${stamp}] ${feedback.title}
|
|
655297
|
+
${feedback.message}`);
|
|
655298
|
+
return;
|
|
655299
|
+
}
|
|
655300
|
+
if ((feedback.kind === "camera-frame" || feedback.kind === "face-crop") && feedback.ascii && feedback.displayPath && feedback.renderer) {
|
|
655301
|
+
renderInfo(`[live ${stamp}] ${feedback.message}`);
|
|
655302
|
+
renderImageAsciiPreview(feedback.title, feedback.displayPath, feedback.ascii, feedback.renderer);
|
|
655303
|
+
return;
|
|
655304
|
+
}
|
|
655305
|
+
renderInfo(`[live ${stamp}] ${feedback.title}
|
|
655306
|
+
${feedback.message}`);
|
|
655307
|
+
}
|
|
655308
|
+
function attachLiveSinks(ctx3, manager, agentEnabled = false) {
|
|
655309
|
+
manager.setStatusSink(ctx3.setLiveMediaStatus);
|
|
655310
|
+
manager.setFeedbackSink((feedback) => renderLiveFeedback(ctx3, manager, feedback));
|
|
655311
|
+
if (agentEnabled && ctx3.startBackgroundPrompt) {
|
|
655312
|
+
manager.setAgentActionSink((prompt) => {
|
|
655313
|
+
const id2 = ctx3.startBackgroundPrompt?.(prompt);
|
|
655314
|
+
if (id2) renderInfo(`Live PFC background review started: ${id2}`);
|
|
655315
|
+
});
|
|
655316
|
+
} else {
|
|
655317
|
+
manager.setAgentActionSink(void 0);
|
|
655318
|
+
}
|
|
655319
|
+
}
|
|
655320
|
+
async function startLiveVoicechat(ctx3, manager) {
|
|
655321
|
+
const agentEnabled = Boolean(ctx3.startBackgroundPrompt);
|
|
655322
|
+
attachLiveSinks(ctx3, manager, agentEnabled);
|
|
655323
|
+
await ensureLiveInventory(manager);
|
|
655324
|
+
renderInfo(manager.startRunMode({ agent: agentEnabled, lowLatency: true }));
|
|
655325
|
+
if (!ctx3.voiceChatStart) {
|
|
655326
|
+
renderWarning("Voice chat is not available in this context; live audio/video feedback remains active.");
|
|
655327
|
+
return;
|
|
655328
|
+
}
|
|
655329
|
+
ctx3.voiceSetMode?.("voicechat");
|
|
655330
|
+
if (ctx3.isVoiceChatActive?.()) {
|
|
655331
|
+
renderInfo("Voice chat is already active. Live audio/video feedback remains attached.");
|
|
655332
|
+
return;
|
|
655333
|
+
}
|
|
655334
|
+
try {
|
|
655335
|
+
await ensureVoiceDeps(ctx3);
|
|
655336
|
+
} catch {
|
|
655337
|
+
}
|
|
655338
|
+
try {
|
|
655339
|
+
renderInfo("Starting live voicechat with audio/video context...");
|
|
655340
|
+
await ctx3.voiceChatStart();
|
|
655341
|
+
renderInfo("Live voicechat active: low-latency audio/video context is updating while voice conversation continues.");
|
|
655342
|
+
} catch (err) {
|
|
655343
|
+
renderError(`Live voicechat failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
655344
|
+
}
|
|
655345
|
+
}
|
|
654420
655346
|
async function chooseLiveDevice(ctx3, title, devices, activeKey) {
|
|
654421
655347
|
if (devices.length === 0) {
|
|
654422
655348
|
renderWarning("No devices found. Run /live refresh after connecting hardware.");
|
|
@@ -654439,7 +655365,7 @@ async function chooseLiveDevice(ctx3, title, devices, activeKey) {
|
|
|
654439
655365
|
}
|
|
654440
655366
|
async function handleLiveCommand(ctx3, arg) {
|
|
654441
655367
|
const manager = getLiveSensorManager(ctx3.repoRoot);
|
|
654442
|
-
|
|
655368
|
+
attachLiveSinks(ctx3, manager);
|
|
654443
655369
|
const parts = arg.trim().split(/\s+/).filter(Boolean);
|
|
654444
655370
|
const sub2 = (parts[0] || "").toLowerCase();
|
|
654445
655371
|
const value2 = parts.slice(1).join(" ");
|
|
@@ -654460,9 +655386,48 @@ async function handleLiveCommand(ctx3, arg) {
|
|
|
654460
655386
|
}
|
|
654461
655387
|
if (sub2 === "stop" || sub2 === "off" || sub2 === "disable") {
|
|
654462
655388
|
manager.stopAll();
|
|
655389
|
+
manager.setAgentActionSink(void 0);
|
|
654463
655390
|
renderInfo("Live sensor streams disabled.");
|
|
654464
655391
|
return;
|
|
654465
655392
|
}
|
|
655393
|
+
if (sub2 === "run" || sub2 === "start" || sub2 === "pfc") {
|
|
655394
|
+
const agentEnabled = sub2 === "pfc" || /\bagent|pfc|action\b/i.test(value2);
|
|
655395
|
+
const effectiveAgent = agentEnabled && Boolean(ctx3.startBackgroundPrompt);
|
|
655396
|
+
attachLiveSinks(ctx3, manager, effectiveAgent);
|
|
655397
|
+
if (agentEnabled && !ctx3.startBackgroundPrompt) {
|
|
655398
|
+
renderWarning("Live PFC agent review is unavailable in this context; starting visual/audio feedback only.");
|
|
655399
|
+
}
|
|
655400
|
+
await ensureLiveInventory(manager);
|
|
655401
|
+
renderInfo(manager.startRunMode({ agent: effectiveAgent, lowLatency: true }));
|
|
655402
|
+
return;
|
|
655403
|
+
}
|
|
655404
|
+
if (sub2 === "chat" || sub2 === "voicechat" || sub2 === "voice") {
|
|
655405
|
+
await startLiveVoicechat(ctx3, manager);
|
|
655406
|
+
return;
|
|
655407
|
+
}
|
|
655408
|
+
if (sub2 === "rotate" || sub2 === "rotation" || sub2 === "orient" || sub2 === "orientation") {
|
|
655409
|
+
await ensureLiveInventory(manager);
|
|
655410
|
+
const mode = (parts[1] || "cycle").toLowerCase();
|
|
655411
|
+
const deviceArg = parts.slice(2).join(" ") || manager.getConfig().selectedCamera;
|
|
655412
|
+
if (mode === "auto" || mode === "detect") {
|
|
655413
|
+
renderInfo("Detecting camera orientation...");
|
|
655414
|
+
renderInfo(await manager.autoOrientCamera(deviceArg));
|
|
655415
|
+
return;
|
|
655416
|
+
}
|
|
655417
|
+
if (mode === "cycle") {
|
|
655418
|
+
const orientation = manager.cycleCameraRotation(deviceArg);
|
|
655419
|
+
renderInfo(orientation ? `Camera rotation set to ${formatCameraRotation(orientation.rotation)} for ${deviceArg || manager.getConfig().selectedCamera || "selected camera"}.` : "No camera selected. Use /live camera <device> first.");
|
|
655420
|
+
return;
|
|
655421
|
+
}
|
|
655422
|
+
try {
|
|
655423
|
+
const rotation = normalizeLiveCameraRotation(mode);
|
|
655424
|
+
const orientation = manager.setCameraRotation(deviceArg, rotation, "manual", `/live rotate ${mode}`);
|
|
655425
|
+
renderInfo(orientation ? `Camera rotation set to ${formatCameraRotation(orientation.rotation)} for ${deviceArg || manager.getConfig().selectedCamera || "selected camera"}.` : "No camera selected. Use /live camera <device> first.");
|
|
655426
|
+
} catch (err) {
|
|
655427
|
+
renderWarning(err instanceof Error ? err.message : String(err));
|
|
655428
|
+
}
|
|
655429
|
+
return;
|
|
655430
|
+
}
|
|
654466
655431
|
if (sub2 === "video") {
|
|
654467
655432
|
const cfg = manager.getConfig();
|
|
654468
655433
|
manager.configure({ videoEnabled: setBool(value2, !cfg.videoEnabled) });
|
|
@@ -654508,6 +655473,15 @@ async function handleLiveCommand(ctx3, arg) {
|
|
|
654508
655473
|
}
|
|
654509
655474
|
return;
|
|
654510
655475
|
}
|
|
655476
|
+
if (sub2 === "clip") {
|
|
655477
|
+
const cfg = manager.getConfig();
|
|
655478
|
+
manager.configure({
|
|
655479
|
+
videoEnabled: true,
|
|
655480
|
+
clipEnabled: setBool(value2, !cfg.clipEnabled)
|
|
655481
|
+
});
|
|
655482
|
+
renderInfo(formatLiveStatus(manager.getSnapshot()));
|
|
655483
|
+
return;
|
|
655484
|
+
}
|
|
654511
655485
|
if (sub2 === "camera") {
|
|
654512
655486
|
await ensureLiveInventory(manager);
|
|
654513
655487
|
const selected = value2 || manager.getConfig().selectedCamera;
|
|
@@ -654551,9 +655525,24 @@ async function showLiveMenu(ctx3, manager) {
|
|
|
654551
655525
|
{
|
|
654552
655526
|
key: "info:status",
|
|
654553
655527
|
label: selectColors.dim(
|
|
654554
|
-
` video ${liveEnabled(cfg.videoEnabled)} · infer ${liveEnabled(cfg.inferEnabled)} · audio ${liveEnabled(cfg.audioEnabled)} · asr ${liveEnabled(cfg.asrEnabled)} · sounds ${liveEnabled(cfg.audioAnalysisEnabled)}`
|
|
655528
|
+
` video ${liveEnabled(cfg.videoEnabled)} · infer ${liveEnabled(cfg.inferEnabled)} · clip ${liveEnabled(cfg.clipEnabled)} · audio ${liveEnabled(cfg.audioEnabled)} · asr ${liveEnabled(cfg.asrEnabled)} · sounds ${liveEnabled(cfg.audioAnalysisEnabled)}`
|
|
654555
655529
|
)
|
|
654556
655530
|
},
|
|
655531
|
+
{
|
|
655532
|
+
key: "run-live",
|
|
655533
|
+
label: "Start Live Run",
|
|
655534
|
+
detail: "low-latency visual/audio feedback + context injection"
|
|
655535
|
+
},
|
|
655536
|
+
{
|
|
655537
|
+
key: "run-agent",
|
|
655538
|
+
label: "Start Live PFC Run",
|
|
655539
|
+
detail: "live feedback + background agent review for significant events"
|
|
655540
|
+
},
|
|
655541
|
+
{
|
|
655542
|
+
key: "run-chat",
|
|
655543
|
+
label: "Start Live Voicechat",
|
|
655544
|
+
detail: "low-latency voice conversation with live audio/video context"
|
|
655545
|
+
},
|
|
654557
655546
|
{
|
|
654558
655547
|
key: "toggle-video",
|
|
654559
655548
|
label: cfg.videoEnabled ? "Disable Video Context" : "Enable Video Context",
|
|
@@ -654569,11 +655558,26 @@ async function showLiveMenu(ctx3, manager) {
|
|
|
654569
655558
|
label: "View Selected Camera",
|
|
654570
655559
|
detail: "capture frame + image-to-ascii preview"
|
|
654571
655560
|
},
|
|
655561
|
+
{
|
|
655562
|
+
key: "auto-orient-camera",
|
|
655563
|
+
label: "Auto-Orient Camera",
|
|
655564
|
+
detail: `current: ${cfg.selectedCamera ? formatCameraRotation(manager.getCameraRotation(cfg.selectedCamera)) : "no camera selected"}`
|
|
655565
|
+
},
|
|
655566
|
+
{
|
|
655567
|
+
key: "cycle-rotation",
|
|
655568
|
+
label: "Cycle Camera Rotation",
|
|
655569
|
+
detail: "manual override: upright -> clockwise -> 180 -> counter-clockwise"
|
|
655570
|
+
},
|
|
654572
655571
|
{
|
|
654573
655572
|
key: "toggle-infer",
|
|
654574
655573
|
label: cfg.inferEnabled ? "Disable Frame Inference" : "Enable Frame Inference",
|
|
654575
655574
|
detail: "YOLO object/location stream into context"
|
|
654576
655575
|
},
|
|
655576
|
+
{
|
|
655577
|
+
key: "toggle-clip",
|
|
655578
|
+
label: cfg.clipEnabled ? "Disable CLIP Visual Memory" : "Enable CLIP Visual Memory",
|
|
655579
|
+
detail: "recognize taught objects from camera frames"
|
|
655580
|
+
},
|
|
654577
655581
|
{
|
|
654578
655582
|
key: "infer-now",
|
|
654579
655583
|
label: "Infer Current Camera Frame",
|
|
@@ -654647,6 +655651,20 @@ async function showLiveMenu(ctx3, manager) {
|
|
|
654647
655651
|
case "toggle-video":
|
|
654648
655652
|
manager.configure({ videoEnabled: !cfg.videoEnabled });
|
|
654649
655653
|
continue;
|
|
655654
|
+
case "run-live":
|
|
655655
|
+
attachLiveSinks(ctx3, manager, false);
|
|
655656
|
+
renderInfo(manager.startRunMode({ agent: false, lowLatency: true }));
|
|
655657
|
+
continue;
|
|
655658
|
+
case "run-agent":
|
|
655659
|
+
attachLiveSinks(ctx3, manager, true);
|
|
655660
|
+
if (!ctx3.startBackgroundPrompt) {
|
|
655661
|
+
renderWarning("Live PFC agent review is unavailable in this context; starting visual/audio feedback only.");
|
|
655662
|
+
}
|
|
655663
|
+
renderInfo(manager.startRunMode({ agent: Boolean(ctx3.startBackgroundPrompt), lowLatency: true }));
|
|
655664
|
+
continue;
|
|
655665
|
+
case "run-chat":
|
|
655666
|
+
await startLiveVoicechat(ctx3, manager);
|
|
655667
|
+
continue;
|
|
654650
655668
|
case "toggle-audio":
|
|
654651
655669
|
manager.configure({ audioEnabled: !cfg.audioEnabled });
|
|
654652
655670
|
continue;
|
|
@@ -654656,6 +655674,9 @@ async function showLiveMenu(ctx3, manager) {
|
|
|
654656
655674
|
case "toggle-infer":
|
|
654657
655675
|
manager.configure({ videoEnabled: true, inferEnabled: !cfg.inferEnabled });
|
|
654658
655676
|
continue;
|
|
655677
|
+
case "toggle-clip":
|
|
655678
|
+
manager.configure({ videoEnabled: true, clipEnabled: !cfg.clipEnabled });
|
|
655679
|
+
continue;
|
|
654659
655680
|
case "toggle-asr":
|
|
654660
655681
|
manager.configure({ audioEnabled: true, asrEnabled: !cfg.asrEnabled });
|
|
654661
655682
|
continue;
|
|
@@ -654683,6 +655704,15 @@ async function showLiveMenu(ctx3, manager) {
|
|
|
654683
655704
|
case "preview-camera":
|
|
654684
655705
|
renderLivePreview(await manager.previewCamera());
|
|
654685
655706
|
continue;
|
|
655707
|
+
case "auto-orient-camera":
|
|
655708
|
+
renderInfo("Detecting camera orientation...");
|
|
655709
|
+
renderInfo(await manager.autoOrientCamera(cfg.selectedCamera));
|
|
655710
|
+
continue;
|
|
655711
|
+
case "cycle-rotation": {
|
|
655712
|
+
const orientation = manager.cycleCameraRotation(cfg.selectedCamera);
|
|
655713
|
+
renderInfo(orientation ? `Camera rotation set to ${formatCameraRotation(orientation.rotation)} for ${cfg.selectedCamera || manager.getConfig().selectedCamera || "selected camera"}.` : "No camera selected. Use Select Camera first.");
|
|
655714
|
+
continue;
|
|
655715
|
+
}
|
|
654686
655716
|
case "infer-now":
|
|
654687
655717
|
manager.configure({ videoEnabled: true, inferEnabled: true });
|
|
654688
655718
|
renderInfo("Sampling camera inference...");
|
|
@@ -659775,7 +660805,7 @@ async function showExposeDashboard(gateway, rl, ctx3) {
|
|
|
659775
660805
|
renderInfo("Expose gateway stopped.");
|
|
659776
660806
|
}
|
|
659777
660807
|
}
|
|
659778
|
-
var OMNIUS_PINNED_DEPENDENCY_SPECS, NEXUS_DIRECTORY_ORIGIN, NEXUS_SPONSORS_URL, _sponsorHeartbeatTimer, _lastRegisteredSponsorPayload, __COMMAND_REGISTRY, DASH_INTERNAL, localGpuMetricsCache, localGpuMetricsProbeAt, localGpuMetricsProbeInFlight;
|
|
660808
|
+
var OMNIUS_PINNED_DEPENDENCY_SPECS, NEXUS_DIRECTORY_ORIGIN, NEXUS_SPONSORS_URL, _sponsorHeartbeatTimer, _lastRegisteredSponsorPayload, __COMMAND_REGISTRY, liveDashboardBlocks, DASH_INTERNAL, localGpuMetricsCache, localGpuMetricsProbeAt, localGpuMetricsProbeInFlight;
|
|
659779
660809
|
var init_commands = __esm({
|
|
659780
660810
|
"packages/cli/src/tui/commands.ts"() {
|
|
659781
660811
|
"use strict";
|
|
@@ -660093,6 +661123,7 @@ var init_commands = __esm({
|
|
|
660093
661123
|
}
|
|
660094
661124
|
});
|
|
660095
661125
|
})();
|
|
661126
|
+
liveDashboardBlocks = /* @__PURE__ */ new Map();
|
|
660096
661127
|
DASH_INTERNAL = /* @__PURE__ */ new Set(["system_metrics", "__list_capabilities"]);
|
|
660097
661128
|
localGpuMetricsCache = null;
|
|
660098
661129
|
localGpuMetricsProbeAt = 0;
|
|
@@ -661333,7 +662364,7 @@ import {
|
|
|
661333
662364
|
readFileSync as readFileSync113,
|
|
661334
662365
|
readdirSync as readdirSync49,
|
|
661335
662366
|
writeFileSync as writeFileSync74,
|
|
661336
|
-
renameSync as
|
|
662367
|
+
renameSync as renameSync13,
|
|
661337
662368
|
mkdirSync as mkdirSync85,
|
|
661338
662369
|
unlinkSync as unlinkSync29,
|
|
661339
662370
|
appendFileSync as appendFileSync16
|
|
@@ -661357,7 +662388,7 @@ function persistSession(s2) {
|
|
|
661357
662388
|
const final2 = sessionPath(s2.id);
|
|
661358
662389
|
const tmp = `${final2}.tmp.${process.pid}.${Date.now()}`;
|
|
661359
662390
|
writeFileSync74(tmp, JSON.stringify(s2, null, 2), "utf-8");
|
|
661360
|
-
|
|
662391
|
+
renameSync13(tmp, final2);
|
|
661361
662392
|
} catch {
|
|
661362
662393
|
}
|
|
661363
662394
|
}
|
|
@@ -661367,7 +662398,7 @@ function persistInFlight(j) {
|
|
|
661367
662398
|
const final2 = inFlightPath(j.sessionId);
|
|
661368
662399
|
const tmp = `${final2}.tmp.${process.pid}.${Date.now()}`;
|
|
661369
662400
|
writeFileSync74(tmp, JSON.stringify(j, null, 2), "utf-8");
|
|
661370
|
-
|
|
662401
|
+
renameSync13(tmp, final2);
|
|
661371
662402
|
} catch {
|
|
661372
662403
|
}
|
|
661373
662404
|
}
|
|
@@ -694197,7 +695228,7 @@ __export(projects_exports, {
|
|
|
694197
695228
|
setCurrentProject: () => setCurrentProject,
|
|
694198
695229
|
unregisterProject: () => unregisterProject
|
|
694199
695230
|
});
|
|
694200
|
-
import { readFileSync as readFileSync124, writeFileSync as writeFileSync83, mkdirSync as mkdirSync95, existsSync as existsSync152, statSync as statSync55, renameSync as
|
|
695231
|
+
import { readFileSync as readFileSync124, writeFileSync as writeFileSync83, mkdirSync as mkdirSync95, existsSync as existsSync152, statSync as statSync55, renameSync as renameSync14 } from "node:fs";
|
|
694201
695232
|
import { homedir as homedir55 } from "node:os";
|
|
694202
695233
|
import { basename as basename40, join as join166, resolve as resolve68 } from "node:path";
|
|
694203
695234
|
import { randomUUID as randomUUID19 } from "node:crypto";
|
|
@@ -694216,7 +695247,7 @@ function writeAll(file) {
|
|
|
694216
695247
|
mkdirSync95(OMNIUS_DIR3, { recursive: true });
|
|
694217
695248
|
const tmp = `${PROJECTS_FILE}.${randomUUID19().slice(0, 8)}.tmp`;
|
|
694218
695249
|
writeFileSync83(tmp, JSON.stringify(file, null, 2), "utf8");
|
|
694219
|
-
|
|
695250
|
+
renameSync14(tmp, PROJECTS_FILE);
|
|
694220
695251
|
}
|
|
694221
695252
|
function listProjects() {
|
|
694222
695253
|
const { projects } = readAll2();
|
|
@@ -695195,7 +696226,7 @@ var init_access_policy = __esm({
|
|
|
695195
696226
|
|
|
695196
696227
|
// packages/cli/src/api/project-preferences.ts
|
|
695197
696228
|
import { createHash as createHash43 } from "node:crypto";
|
|
695198
|
-
import { existsSync as existsSync153, mkdirSync as mkdirSync96, readFileSync as readFileSync125, renameSync as
|
|
696229
|
+
import { existsSync as existsSync153, mkdirSync as mkdirSync96, readFileSync as readFileSync125, renameSync as renameSync15, writeFileSync as writeFileSync84, unlinkSync as unlinkSync35 } from "node:fs";
|
|
695199
696230
|
import { homedir as homedir56 } from "node:os";
|
|
695200
696231
|
import { join as join167, resolve as resolve69 } from "node:path";
|
|
695201
696232
|
import { randomUUID as randomUUID20 } from "node:crypto";
|
|
@@ -695249,7 +696280,7 @@ function writeProjectPreferences(root, partial) {
|
|
|
695249
696280
|
const tmp = `${file}.${randomUUID20().slice(0, 8)}.tmp`;
|
|
695250
696281
|
writeFileSync84(tmp, JSON.stringify(merged, null, 2), "utf8");
|
|
695251
696282
|
try {
|
|
695252
|
-
|
|
696283
|
+
renameSync15(tmp, file);
|
|
695253
696284
|
} catch (err) {
|
|
695254
696285
|
try {
|
|
695255
696286
|
writeFileSync84(file, JSON.stringify(merged, null, 2), "utf8");
|
|
@@ -696305,7 +697336,7 @@ var init_direct_tool_registry = __esm({
|
|
|
696305
697336
|
});
|
|
696306
697337
|
|
|
696307
697338
|
// packages/cli/src/api/external-tool-registry.ts
|
|
696308
|
-
import { existsSync as existsSync157, mkdirSync as mkdirSync100, readFileSync as readFileSync127, writeFileSync as writeFileSync85, renameSync as
|
|
697339
|
+
import { existsSync as existsSync157, mkdirSync as mkdirSync100, readFileSync as readFileSync127, writeFileSync as writeFileSync85, renameSync as renameSync16 } from "node:fs";
|
|
696309
697340
|
import { join as join170 } from "node:path";
|
|
696310
697341
|
function externalToolStorePath(workingDir) {
|
|
696311
697342
|
return join170(workingDir, ".omnius", "external-tools.json");
|
|
@@ -696329,7 +697360,7 @@ function persist3(workingDir, tools) {
|
|
|
696329
697360
|
const file = { version: STORE_VERSION, tools };
|
|
696330
697361
|
const tmp = `${path12}.tmp`;
|
|
696331
697362
|
writeFileSync85(tmp, JSON.stringify(file, null, 2), { mode: 384 });
|
|
696332
|
-
|
|
697363
|
+
renameSync16(tmp, path12);
|
|
696333
697364
|
}
|
|
696334
697365
|
function validateManifest2(input, existing) {
|
|
696335
697366
|
if (!input || typeof input !== "object") {
|
|
@@ -714217,7 +715248,7 @@ import {
|
|
|
714217
715248
|
readdirSync as readdirSync58,
|
|
714218
715249
|
existsSync as existsSync165,
|
|
714219
715250
|
watch as fsWatch4,
|
|
714220
|
-
renameSync as
|
|
715251
|
+
renameSync as renameSync17,
|
|
714221
715252
|
unlinkSync as unlinkSync36,
|
|
714222
715253
|
statSync as statSync60,
|
|
714223
715254
|
openSync as openSync6,
|
|
@@ -715874,7 +716905,7 @@ function atomicJobWrite(dir, id2, job) {
|
|
|
715874
716905
|
const tmpPath = `${finalPath}.tmp.${process.pid}.${Date.now()}`;
|
|
715875
716906
|
try {
|
|
715876
716907
|
writeFileSync90(tmpPath, JSON.stringify(job, null, 2), "utf-8");
|
|
715877
|
-
|
|
716908
|
+
renameSync17(tmpPath, finalPath);
|
|
715878
716909
|
} catch {
|
|
715879
716910
|
try {
|
|
715880
716911
|
writeFileSync90(finalPath, JSON.stringify(job, null, 2), "utf-8");
|
|
@@ -717712,7 +718743,7 @@ function writeUpdateState(state) {
|
|
|
717712
718743
|
const finalPath = updateStateFile();
|
|
717713
718744
|
const tmpPath = `${finalPath}.tmp.${process.pid}`;
|
|
717714
718745
|
writeFileSync90(tmpPath, JSON.stringify(state, null, 2), "utf-8");
|
|
717715
|
-
|
|
718746
|
+
renameSync17(tmpPath, finalPath);
|
|
717716
718747
|
} catch {
|
|
717717
718748
|
}
|
|
717718
718749
|
}
|
|
@@ -732414,6 +733445,11 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
|
|
|
732414
733445
|
setLiveMediaStatus(status) {
|
|
732415
733446
|
statusBar.setLiveMediaStatus(status);
|
|
732416
733447
|
},
|
|
733448
|
+
liveDynamicBlockHost: {
|
|
733449
|
+
registerDynamicBlock: (id2, render2) => statusBar.registerDynamicBlock(id2, render2),
|
|
733450
|
+
appendDynamicBlock: (id2) => statusBar.appendDynamicBlock(id2),
|
|
733451
|
+
refreshDynamicBlocks: () => statusBar.refreshDynamicBlocks()
|
|
733452
|
+
},
|
|
732417
733453
|
// Voice call session — standalone cloudflared tunnel for /call
|
|
732418
733454
|
// Each connecting WebSocket client gets a dedicated CallSubAgent.
|
|
732419
733455
|
// Admin callers (matching session key) get full tool access; public callers get read-only.
|