omnius 1.0.406 → 1.0.408

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -542978,9 +542978,45 @@ var init_process_health = __esm({
542978
542978
  });
542979
542979
 
542980
542980
  // packages/execution/dist/tools/camera-capture.js
542981
- import { accessSync, constants, readFileSync as readFileSync44, writeFileSync as writeFileSync28, unlinkSync as unlinkSync11, existsSync as existsSync60, mkdirSync as mkdirSync34, readdirSync as readdirSync21 } from "node:fs";
542981
+ import { accessSync, constants, readFileSync as readFileSync44, writeFileSync as writeFileSync28, unlinkSync as unlinkSync11, existsSync as existsSync60, mkdirSync as mkdirSync34, readdirSync as readdirSync21, renameSync as renameSync7 } from "node:fs";
542982
542982
  import { join as join75 } from "node:path";
542983
542983
  import { tmpdir as tmpdir10 } from "node:os";
542984
+ function normalizeCameraRotationDegrees(value2) {
542985
+ if (value2 === void 0 || value2 === null || value2 === "")
542986
+ return 0;
542987
+ if (typeof value2 === "number" && Number.isFinite(value2)) {
542988
+ const n2 = (Math.round(value2) % 360 + 360) % 360;
542989
+ if (n2 === 0 || n2 === 90 || n2 === 180 || n2 === 270)
542990
+ return n2;
542991
+ }
542992
+ const raw = String(value2).trim().toLowerCase();
542993
+ if (!raw)
542994
+ return 0;
542995
+ const numeric = Number(raw.replace(/deg(?:rees)?$/i, ""));
542996
+ if (Number.isFinite(numeric))
542997
+ return normalizeCameraRotationDegrees(numeric);
542998
+ if (["none", "off", "upright", "normal", "0"].includes(raw))
542999
+ return 0;
543000
+ if (["cw", "clockwise", "right", "rotate-cw", "rotate_cw", "90"].includes(raw))
543001
+ return 90;
543002
+ if (["180", "upside-down", "upside_down", "flip", "inverted"].includes(raw))
543003
+ return 180;
543004
+ if (["ccw", "counterclockwise", "counter-clockwise", "anticlockwise", "left", "rotate-ccw", "rotate_ccw", "270"].includes(raw))
543005
+ return 270;
543006
+ throw new Error(`Unsupported camera rotation: ${String(value2)}. Use 0, 90, 180, 270, cw, ccw, or none.`);
543007
+ }
543008
+ function ffmpegRotateFilterForDegrees(rotation) {
543009
+ switch (rotation) {
543010
+ case 0:
543011
+ return null;
543012
+ case 90:
543013
+ return "transpose=1";
543014
+ case 180:
543015
+ return "hflip,vflip";
543016
+ case 270:
543017
+ return "transpose=2";
543018
+ }
543019
+ }
542984
543020
  function sleep3(ms) {
542985
543021
  return new Promise((resolve76) => setTimeout(resolve76, ms));
542986
543022
  }
@@ -543216,6 +543252,21 @@ var init_camera_capture = __esm({
543216
543252
  type: "number",
543217
543253
  description: "Capture height in pixels (default: 720 for USB, native for QooCam)"
543218
543254
  },
543255
+ rotate_degrees: {
543256
+ type: "number",
543257
+ enum: [0, 90, 180, 270],
543258
+ description: "Optional clockwise correction applied to the captured frame before saving/returning. Use 90 for clockwise, 270 for counter-clockwise."
543259
+ },
543260
+ rotation_degrees: {
543261
+ type: "number",
543262
+ enum: [0, 90, 180, 270],
543263
+ description: "Alias for rotate_degrees."
543264
+ },
543265
+ rotate: {
543266
+ type: "string",
543267
+ enum: ["none", "cw", "ccw", "180", "clockwise", "counterclockwise"],
543268
+ description: "Optional human-readable rotation alias. Equivalent to rotate_degrees."
543269
+ },
543219
543270
  output_path: {
543220
543271
  type: "string",
543221
543272
  description: "Save captured image to this path instead of returning base64"
@@ -543426,6 +543477,7 @@ ${formatRouterPlan(plan)}`,
543426
543477
  const width = args["width"] || 1280;
543427
543478
  const height = args["height"] || 720;
543428
543479
  const outputPath3 = args["output_path"];
543480
+ const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
543429
543481
  const captureDir = join75(tmpdir10(), "omnius-camera");
543430
543482
  if (!existsSync60(captureDir))
543431
543483
  mkdirSync34(captureDir, { recursive: true });
@@ -543484,10 +543536,25 @@ ${formatRouterPlan(plan)}`,
543484
543536
  return false;
543485
543537
  };
543486
543538
  if (await runAttempts()) {
543539
+ if (rotation !== 0) {
543540
+ try {
543541
+ await this.rotateCapturedImage(tempFile, rotation);
543542
+ } catch (err) {
543543
+ return {
543544
+ success: false,
543545
+ output: "",
543546
+ error: `Captured from ${device}, but failed to apply ${rotation} degree rotation: ${err instanceof Error ? err.message : String(err)}`,
543547
+ durationMs: performance.now() - start2
543548
+ };
543549
+ }
543550
+ }
543487
543551
  const firstSuccess = errors.length > 0 ? `
543488
543552
  Recovered with capture profile: ${captureState.successfulAttempt?.label ?? "unknown"}. Last failed attempt: ${errors[errors.length - 1].slice(0, 260)}` : "";
543553
+ const rotationNote = rotation !== 0 ? `
543554
+ Applied camera rotation correction: ${rotation} degrees clockwise.` : "";
543489
543555
  const result = this.returnCapturedFile(tempFile, captureState.successfulAttempt?.videoSize ?? `${width}x${height}`, device, outputPath3, start2);
543490
- return result.success && firstSuccess ? { ...result, output: result.output + firstSuccess, llmContent: result.llmContent ? result.llmContent + firstSuccess : result.llmContent } : result;
543556
+ const notes = `${firstSuccess}${rotationNote}`;
543557
+ return result.success && notes ? { ...result, output: result.output + notes, llmContent: result.llmContent ? result.llmContent + notes : result.llmContent } : result;
543491
543558
  }
543492
543559
  const allErrors = errors.join("\n\n");
543493
543560
  if (permissionBlocked || cameraFfmpegErrorIsPermissionDenied(allErrors)) {
@@ -543576,6 +543643,37 @@ ${allErrors.slice(0, 900)}`, durationMs: performance.now() - start2 };
543576
543643
  argv.push("-y", outputPath3);
543577
543644
  return argv;
543578
543645
  }
543646
+ async rotateCapturedImage(filePath, rotation) {
543647
+ const vf = ffmpegRotateFilterForDegrees(rotation);
543648
+ if (!vf)
543649
+ return;
543650
+ const rotatedPath = `${filePath}.rotated-${Date.now()}.jpg`;
543651
+ try {
543652
+ await runProcessBuffer(ffmpegBin2(), [
543653
+ "-hide_banner",
543654
+ "-loglevel",
543655
+ "error",
543656
+ "-y",
543657
+ "-i",
543658
+ filePath,
543659
+ "-vf",
543660
+ vf,
543661
+ "-q:v",
543662
+ "2",
543663
+ rotatedPath
543664
+ ], { timeout: 15e3 });
543665
+ if (!existsSync60(rotatedPath) || readFileSync44(rotatedPath).length === 0) {
543666
+ throw new Error("ffmpeg produced no rotated image bytes");
543667
+ }
543668
+ renameSync7(rotatedPath, filePath);
543669
+ } finally {
543670
+ try {
543671
+ if (existsSync60(rotatedPath))
543672
+ unlinkSync11(rotatedPath);
543673
+ } catch {
543674
+ }
543675
+ }
543676
+ }
543579
543677
  async v4l2Info(device, start2) {
543580
543678
  let info = `Camera: ${device} [v4l2]
543581
543679
  `;
@@ -543726,6 +543824,7 @@ Auto-installed camera dependencies: ${[
543726
543824
  /** Capture a frame from QooCam via OSC API */
543727
543825
  async captureQooCam(args, start2) {
543728
543826
  const outputPath3 = args["output_path"];
543827
+ const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
543729
543828
  const conn = await this.connectQooCamWiFi();
543730
543829
  if (!conn.connected) {
543731
543830
  return { success: false, output: "", error: conn.error || "Could not connect to QooCam", durationMs: performance.now() - start2 };
@@ -543745,13 +543844,13 @@ Auto-installed camera dependencies: ${[
543745
543844
  let width = 0;
543746
543845
  let height = 0;
543747
543846
  try {
543748
- const result = await this.oscFetch("POST", "/osc/commands/execute", {
543847
+ const result2 = await this.oscFetch("POST", "/osc/commands/execute", {
543749
543848
  name: "camera.takePicture"
543750
543849
  });
543751
- if (result?.state === "error") {
543752
- return { success: false, output: "", error: `QooCam capture failed: ${result.error?.message || JSON.stringify(result.error)}`, durationMs: performance.now() - start2 };
543850
+ if (result2?.state === "error") {
543851
+ return { success: false, output: "", error: `QooCam capture failed: ${result2.error?.message || JSON.stringify(result2.error)}`, durationMs: performance.now() - start2 };
543753
543852
  }
543754
- const entry = result?.results?.entries?.[0];
543853
+ const entry = result2?.results?.entries?.[0];
543755
543854
  if (!entry?.fileUrl) {
543756
543855
  return { success: false, output: "", error: "QooCam took a picture but returned no file URL", durationMs: performance.now() - start2 };
543757
543856
  }
@@ -543775,8 +543874,27 @@ Auto-installed camera dependencies: ${[
543775
543874
  } catch (err) {
543776
543875
  return { success: false, output: "", error: `Failed to download frame from QooCam: ${err instanceof Error ? err.message : String(err)}`, durationMs: performance.now() - start2 };
543777
543876
  }
543877
+ if (rotation !== 0) {
543878
+ try {
543879
+ await this.rotateCapturedImage(tempFile, rotation);
543880
+ } catch (err) {
543881
+ return {
543882
+ success: false,
543883
+ output: "",
543884
+ error: `Downloaded QooCam frame, but failed to apply ${rotation} degree rotation: ${err instanceof Error ? err.message : String(err)}`,
543885
+ durationMs: performance.now() - start2
543886
+ };
543887
+ }
543888
+ }
543778
543889
  const res = width && height ? `${width}x${height}` : "native";
543779
- return this.returnCapturedFile(tempFile, res, "QooCam 8K (OSC/WiFi)", outputPath3, start2);
543890
+ const result = this.returnCapturedFile(tempFile, res, "QooCam 8K (OSC/WiFi)", outputPath3, start2);
543891
+ return result.success && rotation !== 0 ? {
543892
+ ...result,
543893
+ output: `${result.output}
543894
+ Applied camera rotation correction: ${rotation} degrees clockwise.`,
543895
+ llmContent: result.llmContent ? `${result.llmContent}
543896
+ Applied camera rotation correction: ${rotation} degrees clockwise.` : result.llmContent
543897
+ } : result;
543780
543898
  }
543781
543899
  /** Get QooCam info via OSC */
543782
543900
  async qoocamInfo(start2) {
@@ -553723,7 +553841,7 @@ var init_plugin_system = __esm({
553723
553841
  });
553724
553842
 
553725
553843
  // packages/execution/dist/tools/notebook-edit.js
553726
- import { readFileSync as readFileSync54, writeFileSync as writeFileSync37, existsSync as existsSync74, renameSync as renameSync7, unlinkSync as unlinkSync15 } from "node:fs";
553844
+ import { readFileSync as readFileSync54, writeFileSync as writeFileSync37, existsSync as existsSync74, renameSync as renameSync8, unlinkSync as unlinkSync15 } from "node:fs";
553727
553845
  import { resolve as resolve43 } from "node:path";
553728
553846
  function readNotebook(path12) {
553729
553847
  if (!existsSync74(path12))
@@ -553744,7 +553862,7 @@ function writeNotebook(path12, nb) {
553744
553862
  const tempPath = `${path12}.tmp-${process.pid}-${Date.now()}`;
553745
553863
  writeFileSync37(tempPath, JSON.stringify(nb, null, 1) + "\n", "utf8");
553746
553864
  try {
553747
- renameSync7(tempPath, path12);
553865
+ renameSync8(tempPath, path12);
553748
553866
  } catch (err) {
553749
553867
  try {
553750
553868
  unlinkSync15(tempPath);
@@ -555483,6 +555601,15 @@ function makePythonScript() {
555483
555601
  return String.raw`
555484
555602
  import json, os, sys, time
555485
555603
 
555604
+ def apply_rotation(frame, rotate_degrees, cv2):
555605
+ if rotate_degrees == 90:
555606
+ return cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
555607
+ if rotate_degrees == 180:
555608
+ return cv2.rotate(frame, cv2.ROTATE_180)
555609
+ if rotate_degrees == 270:
555610
+ return cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE)
555611
+ return frame
555612
+
555486
555613
  def main():
555487
555614
  cfg_path = sys.argv[1]
555488
555615
  with open(cfg_path, "r", encoding="utf-8") as f:
@@ -555494,6 +555621,7 @@ def main():
555494
555621
  source_kind = cfg["source_kind"]
555495
555622
  source = cfg["source"]
555496
555623
  media_path = None
555624
+ rotate_degrees = int(cfg.get("rotate_degrees") or 0)
555497
555625
 
555498
555626
  try:
555499
555627
  import cv2
@@ -555567,6 +555695,7 @@ def main():
555567
555695
  ret, frame = cap.read()
555568
555696
  if not ret:
555569
555697
  break
555698
+ frame = apply_rotation(frame, rotate_degrees, cv2)
555570
555699
  ts = cap.get(cv2.CAP_PROP_POS_MSEC) / 1000.0
555571
555700
  if not ts or ts < 0:
555572
555701
  ts = time.time() - start_wall
@@ -555649,6 +555778,7 @@ def main():
555649
555778
  "output_dir": out_dir,
555650
555779
  "yolo_model": yolo_model,
555651
555780
  "yolo_live": True,
555781
+ "rotate_degrees": rotate_degrees,
555652
555782
  "frames_sampled": frame_index,
555653
555783
  "duration_observed_sec": round(float(duration if source_kind != "camera" else time.time() - start_wall), 3),
555654
555784
  "objects": objects,
@@ -555662,6 +555792,7 @@ if __name__ == "__main__":
555662
555792
  }
555663
555793
  function mockResult(args, workingDir) {
555664
555794
  const kind = sourceKind(args);
555795
+ const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
555665
555796
  const outDir = join92(workingDir, ".omnius", "live-media", `mock-${Date.now().toString(36)}`);
555666
555797
  mkdirSync47(join92(outDir, "crops"), { recursive: true });
555667
555798
  const facePath = join92(outDir, "crops", "face_f0000_00.jpg");
@@ -555673,6 +555804,7 @@ function mockResult(args, workingDir) {
555673
555804
  output_dir: outDir,
555674
555805
  yolo_model: String(args["yolo_model"] ?? DEFAULT_YOLO26_MODEL),
555675
555806
  yolo_live: false,
555807
+ rotate_degrees: rotation,
555676
555808
  frames_sampled: 3,
555677
555809
  duration_observed_sec: Number(args["duration_sec"] ?? 3),
555678
555810
  objects: [
@@ -555694,6 +555826,7 @@ var init_live_media_loop = __esm({
555694
555826
  init_process_async();
555695
555827
  init_venv_paths();
555696
555828
  init_audio_analyze();
555829
+ init_camera_capture();
555697
555830
  init_transcribe_tool();
555698
555831
  init_visual_memory();
555699
555832
  init_visual_trigger();
@@ -555713,6 +555846,9 @@ var init_live_media_loop = __esm({
555713
555846
  path: { type: "string", description: "Local media path" },
555714
555847
  file: { type: "string", description: "Alias for path" },
555715
555848
  camera: { type: ["string", "number"], description: "Camera device index/path" },
555849
+ rotate_degrees: { type: "number", enum: [0, 90, 180, 270], description: "Clockwise correction applied to sampled frames before object tracking and face crops." },
555850
+ rotation_degrees: { type: "number", enum: [0, 90, 180, 270], description: "Alias for rotate_degrees." },
555851
+ rotate: { type: "string", enum: ["none", "cw", "ccw", "180", "clockwise", "counterclockwise"], description: "Optional human-readable rotation alias." },
555716
555852
  duration_sec: { type: "number", description: "Bounded watch duration, default 8" },
555717
555853
  sample_interval_sec: { type: "number", description: "Seconds between sampled frames, default 1" },
555718
555854
  max_frames: { type: "number", description: "Max sampled frames, default 8" },
@@ -555736,6 +555872,7 @@ var init_live_media_loop = __esm({
555736
555872
  const start2 = performance.now();
555737
555873
  const action = String(args["action"] ?? "watch").toLowerCase();
555738
555874
  const yoloModel = String(args["yolo_model"] ?? DEFAULT_YOLO26_MODEL);
555875
+ const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
555739
555876
  const plan = buildYolo26BootstrapPlan(yoloModel);
555740
555877
  if (action === "status" || action === "ensure") {
555741
555878
  const runtime = action === "ensure" ? await ensureRuntime(args["auto_bootstrap"] !== false) : await ensureRuntime(false);
@@ -555780,6 +555917,7 @@ var init_live_media_loop = __esm({
555780
555917
  duration_sec: Number(args["duration_sec"] ?? 8),
555781
555918
  sample_interval_sec: Number(args["sample_interval_sec"] ?? 1),
555782
555919
  max_frames: Number(args["max_frames"] ?? 8),
555920
+ rotate_degrees: rotation,
555783
555921
  detect_objects: args["detect_objects"] !== false,
555784
555922
  track_objects: args["track_objects"] !== false,
555785
555923
  crop_faces: args["crop_faces"] !== false,
@@ -555872,6 +556010,8 @@ var init_live_media_loop = __esm({
555872
556010
  if (result.media_path)
555873
556011
  lines.push(`media: ${result.media_path}`);
555874
556012
  lines.push(`YOLO model: ${result.yolo_model} (${result.yolo_live ? "live" : "mock/harness"})`);
556013
+ if (result.rotate_degrees)
556014
+ lines.push(`frame rotation correction: ${result.rotate_degrees} degrees clockwise`);
555875
556015
  lines.push(`sampled: ${result.frames_sampled} frame(s) over ${result.duration_observed_sec.toFixed(1)}s`);
555876
556016
  lines.push(`objects: ${result.objects.length} detection(s), tracks: ${result.tracks.length}, face crops: ${result.faces.length}`);
555877
556017
  const objectSummary = summarizeObjects(result.objects);
@@ -555918,6 +556058,7 @@ var init_live_media_loop = __esm({
555918
556058
  media_path: result.media_path,
555919
556059
  yolo_model: result.yolo_model,
555920
556060
  yolo_live: result.yolo_live,
556061
+ rotate_degrees: result.rotate_degrees ?? 0,
555921
556062
  frames_sampled: result.frames_sampled,
555922
556063
  objects: result.objects.slice(0, 80),
555923
556064
  tracks: result.tracks.slice(0, 40),
@@ -569468,7 +569609,7 @@ var init_memory_consolidation = __esm({
569468
569609
  });
569469
569610
 
569470
569611
  // packages/orchestrator/dist/consolidation-runtime.js
569471
- import { existsSync as existsSync97, mkdirSync as mkdirSync56, readFileSync as readFileSync74, writeFileSync as writeFileSync46, renameSync as renameSync8 } from "node:fs";
569612
+ import { existsSync as existsSync97, mkdirSync as mkdirSync56, readFileSync as readFileSync74, writeFileSync as writeFileSync46, renameSync as renameSync9 } from "node:fs";
569472
569613
  import { homedir as homedir33 } from "node:os";
569473
569614
  import { join as join108, dirname as dirname35 } from "node:path";
569474
569615
  function storePath2() {
@@ -569490,7 +569631,7 @@ function save2(s2) {
569490
569631
  mkdirSync56(dirname35(p2), { recursive: true });
569491
569632
  const tmp = `${p2}.tmp`;
569492
569633
  writeFileSync46(tmp, JSON.stringify(s2), "utf8");
569493
- renameSync8(tmp, p2);
569634
+ renameSync9(tmp, p2);
569494
569635
  } catch {
569495
569636
  }
569496
569637
  }
@@ -597786,7 +597927,7 @@ var init_agent_task = __esm({
597786
597927
  });
597787
597928
 
597788
597929
  // packages/orchestrator/dist/task-recovery.js
597789
- import { existsSync as existsSync105, readFileSync as readFileSync81, writeFileSync as writeFileSync51, mkdirSync as mkdirSync61, readdirSync as readdirSync33, renameSync as renameSync9, unlinkSync as unlinkSync19 } from "node:fs";
597930
+ import { existsSync as existsSync105, readFileSync as readFileSync81, writeFileSync as writeFileSync51, mkdirSync as mkdirSync61, readdirSync as readdirSync33, renameSync as renameSync10, unlinkSync as unlinkSync19 } from "node:fs";
597790
597931
  import { join as join117 } from "node:path";
597791
597932
  import { homedir as homedir36 } from "node:os";
597792
597933
  function sidecarDir() {
@@ -597803,7 +597944,7 @@ function persistAgentTaskSidecar(task) {
597803
597944
  const tmpPath = `${finalPath}.tmp.${process.pid}.${Date.now()}`;
597804
597945
  const { abortController: _ignore, ...serializable } = task;
597805
597946
  writeFileSync51(tmpPath, JSON.stringify(serializable, null, 2), "utf-8");
597806
- renameSync9(tmpPath, finalPath);
597947
+ renameSync10(tmpPath, finalPath);
597807
597948
  } catch {
597808
597949
  }
597809
597950
  }
@@ -601956,6 +602097,172 @@ function parseJsonObject3(value2) {
601956
602097
  return null;
601957
602098
  }
601958
602099
  }
602100
+ function parseJsonObjectFromText(value2) {
602101
+ const direct = parseJsonObject3(value2);
602102
+ if (direct) return direct;
602103
+ const text2 = String(value2 ?? "");
602104
+ const start2 = text2.indexOf("{");
602105
+ const end = text2.lastIndexOf("}");
602106
+ if (start2 < 0 || end <= start2) return null;
602107
+ return parseJsonObject3(text2.slice(start2, end + 1));
602108
+ }
602109
+ function normalizeLiveCameraRotation(value2) {
602110
+ if (value2 === void 0 || value2 === null || value2 === "") return 0;
602111
+ if (typeof value2 === "number" && Number.isFinite(value2)) {
602112
+ const n2 = (Math.round(value2) % 360 + 360) % 360;
602113
+ if (n2 === 0 || n2 === 90 || n2 === 180 || n2 === 270) return n2;
602114
+ }
602115
+ const raw = String(value2).trim().toLowerCase();
602116
+ const numeric = Number(raw.replace(/deg(?:rees)?$/i, ""));
602117
+ if (Number.isFinite(numeric)) return normalizeLiveCameraRotation(numeric);
602118
+ if (["none", "off", "upright", "normal", "0"].includes(raw)) return 0;
602119
+ if (["cw", "clockwise", "right", "rotate-cw", "rotate_cw", "90"].includes(raw)) return 90;
602120
+ if (["180", "upside-down", "upside_down", "flip", "inverted"].includes(raw)) return 180;
602121
+ if (["ccw", "counterclockwise", "counter-clockwise", "counter_clockwise", "anticlockwise", "left", "rotate-ccw", "rotate_ccw", "270"].includes(raw)) return 270;
602122
+ throw new Error(`Unsupported camera rotation: ${String(value2)}. Use 0, 90, 180, 270, cw, ccw, or none.`);
602123
+ }
602124
+ function parseCameraOrientationDecision(value2) {
602125
+ const parsed = parseJsonObjectFromText(value2);
602126
+ if (!parsed) return null;
602127
+ const rawRotation = parsed["rotation_degrees"] ?? parsed["correction_degrees"] ?? parsed["rotate_degrees"] ?? parsed["rotation"] ?? parsed["correction"];
602128
+ if (rawRotation === void 0 || rawRotation === null || rawRotation === "") return null;
602129
+ let rotation;
602130
+ try {
602131
+ rotation = normalizeLiveCameraRotation(rawRotation);
602132
+ } catch {
602133
+ return null;
602134
+ }
602135
+ const confidenceNumber = Number(parsed["confidence"]);
602136
+ const confidence2 = Number.isFinite(confidenceNumber) ? Math.max(0, Math.min(1, confidenceNumber)) : void 0;
602137
+ const reason = typeof parsed["reason"] === "string" ? boundedText(parsed["reason"], 240) : typeof parsed["evidence"] === "string" ? boundedText(parsed["evidence"], 240) : void 0;
602138
+ return { rotation, confidence: confidence2, reason };
602139
+ }
602140
+ function formatCameraRotation(rotation) {
602141
+ if (rotation === 0) return "upright/no correction";
602142
+ if (rotation === 90) return "90 degrees clockwise";
602143
+ if (rotation === 180) return "180 degrees";
602144
+ return "90 degrees counter-clockwise";
602145
+ }
602146
+ function sanitizeCameraOrientation(value2) {
602147
+ if (typeof value2 !== "object" || value2 === null || Array.isArray(value2)) return {};
602148
+ const out = {};
602149
+ for (const [device, raw] of Object.entries(value2)) {
602150
+ if (!device || typeof raw !== "object" || raw === null || Array.isArray(raw)) continue;
602151
+ const entry = raw;
602152
+ try {
602153
+ out[device] = {
602154
+ rotation: normalizeLiveCameraRotation(entry["rotation"]),
602155
+ source: entry["source"] === "manual" ? "manual" : "auto",
602156
+ updatedAt: Math.max(0, Number(entry["updatedAt"] ?? Date.now())),
602157
+ confidence: typeof entry["confidence"] === "number" ? Math.max(0, Math.min(1, entry["confidence"])) : void 0,
602158
+ evidence: typeof entry["evidence"] === "string" ? boundedText(entry["evidence"], 800) : void 0
602159
+ };
602160
+ } catch {
602161
+ }
602162
+ }
602163
+ return out;
602164
+ }
602165
+ function describeCameraOrientation(orientation) {
602166
+ if (!orientation) return "upright/no correction (default)";
602167
+ return `${formatCameraRotation(orientation.rotation)} (${orientation.source}${orientation.confidence !== void 0 ? ` confidence=${orientation.confidence.toFixed(2)}` : ""}, updated=${nowIso3(orientation.updatedAt)})`;
602168
+ }
602169
+ function chooseCameraOrientationFromScores(scores) {
602170
+ const sorted = [...scores].filter((score) => Number.isFinite(score.score)).sort((a2, b) => b.score - a2.score);
602171
+ const best = sorted[0];
602172
+ if (!best || best.score <= 0 || best.faces <= 0) return null;
602173
+ const second3 = sorted[1];
602174
+ const gap = best.score - (second3?.score ?? 0);
602175
+ const total = sorted.reduce((sum, score) => sum + Math.max(0, score.score), 0);
602176
+ const share = total > 0 ? best.score / total : 1;
602177
+ const confidence2 = Math.max(0.55, Math.min(0.98, share * 0.75 + Math.min(0.23, gap / Math.max(1, best.score))));
602178
+ if (confidence2 < 0.58 && gap < 0.4) return null;
602179
+ return {
602180
+ rotation: best.rotation,
602181
+ confidence: confidence2,
602182
+ reason: `cv-face-orientation faces=${best.faces} score=${best.score.toFixed(2)} next=${(second3?.score ?? 0).toFixed(2)} area=${(best.faceAreaRatio ?? 0).toFixed(3)}`
602183
+ };
602184
+ }
602185
+ function trimOneLine(value2, maxChars) {
602186
+ return String(value2 ?? "").replace(/\r/g, " ").replace(/\n/g, " ").replace(/\s+/g, " ").trim().slice(0, maxChars);
602187
+ }
602188
+ function compactSourceId(source) {
602189
+ return source.replace(/^\/dev\//, "");
602190
+ }
602191
+ function truncateCell(value2, width) {
602192
+ if (width <= 1) return value2.slice(0, Math.max(0, width));
602193
+ return value2.length <= width ? value2.padEnd(width, " ") : `${value2.slice(0, Math.max(1, width - 1))}…`;
602194
+ }
602195
+ function summarizeInference(value2) {
602196
+ if (!value2) return "objects: pending";
602197
+ const lines = value2.split("\n").map((line) => line.trim()).filter(Boolean);
602198
+ const objectsIndex = lines.findIndex((line) => /^##\s+Objects/i.test(line));
602199
+ if (objectsIndex >= 0) {
602200
+ const objects = lines.slice(objectsIndex + 1).filter((line) => line.startsWith("- ")).slice(0, 4).map((line) => line.replace(/^-\s*/, ""));
602201
+ if (objects.length > 0) return `objects: ${objects.join("; ")}`;
602202
+ }
602203
+ const sampled = lines.find((line) => /^objects:/i.test(line));
602204
+ return sampled ? trimOneLine(sampled, 160) : trimOneLine(lines.slice(0, 4).join("; "), 160);
602205
+ }
602206
+ function cameraSnapshotSummary(camera) {
602207
+ if (camera.error) return `error: ${trimOneLine(camera.error, 160)}`;
602208
+ return camera.summary || summarizeInference(camera.inference);
602209
+ }
602210
+ function formatLiveDashboardFromSnapshot(snapshot, opts = {}) {
602211
+ const width = Math.max(60, opts.width ?? 100);
602212
+ const now2 = opts.now ?? Date.now();
602213
+ const maxCameras = opts.maxCameras ?? 4;
602214
+ const cameras = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {})).sort((a2, b) => {
602215
+ if (a2.source === snapshot.config.selectedCamera) return -1;
602216
+ if (b.source === snapshot.config.selectedCamera) return 1;
602217
+ return a2.source.localeCompare(b.source);
602218
+ }).slice(0, maxCameras);
602219
+ const audioAge = snapshot.audio?.updatedAt ? `${Math.max(0, Math.round((now2 - snapshot.audio.updatedAt) / 1e3))}s` : "none";
602220
+ const title = `live audio/video cameras=${cameras.length}/${snapshot.devices.video.length} audio=${snapshot.config.audioEnabled ? "on" : "off"} age=${audioAge}`;
602221
+ const horizontal = "─".repeat(Math.max(0, width - 2));
602222
+ const lines = [`╭${horizontal}╮`];
602223
+ lines.push(`│${truncateCell(` ${title}`, width - 2)}│`);
602224
+ if (cameras.length === 0) {
602225
+ lines.push(`│${truncateCell(" no camera frames captured yet", width - 2)}│`);
602226
+ }
602227
+ for (const camera of cameras) {
602228
+ const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
602229
+ const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
602230
+ lines.push(`├${horizontal}┤`);
602231
+ lines.push(`│${truncateCell(` ${compactSourceId(camera.source)} age=${age}s rotation=${rotation}`, width - 2)}│`);
602232
+ const asciiLines = (camera.ascii ?? "").split("\n").filter((line) => line.length > 0).slice(0, 10);
602233
+ const detailLines = [
602234
+ cameraSnapshotSummary(camera),
602235
+ camera.framePath ? `frame: ${camera.displayPath || camera.framePath}` : "",
602236
+ camera.orientation?.evidence ? `orientation: ${trimOneLine(camera.orientation.evidence, 120)}` : ""
602237
+ ].filter(Boolean);
602238
+ if (asciiLines.length === 0) {
602239
+ for (const detail of detailLines.length ? detailLines : ["preview pending"]) {
602240
+ lines.push(`│${truncateCell(` ${detail}`, width - 2)}│`);
602241
+ }
602242
+ continue;
602243
+ }
602244
+ const imageWidth = Math.max(24, Math.min(52, Math.floor((width - 5) * 0.58)));
602245
+ const detailWidth = Math.max(18, width - imageWidth - 5);
602246
+ const rowCount = Math.max(asciiLines.length, detailLines.length);
602247
+ for (let i2 = 0; i2 < rowCount; i2++) {
602248
+ const image = truncateCell(asciiLines[i2] ?? "", imageWidth);
602249
+ const detail = truncateCell(detailLines[i2] ?? "", detailWidth);
602250
+ lines.push(`│ ${image} │ ${detail}│`);
602251
+ }
602252
+ }
602253
+ if (snapshot.audio) {
602254
+ lines.push(`├${horizontal}┤`);
602255
+ const audioBits = [
602256
+ `audio input=${snapshot.audio.input || snapshot.config.selectedAudioInput || "none"}`,
602257
+ snapshot.audio.error ? `error=${trimOneLine(snapshot.audio.error, 120)}` : "",
602258
+ snapshot.audio.transcript ? `asr=${trimOneLine(snapshot.audio.transcript, 120)}` : "",
602259
+ snapshot.audio.analysis ? `sound=${trimOneLine(snapshot.audio.analysis, 120)}` : ""
602260
+ ].filter(Boolean);
602261
+ for (const item of audioBits.slice(0, 4)) lines.push(`│${truncateCell(` ${item}`, width - 2)}│`);
602262
+ }
602263
+ lines.push(`╰${horizontal}╯`);
602264
+ return lines;
602265
+ }
601959
602266
  function parseLiveMediaPayload(value2) {
601960
602267
  const parsed = parseJsonObject3(value2);
601961
602268
  if (!parsed) return null;
@@ -602180,6 +602487,7 @@ function loadLiveSensorConfig(repoRoot) {
602180
602487
  return {
602181
602488
  ...DEFAULT_CONFIG7,
602182
602489
  ...parsed,
602490
+ cameraOrientation: sanitizeCameraOrientation(parsed.cameraOrientation),
602183
602491
  videoIntervalMs: Math.max(1500, Number(parsed.videoIntervalMs ?? DEFAULT_CONFIG7.videoIntervalMs)),
602184
602492
  audioIntervalMs: Math.max(3e3, Number(parsed.audioIntervalMs ?? DEFAULT_CONFIG7.audioIntervalMs))
602185
602493
  };
@@ -602208,11 +602516,30 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
602208
602516
  lines.push("Live sensor context is operator-enabled ambient input. Treat it as current perceptual evidence for this turn; if it is stale, missing, or insufficient, use camera/audio tools to refresh before answering what you see or hear.");
602209
602517
  lines.push("Identity rule: never invent names for observed people. If a person is unknown and identity matters, ask the user or the person for their name, then store the association with visual_memory(action='enroll', image='<face crop>', name='<name>') when a face crop is available, or multimodal_memory(action='meet', person_name='<name>') when live voice/name context is available.");
602210
602518
  lines.push(`Snapshot file: ${liveSnapshotPath(snapshot.repoRoot)}`);
602211
- lines.push(`Streams: video=${snapshot.config.videoEnabled ? "on" : "off"} infer=${snapshot.config.inferEnabled ? "on" : "off"} audio=${snapshot.config.audioEnabled ? "on" : "off"} asr=${snapshot.config.asrEnabled ? "on" : "off"} sounds=${snapshot.config.audioAnalysisEnabled ? "on" : "off"} output-monitor=${snapshot.config.audioOutputEnabled ? "on" : "off"}`);
602212
- if (snapshot.config.selectedCamera) lines.push(`Selected camera: ${snapshot.config.selectedCamera}`);
602519
+ lines.push(`Streams: video=${snapshot.config.videoEnabled ? "on" : "off"} infer=${snapshot.config.inferEnabled ? "on" : "off"} clip=${snapshot.config.clipEnabled ? "on" : "off"} audio=${snapshot.config.audioEnabled ? "on" : "off"} asr=${snapshot.config.asrEnabled ? "on" : "off"} sounds=${snapshot.config.audioAnalysisEnabled ? "on" : "off"} output-monitor=${snapshot.config.audioOutputEnabled ? "on" : "off"}`);
602520
+ if (snapshot.config.selectedCamera) {
602521
+ const orientation = snapshot.config.cameraOrientation?.[snapshot.config.selectedCamera];
602522
+ lines.push(`Selected camera: ${snapshot.config.selectedCamera}`);
602523
+ lines.push(`Camera orientation correction: ${describeCameraOrientation(orientation)}`);
602524
+ }
602525
+ const cameraSnapshots = Object.values(snapshot.cameras ?? (snapshot.video ? { [snapshot.video.source]: snapshot.video } : {})).sort((a2, b) => {
602526
+ if (a2.source === snapshot.config.selectedCamera) return -1;
602527
+ if (b.source === snapshot.config.selectedCamera) return 1;
602528
+ return b.updatedAt - a2.updatedAt;
602529
+ });
602530
+ if (cameraSnapshots.length > 0) {
602531
+ lines.push("Camera snapshot set:");
602532
+ for (const camera of cameraSnapshots.slice(0, 6)) {
602533
+ const age = Math.max(0, Math.round((now2 - camera.updatedAt) / 1e3));
602534
+ const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
602535
+ lines.push(`- ${camera.source} age=${age}s rotation=${rotation}${camera.framePath ? ` frame=${camera.framePath}` : ""}${camera.error ? ` error=${trimOneLine(camera.error, 240)}` : ""}`);
602536
+ const summary = cameraSnapshotSummary(camera);
602537
+ if (summary) lines.push(` summary: ${summary}`);
602538
+ }
602539
+ }
602213
602540
  if (snapshot.config.selectedAudioInput) lines.push(`Selected audio input: ${snapshot.config.selectedAudioInput}`);
602214
602541
  if (snapshot.config.selectedAudioOutput) lines.push(`Selected audio output: ${snapshot.config.selectedAudioOutput}`);
602215
- const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 18);
602542
+ const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "clip_recognition" || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 18);
602216
602543
  if (activeVideo) {
602217
602544
  lines.push("");
602218
602545
  lines.push("## LIVE CAMERA EVENTS");
@@ -602257,7 +602584,7 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
602257
602584
  const age = now2 - snapshot.video.updatedAt;
602258
602585
  lines.push("");
602259
602586
  lines.push("## LIVE CAMERA LATEST FRAME");
602260
- lines.push(`timestamp: ${nowIso3(snapshot.video.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s source=${snapshot.video.source}${age > maxAgeMs ? " STALE" : ""}`);
602587
+ lines.push(`timestamp: ${nowIso3(snapshot.video.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s source=${snapshot.video.source}${snapshot.video.orientation ? ` rotation=${formatCameraRotation(snapshot.video.orientation.rotation)}` : ""}${age > maxAgeMs ? " STALE" : ""}`);
602261
602588
  if (snapshot.video.error) lines.push(`error: ${snapshot.video.error}`);
602262
602589
  if (snapshot.video.framePath) lines.push(`frame: ${snapshot.video.framePath}`);
602263
602590
  if (snapshot.video.asciiContext) lines.push(snapshot.video.asciiContext);
@@ -602266,6 +602593,19 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
602266
602593
  lines.push(cleanPreview(snapshot.video.inference, 1800));
602267
602594
  }
602268
602595
  }
602596
+ if (cameraSnapshots.length > 1) {
602597
+ lines.push("");
602598
+ lines.push("## LIVE CAMERA LATEST FRAMES");
602599
+ for (const camera of cameraSnapshots.slice(0, 6)) {
602600
+ const age = now2 - camera.updatedAt;
602601
+ const rotation = camera.orientation ? formatCameraRotation(camera.orientation.rotation) : "upright/no correction";
602602
+ lines.push(`### ${camera.source}`);
602603
+ lines.push(`timestamp: ${nowIso3(camera.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s rotation=${rotation}${age > maxAgeMs ? " STALE" : ""}`);
602604
+ if (camera.error) lines.push(`error: ${camera.error}`);
602605
+ if (camera.framePath) lines.push(`frame: ${camera.framePath}`);
602606
+ lines.push(`summary: ${cameraSnapshotSummary(camera)}`);
602607
+ }
602608
+ }
602269
602609
  if (snapshot.audio) {
602270
602610
  const audioEvents = (snapshot.events ?? []).filter((event) => event.kind === "audio_transcript" || event.kind === "audio_sound").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 8);
602271
602611
  if (audioEvents.length > 0) {
@@ -602372,6 +602712,93 @@ async function recordAudioSample(device, durationSec, outputPath3) {
602372
602712
  outputPath3
602373
602713
  ], { timeout: (Number(seconds) + 10) * 1e3 });
602374
602714
  }
602715
+ async function scoreCameraOrientationWithOpenCv(framePath) {
602716
+ if (!await commandExists2("python3", 1200)) return null;
602717
+ const script = String.raw`
602718
+ import json, sys
602719
+ try:
602720
+ import cv2
602721
+ except Exception as exc:
602722
+ print(json.dumps({"ok": False, "error": str(exc)}))
602723
+ raise SystemExit(0)
602724
+
602725
+ path = sys.argv[1]
602726
+ img = cv2.imread(path)
602727
+ if img is None:
602728
+ print(json.dumps({"ok": False, "error": "could not read image"}))
602729
+ raise SystemExit(0)
602730
+
602731
+ face = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_default.xml")
602732
+ eye = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_eye.xml")
602733
+
602734
+ def rotate(mat, deg):
602735
+ if deg == 90:
602736
+ return cv2.rotate(mat, cv2.ROTATE_90_CLOCKWISE)
602737
+ if deg == 180:
602738
+ return cv2.rotate(mat, cv2.ROTATE_180)
602739
+ if deg == 270:
602740
+ return cv2.rotate(mat, cv2.ROTATE_90_COUNTERCLOCKWISE)
602741
+ return mat
602742
+
602743
+ scores = []
602744
+ for deg in [0, 90, 180, 270]:
602745
+ frame = rotate(img, deg)
602746
+ gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
602747
+ h, w = gray.shape[:2]
602748
+ faces = face.detectMultiScale(gray, scaleFactor=1.08, minNeighbors=4, minSize=(32, 32))
602749
+ face_area = 0.0
602750
+ eye_count = 0
602751
+ for (x, y, fw, fh) in faces[:8]:
602752
+ face_area += float(fw * fh) / max(1.0, float(w * h))
602753
+ roi = gray[y:y+fh, x:x+fw]
602754
+ try:
602755
+ eye_count += len(eye.detectMultiScale(roi, scaleFactor=1.1, minNeighbors=3, minSize=(8, 8)))
602756
+ except Exception:
602757
+ pass
602758
+ score = len(faces) * 2.5 + min(4.0, face_area * 35.0) + min(2.0, eye_count * 0.35)
602759
+ scores.append({"rotation": deg, "score": score, "faces": int(len(faces)), "faceAreaRatio": face_area})
602760
+ print(json.dumps({"ok": True, "scores": scores}))
602761
+ `;
602762
+ try {
602763
+ const raw = await execFileText4("python3", ["-c", script, framePath], { timeout: 15e3, maxBuffer: 1024 * 1024 });
602764
+ const parsed = parseJsonObjectFromText(raw);
602765
+ const scores = Array.isArray(parsed?.["scores"]) ? parsed["scores"] : [];
602766
+ if (scores.length === 0) return null;
602767
+ return scores.map((score) => ({
602768
+ rotation: normalizeLiveCameraRotation(score["rotation"]),
602769
+ score: Math.max(0, Number(score["score"] ?? 0)),
602770
+ faces: Math.max(0, Number(score["faces"] ?? 0)),
602771
+ faceAreaRatio: Number.isFinite(Number(score["faceAreaRatio"])) ? Number(score["faceAreaRatio"]) : void 0
602772
+ }));
602773
+ } catch {
602774
+ return null;
602775
+ }
602776
+ }
602777
+ async function tryRecordAudioDevice(device, durationSec, outputPath3) {
602778
+ await recordAudioSample(device, durationSec, outputPath3);
602779
+ }
602780
+ async function recordAudioSampleWithRecovery(preferred, devices, durationSec, outputPath3) {
602781
+ const ordered = [];
602782
+ const add3 = (id2) => {
602783
+ const clean5 = String(id2 ?? "").trim();
602784
+ if (clean5 && !ordered.includes(clean5)) ordered.push(clean5);
602785
+ };
602786
+ add3(preferred);
602787
+ for (const device of devices.filter((entry) => entry.source === "pulse" || entry.source === "pipewire")) add3(device.id);
602788
+ add3("pulse:default");
602789
+ add3("default");
602790
+ for (const device of devices.filter((entry) => entry.source !== "pulse" && entry.source !== "pipewire")) add3(device.id);
602791
+ const errors = [];
602792
+ for (const candidate of ordered) {
602793
+ try {
602794
+ await tryRecordAudioDevice(candidate, durationSec, outputPath3);
602795
+ return { ok: true, device: candidate, method: candidate.startsWith("pulse:") ? "pulse/ffmpeg" : "arecord", errors };
602796
+ } catch (err) {
602797
+ errors.push(`${candidate}: ${err instanceof Error ? err.message : String(err)}`.slice(0, 360));
602798
+ }
602799
+ }
602800
+ return { ok: false, device: preferred || "default", method: "none", errors };
602801
+ }
602375
602802
  function firstDeviceId(devices) {
602376
602803
  return devices.find((device) => device.id && !/metadata\/non-capture/i.test(device.detail ?? ""))?.id ?? devices[0]?.id;
602377
602804
  }
@@ -602401,8 +602828,9 @@ function formatLiveInventory(inventory) {
602401
602828
  function formatLiveStatus(snapshot) {
602402
602829
  const cfg = snapshot.config;
602403
602830
  const lines = [
602404
- `Live streams: video=${cfg.videoEnabled ? "on" : "off"} infer=${cfg.inferEnabled ? "on" : "off"} audio=${cfg.audioEnabled ? "on" : "off"} asr=${cfg.asrEnabled ? "on" : "off"} sounds=${cfg.audioAnalysisEnabled ? "on" : "off"} output=${cfg.audioOutputEnabled ? "on" : "off"}`,
602831
+ `Live streams: video=${cfg.videoEnabled ? "on" : "off"} infer=${cfg.inferEnabled ? "on" : "off"} clip=${cfg.clipEnabled ? "on" : "off"} audio=${cfg.audioEnabled ? "on" : "off"} asr=${cfg.asrEnabled ? "on" : "off"} sounds=${cfg.audioAnalysisEnabled ? "on" : "off"} output=${cfg.audioOutputEnabled ? "on" : "off"}`,
602405
602832
  `Camera: ${cfg.selectedCamera || "none"}`,
602833
+ `Camera orientation: ${cfg.selectedCamera ? describeCameraOrientation(cfg.cameraOrientation?.[cfg.selectedCamera]) : "none"}`,
602406
602834
  `Audio input: ${cfg.selectedAudioInput || "none"}`,
602407
602835
  `Audio output: ${cfg.selectedAudioOutput || "none"}`,
602408
602836
  `Snapshot: ${liveSnapshotPath(snapshot.repoRoot)}`
@@ -602427,8 +602855,10 @@ var init_live_sensors = __esm({
602427
602855
  audioEnabled: false,
602428
602856
  audioOutputEnabled: false,
602429
602857
  inferEnabled: false,
602858
+ clipEnabled: false,
602430
602859
  asrEnabled: false,
602431
602860
  audioAnalysisEnabled: false,
602861
+ cameraOrientation: {},
602432
602862
  videoIntervalMs: 5e3,
602433
602863
  audioIntervalMs: 9e3
602434
602864
  };
@@ -602454,16 +602884,155 @@ var init_live_sensors = __esm({
602454
602884
  videoInFlight = false;
602455
602885
  audioInFlight = false;
602456
602886
  statusSink = null;
602887
+ feedbackSink = null;
602888
+ agentActionSink = null;
602889
+ agentActionEnabled = false;
602890
+ lastAgentReviewAt = 0;
602891
+ autoOrientationInFlight = /* @__PURE__ */ new Set();
602892
+ lastFeedbackAt = /* @__PURE__ */ new Map();
602457
602893
  setStatusSink(sink) {
602458
602894
  this.statusSink = sink ?? null;
602459
602895
  this.emitStatus();
602460
602896
  }
602897
+ setFeedbackSink(sink) {
602898
+ this.feedbackSink = sink ?? null;
602899
+ }
602900
+ setAgentActionSink(sink) {
602901
+ this.agentActionSink = sink ?? null;
602902
+ }
602461
602903
  getConfig() {
602462
602904
  return { ...this.config };
602463
602905
  }
602464
602906
  getSnapshot() {
602465
602907
  return this.snapshot;
602466
602908
  }
602909
+ activeVideoSources() {
602910
+ const ordered = [];
602911
+ const add3 = (id2) => {
602912
+ const clean5 = String(id2 ?? "").trim();
602913
+ if (!clean5 || ordered.includes(clean5)) return;
602914
+ const device = this.devices.video.find((entry) => entry.id === clean5);
602915
+ if (device && /metadata\/non-capture/i.test(device.detail ?? "")) return;
602916
+ ordered.push(clean5);
602917
+ };
602918
+ add3(this.config.selectedCamera);
602919
+ for (const device of this.devices.video) add3(device.id);
602920
+ if (ordered.length === 0) add3(firstDeviceId(this.devices.video));
602921
+ return ordered.slice(0, 4);
602922
+ }
602923
+ getCameraOrientation(device = this.config.selectedCamera) {
602924
+ return device ? this.config.cameraOrientation?.[device] : void 0;
602925
+ }
602926
+ getCameraRotation(device = this.config.selectedCamera) {
602927
+ return this.getCameraOrientation(device)?.rotation ?? 0;
602928
+ }
602929
+ setCameraRotation(device, rotation, source, evidence, confidence2) {
602930
+ const target = device || this.config.selectedCamera || firstDeviceId(this.devices.video);
602931
+ if (!target) return null;
602932
+ const orientation = {
602933
+ rotation,
602934
+ source,
602935
+ updatedAt: Date.now(),
602936
+ confidence: confidence2,
602937
+ evidence: evidence ? boundedText(evidence, 800) : void 0
602938
+ };
602939
+ this.config = {
602940
+ ...this.config,
602941
+ selectedCamera: target,
602942
+ cameraOrientation: {
602943
+ ...this.config.cameraOrientation ?? {},
602944
+ [target]: orientation
602945
+ }
602946
+ };
602947
+ this.snapshot.events = mergeRecentById(this.snapshot.events, [{
602948
+ id: stableEventId("camera_orientation", target, orientation.updatedAt, `${source}:${rotation}`),
602949
+ kind: "camera_orientation",
602950
+ observedAt: orientation.updatedAt,
602951
+ source: target,
602952
+ title: "Camera orientation correction updated",
602953
+ summary: `${target}: ${formatCameraRotation(rotation)} via ${source}${confidence2 !== void 0 ? ` confidence=${confidence2.toFixed(2)}` : ""}${evidence ? ` evidence=${boundedText(evidence, 300)}` : ""}`,
602954
+ confidence: confidence2
602955
+ }], 50);
602956
+ this.persist();
602957
+ this.emitFeedback({
602958
+ kind: "orientation",
602959
+ title: "Camera orientation correction updated",
602960
+ observedAt: orientation.updatedAt,
602961
+ message: `${target}: ${describeCameraOrientation(orientation)}`
602962
+ });
602963
+ return orientation;
602964
+ }
602965
+ cycleCameraRotation(device = this.config.selectedCamera) {
602966
+ const current = this.getCameraRotation(device);
602967
+ const next = current === 0 ? 90 : current === 90 ? 180 : current === 180 ? 270 : 0;
602968
+ return this.setCameraRotation(device, next, "manual", "manual cycle from /live menu");
602969
+ }
602970
+ async autoOrientCamera(device = this.config.selectedCamera, options2 = {}) {
602971
+ const target = device || this.config.selectedCamera || firstDeviceId(this.devices.video);
602972
+ if (!target) return "No camera selected. Use /live camera <device> or select a camera in /live.";
602973
+ if (options2.onlyIfUnset && this.getCameraOrientation(target)) {
602974
+ return `Camera orientation already set for ${target}: ${describeCameraOrientation(this.getCameraOrientation(target))}`;
602975
+ }
602976
+ if (this.autoOrientationInFlight.has(target)) {
602977
+ return `Camera orientation detection is already running for ${target}.`;
602978
+ }
602979
+ this.autoOrientationInFlight.add(target);
602980
+ try {
602981
+ const captured = await new CameraCaptureTool().execute({
602982
+ action: "capture",
602983
+ device: target,
602984
+ width: 640,
602985
+ height: 480,
602986
+ rotate_degrees: 0
602987
+ });
602988
+ if (!captured.success) {
602989
+ return `Camera orientation detection could not capture ${target}: ${captured.error || captured.output || "unknown error"}`;
602990
+ }
602991
+ const framePath = extractSavedImagePath(captured.output, this.repoRoot);
602992
+ if (!framePath) return `Camera orientation detection captured ${target}, but no frame path was returned.`;
602993
+ const cvScores = await scoreCameraOrientationWithOpenCv(framePath);
602994
+ const cvDecision = cvScores ? chooseCameraOrientationFromScores(cvScores) : null;
602995
+ if (cvDecision && cvDecision.confidence >= 0.68) {
602996
+ const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
602997
+ return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
602998
+ }
602999
+ const prompt = [
603000
+ "Determine whether this camera frame is upright for a human viewer.",
603001
+ "Return only JSON with this schema:",
603002
+ '{"rotation_degrees":0|90|180|270,"confidence":0.0-1.0,"reason":"brief visual evidence"}',
603003
+ "rotation_degrees is the clockwise correction to apply to future frames so they are upright.",
603004
+ "Use 90 if the image must rotate clockwise; use 270 if it must rotate counter-clockwise; use 180 if upside down; use 0 if already upright or inconclusive."
603005
+ ].join("\n");
603006
+ const vision = await new VisionTool(this.repoRoot).execute({
603007
+ action: "query",
603008
+ image: framePath,
603009
+ prompt,
603010
+ model: "moondream3-preview"
603011
+ });
603012
+ if (!vision.success) {
603013
+ if (cvDecision) {
603014
+ const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
603015
+ return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
603016
+ }
603017
+ return `Camera orientation detection captured ${target}, but vision inference was unavailable: ${vision.error || vision.output || "unknown error"}`;
603018
+ }
603019
+ const decision2 = parseCameraOrientationDecision(vision.llmContent || vision.output);
603020
+ if (!decision2) {
603021
+ if (cvDecision) {
603022
+ const orientation2 = this.setCameraRotation(target, cvDecision.rotation, "auto", cvDecision.reason, cvDecision.confidence);
603023
+ return orientation2 ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation2)}` : `Auto orientation for ${target} could not be persisted.`;
603024
+ }
603025
+ return `Camera orientation detection for ${target} was inconclusive. Vision output did not contain a structured rotation decision.`;
603026
+ }
603027
+ const useCv = cvDecision && (cvDecision.confidence > (decision2.confidence ?? 0) + 0.12 || cvDecision.rotation !== decision2.rotation && cvDecision.confidence >= 0.62 && (decision2.confidence ?? 0) < 0.76);
603028
+ const selected = useCv ? cvDecision : decision2;
603029
+ const evidence = useCv ? cvDecision.reason : `${decision2.reason || boundedText(vision.llmContent || vision.output, 320)}${cvDecision ? `; cv=${cvDecision.reason}` : ""}`;
603030
+ const orientation = this.setCameraRotation(target, selected.rotation, "auto", evidence, selected.confidence);
603031
+ return orientation ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation)}` : `Auto orientation for ${target} could not be persisted.`;
603032
+ } finally {
603033
+ this.autoOrientationInFlight.delete(target);
603034
+ }
603035
+ }
602467
603036
  async refreshDevices() {
602468
603037
  const errors = [];
602469
603038
  let video = [];
@@ -602504,31 +603073,105 @@ var init_live_sensors = __esm({
602504
603073
  return this.getConfig();
602505
603074
  }
602506
603075
  stopAll() {
603076
+ this.agentActionEnabled = false;
602507
603077
  this.configure({
602508
603078
  videoEnabled: false,
602509
603079
  audioEnabled: false,
602510
603080
  audioOutputEnabled: false,
602511
603081
  inferEnabled: false,
603082
+ clipEnabled: false,
602512
603083
  asrEnabled: false,
602513
603084
  audioAnalysisEnabled: false
602514
603085
  });
602515
603086
  }
602516
- async previewCamera(device = this.config.selectedCamera) {
603087
+ startRunMode(options2 = {}) {
603088
+ this.agentActionEnabled = Boolean(options2.agent);
603089
+ this.configure({
603090
+ videoEnabled: true,
603091
+ inferEnabled: true,
603092
+ clipEnabled: true,
603093
+ audioEnabled: true,
603094
+ asrEnabled: true,
603095
+ audioAnalysisEnabled: true,
603096
+ videoIntervalMs: options2.lowLatency === false ? this.config.videoIntervalMs : Math.min(this.config.videoIntervalMs, 1500),
603097
+ audioIntervalMs: options2.lowLatency === false ? this.config.audioIntervalMs : Math.min(this.config.audioIntervalMs, 3e3)
603098
+ });
603099
+ for (const source of this.activeVideoSources()) {
603100
+ const current = this.getCameraOrientation(source);
603101
+ if (current?.source === "manual") continue;
603102
+ if (current && current.confidence !== void 0 && current.confidence >= 0.75 && Date.now() - current.updatedAt < 15 * 6e4) continue;
603103
+ void this.autoOrientCamera(source).then((message2) => {
603104
+ this.emitFeedbackThrottled(`orient:${source}`, {
603105
+ kind: "orientation",
603106
+ title: "Camera auto-orientation",
603107
+ observedAt: Date.now(),
603108
+ message: message2
603109
+ }, 3e4);
603110
+ this.emitDashboard();
603111
+ }).catch((err) => {
603112
+ this.emitFeedbackThrottled(`orient-error:${source}`, {
603113
+ kind: "error",
603114
+ title: "Camera auto-orientation failed",
603115
+ observedAt: Date.now(),
603116
+ message: err instanceof Error ? err.message : String(err)
603117
+ }, 6e4);
603118
+ });
603119
+ }
603120
+ void this.sampleVideoNow(true).catch((err) => {
603121
+ this.emitFeedback({
603122
+ kind: "error",
603123
+ title: "Live video sample failed",
603124
+ observedAt: Date.now(),
603125
+ message: err instanceof Error ? err.message : String(err)
603126
+ });
603127
+ });
603128
+ void this.sampleAudioNow().catch((err) => {
603129
+ this.emitFeedback({
603130
+ kind: "error",
603131
+ title: "Live audio sample failed",
603132
+ observedAt: Date.now(),
603133
+ message: err instanceof Error ? err.message : String(err)
603134
+ });
603135
+ });
603136
+ this.emitFeedback({
603137
+ kind: "status",
603138
+ title: options2.agent ? "Live PFC run started" : "Live run started",
603139
+ observedAt: Date.now(),
603140
+ message: "Low-latency video, face, CLIP, ASR, and sound loops are active. Latest observations are injected into each agent turn."
603141
+ });
603142
+ return options2.agent ? "Live PFC run enabled: visual/audio feedback is active and significant events can launch background agent review." : "Live run enabled: visual/audio feedback is active and live context will update each turn.";
603143
+ }
603144
+ async previewCamera(device = this.config.selectedCamera, options2 = {}) {
602517
603145
  const source = device || firstDeviceId(this.devices.video);
602518
603146
  if (!source) return { ok: false, message: "No camera selected. Use /live to select a camera after device discovery." };
603147
+ const orientation = this.getCameraOrientation(source);
603148
+ const rotation = orientation?.rotation ?? 0;
602519
603149
  const result = await new CameraCaptureTool().execute({
602520
603150
  action: "capture",
602521
603151
  device: source,
602522
603152
  width: 640,
602523
- height: 480
603153
+ height: 480,
603154
+ rotate_degrees: rotation
602524
603155
  });
602525
603156
  if (!result.success) {
602526
603157
  const error = result.error || result.output || "camera capture failed";
602527
603158
  const observedAt2 = Date.now();
602528
- this.snapshot.video = {
603159
+ const errorView = {
602529
603160
  updatedAt: observedAt2,
602530
603161
  source,
602531
- error
603162
+ error,
603163
+ rotation,
603164
+ orientation
603165
+ };
603166
+ if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = errorView;
603167
+ this.snapshot.cameras = {
603168
+ ...this.snapshot.cameras ?? {},
603169
+ [source]: {
603170
+ ...this.snapshot.cameras?.[source] ?? { source, updatedAt: observedAt2 },
603171
+ ...errorView,
603172
+ updatedAt: observedAt2,
603173
+ source
603174
+ }
602532
603175
  };
602533
603176
  this.snapshot.events = mergeRecentById(this.snapshot.events, [{
602534
603177
  id: stableEventId("camera_error", source, observedAt2, error.slice(0, 80)),
@@ -602539,6 +603182,14 @@ var init_live_sensors = __esm({
602539
603182
  summary: error
602540
603183
  }], 50);
602541
603184
  this.persist();
603185
+ if (options2.emit !== false) {
603186
+ this.emitFeedbackThrottled(`camera-capture:${source}:${error.slice(0, 80)}`, {
603187
+ kind: "error",
603188
+ title: "Camera capture failed",
603189
+ observedAt: observedAt2,
603190
+ message: error
603191
+ }, 3e4);
603192
+ }
602542
603193
  return { ok: false, message: error };
602543
603194
  }
602544
603195
  const framePath = extractSavedImagePath(result.output, this.repoRoot);
@@ -602547,12 +603198,25 @@ var init_live_sensors = __esm({
602547
603198
  const preview = await buildImageAsciiPreview(framePath, { width: 72, height: 22 });
602548
603199
  const asciiContext = preview ? formatImageAsciiContext(preview, displayPath) : void 0;
602549
603200
  const observedAt = Date.now();
602550
- this.snapshot.video = {
603201
+ const cameraView = {
602551
603202
  updatedAt: observedAt,
602552
603203
  source,
602553
603204
  framePath,
602554
603205
  displayPath,
602555
- asciiContext
603206
+ ascii: preview?.ascii,
603207
+ renderer: preview?.renderer,
603208
+ asciiContext,
603209
+ rotation,
603210
+ orientation
603211
+ };
603212
+ if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = cameraView;
603213
+ this.snapshot.cameras = {
603214
+ ...this.snapshot.cameras ?? {},
603215
+ [source]: {
603216
+ ...cameraView,
603217
+ summary: this.snapshot.cameras?.[source]?.summary,
603218
+ inference: this.snapshot.cameras?.[source]?.inference
603219
+ }
602556
603220
  };
602557
603221
  this.snapshot.events = mergeRecentById(this.snapshot.events, [{
602558
603222
  id: stableEventId("camera_frame", source, observedAt, framePath),
@@ -602560,10 +603224,22 @@ var init_live_sensors = __esm({
602560
603224
  observedAt,
602561
603225
  source,
602562
603226
  title: "Captured camera frame",
602563
- summary: `frame=${framePath}${displayPath ? ` display=${displayPath}` : ""}`,
603227
+ summary: `frame=${framePath}${displayPath ? ` display=${displayPath}` : ""}${orientation ? ` rotation=${formatCameraRotation(orientation.rotation)}` : ""}`,
602564
603228
  evidencePath: framePath
602565
603229
  }], 50);
602566
603230
  this.persist();
603231
+ if (options2.emit !== false) {
603232
+ this.emitFeedback({
603233
+ kind: "camera-frame",
603234
+ title: "Live camera frame",
603235
+ observedAt,
603236
+ message: `Captured from ${source}${orientation ? ` (${formatCameraRotation(orientation.rotation)})` : ""}: ${framePath}`,
603237
+ imagePath: framePath,
603238
+ displayPath,
603239
+ ascii: preview?.ascii,
603240
+ renderer: preview?.renderer
603241
+ });
603242
+ }
602567
603243
  return {
602568
603244
  ok: true,
602569
603245
  message: `Camera frame captured from ${source}: ${framePath}`,
@@ -602573,46 +603249,144 @@ var init_live_sensors = __esm({
602573
603249
  renderer: preview?.renderer
602574
603250
  };
602575
603251
  }
603252
+ async recognizeFrameObjects(framePath, source, observedAt = Date.now(), options2 = {}) {
603253
+ try {
603254
+ const result = await new VisualMemoryTool().execute({
603255
+ action: "recognize",
603256
+ image: framePath,
603257
+ format: "json",
603258
+ auto_setup: true,
603259
+ timeout_ms: 45e3
603260
+ });
603261
+ const summary = result.success ? boundedText(result.llmContent || result.output || "No visual-memory object matches returned.", 1400) : `CLIP/visual-memory unavailable: ${result.error || result.output || "unknown error"}`;
603262
+ this.snapshot.events = mergeRecentById(this.snapshot.events, [{
603263
+ id: stableEventId("clip_recognition", source, observedAt, framePath),
603264
+ kind: "clip_recognition",
603265
+ observedAt,
603266
+ source,
603267
+ title: result.success ? "CLIP visual-memory recognition" : "CLIP visual-memory unavailable",
603268
+ summary,
603269
+ evidencePath: framePath
603270
+ }], 50);
603271
+ this.persist();
603272
+ if (options2.emit !== false) {
603273
+ this.emitFeedback({
603274
+ kind: "clip",
603275
+ title: result.success ? "CLIP visual-memory recognition" : "CLIP visual-memory unavailable",
603276
+ observedAt,
603277
+ message: summary
603278
+ });
603279
+ }
603280
+ return summary;
603281
+ } catch (err) {
603282
+ const message2 = `CLIP/visual-memory failed: ${err instanceof Error ? err.message : String(err)}`;
603283
+ if (options2.emit !== false) {
603284
+ this.emitFeedbackThrottled(`clip:${source}`, {
603285
+ kind: "error",
603286
+ title: "CLIP visual-memory failed",
603287
+ observedAt,
603288
+ message: message2
603289
+ }, 6e4);
603290
+ }
603291
+ return message2;
603292
+ }
603293
+ }
603294
+ async sampleSingleVideoNow(source, forceInfer) {
603295
+ const preview = await this.previewCamera(source, { emit: false });
603296
+ let inference = "";
603297
+ let clipSummary = "";
603298
+ const orientation = this.getCameraOrientation(source);
603299
+ const sampledAt = Date.now();
603300
+ if (forceInfer && source) {
603301
+ const loop = new LiveMediaLoopTool(this.repoRoot);
603302
+ const result = await loop.execute({
603303
+ action: "watch",
603304
+ source_kind: "camera",
603305
+ camera: source,
603306
+ duration_sec: 2.5,
603307
+ sample_interval_sec: 1,
603308
+ max_frames: 2,
603309
+ rotate_degrees: orientation?.rotation ?? 0,
603310
+ auto_bootstrap: true,
603311
+ audio_mode: "none",
603312
+ detect_objects: true,
603313
+ track_objects: true,
603314
+ crop_faces: true,
603315
+ recognize_faces: true
603316
+ });
603317
+ inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
603318
+ const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
603319
+ const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
603320
+ this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
603321
+ this.snapshot.people = mergeRecentById(this.snapshot.people, observations.people, 24);
603322
+ const existing = this.snapshot.cameras?.[source] ?? this.snapshot.video ?? { updatedAt: Date.now(), source };
603323
+ const camera = {
603324
+ ...existing,
603325
+ updatedAt: sampledAt,
603326
+ source,
603327
+ inference,
603328
+ summary: summarizeInference(inference),
603329
+ rotation: orientation?.rotation ?? 0,
603330
+ orientation,
603331
+ ...preview.ok ? { error: void 0 } : { error: preview.message }
603332
+ };
603333
+ this.snapshot.cameras = {
603334
+ ...this.snapshot.cameras ?? {},
603335
+ [source]: camera
603336
+ };
603337
+ if (source === this.config.selectedCamera || !this.snapshot.video) this.snapshot.video = camera;
603338
+ this.persist();
603339
+ if (!result.success) {
603340
+ this.emitFeedbackThrottled(`video-infer:${source}`, {
603341
+ kind: "error",
603342
+ title: "Live video inference failed",
603343
+ observedAt: sampledAt,
603344
+ message: boundedText(inference, 1200)
603345
+ }, 45e3);
603346
+ }
603347
+ const unknownPeople = observations.people.filter((entry) => entry.status === "unknown");
603348
+ if (unknownPeople.length > 0) {
603349
+ this.maybeRequestAgentReview(
603350
+ `${unknownPeople.length} unknown individual(s) observed on live camera ${source}`,
603351
+ unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; ")
603352
+ );
603353
+ }
603354
+ const triggerEvents = observations.events.filter((entry) => entry.kind === "visual_trigger");
603355
+ if (triggerEvents.length > 0) {
603356
+ this.maybeRequestAgentReview(
603357
+ `Visual trigger hit in live camera stream ${source}`,
603358
+ triggerEvents.map((entry) => entry.summary).join("; ")
603359
+ );
603360
+ }
603361
+ }
603362
+ if (this.config.clipEnabled && preview.ok && preview.framePath && source) {
603363
+ clipSummary = await this.recognizeFrameObjects(preview.framePath, source, sampledAt, { emit: false });
603364
+ const current = this.snapshot.cameras?.[source];
603365
+ if (current) {
603366
+ current.summary = [current.summary, clipSummary ? `clip: ${trimOneLine(clipSummary, 160)}` : ""].filter(Boolean).join(" | ");
603367
+ this.snapshot.cameras = { ...this.snapshot.cameras ?? {}, [source]: current };
603368
+ if (this.snapshot.video?.source === source) this.snapshot.video = current;
603369
+ this.persist();
603370
+ }
603371
+ }
603372
+ return [preview.message, inference ? `
603373
+ ${inference}` : "", clipSummary ? `
603374
+ CLIP visual memory:
603375
+ ${clipSummary}` : ""].filter(Boolean).join("\n");
603376
+ }
602576
603377
  async sampleVideoNow(forceInfer = this.config.inferEnabled) {
602577
603378
  if (this.videoInFlight) return "Video sampler is already running.";
602578
603379
  this.videoInFlight = true;
602579
603380
  try {
602580
- const preview = await this.previewCamera(this.config.selectedCamera);
602581
- let inference = "";
602582
- const source = this.config.selectedCamera || firstDeviceId(this.devices.video);
602583
- if (forceInfer && source) {
602584
- const loop = new LiveMediaLoopTool(this.repoRoot);
602585
- const result = await loop.execute({
602586
- action: "watch",
602587
- source_kind: "camera",
602588
- camera: source,
602589
- duration_sec: 4,
602590
- sample_interval_sec: 1,
602591
- max_frames: 4,
602592
- auto_bootstrap: true,
602593
- audio_mode: "none",
602594
- detect_objects: true,
602595
- track_objects: true,
602596
- crop_faces: true,
602597
- recognize_faces: true
602598
- });
602599
- const sampledAt = Date.now();
602600
- inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
602601
- const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
602602
- const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
602603
- this.snapshot.events = mergeRecentById(this.snapshot.events, observations.events, 50);
602604
- this.snapshot.people = mergeRecentById(this.snapshot.people, observations.people, 24);
602605
- this.snapshot.video = {
602606
- ...this.snapshot.video ?? { updatedAt: Date.now(), source },
602607
- updatedAt: sampledAt,
602608
- source,
602609
- inference,
602610
- ...preview.ok ? {} : { error: preview.message }
602611
- };
602612
- this.persist();
603381
+ const sources = this.activeVideoSources();
603382
+ if (sources.length === 0) return "No camera selected. Use /live refresh after connecting hardware.";
603383
+ const outputs = [];
603384
+ for (const source of sources) {
603385
+ outputs.push(await this.sampleSingleVideoNow(source, forceInfer));
602613
603386
  }
602614
- return [preview.message, inference ? `
602615
- ${inference}` : ""].filter(Boolean).join("\n");
603387
+ this.emitDashboard();
603388
+ return outputs.map((output, index) => `## Camera ${sources[index]}
603389
+ ${output}`).join("\n\n");
602616
603390
  } finally {
602617
603391
  this.videoInFlight = false;
602618
603392
  }
@@ -602627,10 +603401,9 @@ ${inference}` : ""].filter(Boolean).join("\n");
602627
603401
  const recordingPath = join124(outputDir2, `audio-${Date.now()}.wav`);
602628
603402
  let transcript = "";
602629
603403
  let analysis = "";
602630
- try {
602631
- await recordAudioSample(input, 3, recordingPath);
602632
- } catch (err) {
602633
- const message2 = `Audio capture failed from ${input}: ${err instanceof Error ? err.message : String(err)}`;
603404
+ const capture = await recordAudioSampleWithRecovery(input, this.devices.audioInputs, 3, recordingPath);
603405
+ if (!capture.ok) {
603406
+ const message2 = `Audio capture failed from ${input}; tried fallbacks: ${capture.errors.slice(0, 5).join(" | ")}`;
602634
603407
  this.snapshot.audio = {
602635
603408
  updatedAt: Date.now(),
602636
603409
  input,
@@ -602638,8 +603411,18 @@ ${inference}` : ""].filter(Boolean).join("\n");
602638
603411
  error: message2
602639
603412
  };
602640
603413
  this.persist();
603414
+ this.emitFeedbackThrottled(`audio-capture:${input}`, {
603415
+ kind: "error",
603416
+ title: "Live audio capture failed",
603417
+ observedAt: this.snapshot.audio.updatedAt,
603418
+ message: message2
603419
+ }, 6e4);
603420
+ this.emitDashboard();
602641
603421
  return message2;
602642
603422
  }
603423
+ if (capture.device !== this.config.selectedAudioInput) {
603424
+ this.config.selectedAudioInput = capture.device;
603425
+ }
602643
603426
  if (this.config.asrEnabled) {
602644
603427
  try {
602645
603428
  const asr = await new TranscribeFileTool(this.repoRoot).execute({ path: recordingPath, model: "tiny" });
@@ -602667,7 +603450,7 @@ ${inference}` : ""].filter(Boolean).join("\n");
602667
603450
  }
602668
603451
  this.snapshot.audio = {
602669
603452
  updatedAt: Date.now(),
602670
- input,
603453
+ input: capture.device,
602671
603454
  output: this.config.selectedAudioOutput,
602672
603455
  recordingPath,
602673
603456
  transcript,
@@ -602676,10 +603459,10 @@ ${inference}` : ""].filter(Boolean).join("\n");
602676
603459
  const audioEvents = [];
602677
603460
  if (transcript) {
602678
603461
  audioEvents.push({
602679
- id: stableEventId("audio_transcript", input, this.snapshot.audio.updatedAt, recordingPath),
603462
+ id: stableEventId("audio_transcript", capture.device, this.snapshot.audio.updatedAt, recordingPath),
602680
603463
  kind: "audio_transcript",
602681
603464
  observedAt: this.snapshot.audio.updatedAt,
602682
- source: input,
603465
+ source: capture.device,
602683
603466
  title: "Live ASR transcript",
602684
603467
  summary: transcript,
602685
603468
  evidencePath: recordingPath
@@ -602687,10 +603470,10 @@ ${inference}` : ""].filter(Boolean).join("\n");
602687
603470
  }
602688
603471
  if (analysis) {
602689
603472
  audioEvents.push({
602690
- id: stableEventId("audio_sound", input, this.snapshot.audio.updatedAt, recordingPath),
603473
+ id: stableEventId("audio_sound", capture.device, this.snapshot.audio.updatedAt, recordingPath),
602691
603474
  kind: "audio_sound",
602692
603475
  observedAt: this.snapshot.audio.updatedAt,
602693
- source: input,
603476
+ source: capture.device,
602694
603477
  title: "Live sound analysis",
602695
603478
  summary: analysis,
602696
603479
  evidencePath: recordingPath
@@ -602698,8 +603481,9 @@ ${inference}` : ""].filter(Boolean).join("\n");
602698
603481
  }
602699
603482
  this.snapshot.events = mergeRecentById(this.snapshot.events, audioEvents, 50);
602700
603483
  this.persist();
603484
+ this.emitDashboard();
602701
603485
  return [
602702
- `Audio sample captured from ${input}: ${recordingPath}`,
603486
+ `Audio sample captured from ${capture.device}${capture.device !== input ? ` (recovered from ${input})` : ""}: ${recordingPath}`,
602703
603487
  transcript ? `
602704
603488
  ASR:
602705
603489
  ${transcript}` : "",
@@ -602713,10 +603497,10 @@ ${analysis}` : ""
602713
603497
  }
602714
603498
  ensureLoops() {
602715
603499
  if (this.config.videoEnabled && !this.videoTimer) {
602716
- void this.sampleVideoNow(false).catch(() => {
603500
+ void this.sampleVideoNow(this.config.inferEnabled || this.config.clipEnabled).catch(() => {
602717
603501
  });
602718
603502
  this.videoTimer = setInterval(() => {
602719
- void this.sampleVideoNow(this.config.inferEnabled).catch(() => {
603503
+ void this.sampleVideoNow(this.config.inferEnabled || this.config.clipEnabled).catch(() => {
602720
603504
  });
602721
603505
  }, this.config.videoIntervalMs);
602722
603506
  this.videoTimer.unref?.();
@@ -602745,6 +603529,53 @@ ${analysis}` : ""
602745
603529
  label: "live"
602746
603530
  });
602747
603531
  }
603532
+ emitFeedback(feedback) {
603533
+ try {
603534
+ this.feedbackSink?.(feedback);
603535
+ } catch {
603536
+ }
603537
+ }
603538
+ emitFeedbackThrottled(key, feedback, minIntervalMs) {
603539
+ const now2 = Date.now();
603540
+ const last2 = this.lastFeedbackAt.get(key) ?? 0;
603541
+ if (now2 - last2 < minIntervalMs) return;
603542
+ this.lastFeedbackAt.set(key, now2);
603543
+ this.emitFeedback(feedback);
603544
+ }
603545
+ emitDashboard() {
603546
+ this.emitFeedback({
603547
+ kind: "dashboard",
603548
+ title: "Live audio/video monitor",
603549
+ observedAt: Date.now(),
603550
+ message: "live dashboard refresh"
603551
+ });
603552
+ }
603553
+ maybeRequestAgentReview(reason, evidence) {
603554
+ if (!this.agentActionEnabled || !this.agentActionSink) return;
603555
+ const now2 = Date.now();
603556
+ if (now2 - this.lastAgentReviewAt < 2e4) return;
603557
+ this.lastAgentReviewAt = now2;
603558
+ const prompt = [
603559
+ "Live environment PFC review.",
603560
+ "Inspect the current live sensor evidence and decide whether any immediate action is needed.",
603561
+ "Do not invent person names. If a person is unknown and identity matters, ask the user or person for their name and then store the association with visual_memory or multimodal_memory.",
603562
+ "Prefer minimal, reversible actions. If no action is needed, summarize the observation and stand by.",
603563
+ "",
603564
+ `Reason: ${reason}`,
603565
+ `Evidence: ${evidence}`,
603566
+ `Live snapshot: ${liveSnapshotPath(this.repoRoot)}`
603567
+ ].join("\n");
603568
+ this.emitFeedback({
603569
+ kind: "agent-review",
603570
+ title: "Live PFC review queued",
603571
+ observedAt: now2,
603572
+ message: reason
603573
+ });
603574
+ try {
603575
+ this.agentActionSink(prompt);
603576
+ } catch {
603577
+ }
603578
+ }
602748
603579
  persist() {
602749
603580
  ensureLiveDir(this.repoRoot);
602750
603581
  writeFileSync56(liveConfigPath(this.repoRoot), JSON.stringify(this.config, null, 2), "utf8");
@@ -608474,8 +609305,13 @@ var init_command_registry = __esm({
608474
609305
  ["/listen stop", "Stop listening"],
608475
609306
  ["/live", "Open live sensor stream menu for video/audio context"],
608476
609307
  ["/live status", "Show live sensor devices, selected streams, and latest snapshot"],
609308
+ ["/live run [agent]", "Start low-latency video/audio feedback; agent enables background PFC review"],
609309
+ ["/live chat", "Start low-latency voicechat with live audio/video context and async agent review"],
609310
+ ["/livechat", "Alias for /live chat, used by the top live button"],
608477
609311
  ["/live camera <device>", "Select a camera and render an ASCII preview"],
609312
+ ["/live rotate auto|cw|ccw|180|none [camera]", "Detect or set persistent camera orientation correction"],
608478
609313
  ["/live infer [now|on|off]", "Toggle or run camera object/location inference"],
609314
+ ["/live clip [on|off]", "Toggle CLIP visual-memory recognition on live frames"],
608479
609315
  ["/live audio|asr|sounds on|off", "Toggle live audio context streams"],
608480
609316
  ["/live stop", "Disable all live sensor streams"],
608481
609317
  ["/paste", "Attach clipboard image content to the current or next prompt"],
@@ -618863,7 +619699,7 @@ __export(omnius_directory_exports, {
618863
619699
  writeIndexMeta: () => writeIndexMeta,
618864
619700
  writeTaskHandoff: () => writeTaskHandoff2
618865
619701
  });
618866
- import { appendFileSync as appendFileSync13, cpSync as cpSync2, existsSync as existsSync117, mkdirSync as mkdirSync74, readFileSync as readFileSync95, writeFileSync as writeFileSync63, readdirSync as readdirSync40, statSync as statSync44, unlinkSync as unlinkSync22, openSync as openSync2, closeSync as closeSync2, renameSync as renameSync10, watch as fsWatch2 } from "node:fs";
619702
+ import { appendFileSync as appendFileSync13, cpSync as cpSync2, existsSync as existsSync117, mkdirSync as mkdirSync74, readFileSync as readFileSync95, writeFileSync as writeFileSync63, readdirSync as readdirSync40, statSync as statSync44, unlinkSync as unlinkSync22, openSync as openSync2, closeSync as closeSync2, renameSync as renameSync11, watch as fsWatch2 } from "node:fs";
618867
619703
  import { join as join133, relative as relative13, basename as basename25, dirname as dirname40, resolve as resolve57 } from "node:path";
618868
619704
  import { homedir as homedir41 } from "node:os";
618869
619705
  import { createHash as createHash37 } from "node:crypto";
@@ -619269,7 +620105,7 @@ function writeTaskHandoff2(repoRoot, handoff) {
619269
620105
  const tempPath = filePath + ".tmp";
619270
620106
  writeFileSync63(tempPath, JSON.stringify(handoff, null, 2) + "\n", "utf-8");
619271
620107
  try {
619272
- renameSync10(tempPath, filePath);
620108
+ renameSync11(tempPath, filePath);
619273
620109
  } catch {
619274
620110
  writeFileSync63(filePath, JSON.stringify(handoff, null, 2) + "\n", "utf-8");
619275
620111
  try {
@@ -619579,7 +620415,7 @@ function saveSessionContext(repoRoot, entry) {
619579
620415
  const tempFilePath = filePath + ".tmp";
619580
620416
  writeFileSync63(tempFilePath, JSON.stringify(ctx3, null, 2) + "\n", "utf-8");
619581
620417
  try {
619582
- renameSync10(tempFilePath, filePath);
620418
+ renameSync11(tempFilePath, filePath);
619583
620419
  } catch {
619584
620420
  writeFileSync63(filePath, JSON.stringify(ctx3, null, 2) + "\n", "utf-8");
619585
620421
  try {
@@ -622522,7 +623358,7 @@ function setTerminalTitle(task, version5) {
622522
623358
  process.stdout.write(data);
622523
623359
  }
622524
623360
  }
622525
- var EXPERT_TOOL_BASELINES, CONTEXT_SWITCH_OVERHEAD, TURN_PLANNING_OVERHEAD, DEFAULT_TOOL_BASELINE, CODE_READ_CHARS_PER_SEC, PROSE_READ_CHARS_PER_SEC, MIN_CONTENT_FOR_READING, CODE_CONTENT_TOOLS, PROSE_CONTENT_TOOLS, HumanSpeedTracker, PANEL_BG_SEQ, CONTENT_BG_SEQ, BOX_FG, TEXT_PRIMARY, TEXT_DIM, NO_SUB_AGENTS_HEADER_LABEL, HEADER_BUTTON_GLYPH_FG, HEADER_BUTTON_BG, HEADER_BUTTON_FG, HEADER_ACCENT_BOLD_FG, HEADER_TELEGRAM_FG, BOX_TL3, BOX_TR3, BOX_BL3, BOX_BR3, BOX_H3, BOX_V3, BOX_BJ, _globalFooterLock, RESET4, CURSOR_BLINK_BLOCK, _isWindows, SPONSOR_HEADER_LABEL_MAX, _termTitleWriter, StatusBar;
623361
+ var EXPERT_TOOL_BASELINES, CONTEXT_SWITCH_OVERHEAD, TURN_PLANNING_OVERHEAD, DEFAULT_TOOL_BASELINE, CODE_READ_CHARS_PER_SEC, PROSE_READ_CHARS_PER_SEC, MIN_CONTENT_FOR_READING, CODE_CONTENT_TOOLS, PROSE_CONTENT_TOOLS, HumanSpeedTracker, PANEL_BG_SEQ, CONTENT_BG_SEQ, BOX_FG, TEXT_PRIMARY, TEXT_DIM, NO_SUB_AGENTS_HEADER_LABEL, HEADER_BUTTON_GLYPH_FG, HEADER_BUTTON_BG, HEADER_BUTTON_FG, HEADER_ACCENT_BOLD_FG, HEADER_TELEGRAM_FG, BOX_TL3, BOX_TR3, BOX_BL3, BOX_BR3, BOX_H3, BOX_V3, BOX_BJ, _globalFooterLock, RESET4, CURSOR_BLINK_BLOCK, _isWindows, HEADER_BUTTON_LEFT, HEADER_BUTTON_RIGHT, SPONSOR_HEADER_LABEL_MAX, _termTitleWriter, StatusBar;
622526
623362
  var init_status_bar = __esm({
622527
623363
  "packages/cli/src/tui/status-bar.ts"() {
622528
623364
  "use strict";
@@ -622719,6 +623555,8 @@ var init_status_bar = __esm({
622719
623555
  RESET4 = "\x1B[0m";
622720
623556
  CURSOR_BLINK_BLOCK = "\x1B[1 q";
622721
623557
  _isWindows = process.platform === "win32";
623558
+ HEADER_BUTTON_LEFT = _isWindows ? "[" : "🭁";
623559
+ HEADER_BUTTON_RIGHT = _isWindows ? "]" : "🭝";
622722
623560
  SPONSOR_HEADER_LABEL_MAX = 48;
622723
623561
  _termTitleWriter = null;
622724
623562
  StatusBar = class _StatusBar {
@@ -623267,27 +624105,30 @@ var init_status_bar = __esm({
623267
624105
  const buttonFg = (cmd) => {
623268
624106
  let fg2 = TEXT_DIM;
623269
624107
  if (cmd === "voice" && this._voiceActive) fg2 = 82;
623270
- if (cmd === "live" && (this._liveMediaStatus.audio || this._liveMediaStatus.video)) fg2 = 82;
624108
+ if (cmd.startsWith("live") && (this._liveMediaStatus.audio || this._liveMediaStatus.video)) fg2 = 82;
623271
624109
  if (cmd === "nexus") {
623272
624110
  fg2 = this._nexusStatus === "connected" ? 82 : this._nexusStatus === "connecting" ? 208 : 196;
623273
624111
  }
623274
624112
  return fg2;
623275
624113
  };
623276
624114
  const decorateMenuButton = (cmd, label) => {
623277
- return `${HEADER_BUTTON_GLYPH_FG}[\x1B[0m${HEADER_BUTTON_BG}${HEADER_BUTTON_FG}${label}\x1B[0m${PANEL_BG_SEQ}${HEADER_BUTTON_GLYPH_FG}]\x1B[0m${PANEL_BG_SEQ}`;
624115
+ return `${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_LEFT}\x1B[0m${HEADER_BUTTON_BG}${HEADER_BUTTON_FG}${label}\x1B[0m${PANEL_BG_SEQ}${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_RIGHT}\x1B[0m${PANEL_BG_SEQ}`;
623278
624116
  };
623279
624117
  const decorateAgentButton = (content, color, active) => {
623280
624118
  const bg = `\x1B[48;5;${color}m`;
623281
624119
  const fg2 = `\x1B[38;5;${contrastTextColor(color)}m`;
623282
624120
  const weight = active ? "\x1B[1m" : "";
623283
- return `\x1B[38;5;${color}m[\x1B[0m${weight}${fg2}${bg}${content}\x1B[0m${PANEL_BG_SEQ}\x1B[38;5;${color}m]\x1B[0m${PANEL_BG_SEQ}`;
624121
+ return `\x1B[38;5;${color}m${HEADER_BUTTON_LEFT}\x1B[0m${weight}${fg2}${bg}${content}\x1B[0m${PANEL_BG_SEQ}\x1B[38;5;${color}m${HEADER_BUTTON_RIGHT}\x1B[0m${PANEL_BG_SEQ}`;
623284
624122
  };
623285
624123
  const renderBtn = (cmd, label) => {
623286
624124
  const fg2 = buttonFg(cmd);
623287
- const btnW = label.length;
624125
+ const btnW = label.length + 2;
623288
624126
  const availW2 = getTermWidth() - identity3.width - 1;
623289
624127
  if (btnW > availW2) return linkify(cmd, `\x1B[38;5;${fg2}m${label}`);
623290
- return linkify(cmd, `${PANEL_BG_SEQ}\x1B[38;5;${fg2}m${label}\x1B[0m${PANEL_BG_SEQ}`);
624128
+ return linkify(
624129
+ cmd,
624130
+ `${PANEL_BG_SEQ}${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_LEFT}\x1B[0m${PANEL_BG_SEQ}\x1B[38;5;${fg2}m${label}\x1B[0m${PANEL_BG_SEQ}${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_RIGHT}\x1B[0m${PANEL_BG_SEQ}`
624131
+ );
623291
624132
  };
623292
624133
  const identity3 = this.buildHeaderIdentityRender();
623293
624134
  const modelLabel = this.summarizeHeaderModelName() || "model";
@@ -623301,6 +624142,8 @@ var init_status_bar = __esm({
623301
624142
  w: (this._voiceActive ? this._voiceModelId || "voice" : "voice").length + 2
623302
624143
  },
623303
624144
  // +2 for spaces
624145
+ { cmd: "livechat", label: "live", w: "live".length + 2 },
624146
+ // +2 for spaces
623304
624147
  { cmd: "model", label: modelLabel, w: modelLabel.length + 2 },
623305
624148
  // +2 for spaces
623306
624149
  { cmd: "endpoint", label: endpointLabel, w: endpointLabel.length + 2 }
@@ -648807,6 +649650,10 @@ sleep 1
648807
649650
  await handleLiveCommand(ctx3, arg);
648808
649651
  return "handled";
648809
649652
  }
649653
+ case "livechat": {
649654
+ await handleLiveCommand(ctx3, "chat");
649655
+ return "handled";
649656
+ }
648810
649657
  case "call": {
648811
649658
  if (!ctx3.callStart) {
648812
649659
  renderWarning("Call mode not available in this context.");
@@ -654417,6 +655264,85 @@ function renderLivePreview(preview) {
654417
655264
  renderWarning("Camera preview was captured, but ASCII rendering was unavailable.");
654418
655265
  }
654419
655266
  }
655267
+ function renderLiveDashboard(ctx3, manager) {
655268
+ const host = ctx3.liveDynamicBlockHost;
655269
+ if (!host) {
655270
+ renderInfo(formatLiveDashboardFromSnapshot(manager.getSnapshot(), { width: 100 }).join("\n"));
655271
+ return;
655272
+ }
655273
+ const key = ctx3.repoRoot;
655274
+ let state = liveDashboardBlocks.get(key);
655275
+ if (!state || state.host !== host) {
655276
+ const id2 = `live-dashboard-${Math.random().toString(36).slice(2, 9)}`;
655277
+ state = { host, id: id2, manager };
655278
+ liveDashboardBlocks.set(key, state);
655279
+ host.registerDynamicBlock(
655280
+ id2,
655281
+ (width) => formatLiveDashboardFromSnapshot(manager.getSnapshot(), { width })
655282
+ );
655283
+ host.appendDynamicBlock(id2);
655284
+ return;
655285
+ }
655286
+ state.manager = manager;
655287
+ host.refreshDynamicBlocks?.();
655288
+ }
655289
+ function renderLiveFeedback(ctx3, manager, feedback) {
655290
+ if (feedback.kind === "dashboard") {
655291
+ renderLiveDashboard(ctx3, manager);
655292
+ return;
655293
+ }
655294
+ const stamp = new Date(feedback.observedAt).toISOString();
655295
+ if (feedback.kind === "error") {
655296
+ renderWarning(`[live ${stamp}] ${feedback.title}
655297
+ ${feedback.message}`);
655298
+ return;
655299
+ }
655300
+ if ((feedback.kind === "camera-frame" || feedback.kind === "face-crop") && feedback.ascii && feedback.displayPath && feedback.renderer) {
655301
+ renderInfo(`[live ${stamp}] ${feedback.message}`);
655302
+ renderImageAsciiPreview(feedback.title, feedback.displayPath, feedback.ascii, feedback.renderer);
655303
+ return;
655304
+ }
655305
+ renderInfo(`[live ${stamp}] ${feedback.title}
655306
+ ${feedback.message}`);
655307
+ }
655308
+ function attachLiveSinks(ctx3, manager, agentEnabled = false) {
655309
+ manager.setStatusSink(ctx3.setLiveMediaStatus);
655310
+ manager.setFeedbackSink((feedback) => renderLiveFeedback(ctx3, manager, feedback));
655311
+ if (agentEnabled && ctx3.startBackgroundPrompt) {
655312
+ manager.setAgentActionSink((prompt) => {
655313
+ const id2 = ctx3.startBackgroundPrompt?.(prompt);
655314
+ if (id2) renderInfo(`Live PFC background review started: ${id2}`);
655315
+ });
655316
+ } else {
655317
+ manager.setAgentActionSink(void 0);
655318
+ }
655319
+ }
655320
+ async function startLiveVoicechat(ctx3, manager) {
655321
+ const agentEnabled = Boolean(ctx3.startBackgroundPrompt);
655322
+ attachLiveSinks(ctx3, manager, agentEnabled);
655323
+ await ensureLiveInventory(manager);
655324
+ renderInfo(manager.startRunMode({ agent: agentEnabled, lowLatency: true }));
655325
+ if (!ctx3.voiceChatStart) {
655326
+ renderWarning("Voice chat is not available in this context; live audio/video feedback remains active.");
655327
+ return;
655328
+ }
655329
+ ctx3.voiceSetMode?.("voicechat");
655330
+ if (ctx3.isVoiceChatActive?.()) {
655331
+ renderInfo("Voice chat is already active. Live audio/video feedback remains attached.");
655332
+ return;
655333
+ }
655334
+ try {
655335
+ await ensureVoiceDeps(ctx3);
655336
+ } catch {
655337
+ }
655338
+ try {
655339
+ renderInfo("Starting live voicechat with audio/video context...");
655340
+ await ctx3.voiceChatStart();
655341
+ renderInfo("Live voicechat active: low-latency audio/video context is updating while voice conversation continues.");
655342
+ } catch (err) {
655343
+ renderError(`Live voicechat failed: ${err instanceof Error ? err.message : String(err)}`);
655344
+ }
655345
+ }
654420
655346
  async function chooseLiveDevice(ctx3, title, devices, activeKey) {
654421
655347
  if (devices.length === 0) {
654422
655348
  renderWarning("No devices found. Run /live refresh after connecting hardware.");
@@ -654439,7 +655365,7 @@ async function chooseLiveDevice(ctx3, title, devices, activeKey) {
654439
655365
  }
654440
655366
  async function handleLiveCommand(ctx3, arg) {
654441
655367
  const manager = getLiveSensorManager(ctx3.repoRoot);
654442
- manager.setStatusSink(ctx3.setLiveMediaStatus);
655368
+ attachLiveSinks(ctx3, manager);
654443
655369
  const parts = arg.trim().split(/\s+/).filter(Boolean);
654444
655370
  const sub2 = (parts[0] || "").toLowerCase();
654445
655371
  const value2 = parts.slice(1).join(" ");
@@ -654460,9 +655386,48 @@ async function handleLiveCommand(ctx3, arg) {
654460
655386
  }
654461
655387
  if (sub2 === "stop" || sub2 === "off" || sub2 === "disable") {
654462
655388
  manager.stopAll();
655389
+ manager.setAgentActionSink(void 0);
654463
655390
  renderInfo("Live sensor streams disabled.");
654464
655391
  return;
654465
655392
  }
655393
+ if (sub2 === "run" || sub2 === "start" || sub2 === "pfc") {
655394
+ const agentEnabled = sub2 === "pfc" || /\bagent|pfc|action\b/i.test(value2);
655395
+ const effectiveAgent = agentEnabled && Boolean(ctx3.startBackgroundPrompt);
655396
+ attachLiveSinks(ctx3, manager, effectiveAgent);
655397
+ if (agentEnabled && !ctx3.startBackgroundPrompt) {
655398
+ renderWarning("Live PFC agent review is unavailable in this context; starting visual/audio feedback only.");
655399
+ }
655400
+ await ensureLiveInventory(manager);
655401
+ renderInfo(manager.startRunMode({ agent: effectiveAgent, lowLatency: true }));
655402
+ return;
655403
+ }
655404
+ if (sub2 === "chat" || sub2 === "voicechat" || sub2 === "voice") {
655405
+ await startLiveVoicechat(ctx3, manager);
655406
+ return;
655407
+ }
655408
+ if (sub2 === "rotate" || sub2 === "rotation" || sub2 === "orient" || sub2 === "orientation") {
655409
+ await ensureLiveInventory(manager);
655410
+ const mode = (parts[1] || "cycle").toLowerCase();
655411
+ const deviceArg = parts.slice(2).join(" ") || manager.getConfig().selectedCamera;
655412
+ if (mode === "auto" || mode === "detect") {
655413
+ renderInfo("Detecting camera orientation...");
655414
+ renderInfo(await manager.autoOrientCamera(deviceArg));
655415
+ return;
655416
+ }
655417
+ if (mode === "cycle") {
655418
+ const orientation = manager.cycleCameraRotation(deviceArg);
655419
+ renderInfo(orientation ? `Camera rotation set to ${formatCameraRotation(orientation.rotation)} for ${deviceArg || manager.getConfig().selectedCamera || "selected camera"}.` : "No camera selected. Use /live camera <device> first.");
655420
+ return;
655421
+ }
655422
+ try {
655423
+ const rotation = normalizeLiveCameraRotation(mode);
655424
+ const orientation = manager.setCameraRotation(deviceArg, rotation, "manual", `/live rotate ${mode}`);
655425
+ renderInfo(orientation ? `Camera rotation set to ${formatCameraRotation(orientation.rotation)} for ${deviceArg || manager.getConfig().selectedCamera || "selected camera"}.` : "No camera selected. Use /live camera <device> first.");
655426
+ } catch (err) {
655427
+ renderWarning(err instanceof Error ? err.message : String(err));
655428
+ }
655429
+ return;
655430
+ }
654466
655431
  if (sub2 === "video") {
654467
655432
  const cfg = manager.getConfig();
654468
655433
  manager.configure({ videoEnabled: setBool(value2, !cfg.videoEnabled) });
@@ -654508,6 +655473,15 @@ async function handleLiveCommand(ctx3, arg) {
654508
655473
  }
654509
655474
  return;
654510
655475
  }
655476
+ if (sub2 === "clip") {
655477
+ const cfg = manager.getConfig();
655478
+ manager.configure({
655479
+ videoEnabled: true,
655480
+ clipEnabled: setBool(value2, !cfg.clipEnabled)
655481
+ });
655482
+ renderInfo(formatLiveStatus(manager.getSnapshot()));
655483
+ return;
655484
+ }
654511
655485
  if (sub2 === "camera") {
654512
655486
  await ensureLiveInventory(manager);
654513
655487
  const selected = value2 || manager.getConfig().selectedCamera;
@@ -654551,9 +655525,24 @@ async function showLiveMenu(ctx3, manager) {
654551
655525
  {
654552
655526
  key: "info:status",
654553
655527
  label: selectColors.dim(
654554
- ` video ${liveEnabled(cfg.videoEnabled)} · infer ${liveEnabled(cfg.inferEnabled)} · audio ${liveEnabled(cfg.audioEnabled)} · asr ${liveEnabled(cfg.asrEnabled)} · sounds ${liveEnabled(cfg.audioAnalysisEnabled)}`
655528
+ ` video ${liveEnabled(cfg.videoEnabled)} · infer ${liveEnabled(cfg.inferEnabled)} · clip ${liveEnabled(cfg.clipEnabled)} · audio ${liveEnabled(cfg.audioEnabled)} · asr ${liveEnabled(cfg.asrEnabled)} · sounds ${liveEnabled(cfg.audioAnalysisEnabled)}`
654555
655529
  )
654556
655530
  },
655531
+ {
655532
+ key: "run-live",
655533
+ label: "Start Live Run",
655534
+ detail: "low-latency visual/audio feedback + context injection"
655535
+ },
655536
+ {
655537
+ key: "run-agent",
655538
+ label: "Start Live PFC Run",
655539
+ detail: "live feedback + background agent review for significant events"
655540
+ },
655541
+ {
655542
+ key: "run-chat",
655543
+ label: "Start Live Voicechat",
655544
+ detail: "low-latency voice conversation with live audio/video context"
655545
+ },
654557
655546
  {
654558
655547
  key: "toggle-video",
654559
655548
  label: cfg.videoEnabled ? "Disable Video Context" : "Enable Video Context",
@@ -654569,11 +655558,26 @@ async function showLiveMenu(ctx3, manager) {
654569
655558
  label: "View Selected Camera",
654570
655559
  detail: "capture frame + image-to-ascii preview"
654571
655560
  },
655561
+ {
655562
+ key: "auto-orient-camera",
655563
+ label: "Auto-Orient Camera",
655564
+ detail: `current: ${cfg.selectedCamera ? formatCameraRotation(manager.getCameraRotation(cfg.selectedCamera)) : "no camera selected"}`
655565
+ },
655566
+ {
655567
+ key: "cycle-rotation",
655568
+ label: "Cycle Camera Rotation",
655569
+ detail: "manual override: upright -> clockwise -> 180 -> counter-clockwise"
655570
+ },
654572
655571
  {
654573
655572
  key: "toggle-infer",
654574
655573
  label: cfg.inferEnabled ? "Disable Frame Inference" : "Enable Frame Inference",
654575
655574
  detail: "YOLO object/location stream into context"
654576
655575
  },
655576
+ {
655577
+ key: "toggle-clip",
655578
+ label: cfg.clipEnabled ? "Disable CLIP Visual Memory" : "Enable CLIP Visual Memory",
655579
+ detail: "recognize taught objects from camera frames"
655580
+ },
654577
655581
  {
654578
655582
  key: "infer-now",
654579
655583
  label: "Infer Current Camera Frame",
@@ -654647,6 +655651,20 @@ async function showLiveMenu(ctx3, manager) {
654647
655651
  case "toggle-video":
654648
655652
  manager.configure({ videoEnabled: !cfg.videoEnabled });
654649
655653
  continue;
655654
+ case "run-live":
655655
+ attachLiveSinks(ctx3, manager, false);
655656
+ renderInfo(manager.startRunMode({ agent: false, lowLatency: true }));
655657
+ continue;
655658
+ case "run-agent":
655659
+ attachLiveSinks(ctx3, manager, true);
655660
+ if (!ctx3.startBackgroundPrompt) {
655661
+ renderWarning("Live PFC agent review is unavailable in this context; starting visual/audio feedback only.");
655662
+ }
655663
+ renderInfo(manager.startRunMode({ agent: Boolean(ctx3.startBackgroundPrompt), lowLatency: true }));
655664
+ continue;
655665
+ case "run-chat":
655666
+ await startLiveVoicechat(ctx3, manager);
655667
+ continue;
654650
655668
  case "toggle-audio":
654651
655669
  manager.configure({ audioEnabled: !cfg.audioEnabled });
654652
655670
  continue;
@@ -654656,6 +655674,9 @@ async function showLiveMenu(ctx3, manager) {
654656
655674
  case "toggle-infer":
654657
655675
  manager.configure({ videoEnabled: true, inferEnabled: !cfg.inferEnabled });
654658
655676
  continue;
655677
+ case "toggle-clip":
655678
+ manager.configure({ videoEnabled: true, clipEnabled: !cfg.clipEnabled });
655679
+ continue;
654659
655680
  case "toggle-asr":
654660
655681
  manager.configure({ audioEnabled: true, asrEnabled: !cfg.asrEnabled });
654661
655682
  continue;
@@ -654683,6 +655704,15 @@ async function showLiveMenu(ctx3, manager) {
654683
655704
  case "preview-camera":
654684
655705
  renderLivePreview(await manager.previewCamera());
654685
655706
  continue;
655707
+ case "auto-orient-camera":
655708
+ renderInfo("Detecting camera orientation...");
655709
+ renderInfo(await manager.autoOrientCamera(cfg.selectedCamera));
655710
+ continue;
655711
+ case "cycle-rotation": {
655712
+ const orientation = manager.cycleCameraRotation(cfg.selectedCamera);
655713
+ renderInfo(orientation ? `Camera rotation set to ${formatCameraRotation(orientation.rotation)} for ${cfg.selectedCamera || manager.getConfig().selectedCamera || "selected camera"}.` : "No camera selected. Use Select Camera first.");
655714
+ continue;
655715
+ }
654686
655716
  case "infer-now":
654687
655717
  manager.configure({ videoEnabled: true, inferEnabled: true });
654688
655718
  renderInfo("Sampling camera inference...");
@@ -659775,7 +660805,7 @@ async function showExposeDashboard(gateway, rl, ctx3) {
659775
660805
  renderInfo("Expose gateway stopped.");
659776
660806
  }
659777
660807
  }
659778
- var OMNIUS_PINNED_DEPENDENCY_SPECS, NEXUS_DIRECTORY_ORIGIN, NEXUS_SPONSORS_URL, _sponsorHeartbeatTimer, _lastRegisteredSponsorPayload, __COMMAND_REGISTRY, DASH_INTERNAL, localGpuMetricsCache, localGpuMetricsProbeAt, localGpuMetricsProbeInFlight;
660808
+ var OMNIUS_PINNED_DEPENDENCY_SPECS, NEXUS_DIRECTORY_ORIGIN, NEXUS_SPONSORS_URL, _sponsorHeartbeatTimer, _lastRegisteredSponsorPayload, __COMMAND_REGISTRY, liveDashboardBlocks, DASH_INTERNAL, localGpuMetricsCache, localGpuMetricsProbeAt, localGpuMetricsProbeInFlight;
659779
660809
  var init_commands = __esm({
659780
660810
  "packages/cli/src/tui/commands.ts"() {
659781
660811
  "use strict";
@@ -660093,6 +661123,7 @@ var init_commands = __esm({
660093
661123
  }
660094
661124
  });
660095
661125
  })();
661126
+ liveDashboardBlocks = /* @__PURE__ */ new Map();
660096
661127
  DASH_INTERNAL = /* @__PURE__ */ new Set(["system_metrics", "__list_capabilities"]);
660097
661128
  localGpuMetricsCache = null;
660098
661129
  localGpuMetricsProbeAt = 0;
@@ -661333,7 +662364,7 @@ import {
661333
662364
  readFileSync as readFileSync113,
661334
662365
  readdirSync as readdirSync49,
661335
662366
  writeFileSync as writeFileSync74,
661336
- renameSync as renameSync12,
662367
+ renameSync as renameSync13,
661337
662368
  mkdirSync as mkdirSync85,
661338
662369
  unlinkSync as unlinkSync29,
661339
662370
  appendFileSync as appendFileSync16
@@ -661357,7 +662388,7 @@ function persistSession(s2) {
661357
662388
  const final2 = sessionPath(s2.id);
661358
662389
  const tmp = `${final2}.tmp.${process.pid}.${Date.now()}`;
661359
662390
  writeFileSync74(tmp, JSON.stringify(s2, null, 2), "utf-8");
661360
- renameSync12(tmp, final2);
662391
+ renameSync13(tmp, final2);
661361
662392
  } catch {
661362
662393
  }
661363
662394
  }
@@ -661367,7 +662398,7 @@ function persistInFlight(j) {
661367
662398
  const final2 = inFlightPath(j.sessionId);
661368
662399
  const tmp = `${final2}.tmp.${process.pid}.${Date.now()}`;
661369
662400
  writeFileSync74(tmp, JSON.stringify(j, null, 2), "utf-8");
661370
- renameSync12(tmp, final2);
662401
+ renameSync13(tmp, final2);
661371
662402
  } catch {
661372
662403
  }
661373
662404
  }
@@ -694197,7 +695228,7 @@ __export(projects_exports, {
694197
695228
  setCurrentProject: () => setCurrentProject,
694198
695229
  unregisterProject: () => unregisterProject
694199
695230
  });
694200
- import { readFileSync as readFileSync124, writeFileSync as writeFileSync83, mkdirSync as mkdirSync95, existsSync as existsSync152, statSync as statSync55, renameSync as renameSync13 } from "node:fs";
695231
+ import { readFileSync as readFileSync124, writeFileSync as writeFileSync83, mkdirSync as mkdirSync95, existsSync as existsSync152, statSync as statSync55, renameSync as renameSync14 } from "node:fs";
694201
695232
  import { homedir as homedir55 } from "node:os";
694202
695233
  import { basename as basename40, join as join166, resolve as resolve68 } from "node:path";
694203
695234
  import { randomUUID as randomUUID19 } from "node:crypto";
@@ -694216,7 +695247,7 @@ function writeAll(file) {
694216
695247
  mkdirSync95(OMNIUS_DIR3, { recursive: true });
694217
695248
  const tmp = `${PROJECTS_FILE}.${randomUUID19().slice(0, 8)}.tmp`;
694218
695249
  writeFileSync83(tmp, JSON.stringify(file, null, 2), "utf8");
694219
- renameSync13(tmp, PROJECTS_FILE);
695250
+ renameSync14(tmp, PROJECTS_FILE);
694220
695251
  }
694221
695252
  function listProjects() {
694222
695253
  const { projects } = readAll2();
@@ -695195,7 +696226,7 @@ var init_access_policy = __esm({
695195
696226
 
695196
696227
  // packages/cli/src/api/project-preferences.ts
695197
696228
  import { createHash as createHash43 } from "node:crypto";
695198
- import { existsSync as existsSync153, mkdirSync as mkdirSync96, readFileSync as readFileSync125, renameSync as renameSync14, writeFileSync as writeFileSync84, unlinkSync as unlinkSync35 } from "node:fs";
696229
+ import { existsSync as existsSync153, mkdirSync as mkdirSync96, readFileSync as readFileSync125, renameSync as renameSync15, writeFileSync as writeFileSync84, unlinkSync as unlinkSync35 } from "node:fs";
695199
696230
  import { homedir as homedir56 } from "node:os";
695200
696231
  import { join as join167, resolve as resolve69 } from "node:path";
695201
696232
  import { randomUUID as randomUUID20 } from "node:crypto";
@@ -695249,7 +696280,7 @@ function writeProjectPreferences(root, partial) {
695249
696280
  const tmp = `${file}.${randomUUID20().slice(0, 8)}.tmp`;
695250
696281
  writeFileSync84(tmp, JSON.stringify(merged, null, 2), "utf8");
695251
696282
  try {
695252
- renameSync14(tmp, file);
696283
+ renameSync15(tmp, file);
695253
696284
  } catch (err) {
695254
696285
  try {
695255
696286
  writeFileSync84(file, JSON.stringify(merged, null, 2), "utf8");
@@ -696305,7 +697336,7 @@ var init_direct_tool_registry = __esm({
696305
697336
  });
696306
697337
 
696307
697338
  // packages/cli/src/api/external-tool-registry.ts
696308
- import { existsSync as existsSync157, mkdirSync as mkdirSync100, readFileSync as readFileSync127, writeFileSync as writeFileSync85, renameSync as renameSync15 } from "node:fs";
697339
+ import { existsSync as existsSync157, mkdirSync as mkdirSync100, readFileSync as readFileSync127, writeFileSync as writeFileSync85, renameSync as renameSync16 } from "node:fs";
696309
697340
  import { join as join170 } from "node:path";
696310
697341
  function externalToolStorePath(workingDir) {
696311
697342
  return join170(workingDir, ".omnius", "external-tools.json");
@@ -696329,7 +697360,7 @@ function persist3(workingDir, tools) {
696329
697360
  const file = { version: STORE_VERSION, tools };
696330
697361
  const tmp = `${path12}.tmp`;
696331
697362
  writeFileSync85(tmp, JSON.stringify(file, null, 2), { mode: 384 });
696332
- renameSync15(tmp, path12);
697363
+ renameSync16(tmp, path12);
696333
697364
  }
696334
697365
  function validateManifest2(input, existing) {
696335
697366
  if (!input || typeof input !== "object") {
@@ -714217,7 +715248,7 @@ import {
714217
715248
  readdirSync as readdirSync58,
714218
715249
  existsSync as existsSync165,
714219
715250
  watch as fsWatch4,
714220
- renameSync as renameSync16,
715251
+ renameSync as renameSync17,
714221
715252
  unlinkSync as unlinkSync36,
714222
715253
  statSync as statSync60,
714223
715254
  openSync as openSync6,
@@ -715874,7 +716905,7 @@ function atomicJobWrite(dir, id2, job) {
715874
716905
  const tmpPath = `${finalPath}.tmp.${process.pid}.${Date.now()}`;
715875
716906
  try {
715876
716907
  writeFileSync90(tmpPath, JSON.stringify(job, null, 2), "utf-8");
715877
- renameSync16(tmpPath, finalPath);
716908
+ renameSync17(tmpPath, finalPath);
715878
716909
  } catch {
715879
716910
  try {
715880
716911
  writeFileSync90(finalPath, JSON.stringify(job, null, 2), "utf-8");
@@ -717712,7 +718743,7 @@ function writeUpdateState(state) {
717712
718743
  const finalPath = updateStateFile();
717713
718744
  const tmpPath = `${finalPath}.tmp.${process.pid}`;
717714
718745
  writeFileSync90(tmpPath, JSON.stringify(state, null, 2), "utf-8");
717715
- renameSync16(tmpPath, finalPath);
718746
+ renameSync17(tmpPath, finalPath);
717716
718747
  } catch {
717717
718748
  }
717718
718749
  }
@@ -732414,6 +733445,11 @@ Respond concisely and safely. Remember: you are talking to the general public.`;
732414
733445
  setLiveMediaStatus(status) {
732415
733446
  statusBar.setLiveMediaStatus(status);
732416
733447
  },
733448
+ liveDynamicBlockHost: {
733449
+ registerDynamicBlock: (id2, render2) => statusBar.registerDynamicBlock(id2, render2),
733450
+ appendDynamicBlock: (id2) => statusBar.appendDynamicBlock(id2),
733451
+ refreshDynamicBlocks: () => statusBar.refreshDynamicBlocks()
733452
+ },
732417
733453
  // Voice call session — standalone cloudflared tunnel for /call
732418
733454
  // Each connecting WebSocket client gets a dedicated CallSubAgent.
732419
733455
  // Admin callers (matching session key) get full tool access; public callers get read-only.