omnius 1.0.406 → 1.0.407

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -542978,9 +542978,45 @@ var init_process_health = __esm({
542978
542978
  });
542979
542979
 
542980
542980
  // packages/execution/dist/tools/camera-capture.js
542981
- import { accessSync, constants, readFileSync as readFileSync44, writeFileSync as writeFileSync28, unlinkSync as unlinkSync11, existsSync as existsSync60, mkdirSync as mkdirSync34, readdirSync as readdirSync21 } from "node:fs";
542981
+ import { accessSync, constants, readFileSync as readFileSync44, writeFileSync as writeFileSync28, unlinkSync as unlinkSync11, existsSync as existsSync60, mkdirSync as mkdirSync34, readdirSync as readdirSync21, renameSync as renameSync7 } from "node:fs";
542982
542982
  import { join as join75 } from "node:path";
542983
542983
  import { tmpdir as tmpdir10 } from "node:os";
542984
+ function normalizeCameraRotationDegrees(value2) {
542985
+ if (value2 === void 0 || value2 === null || value2 === "")
542986
+ return 0;
542987
+ if (typeof value2 === "number" && Number.isFinite(value2)) {
542988
+ const n2 = (Math.round(value2) % 360 + 360) % 360;
542989
+ if (n2 === 0 || n2 === 90 || n2 === 180 || n2 === 270)
542990
+ return n2;
542991
+ }
542992
+ const raw = String(value2).trim().toLowerCase();
542993
+ if (!raw)
542994
+ return 0;
542995
+ const numeric = Number(raw.replace(/deg(?:rees)?$/i, ""));
542996
+ if (Number.isFinite(numeric))
542997
+ return normalizeCameraRotationDegrees(numeric);
542998
+ if (["none", "off", "upright", "normal", "0"].includes(raw))
542999
+ return 0;
543000
+ if (["cw", "clockwise", "right", "rotate-cw", "rotate_cw", "90"].includes(raw))
543001
+ return 90;
543002
+ if (["180", "upside-down", "upside_down", "flip", "inverted"].includes(raw))
543003
+ return 180;
543004
+ if (["ccw", "counterclockwise", "counter-clockwise", "anticlockwise", "left", "rotate-ccw", "rotate_ccw", "270"].includes(raw))
543005
+ return 270;
543006
+ throw new Error(`Unsupported camera rotation: ${String(value2)}. Use 0, 90, 180, 270, cw, ccw, or none.`);
543007
+ }
543008
+ function ffmpegRotateFilterForDegrees(rotation) {
543009
+ switch (rotation) {
543010
+ case 0:
543011
+ return null;
543012
+ case 90:
543013
+ return "transpose=1";
543014
+ case 180:
543015
+ return "hflip,vflip";
543016
+ case 270:
543017
+ return "transpose=2";
543018
+ }
543019
+ }
542984
543020
  function sleep3(ms) {
542985
543021
  return new Promise((resolve76) => setTimeout(resolve76, ms));
542986
543022
  }
@@ -543216,6 +543252,21 @@ var init_camera_capture = __esm({
543216
543252
  type: "number",
543217
543253
  description: "Capture height in pixels (default: 720 for USB, native for QooCam)"
543218
543254
  },
543255
+ rotate_degrees: {
543256
+ type: "number",
543257
+ enum: [0, 90, 180, 270],
543258
+ description: "Optional clockwise correction applied to the captured frame before saving/returning. Use 90 for clockwise, 270 for counter-clockwise."
543259
+ },
543260
+ rotation_degrees: {
543261
+ type: "number",
543262
+ enum: [0, 90, 180, 270],
543263
+ description: "Alias for rotate_degrees."
543264
+ },
543265
+ rotate: {
543266
+ type: "string",
543267
+ enum: ["none", "cw", "ccw", "180", "clockwise", "counterclockwise"],
543268
+ description: "Optional human-readable rotation alias. Equivalent to rotate_degrees."
543269
+ },
543219
543270
  output_path: {
543220
543271
  type: "string",
543221
543272
  description: "Save captured image to this path instead of returning base64"
@@ -543426,6 +543477,7 @@ ${formatRouterPlan(plan)}`,
543426
543477
  const width = args["width"] || 1280;
543427
543478
  const height = args["height"] || 720;
543428
543479
  const outputPath3 = args["output_path"];
543480
+ const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
543429
543481
  const captureDir = join75(tmpdir10(), "omnius-camera");
543430
543482
  if (!existsSync60(captureDir))
543431
543483
  mkdirSync34(captureDir, { recursive: true });
@@ -543484,10 +543536,25 @@ ${formatRouterPlan(plan)}`,
543484
543536
  return false;
543485
543537
  };
543486
543538
  if (await runAttempts()) {
543539
+ if (rotation !== 0) {
543540
+ try {
543541
+ await this.rotateCapturedImage(tempFile, rotation);
543542
+ } catch (err) {
543543
+ return {
543544
+ success: false,
543545
+ output: "",
543546
+ error: `Captured from ${device}, but failed to apply ${rotation} degree rotation: ${err instanceof Error ? err.message : String(err)}`,
543547
+ durationMs: performance.now() - start2
543548
+ };
543549
+ }
543550
+ }
543487
543551
  const firstSuccess = errors.length > 0 ? `
543488
543552
  Recovered with capture profile: ${captureState.successfulAttempt?.label ?? "unknown"}. Last failed attempt: ${errors[errors.length - 1].slice(0, 260)}` : "";
543553
+ const rotationNote = rotation !== 0 ? `
543554
+ Applied camera rotation correction: ${rotation} degrees clockwise.` : "";
543489
543555
  const result = this.returnCapturedFile(tempFile, captureState.successfulAttempt?.videoSize ?? `${width}x${height}`, device, outputPath3, start2);
543490
- return result.success && firstSuccess ? { ...result, output: result.output + firstSuccess, llmContent: result.llmContent ? result.llmContent + firstSuccess : result.llmContent } : result;
543556
+ const notes = `${firstSuccess}${rotationNote}`;
543557
+ return result.success && notes ? { ...result, output: result.output + notes, llmContent: result.llmContent ? result.llmContent + notes : result.llmContent } : result;
543491
543558
  }
543492
543559
  const allErrors = errors.join("\n\n");
543493
543560
  if (permissionBlocked || cameraFfmpegErrorIsPermissionDenied(allErrors)) {
@@ -543576,6 +543643,37 @@ ${allErrors.slice(0, 900)}`, durationMs: performance.now() - start2 };
543576
543643
  argv.push("-y", outputPath3);
543577
543644
  return argv;
543578
543645
  }
543646
+ async rotateCapturedImage(filePath, rotation) {
543647
+ const vf = ffmpegRotateFilterForDegrees(rotation);
543648
+ if (!vf)
543649
+ return;
543650
+ const rotatedPath = `${filePath}.rotated-${Date.now()}.jpg`;
543651
+ try {
543652
+ await runProcessBuffer(ffmpegBin2(), [
543653
+ "-hide_banner",
543654
+ "-loglevel",
543655
+ "error",
543656
+ "-y",
543657
+ "-i",
543658
+ filePath,
543659
+ "-vf",
543660
+ vf,
543661
+ "-q:v",
543662
+ "2",
543663
+ rotatedPath
543664
+ ], { timeout: 15e3 });
543665
+ if (!existsSync60(rotatedPath) || readFileSync44(rotatedPath).length === 0) {
543666
+ throw new Error("ffmpeg produced no rotated image bytes");
543667
+ }
543668
+ renameSync7(rotatedPath, filePath);
543669
+ } finally {
543670
+ try {
543671
+ if (existsSync60(rotatedPath))
543672
+ unlinkSync11(rotatedPath);
543673
+ } catch {
543674
+ }
543675
+ }
543676
+ }
543579
543677
  async v4l2Info(device, start2) {
543580
543678
  let info = `Camera: ${device} [v4l2]
543581
543679
  `;
@@ -543726,6 +543824,7 @@ Auto-installed camera dependencies: ${[
543726
543824
  /** Capture a frame from QooCam via OSC API */
543727
543825
  async captureQooCam(args, start2) {
543728
543826
  const outputPath3 = args["output_path"];
543827
+ const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
543729
543828
  const conn = await this.connectQooCamWiFi();
543730
543829
  if (!conn.connected) {
543731
543830
  return { success: false, output: "", error: conn.error || "Could not connect to QooCam", durationMs: performance.now() - start2 };
@@ -543745,13 +543844,13 @@ Auto-installed camera dependencies: ${[
543745
543844
  let width = 0;
543746
543845
  let height = 0;
543747
543846
  try {
543748
- const result = await this.oscFetch("POST", "/osc/commands/execute", {
543847
+ const result2 = await this.oscFetch("POST", "/osc/commands/execute", {
543749
543848
  name: "camera.takePicture"
543750
543849
  });
543751
- if (result?.state === "error") {
543752
- return { success: false, output: "", error: `QooCam capture failed: ${result.error?.message || JSON.stringify(result.error)}`, durationMs: performance.now() - start2 };
543850
+ if (result2?.state === "error") {
543851
+ return { success: false, output: "", error: `QooCam capture failed: ${result2.error?.message || JSON.stringify(result2.error)}`, durationMs: performance.now() - start2 };
543753
543852
  }
543754
- const entry = result?.results?.entries?.[0];
543853
+ const entry = result2?.results?.entries?.[0];
543755
543854
  if (!entry?.fileUrl) {
543756
543855
  return { success: false, output: "", error: "QooCam took a picture but returned no file URL", durationMs: performance.now() - start2 };
543757
543856
  }
@@ -543775,8 +543874,27 @@ Auto-installed camera dependencies: ${[
543775
543874
  } catch (err) {
543776
543875
  return { success: false, output: "", error: `Failed to download frame from QooCam: ${err instanceof Error ? err.message : String(err)}`, durationMs: performance.now() - start2 };
543777
543876
  }
543877
+ if (rotation !== 0) {
543878
+ try {
543879
+ await this.rotateCapturedImage(tempFile, rotation);
543880
+ } catch (err) {
543881
+ return {
543882
+ success: false,
543883
+ output: "",
543884
+ error: `Downloaded QooCam frame, but failed to apply ${rotation} degree rotation: ${err instanceof Error ? err.message : String(err)}`,
543885
+ durationMs: performance.now() - start2
543886
+ };
543887
+ }
543888
+ }
543778
543889
  const res = width && height ? `${width}x${height}` : "native";
543779
- return this.returnCapturedFile(tempFile, res, "QooCam 8K (OSC/WiFi)", outputPath3, start2);
543890
+ const result = this.returnCapturedFile(tempFile, res, "QooCam 8K (OSC/WiFi)", outputPath3, start2);
543891
+ return result.success && rotation !== 0 ? {
543892
+ ...result,
543893
+ output: `${result.output}
543894
+ Applied camera rotation correction: ${rotation} degrees clockwise.`,
543895
+ llmContent: result.llmContent ? `${result.llmContent}
543896
+ Applied camera rotation correction: ${rotation} degrees clockwise.` : result.llmContent
543897
+ } : result;
543780
543898
  }
543781
543899
  /** Get QooCam info via OSC */
543782
543900
  async qoocamInfo(start2) {
@@ -553723,7 +553841,7 @@ var init_plugin_system = __esm({
553723
553841
  });
553724
553842
 
553725
553843
  // packages/execution/dist/tools/notebook-edit.js
553726
- import { readFileSync as readFileSync54, writeFileSync as writeFileSync37, existsSync as existsSync74, renameSync as renameSync7, unlinkSync as unlinkSync15 } from "node:fs";
553844
+ import { readFileSync as readFileSync54, writeFileSync as writeFileSync37, existsSync as existsSync74, renameSync as renameSync8, unlinkSync as unlinkSync15 } from "node:fs";
553727
553845
  import { resolve as resolve43 } from "node:path";
553728
553846
  function readNotebook(path12) {
553729
553847
  if (!existsSync74(path12))
@@ -553744,7 +553862,7 @@ function writeNotebook(path12, nb) {
553744
553862
  const tempPath = `${path12}.tmp-${process.pid}-${Date.now()}`;
553745
553863
  writeFileSync37(tempPath, JSON.stringify(nb, null, 1) + "\n", "utf8");
553746
553864
  try {
553747
- renameSync7(tempPath, path12);
553865
+ renameSync8(tempPath, path12);
553748
553866
  } catch (err) {
553749
553867
  try {
553750
553868
  unlinkSync15(tempPath);
@@ -555483,6 +555601,15 @@ function makePythonScript() {
555483
555601
  return String.raw`
555484
555602
  import json, os, sys, time
555485
555603
 
555604
+ def apply_rotation(frame, rotate_degrees, cv2):
555605
+ if rotate_degrees == 90:
555606
+ return cv2.rotate(frame, cv2.ROTATE_90_CLOCKWISE)
555607
+ if rotate_degrees == 180:
555608
+ return cv2.rotate(frame, cv2.ROTATE_180)
555609
+ if rotate_degrees == 270:
555610
+ return cv2.rotate(frame, cv2.ROTATE_90_COUNTERCLOCKWISE)
555611
+ return frame
555612
+
555486
555613
  def main():
555487
555614
  cfg_path = sys.argv[1]
555488
555615
  with open(cfg_path, "r", encoding="utf-8") as f:
@@ -555494,6 +555621,7 @@ def main():
555494
555621
  source_kind = cfg["source_kind"]
555495
555622
  source = cfg["source"]
555496
555623
  media_path = None
555624
+ rotate_degrees = int(cfg.get("rotate_degrees") or 0)
555497
555625
 
555498
555626
  try:
555499
555627
  import cv2
@@ -555567,6 +555695,7 @@ def main():
555567
555695
  ret, frame = cap.read()
555568
555696
  if not ret:
555569
555697
  break
555698
+ frame = apply_rotation(frame, rotate_degrees, cv2)
555570
555699
  ts = cap.get(cv2.CAP_PROP_POS_MSEC) / 1000.0
555571
555700
  if not ts or ts < 0:
555572
555701
  ts = time.time() - start_wall
@@ -555649,6 +555778,7 @@ def main():
555649
555778
  "output_dir": out_dir,
555650
555779
  "yolo_model": yolo_model,
555651
555780
  "yolo_live": True,
555781
+ "rotate_degrees": rotate_degrees,
555652
555782
  "frames_sampled": frame_index,
555653
555783
  "duration_observed_sec": round(float(duration if source_kind != "camera" else time.time() - start_wall), 3),
555654
555784
  "objects": objects,
@@ -555662,6 +555792,7 @@ if __name__ == "__main__":
555662
555792
  }
555663
555793
  function mockResult(args, workingDir) {
555664
555794
  const kind = sourceKind(args);
555795
+ const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
555665
555796
  const outDir = join92(workingDir, ".omnius", "live-media", `mock-${Date.now().toString(36)}`);
555666
555797
  mkdirSync47(join92(outDir, "crops"), { recursive: true });
555667
555798
  const facePath = join92(outDir, "crops", "face_f0000_00.jpg");
@@ -555673,6 +555804,7 @@ function mockResult(args, workingDir) {
555673
555804
  output_dir: outDir,
555674
555805
  yolo_model: String(args["yolo_model"] ?? DEFAULT_YOLO26_MODEL),
555675
555806
  yolo_live: false,
555807
+ rotate_degrees: rotation,
555676
555808
  frames_sampled: 3,
555677
555809
  duration_observed_sec: Number(args["duration_sec"] ?? 3),
555678
555810
  objects: [
@@ -555694,6 +555826,7 @@ var init_live_media_loop = __esm({
555694
555826
  init_process_async();
555695
555827
  init_venv_paths();
555696
555828
  init_audio_analyze();
555829
+ init_camera_capture();
555697
555830
  init_transcribe_tool();
555698
555831
  init_visual_memory();
555699
555832
  init_visual_trigger();
@@ -555713,6 +555846,9 @@ var init_live_media_loop = __esm({
555713
555846
  path: { type: "string", description: "Local media path" },
555714
555847
  file: { type: "string", description: "Alias for path" },
555715
555848
  camera: { type: ["string", "number"], description: "Camera device index/path" },
555849
+ rotate_degrees: { type: "number", enum: [0, 90, 180, 270], description: "Clockwise correction applied to sampled frames before object tracking and face crops." },
555850
+ rotation_degrees: { type: "number", enum: [0, 90, 180, 270], description: "Alias for rotate_degrees." },
555851
+ rotate: { type: "string", enum: ["none", "cw", "ccw", "180", "clockwise", "counterclockwise"], description: "Optional human-readable rotation alias." },
555716
555852
  duration_sec: { type: "number", description: "Bounded watch duration, default 8" },
555717
555853
  sample_interval_sec: { type: "number", description: "Seconds between sampled frames, default 1" },
555718
555854
  max_frames: { type: "number", description: "Max sampled frames, default 8" },
@@ -555736,6 +555872,7 @@ var init_live_media_loop = __esm({
555736
555872
  const start2 = performance.now();
555737
555873
  const action = String(args["action"] ?? "watch").toLowerCase();
555738
555874
  const yoloModel = String(args["yolo_model"] ?? DEFAULT_YOLO26_MODEL);
555875
+ const rotation = normalizeCameraRotationDegrees(args["rotate_degrees"] ?? args["rotation_degrees"] ?? args["rotate"]);
555739
555876
  const plan = buildYolo26BootstrapPlan(yoloModel);
555740
555877
  if (action === "status" || action === "ensure") {
555741
555878
  const runtime = action === "ensure" ? await ensureRuntime(args["auto_bootstrap"] !== false) : await ensureRuntime(false);
@@ -555780,6 +555917,7 @@ var init_live_media_loop = __esm({
555780
555917
  duration_sec: Number(args["duration_sec"] ?? 8),
555781
555918
  sample_interval_sec: Number(args["sample_interval_sec"] ?? 1),
555782
555919
  max_frames: Number(args["max_frames"] ?? 8),
555920
+ rotate_degrees: rotation,
555783
555921
  detect_objects: args["detect_objects"] !== false,
555784
555922
  track_objects: args["track_objects"] !== false,
555785
555923
  crop_faces: args["crop_faces"] !== false,
@@ -555872,6 +556010,8 @@ var init_live_media_loop = __esm({
555872
556010
  if (result.media_path)
555873
556011
  lines.push(`media: ${result.media_path}`);
555874
556012
  lines.push(`YOLO model: ${result.yolo_model} (${result.yolo_live ? "live" : "mock/harness"})`);
556013
+ if (result.rotate_degrees)
556014
+ lines.push(`frame rotation correction: ${result.rotate_degrees} degrees clockwise`);
555875
556015
  lines.push(`sampled: ${result.frames_sampled} frame(s) over ${result.duration_observed_sec.toFixed(1)}s`);
555876
556016
  lines.push(`objects: ${result.objects.length} detection(s), tracks: ${result.tracks.length}, face crops: ${result.faces.length}`);
555877
556017
  const objectSummary = summarizeObjects(result.objects);
@@ -555918,6 +556058,7 @@ var init_live_media_loop = __esm({
555918
556058
  media_path: result.media_path,
555919
556059
  yolo_model: result.yolo_model,
555920
556060
  yolo_live: result.yolo_live,
556061
+ rotate_degrees: result.rotate_degrees ?? 0,
555921
556062
  frames_sampled: result.frames_sampled,
555922
556063
  objects: result.objects.slice(0, 80),
555923
556064
  tracks: result.tracks.slice(0, 40),
@@ -569468,7 +569609,7 @@ var init_memory_consolidation = __esm({
569468
569609
  });
569469
569610
 
569470
569611
  // packages/orchestrator/dist/consolidation-runtime.js
569471
- import { existsSync as existsSync97, mkdirSync as mkdirSync56, readFileSync as readFileSync74, writeFileSync as writeFileSync46, renameSync as renameSync8 } from "node:fs";
569612
+ import { existsSync as existsSync97, mkdirSync as mkdirSync56, readFileSync as readFileSync74, writeFileSync as writeFileSync46, renameSync as renameSync9 } from "node:fs";
569472
569613
  import { homedir as homedir33 } from "node:os";
569473
569614
  import { join as join108, dirname as dirname35 } from "node:path";
569474
569615
  function storePath2() {
@@ -569490,7 +569631,7 @@ function save2(s2) {
569490
569631
  mkdirSync56(dirname35(p2), { recursive: true });
569491
569632
  const tmp = `${p2}.tmp`;
569492
569633
  writeFileSync46(tmp, JSON.stringify(s2), "utf8");
569493
- renameSync8(tmp, p2);
569634
+ renameSync9(tmp, p2);
569494
569635
  } catch {
569495
569636
  }
569496
569637
  }
@@ -597786,7 +597927,7 @@ var init_agent_task = __esm({
597786
597927
  });
597787
597928
 
597788
597929
  // packages/orchestrator/dist/task-recovery.js
597789
- import { existsSync as existsSync105, readFileSync as readFileSync81, writeFileSync as writeFileSync51, mkdirSync as mkdirSync61, readdirSync as readdirSync33, renameSync as renameSync9, unlinkSync as unlinkSync19 } from "node:fs";
597930
+ import { existsSync as existsSync105, readFileSync as readFileSync81, writeFileSync as writeFileSync51, mkdirSync as mkdirSync61, readdirSync as readdirSync33, renameSync as renameSync10, unlinkSync as unlinkSync19 } from "node:fs";
597790
597931
  import { join as join117 } from "node:path";
597791
597932
  import { homedir as homedir36 } from "node:os";
597792
597933
  function sidecarDir() {
@@ -597803,7 +597944,7 @@ function persistAgentTaskSidecar(task) {
597803
597944
  const tmpPath = `${finalPath}.tmp.${process.pid}.${Date.now()}`;
597804
597945
  const { abortController: _ignore, ...serializable } = task;
597805
597946
  writeFileSync51(tmpPath, JSON.stringify(serializable, null, 2), "utf-8");
597806
- renameSync9(tmpPath, finalPath);
597947
+ renameSync10(tmpPath, finalPath);
597807
597948
  } catch {
597808
597949
  }
597809
597950
  }
@@ -601956,6 +602097,75 @@ function parseJsonObject3(value2) {
601956
602097
  return null;
601957
602098
  }
601958
602099
  }
602100
+ function parseJsonObjectFromText(value2) {
602101
+ const direct = parseJsonObject3(value2);
602102
+ if (direct) return direct;
602103
+ const text2 = String(value2 ?? "");
602104
+ const start2 = text2.indexOf("{");
602105
+ const end = text2.lastIndexOf("}");
602106
+ if (start2 < 0 || end <= start2) return null;
602107
+ return parseJsonObject3(text2.slice(start2, end + 1));
602108
+ }
602109
+ function normalizeLiveCameraRotation(value2) {
602110
+ if (value2 === void 0 || value2 === null || value2 === "") return 0;
602111
+ if (typeof value2 === "number" && Number.isFinite(value2)) {
602112
+ const n2 = (Math.round(value2) % 360 + 360) % 360;
602113
+ if (n2 === 0 || n2 === 90 || n2 === 180 || n2 === 270) return n2;
602114
+ }
602115
+ const raw = String(value2).trim().toLowerCase();
602116
+ const numeric = Number(raw.replace(/deg(?:rees)?$/i, ""));
602117
+ if (Number.isFinite(numeric)) return normalizeLiveCameraRotation(numeric);
602118
+ if (["none", "off", "upright", "normal", "0"].includes(raw)) return 0;
602119
+ if (["cw", "clockwise", "right", "rotate-cw", "rotate_cw", "90"].includes(raw)) return 90;
602120
+ if (["180", "upside-down", "upside_down", "flip", "inverted"].includes(raw)) return 180;
602121
+ if (["ccw", "counterclockwise", "counter-clockwise", "counter_clockwise", "anticlockwise", "left", "rotate-ccw", "rotate_ccw", "270"].includes(raw)) return 270;
602122
+ throw new Error(`Unsupported camera rotation: ${String(value2)}. Use 0, 90, 180, 270, cw, ccw, or none.`);
602123
+ }
602124
+ function parseCameraOrientationDecision(value2) {
602125
+ const parsed = parseJsonObjectFromText(value2);
602126
+ if (!parsed) return null;
602127
+ const rawRotation = parsed["rotation_degrees"] ?? parsed["correction_degrees"] ?? parsed["rotate_degrees"] ?? parsed["rotation"] ?? parsed["correction"];
602128
+ if (rawRotation === void 0 || rawRotation === null || rawRotation === "") return null;
602129
+ let rotation;
602130
+ try {
602131
+ rotation = normalizeLiveCameraRotation(rawRotation);
602132
+ } catch {
602133
+ return null;
602134
+ }
602135
+ const confidenceNumber = Number(parsed["confidence"]);
602136
+ const confidence2 = Number.isFinite(confidenceNumber) ? Math.max(0, Math.min(1, confidenceNumber)) : void 0;
602137
+ const reason = typeof parsed["reason"] === "string" ? boundedText(parsed["reason"], 240) : typeof parsed["evidence"] === "string" ? boundedText(parsed["evidence"], 240) : void 0;
602138
+ return { rotation, confidence: confidence2, reason };
602139
+ }
602140
+ function formatCameraRotation(rotation) {
602141
+ if (rotation === 0) return "upright/no correction";
602142
+ if (rotation === 90) return "90 degrees clockwise";
602143
+ if (rotation === 180) return "180 degrees";
602144
+ return "90 degrees counter-clockwise";
602145
+ }
602146
+ function sanitizeCameraOrientation(value2) {
602147
+ if (typeof value2 !== "object" || value2 === null || Array.isArray(value2)) return {};
602148
+ const out = {};
602149
+ for (const [device, raw] of Object.entries(value2)) {
602150
+ if (!device || typeof raw !== "object" || raw === null || Array.isArray(raw)) continue;
602151
+ const entry = raw;
602152
+ try {
602153
+ out[device] = {
602154
+ rotation: normalizeLiveCameraRotation(entry["rotation"]),
602155
+ source: entry["source"] === "manual" ? "manual" : "auto",
602156
+ updatedAt: Math.max(0, Number(entry["updatedAt"] ?? Date.now())),
602157
+ confidence: typeof entry["confidence"] === "number" ? Math.max(0, Math.min(1, entry["confidence"])) : void 0,
602158
+ evidence: typeof entry["evidence"] === "string" ? boundedText(entry["evidence"], 800) : void 0
602159
+ };
602160
+ } catch {
602161
+ }
602162
+ }
602163
+ return out;
602164
+ }
602165
+ function describeCameraOrientation(orientation) {
602166
+ if (!orientation) return "upright/no correction (default)";
602167
+ return `${formatCameraRotation(orientation.rotation)} (${orientation.source}${orientation.confidence !== void 0 ? ` confidence=${orientation.confidence.toFixed(2)}` : ""}, updated=${nowIso3(orientation.updatedAt)})`;
602168
+ }
601959
602169
  function parseLiveMediaPayload(value2) {
601960
602170
  const parsed = parseJsonObject3(value2);
601961
602171
  if (!parsed) return null;
@@ -602180,6 +602390,7 @@ function loadLiveSensorConfig(repoRoot) {
602180
602390
  return {
602181
602391
  ...DEFAULT_CONFIG7,
602182
602392
  ...parsed,
602393
+ cameraOrientation: sanitizeCameraOrientation(parsed.cameraOrientation),
602183
602394
  videoIntervalMs: Math.max(1500, Number(parsed.videoIntervalMs ?? DEFAULT_CONFIG7.videoIntervalMs)),
602184
602395
  audioIntervalMs: Math.max(3e3, Number(parsed.audioIntervalMs ?? DEFAULT_CONFIG7.audioIntervalMs))
602185
602396
  };
@@ -602208,11 +602419,15 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
602208
602419
  lines.push("Live sensor context is operator-enabled ambient input. Treat it as current perceptual evidence for this turn; if it is stale, missing, or insufficient, use camera/audio tools to refresh before answering what you see or hear.");
602209
602420
  lines.push("Identity rule: never invent names for observed people. If a person is unknown and identity matters, ask the user or the person for their name, then store the association with visual_memory(action='enroll', image='<face crop>', name='<name>') when a face crop is available, or multimodal_memory(action='meet', person_name='<name>') when live voice/name context is available.");
602210
602421
  lines.push(`Snapshot file: ${liveSnapshotPath(snapshot.repoRoot)}`);
602211
- lines.push(`Streams: video=${snapshot.config.videoEnabled ? "on" : "off"} infer=${snapshot.config.inferEnabled ? "on" : "off"} audio=${snapshot.config.audioEnabled ? "on" : "off"} asr=${snapshot.config.asrEnabled ? "on" : "off"} sounds=${snapshot.config.audioAnalysisEnabled ? "on" : "off"} output-monitor=${snapshot.config.audioOutputEnabled ? "on" : "off"}`);
602212
- if (snapshot.config.selectedCamera) lines.push(`Selected camera: ${snapshot.config.selectedCamera}`);
602422
+ lines.push(`Streams: video=${snapshot.config.videoEnabled ? "on" : "off"} infer=${snapshot.config.inferEnabled ? "on" : "off"} clip=${snapshot.config.clipEnabled ? "on" : "off"} audio=${snapshot.config.audioEnabled ? "on" : "off"} asr=${snapshot.config.asrEnabled ? "on" : "off"} sounds=${snapshot.config.audioAnalysisEnabled ? "on" : "off"} output-monitor=${snapshot.config.audioOutputEnabled ? "on" : "off"}`);
602423
+ if (snapshot.config.selectedCamera) {
602424
+ const orientation = snapshot.config.cameraOrientation?.[snapshot.config.selectedCamera];
602425
+ lines.push(`Selected camera: ${snapshot.config.selectedCamera}`);
602426
+ lines.push(`Camera orientation correction: ${describeCameraOrientation(orientation)}`);
602427
+ }
602213
602428
  if (snapshot.config.selectedAudioInput) lines.push(`Selected audio input: ${snapshot.config.selectedAudioInput}`);
602214
602429
  if (snapshot.config.selectedAudioOutput) lines.push(`Selected audio output: ${snapshot.config.selectedAudioOutput}`);
602215
- const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 18);
602430
+ const cameraEvents = (snapshot.events ?? []).filter((event) => event.kind.startsWith("camera_") || event.kind === "clip_recognition" || event.kind === "visual_trigger").sort((a2, b) => b.observedAt - a2.observedAt).slice(0, 18);
602216
602431
  if (activeVideo) {
602217
602432
  lines.push("");
602218
602433
  lines.push("## LIVE CAMERA EVENTS");
@@ -602257,7 +602472,7 @@ function formatLiveSensorContextFromSnapshot(snapshot, opts = {}) {
602257
602472
  const age = now2 - snapshot.video.updatedAt;
602258
602473
  lines.push("");
602259
602474
  lines.push("## LIVE CAMERA LATEST FRAME");
602260
- lines.push(`timestamp: ${nowIso3(snapshot.video.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s source=${snapshot.video.source}${age > maxAgeMs ? " STALE" : ""}`);
602475
+ lines.push(`timestamp: ${nowIso3(snapshot.video.updatedAt)} age=${Math.max(0, Math.round(age / 1e3))}s source=${snapshot.video.source}${snapshot.video.orientation ? ` rotation=${formatCameraRotation(snapshot.video.orientation.rotation)}` : ""}${age > maxAgeMs ? " STALE" : ""}`);
602261
602476
  if (snapshot.video.error) lines.push(`error: ${snapshot.video.error}`);
602262
602477
  if (snapshot.video.framePath) lines.push(`frame: ${snapshot.video.framePath}`);
602263
602478
  if (snapshot.video.asciiContext) lines.push(snapshot.video.asciiContext);
@@ -602401,8 +602616,9 @@ function formatLiveInventory(inventory) {
602401
602616
  function formatLiveStatus(snapshot) {
602402
602617
  const cfg = snapshot.config;
602403
602618
  const lines = [
602404
- `Live streams: video=${cfg.videoEnabled ? "on" : "off"} infer=${cfg.inferEnabled ? "on" : "off"} audio=${cfg.audioEnabled ? "on" : "off"} asr=${cfg.asrEnabled ? "on" : "off"} sounds=${cfg.audioAnalysisEnabled ? "on" : "off"} output=${cfg.audioOutputEnabled ? "on" : "off"}`,
602619
+ `Live streams: video=${cfg.videoEnabled ? "on" : "off"} infer=${cfg.inferEnabled ? "on" : "off"} clip=${cfg.clipEnabled ? "on" : "off"} audio=${cfg.audioEnabled ? "on" : "off"} asr=${cfg.asrEnabled ? "on" : "off"} sounds=${cfg.audioAnalysisEnabled ? "on" : "off"} output=${cfg.audioOutputEnabled ? "on" : "off"}`,
602405
602620
  `Camera: ${cfg.selectedCamera || "none"}`,
602621
+ `Camera orientation: ${cfg.selectedCamera ? describeCameraOrientation(cfg.cameraOrientation?.[cfg.selectedCamera]) : "none"}`,
602406
602622
  `Audio input: ${cfg.selectedAudioInput || "none"}`,
602407
602623
  `Audio output: ${cfg.selectedAudioOutput || "none"}`,
602408
602624
  `Snapshot: ${liveSnapshotPath(snapshot.repoRoot)}`
@@ -602427,8 +602643,10 @@ var init_live_sensors = __esm({
602427
602643
  audioEnabled: false,
602428
602644
  audioOutputEnabled: false,
602429
602645
  inferEnabled: false,
602646
+ clipEnabled: false,
602430
602647
  asrEnabled: false,
602431
602648
  audioAnalysisEnabled: false,
602649
+ cameraOrientation: {},
602432
602650
  videoIntervalMs: 5e3,
602433
602651
  audioIntervalMs: 9e3
602434
602652
  };
@@ -602454,16 +602672,124 @@ var init_live_sensors = __esm({
602454
602672
  videoInFlight = false;
602455
602673
  audioInFlight = false;
602456
602674
  statusSink = null;
602675
+ feedbackSink = null;
602676
+ agentActionSink = null;
602677
+ agentActionEnabled = false;
602678
+ lastAgentReviewAt = 0;
602679
+ autoOrientationInFlight = /* @__PURE__ */ new Set();
602457
602680
  setStatusSink(sink) {
602458
602681
  this.statusSink = sink ?? null;
602459
602682
  this.emitStatus();
602460
602683
  }
602684
+ setFeedbackSink(sink) {
602685
+ this.feedbackSink = sink ?? null;
602686
+ }
602687
+ setAgentActionSink(sink) {
602688
+ this.agentActionSink = sink ?? null;
602689
+ }
602461
602690
  getConfig() {
602462
602691
  return { ...this.config };
602463
602692
  }
602464
602693
  getSnapshot() {
602465
602694
  return this.snapshot;
602466
602695
  }
602696
+ getCameraOrientation(device = this.config.selectedCamera) {
602697
+ return device ? this.config.cameraOrientation?.[device] : void 0;
602698
+ }
602699
+ getCameraRotation(device = this.config.selectedCamera) {
602700
+ return this.getCameraOrientation(device)?.rotation ?? 0;
602701
+ }
602702
+ setCameraRotation(device, rotation, source, evidence, confidence2) {
602703
+ const target = device || this.config.selectedCamera || firstDeviceId(this.devices.video);
602704
+ if (!target) return null;
602705
+ const orientation = {
602706
+ rotation,
602707
+ source,
602708
+ updatedAt: Date.now(),
602709
+ confidence: confidence2,
602710
+ evidence: evidence ? boundedText(evidence, 800) : void 0
602711
+ };
602712
+ this.config = {
602713
+ ...this.config,
602714
+ selectedCamera: target,
602715
+ cameraOrientation: {
602716
+ ...this.config.cameraOrientation ?? {},
602717
+ [target]: orientation
602718
+ }
602719
+ };
602720
+ this.snapshot.events = mergeRecentById(this.snapshot.events, [{
602721
+ id: stableEventId("camera_orientation", target, orientation.updatedAt, `${source}:${rotation}`),
602722
+ kind: "camera_orientation",
602723
+ observedAt: orientation.updatedAt,
602724
+ source: target,
602725
+ title: "Camera orientation correction updated",
602726
+ summary: `${target}: ${formatCameraRotation(rotation)} via ${source}${confidence2 !== void 0 ? ` confidence=${confidence2.toFixed(2)}` : ""}${evidence ? ` evidence=${boundedText(evidence, 300)}` : ""}`,
602727
+ confidence: confidence2
602728
+ }], 50);
602729
+ this.persist();
602730
+ this.emitFeedback({
602731
+ kind: "orientation",
602732
+ title: "Camera orientation correction updated",
602733
+ observedAt: orientation.updatedAt,
602734
+ message: `${target}: ${describeCameraOrientation(orientation)}`
602735
+ });
602736
+ return orientation;
602737
+ }
602738
+ cycleCameraRotation(device = this.config.selectedCamera) {
602739
+ const current = this.getCameraRotation(device);
602740
+ const next = current === 0 ? 90 : current === 90 ? 180 : current === 180 ? 270 : 0;
602741
+ return this.setCameraRotation(device, next, "manual", "manual cycle from /live menu");
602742
+ }
602743
+ async autoOrientCamera(device = this.config.selectedCamera, options2 = {}) {
602744
+ const target = device || this.config.selectedCamera || firstDeviceId(this.devices.video);
602745
+ if (!target) return "No camera selected. Use /live camera <device> or select a camera in /live.";
602746
+ if (options2.onlyIfUnset && this.getCameraOrientation(target)) {
602747
+ return `Camera orientation already set for ${target}: ${describeCameraOrientation(this.getCameraOrientation(target))}`;
602748
+ }
602749
+ if (this.autoOrientationInFlight.has(target)) {
602750
+ return `Camera orientation detection is already running for ${target}.`;
602751
+ }
602752
+ this.autoOrientationInFlight.add(target);
602753
+ try {
602754
+ const captured = await new CameraCaptureTool().execute({
602755
+ action: "capture",
602756
+ device: target,
602757
+ width: 640,
602758
+ height: 480,
602759
+ rotate_degrees: 0
602760
+ });
602761
+ if (!captured.success) {
602762
+ return `Camera orientation detection could not capture ${target}: ${captured.error || captured.output || "unknown error"}`;
602763
+ }
602764
+ const framePath = extractSavedImagePath(captured.output, this.repoRoot);
602765
+ if (!framePath) return `Camera orientation detection captured ${target}, but no frame path was returned.`;
602766
+ const prompt = [
602767
+ "Determine whether this camera frame is upright for a human viewer.",
602768
+ "Return only JSON with this schema:",
602769
+ '{"rotation_degrees":0|90|180|270,"confidence":0.0-1.0,"reason":"brief visual evidence"}',
602770
+ "rotation_degrees is the clockwise correction to apply to future frames so they are upright.",
602771
+ "Use 90 if the image must rotate clockwise; use 270 if it must rotate counter-clockwise; use 180 if upside down; use 0 if already upright or inconclusive."
602772
+ ].join("\n");
602773
+ const vision = await new VisionTool(this.repoRoot).execute({
602774
+ action: "query",
602775
+ image: framePath,
602776
+ prompt,
602777
+ model: "moondream3-preview"
602778
+ });
602779
+ if (!vision.success) {
602780
+ return `Camera orientation detection captured ${target}, but vision inference was unavailable: ${vision.error || vision.output || "unknown error"}`;
602781
+ }
602782
+ const decision2 = parseCameraOrientationDecision(vision.llmContent || vision.output);
602783
+ if (!decision2) {
602784
+ return `Camera orientation detection for ${target} was inconclusive. Vision output did not contain a structured rotation decision.`;
602785
+ }
602786
+ const evidence = decision2.reason || boundedText(vision.llmContent || vision.output, 400);
602787
+ const orientation = this.setCameraRotation(target, decision2.rotation, "auto", evidence, decision2.confidence);
602788
+ return orientation ? `Auto orientation for ${target}: ${describeCameraOrientation(orientation)}` : `Auto orientation for ${target} could not be persisted.`;
602789
+ } finally {
602790
+ this.autoOrientationInFlight.delete(target);
602791
+ }
602792
+ }
602467
602793
  async refreshDevices() {
602468
602794
  const errors = [];
602469
602795
  let video = [];
@@ -602504,23 +602830,82 @@ var init_live_sensors = __esm({
602504
602830
  return this.getConfig();
602505
602831
  }
602506
602832
  stopAll() {
602833
+ this.agentActionEnabled = false;
602507
602834
  this.configure({
602508
602835
  videoEnabled: false,
602509
602836
  audioEnabled: false,
602510
602837
  audioOutputEnabled: false,
602511
602838
  inferEnabled: false,
602839
+ clipEnabled: false,
602512
602840
  asrEnabled: false,
602513
602841
  audioAnalysisEnabled: false
602514
602842
  });
602515
602843
  }
602844
+ startRunMode(options2 = {}) {
602845
+ this.agentActionEnabled = Boolean(options2.agent);
602846
+ this.configure({
602847
+ videoEnabled: true,
602848
+ inferEnabled: true,
602849
+ clipEnabled: true,
602850
+ audioEnabled: true,
602851
+ asrEnabled: true,
602852
+ audioAnalysisEnabled: true,
602853
+ videoIntervalMs: options2.lowLatency === false ? this.config.videoIntervalMs : Math.min(this.config.videoIntervalMs, 1500),
602854
+ audioIntervalMs: options2.lowLatency === false ? this.config.audioIntervalMs : Math.min(this.config.audioIntervalMs, 3e3)
602855
+ });
602856
+ const source = this.config.selectedCamera || firstDeviceId(this.devices.video);
602857
+ if (source && !this.getCameraOrientation(source)) {
602858
+ void this.autoOrientCamera(source, { onlyIfUnset: true }).then((message2) => {
602859
+ this.emitFeedback({
602860
+ kind: "orientation",
602861
+ title: "Camera auto-orientation",
602862
+ observedAt: Date.now(),
602863
+ message: message2
602864
+ });
602865
+ }).catch((err) => {
602866
+ this.emitFeedback({
602867
+ kind: "error",
602868
+ title: "Camera auto-orientation failed",
602869
+ observedAt: Date.now(),
602870
+ message: err instanceof Error ? err.message : String(err)
602871
+ });
602872
+ });
602873
+ }
602874
+ void this.sampleVideoNow(true).catch((err) => {
602875
+ this.emitFeedback({
602876
+ kind: "error",
602877
+ title: "Live video sample failed",
602878
+ observedAt: Date.now(),
602879
+ message: err instanceof Error ? err.message : String(err)
602880
+ });
602881
+ });
602882
+ void this.sampleAudioNow().catch((err) => {
602883
+ this.emitFeedback({
602884
+ kind: "error",
602885
+ title: "Live audio sample failed",
602886
+ observedAt: Date.now(),
602887
+ message: err instanceof Error ? err.message : String(err)
602888
+ });
602889
+ });
602890
+ this.emitFeedback({
602891
+ kind: "status",
602892
+ title: options2.agent ? "Live PFC run started" : "Live run started",
602893
+ observedAt: Date.now(),
602894
+ message: "Low-latency video, face, CLIP, ASR, and sound loops are active. Latest observations are injected into each agent turn."
602895
+ });
602896
+ return options2.agent ? "Live PFC run enabled: visual/audio feedback is active and significant events can launch background agent review." : "Live run enabled: visual/audio feedback is active and live context will update each turn.";
602897
+ }
602516
602898
  async previewCamera(device = this.config.selectedCamera) {
602517
602899
  const source = device || firstDeviceId(this.devices.video);
602518
602900
  if (!source) return { ok: false, message: "No camera selected. Use /live to select a camera after device discovery." };
602901
+ const orientation = this.getCameraOrientation(source);
602902
+ const rotation = orientation?.rotation ?? 0;
602519
602903
  const result = await new CameraCaptureTool().execute({
602520
602904
  action: "capture",
602521
602905
  device: source,
602522
602906
  width: 640,
602523
- height: 480
602907
+ height: 480,
602908
+ rotate_degrees: rotation
602524
602909
  });
602525
602910
  if (!result.success) {
602526
602911
  const error = result.error || result.output || "camera capture failed";
@@ -602539,6 +602924,12 @@ var init_live_sensors = __esm({
602539
602924
  summary: error
602540
602925
  }], 50);
602541
602926
  this.persist();
602927
+ this.emitFeedback({
602928
+ kind: "error",
602929
+ title: "Camera capture failed",
602930
+ observedAt: observedAt2,
602931
+ message: error
602932
+ });
602542
602933
  return { ok: false, message: error };
602543
602934
  }
602544
602935
  const framePath = extractSavedImagePath(result.output, this.repoRoot);
@@ -602552,7 +602943,9 @@ var init_live_sensors = __esm({
602552
602943
  source,
602553
602944
  framePath,
602554
602945
  displayPath,
602555
- asciiContext
602946
+ asciiContext,
602947
+ rotation,
602948
+ orientation
602556
602949
  };
602557
602950
  this.snapshot.events = mergeRecentById(this.snapshot.events, [{
602558
602951
  id: stableEventId("camera_frame", source, observedAt, framePath),
@@ -602560,10 +602953,20 @@ var init_live_sensors = __esm({
602560
602953
  observedAt,
602561
602954
  source,
602562
602955
  title: "Captured camera frame",
602563
- summary: `frame=${framePath}${displayPath ? ` display=${displayPath}` : ""}`,
602956
+ summary: `frame=${framePath}${displayPath ? ` display=${displayPath}` : ""}${orientation ? ` rotation=${formatCameraRotation(orientation.rotation)}` : ""}`,
602564
602957
  evidencePath: framePath
602565
602958
  }], 50);
602566
602959
  this.persist();
602960
+ this.emitFeedback({
602961
+ kind: "camera-frame",
602962
+ title: "Live camera frame",
602963
+ observedAt,
602964
+ message: `Captured from ${source}${orientation ? ` (${formatCameraRotation(orientation.rotation)})` : ""}: ${framePath}`,
602965
+ imagePath: framePath,
602966
+ displayPath,
602967
+ ascii: preview?.ascii,
602968
+ renderer: preview?.renderer
602969
+ });
602567
602970
  return {
602568
602971
  ok: true,
602569
602972
  message: `Camera frame captured from ${source}: ${framePath}`,
@@ -602573,13 +602976,54 @@ var init_live_sensors = __esm({
602573
602976
  renderer: preview?.renderer
602574
602977
  };
602575
602978
  }
602979
+ async recognizeFrameObjects(framePath, source, observedAt = Date.now()) {
602980
+ try {
602981
+ const result = await new VisualMemoryTool().execute({
602982
+ action: "recognize",
602983
+ image: framePath,
602984
+ format: "json",
602985
+ auto_setup: true,
602986
+ timeout_ms: 45e3
602987
+ });
602988
+ const summary = result.success ? boundedText(result.llmContent || result.output || "No visual-memory object matches returned.", 1400) : `CLIP/visual-memory unavailable: ${result.error || result.output || "unknown error"}`;
602989
+ this.snapshot.events = mergeRecentById(this.snapshot.events, [{
602990
+ id: stableEventId("clip_recognition", source, observedAt, framePath),
602991
+ kind: "clip_recognition",
602992
+ observedAt,
602993
+ source,
602994
+ title: result.success ? "CLIP visual-memory recognition" : "CLIP visual-memory unavailable",
602995
+ summary,
602996
+ evidencePath: framePath
602997
+ }], 50);
602998
+ this.persist();
602999
+ this.emitFeedback({
603000
+ kind: "clip",
603001
+ title: result.success ? "CLIP visual-memory recognition" : "CLIP visual-memory unavailable",
603002
+ observedAt,
603003
+ message: summary
603004
+ });
603005
+ return summary;
603006
+ } catch (err) {
603007
+ const message2 = `CLIP/visual-memory failed: ${err instanceof Error ? err.message : String(err)}`;
603008
+ this.emitFeedback({
603009
+ kind: "error",
603010
+ title: "CLIP visual-memory failed",
603011
+ observedAt,
603012
+ message: message2
603013
+ });
603014
+ return message2;
603015
+ }
603016
+ }
602576
603017
  async sampleVideoNow(forceInfer = this.config.inferEnabled) {
602577
603018
  if (this.videoInFlight) return "Video sampler is already running.";
602578
603019
  this.videoInFlight = true;
602579
603020
  try {
602580
603021
  const preview = await this.previewCamera(this.config.selectedCamera);
602581
603022
  let inference = "";
603023
+ let clipSummary = "";
602582
603024
  const source = this.config.selectedCamera || firstDeviceId(this.devices.video);
603025
+ const orientation = source ? this.getCameraOrientation(source) : void 0;
603026
+ const sampledAt = Date.now();
602583
603027
  if (forceInfer && source) {
602584
603028
  const loop = new LiveMediaLoopTool(this.repoRoot);
602585
603029
  const result = await loop.execute({
@@ -602589,6 +603033,7 @@ var init_live_sensors = __esm({
602589
603033
  duration_sec: 4,
602590
603034
  sample_interval_sec: 1,
602591
603035
  max_frames: 4,
603036
+ rotate_degrees: orientation?.rotation ?? 0,
602592
603037
  auto_bootstrap: true,
602593
603038
  audio_mode: "none",
602594
603039
  detect_objects: true,
@@ -602596,7 +603041,6 @@ var init_live_sensors = __esm({
602596
603041
  crop_faces: true,
602597
603042
  recognize_faces: true
602598
603043
  });
602599
- const sampledAt = Date.now();
602600
603044
  inference = result.success ? String(result.output || result.llmContent || "") : `Video inference failed: ${result.error || result.output || "unknown error"}`;
602601
603045
  const livePayload = result.success ? parseLiveMediaPayload(result.llmContent || result.output) : null;
602602
603046
  const observations = observationsFromLiveMediaPayload(livePayload, sampledAt, source);
@@ -602607,12 +603051,54 @@ var init_live_sensors = __esm({
602607
603051
  updatedAt: sampledAt,
602608
603052
  source,
602609
603053
  inference,
603054
+ rotation: orientation?.rotation ?? 0,
603055
+ orientation,
602610
603056
  ...preview.ok ? {} : { error: preview.message }
602611
603057
  };
602612
603058
  this.persist();
603059
+ this.emitFeedback({
603060
+ kind: result.success ? "video-inference" : "error",
603061
+ title: result.success ? "Live video inference" : "Live video inference failed",
603062
+ observedAt: sampledAt,
603063
+ message: boundedText(inference, 1800)
603064
+ });
603065
+ for (const person of observations.people.filter((entry) => entry.cropPath).slice(0, 4)) {
603066
+ const cropPath = person.cropPath;
603067
+ const displayPath = relative11(this.repoRoot, cropPath).startsWith("..") ? cropPath : relative11(this.repoRoot, cropPath);
603068
+ const facePreview = await buildImageAsciiPreview(cropPath, { width: 38, height: 14 });
603069
+ this.emitFeedback({
603070
+ kind: "face-crop",
603071
+ title: person.status === "known" && person.name ? `Known face: ${person.name}` : "Unknown face awaiting identification",
603072
+ observedAt: person.observedAt,
603073
+ message: person.status === "known" && person.name ? `Identified ${person.name}${person.confidence !== void 0 ? ` confidence=${person.confidence.toFixed(2)}` : ""}` : `Face crop saved and awaiting identification${person.confidence !== void 0 ? ` confidence=${person.confidence.toFixed(2)}` : ""}`,
603074
+ imagePath: cropPath,
603075
+ displayPath,
603076
+ ascii: facePreview?.ascii,
603077
+ renderer: facePreview?.renderer
603078
+ });
603079
+ }
603080
+ const unknownPeople = observations.people.filter((entry) => entry.status === "unknown");
603081
+ if (unknownPeople.length > 0) {
603082
+ this.maybeRequestAgentReview(
603083
+ `${unknownPeople.length} unknown individual(s) observed on live camera`,
603084
+ unknownPeople.map((entry) => `${entry.cropPath || entry.trackId || entry.source} ${entry.evidence || ""}`).join("; ")
603085
+ );
603086
+ }
603087
+ const triggerEvents = observations.events.filter((entry) => entry.kind === "visual_trigger");
603088
+ if (triggerEvents.length > 0) {
603089
+ this.maybeRequestAgentReview(
603090
+ "Visual trigger hit in live camera stream",
603091
+ triggerEvents.map((entry) => entry.summary).join("; ")
603092
+ );
603093
+ }
603094
+ }
603095
+ if (this.config.clipEnabled && preview.ok && preview.framePath && source) {
603096
+ clipSummary = await this.recognizeFrameObjects(preview.framePath, source, sampledAt);
602613
603097
  }
602614
603098
  return [preview.message, inference ? `
602615
- ${inference}` : ""].filter(Boolean).join("\n");
603099
+ ${inference}` : "", clipSummary ? `
603100
+ CLIP visual memory:
603101
+ ${clipSummary}` : ""].filter(Boolean).join("\n");
602616
603102
  } finally {
602617
603103
  this.videoInFlight = false;
602618
603104
  }
@@ -602638,6 +603124,12 @@ ${inference}` : ""].filter(Boolean).join("\n");
602638
603124
  error: message2
602639
603125
  };
602640
603126
  this.persist();
603127
+ this.emitFeedback({
603128
+ kind: "error",
603129
+ title: "Live audio capture failed",
603130
+ observedAt: this.snapshot.audio.updatedAt,
603131
+ message: message2
603132
+ });
602641
603133
  return message2;
602642
603134
  }
602643
603135
  if (this.config.asrEnabled) {
@@ -602698,6 +603190,19 @@ ${inference}` : ""].filter(Boolean).join("\n");
602698
603190
  }
602699
603191
  this.snapshot.events = mergeRecentById(this.snapshot.events, audioEvents, 50);
602700
603192
  this.persist();
603193
+ if (transcript || analysis) {
603194
+ this.emitFeedback({
603195
+ kind: "audio",
603196
+ title: "Live audio sample",
603197
+ observedAt: this.snapshot.audio.updatedAt,
603198
+ message: [
603199
+ transcript ? `ASR:
603200
+ ${boundedText(transcript, 700)}` : "",
603201
+ analysis ? `Sounds:
603202
+ ${boundedText(analysis, 900)}` : ""
603203
+ ].filter(Boolean).join("\n\n")
603204
+ });
603205
+ }
602701
603206
  return [
602702
603207
  `Audio sample captured from ${input}: ${recordingPath}`,
602703
603208
  transcript ? `
@@ -602713,10 +603218,10 @@ ${analysis}` : ""
602713
603218
  }
602714
603219
  ensureLoops() {
602715
603220
  if (this.config.videoEnabled && !this.videoTimer) {
602716
- void this.sampleVideoNow(false).catch(() => {
603221
+ void this.sampleVideoNow(this.config.inferEnabled || this.config.clipEnabled).catch(() => {
602717
603222
  });
602718
603223
  this.videoTimer = setInterval(() => {
602719
- void this.sampleVideoNow(this.config.inferEnabled).catch(() => {
603224
+ void this.sampleVideoNow(this.config.inferEnabled || this.config.clipEnabled).catch(() => {
602720
603225
  });
602721
603226
  }, this.config.videoIntervalMs);
602722
603227
  this.videoTimer.unref?.();
@@ -602745,6 +603250,38 @@ ${analysis}` : ""
602745
603250
  label: "live"
602746
603251
  });
602747
603252
  }
603253
+ emitFeedback(feedback) {
603254
+ try {
603255
+ this.feedbackSink?.(feedback);
603256
+ } catch {
603257
+ }
603258
+ }
603259
+ maybeRequestAgentReview(reason, evidence) {
603260
+ if (!this.agentActionEnabled || !this.agentActionSink) return;
603261
+ const now2 = Date.now();
603262
+ if (now2 - this.lastAgentReviewAt < 2e4) return;
603263
+ this.lastAgentReviewAt = now2;
603264
+ const prompt = [
603265
+ "Live environment PFC review.",
603266
+ "Inspect the current live sensor evidence and decide whether any immediate action is needed.",
603267
+ "Do not invent person names. If a person is unknown and identity matters, ask the user or person for their name and then store the association with visual_memory or multimodal_memory.",
603268
+ "Prefer minimal, reversible actions. If no action is needed, summarize the observation and stand by.",
603269
+ "",
603270
+ `Reason: ${reason}`,
603271
+ `Evidence: ${evidence}`,
603272
+ `Live snapshot: ${liveSnapshotPath(this.repoRoot)}`
603273
+ ].join("\n");
603274
+ this.emitFeedback({
603275
+ kind: "agent-review",
603276
+ title: "Live PFC review queued",
603277
+ observedAt: now2,
603278
+ message: reason
603279
+ });
603280
+ try {
603281
+ this.agentActionSink(prompt);
603282
+ } catch {
603283
+ }
603284
+ }
602748
603285
  persist() {
602749
603286
  ensureLiveDir(this.repoRoot);
602750
603287
  writeFileSync56(liveConfigPath(this.repoRoot), JSON.stringify(this.config, null, 2), "utf8");
@@ -608474,8 +609011,13 @@ var init_command_registry = __esm({
608474
609011
  ["/listen stop", "Stop listening"],
608475
609012
  ["/live", "Open live sensor stream menu for video/audio context"],
608476
609013
  ["/live status", "Show live sensor devices, selected streams, and latest snapshot"],
609014
+ ["/live run [agent]", "Start low-latency video/audio feedback; agent enables background PFC review"],
609015
+ ["/live chat", "Start low-latency voicechat with live audio/video context and async agent review"],
609016
+ ["/livechat", "Alias for /live chat, used by the top live button"],
608477
609017
  ["/live camera <device>", "Select a camera and render an ASCII preview"],
609018
+ ["/live rotate auto|cw|ccw|180|none [camera]", "Detect or set persistent camera orientation correction"],
608478
609019
  ["/live infer [now|on|off]", "Toggle or run camera object/location inference"],
609020
+ ["/live clip [on|off]", "Toggle CLIP visual-memory recognition on live frames"],
608479
609021
  ["/live audio|asr|sounds on|off", "Toggle live audio context streams"],
608480
609022
  ["/live stop", "Disable all live sensor streams"],
608481
609023
  ["/paste", "Attach clipboard image content to the current or next prompt"],
@@ -618863,7 +619405,7 @@ __export(omnius_directory_exports, {
618863
619405
  writeIndexMeta: () => writeIndexMeta,
618864
619406
  writeTaskHandoff: () => writeTaskHandoff2
618865
619407
  });
618866
- import { appendFileSync as appendFileSync13, cpSync as cpSync2, existsSync as existsSync117, mkdirSync as mkdirSync74, readFileSync as readFileSync95, writeFileSync as writeFileSync63, readdirSync as readdirSync40, statSync as statSync44, unlinkSync as unlinkSync22, openSync as openSync2, closeSync as closeSync2, renameSync as renameSync10, watch as fsWatch2 } from "node:fs";
619408
+ import { appendFileSync as appendFileSync13, cpSync as cpSync2, existsSync as existsSync117, mkdirSync as mkdirSync74, readFileSync as readFileSync95, writeFileSync as writeFileSync63, readdirSync as readdirSync40, statSync as statSync44, unlinkSync as unlinkSync22, openSync as openSync2, closeSync as closeSync2, renameSync as renameSync11, watch as fsWatch2 } from "node:fs";
618867
619409
  import { join as join133, relative as relative13, basename as basename25, dirname as dirname40, resolve as resolve57 } from "node:path";
618868
619410
  import { homedir as homedir41 } from "node:os";
618869
619411
  import { createHash as createHash37 } from "node:crypto";
@@ -619269,7 +619811,7 @@ function writeTaskHandoff2(repoRoot, handoff) {
619269
619811
  const tempPath = filePath + ".tmp";
619270
619812
  writeFileSync63(tempPath, JSON.stringify(handoff, null, 2) + "\n", "utf-8");
619271
619813
  try {
619272
- renameSync10(tempPath, filePath);
619814
+ renameSync11(tempPath, filePath);
619273
619815
  } catch {
619274
619816
  writeFileSync63(filePath, JSON.stringify(handoff, null, 2) + "\n", "utf-8");
619275
619817
  try {
@@ -619579,7 +620121,7 @@ function saveSessionContext(repoRoot, entry) {
619579
620121
  const tempFilePath = filePath + ".tmp";
619580
620122
  writeFileSync63(tempFilePath, JSON.stringify(ctx3, null, 2) + "\n", "utf-8");
619581
620123
  try {
619582
- renameSync10(tempFilePath, filePath);
620124
+ renameSync11(tempFilePath, filePath);
619583
620125
  } catch {
619584
620126
  writeFileSync63(filePath, JSON.stringify(ctx3, null, 2) + "\n", "utf-8");
619585
620127
  try {
@@ -622522,7 +623064,7 @@ function setTerminalTitle(task, version5) {
622522
623064
  process.stdout.write(data);
622523
623065
  }
622524
623066
  }
622525
- var EXPERT_TOOL_BASELINES, CONTEXT_SWITCH_OVERHEAD, TURN_PLANNING_OVERHEAD, DEFAULT_TOOL_BASELINE, CODE_READ_CHARS_PER_SEC, PROSE_READ_CHARS_PER_SEC, MIN_CONTENT_FOR_READING, CODE_CONTENT_TOOLS, PROSE_CONTENT_TOOLS, HumanSpeedTracker, PANEL_BG_SEQ, CONTENT_BG_SEQ, BOX_FG, TEXT_PRIMARY, TEXT_DIM, NO_SUB_AGENTS_HEADER_LABEL, HEADER_BUTTON_GLYPH_FG, HEADER_BUTTON_BG, HEADER_BUTTON_FG, HEADER_ACCENT_BOLD_FG, HEADER_TELEGRAM_FG, BOX_TL3, BOX_TR3, BOX_BL3, BOX_BR3, BOX_H3, BOX_V3, BOX_BJ, _globalFooterLock, RESET4, CURSOR_BLINK_BLOCK, _isWindows, SPONSOR_HEADER_LABEL_MAX, _termTitleWriter, StatusBar;
623067
+ var EXPERT_TOOL_BASELINES, CONTEXT_SWITCH_OVERHEAD, TURN_PLANNING_OVERHEAD, DEFAULT_TOOL_BASELINE, CODE_READ_CHARS_PER_SEC, PROSE_READ_CHARS_PER_SEC, MIN_CONTENT_FOR_READING, CODE_CONTENT_TOOLS, PROSE_CONTENT_TOOLS, HumanSpeedTracker, PANEL_BG_SEQ, CONTENT_BG_SEQ, BOX_FG, TEXT_PRIMARY, TEXT_DIM, NO_SUB_AGENTS_HEADER_LABEL, HEADER_BUTTON_GLYPH_FG, HEADER_BUTTON_BG, HEADER_BUTTON_FG, HEADER_ACCENT_BOLD_FG, HEADER_TELEGRAM_FG, BOX_TL3, BOX_TR3, BOX_BL3, BOX_BR3, BOX_H3, BOX_V3, BOX_BJ, _globalFooterLock, RESET4, CURSOR_BLINK_BLOCK, _isWindows, HEADER_BUTTON_LEFT, HEADER_BUTTON_RIGHT, SPONSOR_HEADER_LABEL_MAX, _termTitleWriter, StatusBar;
622526
623068
  var init_status_bar = __esm({
622527
623069
  "packages/cli/src/tui/status-bar.ts"() {
622528
623070
  "use strict";
@@ -622719,6 +623261,8 @@ var init_status_bar = __esm({
622719
623261
  RESET4 = "\x1B[0m";
622720
623262
  CURSOR_BLINK_BLOCK = "\x1B[1 q";
622721
623263
  _isWindows = process.platform === "win32";
623264
+ HEADER_BUTTON_LEFT = _isWindows ? "[" : "🭁";
623265
+ HEADER_BUTTON_RIGHT = _isWindows ? "]" : "🭝";
622722
623266
  SPONSOR_HEADER_LABEL_MAX = 48;
622723
623267
  _termTitleWriter = null;
622724
623268
  StatusBar = class _StatusBar {
@@ -623267,27 +623811,30 @@ var init_status_bar = __esm({
623267
623811
  const buttonFg = (cmd) => {
623268
623812
  let fg2 = TEXT_DIM;
623269
623813
  if (cmd === "voice" && this._voiceActive) fg2 = 82;
623270
- if (cmd === "live" && (this._liveMediaStatus.audio || this._liveMediaStatus.video)) fg2 = 82;
623814
+ if (cmd.startsWith("live") && (this._liveMediaStatus.audio || this._liveMediaStatus.video)) fg2 = 82;
623271
623815
  if (cmd === "nexus") {
623272
623816
  fg2 = this._nexusStatus === "connected" ? 82 : this._nexusStatus === "connecting" ? 208 : 196;
623273
623817
  }
623274
623818
  return fg2;
623275
623819
  };
623276
623820
  const decorateMenuButton = (cmd, label) => {
623277
- return `${HEADER_BUTTON_GLYPH_FG}[\x1B[0m${HEADER_BUTTON_BG}${HEADER_BUTTON_FG}${label}\x1B[0m${PANEL_BG_SEQ}${HEADER_BUTTON_GLYPH_FG}]\x1B[0m${PANEL_BG_SEQ}`;
623821
+ return `${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_LEFT}\x1B[0m${HEADER_BUTTON_BG}${HEADER_BUTTON_FG}${label}\x1B[0m${PANEL_BG_SEQ}${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_RIGHT}\x1B[0m${PANEL_BG_SEQ}`;
623278
623822
  };
623279
623823
  const decorateAgentButton = (content, color, active) => {
623280
623824
  const bg = `\x1B[48;5;${color}m`;
623281
623825
  const fg2 = `\x1B[38;5;${contrastTextColor(color)}m`;
623282
623826
  const weight = active ? "\x1B[1m" : "";
623283
- return `\x1B[38;5;${color}m[\x1B[0m${weight}${fg2}${bg}${content}\x1B[0m${PANEL_BG_SEQ}\x1B[38;5;${color}m]\x1B[0m${PANEL_BG_SEQ}`;
623827
+ return `\x1B[38;5;${color}m${HEADER_BUTTON_LEFT}\x1B[0m${weight}${fg2}${bg}${content}\x1B[0m${PANEL_BG_SEQ}\x1B[38;5;${color}m${HEADER_BUTTON_RIGHT}\x1B[0m${PANEL_BG_SEQ}`;
623284
623828
  };
623285
623829
  const renderBtn = (cmd, label) => {
623286
623830
  const fg2 = buttonFg(cmd);
623287
- const btnW = label.length;
623831
+ const btnW = label.length + 2;
623288
623832
  const availW2 = getTermWidth() - identity3.width - 1;
623289
623833
  if (btnW > availW2) return linkify(cmd, `\x1B[38;5;${fg2}m${label}`);
623290
- return linkify(cmd, `${PANEL_BG_SEQ}\x1B[38;5;${fg2}m${label}\x1B[0m${PANEL_BG_SEQ}`);
623834
+ return linkify(
623835
+ cmd,
623836
+ `${PANEL_BG_SEQ}${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_LEFT}\x1B[0m${PANEL_BG_SEQ}\x1B[38;5;${fg2}m${label}\x1B[0m${PANEL_BG_SEQ}${HEADER_BUTTON_GLYPH_FG}${HEADER_BUTTON_RIGHT}\x1B[0m${PANEL_BG_SEQ}`
623837
+ );
623291
623838
  };
623292
623839
  const identity3 = this.buildHeaderIdentityRender();
623293
623840
  const modelLabel = this.summarizeHeaderModelName() || "model";
@@ -623301,6 +623848,8 @@ var init_status_bar = __esm({
623301
623848
  w: (this._voiceActive ? this._voiceModelId || "voice" : "voice").length + 2
623302
623849
  },
623303
623850
  // +2 for spaces
623851
+ { cmd: "livechat", label: "live", w: "live".length + 2 },
623852
+ // +2 for spaces
623304
623853
  { cmd: "model", label: modelLabel, w: modelLabel.length + 2 },
623305
623854
  // +2 for spaces
623306
623855
  { cmd: "endpoint", label: endpointLabel, w: endpointLabel.length + 2 }
@@ -648807,6 +649356,10 @@ sleep 1
648807
649356
  await handleLiveCommand(ctx3, arg);
648808
649357
  return "handled";
648809
649358
  }
649359
+ case "livechat": {
649360
+ await handleLiveCommand(ctx3, "chat");
649361
+ return "handled";
649362
+ }
648810
649363
  case "call": {
648811
649364
  if (!ctx3.callStart) {
648812
649365
  renderWarning("Call mode not available in this context.");
@@ -654417,6 +654970,59 @@ function renderLivePreview(preview) {
654417
654970
  renderWarning("Camera preview was captured, but ASCII rendering was unavailable.");
654418
654971
  }
654419
654972
  }
654973
+ function renderLiveFeedback(feedback) {
654974
+ const stamp = new Date(feedback.observedAt).toISOString();
654975
+ if (feedback.kind === "error") {
654976
+ renderWarning(`[live ${stamp}] ${feedback.title}
654977
+ ${feedback.message}`);
654978
+ return;
654979
+ }
654980
+ if ((feedback.kind === "camera-frame" || feedback.kind === "face-crop") && feedback.ascii && feedback.displayPath && feedback.renderer) {
654981
+ renderInfo(`[live ${stamp}] ${feedback.message}`);
654982
+ renderImageAsciiPreview(feedback.title, feedback.displayPath, feedback.ascii, feedback.renderer);
654983
+ return;
654984
+ }
654985
+ renderInfo(`[live ${stamp}] ${feedback.title}
654986
+ ${feedback.message}`);
654987
+ }
654988
+ function attachLiveSinks(ctx3, manager, agentEnabled = false) {
654989
+ manager.setStatusSink(ctx3.setLiveMediaStatus);
654990
+ manager.setFeedbackSink((feedback) => renderLiveFeedback(feedback));
654991
+ if (agentEnabled && ctx3.startBackgroundPrompt) {
654992
+ manager.setAgentActionSink((prompt) => {
654993
+ const id2 = ctx3.startBackgroundPrompt?.(prompt);
654994
+ if (id2) renderInfo(`Live PFC background review started: ${id2}`);
654995
+ });
654996
+ } else {
654997
+ manager.setAgentActionSink(void 0);
654998
+ }
654999
+ }
655000
+ async function startLiveVoicechat(ctx3, manager) {
655001
+ const agentEnabled = Boolean(ctx3.startBackgroundPrompt);
655002
+ attachLiveSinks(ctx3, manager, agentEnabled);
655003
+ await ensureLiveInventory(manager);
655004
+ renderInfo(manager.startRunMode({ agent: agentEnabled, lowLatency: true }));
655005
+ if (!ctx3.voiceChatStart) {
655006
+ renderWarning("Voice chat is not available in this context; live audio/video feedback remains active.");
655007
+ return;
655008
+ }
655009
+ ctx3.voiceSetMode?.("voicechat");
655010
+ if (ctx3.isVoiceChatActive?.()) {
655011
+ renderInfo("Voice chat is already active. Live audio/video feedback remains attached.");
655012
+ return;
655013
+ }
655014
+ try {
655015
+ await ensureVoiceDeps(ctx3);
655016
+ } catch {
655017
+ }
655018
+ try {
655019
+ renderInfo("Starting live voicechat with audio/video context...");
655020
+ await ctx3.voiceChatStart();
655021
+ renderInfo("Live voicechat active: low-latency audio/video context is updating while voice conversation continues.");
655022
+ } catch (err) {
655023
+ renderError(`Live voicechat failed: ${err instanceof Error ? err.message : String(err)}`);
655024
+ }
655025
+ }
654420
655026
  async function chooseLiveDevice(ctx3, title, devices, activeKey) {
654421
655027
  if (devices.length === 0) {
654422
655028
  renderWarning("No devices found. Run /live refresh after connecting hardware.");
@@ -654439,7 +655045,7 @@ async function chooseLiveDevice(ctx3, title, devices, activeKey) {
654439
655045
  }
654440
655046
  async function handleLiveCommand(ctx3, arg) {
654441
655047
  const manager = getLiveSensorManager(ctx3.repoRoot);
654442
- manager.setStatusSink(ctx3.setLiveMediaStatus);
655048
+ attachLiveSinks(ctx3, manager);
654443
655049
  const parts = arg.trim().split(/\s+/).filter(Boolean);
654444
655050
  const sub2 = (parts[0] || "").toLowerCase();
654445
655051
  const value2 = parts.slice(1).join(" ");
@@ -654460,9 +655066,48 @@ async function handleLiveCommand(ctx3, arg) {
654460
655066
  }
654461
655067
  if (sub2 === "stop" || sub2 === "off" || sub2 === "disable") {
654462
655068
  manager.stopAll();
655069
+ manager.setAgentActionSink(void 0);
654463
655070
  renderInfo("Live sensor streams disabled.");
654464
655071
  return;
654465
655072
  }
655073
+ if (sub2 === "run" || sub2 === "start" || sub2 === "pfc") {
655074
+ const agentEnabled = sub2 === "pfc" || /\bagent|pfc|action\b/i.test(value2);
655075
+ const effectiveAgent = agentEnabled && Boolean(ctx3.startBackgroundPrompt);
655076
+ attachLiveSinks(ctx3, manager, effectiveAgent);
655077
+ if (agentEnabled && !ctx3.startBackgroundPrompt) {
655078
+ renderWarning("Live PFC agent review is unavailable in this context; starting visual/audio feedback only.");
655079
+ }
655080
+ await ensureLiveInventory(manager);
655081
+ renderInfo(manager.startRunMode({ agent: effectiveAgent, lowLatency: true }));
655082
+ return;
655083
+ }
655084
+ if (sub2 === "chat" || sub2 === "voicechat" || sub2 === "voice") {
655085
+ await startLiveVoicechat(ctx3, manager);
655086
+ return;
655087
+ }
655088
+ if (sub2 === "rotate" || sub2 === "rotation" || sub2 === "orient" || sub2 === "orientation") {
655089
+ await ensureLiveInventory(manager);
655090
+ const mode = (parts[1] || "cycle").toLowerCase();
655091
+ const deviceArg = parts.slice(2).join(" ") || manager.getConfig().selectedCamera;
655092
+ if (mode === "auto" || mode === "detect") {
655093
+ renderInfo("Detecting camera orientation...");
655094
+ renderInfo(await manager.autoOrientCamera(deviceArg));
655095
+ return;
655096
+ }
655097
+ if (mode === "cycle") {
655098
+ const orientation = manager.cycleCameraRotation(deviceArg);
655099
+ renderInfo(orientation ? `Camera rotation set to ${formatCameraRotation(orientation.rotation)} for ${deviceArg || manager.getConfig().selectedCamera || "selected camera"}.` : "No camera selected. Use /live camera <device> first.");
655100
+ return;
655101
+ }
655102
+ try {
655103
+ const rotation = normalizeLiveCameraRotation(mode);
655104
+ const orientation = manager.setCameraRotation(deviceArg, rotation, "manual", `/live rotate ${mode}`);
655105
+ renderInfo(orientation ? `Camera rotation set to ${formatCameraRotation(orientation.rotation)} for ${deviceArg || manager.getConfig().selectedCamera || "selected camera"}.` : "No camera selected. Use /live camera <device> first.");
655106
+ } catch (err) {
655107
+ renderWarning(err instanceof Error ? err.message : String(err));
655108
+ }
655109
+ return;
655110
+ }
654466
655111
  if (sub2 === "video") {
654467
655112
  const cfg = manager.getConfig();
654468
655113
  manager.configure({ videoEnabled: setBool(value2, !cfg.videoEnabled) });
@@ -654508,6 +655153,15 @@ async function handleLiveCommand(ctx3, arg) {
654508
655153
  }
654509
655154
  return;
654510
655155
  }
655156
+ if (sub2 === "clip") {
655157
+ const cfg = manager.getConfig();
655158
+ manager.configure({
655159
+ videoEnabled: true,
655160
+ clipEnabled: setBool(value2, !cfg.clipEnabled)
655161
+ });
655162
+ renderInfo(formatLiveStatus(manager.getSnapshot()));
655163
+ return;
655164
+ }
654511
655165
  if (sub2 === "camera") {
654512
655166
  await ensureLiveInventory(manager);
654513
655167
  const selected = value2 || manager.getConfig().selectedCamera;
@@ -654551,9 +655205,24 @@ async function showLiveMenu(ctx3, manager) {
654551
655205
  {
654552
655206
  key: "info:status",
654553
655207
  label: selectColors.dim(
654554
- ` video ${liveEnabled(cfg.videoEnabled)} · infer ${liveEnabled(cfg.inferEnabled)} · audio ${liveEnabled(cfg.audioEnabled)} · asr ${liveEnabled(cfg.asrEnabled)} · sounds ${liveEnabled(cfg.audioAnalysisEnabled)}`
655208
+ ` video ${liveEnabled(cfg.videoEnabled)} · infer ${liveEnabled(cfg.inferEnabled)} · clip ${liveEnabled(cfg.clipEnabled)} · audio ${liveEnabled(cfg.audioEnabled)} · asr ${liveEnabled(cfg.asrEnabled)} · sounds ${liveEnabled(cfg.audioAnalysisEnabled)}`
654555
655209
  )
654556
655210
  },
655211
+ {
655212
+ key: "run-live",
655213
+ label: "Start Live Run",
655214
+ detail: "low-latency visual/audio feedback + context injection"
655215
+ },
655216
+ {
655217
+ key: "run-agent",
655218
+ label: "Start Live PFC Run",
655219
+ detail: "live feedback + background agent review for significant events"
655220
+ },
655221
+ {
655222
+ key: "run-chat",
655223
+ label: "Start Live Voicechat",
655224
+ detail: "low-latency voice conversation with live audio/video context"
655225
+ },
654557
655226
  {
654558
655227
  key: "toggle-video",
654559
655228
  label: cfg.videoEnabled ? "Disable Video Context" : "Enable Video Context",
@@ -654569,11 +655238,26 @@ async function showLiveMenu(ctx3, manager) {
654569
655238
  label: "View Selected Camera",
654570
655239
  detail: "capture frame + image-to-ascii preview"
654571
655240
  },
655241
+ {
655242
+ key: "auto-orient-camera",
655243
+ label: "Auto-Orient Camera",
655244
+ detail: `current: ${cfg.selectedCamera ? formatCameraRotation(manager.getCameraRotation(cfg.selectedCamera)) : "no camera selected"}`
655245
+ },
655246
+ {
655247
+ key: "cycle-rotation",
655248
+ label: "Cycle Camera Rotation",
655249
+ detail: "manual override: upright -> clockwise -> 180 -> counter-clockwise"
655250
+ },
654572
655251
  {
654573
655252
  key: "toggle-infer",
654574
655253
  label: cfg.inferEnabled ? "Disable Frame Inference" : "Enable Frame Inference",
654575
655254
  detail: "YOLO object/location stream into context"
654576
655255
  },
655256
+ {
655257
+ key: "toggle-clip",
655258
+ label: cfg.clipEnabled ? "Disable CLIP Visual Memory" : "Enable CLIP Visual Memory",
655259
+ detail: "recognize taught objects from camera frames"
655260
+ },
654577
655261
  {
654578
655262
  key: "infer-now",
654579
655263
  label: "Infer Current Camera Frame",
@@ -654647,6 +655331,20 @@ async function showLiveMenu(ctx3, manager) {
654647
655331
  case "toggle-video":
654648
655332
  manager.configure({ videoEnabled: !cfg.videoEnabled });
654649
655333
  continue;
655334
+ case "run-live":
655335
+ attachLiveSinks(ctx3, manager, false);
655336
+ renderInfo(manager.startRunMode({ agent: false, lowLatency: true }));
655337
+ continue;
655338
+ case "run-agent":
655339
+ attachLiveSinks(ctx3, manager, true);
655340
+ if (!ctx3.startBackgroundPrompt) {
655341
+ renderWarning("Live PFC agent review is unavailable in this context; starting visual/audio feedback only.");
655342
+ }
655343
+ renderInfo(manager.startRunMode({ agent: Boolean(ctx3.startBackgroundPrompt), lowLatency: true }));
655344
+ continue;
655345
+ case "run-chat":
655346
+ await startLiveVoicechat(ctx3, manager);
655347
+ continue;
654650
655348
  case "toggle-audio":
654651
655349
  manager.configure({ audioEnabled: !cfg.audioEnabled });
654652
655350
  continue;
@@ -654656,6 +655354,9 @@ async function showLiveMenu(ctx3, manager) {
654656
655354
  case "toggle-infer":
654657
655355
  manager.configure({ videoEnabled: true, inferEnabled: !cfg.inferEnabled });
654658
655356
  continue;
655357
+ case "toggle-clip":
655358
+ manager.configure({ videoEnabled: true, clipEnabled: !cfg.clipEnabled });
655359
+ continue;
654659
655360
  case "toggle-asr":
654660
655361
  manager.configure({ audioEnabled: true, asrEnabled: !cfg.asrEnabled });
654661
655362
  continue;
@@ -654683,6 +655384,15 @@ async function showLiveMenu(ctx3, manager) {
654683
655384
  case "preview-camera":
654684
655385
  renderLivePreview(await manager.previewCamera());
654685
655386
  continue;
655387
+ case "auto-orient-camera":
655388
+ renderInfo("Detecting camera orientation...");
655389
+ renderInfo(await manager.autoOrientCamera(cfg.selectedCamera));
655390
+ continue;
655391
+ case "cycle-rotation": {
655392
+ const orientation = manager.cycleCameraRotation(cfg.selectedCamera);
655393
+ renderInfo(orientation ? `Camera rotation set to ${formatCameraRotation(orientation.rotation)} for ${cfg.selectedCamera || manager.getConfig().selectedCamera || "selected camera"}.` : "No camera selected. Use Select Camera first.");
655394
+ continue;
655395
+ }
654686
655396
  case "infer-now":
654687
655397
  manager.configure({ videoEnabled: true, inferEnabled: true });
654688
655398
  renderInfo("Sampling camera inference...");
@@ -661333,7 +662043,7 @@ import {
661333
662043
  readFileSync as readFileSync113,
661334
662044
  readdirSync as readdirSync49,
661335
662045
  writeFileSync as writeFileSync74,
661336
- renameSync as renameSync12,
662046
+ renameSync as renameSync13,
661337
662047
  mkdirSync as mkdirSync85,
661338
662048
  unlinkSync as unlinkSync29,
661339
662049
  appendFileSync as appendFileSync16
@@ -661357,7 +662067,7 @@ function persistSession(s2) {
661357
662067
  const final2 = sessionPath(s2.id);
661358
662068
  const tmp = `${final2}.tmp.${process.pid}.${Date.now()}`;
661359
662069
  writeFileSync74(tmp, JSON.stringify(s2, null, 2), "utf-8");
661360
- renameSync12(tmp, final2);
662070
+ renameSync13(tmp, final2);
661361
662071
  } catch {
661362
662072
  }
661363
662073
  }
@@ -661367,7 +662077,7 @@ function persistInFlight(j) {
661367
662077
  const final2 = inFlightPath(j.sessionId);
661368
662078
  const tmp = `${final2}.tmp.${process.pid}.${Date.now()}`;
661369
662079
  writeFileSync74(tmp, JSON.stringify(j, null, 2), "utf-8");
661370
- renameSync12(tmp, final2);
662080
+ renameSync13(tmp, final2);
661371
662081
  } catch {
661372
662082
  }
661373
662083
  }
@@ -694197,7 +694907,7 @@ __export(projects_exports, {
694197
694907
  setCurrentProject: () => setCurrentProject,
694198
694908
  unregisterProject: () => unregisterProject
694199
694909
  });
694200
- import { readFileSync as readFileSync124, writeFileSync as writeFileSync83, mkdirSync as mkdirSync95, existsSync as existsSync152, statSync as statSync55, renameSync as renameSync13 } from "node:fs";
694910
+ import { readFileSync as readFileSync124, writeFileSync as writeFileSync83, mkdirSync as mkdirSync95, existsSync as existsSync152, statSync as statSync55, renameSync as renameSync14 } from "node:fs";
694201
694911
  import { homedir as homedir55 } from "node:os";
694202
694912
  import { basename as basename40, join as join166, resolve as resolve68 } from "node:path";
694203
694913
  import { randomUUID as randomUUID19 } from "node:crypto";
@@ -694216,7 +694926,7 @@ function writeAll(file) {
694216
694926
  mkdirSync95(OMNIUS_DIR3, { recursive: true });
694217
694927
  const tmp = `${PROJECTS_FILE}.${randomUUID19().slice(0, 8)}.tmp`;
694218
694928
  writeFileSync83(tmp, JSON.stringify(file, null, 2), "utf8");
694219
- renameSync13(tmp, PROJECTS_FILE);
694929
+ renameSync14(tmp, PROJECTS_FILE);
694220
694930
  }
694221
694931
  function listProjects() {
694222
694932
  const { projects } = readAll2();
@@ -695195,7 +695905,7 @@ var init_access_policy = __esm({
695195
695905
 
695196
695906
  // packages/cli/src/api/project-preferences.ts
695197
695907
  import { createHash as createHash43 } from "node:crypto";
695198
- import { existsSync as existsSync153, mkdirSync as mkdirSync96, readFileSync as readFileSync125, renameSync as renameSync14, writeFileSync as writeFileSync84, unlinkSync as unlinkSync35 } from "node:fs";
695908
+ import { existsSync as existsSync153, mkdirSync as mkdirSync96, readFileSync as readFileSync125, renameSync as renameSync15, writeFileSync as writeFileSync84, unlinkSync as unlinkSync35 } from "node:fs";
695199
695909
  import { homedir as homedir56 } from "node:os";
695200
695910
  import { join as join167, resolve as resolve69 } from "node:path";
695201
695911
  import { randomUUID as randomUUID20 } from "node:crypto";
@@ -695249,7 +695959,7 @@ function writeProjectPreferences(root, partial) {
695249
695959
  const tmp = `${file}.${randomUUID20().slice(0, 8)}.tmp`;
695250
695960
  writeFileSync84(tmp, JSON.stringify(merged, null, 2), "utf8");
695251
695961
  try {
695252
- renameSync14(tmp, file);
695962
+ renameSync15(tmp, file);
695253
695963
  } catch (err) {
695254
695964
  try {
695255
695965
  writeFileSync84(file, JSON.stringify(merged, null, 2), "utf8");
@@ -696305,7 +697015,7 @@ var init_direct_tool_registry = __esm({
696305
697015
  });
696306
697016
 
696307
697017
  // packages/cli/src/api/external-tool-registry.ts
696308
- import { existsSync as existsSync157, mkdirSync as mkdirSync100, readFileSync as readFileSync127, writeFileSync as writeFileSync85, renameSync as renameSync15 } from "node:fs";
697018
+ import { existsSync as existsSync157, mkdirSync as mkdirSync100, readFileSync as readFileSync127, writeFileSync as writeFileSync85, renameSync as renameSync16 } from "node:fs";
696309
697019
  import { join as join170 } from "node:path";
696310
697020
  function externalToolStorePath(workingDir) {
696311
697021
  return join170(workingDir, ".omnius", "external-tools.json");
@@ -696329,7 +697039,7 @@ function persist3(workingDir, tools) {
696329
697039
  const file = { version: STORE_VERSION, tools };
696330
697040
  const tmp = `${path12}.tmp`;
696331
697041
  writeFileSync85(tmp, JSON.stringify(file, null, 2), { mode: 384 });
696332
- renameSync15(tmp, path12);
697042
+ renameSync16(tmp, path12);
696333
697043
  }
696334
697044
  function validateManifest2(input, existing) {
696335
697045
  if (!input || typeof input !== "object") {
@@ -714217,7 +714927,7 @@ import {
714217
714927
  readdirSync as readdirSync58,
714218
714928
  existsSync as existsSync165,
714219
714929
  watch as fsWatch4,
714220
- renameSync as renameSync16,
714930
+ renameSync as renameSync17,
714221
714931
  unlinkSync as unlinkSync36,
714222
714932
  statSync as statSync60,
714223
714933
  openSync as openSync6,
@@ -715874,7 +716584,7 @@ function atomicJobWrite(dir, id2, job) {
715874
716584
  const tmpPath = `${finalPath}.tmp.${process.pid}.${Date.now()}`;
715875
716585
  try {
715876
716586
  writeFileSync90(tmpPath, JSON.stringify(job, null, 2), "utf-8");
715877
- renameSync16(tmpPath, finalPath);
716587
+ renameSync17(tmpPath, finalPath);
715878
716588
  } catch {
715879
716589
  try {
715880
716590
  writeFileSync90(finalPath, JSON.stringify(job, null, 2), "utf-8");
@@ -717712,7 +718422,7 @@ function writeUpdateState(state) {
717712
718422
  const finalPath = updateStateFile();
717713
718423
  const tmpPath = `${finalPath}.tmp.${process.pid}`;
717714
718424
  writeFileSync90(tmpPath, JSON.stringify(state, null, 2), "utf-8");
717715
- renameSync16(tmpPath, finalPath);
718425
+ renameSync17(tmpPath, finalPath);
717716
718426
  } catch {
717717
718427
  }
717718
718428
  }