infinicode 2.8.46 → 2.8.51

package/packages/robopark/scheduler/preview_agent.py

@@ -69,6 +69,7 @@ logger = logging.getLogger("robopark.preview_agent")
 _DEVICE_INVENTORY_CACHE: Optional[dict] = None
 _DEVICE_INVENTORY_CACHE_AT = 0.0
 DEVICE_INVENTORY_CACHE_SECONDS = 30.0
+ROBOVISION_MEDIA_URL = os.getenv("ROBOVISION_MEDIA_URL", "http://127.0.0.1:5000/api/media/inventory")
 
 
 def _normalize_livekit_url(url: Optional[str]) -> Optional[str]:
@@ -141,50 +142,67 @@ def _get_device_inventory() -> dict:
             and now - _DEVICE_INVENTORY_CACHE_AT < DEVICE_INVENTORY_CACHE_SECONDS):
         return _DEVICE_INVENTORY_CACHE
 
-    inventory = {"video": [], "audio_input": [], "audio_output": [], "platform": sys.platform}
-    inventory["video"].append({"id": "auto", "name": "Auto detect"})
-    inventory["video"].append({"id": "none", "name": "Disable camera"})
-
-    try:
-        import cv2
-        candidates = sorted(glob.glob("/dev/video*")) if os.name != "nt" else [str(i) for i in range(10)]
-        for candidate in candidates:
-            value = int(candidate) if os.name == "nt" else candidate
-            backend = cv2.CAP_DSHOW if os.name == "nt" else cv2.CAP_ANY
-            cap = cv2.VideoCapture(value, backend)
-            if cap.isOpened():
-                device_id = str(value)
-                inventory["video"].append({"id": device_id, "name": f"Camera {candidate}", "backend": "dshow" if os.name == "nt" else "v4l2"})
-            cap.release()
-    except Exception as e:
-        logger.debug(f"camera inventory unavailable: {e}")
-
-    try:
-        import pyaudio
-        pa = pyaudio.PyAudio()
-        default_in = None
-        default_out = None
-        try:
-            wasapi = pa.get_host_api_info_by_type(pyaudio.paWASAPI)
-            default_in = wasapi.get("defaultInputDevice")
-            default_out = wasapi.get("defaultOutputDevice")
-        except Exception:
-            pass
-        inventory["audio_input"].append({"id": "default", "name": "System default input"})
-        inventory["audio_output"].append({"id": "default", "name": "System default output"})
-        for i in range(pa.get_device_count()):
-            info = pa.get_device_info_by_index(i)
-            name = str(info.get("name", f"Audio device {i}"))
-            item = {"id": str(i), "name": name, "host_api": str(info.get("hostApi", ""))}
-            if info.get("maxInputChannels", 0) > 0:
-                item["default"] = i == default_in
-                inventory["audio_input"].append(item.copy())
-            if info.get("maxOutputChannels", 0) > 0:
-                item["default"] = i == default_out
-                inventory["audio_output"].append(item.copy())
-        pa.terminate()
-    except Exception as e:
-        logger.debug(f"audio inventory unavailable: {e}")
+    inventory = {"video": [], "audio_input": [], "audio_output": [], "platform": sys.platform}
+    inventory["video"].append({"id": "auto", "name": "Auto detect"})
+    inventory["video"].append({"id": "none", "name": "Disable camera"})
+
+    # RoboVisionAI_PI owns the production camera. Prefer its native inventory
+    # so this process never probes an already-open V4L2 device just to fill a
+    # dashboard dropdown.
+    robovision_inventory = None
+    try:
+        response = httpx.get(ROBOVISION_MEDIA_URL, timeout=0.8)
+        if response.is_success:
+            robovision_inventory = response.json()
+            for key in ("video", "audio_input", "audio_output"):
+                if isinstance(robovision_inventory.get(key), list):
+                    inventory[key] = robovision_inventory[key]
+            inventory["source"] = robovision_inventory.get("source", "robovision_pi")
+    except Exception:
+        pass
+
+    if not robovision_inventory:
+        try:
+            import cv2
+            candidates = sorted(glob.glob("/dev/video*")) if os.name != "nt" else [str(i) for i in range(10)]
+            for candidate in candidates:
+                value = int(candidate) if os.name == "nt" else candidate
+                backend = cv2.CAP_DSHOW if os.name == "nt" else cv2.CAP_ANY
+                cap = cv2.VideoCapture(value, backend)
+                if cap.isOpened():
+                    device_id = str(value)
+                    inventory["video"].append({"id": device_id, "name": f"Camera {candidate}", "backend": "dshow" if os.name == "nt" else "v4l2"})
+                cap.release()
+        except Exception as e:
+            logger.debug(f"camera inventory unavailable: {e}")
+
+    if not robovision_inventory:
+        try:
+            import pyaudio
+            pa = pyaudio.PyAudio()
+            default_in = None
+            default_out = None
+            try:
+                wasapi = pa.get_host_api_info_by_type(pyaudio.paWASAPI)
+                default_in = wasapi.get("defaultInputDevice")
+                default_out = wasapi.get("defaultOutputDevice")
+            except Exception:
+                pass
+            inventory["audio_input"].append({"id": "default", "name": "System default input"})
+            inventory["audio_output"].append({"id": "default", "name": "System default output"})
+            for i in range(pa.get_device_count()):
+                info = pa.get_device_info_by_index(i)
+                name = str(info.get("name", f"Audio device {i}"))
+                item = {"id": str(i), "name": name, "host_api": str(info.get("hostApi", ""))}
+                if info.get("maxInputChannels", 0) > 0:
+                    item["default"] = i == default_in
+                    inventory["audio_input"].append(item.copy())
+                if info.get("maxOutputChannels", 0) > 0:
+                    item["default"] = i == default_out
+                    inventory["audio_output"].append(item.copy())
+            pa.terminate()
+        except Exception as e:
+            logger.debug(f"audio inventory unavailable: {e}")
 
     _DEVICE_INVENTORY_CACHE = inventory
     _DEVICE_INVENTORY_CACHE_AT = now
@@ -318,8 +336,10 @@ class PreviewAgent:
         self.device_id: Optional[str] = cfg.get("device_id")
         self.enrollment_token: Optional[str] = cfg.get("enrollment_token")
         self.video_device = cfg.get("video_device", os.getenv("VIDEO_DEVICE", "auto"))
-        self.audio_device = cfg.get("audio_device", os.getenv("AUDIO_DEVICE", "default"))
-        self.audio_output_device = cfg.get("audio_output_device", os.getenv("AUDIO_OUTPUT_DEVICE", "default"))
+        self.audio_device = cfg.get("audio_device", os.getenv("AUDIO_DEVICE", "default"))
+        self.audio_output_device = cfg.get("audio_output_device", os.getenv("AUDIO_OUTPUT_DEVICE", "default"))
+        self.robovision_url = cfg.get("robovision_url", os.getenv("ROBOVISION_URL", "http://127.0.0.1:5000"))
+        self.use_robovision_camera = bool(cfg.get("use_robovision_camera", False))
         self.width = int(cfg.get("video_width", os.getenv("VIDEO_WIDTH", DEFAULT_VIDEO_WIDTH)))
         self.height = int(cfg.get("video_height", os.getenv("VIDEO_HEIGHT", DEFAULT_VIDEO_HEIGHT)))
         self.fps = int(cfg.get("video_fps", os.getenv("VIDEO_FPS", DEFAULT_FPS)))
@@ -524,7 +544,7 @@ class PreviewAgent:
         if data.get("trigger"):
             await self._on_vision_motion({"source": data.get("source", "dashboard")})
 
-    async def _poll_device_config(self) -> None:
+    async def _poll_device_config(self) -> None:
         """Apply dashboard-selected camera and microphone IDs before preview."""
         if not self._session or not self.device_id or not self.device_token:
             return
@@ -538,16 +558,41 @@ class PreviewAgent:
                 headers={"Authorization": f"Bearer {self.device_token}"},
                 timeout=10.0,
             )
-            r.raise_for_status()
-            data = r.json()
-            if data.get("video_device") is not None:
-                self.video_device = data["video_device"]
-            if data.get("audio_device") is not None:
-                self.audio_device = data["audio_device"]
-            if data.get("audio_output_device") is not None:
-                self.audio_output_device = data["audio_output_device"]
-        except Exception as e:
-            logger.debug(f"device config poll failed: {e}")
+            r.raise_for_status()
+            data = r.json()
+            changed = False
+            if data.get("video_device") is not None:
+                changed = changed or self.video_device != data["video_device"]
+                self.video_device = data["video_device"]
+            if data.get("audio_device") is not None:
+                changed = changed or self.audio_device != data["audio_device"]
+                self.audio_device = data["audio_device"]
+            if data.get("audio_output_device") is not None:
+                changed = changed or self.audio_output_device != data["audio_output_device"]
+                self.audio_output_device = data["audio_output_device"]
+            if changed:
+                await self._sync_robovision_media_config()
+        except Exception as e:
+            logger.debug(f"device config poll failed: {e}")
+
+    async def _sync_robovision_media_config(self) -> None:
+        """Apply scheduler media choices to the local RoboVisionAI_PI owner."""
+        try:
+            response = await self._session.post(
+                f"{self.robovision_url.rstrip('/')}/api/media/config",
+                json={
+                    "video_device": self.video_device,
+                    "audio_device": self.audio_device,
+                    "audio_output_device": self.audio_output_device,
+                },
+                timeout=1.0,
+            )
+            if response.is_success:
+                global _DEVICE_INVENTORY_CACHE
+                _DEVICE_INVENTORY_CACHE = None
+        except Exception:
+            # RoboVision is optional on laptops and simulation nodes.
+            pass
 
     async def _check_vision_session(self) -> None:
         """Decide whether the active motion-triggered session should keep
@@ -908,7 +953,8 @@ class LiveKitPublisher:
         if video_enabled:
             try:
                 self._capture = await asyncio.to_thread(
-                    create_video_capture, self.agent.video_device, self.agent.width, self.agent.height, self.agent.fps
+                            create_video_capture, self.agent.video_device, self.agent.width, self.agent.height, self.agent.fps,
+                            self.agent.robovision_url if self.agent.use_robovision_camera else None,
                 )
             except Exception as e:
                 # Camera availability must not block the microphone/session.
@@ -1389,15 +1435,26 @@ class Picamera2Capture(VideoCapture):
             pass
 
 
-def create_video_capture(device: str, width: int, height: int, fps: int) -> Optional[VideoCapture]:
+def create_video_capture(device: str, width: int, height: int, fps: int,
+                         robovision_url: Optional[str] = None) -> Optional[VideoCapture]:
     if device.lower() in ("none", "", "false", "null"):
         return None
-    try:
+    try:
         from livekit import rtc
         _ = rtc.VideoSource
     except Exception as e:
         logger.error(f"livekit python sdk not installed: {e}")
-        return None
+        return None
+
+    if robovision_url:
+        stream_url = f"{robovision_url.rstrip('/')}/video_feed"
+        try:
+            cap = OpencvVideoCapture(stream_url, width, height, fps)
+            logger.info(f"using RoboVisionAI_PI camera stream {stream_url}")
+            return cap
+        except Exception as e:
+            logger.error(f"RoboVision camera stream unavailable: {e}")
+            return None
 
     # Auto-detect: prefer first V4L2 device, fall back to picamera2 if available.
     if device.lower() in ("auto", "default", "first"):
@@ -1538,7 +1595,9 @@ def main() -> None:
     parser.add_argument("--device-token", default=os.getenv("DEVICE_TOKEN"))
     parser.add_argument("--enrollment-token", default=os.getenv("ENROLLMENT_TOKEN"))
     parser.add_argument("--video-device", default=os.getenv("VIDEO_DEVICE", "auto"))
-    parser.add_argument("--audio-device", default=os.getenv("AUDIO_DEVICE", "default"))
+    parser.add_argument("--audio-device", default=os.getenv("AUDIO_DEVICE", "default"))
+    parser.add_argument("--robovision-url", default=None,
+                        help="use RoboVisionAI_PI's /video_feed as the LiveKit camera source")
     parser.add_argument("--width", type=int, default=int(os.getenv("VIDEO_WIDTH", DEFAULT_VIDEO_WIDTH)))
     parser.add_argument("--height", type=int, default=int(os.getenv("VIDEO_HEIGHT", DEFAULT_VIDEO_HEIGHT)))
     parser.add_argument("--fps", type=int, default=int(os.getenv("VIDEO_FPS", DEFAULT_FPS)))
@@ -1561,7 +1620,9 @@ def main() -> None:
         "scheduler_url": args.scheduler_url,
         "robot_id": args.robot_id,
         "video_device": args.video_device,
-        "audio_device": args.audio_device,
+        "audio_device": args.audio_device,
+        "robovision_url": args.robovision_url or os.getenv("ROBOVISION_URL", "http://127.0.0.1:5000"),
+        "use_robovision_camera": bool(args.robovision_url),
         "video_width": args.width,
         "video_height": args.height,
         "video_fps": args.fps,

package/packages/robopark/vision/app_pi_clean.py

@@ -1,7 +1,8 @@
 from flask import Flask, Response, jsonify, request
 from flask_cors import CORS
-import argparse
-import cv2
+import argparse
+import glob
+import cv2
 import os
 import sys
 import time
@@ -14,8 +15,10 @@ app = Flask(__name__)
 CORS(app)
 
 # Camera management
-current_camera_index = 0
-camera = None
+current_camera_index = 0
+camera = None
+audio_input_device = "default"
+audio_output_device = "default"
 
 def _open_camera(index):
     # On Windows, cv2.VideoCapture(index) with no explicit backend can
@@ -312,21 +315,24 @@ def caption_mode():
     caption_mode_enabled = bool(data.get('enabled', False))
     return jsonify({"enabled": caption_mode_enabled})
 
-@app.route('/api/cameras', methods=['GET'])
-def list_cameras():
-    available = []
-    for i in range(4):
-        cap = _open_camera(i)
-        if cap.isOpened():
-            available.append({"index": i, "name": f"Camera {i}"})
-            cap.release()
-    return jsonify({"cameras": available, "current": current_camera_index})
+@app.route('/api/cameras', methods=['GET'])
+def list_cameras():
+    available = []
+    candidates = sorted(glob.glob('/dev/video*')) if sys.platform != 'win32' else list(range(4))
+    for candidate in candidates:
+        cap = _open_camera(candidate)
+        if cap.isOpened():
+            available.append({"index": candidate, "id": str(candidate), "name": f"Camera {candidate}", "backend": "v4l2" if sys.platform != 'win32' else "dshow"})
+            cap.release()
+    return jsonify({"cameras": available, "current": current_camera_index})
 
 @app.route('/api/camera/switch', methods=['POST'])
-def switch_camera():
+def switch_camera():
     global camera, current_camera_index
     data = request.json or {}
-    new_index = int(data.get('index', 0))
+    new_index = data.get('device', data.get('index', 0))
+    if isinstance(new_index, str) and new_index.isdigit():
+        new_index = int(new_index)
 
     if camera:
         camera.release()
@@ -334,7 +340,64 @@ def switch_camera():
     current_camera_index = new_index
     camera = None
 
-    return jsonify({"status": "ok", "camera_index": current_camera_index})
+    return jsonify({"status": "ok", "camera_index": current_camera_index})
+
+
+def _audio_inventory():
+    inputs = [{"id": "default", "name": "System default input"}]
+    outputs = [{"id": "default", "name": "System default output"}]
+    try:
+        import pyaudio
+        pa = pyaudio.PyAudio()
+        for index in range(pa.get_device_count()):
+            info = pa.get_device_info_by_index(index)
+            item = {"id": str(index), "name": str(info.get("name", f"Audio device {index}")), "host_api": str(info.get("hostApi", ""))}
+            if info.get("maxInputChannels", 0) > 0:
+                inputs.append(item.copy())
+            if info.get("maxOutputChannels", 0) > 0:
+                outputs.append(item.copy())
+        pa.terminate()
+    except Exception:
+        pass
+    return inputs, outputs
+
+
+@app.route('/api/media/inventory', methods=['GET'])
+def media_inventory():
+    cameras = list_cameras().get_json().get("cameras", [])
+    if not cameras and sys.platform != 'win32':
+        # The active V4L2 handle may reject a second probe. The node path is
+        # still a valid selectable camera and is safer than hiding it.
+        cameras = [{"id": path, "index": path, "name": f"Camera {path}", "backend": "v4l2"}
+                   for path in sorted(glob.glob('/dev/video*'))]
+    inputs, outputs = _audio_inventory()
+    return jsonify({
+        "video": [{"id": "auto", "name": "Auto detect"}, {"id": "none", "name": "Disable camera"}] + cameras,
+        "audio_input": inputs,
+        "audio_output": outputs,
+        "selected": {"video_device": str(current_camera_index), "audio_device": audio_input_device, "audio_output_device": audio_output_device},
+        "source": "robovision_pi",
+    })
+
+
+@app.route('/api/media/config', methods=['GET', 'POST'])
+def media_config():
+    global audio_input_device, audio_output_device, camera, current_camera_index
+    if request.method == 'POST':
+        data = request.json or {}
+        if "video_device" in data:
+            value = str(data["video_device"])
+            if value not in ("", "auto", "none"):
+                switch_camera_value = value
+                if camera:
+                    camera.release()
+                current_camera_index = int(switch_camera_value) if switch_camera_value.isdigit() else switch_camera_value
+                camera = None
+        if "audio_device" in data:
+            audio_input_device = str(data["audio_device"])
+        if "audio_output_device" in data:
+            audio_output_device = str(data["audio_output_device"])
+    return jsonify({"video_device": str(current_camera_index), "audio_device": audio_input_device, "audio_output_device": audio_output_device, "source": "robovision_pi"})
 
 @app.route('/api/motion/status', methods=['GET'])
 def motion_status():