infinicode 2.8.36 → 2.8.38

package/packages/robopark/scheduler/preview_agent.py

@@ -45,10 +45,14 @@ from __future__ import annotations
 
 import argparse
 import asyncio
+import glob
 import json
 import logging
+import math
 import os
 import signal
+import socket
+import struct
 import sys
 import threading
 import time
@@ -62,6 +66,130 @@ import httpx
 
 logger = logging.getLogger("robopark.preview_agent")
 
+_DEVICE_INVENTORY_CACHE: Optional[dict] = None
+
+
+def _normalize_livekit_url(url: Optional[str]) -> Optional[str]:
+    """Avoid Windows localhost IPv6/IPv4 ambiguity for local LiveKit."""
+    if not url:
+        return url
+    for scheme in ("ws", "wss"):
+        prefix = f"{scheme}://localhost"
+        if url.startswith(prefix):
+            return f"{scheme}://127.0.0.1" + url[len(prefix):]
+    return url
+
+
+def _play_audio_effect(selected_output: str | None, effect: str) -> None:
+    """Play a short local cue without involving the voice pipeline."""
+    try:
+        import pyaudio
+    except Exception:
+        return
+    sample_rate = 48000
+    channels = 2
+    if effect == "motion":
+        notes = ((880, 0.09), (1320, 0.13))
+    else:
+        notes = ((660, 0.10), (440, 0.16))
+    pa = pyaudio.PyAudio()
+    device_index = None
+    selected = str(selected_output or "default")
+    try:
+        if selected.strip().isdigit():
+            device_index = int(selected.strip())
+        elif selected.lower() == "default":
+            wasapi = pa.get_host_api_info_by_type(pyaudio.paWASAPI)
+            device_index = wasapi.get("defaultOutputDevice")
+        else:
+            needle = selected.lower()
+            for i in range(pa.get_device_count()):
+                info = pa.get_device_info_by_index(i)
+                if info.get("maxOutputChannels", 0) > 0 and needle in str(info.get("name", "")).lower():
+                    device_index = i
+                    break
+        stream = pa.open(
+            format=pyaudio.paInt16,
+            channels=channels,
+            rate=sample_rate,
+            output=True,
+            output_device_index=device_index,
+        )
+        for frequency, duration in notes:
+            count = int(sample_rate * duration)
+            frames = bytearray()
+            for n in range(count):
+                envelope = min(1.0, n / 240.0, (count - n) / 1200.0)
+                value = int(5000 * envelope * math.sin(2 * math.pi * frequency * n / sample_rate))
+                frames.extend(struct.pack("<hh", value, value))
+            stream.write(bytes(frames))
+        stream.stop_stream()
+        stream.close()
+    except Exception as e:
+        logger.debug(f"audio effect unavailable: {e}")
+    finally:
+        pa.terminate()
+
+
+def _get_device_inventory() -> dict:
+    """Return discoverable camera and audio devices for dashboard selection."""
+    global _DEVICE_INVENTORY_CACHE
+    if _DEVICE_INVENTORY_CACHE is not None:
+        return _DEVICE_INVENTORY_CACHE
+
+    inventory = {"video": [], "audio_input": [], "audio_output": [], "platform": sys.platform}
+    inventory["video"].append({"id": "auto", "name": "Auto detect"})
+    inventory["video"].append({"id": "none", "name": "Disable camera"})
+
+    try:
+        import cv2
+        candidates = sorted(glob.glob("/dev/video*")) if os.name != "nt" else [str(i) for i in range(10)]
+        for candidate in candidates:
+            value = int(candidate) if os.name == "nt" else candidate
+            backend = cv2.CAP_DSHOW if os.name == "nt" else cv2.CAP_ANY
+            cap = cv2.VideoCapture(value, backend)
+            if cap.isOpened():
+                device_id = str(value)
+                inventory["video"].append({"id": device_id, "name": f"Camera {candidate}", "backend": "dshow" if os.name == "nt" else "v4l2"})
+            cap.release()
+    except Exception as e:
+        logger.debug(f"camera inventory unavailable: {e}")
+
+    try:
+        import pyaudio
+        pa = pyaudio.PyAudio()
+        default_in = None
+        default_out = None
+        try:
+            wasapi = pa.get_host_api_info_by_type(pyaudio.paWASAPI)
+            default_in = wasapi.get("defaultInputDevice")
+            default_out = wasapi.get("defaultOutputDevice")
+        except Exception:
+            pass
+        inventory["audio_input"].append({"id": "default", "name": "System default input"})
+        inventory["audio_output"].append({"id": "default", "name": "System default output"})
+        for i in range(pa.get_device_count()):
+            info = pa.get_device_info_by_index(i)
+            name = str(info.get("name", f"Audio device {i}"))
+            item = {"id": str(i), "name": name, "host_api": str(info.get("hostApi", ""))}
+            if info.get("maxInputChannels", 0) > 0:
+                item["default"] = i == default_in
+                inventory["audio_input"].append(item.copy())
+            if info.get("maxOutputChannels", 0) > 0:
+                item["default"] = i == default_out
+                inventory["audio_output"].append(item.copy())
+        pa.terminate()
+    except Exception as e:
+        logger.debug(f"audio inventory unavailable: {e}")
+
+    _DEVICE_INVENTORY_CACHE = inventory
+    logger.info(
+        f"device inventory: {len(inventory['video']) - 2} cameras, "
+        f"{len(inventory['audio_input']) - 1} inputs, "
+        f"{len(inventory['audio_output']) - 1} outputs"
+    )
+    return inventory
+
 CONFIG_DIR = Path.home() / ".robopark"
 CONFIG_FILE = CONFIG_DIR / "preview_agent.json"
 TOKEN_FILE = CONFIG_DIR / "device_token"
@@ -70,7 +198,7 @@ DEFAULT_VIDEO_WIDTH = 640
 DEFAULT_VIDEO_HEIGHT = 480
 DEFAULT_FPS = 15
 DEFAULT_POLL_INTERVAL = 3.0
-DEFAULT_HEARTBEAT_INTERVAL = 30.0
+DEFAULT_HEARTBEAT_INTERVAL = 5.0
 DEFAULT_VISION_WEBHOOK_PORT = 5057
 DEFAULT_VISION_TRIGGER_COOLDOWN = 20.0
 DEFAULT_VISION_SESSION_SECONDS = 90.0
@@ -78,7 +206,10 @@ DEFAULT_VISION_SESSION_SECONDS = 90.0
 # (POST /api/sessions/{id}/keepalive, called by the voice agent — a
 # different repo, not this one) before preview_agent tears it down.
 # vision_session_seconds remains a hard ceiling regardless of activity.
-DEFAULT_VISION_SILENCE_TIMEOUT = 15.0
+# Cloud STT/LLM/TTS can legitimately leave a session quiet for several
+# seconds. The voice agent has its own VAD-aware 60s idle policy; this remote
+# safety timeout must not preempt a turn while it is thinking or speaking.
+DEFAULT_VISION_SILENCE_TIMEOUT = 60.0
 
 
 def _load_config() -> dict:
@@ -148,17 +279,20 @@ def _get_lan_ip() -> Optional[str]:
         return None
 
 
-async def _send_heartbeat(scheduler_url: str, device_id: str, token: str) -> None:
+async def _send_heartbeat(scheduler_url: str, device_id: str, token: str) -> Optional[bool]:
     try:
         async with httpx.AsyncClient() as client:
-            await client.post(
+            response = await client.post(
                 f"{scheduler_url.rstrip('/')}/api/devices/{device_id}/heartbeat",
-                json={"status": "online", "ip": _get_lan_ip()},
+                json={"status": "online", "ip": _get_lan_ip(), "device_inventory": _get_device_inventory()},
                 headers={"Authorization": f"Bearer {token}"},
                 timeout=10.0,
             )
+            response.raise_for_status()
+            return bool(response.json().get("production_mode", False))
     except Exception as e:
         logger.debug(f"heartbeat failed: {e}")
+    return None
 
 
 @dataclass
@@ -167,6 +301,8 @@ class PreviewState:
     url: Optional[str] = None
     token: Optional[str] = None
     room: Optional[str] = None
+    mode: str = "preview"
+    session_id: Optional[str] = None
 
 
 class PreviewAgent:
@@ -178,6 +314,7 @@ class PreviewAgent:
         self.enrollment_token: Optional[str] = cfg.get("enrollment_token")
         self.video_device = cfg.get("video_device", os.getenv("VIDEO_DEVICE", "auto"))
         self.audio_device = cfg.get("audio_device", os.getenv("AUDIO_DEVICE", "default"))
+        self.audio_output_device = cfg.get("audio_output_device", os.getenv("AUDIO_OUTPUT_DEVICE", "default"))
         self.width = int(cfg.get("video_width", os.getenv("VIDEO_WIDTH", DEFAULT_VIDEO_WIDTH)))
         self.height = int(cfg.get("video_height", os.getenv("VIDEO_HEIGHT", DEFAULT_VIDEO_HEIGHT)))
         self.fps = int(cfg.get("video_fps", os.getenv("VIDEO_FPS", DEFAULT_FPS)))
@@ -187,6 +324,12 @@ class PreviewAgent:
         self.vision_trigger_cooldown = float(cfg.get("vision_trigger_cooldown", os.getenv("VISION_TRIGGER_COOLDOWN", DEFAULT_VISION_TRIGGER_COOLDOWN)))
         self.vision_session_seconds = float(cfg.get("vision_session_seconds", os.getenv("VISION_SESSION_SECONDS", DEFAULT_VISION_SESSION_SECONDS)))
         self.vision_silence_timeout = float(cfg.get("vision_silence_timeout", os.getenv("VISION_SILENCE_TIMEOUT", DEFAULT_VISION_SILENCE_TIMEOUT)))
+        self.local_camera_motion = str(
+            # Production needs an always-on detector. The preview publisher
+            # remains the single camera owner when a session starts, so this
+            # does not require the separate OpenCV vision server.
+            cfg.get("local_camera_motion", os.getenv("LOCAL_CAMERA_MOTION", "true"))
+        ).lower() in ("1", "true", "yes", "on")
 
         self._shutdown = asyncio.Event()
         self._task: Optional[asyncio.Task] = None
@@ -200,8 +343,16 @@ class PreviewAgent:
         # with a hard ceiling — see DEFAULT_VISION_SILENCE_TIMEOUT above and
         # _check_vision_session() below for the full design.
         self._vision_session_id: Optional[str] = None
+        self._vision_trigger_in_flight = False
         self._vision_hard_deadline: float = 0.0
         self._vision_last_activity: float = 0.0
+        self.production_mode = False
+        self._remote_session_ended = False
+        self._last_device_config_poll = 0.0
+        self._motion_reference = None
+        self._last_motion_sample = 0.0
+        self._motion_capture: Optional[VideoCapture] = None
+        self._motion_capture_lock = asyncio.Lock()
 
     async def run(self) -> None:
         enrollment_token = os.getenv("ENROLLMENT_TOKEN") or self.enrollment_token
@@ -231,6 +382,11 @@ class PreviewAgent:
             asyncio.create_task(self._poll_loop()),
             asyncio.create_task(self._heartbeat_loop()),
         ]
+        # In production the robot's motion detector sends the webhook. Do not
+        # open the same Windows camera locally unless explicitly requested;
+        # doing both creates a DirectShow ownership conflict with LiveKit.
+        if self.local_camera_motion:
+            tasks.append(asyncio.create_task(self._motion_loop()))
         await self._shutdown.wait()
         for t in tasks:
             t.cancel()
@@ -241,8 +397,70 @@ class PreviewAgent:
         if self._vision_server:
             self._vision_server.shutdown()
         await self._stop_publisher()
+        await self._stop_motion_capture()
         await self._session.aclose()
 
+    async def _stop_motion_capture(self) -> None:
+        async with self._motion_capture_lock:
+            capture = self._motion_capture
+            self._motion_capture = None
+            if capture is not None:
+                await asyncio.to_thread(capture.stop)
+
+    async def _motion_loop(self) -> None:
+        """Own the camera while idle and detect motion without a second process."""
+        while not self._shutdown.is_set():
+            if self._vision_session_id or self._current_state.active:
+                await self._stop_motion_capture()
+                await asyncio.sleep(0.25)
+                continue
+            try:
+                async with self._motion_capture_lock:
+                    if self._motion_capture is None:
+                        self._motion_capture = await asyncio.to_thread(
+                            create_video_capture,
+                            self.video_device,
+                            self.width,
+                            self.height,
+                            self.fps,
+                        )
+                        self._motion_reference = None
+                        if self._motion_capture is None:
+                            continue
+                        logger.info("motion camera opened by preview agent")
+                    frame = await asyncio.to_thread(self._motion_capture.read)
+                if frame is not None:
+                    self._detect_motion(frame)
+            except Exception as e:
+                logger.warning(f"motion camera unavailable: {e}")
+                await self._stop_motion_capture()
+                await asyncio.sleep(2.0)
+            await asyncio.sleep(0.1)
+
+    def _detect_motion(self, frame) -> None:
+        """Compare sparse RGB samples and trigger only meaningful scene changes."""
+        now = time.monotonic()
+        if now - self._last_motion_sample < 0.25:
+            return
+        self._last_motion_sample = now
+        try:
+            import numpy as np
+
+            data = np.frombuffer(bytes(frame.data), dtype=np.uint8)
+            sample = data.reshape(frame.height, frame.width, 3)[::12, ::12].mean(axis=2)
+            previous = self._motion_reference
+            self._motion_reference = sample
+            if previous is None or previous.shape != sample.shape:
+                return
+            change = float(np.abs(sample - previous).mean())
+            threshold = float(os.getenv("VISION_MOTION_THRESHOLD", "12"))
+            if change >= threshold:
+                asyncio.create_task(
+                    self._on_vision_motion({"source": "preview_camera", "change": change})
+                )
+        except Exception as e:
+            logger.debug(f"preview motion sampling failed: {e}")
+
     async def _resolve_device_id(self) -> None:
         """Look up our device_id from the scheduler using the token."""
         if not self._session or not self.device_token:
@@ -274,6 +492,9 @@ class PreviewAgent:
                     # operator-preview state tear it down.
                     await self._check_vision_session()
                 if not self._vision_session_id:
+                    await self._poll_trigger_command()
+                if not self._vision_session_id:
+                    await self._poll_device_config()
                     state = await self._fetch_preview_state()
                     await self._apply_state(state)
             except Exception as e:
@@ -283,6 +504,46 @@ class PreviewAgent:
             except asyncio.TimeoutError:
                 pass
 
+    async def _poll_trigger_command(self) -> None:
+        """Consume dashboard test triggers through the authenticated device path."""
+        if not self._session or not self.device_id or not self.device_token:
+            return
+        r = await self._session.get(
+            f"{self.scheduler_url.rstrip('/')}/api/devices/{self.device_id}/trigger-command",
+            headers={"Authorization": f"Bearer {self.device_token}"},
+            timeout=10.0,
+        )
+        r.raise_for_status()
+        data = r.json()
+        self.production_mode = bool(data.get("production_mode", self.production_mode))
+        if data.get("trigger"):
+            await self._on_vision_motion({"source": data.get("source", "dashboard")})
+
+    async def _poll_device_config(self) -> None:
+        """Apply dashboard-selected camera and microphone IDs before preview."""
+        if not self._session or not self.device_id or not self.device_token:
+            return
+        now = time.monotonic()
+        if now - self._last_device_config_poll < 5.0:
+            return
+        self._last_device_config_poll = now
+        try:
+            r = await self._session.get(
+                f"{self.scheduler_url.rstrip('/')}/api/devices/{self.device_id}/config",
+                headers={"Authorization": f"Bearer {self.device_token}"},
+                timeout=10.0,
+            )
+            r.raise_for_status()
+            data = r.json()
+            if data.get("video_device") is not None:
+                self.video_device = data["video_device"]
+            if data.get("audio_device") is not None:
+                self.audio_device = data["audio_device"]
+            if data.get("audio_output_device") is not None:
+                self.audio_output_device = data["audio_output_device"]
+        except Exception as e:
+            logger.debug(f"device config poll failed: {e}")
+
     async def _check_vision_session(self) -> None:
         """Decide whether the active motion-triggered session should keep
         holding the publisher.
@@ -314,6 +575,10 @@ class PreviewAgent:
 
         try:
             last_activity = await self._fetch_session_last_activity(self._vision_session_id)
+            if self._remote_session_ended:
+                logger.info("scheduler ended the vision session — stopping publisher")
+                await self._end_vision_session()
+                return
             if last_activity is not None and last_activity > self._vision_last_activity:
                 self._vision_last_activity = last_activity
         except Exception as e:
@@ -341,6 +606,9 @@ class PreviewAgent:
         )
         r.raise_for_status()
         data = r.json()
+        if data.get("ended_at"):
+            self._remote_session_ended = True
+            return None
         ts = data.get("last_activity_at") or data.get("started_at")
         if not ts:
             return None
@@ -357,7 +625,8 @@ class PreviewAgent:
         # picked up (unrelated to this specific dispatch, but a real bug:
         # every silence-timeout before this fix leaked a permanently
         # "active" session).
-        if self._session and self.device_token:
+        remote_session_ended = self._remote_session_ended
+        if not remote_session_ended and self._session and self.device_token:
             try:
                 await self._session.post(
                     f"{self.scheduler_url.rstrip('/')}/api/robots/{self.device_id or self.robot_id}/end-session",
@@ -368,15 +637,24 @@ class PreviewAgent:
             except Exception as e:
                 logger.warning(f"failed to notify scheduler of session end: {e}")
         self._vision_session_id = None
+        self._vision_trigger_in_flight = False
         self._vision_hard_deadline = 0.0
         self._vision_last_activity = 0.0
+        self._remote_session_ended = False
         await self._stop_publisher()
+        await asyncio.to_thread(_play_audio_effect, self.audio_output_device, "disconnect")
         self._current_state = PreviewState()
 
     async def _heartbeat_loop(self) -> None:
         while not self._shutdown.is_set():
             if self.device_id and self.device_token:
-                await _send_heartbeat(self.scheduler_url, self.device_id, self.device_token)
+                production_mode = await _send_heartbeat(
+                    self.scheduler_url, self.device_id, self.device_token
+                )
+                if production_mode is not None:
+                    self.production_mode = production_mode
+                    if not production_mode and self._vision_session_id:
+                        await self._end_vision_session()
             try:
                 await asyncio.wait_for(self._shutdown.wait(), timeout=self.heartbeat_interval)
             except asyncio.TimeoutError:
@@ -396,17 +674,20 @@ class PreviewAgent:
             return PreviewState()
         return PreviewState(
             active=True,
-            url=data.get("url"),
+            url=_normalize_livekit_url(data.get("url")),
             token=data.get("token"),
             room=data.get("room"),
+            mode=data.get("mode", "preview"),
+            session_id=data.get("session_id"),
         )
 
     async def _apply_state(self, state: PreviewState) -> None:
         same = (
             state.active == self._current_state.active
             and state.room == self._current_state.room
-            and state.token == self._current_state.token
             and state.url == self._current_state.url
+            and state.mode == self._current_state.mode
+            and state.session_id == self._current_state.session_id
         )
         if same:
             return
@@ -420,6 +701,7 @@ class PreviewAgent:
         await self._stop_publisher()
         if not state.url or not state.token or not state.room:
             return
+        pub = None
         try:
             pub = LiveKitPublisher(state.url, state.token, state.room, self)
             await pub.start()
@@ -427,6 +709,11 @@ class PreviewAgent:
             logger.info(f"joined preview room {state.room}")
         except Exception as e:
             logger.error(f"failed to start publisher: {e}")
+            if pub is not None:
+                try:
+                    await pub.stop()
+                except Exception as cleanup_error:
+                    logger.debug(f"publisher cleanup after start failure: {cleanup_error}")
 
     async def _stop_publisher(self) -> None:
         if self._publisher:
@@ -482,15 +769,40 @@ class PreviewAgent:
             self._vision_server = None
 
     async def _on_vision_motion(self, payload: dict) -> None:
+        # vision_trigger_cooldown is a MINIMUM SPACING between trigger
+        # attempts, not "an active session is fine to interrupt" -- without
+        # this separate check, continuous ambient motion (someone standing
+        # in frame) re-fires every cooldown window regardless of whether a
+        # conversation is already in progress, tearing down the room and
+        # restarting it before the greeting/response ever finishes playing.
+        # Only allow a new trigger once the previous vision session has
+        # actually ended (silence timeout, hard ceiling, or remote end).
+        if self._vision_session_id or self._vision_trigger_in_flight:
+            logger.debug("vision trigger suppressed (a vision session is already active)")
+            return
         now = time.time()
         if now - self._last_vision_trigger < self.vision_trigger_cooldown:
             logger.debug("vision trigger suppressed (cooldown)")
             return
         self._last_vision_trigger = now
+        if not self.production_mode:
+            logger.info("motion event ignored because production mode is OFF")
+            return
         if not self.device_id or not self.device_token or not self._session:
             logger.warning("vision motion event received but not enrolled yet — ignoring")
             return
+        self._vision_trigger_in_flight = True
         logger.info("motion detected by RoboVisionAI_PI — requesting a session")
+        # The cue is feedback only; never block session allocation on speaker
+        await self._stop_motion_capture()
+        # DirectShow can return a stale handle for a short interval after
+        # release. Give the driver time to finish teardown before reopening it
+        # for the LiveKit publisher.
+        await asyncio.sleep(0.5)
+        # I/O. Camera/mic join and the preset greeting are the critical path.
+        asyncio.create_task(
+            asyncio.to_thread(_play_audio_effect, self.audio_output_device, "motion")
+        )
         try:
             r = await self._session.post(
                 f"{self.scheduler_url.rstrip('/')}/api/devices/{self.device_id}/request-session",
@@ -501,10 +813,11 @@ class PreviewAgent:
             data = r.json()
         except Exception as e:
             logger.error(f"request-session failed: {e}")
+            self._vision_trigger_in_flight = False
             return
         state = PreviewState(
             active=True,
-            url=data.get("server_url"),
+            url=_normalize_livekit_url(data.get("server_url")),
             token=data.get("token"),
             room=data.get("room_name"),
         )
@@ -512,8 +825,10 @@ class PreviewAgent:
         voice_config = data.get("voice_config")
         now = time.time()
         self._vision_session_id = session_id
+        self._vision_trigger_in_flight = False
         self._vision_hard_deadline = now + self.vision_session_seconds
         self._vision_last_activity = now
+        self._remote_session_ended = False
         self._current_state = state
         # If the scheduler returned a voice config, let the agent know by
         # posting it to our local webhook endpoint. The preview agent itself
@@ -554,32 +869,18 @@ class LiveKitPublisher:
         self._stop_event = asyncio.Event()
         self._tasks: list[asyncio.Task] = []
         self._capture: Optional["VideoCapture"] = None
+        # Motion sampling state belongs to the publisher instance.  Keeping it
+        # initialized here prevents shutdown/reopen paths from raising while
+        # the camera is being handed between preview and voice sessions.
+        self._last_motion_sample = 0.0
 
     async def start(self) -> None:
         self.room = self.rtc.Room()
         await self.room.connect(self.url, self.token)
 
-        # Video
-        self.video_source = self.rtc.VideoSource(self.agent.width, self.agent.height)
-        self.video_track = self.rtc.LocalVideoTrack.create_video_track("camera", self.video_source)
-        vopts = self.rtc.TrackPublishOptions()
-        vopts.source = self.rtc.TrackSource.SOURCE_CAMERA
-        await self.room.local_participant.publish_track(self.video_track, vopts)
-
-        # Audio
-        self.audio_source = self.rtc.AudioSource(48000, 1)
-        self.audio_track = self.rtc.LocalAudioTrack.create_audio_track("microphone", self.audio_source)
-        aopts = self.rtc.TrackPublishOptions()
-        aopts.source = self.rtc.TrackSource.SOURCE_MICROPHONE
-        await self.room.local_participant.publish_track(self.audio_track, aopts)
-
-        # Register speaker playback (subscribe to the voice agent's TTS audio
-        # track) BEFORE opening the camera. Camera open is a slow/occasionally
-        # hanging blocking call (see create_video_capture below), and since
-        # asyncio is single-threaded, running it inline here would stall the
-        # entire event loop — including receiving the agent's greeting audio
-        # — until it finished, so a short "Hello friend!" greeting could be
-        # over and gone before we ever got a chance to subscribe to it.
+        # Register after signaling. Registering during Room.connect can invoke
+        # callbacks while the native LiveKit participant state is incomplete;
+        # on Windows that has caused an unrecoverable native client abort.
         self._playback_streams: dict = {}
 
         def _on_track_subscribed(track, publication, participant):
@@ -587,22 +888,79 @@ class LiveKitPublisher:
             if track.kind != self.rtc.TrackKind.KIND_AUDIO:
                 return
             if participant.identity == self.room.local_participant.identity:
-                return  # don't play our own mic back
+                return
             self._tasks.append(asyncio.create_task(self._play_remote_audio(track, publication.sid)))
 
         self.room.on("track_subscribed", _on_track_subscribed)
         logger.info("audio out: track_subscribed listener registered")
 
-        if has_audio():
+        # Open and warm the camera before creating the native LiveKit source.
+        # Windows webcams may ignore the requested 640x480 mode and return a
+        # different size (for example 480x360); capturing that frame into a
+        # mismatched VideoSource can abort the native SDK.
+        video_enabled = str(self.agent.video_device).lower() not in ("none", "", "false", "null")
+        first_frame = None
+        if video_enabled:
+            try:
+                self._capture = await asyncio.to_thread(
+                    create_video_capture, self.agent.video_device, self.agent.width, self.agent.height, self.agent.fps
+                )
+            except Exception as e:
+                # Camera availability must not block the microphone/session.
+                # Windows camera drivers can fail during a reopen while the
+                # audio path remains healthy and should still accept speech.
+                logger.warning(f"camera unavailable for this session; continuing audio-only: {e}")
+                self._capture = None
+            if self._capture:
+                for _ in range(12):
+                    try:
+                        first_frame = await asyncio.to_thread(self._capture.read)
+                    except Exception as e:
+                        logger.warning(f"camera read failed during warm-up; continuing audio-only: {e}")
+                        first_frame = None
+                        break
+                    if first_frame is not None:
+                        logger.info(
+                            "camera warm-up produced a valid frame (%sx%s)",
+                            first_frame.width,
+                            first_frame.height,
+                        )
+                        break
+                    await asyncio.sleep(0.08)
+                if first_frame is None:
+                    self._capture.stop()
+                    self._capture = None
+
+        # Do not publish dummy tracks for explicitly disabled devices. Apart
+        # from misleading the worker, creating native LiveKit sources for a
+        # disabled Windows device has caused unstable track publication.
+        audio_enabled = str(self.agent.audio_device).lower() not in ("none", "", "false", "null")
+        if first_frame is not None:
+            self.video_source = self.rtc.VideoSource(first_frame.width, first_frame.height)
+            self.video_track = self.rtc.LocalVideoTrack.create_video_track("camera", self.video_source)
+            vopts = self.rtc.TrackPublishOptions()
+            vopts.source = self.rtc.TrackSource.SOURCE_CAMERA
+            await self.room.local_participant.publish_track(self.video_track, vopts)
+            self.video_source.capture_frame(first_frame)
+
+        if audio_enabled:
+            self.audio_source = self.rtc.AudioSource(48000, 1)
+            self.audio_track = self.rtc.LocalAudioTrack.create_audio_track("microphone", self.audio_source)
+            aopts = self.rtc.TrackPublishOptions()
+            aopts.source = self.rtc.TrackSource.SOURCE_MICROPHONE
+            await self.room.local_participant.publish_track(self.audio_track, aopts)
+
+        # Register speaker playback (subscribe to the voice agent's TTS audio
+        # track) BEFORE opening the camera. Camera open is a slow/occasionally
+        # hanging blocking call (see create_video_capture below), and since
+        # asyncio is single-threaded, running it inline here would stall the
+        # entire event loop — including receiving the agent's greeting audio
+        # — until it finished, so a short "Hello friend!" greeting could be
+        # over and gone before we ever got a chance to subscribe to it.
+        if has_audio() and audio_enabled:
             self._tasks.append(asyncio.create_task(self._audio_loop()))
 
-        # Camera open can block for a long time (flaky USB/driver reopen) —
-        # run it in a thread so it can't stall the event loop (and therefore
-        # incoming agent audio / heartbeats / the playback subscribed above).
-        self._capture = await asyncio.to_thread(
-            create_video_capture, self.agent.video_device, self.agent.width, self.agent.height, self.agent.fps
-        )
-        if self._capture:
+        if self._capture and self.video_source:
             self._tasks.append(asyncio.create_task(self._video_loop()))
 
     async def _play_remote_audio(self, track, sid: str) -> None:
@@ -612,29 +970,174 @@ class LiveKitPublisher:
             logger.warning(f"pyaudio unavailable for playback: {e}")
             return
         OUT_RATE = 48000
+        OUT_CHANNELS = 2
         pa = pyaudio.PyAudio()
-        out = pa.open(format=pyaudio.paInt16, channels=1, rate=OUT_RATE, output=True)
-        logger.info(f"audio out: opened playback stream for {sid}")
+        # Same MME-vs-WASAPI gotcha as mic capture (see PyAudioCapture._resolve_device):
+        # PyAudio's global default output device resolves through MME, which on this
+        # machine will accept writes and report success with zero exceptions while
+        # never actually reaching the real speakers. Prefer WASAPI's default output,
+        # which is what Windows' own volume mixer/meter is backed by.
+        output_device_index = None
+        selected_output = str(self.agent.audio_output_device or "default")
+        if selected_output.strip().isdigit():
+            output_device_index = int(selected_output.strip())
+        try:
+            if selected_output.lower() == "default":
+                wasapi = pa.get_host_api_info_by_type(pyaudio.paWASAPI)
+                idx = wasapi.get("defaultOutputDevice")
+                if idx is not None and idx >= 0:
+                    output_device_index = idx
+            elif output_device_index is None:
+                needle = selected_output.lower()
+                for i in range(pa.get_device_count()):
+                    info = pa.get_device_info_by_index(i)
+                    if info.get("maxOutputChannels", 0) > 0 and needle in str(info.get("name", "")).lower():
+                        output_device_index = i
+                        break
+        except Exception as e:
+            logger.debug(f"audio out: WASAPI default output lookup failed, using PyAudio default: {e}")
+        out = pa.open(
+            format=pyaudio.paInt16, channels=OUT_CHANNELS, rate=OUT_RATE, output=True,
+            output_device_index=output_device_index,
+            frames_per_buffer=960,
+        )
+        logger.info(f"audio out: opened playback stream for {sid} (device_index={output_device_index})")
         frame_count = 0
+        mismatch_logged = False
+        try:
+            import numpy as np
+        except Exception as e:
+            np = None
+            logger.warning(f"numpy unavailable for playback resampling: {e}")
+
+        # Frames arrive over the network in irregular bursts (TTS streaming,
+        # scheduling jitter); writing each one straight to a blocking PyAudio
+        # stream inline ties the audio device's write timing to that jitter,
+        # which is what produced "heavily distorted/staticky" playback even
+        # with matching rate/channels. Decouple the two with a small jitter
+        # buffer: a writer thread drains a queue into PyAudio on its own
+        # steady pace, independent of how unevenly frames actually arrive.
+        import queue
+        import threading
+        write_queue: "queue.Queue[Optional[bytes]]" = queue.Queue()
+        written_frames = [0]
+        PREBUFFER_CHUNKS = 5
+        stream_failed = threading.Event()
+
+        def _safe_write(chunk: bytes) -> bool:
+            if stream_failed.is_set():
+                return False
+            try:
+                out.write(chunk)
+                return True
+            except Exception as e:
+                stream_failed.set()
+                logger.warning(f"audio out stream closed; disabling playback for this track: {e}")
+                return False
+
+        def _writer():
+            buffered = []
+            started = False
+            while True:
+                try:
+                    chunk = write_queue.get()
+                except queue.Empty:
+                    continue
+                if chunk is None:
+                    # Drain any short tail when the remote track ends.
+                    for pending in buffered:
+                        if not _safe_write(pending):
+                            break
+                    break
+                if not started:
+                    buffered.append(chunk)
+                    if len(buffered) < PREBUFFER_CHUNKS:
+                        continue
+                    for pending in buffered:
+                        if not _safe_write(pending):
+                            break
+                    buffered.clear()
+                    if stream_failed.is_set():
+                        break
+                    started = True
+                    continue
+                try:
+                    if _safe_write(chunk):
+                        written_frames[0] += 1
+                        if written_frames[0] == 1:
+                            logger.info(f"audio out: writer thread wrote its first chunk for {sid}")
+                    elif stream_failed.is_set():
+                        break
+                except Exception as e:
+                    # A single write failing (e.g. a transient device hiccup) must
+                    # not silently kill the whole thread — that leaves every
+                    # later frame queued with nothing left to consume them,
+                    # which looks exactly like "no audio at all" even though
+                    # frames kept arriving fine. Log loudly and keep going.
+                    logger.warning(f"audio out write error (continuing): {e}")
+
+        writer_thread = threading.Thread(target=_writer, daemon=True)
+        writer_thread.start()
+
         try:
             stream = self.rtc.AudioStream(track=track)
             async for frame in stream:
                 af = frame.frame if hasattr(frame, "frame") else frame
-                data = bytes(af.data)
                 frame_count += 1
                 if frame_count == 1:
-                    logger.info(f"audio out: first frame received for {sid} (rate={af.sample_rate})")
-                if af.sample_rate != OUT_RATE:
-                    # Frames from Kokoro/TTS are already 48kHz in this stack;
-                    # a mismatch here would need resampling like the Pi
-                    # client does, but isn't expected on the local test loop.
-                    logger.debug(
-                        f"audio out: unexpected sample rate {af.sample_rate}, expected {OUT_RATE}"
+                    logger.info(
+                        f"audio out: first frame received for {sid} "
+                        f"(rate={af.sample_rate}, channels={getattr(af, 'num_channels', 1)})"
                     )
-                out.write(data)
+                in_rate = af.sample_rate
+                in_channels = int(getattr(af, "num_channels", 1) or 1)
+                raw = bytes(af.data)
+                if np is not None:
+                    in_data = np.frombuffer(raw, dtype=np.int16)
+                    if in_channels > 1:
+                        in_data = in_data.reshape(-1, in_channels).mean(axis=1).astype(np.int16)
+                else:
+                    in_data = raw
+
+                if in_rate == OUT_RATE or np is None:
+                    if in_rate != OUT_RATE and not mismatch_logged:
+                        mismatch_logged = True
+                        logger.warning(
+                            f"audio out: sample rate {in_rate} != {OUT_RATE} and numpy "
+                            f"unavailable — playing unresampled (expect distortion)"
+                        )
+                    mono_data = in_data
+                else:
+                    # Mirror the real Pi client's resampling (pi-client/livekit_bridge.py):
+                    # writing raw bytes straight to a fixed-rate output stream when the
+                    # source rate differs (e.g. Kokoro's native rate vs our 48kHz stream)
+                    # is what produced the "heavily distorted/staticky" playback — every
+                    # frame needs resampling to OUT_RATE first, not just the ones that
+                    # happen to already match.
+                    if not mismatch_logged:
+                        mismatch_logged = True
+                        logger.info(f"audio out: resampling {in_rate}Hz -> {OUT_RATE}Hz for {sid}")
+                    ratio = OUT_RATE / in_rate
+                    n_out = int(len(in_data) * ratio)
+                    indices = np.linspace(0, len(in_data) - 1, n_out)
+                    out_data = np.interp(indices, np.arange(len(in_data)), in_data.astype(np.float64)).astype(np.int16)
+                    mono_data = out_data
+
+                if np is not None:
+                    # The Windows endpoint is stereo; duplicate mono LiveKit audio explicitly.
+                    data = np.repeat(mono_data[:, None], OUT_CHANNELS, axis=1).astype(np.int16).tobytes()
+                else:
+                    data = b"".join(raw[i:i + 2] * OUT_CHANNELS for i in range(0, len(raw), 2))
+                write_queue.put(data)
         except Exception as e:
             logger.warning(f"audio out stream error: {e}")
         finally:
+            write_queue.put(None)
+            writer_thread.join(timeout=2.0)
+            logger.info(
+                f"audio out: {sid} received {frame_count} frames, "
+                f"writer wrote {written_frames[0]} chunks, queue backlog at close={write_queue.qsize()}"
+            )
             out.stop_stream()
             out.close()
             pa.terminate()
@@ -669,6 +1172,7 @@ class LiveKitPublisher:
             frame = await asyncio.to_thread(self._capture.read)
             if frame is not None:
                 self.video_source.capture_frame(frame)
+                self._detect_motion(frame)
             elapsed = time.monotonic() - start
             sleep_for = frame_interval - elapsed
             if sleep_for > 0:
@@ -677,6 +1181,34 @@ class LiveKitPublisher:
                 except asyncio.TimeoutError:
                     pass
 
+    def _detect_motion(self, frame: "rtc.VideoFrame") -> None:
+        """Detect motion from the already-published camera frame.
+
+        The preview publisher is the sole camera owner. Keeping motion
+        detection here avoids opening the Windows camera a second time from a
+        separate detector process, which caused black frames and crashes.
+        """
+        now = time.monotonic()
+        if now - self._last_motion_sample < 0.25:
+            return
+        self._last_motion_sample = now
+        try:
+            import numpy as np
+
+            data = np.frombuffer(bytes(frame.data), dtype=np.uint8)
+            sample = data.reshape(frame.height, frame.width, 3)[::12, ::12].mean(axis=2)
+            previous = self._motion_reference
+            self._motion_reference = sample
+            if previous is None or previous.shape != sample.shape:
+                return
+            change = float(np.abs(sample - previous).mean())
+            if change >= float(os.getenv("VISION_MOTION_THRESHOLD", "12")):
+                asyncio.create_task(
+                    self._on_vision_motion({"source": "preview_camera", "change": change})
+                )
+        except Exception as e:
+            logger.debug(f"preview motion sampling failed: {e}")
+
     async def _audio_loop(self) -> None:
         assert self.audio_source is not None
         mic = create_audio_capture(self.agent.audio_device)
@@ -764,10 +1296,22 @@ class OpencvVideoCapture(VideoCapture):
         self.cv2 = cv2
         if isinstance(device, str) and device.startswith("/dev/video"):
             device = int(device.replace("/dev/video", ""))
-        self.cap = cv2.VideoCapture(device)
+        elif isinstance(device, str) and device.isdigit():
+            device = int(device)
+        # MSMF frequently returns intermittent grab failures for USB cameras
+        # on Windows. DirectShow is more stable for this long-lived stream;
+        # retain the default backend as a fallback for unusual devices.
+        if os.name == "nt" and isinstance(device, int):
+            self.cap = cv2.VideoCapture(device, cv2.CAP_DSHOW)
+            if not self.cap.isOpened():
+                self.cap.release()
+                self.cap = cv2.VideoCapture(device)
+        else:
+            self.cap = cv2.VideoCapture(device)
         self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, width)
         self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, height)
         self.cap.set(cv2.CAP_PROP_FPS, fps)
+        self._last_black_log = 0.0
         self._ensure_frame()
 
     def _ensure_frame(self):
@@ -780,11 +1324,28 @@ class OpencvVideoCapture(VideoCapture):
         ok, bgr = self.cap.read()
         if not ok or bgr is None:
             return None
+        # OpenCV can report a successful read while a Windows camera driver
+        # returns an effectively black frame (privacy shutter, wrong device,
+        # or a stale DirectShow handle). Do not publish misleading video.
+        mean = float(bgr.mean())
+        if mean <= 1.0 and float(bgr.max()) <= 8.0:
+            now = time.monotonic()
+            if now - self._last_black_log >= 10.0:
+                logger.error(
+                    "camera returned black frames from device %s; check camera selection, "
+                    "privacy shutter, and Windows camera permissions",
+                    self.cap,
+                )
+                self._last_black_log = now
+            return None
+        # The current LiveKit RTC SDK exposes RGB/RGBA frame types, not BGR.
+        # OpenCV captures BGR, so convert before constructing the frame.
+        rgb = self.cv2.cvtColor(bgr, self.cv2.COLOR_BGR2RGB)
         return rtc.VideoFrame(
-            width=bgr.shape[1],
-            height=bgr.shape[0],
-            type=rtc.VideoBufferType.BGR,
-            data=bgr.tobytes(),
+            width=rgb.shape[1],
+            height=rgb.shape[0],
+            type=rtc.VideoBufferType.RGB24,
+            data=rgb.tobytes(),
         )
 
     def stop(self):
@@ -812,7 +1373,7 @@ class Picamera2Capture(VideoCapture):
         return rtc.VideoFrame(
             width=self.width,
             height=self.height,
-            type=rtc.VideoBufferType.RGB,
+            type=rtc.VideoBufferType.RGB24,
             data=arr.tobytes(),
         )
 
@@ -890,6 +1451,8 @@ class PyAudioCapture(AudioCapture):
     def _resolve_device(self, device: str | int | None) -> Optional[int]:
         if isinstance(device, int):
             return device
+        if isinstance(device, str) and device.strip().isdigit():
+            return int(device.strip())
         if device is None or device == "default":
             # PyAudio's global default resolves through the MME host API,
             # which on Windows can silently capture pure silence (no error,