infinicode 2.8.94 → 2.8.96

package/dist/robopark/robot-runtime.js

@@ -83,6 +83,10 @@ export async function roboparkRobotRuntime(opts) {
     const children = [];
     let stopping = false;
     let recycling = false;
+    const requestedVideo = String(opts.videoDevice ?? '').trim();
+    const hardwareVideo = !requestedVideo || ['auto', 'default', 'first'].includes(requestedVideo.toLowerCase())
+        ? '/dev/video0'
+        : requestedVideo;
     const infinicode = await resolveInfinicodeBin();
     if (!infinicode)
         throw new Error('could not resolve infinicode CLI');
@@ -100,14 +104,15 @@ export async function roboparkRobotRuntime(opts) {
             args: [cli, 'vision-agent', '--foreground', '--port', '5000',
                 '--motion-webhook-url', 'http://127.0.0.1:5057/'],
             env: {
-                ROBOPARK_CAMERA_DEVICE: opts.videoDevice ?? '/dev/robopark-camera',
+                ROBOPARK_CAMERA_DEVICE: hardwareVideo,
             },
         });
     }
     const previewArgs = [cli, 'preview-agent', '--foreground',
         '--scheduler-url', opts.schedulerUrl, '--robot-id', opts.name,
-        '--video-device', opts.videoDevice ?? '/dev/video0',
+        '--video-device', hardwareVideo,
         '--audio-device', opts.audioDevice ?? 'Usb Audio Device: USB Audio (hw:3,0)',
+        '--robovision-url', 'http://127.0.0.1:5000',
         '--save-config'];
     if (opts.enrollmentToken)
         previewArgs.push('--enrollment-token', opts.enrollmentToken);
@@ -127,6 +132,9 @@ export async function roboparkRobotRuntime(opts) {
                 ? 'http://127.0.0.1:5000/api/media/inventory'
                 : '',
             ROBOVISION_CAMERA: opts.vision !== false ? '1' : '0',
+            // Motion remains enabled, but it reads RoboVision's shared stream. Only
+            // RoboVision may open the physical V4L2 device.
+            LOCAL_CAMERA_MOTION: 'true',
             // Fleet hardware profile: resolve volatile ALSA hw:X,Y coordinates
             // from these USB product labels on every inventory refresh.
             ROBOPARK_AUDIO_INPUT_MATCH: 'Usb Audio Device: USB Audio',

package/docs/ROBOPARK_PRODUCTION_MEMORY.md

@@ -150,3 +150,25 @@ Required result: `playback_started  ok`.
   and microphone publication, TTS subscription, and first robot playback.
 - An unattended readiness pass does not claim an STT/LLM turn occurred. That
   semantic turn still requires injected speech or a person near the robot.
+
+## Camera Ownership Contract (2.8.95+)
+
+- RoboVision is the only process permitted to open the physical V4L2 camera.
+- Dashboard, motion sampling, and conversation video consume RoboVision's
+  shared MJPEG stream; they never open `/dev/video0` independently.
+- Local camera motion remains enabled by default. Production mode controls
+  whether motion creates a conversation, not whether the camera worker runs.
+- `auto`, `default`, and `first` resolve to `/dev/robopark-camera` when present,
+  otherwise `/dev/video0`; those literal strings are never passed to OpenCV.
+
+## Audio Ownership Contract (2.8.96+)
+
+- Fleet microphone selection is resolved by stable USB product label on every
+  inventory refresh. The current stack maps microphone to `hw:3,0` and speaker
+  to `hw:2,0`.
+- A LiveKit microphone track is healthy only after the robot captures and
+  publishes real PCM. Publishing an empty track is not readiness.
+- Linux capture probes `plughw` at 48 kHz, falls back to 44.1 kHz with
+  resampling, and releases the microphone lock after every failed attempt.
+- Pipeline failures include the selected device and exact ALSA startup error.
+  A flat browser meter means missing PCM, not a quiet room.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "infinicode",
-  "version": "2.8.94",
+  "version": "2.8.96",
   "description": "OpenKernel — provider-agnostic AI execution kernel. Native coding agent + mission-driven execution runtime.",
   "type": "module",
   "main": "./dist/kernel/index.js",

package/packages/robopark/scheduler/preview_agent.py

@@ -486,7 +486,7 @@ class PreviewAgent:
         # to its hardware name before opening any playback stream.
         self.audio_playback_device = self.audio_output_device
         self.robovision_url = cfg.get("robovision_url", os.getenv("ROBOVISION_URL", "http://127.0.0.1:5000"))
-        self.use_robovision_camera = bool(cfg.get("use_robovision_camera", False))
+        self.use_robovision_camera = bool(cfg.get("use_robovision_camera", True))
         self.width = int(cfg.get("video_width", os.getenv("VIDEO_WIDTH", DEFAULT_VIDEO_WIDTH)))
         self.height = int(cfg.get("video_height", os.getenv("VIDEO_HEIGHT", DEFAULT_VIDEO_HEIGHT)))
         self.fps = int(cfg.get("video_fps", os.getenv("VIDEO_FPS", DEFAULT_FPS)))
@@ -522,7 +522,8 @@ class PreviewAgent:
         self._speaker_operation_lock = asyncio.Lock()
         self._vision_hard_deadline: float = 0.0
         self._vision_last_activity: float = 0.0
-        self.production_mode = False
+        self.production_mode = False
+        self._robovision_motion_state: Optional[bool] = None
         self._remote_session_ended = False
         self._last_device_config_poll = 0.0
         self._motion_reference = None
@@ -632,9 +633,13 @@ class PreviewAgent:
             if capture is not None:
                 await asyncio.to_thread(capture.stop)
 
-    async def _motion_loop(self) -> None:
-        """Own the camera while idle and detect motion without a second process."""
-        while not self._shutdown.is_set():
+    async def _motion_loop(self) -> None:
+        """Detect motion from RoboVision's shared stream without owning V4L2."""
+        while not self._shutdown.is_set():
+            if not self.production_mode:
+                await self._stop_motion_capture()
+                await asyncio.sleep(0.5)
+                continue
             if self._vision_session_id or self._current_state.active:
                 await self._stop_motion_capture()
                 await asyncio.sleep(0.25)
@@ -643,16 +648,17 @@ class PreviewAgent:
                 async with self._motion_capture_lock:
                     if self._motion_capture is None:
                         self._motion_capture = await asyncio.to_thread(
-                            create_video_capture,
-                            self.video_device,
-                            self.width,
-                            self.height,
-                            self.fps,
-                        )
+                            create_video_capture,
+                            self.video_device,
+                            self.width,
+                            self.height,
+                            self.fps,
+                            self.robovision_url,
+                        )
                         self._motion_reference = None
                         if self._motion_capture is None:
                             continue
-                        logger.info("motion camera opened by preview agent")
+                        logger.info("motion sampler connected to RoboVision shared stream")
                     frame = await asyncio.to_thread(self._motion_capture.read)
                 if frame is not None:
                     self._detect_motion(frame)
@@ -921,6 +927,20 @@ class PreviewAgent:
                     self._next_mesh_bootstrap = 0.0
                 if production_mode is not None:
                     self.production_mode = production_mode
+                    # Exactly one motion detector is active. With the default
+                    # local sampler enabled, preview reads RoboVision's shared
+                    # MJPEG stream and RoboVision only owns capture/encoding.
+                    robovision_motion = bool(production_mode and not self.local_camera_motion)
+                    if self._robovision_motion_state != robovision_motion:
+                        try:
+                            response = await self._session.post(
+                                f"{self.robovision_url.rstrip('/')}/api/motion/toggle",
+                                json={"enabled": robovision_motion}, timeout=1.5,
+                            )
+                            response.raise_for_status()
+                            self._robovision_motion_state = robovision_motion
+                        except Exception as exc:
+                            logger.warning(f"failed to synchronize RoboVision motion mode: {exc}")
                     await self._report_pipeline("robot_online", message="Robot heartbeat accepted")
                     inventory = _get_device_inventory()
                     real_video = [d for d in inventory.get("video", []) if str(d.get("id", "")).lower() not in ("auto", "none")]
@@ -1218,6 +1238,7 @@ class LiveKitPublisher:
         self._capture: Optional["VideoCapture"] = None
         self._mic_capture: Optional["AudioCapture"] = None
         self._mic_streaming = asyncio.Event()
+        self._mic_error: Optional[str] = None
         # Motion sampling state belongs to the publisher instance.  Keeping it
         # initialized here prevents shutdown/reopen paths from raising while
         # the camera is being handed between preview and voice sessions.
@@ -1315,10 +1336,16 @@ class LiveKitPublisher:
             try:
                 await asyncio.wait_for(self._mic_streaming.wait(), timeout=5.0)
             except asyncio.TimeoutError:
+                detail = self._mic_error or f"no PCM received from {self.agent.audio_capture_device}"
                 await self.agent._report_pipeline(
-                    "microphone_published", "blocked", "Microphone track published but PCM capture did not start"
+                    "microphone_published", "blocked",
+                    f"Microphone track published but PCM capture did not start: {detail}",
+                    {"device": self.agent.audio_capture_device, "error": detail},
+                )
+                logger.warning(
+                    "microphone PCM is not ready (%s); keeping the session alive while capture retries",
+                    detail,
                 )
-                logger.warning("microphone PCM is not ready; keeping the session alive while capture retries")
 
         if self._capture and self.video_source:
             self._tasks.append(asyncio.create_task(self._video_loop()))
@@ -1668,6 +1695,7 @@ class LiveKitPublisher:
             except asyncio.CancelledError:
                 raise
             except Exception as e:
+                self._mic_error = str(e)
                 logger.warning(
                     f"audio_loop: capture failed on {self.agent.audio_capture_device}: {e}; retrying"
                 )
@@ -1689,7 +1717,8 @@ class LiveKitPublisher:
             logger.warning("audio_loop: create_audio_capture returned None, mic will not publish")
             return
         self._mic_capture = mic
-        logger.info(f"audio_loop: mic capture started ({type(mic).__name__})")
+        self._mic_error = None
+        logger.info(f"audio_loop: mic capture started ({type(mic).__name__})")
         # Read in bigger batches (100ms) instead of one 20ms frame per
         # asyncio.to_thread() dispatch. Each dispatch/poll cycle has a fixed
         # overhead (~30ms observed on this machine) that dominates when the
@@ -1929,46 +1958,92 @@ class AlsaAudioCapture(AudioCapture):
     """Capture Linux PCM through the same ALSA path used by onsite tests."""
 
     def __init__(self, device: str):
+        import os
         import re
+        import select
         import subprocess
         from media_lock import media_lock
 
         match = re.search(r"\b(hw:\d+,\d+)\b", device)
         if not match:
             raise ValueError(f"no ALSA hardware address in {device!r}")
-        alsa_device = f"plug{match.group(1)}"
         self.buffer = bytearray()
-        self.media_guard = media_lock("microphone", timeout=5.0).acquire()
-        try:
-            self.process = subprocess.Popen(
-                [
-                    "arecord", "-q", "-D", alsa_device, "-t", "raw",
-                    "-f", "S16_LE", "-r", "48000", "-c", "1",
-                ],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-            )
-        except Exception:
-            self.media_guard.release()
-            raise
-        if self.process.stdout is None:
-            self.process.kill()
-            self.process.wait(timeout=1)
+        self.media_guard = media_lock("microphone", timeout=1.5).acquire()
+        self.process = None
+        self.source_rate = 48000
+        self.rate_state = None
+        errors = []
+        # The fleet USB microphone normally accepts 48 kHz through ALSA's
+        # plug layer. Some firmware revisions expose only native 44.1 kHz;
+        # accept that rate and resample below rather than publishing silence.
+        for source_rate in (48000, 44100):
+            alsa_device = f"plug{match.group(1)}"
+            process = None
+            try:
+                process = subprocess.Popen(
+                    [
+                        "arecord", "-q", "-D", alsa_device, "-t", "raw",
+                        "-f", "S16_LE", "-r", str(source_rate), "-c", "1",
+                        "--period-size", str(max(256, source_rate // 50)),
+                    ],
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    bufsize=0,
+                )
+                if process.stdout is None:
+                    raise RuntimeError("arecord did not provide a PCM stream")
+                ready, _, _ = select.select([process.stdout], [], [], 2.0)
+                if not ready:
+                    if process.poll() is None:
+                        raise TimeoutError("arecord produced no PCM within 2 seconds")
+                    detail = process.stderr.read().decode("utf-8", errors="replace").strip() if process.stderr else ""
+                    raise OSError(detail or f"arecord exited {process.returncode}")
+                first = os.read(process.stdout.fileno(), max(2048, source_rate // 25 * 2))
+                if not first:
+                    detail = process.stderr.read().decode("utf-8", errors="replace").strip() if process.stderr else ""
+                    raise OSError(detail or "arecord returned an empty PCM frame")
+                self.process = process
+                self.source_rate = source_rate
+                self.alsa_device = alsa_device
+                if source_rate != 48000:
+                    import audioop
+                    first, self.rate_state = audioop.ratecv(
+                        first, 2, 1, source_rate, 48000, self.rate_state
+                    )
+                self.buffer.extend(first)
+                break
+            except Exception as exc:
+                errors.append(f"{source_rate}Hz: {exc}")
+                if process is not None:
+                    if process.poll() is None:
+                        process.terminate()
+                    try:
+                        process.wait(timeout=1)
+                    except subprocess.TimeoutExpired:
+                        process.kill()
+                        process.wait(timeout=1)
+        if self.process is None:
             self.media_guard.release()
-            raise RuntimeError("arecord did not provide a PCM stream")
-        self.alsa_device = alsa_device
+            raise OSError(f"ALSA capture failed on plug{match.group(1)} ({'; '.join(errors)})")
 
     def read(self, samples_per_frame: int):
+        import audioop
+        import os
         from livekit import rtc
 
         bytes_needed = samples_per_frame * 2
+        source_bytes_needed = max(2, int(samples_per_frame * self.source_rate / 48000) * 2)
         while len(self.buffer) < bytes_needed:
-            chunk = self.process.stdout.read(bytes_needed - len(self.buffer))
+            chunk = os.read(self.process.stdout.fileno(), source_bytes_needed)
             if not chunk:
                 detail = ""
                 if self.process.stderr is not None:
                     detail = self.process.stderr.read().decode("utf-8", errors="replace").strip()
                 raise OSError(f"ALSA capture stopped on {self.alsa_device}: {detail or 'no PCM data'}")
+            if self.source_rate != 48000:
+                chunk, self.rate_state = audioop.ratecv(
+                    chunk, 2, 1, self.source_rate, 48000, self.rate_state
+                )
             self.buffer.extend(chunk)
         data = bytes(self.buffer[:bytes_needed])
         del self.buffer[:bytes_needed]
@@ -2128,7 +2203,7 @@ def main() -> None:
         "audio_device": args.audio_device,
         "robovision_url": args.robovision_url or os.getenv("ROBOVISION_URL", "http://127.0.0.1:5000"),
         "use_robovision_camera": bool(args.robovision_url) or os.getenv(
-            "ROBOVISION_CAMERA", ""
+            "ROBOVISION_CAMERA", "true"
         ).lower() in ("1", "true", "yes", "on"),
         "video_width": args.width,
         "video_height": args.height,

package/packages/robopark/vision/app_pi_clean.py

@@ -17,8 +17,19 @@ CORS(app)
 
 # Camera management. Production installs create /dev/robopark-camera from
 # the primary USB capture interface, avoiding /dev/videoN renumbering.
+def _normalize_camera_device(value):
+    configured = str(value or "").strip()
+    if configured.lower() in ("", "auto", "default", "first"):
+        if sys.platform.startswith("linux"):
+            return "/dev/robopark-camera" if os.path.exists("/dev/robopark-camera") else "/dev/video0"
+        return 0
+    if configured.isdigit():
+        return int(configured)
+    return configured
+
+
 _configured_camera = os.getenv("ROBOPARK_CAMERA_DEVICE", "")
-current_camera_index = _configured_camera or ("/dev/robopark-camera" if os.path.exists("/dev/robopark-camera") else 0)
+current_camera_index = _normalize_camera_device(_configured_camera)
 camera = None
 camera_lock = threading.Lock()
 frame_condition = threading.Condition()
@@ -41,9 +52,17 @@ def _open_camera(index):
     # webcam — isOpened() then stays False forever and callers retry in an
     # infinite doomed loop. Force DirectShow, the backend that actually
     # enumerates standard Windows webcams.
-    if sys.platform == 'win32':
-        return cv2.VideoCapture(index, cv2.CAP_DSHOW)
-    return cv2.VideoCapture(index)
+    if sys.platform == 'win32':
+        return cv2.VideoCapture(index, cv2.CAP_DSHOW)
+    if sys.platform.startswith('linux'):
+        cap = cv2.VideoCapture(index, cv2.CAP_V4L2)
+        if cap.isOpened():
+            cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*'MJPG'))
+            cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
+            cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
+            cap.set(cv2.CAP_PROP_FPS, 15)
+        return cap
+    return cv2.VideoCapture(index)
 
 
 def get_camera():
@@ -469,7 +488,7 @@ def switch_camera():
     with camera_lock:
         if camera:
             camera.release()
-        current_camera_index = new_index
+        current_camera_index = _normalize_camera_device(new_index)
         camera = None
 
     return jsonify({"status": "ok", "camera_index": current_camera_index})
@@ -531,7 +550,7 @@ def media_config():
                 switch_camera_value = value
                 if camera:
                     camera.release()
-                current_camera_index = int(switch_camera_value) if switch_camera_value.isdigit() else switch_camera_value
+                current_camera_index = _normalize_camera_device(switch_camera_value)
                 camera = None
         if "audio_device" in data:
             audio_input_device = str(data["audio_device"])