infinicode 2.8.87 → 2.8.89

package/docs/ROBOPARK_PRODUCTION_MEMORY.md

@@ -60,6 +60,10 @@ ROBOVOICE checkout, and rebuilds Compose once.
 The preview agent previously used PyAudio for the selected speaker. BMW's
 proven output is ALSA `aplay -D plughw:2,0`. Infinicode `2.8.86` uses that
 same endpoint for live TTS whenever a Linux output selection contains `hw:`.
+Motion cues must use that same ALSA endpoint and finish before the greeting
+starts. Running a PyAudio cue concurrently with ALSA TTS creates an exclusive
+device race on `hw:2,0`. `playback_started` is valid only after PCM has been
+written to the physical speaker, not merely received from LiveKit.
 
 ### Silent Conversation After Greeting
 
@@ -70,6 +74,14 @@ for actual PCM frames before reporting `microphone_published`, reports a blocked
 stage when capture does not start within five seconds, and applies bounded mic
 gain before publication.
 
+### Camera Relay With Multiple Viewers
+
+RoboVision must have exactly one OpenCV camera reader. Preview, dashboard, and
+motion detection consume a shared latest-frame stream. Starting one
+`generate_frames()` camera loop per MJPEG client causes concurrent
+`VideoCapture.read()` calls, which can block the second client and make the hub
+relay time out with zero bytes.
+
 ### Duplicate Speaker Tests
 
 Only one robot process may claim a scheduler `speaker_test`. The scheduler

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "infinicode",
-  "version": "2.8.87",
+  "version": "2.8.89",
   "description": "OpenKernel — provider-agnostic AI execution kernel. Native coding agent + mission-driven execution runtime.",
   "type": "module",
   "main": "./dist/kernel/index.js",

package/packages/robopark/scheduler/preview_agent.py

@@ -83,22 +83,51 @@ def _normalize_livekit_url(url: Optional[str]) -> Optional[str]:
     return url
 
 
-def _play_audio_effect(selected_output: str | None, effect: str) -> None:
-    """Play a short local cue without involving the voice pipeline."""
-    try:
-        import pyaudio
-    except Exception:
-        return
-    sample_rate = 48000
-    channels = 2
+def _play_audio_effect(selected_output: str | None, effect: str) -> None:
+    """Play a short local cue without involving the voice pipeline."""
+    sample_rate = 48000
+    channels = 2
     if effect == "motion":
         notes = ((880, 0.09), (1320, 0.13))
     else:
         notes = ((660, 0.10), (440, 0.16))
-    pa = pyaudio.PyAudio()
-    device_index = None
-    selected = str(selected_output or "default")
-    try:
+    selected = str(selected_output or "default")
+    frames = bytearray()
+    for frequency, duration in notes:
+        count = int(sample_rate * duration)
+        for n in range(count):
+            envelope = min(1.0, n / 240.0, (count - n) / 1200.0)
+            value = int(5000 * envelope * math.sin(2 * math.pi * frequency * n / sample_rate))
+            frames.extend(struct.pack("<hh", value, value))
+
+    if sys.platform.startswith("linux") and "hw:" in selected:
+        import re
+        import subprocess
+
+        match = re.search(r"\b(hw:\d+,\d+)\b", selected)
+        if not match:
+            return
+        result = subprocess.run(
+            [
+                "aplay", "-q", "-D", f"plug{match.group(1)}", "-t", "raw",
+                "-f", "S16_LE", "-r", str(sample_rate), "-c", str(channels),
+            ],
+            input=bytes(frames), capture_output=True, timeout=3.0,
+        )
+        if result.returncode:
+            logger.warning(
+                "audio effect failed on %s: %s",
+                match.group(1), result.stderr.decode("utf-8", errors="replace").strip(),
+            )
+        return
+
+    try:
+        import pyaudio
+    except Exception:
+        return
+    pa = pyaudio.PyAudio()
+    device_index = None
+    try:
         if selected.strip().isdigit():
             device_index = int(selected.strip())
         elif selected.lower() == "default":
@@ -118,14 +147,7 @@ def _play_audio_effect(selected_output: str | None, effect: str) -> None:
             output=True,
             output_device_index=device_index,
         )
-        for frequency, duration in notes:
-            count = int(sample_rate * duration)
-            frames = bytearray()
-            for n in range(count):
-                envelope = min(1.0, n / 240.0, (count - n) / 1200.0)
-                value = int(5000 * envelope * math.sin(2 * math.pi * frequency * n / sample_rate))
-                frames.extend(struct.pack("<hh", value, value))
-            stream.write(bytes(frames))
+        stream.write(bytes(frames))
         stream.stop_stream()
         stream.close()
     except Exception as e:
@@ -1041,9 +1063,10 @@ class PreviewAgent:
         # for the LiveKit publisher.
         await asyncio.sleep(0.5)
         # I/O. Camera/mic join and the preset greeting are the critical path.
-        asyncio.create_task(
-            asyncio.to_thread(_play_audio_effect, self.audio_playback_device, "motion")
-        )
+        # The cue and TTS share one physical ALSA output. Finish and release
+        # the cue before LiveKit can deliver the greeting; background playback
+        # races aplay and makes greeting delivery intermittently fail busy.
+        await asyncio.to_thread(_play_audio_effect, self.audio_playback_device, "motion")
         try:
             r = await self._session.post(
                 f"{self.scheduler_url.rstrip('/')}/api/devices/{self.device_id}/request-session",
@@ -1350,17 +1373,27 @@ class LiveKitPublisher:
         # steady pace, independent of how unevenly frames actually arrive.
         import queue
         import threading
-        write_queue: "queue.Queue[Optional[bytes]]" = queue.Queue()
-        written_frames = [0]
-        PREBUFFER_CHUNKS = 5
-        stream_failed = threading.Event()
+        write_queue: "queue.Queue[Optional[bytes]]" = queue.Queue()
+        written_frames = [0]
+        PREBUFFER_CHUNKS = 5
+        stream_failed = threading.Event()
+        playback_reported = threading.Event()
+        event_loop = asyncio.get_running_loop()
 
         def _safe_write(chunk: bytes) -> bool:
             if stream_failed.is_set():
                 return False
-            try:
-                out.write(chunk)
-                return True
+            try:
+                out.write(chunk)
+                if not playback_reported.is_set():
+                    playback_reported.set()
+                    asyncio.run_coroutine_threadsafe(
+                        self.agent._report_pipeline(
+                            "playback_started", "ok", "First TTS audio chunk written to robot speaker"
+                        ),
+                        event_loop,
+                    )
+                return True
             except Exception as e:
                 stream_failed.set()
                 logger.warning(f"audio out stream closed; disabling playback for this track: {e}")
@@ -1420,7 +1453,6 @@ class LiveKitPublisher:
                         f"audio out: first frame received for {sid} "
                         f"(rate={af.sample_rate}, channels={getattr(af, 'num_channels', 1)})"
                     )
-                    await self.agent._report_pipeline("playback_started", "ok", "First TTS audio frame reached the robot")
                 in_rate = af.sample_rate
                 in_channels = int(getattr(af, "num_channels", 1) or 1)
                 raw = bytes(af.data)

package/packages/robopark/vision/app_pi_clean.py

@@ -18,6 +18,11 @@ CORS(app)
 # Camera management
 current_camera_index = 0
 camera = None
+camera_lock = threading.Lock()
+frame_condition = threading.Condition()
+latest_frame_bytes = None
+latest_frame_sequence = 0
+camera_worker_started = False
 audio_input_device = "default"
 audio_output_device = "default"
 ROBOVISION_AUDIO_URL = os.getenv("ROBOVISION_AUDIO_URL", "http://127.0.0.1:8000")
@@ -224,22 +229,25 @@ def send_webhook(frame_data):
     except Exception as e:
         print(f"Error preparing webhook: {e}")
 
-def generate_frames():
-    global latest_detections, motion_detection_active, motion_detected_state
-    global last_motion_time, motion_frame_buffer
-
-    prev_gray = None
-
-    while True:
-        cam = get_camera()
-        if cam is None or not cam.isOpened():
-            time.sleep(1)
-            continue
-
-        success, frame = cam.read()
-        if not success:
-            time.sleep(0.1)
-            continue
+def camera_worker():
+    global latest_detections, motion_detection_active, motion_detected_state
+    global last_motion_time, motion_frame_buffer
+    global latest_frame_bytes, latest_frame_sequence
+
+    prev_gray = None
+
+    while True:
+        with camera_lock:
+            cam = get_camera()
+        if cam is None or not cam.isOpened():
+            time.sleep(1)
+            continue
+
+        with camera_lock:
+            success, frame = cam.read()
+        if not success:
+            time.sleep(0.1)
+            continue
 
         # Run object detection
         processed_frame, detections = detect_objects(frame, conf_threshold=0.5)
@@ -288,11 +296,40 @@ def generate_frames():
 
             prev_gray = gray
 
-        ret, buffer = cv2.imencode('.jpg', processed_frame, [cv2.IMWRITE_JPEG_QUALITY, 80])
-        frame_bytes = buffer.tobytes()
-
-        yield (b'--frame\r\n'
-               b'Content-Type: image/jpeg\r\n\r\n' + frame_bytes + b'\r\n')
+        ret, buffer = cv2.imencode('.jpg', processed_frame, [cv2.IMWRITE_JPEG_QUALITY, 80])
+        if not ret:
+            continue
+        with frame_condition:
+            latest_frame_bytes = buffer.tobytes()
+            latest_frame_sequence += 1
+            frame_condition.notify_all()
+
+
+def _ensure_camera_worker():
+    global camera_worker_started
+    with frame_condition:
+        if camera_worker_started:
+            return
+        camera_worker_started = True
+    threading.Thread(target=camera_worker, name='robovision-camera', daemon=True).start()
+
+
+def generate_frames():
+    _ensure_camera_worker()
+    sequence = -1
+    while True:
+        with frame_condition:
+            frame_condition.wait_for(
+                lambda: latest_frame_bytes is not None and latest_frame_sequence != sequence,
+                timeout=5.0,
+            )
+            if latest_frame_bytes is None or latest_frame_sequence == sequence:
+                continue
+            frame_bytes = latest_frame_bytes
+            sequence = latest_frame_sequence
+
+        yield (b'--frame\r\n'
+               b'Content-Type: image/jpeg\r\n\r\n' + frame_bytes + b'\r\n')
 
 @app.route('/')
 def index():
@@ -379,11 +416,11 @@ def switch_camera():
     if isinstance(new_index, str) and new_index.isdigit():
         new_index = int(new_index)
 
-    if camera:
-        camera.release()
-
-    current_camera_index = new_index
-    camera = None
+    with camera_lock:
+        if camera:
+            camera.release()
+        current_camera_index = new_index
+        camera = None
 
     return jsonify({"status": "ok", "camera_index": current_camera_index})