infinicode 2.8.64 → 2.8.66

package/packages/robopark/scheduler/preview_agent.py

@@ -433,6 +433,31 @@ class PreviewAgent:
         self._motion_capture_lock = asyncio.Lock()
         self._mesh_bootstrap_ready = False
         self._next_mesh_bootstrap = 0.0
+        self._reported_pipeline: set[tuple[str, str]] = set()
+
+    async def _report_pipeline(self, stage: str, status: str = "ok", message: str = "",
+                               details: Optional[dict] = None, once: bool = True) -> None:
+        """Report a real robot-side transition for the Park test timeline."""
+        if not self._session or not self.device_id or not self.device_token:
+            return
+        session_id = self._vision_session_id or self._current_state.session_id
+        key = (session_id or "idle", stage)
+        if once and key in self._reported_pipeline:
+            return
+        try:
+            response = await self._session.post(
+                f"{self.scheduler_url.rstrip('/')}/api/devices/{self.device_id}/pipeline-events",
+                headers={"Authorization": f"Bearer {self.device_token}"},
+                json={"stage": stage, "status": status, "message": message,
+                      "session_id": session_id, "source": "preview_agent",
+                      "details": details or {}},
+                timeout=5.0,
+            )
+            response.raise_for_status()
+            if once:
+                self._reported_pipeline.add(key)
+        except Exception as exc:
+            logger.debug("pipeline event %s was not accepted: %s", stage, exc)
 
     async def _ensure_mesh_identity(self) -> bool:
         if not _mesh_proxy_headers():
@@ -744,7 +769,7 @@ class PreviewAgent:
             dt = dt.replace(tzinfo=timezone.utc)
         return dt.timestamp()
 
-    async def _end_vision_session(self) -> None:
+    async def _end_vision_session(self) -> None:
         # Tell the scheduler too, not just the local publisher/LiveKit
         # connection — otherwise the sessions row never gets ended_at set,
         # and the server's active_sessions count climbs forever until it
@@ -763,7 +788,8 @@ class PreviewAgent:
                 )
             except Exception as e:
                 logger.warning(f"failed to notify scheduler of session end: {e}")
-        self._vision_session_id = None
+        await self._report_pipeline("session_ended", "ok", "Robot conversation loop stopped")
+        self._vision_session_id = None
         self._vision_trigger_in_flight = False
         self._vision_hard_deadline = 0.0
         self._vision_last_activity = 0.0
@@ -784,8 +810,16 @@ class PreviewAgent:
                     self._mesh_bootstrap_ready = False
                     self._next_mesh_bootstrap = 0.0
                 if production_mode is not None:
-                    self.production_mode = production_mode
-                    if not production_mode and self._vision_session_id:
+                    self.production_mode = production_mode
+                    await self._report_pipeline("robot_online", message="Robot heartbeat accepted")
+                    inventory = _get_device_inventory()
+                    real_video = [d for d in inventory.get("video", []) if str(d.get("id", "")).lower() not in ("auto", "none")]
+                    real_inputs = [d for d in inventory.get("audio_input", []) if str(d.get("id", "")).lower() not in ("default", "none")]
+                    real_outputs = [d for d in inventory.get("audio_output", []) if str(d.get("id", "")).lower() not in ("default", "none")]
+                    await self._report_pipeline("camera_ready", "ok" if real_video else "blocked", f"{len(real_video)} camera device(s) detected")
+                    await self._report_pipeline("microphone_ready", "ok" if real_inputs else "blocked", f"{len(real_inputs)} microphone device(s) detected")
+                    await self._report_pipeline("speaker_ready", "ok" if real_outputs else "blocked", f"{len(real_outputs)} speaker device(s) detected")
+                    if not production_mode and self._vision_session_id:
                         await self._end_vision_session()
             try:
                 await asyncio.wait_for(self._shutdown.wait(), timeout=self.heartbeat_interval)
@@ -923,8 +957,9 @@ class PreviewAgent:
         if not self.device_id or not self.device_token or not self._session:
             logger.warning("vision motion event received but not enrolled yet — ignoring")
             return
-        self._vision_trigger_in_flight = True
-        logger.info("motion detected by RoboVisionAI_PI — requesting a session")
+        self._vision_trigger_in_flight = True
+        await self._report_pipeline("motion_detected", "ok", f"Motion received from {payload.get('source', 'camera')}", once=False)
+        logger.info("motion detected by RoboVisionAI_PI — requesting a session")
         # The cue is feedback only; never block session allocation on speaker
         await self._stop_motion_capture()
         # DirectShow can return a stale handle for a short interval after
@@ -943,9 +978,10 @@ class PreviewAgent:
             )
             r.raise_for_status()
             data = r.json()
-        except Exception as e:
-            logger.error(f"request-session failed: {e}")
-            self._vision_trigger_in_flight = False
+        except Exception as e:
+            logger.error(f"request-session failed: {e}")
+            await self._report_pipeline("scheduler_session", "failed", f"Session request failed: {type(e).__name__}", once=False)
+            self._vision_trigger_in_flight = False
             return
         state = PreviewState(
             active=True,
@@ -960,8 +996,9 @@ class PreviewAgent:
         self._vision_trigger_in_flight = False
         self._vision_hard_deadline = now + self.vision_session_seconds
         self._vision_last_activity = now
-        self._remote_session_ended = False
-        self._current_state = state
+        self._remote_session_ended = False
+        self._current_state = state
+        await self._report_pipeline("scheduler_session", "ok", "Scheduler allocated a conversation session")
         # If the scheduler returned a voice config, let the agent know by
         # posting it to our local webhook endpoint. The preview agent itself
         # does not consume it, but this makes the config observable locally
@@ -1006,9 +1043,10 @@ class LiveKitPublisher:
         # the camera is being handed between preview and voice sessions.
         self._last_motion_sample = 0.0
 
-    async def start(self) -> None:
-        self.room = self.rtc.Room()
-        await self.room.connect(self.url, self.token)
+    async def start(self) -> None:
+        self.room = self.rtc.Room()
+        await self.room.connect(self.url, self.token)
+        await self.agent._report_pipeline("livekit_join", "ok", "Robot joined the LiveKit room")
 
         # Register after signaling. Registering during Room.connect can invoke
         # callbacks while the native LiveKit participant state is incomplete;
@@ -1019,9 +1057,10 @@ class LiveKitPublisher:
             logger.info(f"track_subscribed: kind={track.kind} from={participant.identity} sid={publication.sid}")
             if track.kind != self.rtc.TrackKind.KIND_AUDIO:
                 return
-            if participant.identity == self.room.local_participant.identity:
-                return
-            self._tasks.append(asyncio.create_task(self._play_remote_audio(track, publication.sid)))
+            if participant.identity == self.room.local_participant.identity:
+                return
+            self._tasks.append(asyncio.create_task(self.agent._report_pipeline("tts_subscribed", "ok", "Subscribed to remote voice audio")))
+            self._tasks.append(asyncio.create_task(self._play_remote_audio(track, publication.sid)))
 
         self.room.on("track_subscribed", _on_track_subscribed)
         logger.info("audio out: track_subscribed listener registered")
@@ -1073,15 +1112,17 @@ class LiveKitPublisher:
             self.video_track = self.rtc.LocalVideoTrack.create_video_track("camera", self.video_source)
             vopts = self.rtc.TrackPublishOptions()
             vopts.source = self.rtc.TrackSource.SOURCE_CAMERA
-            await self.room.local_participant.publish_track(self.video_track, vopts)
-            self.video_source.capture_frame(first_frame)
+            await self.room.local_participant.publish_track(self.video_track, vopts)
+            self.video_source.capture_frame(first_frame)
+            await self.agent._report_pipeline("camera_published", "ok", "Camera track published")
 
         if audio_enabled:
             self.audio_source = self.rtc.AudioSource(48000, 1)
             self.audio_track = self.rtc.LocalAudioTrack.create_audio_track("microphone", self.audio_source)
             aopts = self.rtc.TrackPublishOptions()
             aopts.source = self.rtc.TrackSource.SOURCE_MICROPHONE
-            await self.room.local_participant.publish_track(self.audio_track, aopts)
+            await self.room.local_participant.publish_track(self.audio_track, aopts)
+            await self.agent._report_pipeline("microphone_published", "ok", "Microphone track published")
 
         # Register speaker playback (subscribe to the voice agent's TTS audio
         # track) BEFORE opening the camera. Camera open is a slow/occasionally
@@ -1217,11 +1258,12 @@ class LiveKitPublisher:
             async for frame in stream:
                 af = frame.frame if hasattr(frame, "frame") else frame
                 frame_count += 1
-                if frame_count == 1:
-                    logger.info(
-                        f"audio out: first frame received for {sid} "
-                        f"(rate={af.sample_rate}, channels={getattr(af, 'num_channels', 1)})"
-                    )
+                if frame_count == 1:
+                    logger.info(
+                        f"audio out: first frame received for {sid} "
+                        f"(rate={af.sample_rate}, channels={getattr(af, 'num_channels', 1)})"
+                    )
+                    await self.agent._report_pipeline("playback_started", "ok", "First TTS audio frame reached the robot")
                 in_rate = af.sample_rate
                 in_channels = int(getattr(af, "num_channels", 1) or 1)
                 raw = bytes(af.data)