infinicode 2.8.13 → 2.8.16

package/packages/robopark/scheduler/main.py

@@ -505,9 +505,11 @@ async def request_session(robot_id: str,
             if server["active"] >= server["max_sessions"]:
                 raise HTTPException(503, "All servers at capacity")
         
-        # Create session
+        # Create session. Room name MUST start with "robopark-" — that's the
+        # literal prefix voice_agent.py checks to enable production-mode
+        # (character/voice/motor resolution, unlimited idle timeout).
         session_id = f"session_{robot_id}_{int(datetime.utcnow().timestamp())}"
-        room_name = f"robopark_{robot_id}_{int(datetime.utcnow().timestamp())}"
+        room_name = f"robopark-{robot_id}-{int(datetime.utcnow().timestamp())}"
         
         await db.execute("""
             INSERT INTO sessions (id, robot_id, server_id, room_name, started_at)
@@ -1370,6 +1372,7 @@ async def robot_stream(robot_id: str,
 # cam. Cam therefore runs ONLY while an operator is watching.
 
 PREVIEW_TTL_SECONDS = 45  # preview auto-expires this long after the last keepalive
+DEFAULT_VISION_PORT = 5000  # RoboVisionAI_PI's default Flask port (/video_feed, /api/motion/*)
 
 async def _pick_preview_server(db, prefer_id: Optional[str] = None):
     """Return a LiveKit server row for a preview: a preferred one if still present,
@@ -1518,12 +1521,29 @@ async def list_devices():
 
 @app.get("/api/devices/by-room/{room_name}")
 async def get_device_by_room(room_name: str):
-    """Look up the device bound to a LiveKit room. Convention: room = 'robopark-<device_id>'."""
-    if not room_name.startswith("robopark-"):
-        raise HTTPException(404, "Room is not a robopark room")
-    device_id = room_name[len("robopark-"):]
+    """Look up the device bound to a LiveKit room.
+
+    Resolves via the sessions table (robust to any room-name format the
+    scheduler generates — request-session's robopark-<id>-<ts>, etc.) rather
+    than parsing the name, since a plain prefix-strip broke the moment
+    request-session started including a trailing timestamp. Falls back to a
+    prefix-strip for the standing-room convention (robopark-<device_id>, no
+    trailing timestamp, no sessions row — e.g. robopark-pi-client's always-on
+    room when production_mode is on)."""
+    device_id = None
     async with aiosqlite.connect(DB_PATH) as db:
         db.row_factory = aiosqlite.Row
+        async with db.execute(
+            "SELECT robot_id FROM sessions WHERE room_name = ? ORDER BY started_at DESC LIMIT 1",
+            (room_name,),
+        ) as c:
+            session = await c.fetchone()
+        if session:
+            device_id = session["robot_id"]
+        elif room_name.startswith("robopark-"):
+            device_id = room_name[len("robopark-"):]
+        if not device_id:
+            raise HTTPException(404, "No device bound to this room")
         async with db.execute("SELECT * FROM devices WHERE id = ?", (device_id,)) as c:
             row = await c.fetchone()
             if not row:
@@ -1889,7 +1909,11 @@ async def device_request_session(device_id: str,
 
         ts = int(datetime.utcnow().timestamp())
         session_id = f"session_{device_id}_{ts}"
-        room_name = f"robopark_{device_id}_{ts}"
+        # Room name MUST start with "robopark-" — the literal prefix
+        # voice_agent.py checks to enable production-mode (character/voice/
+        # motor resolution via GET /api/devices/by-room, unlimited idle
+        # timeout instead of the 30s default).
+        room_name = f"robopark-{device_id}-{ts}"
 
         await db.execute("""
             INSERT INTO sessions (id, robot_id, server_id, room_name, started_at)
@@ -2027,13 +2051,22 @@ async def _robot_telemetry(db, robot_id: str) -> Optional[dict]:
     Returns trigger_count, sessions_total, avg_latency_ms, drop_rate, drops and
     an end_reason breakdown; None if the robot does not exist."""
     async with db.execute(
-        "SELECT trigger_count FROM robots WHERE id = ?", (robot_id,)
+        "SELECT trigger_count, name FROM robots WHERE id = ?", (robot_id,)
     ) as c:
         robot = await c.fetchone()
     if not robot:
         return None
     trigger_count = robot["trigger_count"] or 0
 
+    # Pull the device's freshest known LAN address (heartbeat > enrollment) so
+    # the dashboard can reach the robot directly for e.g. the vision overlay
+    # feed, which the scheduler doesn't proxy.
+    async with db.execute(
+        "SELECT last_seen_ip, lan_ip FROM devices WHERE id = ?", (robot_id,)
+    ) as c:
+        dev = await c.fetchone()
+    lan_ip = (dev["last_seen_ip"] or dev["lan_ip"]) if dev else None
+
     # Average latency over sessions that recorded a room join.
     async with db.execute(
         "SELECT started_at, joined_at FROM sessions WHERE robot_id = ? AND joined_at IS NOT NULL",
@@ -2066,6 +2099,9 @@ async def _robot_telemetry(db, robot_id: str) -> Optional[dict]:
 
     return {
         "robot_id": robot_id,
+        "name": robot["name"],
+        "lan_ip": lan_ip,
+        "vision_port": DEFAULT_VISION_PORT,
         "trigger_count": trigger_count,
         "sessions_total": sessions_total,
         "avg_latency_ms": avg_latency_ms,
@@ -2822,7 +2858,7 @@ async def dashboard():
                                     <video id="preview-video" class="w-full" autoplay playsinline muted></video>
                                     <div class="text-xs text-gray-400 px-2 py-1">{{ preview.room }}</div>
                                 </div>
-                            </div>
+                            </div>
                             
                             <!-- Servers Panel -->
                             <div class="col-span-5 space-y-4">

package/packages/robopark/scheduler/preview_agent.py

@@ -7,6 +7,15 @@ opens the selected camera + microphone and publishes them to the returned
 LiveKit room. When the preview expires or is stopped, the agent leaves the
 room and releases the hardware.
 
+Also runs a small local HTTP listener for RoboVisionAI_PI's motion webhook
+(POST /api/motion/webhook target — see RoboVisionAI_PI's app_pi_clean.py).
+On a motion event it calls the scheduler's presence-triggered
+POST /api/devices/{id}/request-session and publishes into the returned room
+using the same LiveKitPublisher as the operator preview — this is the
+"scene detection" trigger point the scheduler's request-session docstring
+describes; RoboVisionAI_PI supplies the detection, this agent supplies the
+bridge to an actual session.
+
 Configuration (env / ~/.robopark/preview_agent.json):
   SCHEDULER_URL          base URL of the RoboPark scheduler
   ROBOT_ID               robot identity in the scheduler (defaults to hostname)
@@ -18,6 +27,9 @@ Configuration (env / ~/.robopark/preview_agent.json):
   VIDEO_FPS              capture fps (default 15)
   POLL_INTERVAL          seconds between scheduler polls (default 3)
   HEARTBEAT_INTERVAL     seconds between device heartbeats (default 30)
+  VISION_WEBHOOK_PORT    local port for the RoboVision motion webhook (default 5057, 0 disables)
+  VISION_TRIGGER_COOLDOWN  min seconds between motion-triggered sessions (default 20)
+  VISION_SESSION_SECONDS   how long a motion-triggered session holds the publisher (default 90)
 """
 from __future__ import annotations
 
@@ -28,8 +40,10 @@ import logging
 import os
 import signal
 import sys
+import threading
 import time
 from dataclasses import dataclass
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from typing import Optional
 
@@ -46,6 +60,9 @@ DEFAULT_VIDEO_HEIGHT = 480
 DEFAULT_FPS = 15
 DEFAULT_POLL_INTERVAL = 3.0
 DEFAULT_HEARTBEAT_INTERVAL = 30.0
+DEFAULT_VISION_WEBHOOK_PORT = 5057
+DEFAULT_VISION_TRIGGER_COOLDOWN = 20.0
+DEFAULT_VISION_SESSION_SECONDS = 90.0
 
 
 def _load_config() -> dict:
@@ -120,7 +137,7 @@ async def _send_heartbeat(scheduler_url: str, device_id: str, token: str) -> Non
         async with httpx.AsyncClient() as client:
             await client.post(
                 f"{scheduler_url.rstrip('/')}/api/devices/{device_id}/heartbeat",
-                json={"status": "online"},
+                json={"status": "online", "ip": _get_lan_ip()},
                 headers={"Authorization": f"Bearer {token}"},
                 timeout=10.0,
             )
@@ -150,12 +167,19 @@ class PreviewAgent:
         self.fps = int(cfg.get("video_fps", os.getenv("VIDEO_FPS", DEFAULT_FPS)))
         self.poll_interval = float(cfg.get("poll_interval", os.getenv("POLL_INTERVAL", DEFAULT_POLL_INTERVAL)))
         self.heartbeat_interval = float(cfg.get("heartbeat_interval", os.getenv("HEARTBEAT_INTERVAL", DEFAULT_HEARTBEAT_INTERVAL)))
+        self.vision_webhook_port = int(cfg.get("vision_webhook_port", os.getenv("VISION_WEBHOOK_PORT", DEFAULT_VISION_WEBHOOK_PORT)))
+        self.vision_trigger_cooldown = float(cfg.get("vision_trigger_cooldown", os.getenv("VISION_TRIGGER_COOLDOWN", DEFAULT_VISION_TRIGGER_COOLDOWN)))
+        self.vision_session_seconds = float(cfg.get("vision_session_seconds", os.getenv("VISION_SESSION_SECONDS", DEFAULT_VISION_SESSION_SECONDS)))
 
         self._shutdown = asyncio.Event()
         self._task: Optional[asyncio.Task] = None
         self._session: Optional[httpx.AsyncClient] = None
         self._current_state: PreviewState = PreviewState()
         self._publisher: Optional["LiveKitPublisher"] = None
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._vision_server: Optional[ThreadingHTTPServer] = None
+        self._last_vision_trigger: float = 0.0
+        self._vision_active_until: float = 0.0
 
     async def run(self) -> None:
         enrollment_token = os.getenv("ENROLLMENT_TOKEN") or self.enrollment_token
@@ -170,6 +194,8 @@ class PreviewAgent:
             await self._resolve_device_id()
 
         self._session = httpx.AsyncClient()
+        self._loop = asyncio.get_running_loop()
+        self._start_vision_webhook_server()
 
         tasks = [
             asyncio.create_task(self._poll_loop()),
@@ -182,6 +208,8 @@ class PreviewAgent:
                 await t
             except asyncio.CancelledError:
                 pass
+        if self._vision_server:
+            self._vision_server.shutdown()
         await self._stop_publisher()
         await self._session.aclose()
 
@@ -209,8 +237,17 @@ class PreviewAgent:
     async def _poll_loop(self) -> None:
         while not self._shutdown.is_set():
             try:
-                state = await self._fetch_preview_state()
-                await self._apply_state(state)
+                if time.time() >= self._vision_active_until:
+                    # A motion-triggered session (if any) has ended its lease —
+                    # let the operator-preview poll reconcile the publisher again.
+                    if self._vision_active_until:
+                        self._vision_active_until = 0.0
+                        await self._stop_publisher()
+                        self._current_state = PreviewState()
+                    state = await self._fetch_preview_state()
+                    await self._apply_state(state)
+                # else: a vision-triggered session currently owns the publisher —
+                # don't let the (unrelated) operator-preview state tear it down.
             except Exception as e:
                 logger.warning(f"poll error: {e}")
             try:
@@ -281,6 +318,92 @@ class PreviewAgent:
                 logger.warning(f"publisher stop error: {e}")
             self._publisher = None
 
+    # ---- vision (RoboVisionAI_PI motion webhook) -> presence-triggered session ----
+
+    def _start_vision_webhook_server(self) -> None:
+        if not self.vision_webhook_port:
+            return
+        agent = self
+
+        class Handler(BaseHTTPRequestHandler):
+            def log_message(self, fmt, *a):  # noqa: A002 — quiet by default
+                logger.debug("vision webhook: " + fmt, *a)
+
+            def do_POST(self):  # noqa: N802 — http.server's required method name
+                try:
+                    length = int(self.headers.get("Content-Length", 0))
+                    body = self.rfile.read(length) if length else b"{}"
+                    payload = json.loads(body or b"{}")
+                except Exception as e:
+                    self.send_response(400)
+                    self.end_headers()
+                    self.wfile.write(str(e).encode())
+                    return
+                self.send_response(200)
+                self.end_headers()
+                self.wfile.write(b'{"ok":true}')
+                if agent._loop:
+                    asyncio.run_coroutine_threadsafe(agent._on_vision_motion(payload), agent._loop)
+
+            def do_GET(self):  # noqa: N802
+                self.send_response(200)
+                self.end_headers()
+                self.wfile.write(b'{"status":"ok","listening_for":"robovision motion webhook"}')
+
+        try:
+            self._vision_server = ThreadingHTTPServer(("0.0.0.0", self.vision_webhook_port), Handler)
+            thread = threading.Thread(target=self._vision_server.serve_forever, daemon=True)
+            thread.start()
+            logger.info(
+                f"vision webhook listening on :{self.vision_webhook_port} — "
+                f"point RoboVisionAI_PI's POST /api/motion/webhook at "
+                f"http://<this-host>:{self.vision_webhook_port}/"
+            )
+        except Exception as e:
+            logger.error(f"could not start vision webhook server: {e}")
+            self._vision_server = None
+
+    async def _on_vision_motion(self, payload: dict) -> None:
+        now = time.time()
+        if now - self._last_vision_trigger < self.vision_trigger_cooldown:
+            logger.debug("vision trigger suppressed (cooldown)")
+            return
+        self._last_vision_trigger = now
+        if not self.device_id or not self.device_token or not self._session:
+            logger.warning("vision motion event received but not enrolled yet — ignoring")
+            return
+        logger.info("motion detected by RoboVisionAI_PI — requesting a session")
+        try:
+            r = await self._session.post(
+                f"{self.scheduler_url.rstrip('/')}/api/devices/{self.device_id}/request-session",
+                headers={"Authorization": f"Bearer {self.device_token}"},
+                timeout=10.0,
+            )
+            r.raise_for_status()
+            data = r.json()
+        except Exception as e:
+            logger.error(f"request-session failed: {e}")
+            return
+        state = PreviewState(
+            active=True,
+            url=data.get("server_url"),
+            token=data.get("token"),
+            room=data.get("room_name"),
+        )
+        self._vision_active_until = time.time() + self.vision_session_seconds
+        self._current_state = state
+        await self._start_publisher(state)
+        session_id = data.get("session_id")
+        if session_id and self._session:
+            try:
+                await self._session.post(
+                    f"{self.scheduler_url.rstrip('/')}/api/sessions/{session_id}/joined",
+                    headers={"Authorization": f"Bearer {self.device_token}"},
+                    timeout=10.0,
+                )
+            except Exception as e:
+                logger.debug(f"could not mark session joined: {e}")
+
     def shutdown(self) -> None:
         self._shutdown.set()
 
@@ -592,6 +715,9 @@ def main() -> None:
     parser.add_argument("--fps", type=int, default=int(os.getenv("VIDEO_FPS", DEFAULT_FPS)))
     parser.add_argument("--poll-interval", type=float, default=float(os.getenv("POLL_INTERVAL", DEFAULT_POLL_INTERVAL)))
     parser.add_argument("--heartbeat-interval", type=float, default=float(os.getenv("HEARTBEAT_INTERVAL", DEFAULT_HEARTBEAT_INTERVAL)))
+    parser.add_argument("--vision-webhook-port", type=int, default=int(os.getenv("VISION_WEBHOOK_PORT", DEFAULT_VISION_WEBHOOK_PORT)), help="local port for RoboVisionAI_PI's motion webhook (0 disables)")
+    parser.add_argument("--vision-trigger-cooldown", type=float, default=float(os.getenv("VISION_TRIGGER_COOLDOWN", DEFAULT_VISION_TRIGGER_COOLDOWN)))
+    parser.add_argument("--vision-session-seconds", type=float, default=float(os.getenv("VISION_SESSION_SECONDS", DEFAULT_VISION_SESSION_SECONDS)))
     parser.add_argument("--save-config", action="store_true", help="write CLI args to ~/.robopark/preview_agent.json")
     parser.add_argument("-v", "--verbose", action="store_true")
     args = parser.parse_args()
@@ -612,6 +738,9 @@ def main() -> None:
         "video_fps": args.fps,
         "poll_interval": args.poll_interval,
         "heartbeat_interval": args.heartbeat_interval,
+        "vision_webhook_port": args.vision_webhook_port,
+        "vision_trigger_cooldown": args.vision_trigger_cooldown,
+        "vision_session_seconds": args.vision_session_seconds,
     })
     if args.device_token:
         cfg["device_token"] = args.device_token

package/packages/robopark/vision/.env.example

@@ -0,0 +1,7 @@
+GROQ_API_KEY=
+TRANSCRIPTION_ENABLED=true
+WHISPER_MODEL=whisper-large-v3
+AUTO_DETECT_LANGUAGE=true
+BLUETOOTH_DEVICE_NAME=
+WEBHOOK_AUDIO_DETECTED=
+WEBHOOK_NO_AUDIO=

package/packages/robopark/vision/README.md

@@ -0,0 +1,66 @@
+# RoboVision — camera/motion detection (robot-side)
+
+Runs on the robot. A Flask server (`app_pi_clean.py`, port 5000 by default)
+that:
+
+- Streams the camera at `/video_feed` (MJPEG, with OpenCV-drawn bounding
+  boxes/labels baked into the frame) — this is what the Park dashboard's
+  "Show overlay" button in the camera drawer points at directly.
+- Runs OpenCV DNN object detection (MobileNet SSD) on every frame —
+  `/api/detections` for the latest results.
+- Runs frame-diff motion detection, toggled via `POST /api/motion/toggle
+  {"active": true}` — when motion is detected it POSTs a JPEG snapshot to a
+  configurable webhook (`POST /api/motion/webhook {"url": "..."}`).
+
+## Wiring it to an actual session (the production trigger)
+
+RoboVision only *detects*; it doesn't itself start a robot session. The
+scheduler's `preview_agent.py` (in `../scheduler/`) now runs a small built-in
+listener for exactly this webhook and turns a motion event into a real,
+presence-triggered session (`POST /api/devices/{id}/request-session`),
+publishing camera + mic into the resulting LiveKit room. Point RoboVision at
+it:
+
+Both of these calls go to RoboVision's own server (port 5000 by default —
+NOT the scheduler on 8080):
+
+```
+curl -X POST http://localhost:5000/api/motion/webhook \
+  -H "Content-Type: application/json" \
+  -d '{"url": "http://localhost:5057/"}'
+curl -X POST http://localhost:5000/api/motion/toggle \
+  -H "Content-Type: application/json" -d '{"active": true}'
+```
+
+(`5057` is `preview_agent.py`'s default `--vision-webhook-port`; override with
+`--vision-webhook-port` if RoboVision runs on a different host than the
+preview agent.)
+
+## Required model files (NOT bundled)
+
+`app_pi_clean.py` needs two Caffe model files that are **not shipped** in this
+npm package (the weights are ~23MB and would bloat every `infinicode`/
+`robopark` install, including installs that never touch RoboPark):
+
+```
+deploy.prototxt
+mobilenet_iter_73000.caffemodel
+```
+
+Download the standard MobileNet-SSD (VOC) weights (e.g. from
+chuanqi305/MobileNet-SSD) into this directory before running. Without them,
+`app_pi_clean.py` still runs — object detection is silently disabled, motion
+detection still works.
+
+## Running
+
+Pure Python + OpenCV (Flask, cv2, numpy) — cross-platform, no bash required:
+
+```
+pip install -r requirements_pi_unified.txt
+python app_pi_clean.py
+```
+
+`install.sh`/`run.sh` are Linux/Pi convenience wrappers (venv + systemd-style
+process management) — not required on Windows/macOS, just run the script
+directly.