infinicode 2.8.12 → 2.8.15

package/packages/robopark/scheduler/preview_agent.py

@@ -7,6 +7,15 @@ opens the selected camera + microphone and publishes them to the returned
 LiveKit room. When the preview expires or is stopped, the agent leaves the
 room and releases the hardware.
 
+Also runs a small local HTTP listener for RoboVisionAI_PI's motion webhook
+(POST /api/motion/webhook target — see RoboVisionAI_PI's app_pi_clean.py).
+On a motion event it calls the scheduler's presence-triggered
+POST /api/devices/{id}/request-session and publishes into the returned room
+using the same LiveKitPublisher as the operator preview — this is the
+"scene detection" trigger point the scheduler's request-session docstring
+describes; RoboVisionAI_PI supplies the detection, this agent supplies the
+bridge to an actual session.
+
 Configuration (env / ~/.robopark/preview_agent.json):
   SCHEDULER_URL          base URL of the RoboPark scheduler
   ROBOT_ID               robot identity in the scheduler (defaults to hostname)
@@ -18,6 +27,9 @@ Configuration (env / ~/.robopark/preview_agent.json):
   VIDEO_FPS              capture fps (default 15)
   POLL_INTERVAL          seconds between scheduler polls (default 3)
   HEARTBEAT_INTERVAL     seconds between device heartbeats (default 30)
+  VISION_WEBHOOK_PORT    local port for the RoboVision motion webhook (default 5057, 0 disables)
+  VISION_TRIGGER_COOLDOWN  min seconds between motion-triggered sessions (default 20)
+  VISION_SESSION_SECONDS   how long a motion-triggered session holds the publisher (default 90)
 """
 from __future__ import annotations
 
@@ -28,8 +40,10 @@ import logging
 import os
 import signal
 import sys
+import threading
 import time
 from dataclasses import dataclass
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from typing import Optional
 
@@ -46,6 +60,9 @@ DEFAULT_VIDEO_HEIGHT = 480
 DEFAULT_FPS = 15
 DEFAULT_POLL_INTERVAL = 3.0
 DEFAULT_HEARTBEAT_INTERVAL = 30.0
+DEFAULT_VISION_WEBHOOK_PORT = 5057
+DEFAULT_VISION_TRIGGER_COOLDOWN = 20.0
+DEFAULT_VISION_SESSION_SECONDS = 90.0
 
 
 def _load_config() -> dict:
@@ -120,7 +137,7 @@ async def _send_heartbeat(scheduler_url: str, device_id: str, token: str) -> Non
         async with httpx.AsyncClient() as client:
             await client.post(
                 f"{scheduler_url.rstrip('/')}/api/devices/{device_id}/heartbeat",
-                json={"status": "online"},
+                json={"status": "online", "ip": _get_lan_ip()},
                 headers={"Authorization": f"Bearer {token}"},
                 timeout=10.0,
             )
@@ -150,12 +167,19 @@ class PreviewAgent:
         self.fps = int(cfg.get("video_fps", os.getenv("VIDEO_FPS", DEFAULT_FPS)))
         self.poll_interval = float(cfg.get("poll_interval", os.getenv("POLL_INTERVAL", DEFAULT_POLL_INTERVAL)))
         self.heartbeat_interval = float(cfg.get("heartbeat_interval", os.getenv("HEARTBEAT_INTERVAL", DEFAULT_HEARTBEAT_INTERVAL)))
+        self.vision_webhook_port = int(cfg.get("vision_webhook_port", os.getenv("VISION_WEBHOOK_PORT", DEFAULT_VISION_WEBHOOK_PORT)))
+        self.vision_trigger_cooldown = float(cfg.get("vision_trigger_cooldown", os.getenv("VISION_TRIGGER_COOLDOWN", DEFAULT_VISION_TRIGGER_COOLDOWN)))
+        self.vision_session_seconds = float(cfg.get("vision_session_seconds", os.getenv("VISION_SESSION_SECONDS", DEFAULT_VISION_SESSION_SECONDS)))
 
         self._shutdown = asyncio.Event()
         self._task: Optional[asyncio.Task] = None
         self._session: Optional[httpx.AsyncClient] = None
         self._current_state: PreviewState = PreviewState()
         self._publisher: Optional["LiveKitPublisher"] = None
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._vision_server: Optional[ThreadingHTTPServer] = None
+        self._last_vision_trigger: float = 0.0
+        self._vision_active_until: float = 0.0
 
     async def run(self) -> None:
         enrollment_token = os.getenv("ENROLLMENT_TOKEN") or self.enrollment_token
@@ -170,6 +194,8 @@ class PreviewAgent:
             await self._resolve_device_id()
 
         self._session = httpx.AsyncClient()
+        self._loop = asyncio.get_running_loop()
+        self._start_vision_webhook_server()
 
         tasks = [
             asyncio.create_task(self._poll_loop()),
@@ -182,6 +208,8 @@ class PreviewAgent:
                 await t
             except asyncio.CancelledError:
                 pass
+        if self._vision_server:
+            self._vision_server.shutdown()
         await self._stop_publisher()
         await self._session.aclose()
 
@@ -209,8 +237,17 @@ class PreviewAgent:
     async def _poll_loop(self) -> None:
         while not self._shutdown.is_set():
             try:
-                state = await self._fetch_preview_state()
-                await self._apply_state(state)
+                if time.time() >= self._vision_active_until:
+                    # A motion-triggered session (if any) has ended its lease —
+                    # let the operator-preview poll reconcile the publisher again.
+                    if self._vision_active_until:
+                        self._vision_active_until = 0.0
+                        await self._stop_publisher()
+                        self._current_state = PreviewState()
+                    state = await self._fetch_preview_state()
+                    await self._apply_state(state)
+                # else: a vision-triggered session currently owns the publisher —
+                # don't let the (unrelated) operator-preview state tear it down.
             except Exception as e:
                 logger.warning(f"poll error: {e}")
             try:
@@ -281,6 +318,92 @@ class PreviewAgent:
                 logger.warning(f"publisher stop error: {e}")
             self._publisher = None
 
+    # ---- vision (RoboVisionAI_PI motion webhook) -> presence-triggered session ----
+
+    def _start_vision_webhook_server(self) -> None:
+        if not self.vision_webhook_port:
+            return
+        agent = self
+
+        class Handler(BaseHTTPRequestHandler):
+            def log_message(self, fmt, *a):  # noqa: A002 — quiet by default
+                logger.debug("vision webhook: " + fmt, *a)
+
+            def do_POST(self):  # noqa: N802 — http.server's required method name
+                try:
+                    length = int(self.headers.get("Content-Length", 0))
+                    body = self.rfile.read(length) if length else b"{}"
+                    payload = json.loads(body or b"{}")
+                except Exception as e:
+                    self.send_response(400)
+                    self.end_headers()
+                    self.wfile.write(str(e).encode())
+                    return
+                self.send_response(200)
+                self.end_headers()
+                self.wfile.write(b'{"ok":true}')
+                if agent._loop:
+                    asyncio.run_coroutine_threadsafe(agent._on_vision_motion(payload), agent._loop)
+
+            def do_GET(self):  # noqa: N802
+                self.send_response(200)
+                self.end_headers()
+                self.wfile.write(b'{"status":"ok","listening_for":"robovision motion webhook"}')
+
+        try:
+            self._vision_server = ThreadingHTTPServer(("0.0.0.0", self.vision_webhook_port), Handler)
+            thread = threading.Thread(target=self._vision_server.serve_forever, daemon=True)
+            thread.start()
+            logger.info(
+                f"vision webhook listening on :{self.vision_webhook_port} — "
+                f"point RoboVisionAI_PI's POST /api/motion/webhook at "
+                f"http://<this-host>:{self.vision_webhook_port}/"
+            )
+        except Exception as e:
+            logger.error(f"could not start vision webhook server: {e}")
+            self._vision_server = None
+
+    async def _on_vision_motion(self, payload: dict) -> None:
+        now = time.time()
+        if now - self._last_vision_trigger < self.vision_trigger_cooldown:
+            logger.debug("vision trigger suppressed (cooldown)")
+            return
+        self._last_vision_trigger = now
+        if not self.device_id or not self.device_token or not self._session:
+            logger.warning("vision motion event received but not enrolled yet — ignoring")
+            return
+        logger.info("motion detected by RoboVisionAI_PI — requesting a session")
+        try:
+            r = await self._session.post(
+                f"{self.scheduler_url.rstrip('/')}/api/devices/{self.device_id}/request-session",
+                headers={"Authorization": f"Bearer {self.device_token}"},
+                timeout=10.0,
+            )
+            r.raise_for_status()
+            data = r.json()
+        except Exception as e:
+            logger.error(f"request-session failed: {e}")
+            return
+        state = PreviewState(
+            active=True,
+            url=data.get("server_url"),
+            token=data.get("token"),
+            room=data.get("room_name"),
+        )
+        self._vision_active_until = time.time() + self.vision_session_seconds
+        self._current_state = state
+        await self._start_publisher(state)
+        session_id = data.get("session_id")
+        if session_id and self._session:
+            try:
+                await self._session.post(
+                    f"{self.scheduler_url.rstrip('/')}/api/sessions/{session_id}/joined",
+                    headers={"Authorization": f"Bearer {self.device_token}"},
+                    timeout=10.0,
+                )
+            except Exception as e:
+                logger.debug(f"could not mark session joined: {e}")
+
     def shutdown(self) -> None:
         self._shutdown.set()
 
@@ -592,6 +715,9 @@ def main() -> None:
     parser.add_argument("--fps", type=int, default=int(os.getenv("VIDEO_FPS", DEFAULT_FPS)))
     parser.add_argument("--poll-interval", type=float, default=float(os.getenv("POLL_INTERVAL", DEFAULT_POLL_INTERVAL)))
     parser.add_argument("--heartbeat-interval", type=float, default=float(os.getenv("HEARTBEAT_INTERVAL", DEFAULT_HEARTBEAT_INTERVAL)))
+    parser.add_argument("--vision-webhook-port", type=int, default=int(os.getenv("VISION_WEBHOOK_PORT", DEFAULT_VISION_WEBHOOK_PORT)), help="local port for RoboVisionAI_PI's motion webhook (0 disables)")
+    parser.add_argument("--vision-trigger-cooldown", type=float, default=float(os.getenv("VISION_TRIGGER_COOLDOWN", DEFAULT_VISION_TRIGGER_COOLDOWN)))
+    parser.add_argument("--vision-session-seconds", type=float, default=float(os.getenv("VISION_SESSION_SECONDS", DEFAULT_VISION_SESSION_SECONDS)))
     parser.add_argument("--save-config", action="store_true", help="write CLI args to ~/.robopark/preview_agent.json")
     parser.add_argument("-v", "--verbose", action="store_true")
     args = parser.parse_args()
@@ -612,6 +738,9 @@ def main() -> None:
         "video_fps": args.fps,
         "poll_interval": args.poll_interval,
         "heartbeat_interval": args.heartbeat_interval,
+        "vision_webhook_port": args.vision_webhook_port,
+        "vision_trigger_cooldown": args.vision_trigger_cooldown,
+        "vision_session_seconds": args.vision_session_seconds,
     })
     if args.device_token:
         cfg["device_token"] = args.device_token

package/packages/robopark/vision/.env.example

@@ -0,0 +1,7 @@
+GROQ_API_KEY=
+TRANSCRIPTION_ENABLED=true
+WHISPER_MODEL=whisper-large-v3
+AUTO_DETECT_LANGUAGE=true
+BLUETOOTH_DEVICE_NAME=
+WEBHOOK_AUDIO_DETECTED=
+WEBHOOK_NO_AUDIO=

package/packages/robopark/vision/README.md

@@ -0,0 +1,63 @@
+# RoboVision — camera/motion detection (robot-side)
+
+Runs on the robot. A Flask server (`app_pi_clean.py`, port 5000 by default)
+that:
+
+- Streams the camera at `/video_feed` (MJPEG, with OpenCV-drawn bounding
+  boxes/labels baked into the frame) — this is what the Park dashboard's
+  "Show overlay" button in the camera drawer points at directly.
+- Runs OpenCV DNN object detection (MobileNet SSD) on every frame —
+  `/api/detections` for the latest results.
+- Runs frame-diff motion detection, toggled via `POST /api/motion/toggle
+  {"active": true}` — when motion is detected it POSTs a JPEG snapshot to a
+  configurable webhook (`POST /api/motion/webhook {"url": "..."}`).
+
+## Wiring it to an actual session (the production trigger)
+
+RoboVision only *detects*; it doesn't itself start a robot session. The
+scheduler's `preview_agent.py` (in `../scheduler/`) now runs a small built-in
+listener for exactly this webhook and turns a motion event into a real,
+presence-triggered session (`POST /api/devices/{id}/request-session`),
+publishing camera + mic into the resulting LiveKit room. Point RoboVision at
+it:
+
+```
+curl -X POST http://localhost:8080/api/motion/webhook \
+  -H "Content-Type: application/json" \
+  -d '{"url": "http://localhost:5057/"}'
+curl -X POST http://localhost:8080/api/motion/toggle \
+  -H "Content-Type: application/json" -d '{"active": true}'
+```
+
+(`5057` is `preview_agent.py`'s default `--vision-webhook-port`; override with
+`--vision-webhook-port` if RoboVision runs on a different host than the
+preview agent.)
+
+## Required model files (NOT bundled)
+
+`app_pi_clean.py` needs two Caffe model files that are **not shipped** in this
+npm package (the weights are ~23MB and would bloat every `infinicode`/
+`robopark` install, including installs that never touch RoboPark):
+
+```
+deploy.prototxt
+mobilenet_iter_73000.caffemodel
+```
+
+Download the standard MobileNet-SSD (VOC) weights (e.g. from
+chuanqi305/MobileNet-SSD) into this directory before running. Without them,
+`app_pi_clean.py` still runs — object detection is silently disabled, motion
+detection still works.
+
+## Running
+
+Pure Python + OpenCV (Flask, cv2, numpy) — cross-platform, no bash required:
+
+```
+pip install -r requirements_pi_unified.txt
+python app_pi_clean.py
+```
+
+`install.sh`/`run.sh` are Linux/Pi convenience wrappers (venv + systemd-style
+process management) — not required on Windows/macOS, just run the script
+directly.

package/packages/robopark/vision/app_pi_clean.py

@@ -0,0 +1,307 @@
+from flask import Flask, Response, jsonify, request
+from flask_cors import CORS
+import cv2
+import time
+import numpy as np
+import threading
+import requests
+import base64
+
+app = Flask(__name__)
+CORS(app)
+
+# Camera management
+current_camera_index = 0
+camera = None
+
+def get_camera():
+    global camera, current_camera_index
+    if camera is None or not camera.isOpened():
+        camera = cv2.VideoCapture(current_camera_index)
+        camera.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
+        camera.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
+        print(f"Camera {current_camera_index} initialized")
+    return camera
+
+# Global variables
+latest_detections = []
+lock = threading.Lock()
+caption_mode_enabled = False
+motion_detection_active = False
+motion_detected_state = False
+last_motion_time = 0
+motion_frame_buffer = None
+webhook_url = None
+last_webhook_send_time = 0
+webhook_send_interval = 0.5
+
+# Simple object detection using OpenCV DNN (MobileNet SSD)
+try:
+    # Load pre-trained MobileNet SSD model
+    net = cv2.dnn.readNetFromCaffe(
+        'deploy.prototxt',
+        'mobilenet_iter_73000.caffemodel'
+    )
+    DETECTION_AVAILABLE = True
+    print("Object detection model loaded")
+except:
+    DETECTION_AVAILABLE = False
+    print("Object detection model not found - running without detection")
+
+# COCO class labels
+CLASSES = ["background", "aeroplane", "bicycle", "bird", "boat",
+    "bottle", "bus", "car", "cat", "chair", "cow", "diningtable",
+    "dog", "horse", "motorbike", "person", "pottedplant", "sheep",
+    "sofa", "train", "tvmonitor"]
+
+def detect_objects(frame, conf_threshold=0.5):
+    """Detect objects using OpenCV DNN"""
+    if not DETECTION_AVAILABLE:
+        return frame, []
+
+    (h, w) = frame.shape[:2]
+    blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 0.007843, (300, 300), 127.5)
+    net.setInput(blob)
+    detections_dnn = net.forward()
+
+    detected_objects = []
+
+    for i in range(detections_dnn.shape[2]):
+        confidence = detections_dnn[0, 0, i, 2]
+
+        if confidence > conf_threshold:
+            idx = int(detections_dnn[0, 0, i, 1])
+            if idx >= len(CLASSES):
+                continue
+
+            box = detections_dnn[0, 0, i, 3:7] * np.array([w, h, w, h])
+            (startX, startY, endX, endY) = box.astype("int")
+
+            label = CLASSES[idx]
+
+            # Draw bounding box
+            cv2.rectangle(frame, (startX, startY), (endX, endY), (0, 255, 0), 2)
+
+            # Draw label with confidence
+            text = f"{label}: {confidence*100:.1f}%"
+            y = startY - 15 if startY - 15 > 15 else startY + 15
+            cv2.putText(frame, text, (startX, y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
+
+            detected_objects.append({
+                "label": label,
+                "confidence": float(confidence),
+                "bbox": [int(startX), int(startY), int(endX), int(endY)],
+                "is_focus": False
+            })
+
+    return frame, detected_objects
+
+def send_webhook(frame_data):
+    """Send frame to webhook URL"""
+    global webhook_url, last_webhook_send_time
+
+    if not webhook_url:
+        return
+
+    current_time = time.time()
+    if current_time - last_webhook_send_time < webhook_send_interval:
+        return
+
+    try:
+        _, buffer = cv2.imencode('.jpg', frame_data)
+        img_base64 = base64.b64encode(buffer).decode('utf-8')
+
+        payload = {
+            'timestamp': time.strftime('%Y-%m-%dT%H:%M:%S'),
+            'image': img_base64,
+            'format': 'jpeg',
+            'encoding': 'base64'
+        }
+
+        def send_async():
+            try:
+                response = requests.post(webhook_url, json=payload, headers={'Content-Type': 'application/json'}, timeout=5)
+                if response.status_code == 200:
+                    print(f"Webhook sent successfully")
+            except Exception as e:
+                print(f"Webhook error: {e}")
+
+        thread = threading.Thread(target=send_async, daemon=True)
+        thread.start()
+        last_webhook_send_time = current_time
+    except Exception as e:
+        print(f"Error preparing webhook: {e}")
+
+def generate_frames():
+    global latest_detections, motion_detection_active, motion_detected_state
+    global last_motion_time, motion_frame_buffer
+
+    prev_gray = None
+
+    while True:
+        cam = get_camera()
+        if cam is None or not cam.isOpened():
+            time.sleep(1)
+            continue
+
+        success, frame = cam.read()
+        if not success:
+            time.sleep(0.1)
+            continue
+
+        # Run object detection
+        processed_frame, detections = detect_objects(frame, conf_threshold=0.5)
+
+        with lock:
+            latest_detections = detections
+
+        # Motion detection logic
+        if motion_detection_active:
+            gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+            gray = cv2.GaussianBlur(gray, (21, 21), 0)
+
+            if prev_gray is not None:
+                frame_delta = cv2.absdiff(prev_gray, gray)
+                thresh = cv2.threshold(frame_delta, 25, 255, cv2.THRESH_BINARY)[1]
+                thresh = cv2.dilate(thresh, None, iterations=2)
+                contours, _ = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+                motion_detected = False
+                for contour in contours:
+                    if cv2.contourArea(contour) >= 500:
+                        motion_detected = True
+                        (x, y, w, h) = cv2.boundingRect(contour)
+                        cv2.rectangle(processed_frame, (x, y), (x + w, y + h), (0, 0, 255), 2)
+                        break
+
+                if motion_detected:
+                    motion_detected_state = True
+                    last_motion_time = time.time()
+                    motion_frame_buffer = processed_frame.copy()
+                    print(f"Motion detected!")
+                    send_webhook(processed_frame)
+                else:
+                    motion_detected_state = False
+
+            prev_gray = gray
+
+        ret, buffer = cv2.imencode('.jpg', processed_frame, [cv2.IMWRITE_JPEG_QUALITY, 80])
+        frame_bytes = buffer.tobytes()
+
+        yield (b'--frame\r\n'
+               b'Content-Type: image/jpeg\r\n\r\n' + frame_bytes + b'\r\n')
+
+@app.route('/')
+def index():
+    return jsonify({"status": "ok", "message": "RoboVision Pi Server", "version": "1.0"})
+
+@app.route('/video_feed')
+def video_feed():
+    return Response(generate_frames(), mimetype='multipart/x-mixed-replace; boundary=frame')
+
+@app.route('/api/detections')
+def get_detections():
+    with lock:
+        return jsonify(latest_detections)
+
+@app.route('/api/caption')
+def get_caption():
+    return jsonify({"caption": "Awaiting caption..."})
+
+@app.route('/api/caption_mode', methods=['GET', 'POST'])
+def caption_mode():
+    global caption_mode_enabled
+    if request.method == 'GET':
+        return jsonify({"enabled": caption_mode_enabled})
+    data = request.json or {}
+    caption_mode_enabled = bool(data.get('enabled', False))
+    return jsonify({"enabled": caption_mode_enabled})
+
+@app.route('/api/cameras', methods=['GET'])
+def list_cameras():
+    available = []
+    for i in range(4):
+        cap = cv2.VideoCapture(i)
+        if cap.isOpened():
+            available.append({"index": i, "name": f"Camera {i}"})
+            cap.release()
+    return jsonify({"cameras": available, "current": current_camera_index})
+
+@app.route('/api/camera/switch', methods=['POST'])
+def switch_camera():
+    global camera, current_camera_index
+    data = request.json or {}
+    new_index = int(data.get('index', 0))
+
+    if camera:
+        camera.release()
+
+    current_camera_index = new_index
+    camera = None
+
+    return jsonify({"status": "ok", "camera_index": current_camera_index})
+
+@app.route('/api/motion/status', methods=['GET'])
+def motion_status():
+    global motion_detection_active, motion_detected_state, last_motion_time
+    return jsonify({
+        "active": motion_detection_active,
+        "motion_detected": motion_detected_state,
+        "last_motion": last_motion_time,
+        "time_since_motion": time.time() - last_motion_time if last_motion_time > 0 else None
+    })
+
+@app.route('/api/motion/toggle', methods=['POST'])
+def motion_toggle():
+    global motion_detection_active
+    data = request.json or {}
+    motion_detection_active = bool(data.get('active', False))
+    return jsonify({"status": "success", "active": motion_detection_active})
+
+@app.route('/api/motion/snapshot', methods=['GET'])
+def motion_snapshot():
+    global motion_frame_buffer
+    if motion_frame_buffer is not None:
+        _, buffer = cv2.imencode('.jpg', motion_frame_buffer, [cv2.IMWRITE_JPEG_QUALITY, 85])
+        img_base64 = base64.b64encode(buffer).decode('utf-8')
+        return jsonify({
+            "image": img_base64,
+            "timestamp": time.time()
+        })
+    return jsonify({"error": "No frame available"}), 404
+
+@app.route('/api/motion/webhook', methods=['GET', 'POST'])
+def motion_webhook():
+    global webhook_url
+
+    if request.method == 'GET':
+        return jsonify({
+            "webhook_url": webhook_url or "",
+            "configured": webhook_url is not None and len(webhook_url) > 0
+        })
+
+    data = request.json or {}
+    new_url = data.get('url', '').strip()
+
+    if new_url:
+        webhook_url = new_url
+        return jsonify({
+            "status": "success",
+            "message": "Webhook URL configured",
+            "webhook_url": webhook_url
+        })
+    else:
+        webhook_url = None
+        return jsonify({
+            "status": "success",
+            "message": "Webhook URL cleared",
+            "webhook_url": None
+        })
+
+if __name__ == '__main__':
+    print("=" * 60)
+    print("RoboVision - Raspberry Pi Vision Server (Minimal)")
+    print("=" * 60)
+    print("Starting Flask server on http://0.0.0.0:5000")
+    print("=" * 60)
+    app.run(host='0.0.0.0', port=5000, debug=False, threaded=True)