infinicode 2.8.73 → 2.8.76

package/packages/robopark/scheduler/main.py

@@ -3,8 +3,9 @@ RoboPark Session Scheduler - Backend API
 Manages robot fleet, LiveKit servers, and session orchestration
 """
 
-import os
-import asyncio
+import os
+import asyncio
+import base64
 import contextvars
 import json
 import logging
@@ -210,6 +211,7 @@ class DeviceBootstrapRequest(BaseModel):
     name: str
     lan_ip: Optional[str] = None
     tailscale_ip: Optional[str] = None
+    livekit_url: Optional[str] = None
 
 class DeviceEnrollResponse(BaseModel):
     device_id: str
@@ -220,8 +222,9 @@ class DeviceHeartbeat(BaseModel):
     status: Optional[str] = None
     ip: Optional[str] = None
     uptime_seconds: Optional[int] = None
-    production_mode: Optional[bool] = None
+    production_mode: Optional[bool] = None
     device_inventory: Optional[dict] = None
+    livekit_url: Optional[str] = None
 
 class PipelineEventPayload(BaseModel):
     stage: str
@@ -2811,16 +2814,17 @@ async def bootstrap_device(
             await db.execute(
                 """UPDATE devices SET token_hash = ?, status = 'enrolled',
                    lan_ip = COALESCE(?, lan_ip), tailscale_ip = COALESCE(?, tailscale_ip),
+                   livekit_url = COALESCE(?, livekit_url),
                    enrolled_at = COALESCE(enrolled_at, ?) WHERE id = ?""",
-                (new_hash, payload.lan_ip, payload.tailscale_ip, now, device_id),
+                (new_hash, payload.lan_ip, payload.tailscale_ip, payload.livekit_url, now, device_id),
             )
         else:
             device_id = f"dev_{secrets.token_hex(4)}"
             await db.execute(
                 """INSERT INTO devices
-                   (id, name, lan_ip, tailscale_ip, token_hash, status, enrolled_at, created_at)
-                   VALUES (?, ?, ?, ?, ?, 'enrolled', ?, ?)""",
-                (device_id, name, payload.lan_ip, payload.tailscale_ip, new_hash, now, now),
+                   (id, name, lan_ip, tailscale_ip, livekit_url, token_hash, status, enrolled_at, created_at)
+                   VALUES (?, ?, ?, ?, ?, ?, 'enrolled', ?, ?)""",
+                (device_id, name, payload.lan_ip, payload.tailscale_ip, payload.livekit_url, new_hash, now, now),
             )
         await db.execute(
             "INSERT OR IGNORE INTO robots (id, name, status) VALUES (?, ?, 'idle')",
@@ -2988,16 +2992,18 @@ async def device_heartbeat(device_id: str, payload: DeviceHeartbeat,
             """UPDATE devices
                SET status = ?, last_heartbeat = ?,
                    lan_ip = COALESCE(?, lan_ip),
-                   last_seen_ip = COALESCE(?, last_seen_ip),
-                   device_inventory = COALESCE(?, device_inventory)
+                   last_seen_ip = COALESCE(?, last_seen_ip),
+                   device_inventory = COALESCE(?, device_inventory),
+                   livekit_url = COALESCE(?, livekit_url)
                WHERE id = ?""",
             (
                 payload.status or "online",
                 datetime.utcnow().isoformat(),
                 payload.ip,
-                payload.ip,
-                json.dumps(payload.device_inventory) if payload.device_inventory is not None else None,
-                device_id,
+                payload.ip,
+                json.dumps(payload.device_inventory) if payload.device_inventory is not None else None,
+                payload.livekit_url,
+                device_id,
             ),
         ) as cur:
             await db.commit()
@@ -3435,24 +3441,101 @@ async def shell_allowlist():
 # other shell commands (the supervisor treats speaker_test as
 # just another request kind).
 
-class SpeakerTestRequest(BaseModel):
+class SpeakerTestRequest(BaseModel):
+    mode: str = "tone"
+    text: Optional[str] = None
     frequency: Optional[float] = None
     duration: Optional[float] = None
     amplitude: Optional[float] = None
     threshold_db: Optional[float] = None
     output: Optional[str] = None
     input: Optional[str] = None
-    sample_rate: Optional[int] = None
+    sample_rate: Optional[int] = None
+
+
+async def _cached_speaker_tts(robot_id: str, text: str) -> dict:
+    """Generate configured ElevenLabs PCM once, then reuse it for device tests."""
+    phrase = " ".join((text or "Hello, this is a RoboPark speaker test.").split())[:240]
+    config = await _resolve_robot_voice_config(robot_id)
+    stack = config.voice_stack if config else None
+    provider = (stack.tts_provider if stack else "elevenlabs").lower()
+    voice_id = stack.tts_voice if stack else "21m00Tcm4TlvDq8ikWAM"
+    if provider != "elevenlabs":
+        raise HTTPException(400, f"Cached voice test currently requires ElevenLabs; selected stack uses {provider}")
+    if not ELEVENLABS_API_KEY:
+        raise HTTPException(503, "ELEVENLABS_API_KEY is not configured on the scheduler")
+    if not voice_id:
+        raise HTTPException(400, "Selected voice stack has no ElevenLabs voice id")
+    cache_dir = os.path.join(os.path.dirname(DB_PATH), "tts-test-cache")
+    os.makedirs(cache_dir, exist_ok=True)
+    cache_key = hashlib.sha256(f"{provider}|{voice_id}|{phrase}".encode("utf8")).hexdigest()
+    cache_path = os.path.join(cache_dir, cache_key + ".pcm")
+    cache_hit = os.path.isfile(cache_path)
+    if cache_hit:
+        with open(cache_path, "rb") as handle:
+            pcm = handle.read()
+    else:
+        async with httpx.AsyncClient(timeout=20.0) as client:
+            response = await client.post(
+                f"{ELEVENLABS_BASE_URL}/v1/text-to-speech/{voice_id}",
+                params={"output_format": "pcm_24000"},
+                headers={"xi-api-key": ELEVENLABS_API_KEY, "Accept": "audio/pcm"},
+                json={"text": phrase, "model_id": "eleven_flash_v2_5"},
+            )
+        if response.status_code != 200:
+            detail = response.text[:240] or "no response body"
+            raise HTTPException(502, f"ElevenLabs speaker test failed ({response.status_code}): {detail}")
+        pcm = response.content
+        if len(pcm) < 480:
+            raise HTTPException(502, "ElevenLabs returned empty speaker-test audio")
+        temp_path = cache_path + ".tmp"
+        with open(temp_path, "wb") as handle:
+            handle.write(pcm)
+        os.replace(temp_path, cache_path)
+    return {
+        "mode": "tts", "text": phrase, "tts_provider": provider, "tts_voice": voice_id,
+        "sample_rate": 24000, "audio_pcm_base64": base64.b64encode(pcm).decode("ascii"),
+        "cache_hit": cache_hit,
+    }
 
 
 @app.post("/api/robots/{robot_id}/shell/speaker-test")
-async def robot_speaker_test(robot_id: str, payload: SpeakerTestRequest):
+async def robot_speaker_test(robot_id: str, payload: SpeakerTestRequest):
     """Queue a speaker + mic round-trip test on the robot. Returns a
     request_id; poll /api/robots/{id}/shell/wait/{rid} for the result."""
-    device_id = robot_id
-    params = payload.model_dump(exclude_none=True)
-    request_id = await _queue_shell_request_async(device_id, "speaker_test", None, params)
-    return {"status": "queued", "request_id": request_id, "device_id": device_id}
+    async with aiosqlite.connect(DB_PATH) as db:
+        db.row_factory = aiosqlite.Row
+        async with db.execute(
+            "SELECT id FROM devices WHERE id = ? OR lower(name) = lower(?) "
+            "ORDER BY CASE WHEN id = ? THEN 0 ELSE 1 END LIMIT 1",
+            (robot_id, robot_id, robot_id),
+        ) as cursor:
+            device = await cursor.fetchone()
+    if not device:
+        raise HTTPException(404, f"Robot device not found: {robot_id}")
+    device_id = device["id"]
+    params = payload.model_dump(exclude_none=True)
+    if payload.mode == "tts":
+        params.update(await _cached_speaker_tts(robot_id, payload.text or ""))
+    elif payload.mode != "tone":
+        raise HTTPException(422, "mode must be tone or tts")
+    request_id = await _queue_shell_request_async(device_id, "speaker_test", None, params)
+    return {"status": "queued", "request_id": request_id, "device_id": device_id}
+
+
+@app.get("/api/devices/{device_id}/shell/next-speaker-test")
+async def next_device_speaker_test(device_id: str, authorization: Optional[str] = Header(default=None)):
+    """Allow the always-running preview agent to execute audio tests."""
+    if not await _authorize_device(device_id, authorization):
+        raise HTTPException(401, "Invalid device token")
+    async with aiosqlite.connect(DB_PATH) as db:
+        db.row_factory = aiosqlite.Row
+        async with db.execute(
+            "SELECT id, params FROM shell_requests WHERE device_id = ? AND kind = 'speaker_test' "
+            "AND completed_at IS NULL ORDER BY requested_at LIMIT 1", (device_id,),
+        ) as cursor:
+            row = await cursor.fetchone()
+    return {"request": {"id": row["id"], "params": json.loads(row["params"] or "{}")} if row else None}
 
 
 # ── Incident acknowledgement (C4) ──
@@ -3655,7 +3738,7 @@ async def device_request_session(device_id: str,
     return {
         "session_id": session_id,
         "server_id": server["id"],
-        "server_url": server["url"],
+        "server_url": device["livekit_url"] or server["url"],
         "room_name": room_name,
         "token": token,
         "voice_config": voice_config.model_dump() if voice_config else None,

package/packages/robopark/scheduler/preview_agent.py

@@ -278,7 +278,7 @@ async def _bootstrap_mesh_device(scheduler_url: str, robot_id: str) -> tuple[str
     async with httpx.AsyncClient(headers=_mesh_proxy_headers()) as client:
         response = await client.post(
             f"{scheduler_url.rstrip('/')}/api/devices/bootstrap",
-            json={"name": robot_id, "lan_ip": _get_lan_ip()},
+            json={"name": robot_id, "lan_ip": _get_lan_ip(), "livekit_url": os.getenv("ROBOPARK_LIVEKIT_URL") or None},
             timeout=30.0,
         )
         response.raise_for_status()
@@ -302,8 +302,9 @@ async def _enroll(scheduler_url: str, enrollment_token: str, robot_id: str) -> s
     import socket
     payload = {
         "enrollment_token": enrollment_token,
-        "name": robot_id,
-        "lan_ip": _get_lan_ip(),
+        "name": robot_id,
+        "lan_ip": _get_lan_ip(),
+        "livekit_url": os.getenv("ROBOPARK_LIVEKIT_URL") or None,
     }
     async with httpx.AsyncClient(headers=_mesh_proxy_headers()) as client:
         r = await client.post(
@@ -346,7 +347,8 @@ async def _send_heartbeat(
         async with httpx.AsyncClient() as client:
             response = await client.post(
                 f"{scheduler_url.rstrip('/')}/api/devices/{device_id}/heartbeat",
-                json={"status": "online", "ip": _get_lan_ip(), "device_inventory": inventory},
+                json={"status": "online", "ip": _get_lan_ip(), "device_inventory": inventory,
+                      "livekit_url": os.getenv("ROBOPARK_LIVEKIT_URL") or None},
                 headers=headers,
                 timeout=10.0,
             )
@@ -421,7 +423,8 @@ class PreviewAgent:
         # with a hard ceiling — see DEFAULT_VISION_SILENCE_TIMEOUT above and
         # _check_vision_session() below for the full design.
         self._vision_session_id: Optional[str] = None
-        self._vision_trigger_in_flight = False
+        self._vision_trigger_in_flight = False
+        self._speaker_test_in_flight = False
         self._vision_hard_deadline: float = 0.0
         self._vision_last_activity: float = 0.0
         self.production_mode = False
@@ -820,11 +823,41 @@ class PreviewAgent:
                     await self._report_pipeline("microphone_ready", "ok" if real_inputs else "blocked", f"{len(real_inputs)} microphone device(s) detected")
                     await self._report_pipeline("speaker_ready", "ok" if real_outputs else "blocked", f"{len(real_outputs)} speaker device(s) detected")
                     if not production_mode and self._vision_session_id:
-                        await self._end_vision_session()
+                        await self._end_vision_session()
+                    await self._poll_speaker_test()
             try:
                 await asyncio.wait_for(self._shutdown.wait(), timeout=self.heartbeat_interval)
-            except asyncio.TimeoutError:
-                pass
+            except asyncio.TimeoutError:
+                pass
+
+    async def _poll_speaker_test(self) -> None:
+        """Execute queued speaker tests even when no optional supervisor runs."""
+        if self._speaker_test_in_flight or not self._session or not self.device_id or not self.device_token:
+            return
+        headers = {"Authorization": f"Bearer {self.device_token}", **_mesh_proxy_headers()}
+        try:
+            response = await self._session.get(
+                f"{self.scheduler_url.rstrip('/')}/api/devices/{self.device_id}/shell/next-speaker-test",
+                headers=headers, timeout=5.0,
+            )
+            response.raise_for_status()
+            request = response.json().get("request")
+            if not request:
+                return
+            self._speaker_test_in_flight = True
+            from robot_supervisor import _speaker_roundtrip_test
+            result = await asyncio.to_thread(_speaker_roundtrip_test, request.get("params") or {})
+            result_response = await self._session.post(
+                f"{self.scheduler_url.rstrip('/')}/api/devices/{self.device_id}/supervisor-output",
+                json={"kind": "speaker_test", "service": None, "payload": result,
+                      "request_id": request.get("id")},
+                headers=headers, timeout=8.0,
+            )
+            result_response.raise_for_status()
+        except Exception as e:
+            logger.warning(f"speaker test execution failed: {e}")
+        finally:
+            self._speaker_test_in_flight = False
 
     async def _fetch_preview_state(self) -> PreviewState:
         if not self._session or not self.device_token:
@@ -863,23 +896,25 @@ class PreviewAgent:
             return
         await self._start_publisher(state)
 
-    async def _start_publisher(self, state: PreviewState) -> None:
-        await self._stop_publisher()
-        if not state.url or not state.token or not state.room:
-            return
+    async def _start_publisher(self, state: PreviewState) -> bool:
+        await self._stop_publisher()
+        if not state.url or not state.token or not state.room:
+            return False
         pub = None
         try:
             pub = LiveKitPublisher(state.url, state.token, state.room, self)
             await pub.start()
-            self._publisher = pub
-            logger.info(f"joined preview room {state.room}")
+            self._publisher = pub
+            logger.info(f"joined preview room {state.room}")
+            return True
         except Exception as e:
             logger.error(f"failed to start publisher: {e}")
             if pub is not None:
                 try:
                     await pub.stop()
-                except Exception as cleanup_error:
-                    logger.debug(f"publisher cleanup after start failure: {cleanup_error}")
+                except Exception as cleanup_error:
+                    logger.debug(f"publisher cleanup after start failure: {cleanup_error}")
+            return False
 
     async def _stop_publisher(self) -> None:
         if self._publisher:
@@ -1005,14 +1040,35 @@ class PreviewAgent:
         # and lets downstream components (audio server, vision, etc.) adapt.
         if voice_config and session_id:
             logger.info(f"scheduler voice config for session {session_id}: {voice_config}")
-        await self._start_publisher(state)
-        if session_id and self._session:
-            try:
-                await self._session.post(
-                    f"{self.scheduler_url.rstrip('/')}/api/sessions/{session_id}/joined",
-                    headers={"Authorization": f"Bearer {self.device_token}"},
-                    timeout=10.0,
-                )
+        joined = await self._start_publisher(state)
+        if not joined:
+            await self._report_pipeline(
+                "livekit_join", "failed", "Robot could not connect to its assigned LiveKit route",
+                details={"server_url": state.url}, once=False,
+            )
+            if session_id and self._session:
+                try:
+                    response = await self._session.post(
+                        f"{self.scheduler_url.rstrip('/')}/api/robots/{self.device_id}/end-session",
+                        params={"reason": "livekit_join_failed"},
+                        headers={"Authorization": f"Bearer {self.device_token}"}, timeout=10.0,
+                    )
+                    response.raise_for_status()
+                except Exception as e:
+                    logger.debug(f"could not end failed LiveKit session: {e}")
+            self._vision_session_id = None
+            self._vision_hard_deadline = 0.0
+            self._vision_last_activity = 0.0
+            self._current_state = PreviewState()
+            return
+        if session_id and self._session:
+            try:
+                response = await self._session.post(
+                    f"{self.scheduler_url.rstrip('/')}/api/sessions/{session_id}/joined",
+                    headers={"Authorization": f"Bearer {self.device_token}"},
+                    timeout=10.0,
+                )
+                response.raise_for_status()
             except Exception as e:
                 logger.debug(f"could not mark session joined: {e}")
 

package/packages/robopark/scheduler/robot_supervisor.py

@@ -30,7 +30,8 @@ Auto-start on boot:
 """
 from __future__ import annotations
 
-import argparse
+import argparse
+import base64
 import json
 import logging
 import math
@@ -493,7 +494,7 @@ def _resolve_audio_device(pa, selected: str, kind: str) -> Optional[int]:
     return None
 
 
-def _speaker_roundtrip_test(params: dict) -> dict:
+def _speaker_roundtrip_test(params: dict) -> dict:
     """Play a test tone + record the mic simultaneously; return metrics.
 
     params may include {frequency, duration, amplitude, output, input,
@@ -502,20 +503,32 @@ def _speaker_roundtrip_test(params: dict) -> dict:
         import pyaudio
     except ImportError as e:
         return {"ok": False, "error": f"pyaudio not installed on this robot: {e}"}
-    freq = float(params.get("frequency", SPEAKER_TEST_FREQUENCY_HZ))
-    dur = float(params.get("duration", SPEAKER_TEST_DURATION_S))
+    mode = str(params.get("mode", "tone")).lower()
+    freq = float(params.get("frequency", SPEAKER_TEST_FREQUENCY_HZ))
+    dur = float(params.get("duration", SPEAKER_TEST_DURATION_S))
     amp = float(params.get("amplitude", SPEAKER_TEST_AMPLITUDE))
     threshold_db = float(params.get("threshold_db", SPEAKER_TEST_PEAK_DB_THRESHOLD))
     out_name = params.get("output", os.environ.get("ROBOPARK_AUDIO_OUTPUT") or SPEAKER_TEST_OUTPUT_DEVICE)
     in_name = params.get("input", os.environ.get("ROBOPARK_AUDIO_INPUT") or SPEAKER_TEST_INPUT_DEVICE)
     sample_rate = int(params.get("sample_rate", SPEAKER_TEST_SAMPLE_RATE))
-    if not (50.0 <= freq <= 8000.0):
+    if mode not in ("tone", "tts"):
+        return {"ok": False, "error": f"unsupported speaker test mode: {mode}"}
+    if mode == "tone" and not (50.0 <= freq <= 8000.0):
         return {"ok": False, "error": f"frequency {freq}Hz out of allowed range (50..8000)"}
     if not (0.1 <= dur <= 3.0):
         return {"ok": False, "error": f"duration {dur}s out of allowed range (0.1..3.0)"}
     if not (0.05 <= amp <= 1.0):
         return {"ok": False, "error": f"amplitude {amp} out of allowed range (0.05..1.0)"}
-    n_samples = int(sample_rate * dur)
+    mono_pcm = None
+    if mode == "tts":
+        try:
+            mono_pcm = base64.b64decode(params.get("audio_pcm_base64") or "", validate=True)
+        except Exception as e:
+            return {"ok": False, "error": f"invalid cached TTS audio: {e}"}
+        if len(mono_pcm) < 480 or len(mono_pcm) % 2:
+            return {"ok": False, "error": "cached TTS audio is empty or malformed"}
+        dur = len(mono_pcm) / 2 / sample_rate
+    n_samples = int(sample_rate * dur)
     if n_samples < 480:
         return {"ok": False, "error": "duration too short"}
 
@@ -529,13 +542,20 @@ def _speaker_roundtrip_test(params: dict) -> dict:
     out_label = _dev_name(out_idx)
     in_label = _dev_name(in_idx)
 
-    frames = bytearray()
-    peak_value = int(32767 * amp)
-    for n in range(n_samples):
-        env = min(1.0, n / (sample_rate * 0.004), (n_samples - n) / (sample_rate * 0.008))
-        value = int(peak_value * env * math.sin(2 * math.pi * freq * n / sample_rate))
-        frames.extend(struct.pack("<hh", value, value))
-    raw = bytes(frames)
+    if mono_pcm is not None:
+        frames = bytearray()
+        for pos in range(0, len(mono_pcm), 2):
+            sample = mono_pcm[pos:pos + 2]
+            frames.extend(sample + sample)
+        raw = bytes(frames)
+    else:
+        frames = bytearray()
+        peak_value = int(32767 * amp)
+        for n in range(n_samples):
+            env = min(1.0, n / (sample_rate * 0.004), (n_samples - n) / (sample_rate * 0.008))
+            value = int(peak_value * env * math.sin(2 * math.pi * freq * n / sample_rate))
+            frames.extend(struct.pack("<hh", value, value))
+        raw = bytes(frames)
 
     recorded_peak = 0
     recorded_rms = 0.0
@@ -614,9 +634,14 @@ def _speaker_roundtrip_test(params: dict) -> dict:
     else:
         rms_db = -120.0
     pass_ = peak_db >= threshold_db
-    return {
-        "ok": True,
-        "pass": pass_,
+    return {
+        "ok": True,
+        "pass": pass_,
+        "mode": mode,
+        "text": params.get("text"),
+        "cache_hit": bool(params.get("cache_hit")),
+        "tts_provider": params.get("tts_provider"),
+        "tts_voice": params.get("tts_voice"),
         "frequency": freq,
         "duration": dur,
         "duration_ms": duration_ms,