infinicode 2.8.95 → 2.8.97

package/docs/ROBOPARK_PRODUCTION_MEMORY.md

@@ -160,3 +160,21 @@ Required result: `playback_started  ok`.
   whether motion creates a conversation, not whether the camera worker runs.
 - `auto`, `default`, and `first` resolve to `/dev/robopark-camera` when present,
   otherwise `/dev/video0`; those literal strings are never passed to OpenCV.
+
+## Audio Ownership Contract (2.8.96+)
+
+- Fleet microphone selection is resolved by stable USB product label on every
+  inventory refresh. The current stack maps microphone to `hw:3,0` and speaker
+  to `hw:2,0`.
+- A LiveKit microphone track is healthy only after the robot captures and
+  publishes real PCM. Publishing an empty track is not readiness.
+- Linux capture probes `plughw` at 48 kHz, falls back to 44.1 kHz with
+  resampling, and releases the microphone lock after every failed attempt.
+- Pipeline failures include the selected device and exact ALSA startup error.
+  A flat browser meter means missing PCM, not a quiet room.
+- Remote `speaker-test` requests support `mode=mic_groundtruth`, which captures
+  through the same `arecord -D plughw:X,Y -f S16_LE -r 48000 -c 1` command
+  proven onsite and returns bytes, RMS, peak, and nonzero sample counts.
+- Python 3.13 removed `audioop`. Version 2.8.97 replaces microphone gain, peak
+  measurement, and fallback resampling with internal PCM16 helpers; production
+  capture must not import `audioop`.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "infinicode",
-  "version": "2.8.95",
+  "version": "2.8.97",
   "description": "OpenKernel — provider-agnostic AI execution kernel. Native coding agent + mission-driven execution runtime.",
   "type": "module",
   "main": "./dist/kernel/index.js",

package/packages/robopark/scheduler/preview_agent.py

@@ -97,7 +97,7 @@ def _resolve_inventory_audio(items: list, selected: object, preferred: str) -> s
     return str(match.get("name")) if match and match.get("name") else selected_text
 
 
-def _normalize_livekit_url(url: Optional[str]) -> Optional[str]:
+def _normalize_livekit_url(url: Optional[str]) -> Optional[str]:
     """Avoid Windows localhost IPv6/IPv4 ambiguity for local LiveKit."""
     if not url:
         return url
@@ -105,9 +105,54 @@ def _normalize_livekit_url(url: Optional[str]) -> Optional[str]:
         prefix = f"{scheme}://localhost"
         if url.startswith(prefix):
             return f"{scheme}://127.0.0.1" + url[len(prefix):]
-    return url
-
-
+    return url
+
+
+def _pcm16_scale_and_peak(data: bytes, gain: float) -> tuple[bytes, int]:
+    """Apply gain and measure peak without audioop (removed in Python 3.13)."""
+    from array import array
+
+    samples = array("h")
+    samples.frombytes(data[:len(data) - (len(data) % 2)])
+    if sys.byteorder != "little":
+        samples.byteswap()
+    peak = 0
+    for index, value in enumerate(samples):
+        scaled = max(-32768, min(32767, int(value * gain))) if gain != 1.0 else value
+        samples[index] = scaled
+        peak = max(peak, abs(scaled))
+    if sys.byteorder != "little":
+        samples.byteswap()
+    return samples.tobytes(), peak
+
+
+def _pcm16_resample_mono(data: bytes, source_rate: int, target_rate: int) -> bytes:
+    """Linearly resample a PCM16 mono chunk using only the standard library."""
+    from array import array
+
+    if source_rate == target_rate or len(data) < 4:
+        return data
+    source = array("h")
+    source.frombytes(data[:len(data) - (len(data) % 2)])
+    if sys.byteorder != "little":
+        source.byteswap()
+    target_count = max(1, round(len(source) * target_rate / source_rate))
+    target = array("h", [0]) * target_count
+    scale = source_rate / target_rate
+    last = len(source) - 1
+    for index in range(target_count):
+        position = min(last, index * scale)
+        left = int(position)
+        right = min(last, left + 1)
+        fraction = position - left
+        target[index] = max(-32768, min(32767, round(
+            source[left] + (source[right] - source[left]) * fraction
+        )))
+    if sys.byteorder != "little":
+        target.byteswap()
+    return target.tobytes()
+
+
 def _play_audio_effect(selected_output: str | None, effect: str) -> None:
     """Play a short local cue without involving the voice pipeline."""
     sample_rate = 48000
@@ -1238,6 +1283,7 @@ class LiveKitPublisher:
         self._capture: Optional["VideoCapture"] = None
         self._mic_capture: Optional["AudioCapture"] = None
         self._mic_streaming = asyncio.Event()
+        self._mic_error: Optional[str] = None
         # Motion sampling state belongs to the publisher instance.  Keeping it
         # initialized here prevents shutdown/reopen paths from raising while
         # the camera is being handed between preview and voice sessions.
@@ -1335,10 +1381,16 @@ class LiveKitPublisher:
             try:
                 await asyncio.wait_for(self._mic_streaming.wait(), timeout=5.0)
             except asyncio.TimeoutError:
+                detail = self._mic_error or f"no PCM received from {self.agent.audio_capture_device}"
                 await self.agent._report_pipeline(
-                    "microphone_published", "blocked", "Microphone track published but PCM capture did not start"
+                    "microphone_published", "blocked",
+                    f"Microphone track published but PCM capture did not start: {detail}",
+                    {"device": self.agent.audio_capture_device, "error": detail},
+                )
+                logger.warning(
+                    "microphone PCM is not ready (%s); keeping the session alive while capture retries",
+                    detail,
                 )
-                logger.warning("microphone PCM is not ready; keeping the session alive while capture retries")
 
         if self._capture and self.video_source:
             self._tasks.append(asyncio.create_task(self._video_loop()))
@@ -1688,6 +1740,7 @@ class LiveKitPublisher:
             except asyncio.CancelledError:
                 raise
             except Exception as e:
+                self._mic_error = str(e)
                 logger.warning(
                     f"audio_loop: capture failed on {self.agent.audio_capture_device}: {e}; retrying"
                 )
@@ -1709,7 +1762,8 @@ class LiveKitPublisher:
             logger.warning("audio_loop: create_audio_capture returned None, mic will not publish")
             return
         self._mic_capture = mic
-        logger.info(f"audio_loop: mic capture started ({type(mic).__name__})")
+        self._mic_error = None
+        logger.info(f"audio_loop: mic capture started ({type(mic).__name__})")
         # Read in bigger batches (100ms) instead of one 20ms frame per
         # asyncio.to_thread() dispatch. Each dispatch/poll cycle has a fixed
         # overhead (~30ms observed on this machine) that dominates when the
@@ -1724,7 +1778,6 @@ class LiveKitPublisher:
         samples_per_frame = int(48000 * frame_ms / 1000)
         samples_per_batch = int(48000 * BATCH_MS / 1000)
         bytes_per_frame = samples_per_frame * 2  # int16 mono
-        import audioop
         mic_gain = max(1.0, min(float(os.getenv("ROBOPARK_MIC_GAIN", "4.0")), 12.0))
         _diag_peak = 0
         _diag_count = 0
@@ -1742,8 +1795,7 @@ class LiveKitPublisher:
                 _pub_start = time.monotonic()
                 for off in range(0, len(data) - bytes_per_frame + 1, bytes_per_frame):
                     chunk = data[off:off + bytes_per_frame]
-                    if mic_gain != 1.0:
-                        chunk = audioop.mul(chunk, 2, mic_gain)
+                    chunk, p = _pcm16_scale_and_peak(chunk, mic_gain)
                     frame = self.rtc.AudioFrame(
                         data=chunk, sample_rate=48000, num_channels=1, samples_per_channel=samples_per_frame,
                     )
@@ -1754,8 +1806,7 @@ class LiveKitPublisher:
                             "microphone_published", "ok",
                             "Microphone track published with live PCM frames",
                         )
-                    p = audioop.max(chunk, 2)
-                    _diag_peak = max(_diag_peak, p)
+                    _diag_peak = max(_diag_peak, p)
                     _diag_count += 1
                 _diag_read_ms += (_t1 - _t0) * 1000
                 _diag_publish_ms += (time.monotonic() - _pub_start) * 1000
@@ -1949,46 +2000,85 @@ class AlsaAudioCapture(AudioCapture):
     """Capture Linux PCM through the same ALSA path used by onsite tests."""
 
     def __init__(self, device: str):
+        import os
         import re
+        import select
         import subprocess
         from media_lock import media_lock
 
         match = re.search(r"\b(hw:\d+,\d+)\b", device)
         if not match:
             raise ValueError(f"no ALSA hardware address in {device!r}")
-        alsa_device = f"plug{match.group(1)}"
         self.buffer = bytearray()
-        self.media_guard = media_lock("microphone", timeout=5.0).acquire()
-        try:
-            self.process = subprocess.Popen(
-                [
-                    "arecord", "-q", "-D", alsa_device, "-t", "raw",
-                    "-f", "S16_LE", "-r", "48000", "-c", "1",
-                ],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
-            )
-        except Exception:
-            self.media_guard.release()
-            raise
-        if self.process.stdout is None:
-            self.process.kill()
-            self.process.wait(timeout=1)
+        self.media_guard = media_lock("microphone", timeout=1.5).acquire()
+        self.process = None
+        self.source_rate = 48000
+        errors = []
+        # The fleet USB microphone normally accepts 48 kHz through ALSA's
+        # plug layer. Some firmware revisions expose only native 44.1 kHz;
+        # accept that rate and resample below rather than publishing silence.
+        for source_rate in (48000, 44100):
+            alsa_device = f"plug{match.group(1)}"
+            process = None
+            try:
+                process = subprocess.Popen(
+                    [
+                        "arecord", "-q", "-D", alsa_device, "-t", "raw",
+                        "-f", "S16_LE", "-r", str(source_rate), "-c", "1",
+                        "--period-size", str(max(256, source_rate // 50)),
+                    ],
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    bufsize=0,
+                )
+                if process.stdout is None:
+                    raise RuntimeError("arecord did not provide a PCM stream")
+                ready, _, _ = select.select([process.stdout], [], [], 2.0)
+                if not ready:
+                    if process.poll() is None:
+                        raise TimeoutError("arecord produced no PCM within 2 seconds")
+                    detail = process.stderr.read().decode("utf-8", errors="replace").strip() if process.stderr else ""
+                    raise OSError(detail or f"arecord exited {process.returncode}")
+                first = os.read(process.stdout.fileno(), max(2048, source_rate // 25 * 2))
+                if not first:
+                    detail = process.stderr.read().decode("utf-8", errors="replace").strip() if process.stderr else ""
+                    raise OSError(detail or "arecord returned an empty PCM frame")
+                self.process = process
+                self.source_rate = source_rate
+                self.alsa_device = alsa_device
+                if source_rate != 48000:
+                    first = _pcm16_resample_mono(first, source_rate, 48000)
+                self.buffer.extend(first)
+                break
+            except Exception as exc:
+                errors.append(f"{source_rate}Hz: {exc}")
+                if process is not None:
+                    if process.poll() is None:
+                        process.terminate()
+                    try:
+                        process.wait(timeout=1)
+                    except subprocess.TimeoutExpired:
+                        process.kill()
+                        process.wait(timeout=1)
+        if self.process is None:
             self.media_guard.release()
-            raise RuntimeError("arecord did not provide a PCM stream")
-        self.alsa_device = alsa_device
+            raise OSError(f"ALSA capture failed on plug{match.group(1)} ({'; '.join(errors)})")
 
     def read(self, samples_per_frame: int):
+        import os
         from livekit import rtc
 
         bytes_needed = samples_per_frame * 2
+        source_bytes_needed = max(2, int(samples_per_frame * self.source_rate / 48000) * 2)
         while len(self.buffer) < bytes_needed:
-            chunk = self.process.stdout.read(bytes_needed - len(self.buffer))
+            chunk = os.read(self.process.stdout.fileno(), source_bytes_needed)
             if not chunk:
                 detail = ""
                 if self.process.stderr is not None:
                     detail = self.process.stderr.read().decode("utf-8", errors="replace").strip()
                 raise OSError(f"ALSA capture stopped on {self.alsa_device}: {detail or 'no PCM data'}")
+            if self.source_rate != 48000:
+                chunk = _pcm16_resample_mono(chunk, self.source_rate, 48000)
             self.buffer.extend(chunk)
         data = bytes(self.buffer[:bytes_needed])
         del self.buffer[:bytes_needed]

package/packages/robopark/scheduler/robot_supervisor.py

@@ -626,15 +626,92 @@ def _linux_aplay_pcm16(raw: bytes, selected_output: str, sample_rate: int,
     return False, f"{alsa_device}: {last_error}"
 
 
+def _linux_mic_groundtruth(selected_input: str, duration: float = 4.0) -> dict:
+    """Capture the fleet USB mic through the exact onsite-proven ALSA path."""
+    import re
+
+    match = re.search(r"hw:(\d+),(\d+)", str(selected_input), re.IGNORECASE)
+    if not match:
+        return {"ok": False, "error": "selected input has no ALSA hw address"}
+    alsa_device = f"plughw:{match.group(1)},{match.group(2)}"
+    seconds = max(1, min(10, int(round(duration))))
+    command = [
+        "arecord", "-q", "-D", alsa_device, "-t", "raw", "-f", "S16_LE",
+        "-r", "48000", "-c", "1", "-d", str(seconds),
+    ]
+    started = time.monotonic()
+    try:
+        from media_lock import media_lock
+        with media_lock("microphone", timeout=8.0):
+            completed = subprocess.run(
+                command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                timeout=seconds + 4.0,
+                check=False,
+            )
+    except Exception as exc:
+        return {
+            "ok": False,
+            "mode": "mic_groundtruth",
+            "error": f"{type(exc).__name__}: {exc}",
+            "input_device": alsa_device,
+            "duration_ms": int((time.monotonic() - started) * 1000),
+        }
+    raw = completed.stdout or b""
+    stderr = completed.stderr.decode("utf-8", errors="replace").strip()
+    if completed.returncode != 0 or len(raw) < 2:
+        return {
+            "ok": False,
+            "mode": "mic_groundtruth",
+            "error": stderr or f"arecord exited {completed.returncode}",
+            "exit_code": completed.returncode,
+            "bytes": len(raw),
+            "input_device": alsa_device,
+            "command": " ".join(command),
+            "duration_ms": int((time.monotonic() - started) * 1000),
+        }
+    sample_count = len(raw) // 2
+    peak = 0
+    square_sum = 0
+    nonzero = 0
+    for (value,) in struct.iter_unpack("<h", raw[:sample_count * 2]):
+        magnitude = abs(value)
+        peak = max(peak, magnitude)
+        square_sum += value * value
+        if value:
+            nonzero += 1
+    rms = (square_sum / sample_count) ** 0.5
+    peak_db = round(20.0 * math.log10(max(1, peak) / 32767.0), 1)
+    rms_db = round(20.0 * math.log10(max(1.0, rms) / 32767.0), 1)
+    return {
+        "ok": True,
+        "pass": nonzero > 0 and peak > 0,
+        "mode": "mic_groundtruth",
+        "exit_code": completed.returncode,
+        "bytes": len(raw),
+        "samples": sample_count,
+        "nonzero_samples": nonzero,
+        "nonzero_percent": round(nonzero * 100.0 / sample_count, 3),
+        "recorded_peak": peak,
+        "recorded_peak_db": peak_db,
+        "recorded_rms": round(rms, 2),
+        "recorded_rms_db": rms_db,
+        "sample_rate": 48000,
+        "channels": 1,
+        "input_device": alsa_device,
+        "command": " ".join(command),
+        "stderr": stderr,
+        "duration_ms": int((time.monotonic() - started) * 1000),
+        "diagnostic": "ground truth captured through the onsite-proven ALSA arecord path",
+    }
+
+
 def _speaker_roundtrip_test(params: dict) -> dict:
     """Play a test tone + record the mic simultaneously; return metrics.
 
     params may include {frequency, duration, amplitude, output, input,
     threshold_db}. Anything missing falls back to the module constants."""
-    try:
-        import pyaudio
-    except ImportError as e:
-        return {"ok": False, "error": f"pyaudio not installed on this robot: {e}"}
     mode = str(params.get("mode", "tone")).lower()
     playback_only = bool(params.get("playback_only", mode == "tts"))
     freq = float(params.get("frequency", SPEAKER_TEST_FREQUENCY_HZ))
@@ -646,8 +723,16 @@ def _speaker_roundtrip_test(params: dict) -> dict:
     out_name = params.get("output_name") or params.get("output", os.environ.get("ROBOPARK_AUDIO_OUTPUT") or SPEAKER_TEST_OUTPUT_DEVICE)
     in_name = params.get("input_name") or params.get("input", os.environ.get("ROBOPARK_AUDIO_INPUT") or SPEAKER_TEST_INPUT_DEVICE)
     source_rate = int(params.get("sample_rate", SPEAKER_TEST_SAMPLE_RATE))
+    if mode == "mic_groundtruth":
+        if not sys.platform.startswith("linux"):
+            return {"ok": False, "error": "mic_groundtruth requires Linux ALSA"}
+        return _linux_mic_groundtruth(str(in_name), float(params.get("duration", 4.0)))
     if mode not in ("tone", "tts"):
         return {"ok": False, "error": f"unsupported speaker test mode: {mode}"}
+    try:
+        import pyaudio
+    except ImportError as e:
+        return {"ok": False, "error": f"pyaudio not installed on this robot: {e}"}
     if mode == "tone" and not (50.0 <= freq <= 8000.0):
         return {"ok": False, "error": f"frequency {freq}Hz out of allowed range (50..8000)"}
     if not (0.1 <= dur <= 3.0):