infinicode 2.8.37 → 2.8.38

package/packages/robopark/scheduler/robot_supervisor.py

@@ -33,8 +33,10 @@ from __future__ import annotations
 import argparse
 import json
 import logging
+import math
 import os
 import signal
+import struct
 import subprocess
 import sys
 import time
@@ -220,23 +222,422 @@ def _terminate(state: ServiceState) -> None:
 
 
 def _execute_command(state: ServiceState, action: str) -> None:
-    """Operator-initiated action from the dashboard. Currently only
-    "restart" -- terminate if running (the main loop's own restart logic
-    then respawns it), and clear any crash backoff since this isn't a
-    crash, it's a deliberate request that should take effect right away."""
-    if action != "restart":
-        logger.warning(f"ignoring unknown remote command {action!r} for {state.spec.name}")
+    """Operator-initiated action from the dashboard. Supports:
+      - "restart" (original): terminate if running, the main loop
+        respawns it on the next tick.
+      - "start": spawn the service if it's not already running. Used
+        after a deliberate stop.
+      - "stop": graceful terminate and mark `stopped=True` so the main
+        loop does NOT auto-restart it. Re-enable with "start".
+    Anything else is ignored with a warning."""
+    if action == "restart":
+        # clear any deliberate-stop so the next main-loop tick is allowed
+        # to respawn it
+        state.stopped = False
+        if state.proc is not None:
+            _terminate(state)
+            state.proc = None
+        state.failure_count = 0
+        state.next_restart_at = 0.0
         return
-    logger.info(f"remote restart requested for {state.spec.name}")
-    if state.proc is not None:
-        _terminate(state)
-        state.proc = None
-    state.failure_count = 0
-    state.next_restart_at = 0.0  # due immediately -- the main loop respawns it next tick
+    if action == "start":
+        state.stopped = False
+        state.failure_count = 0
+        state.next_restart_at = 0.0
+        if state.proc is None:
+            _spawn(state, _CURRENT_LOG_DIR or LOG_DIR)
+        return
+    if action == "stop":
+        if state.proc is not None:
+            _terminate(state)
+            state.proc = None
+        state.stopped = True
+        return
+    logger.warning(f"ignoring unknown remote command {action!r} for {state.spec.name}")
+
+
+# ── Operator shell (C1 gap-fill) ──
+# Allows the dashboard to read service logs and run a small set of
+# pre-approved diagnostic commands on the robot. Results are POSTed back
+# to the scheduler (POST /api/devices/{id}/supervisor-output) which
+# caches them per (device, command-id) so the dashboard can poll the
+# result without holding a long-lived connection to the robot.
+#
+# SECURITY: the allowlist below is the only thing that gets executed.
+# Operators cannot pass arbitrary commands; the parser rejects anything
+# not on this list with a 400-equivalent ("command not allowed").
+
+import secrets as _secrets  # noqa: E402
+
+# A list of allow-listed command specs. Each entry has a fixed argv;
+# placeholders in <angle brackets> get substituted from the request.
+# A request that doesn't match any entry is rejected.
+SHELL_ALLOWLIST = [
+    {"name": "uptime",        "argv": ["uptime"]},
+    {"name": "free",          "argv": ["free", "-m"]},
+    {"name": "df",            "argv": ["df", "-h"]},
+    {"name": "ps",            "argv": ["ps", "aux"]},
+    {"name": "uname",         "argv": ["uname", "-a"]},
+    {"name": "date",          "argv": ["date"]},
+    {"name": "whoami",        "argv": ["whoami"]},
+    {"name": "hostname",      "argv": ["hostname"]},
+    {"name": "ip",            "argv": ["ip", "addr"]},
+    {"name": "ss",            "argv": ["ss", "-tlnp"]},
+    {"name": "os-release",    "argv": ["cat", "/etc/os-release"]},
+    {"name": "ls-logs",       "argv": ["ls", "-la", "<dir>"]},
+    {"name": "systemctl-status", "argv": ["systemctl", "status", "<name>", "--no-pager", "-n", "30"]},
+    {"name": "journalctl",    "argv": ["journalctl", "-u", "<name>", "-n", "200", "--no-pager"]},
+    {"name": "tail",          "argv": ["tail", "-n", "<n>", "<path>"]},
+]
+
+# Cap output size per command to avoid an operator accidentally filling
+# the supervisor output buffer with 100MB of `ps aux`.
+SHELL_OUTPUT_MAX_BYTES = 64 * 1024
+SHELL_RUN_TIMEOUT_SECONDS = 8.0
+
+
+def _run_shell_command(name: str, params: dict) -> dict:
+    """Resolve `name` against the allowlist, substitute params, and run.
+
+    Returns a dict {ok, exit_code, stdout, stderr, error}. The caller
+    POSTs this to the scheduler as the supervisor-output payload."""
+    spec = None
+    for s in SHELL_ALLOWLIST:
+        if s["name"] == name:
+            spec = s
+            break
+    if spec is None:
+        return {"ok": False, "error": f"command {name!r} is not in the allowlist"}
+    argv = []
+    for piece in spec["argv"]:
+        if piece.startswith("<") and piece.endswith(">"):
+            key = piece[1:-1]
+            val = params.get(key)
+            if val is None or not isinstance(val, str) or not val.strip():
+                return {"ok": False, "error": f"missing required parameter {key!r} for command {name!r}"}
+            # Reject anything that smells like a shell metachar — keep it
+            # strictly to filenames / unit names. Service names are owned
+            # by supervisor.json on this robot, not by the network.
+            if any(c in val for c in ("\x00", "\n", "\r")):
+                return {"ok": False, "error": "invalid characters in parameter"}
+            argv.append(val)
+        else:
+            argv.append(piece)
+    try:
+        proc = subprocess.run(
+            argv,
+            capture_output=True,
+            text=True,
+            timeout=SHELL_RUN_TIMEOUT_SECONDS,
+            check=False,
+        )
+        out = (proc.stdout or "")[:SHELL_OUTPUT_MAX_BYTES]
+        err = (proc.stderr or "")[:SHELL_OUTPUT_MAX_BYTES]
+        return {
+            "ok": True,
+            "exit_code": proc.returncode,
+            "stdout": out,
+            "stderr": err,
+            "truncated": len(proc.stdout or "") > SHELL_OUTPUT_MAX_BYTES or len(proc.stderr or "") > SHELL_OUTPUT_MAX_BYTES,
+        }
+    except subprocess.TimeoutExpired:
+        return {"ok": False, "error": f"command timed out after {SHELL_RUN_TIMEOUT_SECONDS}s"}
+    except FileNotFoundError as e:
+        return {"ok": False, "error": f"command not found: {e}"}
+    except Exception as e:
+        return {"ok": False, "error": f"{type(e).__name__}: {e}"}
+
+
+def _tail_log_file(path: Path, lines: int) -> dict:
+    """Return the last `lines` lines of a log file as a string.
+
+    Resolves `path` against LOG_DIR if it's relative; rejects anything
+    that tries to escape (../) for safety, even though the operator
+    can already read anything on the robot via the allowlist."""
+    try:
+        lines = max(1, min(int(lines), 2000))
+    except (TypeError, ValueError):
+        lines = 200
+    if not path.is_absolute():
+        path = (LOG_DIR / path).resolve()
+    # Belt-and-suspenders: don't let `..` traversal escape the log dir
+    # (the dashboard only ever sends a service name, but defend anyway).
+    try:
+        path.relative_to(LOG_DIR.resolve())
+    except ValueError:
+        return {"ok": False, "error": "log path is outside the supervisor log directory"}
+    if not path.exists():
+        return {"ok": False, "error": f"no such log file: {path}"}
+    try:
+        # Read the tail efficiently: seek to ~32KB from the end and split.
+        size = path.stat().st_size
+        chunk = min(size, 64 * 1024)
+        with path.open("rb") as f:
+            if size > chunk:
+                f.seek(size - chunk)
+            data = f.read().decode("utf-8", errors="replace")
+        all_lines = data.splitlines()
+        tail = all_lines[-lines:]
+        return {"ok": True, "path": str(path), "lines": len(tail), "total_lines": len(all_lines), "content": "\n".join(tail)}
+    except Exception as e:
+        return {"ok": False, "error": f"{type(e).__name__}: {e}"}
+
+
+def _post_supervisor_output(identity, kind: str, service: Optional[str], payload: dict, request_id: Optional[str] = None) -> None:
+    """Send the result of a tail_logs or shell_run back to the scheduler.
+
+    Uses the same httpx call style as the status report itself; no
+    retry, no queue — if the scheduler is down we drop the result.
+    The dashboard times out and reports a clear error in that case."""
+    device_id, scheduler_url, token = identity
+    try:
+        import httpx
+    except ImportError:
+        return
+    url = f"{scheduler_url.rstrip('/')}/api/devices/{device_id}/supervisor-output"
+    body = {"kind": kind, "service": service, "payload": payload, "request_id": request_id}
+    try:
+        httpx.post(url, json=body, headers={"Authorization": f"Bearer {token}"}, timeout=5.0)
+    except Exception as e:
+        logger.debug(f"supervisor-output POST failed: {e}")
+
+
+def _handle_shell_command(identity, cmd: dict) -> None:
+    """Process a single shell/log request returned by the scheduler.
+
+    `cmd` is {id, kind: "shell_run"|"tail_logs"|"speaker_test", service,
+    params}. Runs the request, posts the result back, and returns."""
+    kind = cmd.get("kind", "shell_run")
+    service = cmd.get("service")
+    request_id = cmd.get("id")
+    params = cmd.get("params") or {}
+    if kind == "tail_logs":
+        # params: {lines: N, path?: override path}
+        path_param = params.get("path")
+        if path_param:
+            log_path = Path(path_param)
+        elif service:
+            log_path = (_CURRENT_LOG_DIR or LOG_DIR) / f"{service}.log"
+        else:
+            _post_supervisor_output(identity, kind, service, {"ok": False, "error": "tail_logs requires service or path"}, request_id)
+            return
+        result = _tail_log_file(log_path, int(params.get("lines", 200)))
+    elif kind == "shell_run":
+        result = _run_shell_command(params.get("name", ""), params)
+    elif kind == "speaker_test":
+        result = _speaker_roundtrip_test(params)
+    else:
+        result = {"ok": False, "error": f"unknown shell command kind: {kind!r}"}
+    _post_supervisor_output(identity, kind, service, result, request_id)
+
+
+# ── Speaker roundtrip test (C2 gap-fill) ──
+# Plays a short tone through the robot's configured output device while
+# simultaneously recording from the configured input device. Returns
+# {played, recorded_rms, peak_db, duration_ms, output_device,
+# input_device, error}.
+#
+# Used by the dashboard's "Test speaker" button. Helps catch the
+# classic failure modes: speaker muted, wrong output device, mic
+# pointing the wrong way, USB audio device unplugged. The result is
+# a quick PASS/FAIL plus a peak-dB readout that the operator can
+# read at a glance.
+
+# Default test tone parameters — overridden by per-request params
+SPEAKER_TEST_FREQUENCY_HZ = 1000.0
+SPEAKER_TEST_DURATION_S = 0.6
+SPEAKER_TEST_SAMPLE_RATE = 48000
+SPEAKER_TEST_AMPLITUDE = 0.6   # 0..1; conservative so it doesn't clip
+SPEAKER_TEST_OUTPUT_DEVICE = None  # None = use ROBOPARK_AUDIO_OUTPUT / default
+SPEAKER_TEST_INPUT_DEVICE = None
+# Peak dB threshold: anything below this is reported as "no signal
+# detected" (PASS means the robot heard the tone back; FAIL means
+# the mic didn't pick it up). The default (-30 dB) is conservative
+# for a quiet indoor environment; can be overridden per request.
+SPEAKER_TEST_PEAK_DB_THRESHOLD = -30.0
+
+
+def _resolve_audio_device(pa, selected: str, kind: str) -> Optional[int]:
+    """Resolve a free-form device name/index to a PyAudio device index.
+
+    kind is "input" or "output". None / "default" picks the WASAPI
+    default device (per the same logic preview_agent.py uses for the
+    mic). If the name doesn't match any device, returns None and lets
+    PyAudio pick its global default — that may still work, but the
+    result is recorded so the operator can see it."""
+    if selected is None:
+        selected = "default"
+    s = str(selected).strip()
+    if s == "" or s.lower() == "default":
+        try:
+            wasapi = pa.get_host_api_info_by_type(pyaudio.paWASAPI)
+            key = "defaultInputDevice" if kind == "input" else "defaultOutputDevice"
+            idx = wasapi.get(key)
+            if idx is not None and idx >= 0:
+                return idx
+        except Exception:
+            pass
+        return None
+    if s.isdigit():
+        return int(s)
+    needle = s.lower()
+    want_channels = 1 if kind == "input" else 0  # input: must have >0
+    for i in range(pa.get_device_count()):
+        info = pa.get_device_info_by_index(i)
+        if kind == "output" and info.get("maxOutputChannels", 0) <= 0:
+            continue
+        if kind == "input" and info.get("maxInputChannels", 0) <= 0:
+            continue
+        if needle in str(info.get("name", "")).lower():
+            return i
+    return None
+
+
+def _speaker_roundtrip_test(params: dict) -> dict:
+    """Play a test tone + record the mic simultaneously; return metrics.
+
+    params may include {frequency, duration, amplitude, output, input,
+    threshold_db}. Anything missing falls back to the module constants."""
+    try:
+        import pyaudio
+    except ImportError as e:
+        return {"ok": False, "error": f"pyaudio not installed on this robot: {e}"}
+    freq = float(params.get("frequency", SPEAKER_TEST_FREQUENCY_HZ))
+    dur = float(params.get("duration", SPEAKER_TEST_DURATION_S))
+    amp = float(params.get("amplitude", SPEAKER_TEST_AMPLITUDE))
+    threshold_db = float(params.get("threshold_db", SPEAKER_TEST_PEAK_DB_THRESHOLD))
+    out_name = params.get("output", os.environ.get("ROBOPARK_AUDIO_OUTPUT") or SPEAKER_TEST_OUTPUT_DEVICE)
+    in_name = params.get("input", os.environ.get("ROBOPARK_AUDIO_INPUT") or SPEAKER_TEST_INPUT_DEVICE)
+    sample_rate = int(params.get("sample_rate", SPEAKER_TEST_SAMPLE_RATE))
+    if not (50.0 <= freq <= 8000.0):
+        return {"ok": False, "error": f"frequency {freq}Hz out of allowed range (50..8000)"}
+    if not (0.1 <= dur <= 3.0):
+        return {"ok": False, "error": f"duration {dur}s out of allowed range (0.1..3.0)"}
+    if not (0.05 <= amp <= 1.0):
+        return {"ok": False, "error": f"amplitude {amp} out of allowed range (0.05..1.0)"}
+    n_samples = int(sample_rate * dur)
+    if n_samples < 480:
+        return {"ok": False, "error": "duration too short"}
+
+    pa = pyaudio.PyAudio()
+    out_idx = _resolve_audio_device(pa, out_name, "output")
+    in_idx = _resolve_audio_device(pa, in_name, "input")
+    def _dev_name(idx):
+        if idx is None: return "(default)"
+        try: return pa.get_device_info_by_index(idx).get("name", str(idx))
+        except Exception: return str(idx)
+    out_label = _dev_name(out_idx)
+    in_label = _dev_name(in_idx)
+
+    frames = bytearray()
+    peak_value = int(32767 * amp)
+    for n in range(n_samples):
+        env = min(1.0, n / (sample_rate * 0.004), (n_samples - n) / (sample_rate * 0.008))
+        value = int(peak_value * env * math.sin(2 * math.pi * freq * n / sample_rate))
+        frames.extend(struct.pack("<hh", value, value))
+    raw = bytes(frames)
+
+    recorded_peak = 0
+    recorded_rms = 0.0
+    err = None
+    t0 = time.monotonic()
+    try:
+        out_stream = pa.open(
+            format=pyaudio.paInt16,
+            channels=2,
+            rate=sample_rate,
+            output=True,
+            output_device_index=out_idx,
+        )
+        chunk = 1024
+        in_stream = pa.open(
+            format=pyaudio.paInt16,
+            channels=1,
+            rate=sample_rate,
+            input=True,
+            input_device_index=in_idx,
+            frames_per_buffer=chunk,
+        )
+        in_stream.start_stream()
+        chunk_bytes = chunk * 2 * 2
+        pos = 0
+        peak_acc = 0
+        rms_acc = 0.0
+        n_samp = 0
+        deadline = time.monotonic() + dur + 1.5
+        while pos < len(raw) and time.monotonic() < deadline:
+            end = min(pos + chunk_bytes, len(raw))
+            out_stream.write(raw[pos:end])
+            pos = end
+            try:
+                avail = in_stream.get_read_available()
+                if avail and avail > 0:
+                    data = in_stream.read(avail, exception_on_overflow=False)
+                    for s in range(0, len(data) - 1, 2):
+                        v = int.from_bytes(data[s:s+2], "little", signed=True)
+                        a = abs(v)
+                        if a > peak_acc: peak_acc = a
+                        rms_acc += v * v
+                        n_samp += 1
+            except Exception:
+                pass
+        out_stream.stop_stream(); out_stream.close()
+        try:
+            tail = in_stream.read(n_samples, exception_on_overflow=False)
+            for s in range(0, len(tail) - 1, 2):
+                v = int.from_bytes(tail[s:s+2], "little", signed=True)
+                a = abs(v)
+                if a > peak_acc: peak_acc = a
+                rms_acc += v * v
+                n_samp += 1
+        except Exception:
+            pass
+        in_stream.stop_stream(); in_stream.close()
+        recorded_peak = peak_acc
+        recorded_rms = (rms_acc / max(1, n_samp)) ** 0.5
+    except Exception as e:
+        err = f"{type(e).__name__}: {e}"
+    finally:
+        try: pa.terminate()
+        except Exception: pass
+    duration_ms = int((time.monotonic() - t0) * 1000)
+    if err:
+        return {"ok": False, "error": err, "duration_ms": duration_ms,
+                "output_device": out_label, "input_device": in_label,
+                "frequency": freq, "duration": dur}
+    if recorded_peak > 0:
+        peak_db = 20.0 * math.log10(recorded_peak / 32767.0)
+    else:
+        peak_db = -120.0
+    if recorded_rms > 0:
+        rms_db = 20.0 * math.log10(recorded_rms / 32767.0)
+    else:
+        rms_db = -120.0
+    pass_ = peak_db >= threshold_db
+    return {
+        "ok": True,
+        "pass": pass_,
+        "frequency": freq,
+        "duration": dur,
+        "duration_ms": duration_ms,
+        "sample_rate": sample_rate,
+        "amplitude": amp,
+        "output_device": out_label,
+        "input_device": in_label,
+        "recorded_peak": int(recorded_peak),
+        "recorded_peak_db": round(peak_db, 1),
+        "recorded_rms": round(recorded_rms, 1),
+        "recorded_rms_db": round(rms_db, 1),
+        "threshold_db": threshold_db,
+        "diagnostic": ("speaker + mic round-trip OK" if pass_ else
+                       "no signal detected — check that the speaker is on, the mic isn't muted, and the right output/input devices are selected"),
+    }
 
 
 def run(config_path: Path) -> None:
+    global _CURRENT_LOG_DIR
     services, log_dir = _load_config(config_path)
+    _CURRENT_LOG_DIR = log_dir
     enabled = [s for s in services if s.enabled]
     if not enabled:
         raise SystemExit(f"No enabled services in {config_path} — nothing to supervise.")
@@ -262,7 +663,7 @@ def run(config_path: Path) -> None:
             now = time.monotonic()
             for state in states:
                 if state.proc is None:
-                    if now >= state.next_restart_at:
+                    if not state.stopped and now >= state.next_restart_at:
                         _spawn(state, log_dir)
                     continue
                 ret = state.proc.poll()
@@ -289,11 +690,19 @@ def run(config_path: Path) -> None:
                     commands = _report_status(states, identity)
                     by_name = {s.spec.name: s for s in states}
                     for cmd in commands:
+                        kind = cmd.get("kind", "supervisor_action")
                         target = by_name.get(cmd.get("service_name"))
-                        if target:
+                        if kind == "supervisor_action" and target:
                             _execute_command(target, cmd.get("action", "restart"))
-                        else:
+                        elif kind == "supervisor_action":
                             logger.warning(f"remote command for unknown service {cmd.get('service_name')!r} ignored")
+                        elif kind in ("shell_run", "tail_logs", "speaker_test"):
+                            # run on a background thread so we don't block
+                            # the 2s poll loop on a slow command
+                            import threading
+                            threading.Thread(target=_handle_shell_command, args=(identity, cmd), daemon=True).start()
+                        else:
+                            logger.warning(f"unknown remote command kind: {kind!r}")
                 else:
                     logger.debug("not enrolled yet (no ~/.robopark/preview_agent.json + device_token) -- skipping status report")
             time.sleep(POLL_INTERVAL_SECONDS)