livepilot 1.27.1 → 1.27.3

package/mcp_server/m4l_bridge.py

@@ -23,6 +23,7 @@ OSC address convention:
 from __future__ import annotations
 
 import asyncio
+import atexit
 import base64
 import json
 import random
@@ -30,6 +31,7 @@ import socket
 import struct
 import threading
 import time
+import uuid
 from typing import Any, Callable, Optional
 
 
@@ -158,15 +160,21 @@ class MidiToolCache:
         self._context: Optional[dict] = None
         self._notes: Optional[list] = None
         self._last_seen = 0.0
+        self._request_time = 0.0
         self._connected = False
         self._target_tool: Optional[str] = None
         self._target_params: dict = {}
 
     def set_request(self, context: dict, notes: list) -> None:
         with self._lock:
+            now = time.monotonic()
             self._context = context
             self._notes = notes
-            self._last_seen = time.monotonic()
+            self._last_seen = now
+            # Timestamp the request payload independently of _last_seen so a
+            # later mark_ready() heartbeat (which bumps _last_seen for
+            # is_connected) cannot resurrect an expired context/notes payload.
+            self._request_time = now
             self._connected = True
 
     def mark_ready(self) -> None:
@@ -179,7 +187,7 @@ class MidiToolCache:
         with self._lock:
             if self._context is None:
                 return None
-            age = time.monotonic() - self._last_seen
+            age = time.monotonic() - self._request_time
             if age > self._max_age:
                 return None
             return dict(self._context)
@@ -188,7 +196,7 @@ class MidiToolCache:
         with self._lock:
             if self._notes is None:
                 return None
-            age = time.monotonic() - self._last_seen
+            age = time.monotonic() - self._request_time
             if age > self._max_age:
                 return None
             return list(self._notes)
@@ -477,7 +485,10 @@ class SpectralReceiver(asyncio.DatagramProtocol):
         /rms f                     — RMS level
         /pitch f f                 — MIDI note, amplitude
         /response s                — base64-encoded JSON (single packet)
-        /response_chunk i i s      — chunked response (index, total, data)
+        /response_chunk [s] i i s  — chunked response. New builds prefix the
+                                     per-response request id so chunks of
+                                     different commands never share a bucket;
+                                     legacy builds omit it: (index, total, data)
     """
 
     # Band names keyed by how many bands the .amxd emits. 8 bands is the v1.x
@@ -499,8 +510,18 @@ class SpectralReceiver(asyncio.DatagramProtocol):
         self._chunk_id = 0
         self._chunk_key: Optional[str] = None  # Key of the single active reassembly bucket
         self._response_callback: Optional[asyncio.Future] = None
+        self._response_request_id: Optional[str] = None
+        # Set True the first time a response arrives carrying a request id.
+        # Once the analyzer build is known to stamp ids, a response that
+        # arrives with NO id (or a mismatched one) while a future is live is a
+        # stale straggler and must be dropped — see _handle_response.
+        self._seen_request_id = False
         self._capture_future: Optional[asyncio.Future] = None
         self._miditool_handler: Optional[Callable[[str, dict, list], None]] = None
+        # Source-address allowlist: the M4L device always sends from loopback.
+        # Track which non-loopback senders we've already warned about so a
+        # spoofing/collision source doesn't spam stderr once per packet.
+        self._warned_sources: set[str] = set()
 
     def set_miditool_handler(self, handler: Optional[Callable[[str, dict, list], None]]) -> None:
         """Register a callback invoked on each /miditool/request packet.
@@ -514,7 +535,27 @@ class SpectralReceiver(asyncio.DatagramProtocol):
     def connection_made(self, transport: asyncio.DatagramTransport) -> None:
         self.transport = transport
 
+    # Any local process can send to this UDP port, so a stray/malicious
+    # sender could poison the spectral cache or spoof /response packets to
+    # resolve pending bridge futures. The M4L device always sends from
+    # loopback, so anything else is dropped. This is a source-address
+    # allowlist, not authentication — a full shared-token handshake between
+    # the .amxd and the bridge is a deeper fix, tracked separately.
+    _ALLOWED_HOSTS = ("127.0.0.1", "::1")
+
     def datagram_received(self, data: bytes, addr: tuple) -> None:
+        host = addr[0] if addr else None
+        if host not in self._ALLOWED_HOSTS:
+            if host not in self._warned_sources:
+                self._warned_sources.add(host)
+                import sys
+                print(
+                    f"LivePilot: dropping UDP packet from non-loopback source "
+                    f"{addr!r} on spectral bridge port (further packets from "
+                    "this source will be dropped silently)",
+                    file=sys.stderr,
+                )
+            return
         try:
             self._parse_osc(data)
         except Exception as exc:
@@ -653,7 +694,16 @@ class SpectralReceiver(asyncio.DatagramProtocol):
             self._handle_response(str(args[0]))
 
         elif address == "/response_chunk" and len(args) >= 3:
-            self._handle_chunk(int(args[0]), int(args[1]), str(args[2]))
+            # New builds prefix the request id (string): (rid, index, total, data).
+            # Legacy builds send (index, total, data). Distinguish by the type
+            # of the first arg — OSC ints decode to int, the id to a str.
+            if len(args) >= 4 and isinstance(args[0], str):
+                self._handle_chunk(
+                    int(args[1]), int(args[2]), str(args[3]),
+                    request_id=str(args[0]),
+                )
+            else:
+                self._handle_chunk(int(args[0]), int(args[1]), str(args[2]))
 
         elif address == "/miditool/request" and len(args) >= 1:
             self._handle_miditool_request(str(args[0]))
@@ -665,24 +715,52 @@ class SpectralReceiver(asyncio.DatagramProtocol):
     def _handle_response(self, encoded: str) -> None:
         """Decode a single-packet base64 response.
 
-        Resolves _response_callback exactly once, then clears it. Without the
-        clear, a second late packet could overwrite a future belonging to a
-        different in-flight command. The protocol has no request id yet
-        (livepilot_bridge.js:666 emits bare /response), so correlation relies
-        on the single-command-in-flight invariant enforced by M4LBridge._cmd_lock
-        plus this one-shot clear.
+        Updated analyzer JS echoes ``_livepilot_request_id`` on every response
+        (single-packet here, and on the chunk header for chunked responses).
+        Correlation rules, once this device is known to stamp ids:
+          * a response whose id does NOT match the in-flight future is dropped;
+          * a response with NO id at all is also dropped — it is a stale
+            straggler (e.g. a batched read that outlived its timeout), not a
+            pre-request-id build.
+        Pre-request-id builds (which never stamp an id) stay supported: until
+        the first id is ever seen, no-id responses are accepted.
         """
         try:
             # URL-safe base64 decode (- and _ instead of + and /)
             padded = encoded + "=" * (-len(encoded) % 4)
             decoded = base64.urlsafe_b64decode(padded).decode('utf-8')
             result = _normalize_bridge_payload(json.loads(decoded))
+            response_request_id = None
+            if isinstance(result, dict):
+                response_request_id = result.pop("_livepilot_request_id", None)
+
+            if response_request_id is not None:
+                # This analyzer build stamps request ids — remember it so a
+                # later no-id straggler is recognised as stale rather than
+                # mistaken for an old build that never sends ids.
+                self._seen_request_id = True
+
             cb = self._response_callback
+            expected_request_id = self._response_request_id
+            if cb and not cb.done() and expected_request_id is not None:
+                mismatched = (
+                    response_request_id is not None
+                    and str(response_request_id) != str(expected_request_id)
+                )
+                missing_but_expected = (
+                    response_request_id is None and self._seen_request_id
+                )
+                if mismatched or missing_but_expected:
+                    # Stale/uncorrelated reply — drop it and keep waiting for
+                    # the response that actually matches this command.
+                    return
+
             if cb and not cb.done():
                 cb.set_result(result)
             # Clear regardless — either we consumed it, or it was already
             # done/abandoned. Future packets with no owner get dropped.
             self._response_callback = None
+            self._response_request_id = None
         except Exception as exc:
             import sys
             print(f"LivePilot: failed to decode bridge response: {exc}", file=sys.stderr)
@@ -691,7 +769,8 @@ class SpectralReceiver(asyncio.DatagramProtocol):
         """Decode a /miditool/request packet and dispatch to the configured handler.
 
         Payload: base64(JSON({request_id, context, notes})).
-        Packet arrives on the UDP receive thread; the registered handler is
+        Packet arrives on the asyncio event-loop thread (SpectralReceiver is a
+        DatagramProtocol, not a separate thread); the registered handler is
         expected to schedule work on an event loop rather than block here.
         """
         try:
@@ -718,49 +797,62 @@ class SpectralReceiver(asyncio.DatagramProtocol):
                 import sys
                 print(f"LivePilot: miditool handler error: {exc}", file=sys.stderr)
 
-    def _handle_chunk(self, index: int, total: int, encoded: str) -> None:
+    def _handle_chunk(
+        self,
+        index: int,
+        total: int,
+        encoded: str,
+        request_id: Optional[str] = None,
+    ) -> None:
         """Reassemble chunked responses.
 
-        The previous implementation incremented ``_chunk_id`` only when
-        ``index == 0`` and assumed the first chunk always arrived first.
-        Under UDP reordering (rare on loopback but possible under system
-        load), a chunk with ``index > 0`` arriving before ``index 0`` would
-        be dropped into the PREVIOUS sequence's bucket — silently corrupting
-        that earlier response's payload.
-
-        Until the wire protocol adds an explicit sequence id, the safer
-        behavior is: if we see an out-of-order first-chunk (``index > 0``
-        with no open bucket), start a fresh bucket but log a warning. That
-        way we never poison a prior sequence, and the problem surfaces in
-        logs if it happens.
+        New analyzer builds stamp the per-response request id on every
+        ``/response_chunk`` header, so chunks from different commands land in
+        SEPARATE buckets and can never be interleaved — even if a stale chunk
+        from a timed-out command arrives mid-flight. Legacy builds omit the id
+        (``request_id`` None/""); they fall back to a single rolling bucket,
+        which is safe because ``send_command`` serialises on ``_cmd_lock``.
+
+        Hardening (v1.27.2): an index outside ``[0, total)`` is dropped — a
+        duplicate or malformed packet must never count toward completion — and
+        reassembly only fires once EVERY index ``0..total-1`` is present. The
+        old ``len(parts) == total`` check could KeyError on a duplicate or
+        out-of-range index, silently losing the response and forcing a full
+        timeout.
         """
-        # Only one response is chunked at a time: send_command serialises on
-        # _cmd_lock, so a single active reassembly bucket is sufficient and we
-        # do NOT need the first packet to be index 0. Accepting chunks in any
-        # order fixes the permanent-loss path where an index>0 chunk arriving
-        # before index 0 (UDP loopback reordering under load) used to split
-        # one response across two buckets that never completed.
-        active = self._chunks.get(self._chunk_key)
-        if active is None or active["total"] != total:
-            # No bucket open, OR a chunk with a different `total` arrived,
-            # meaning a new response started (e.g. the previous one timed out
-            # without ever completing). Evict any stale partial and open fresh.
-            if active is not None:
-                self._chunks.pop(self._chunk_key, None)
-                self._chunk_times.pop(self._chunk_key, None)
-            self._chunk_id += 1
-            self._chunk_key = str(self._chunk_id)
-            self._chunks[self._chunk_key] = {"parts": {}, "total": total}
-            self._chunk_times[self._chunk_key] = time.monotonic()
-
-        key = self._chunk_key
-        self._chunks[key]["parts"][index] = encoded
-
-        if len(self._chunks[key]["parts"]) == total:
-            # All chunks received — reassemble
-            full = ""
-            for i in range(total):
-                full += self._chunks[key]["parts"][i]
+        # Drop chunks that cannot belong to a well-formed response.
+        if total <= 0 or index < 0 or index >= total:
+            return
+
+        if request_id:
+            # Collision-free per-response bucket.
+            key = f"rid:{request_id}"
+            active = self._chunks.get(key)
+            if active is None or active["total"] != total:
+                self._chunks[key] = {"parts": {}, "total": total}
+                self._chunk_times[key] = time.monotonic()
+            self._chunk_key = key
+        else:
+            # Legacy single-active-bucket model (no id on the wire). A chunk
+            # with a different `total` means a new response started (e.g. the
+            # previous one timed out without ever completing) — evict + reopen.
+            active = self._chunks.get(self._chunk_key)
+            if active is None or active["total"] != total:
+                if active is not None:
+                    self._chunks.pop(self._chunk_key, None)
+                    self._chunk_times.pop(self._chunk_key, None)
+                self._chunk_id += 1
+                self._chunk_key = str(self._chunk_id)
+                self._chunks[self._chunk_key] = {"parts": {}, "total": total}
+                self._chunk_times[self._chunk_key] = time.monotonic()
+            key = self._chunk_key
+
+        parts = self._chunks[key]["parts"]
+        parts[index] = encoded
+
+        if len(parts) == total and all(i in parts for i in range(total)):
+            # Every chunk present — reassemble in order.
+            full = "".join(parts[i] for i in range(total))
             del self._chunks[key]
             self._chunk_times.pop(key, None)
             self._handle_response(full)
@@ -780,13 +872,22 @@ class SpectralReceiver(asyncio.DatagramProtocol):
             result = _normalize_bridge_payload(json.loads(decoded))
             if self._capture_future and not self._capture_future.done():
                 self._capture_future.set_result(result)
+            # Clear the reference so a completed future isn't retained until the
+            # next capture, and so send_capture's done()-guarded cancel has no
+            # stale object to consider.
+            self._capture_future = None
         except Exception as exc:
             import sys
             print(f"LivePilot: failed to decode capture response: {exc}", file=sys.stderr)
 
-    def set_response_future(self, future: asyncio.Future) -> None:
+    def set_response_future(
+        self,
+        future: Optional[asyncio.Future],
+        request_id: Optional[str] = None,
+    ) -> None:
         """Set a future to be resolved with the next response."""
         self._response_callback = future
+        self._response_request_id = request_id if future is not None else None
 
     def set_capture_future(self, future: asyncio.Future) -> None:
         """Set a future to be resolved when a capture_complete OSC arrives."""
@@ -810,7 +911,14 @@ class M4LBridge:
         self.receiver = receiver
         self.miditool_cache = miditool_cache
         self._sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        # Non-blocking so a momentarily-full send buffer never stalls the
+        # asyncio event loop. The miditool response path sends from inside a
+        # DatagramProtocol callback that runs ON the loop thread — a blocking
+        # sendto there would freeze every pending coroutine. UDP loopback sends
+        # essentially never block; _safe_sendto guards the rare case anyway.
+        self._sock.setblocking(False)
         self._m4l_addr = ("127.0.0.1", 9881)
+        self._closed = False
         self._cmd_lock: Optional[asyncio.Lock] = None
         # BUG-audit-C1: send_capture uses _capture_future, which is
         # independent of _response_callback used by send_command.
@@ -825,13 +933,40 @@ class M4LBridge:
             if self.receiver.miditool_cache is None and miditool_cache is not None:
                 self.receiver.miditool_cache = miditool_cache
 
+        # Best-effort: release this bridge's OUTGOING UDP 9881 send socket on a
+        # clean interpreter exit. NOTE: this is NOT the 9880 analyzer listener —
+        # that socket lives on the SpectralReceiver transport (created in the
+        # server lifespan) and is closed in the lifespan finally. The OS reclaims
+        # BOTH ports on any process death, INCLUDING the SIGTERM/SIGKILL the
+        # orphan-kill runbook uses (which bypasses atexit). The real defense
+        # against an orphan squatting 9880 is the §9b kill-orphan procedure +
+        # reconnect_bridge, not this handler.
+        atexit.register(self.close)
+
+    def _safe_sendto(self, data: bytes) -> None:
+        """Send a UDP packet without ever blocking the event loop.
+
+        The socket is non-blocking; if the OS send buffer is momentarily full
+        (vanishingly rare on loopback) sendto raises BlockingIOError. We drop
+        the packet rather than stall — the caller's timeout/retry handles it.
+        """
+        try:
+            self._sock.sendto(data, self._m4l_addr)
+        except BlockingIOError:
+            import sys
+            print(
+                "LivePilot: M4L UDP send buffer full — packet dropped",
+                file=sys.stderr,
+            )
+
     def _dispatch_miditool_request(
         self, request_id: str, context: dict, notes: list,
     ) -> None:
         """Handle a decoded /miditool/request: run the configured generator,
         send /miditool/response back with {request_id, notes}.
 
-        Invoked from SpectralReceiver on the UDP receive thread. We do not
+        Invoked from SpectralReceiver on the asyncio event-loop thread
+        (it is a DatagramProtocol, not a separate UDP thread). We do not
         await anything here — generators are synchronous, pure Python.
         If no target is configured, pass notes through unchanged (identity).
         """
@@ -869,7 +1004,7 @@ class M4LBridge:
         """
         payload = {"tool_name": tool_name or "", "params": params or {}}
         osc = self._build_osc("miditool/config", (json.dumps(payload),))
-        self._sock.sendto(osc, self._m4l_addr)
+        self._safe_sendto(osc)
 
     def send_miditool_response(self, request_id: str, notes: list) -> None:
         """Send transformed notes back to the JS bridge.
@@ -879,7 +1014,7 @@ class M4LBridge:
         """
         payload = {"request_id": str(request_id or ""), "notes": list(notes or [])}
         osc = self._build_osc("miditool/response", (json.dumps(payload),))
-        self._sock.sendto(osc, self._m4l_addr)
+        self._safe_sendto(osc)
 
     async def send_command(self, command: str, *args: Any, timeout: float = 5.0) -> dict:
         """Send an OSC command to the M4L device and wait for the response."""
@@ -905,12 +1040,14 @@ class M4LBridge:
             # Create a future for the response
             loop = asyncio.get_running_loop()
             future = loop.create_future()
-            self.receiver.set_response_future(future)
+            request_id = uuid.uuid4().hex
+            self.receiver.set_response_future(future, request_id=request_id)
 
             # Build and send OSC message (no leading / — Max udpreceive
             # passes messagename with / intact to JS, breaking dispatch)
-            osc_data = self._build_osc(command, args)
-            self._sock.sendto(osc_data, self._m4l_addr)
+            request_arg = f"__livepilot_request_id:{request_id}"
+            osc_data = self._build_osc(command, (*args, request_arg))
+            self._safe_sendto(osc_data)
 
             # Wait for response with timeout
             try:
@@ -958,7 +1095,7 @@ class M4LBridge:
             self.receiver.set_capture_future(future)
 
             osc_data = self._build_osc(command, args)
-            self._sock.sendto(osc_data, self._m4l_addr)
+            self._safe_sendto(osc_data)
 
             try:
                 result = await asyncio.wait_for(future, timeout=timeout)
@@ -971,7 +1108,7 @@ class M4LBridge:
     async def cancel_capture_future(self) -> None:
         """Resolve any in-progress capture future with a stopped result.
 
-        Does NOT acquire _cmd_lock — send_capture holds it during recording.
+        Does NOT acquire _capture_lock — send_capture holds it during recording.
         Resolving (not cancelling) the future lets send_capture return a
         clean partial-result dict instead of raising CancelledError.
         """
@@ -1025,4 +1162,22 @@ class M4LBridge:
         return addr_bytes + tag_bytes + arg_data
 
     def close(self) -> None:
-        self._sock.close()
+        """Close the outgoing UDP socket and mark this bridge as shut down.
+
+        Idempotent — safe to call multiple times (from the server lifespan
+        finally block AND from the atexit handler registered at construction).
+
+        Releases this bridge's OUTGOING UDP 9881 send socket (M4LBridge._sock).
+        This is NOT the 9880 analyzer listener — that is the SpectralReceiver
+        transport, closed in the server lifespan finally. Called from both the
+        lifespan shutdown and an atexit handler; on abrupt process death the OS
+        reclaims the socket regardless (see the §9b orphan-kill runbook for the
+        actual stale-9880 defense).
+        """
+        if self._closed:
+            return
+        self._closed = True
+        try:
+            self._sock.close()
+        except OSError:
+            pass

package/mcp_server/memory/taste_graph.py

@@ -216,6 +216,49 @@ class TasteGraph:
                 logger.debug("update_novelty failed: %s", exc)
                 pass  # persistence is best-effort
 
+    def record_anti_preference(self, dimension: str, direction: str) -> None:
+        """Record a dimension avoidance and persist it across restarts.
+
+        Mirrors record_move_outcome's write-back contract: updates the
+        in-memory dimension_avoidances map AND persists to disk so the
+        avoidance survives a server restart. build_taste_graph hydrates
+        anti_preferences from the same persistent store, so without this
+        write-back the persisted read branch would stay dead (P2-29).
+        """
+        now = int(time.time() * 1000)
+        self.dimension_avoidances[dimension] = direction
+        self.evidence_count += 1
+        self.last_updated_ms = now
+
+        # Write-back to persistent store
+        if self._persistent_store is not None:
+            try:
+                self._persistent_store.record_anti_preference(dimension, direction)
+            except Exception as exc:
+                logger.debug("record_anti_preference failed: %s", exc)
+                pass  # persistence is best-effort
+
+    def record_dimension_weight(self, dimension: str, value: float) -> None:
+        """Set a dimension preference weight and persist it across restarts.
+
+        Mirrors record_move_outcome's write-back contract: updates the
+        in-memory dimension_weights map AND persists to disk so the weight
+        survives a server restart. build_taste_graph hydrates
+        dimension_weights from the same persistent store, so without this
+        write-back the persisted read branch would stay dead (P2-29).
+        """
+        now = int(time.time() * 1000)
+        self.dimension_weights[dimension] = value
+        self.last_updated_ms = now
+
+        # Write-back to persistent store
+        if self._persistent_store is not None:
+            try:
+                self._persistent_store.record_dimension_weight(dimension, value)
+            except Exception as exc:
+                logger.debug("record_dimension_weight failed: %s", exc)
+                pass  # persistence is best-effort
+
     # ── Ranking ──────────────────────────────────────────────────────
 
     def rank_moves(

package/mcp_server/memory/technique_store.py

@@ -2,6 +2,7 @@
 
 import copy
 import json
+import logging
 import os
 import threading
 import uuid
@@ -10,6 +11,9 @@ from pathlib import Path
 from typing import Any, Optional
 
 
+logger = logging.getLogger(__name__)
+
+
 VALID_TYPES = frozenset(
     ["beat_pattern", "device_chain", "mix_template", "browser_pin", "preference",
      "outcome", "composition_outcome", "technique_card"]
@@ -74,9 +78,28 @@ class TechniqueStore:
                     with open(self._file, "r") as f:
                         self._data = json.load(f)
                     self._loaded_sig = self._file_signature()
-                except (json.JSONDecodeError, ValueError):
-                    corrupt = self._file.with_suffix(".json.corrupt")
-                    self._file.rename(corrupt)
+                except (json.JSONDecodeError, ValueError) as exc:
+                    # Preserve the unparseable file before replacing it with
+                    # defaults, so the user's data stays recoverable. Use a
+                    # timestamped name so repeated corruptions don't clobber
+                    # an earlier backup (a plain ".json.corrupt" target would
+                    # be silently overwritten on POSIX / raise on Windows).
+                    ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S")
+                    corrupt = self._file.with_suffix(f".json.corrupt.{ts}")
+                    try:
+                        self._file.rename(corrupt)
+                        logger.warning(
+                            "techniques.json failed to parse (%s); backed up "
+                            "corrupt file to %s and reinitialized with defaults",
+                            exc, corrupt,
+                        )
+                    except OSError as rename_exc:
+                        logger.warning(
+                            "techniques.json failed to parse (%s) and the "
+                            "corrupt backup could not be written (%s); "
+                            "reinitialized with defaults — prior data lost",
+                            exc, rename_exc,
+                        )
                     self._data = {"version": 1, "techniques": []}
                     self._loaded_sig = None
             else: