livepilot 1.27.2 → 1.27.3

package/mcp_server/experiment/engine.py

@@ -17,7 +17,9 @@ The engine itself is pure orchestration logic.
 
 from __future__ import annotations
 
+import asyncio
 import hashlib
+import threading
 import time
 from typing import Optional
 
@@ -28,8 +30,15 @@ import logging
 logger = logging.getLogger(__name__)
 
 # ── In-memory experiment store ───────────────────────────────────────────────
+#
+# _EXPERIMENTS is mutated from both threadpooled sync tools (create_experiment)
+# and event-loop async tools (run_experiment / commit_branch_async) — a
+# module-level lock around the dict ops keeps inserts/reads race-free under
+# mixed sync/async concurrent callers, mirroring preview_studio.engine and
+# wonder_mode.session.
 
 _EXPERIMENTS: dict[str, ExperimentSet] = {}
+_EXPERIMENTS_LOCK = threading.Lock()
 
 
 def _gen_id(prefix: str, seed: str) -> str:
@@ -89,7 +98,8 @@ def create_experiment_from_seeds(
         created_at_ms=now,
     )
 
-    _EXPERIMENTS[exp_id] = experiment
+    with _EXPERIMENTS_LOCK:
+        _EXPERIMENTS[exp_id] = experiment
     return experiment
 
 
@@ -114,12 +124,15 @@ def create_experiment(
 
 def get_experiment(experiment_id: str) -> Optional[ExperimentSet]:
     """Get an experiment by ID."""
-    return _EXPERIMENTS.get(experiment_id)
+    with _EXPERIMENTS_LOCK:
+        return _EXPERIMENTS.get(experiment_id)
 
 
 def list_experiments() -> list[dict]:
     """List all experiment sets."""
-    return [exp.to_dict() for exp in _EXPERIMENTS.values()]
+    with _EXPERIMENTS_LOCK:
+        exps = list(_EXPERIMENTS.values())
+    return [exp.to_dict() for exp in exps]
 
 # ── Run experiments (requires Ableton connection) ────────────────────────────
 
@@ -211,12 +224,16 @@ async def run_branch_async(
     analyze_mix) are skipped in the apply pass — they're used for snapshot
     capture separately.
     """
-    from ..runtime.execution_router import execute_plan_steps_async, filter_apply_steps
+    from ..runtime.execution_router import (
+        execute_plan_steps_async,
+        filter_apply_steps,
+        undo_remote_steps,
+    )
 
     branch.status = "running"
     branch.compiled_plan = compiled_plan
 
-    branch.before_snapshot = capture_fn()
+    branch.before_snapshot = await asyncio.to_thread(capture_fn)
 
     # Filter out read-only verification steps from the apply pass (canonical
     # list lives in execution_router.READ_ONLY_TOOLS).
@@ -244,24 +261,13 @@ async def run_branch_async(
     ]
 
     steps_executed = sum(1 for r in exec_results if r.ok)
-    # Only remote_command steps land on Ableton's linear undo stack. Bridge
-    # (M4L/OSC) and mcp_tool mutations do NOT, so issuing one `undo` per
-    # successful step over-undoes and walks back unrelated prior user edits.
-    # Count undos from remote_command successes alone.
-    undo_count = sum(
-        1 for r in exec_results if r.ok and r.backend == "remote_command"
-    )
     branch.executed_at_ms = int(time.time() * 1000)
-    branch.after_snapshot = capture_fn()
+    branch.after_snapshot = await asyncio.to_thread(capture_fn)
 
-    # Undo only the remote_command steps back to checkpoint. Undo is itself a
-    # remote_command, routed through the normal ableton.send_command path.
-    for _ in range(undo_count):
-        try:
-            ableton.send_command("undo", {})
-        except Exception as exc:
-            logger.debug("run_branch_async failed: %s", exc)
-            break
+    # Undo only the remote_command steps back to checkpoint (bridge/mcp_tool
+    # mutations don't land on Ableton's linear undo stack — see
+    # undo_remote_steps' docstring).
+    await undo_remote_steps(ableton, exec_results)
 
     # A branch is "evaluated" only if it actually applied at least one step.
     # If every step failed, mark it "failed" — this is the truth-release
@@ -442,7 +448,8 @@ def commit_branch(
 
 def discard_experiment(experiment_id: str) -> dict:
     """Discard an entire experiment set."""
-    exp = _EXPERIMENTS.get(experiment_id)
+    with _EXPERIMENTS_LOCK:
+        exp = _EXPERIMENTS.get(experiment_id)
     if not exp:
         return {"error": f"Experiment {experiment_id} not found"}
 

package/mcp_server/experiment/tools.py

@@ -10,6 +10,8 @@
 
 from __future__ import annotations
 
+import asyncio
+import functools
 import time
 from typing import Optional
 
@@ -351,7 +353,7 @@ async def run_experiment(
     if experiment.baseline_transport is None:
         from .baseline import capture_baseline
         try:
-            experiment.baseline_transport = capture_baseline(ableton)
+            experiment.baseline_transport = await asyncio.to_thread(capture_baseline, ableton)
         except Exception as exc:
             logger.debug("baseline capture failed: %s", exc)
             experiment.baseline_transport = None
@@ -365,7 +367,8 @@ async def run_experiment(
         # Between branches (not before the first), restore the baseline so
         # the next before_snapshot reads from the same reference state.
         if pending_seen > 0:
-            engine.prepare_for_next_branch(
+            await asyncio.to_thread(
+                engine.prepare_for_next_branch,
                 ableton, experiment.baseline_transport, stabilize_ms=300,
             )
         pending_seen += 1
@@ -414,7 +417,7 @@ async def run_experiment(
                 results.append(branch.to_dict())
                 continue
 
-            session_info = ableton.send_command("get_session_info")
+            session_info = await ableton.send_command_async("get_session_info")
             kernel = {"session_info": session_info, "mode": "explore"}
             plan = compiler.compile(move, kernel)
             compiled_dict = plan.to_dict()
@@ -422,12 +425,19 @@ async def run_experiment(
         # Pick the capture function — render-verify mode captures audio
         # and extracts a TimbralFingerprint, adding latency but giving
         # classify_branch_outcome real measurable evidence.
+        #
+        # NOTE: bound via functools.partial (not a lambda) so the blocking
+        # `_capture_snapshot*` call is never a "bare" call site in this
+        # function's body — it is only ever invoked downstream inside
+        # `engine.run_branch_async` via `await asyncio.to_thread(capture_fn)`,
+        # which is where the actual off-loop offload happens.
         if render_verify:
-            capture_fn = lambda: _capture_snapshot_with_render_verify(
+            capture_fn = functools.partial(
+                _capture_snapshot_with_render_verify,
                 ctx, duration_seconds=render_duration_seconds,
             )
         else:
-            capture_fn = lambda: _capture_snapshot(ctx)
+            capture_fn = functools.partial(_capture_snapshot, ctx)
 
         # Run the branch through the async router
         await engine.run_branch_async(

package/mcp_server/m4l_bridge.py

@@ -23,6 +23,7 @@ OSC address convention:
 from __future__ import annotations
 
 import asyncio
+import atexit
 import base64
 import json
 import random
@@ -159,15 +160,21 @@ class MidiToolCache:
         self._context: Optional[dict] = None
         self._notes: Optional[list] = None
         self._last_seen = 0.0
+        self._request_time = 0.0
         self._connected = False
         self._target_tool: Optional[str] = None
         self._target_params: dict = {}
 
     def set_request(self, context: dict, notes: list) -> None:
         with self._lock:
+            now = time.monotonic()
             self._context = context
             self._notes = notes
-            self._last_seen = time.monotonic()
+            self._last_seen = now
+            # Timestamp the request payload independently of _last_seen so a
+            # later mark_ready() heartbeat (which bumps _last_seen for
+            # is_connected) cannot resurrect an expired context/notes payload.
+            self._request_time = now
             self._connected = True
 
     def mark_ready(self) -> None:
@@ -180,7 +187,7 @@ class MidiToolCache:
         with self._lock:
             if self._context is None:
                 return None
-            age = time.monotonic() - self._last_seen
+            age = time.monotonic() - self._request_time
             if age > self._max_age:
                 return None
             return dict(self._context)
@@ -189,7 +196,7 @@ class MidiToolCache:
         with self._lock:
             if self._notes is None:
                 return None
-            age = time.monotonic() - self._last_seen
+            age = time.monotonic() - self._request_time
             if age > self._max_age:
                 return None
             return list(self._notes)
@@ -511,6 +518,10 @@ class SpectralReceiver(asyncio.DatagramProtocol):
         self._seen_request_id = False
         self._capture_future: Optional[asyncio.Future] = None
         self._miditool_handler: Optional[Callable[[str, dict, list], None]] = None
+        # Source-address allowlist: the M4L device always sends from loopback.
+        # Track which non-loopback senders we've already warned about so a
+        # spoofing/collision source doesn't spam stderr once per packet.
+        self._warned_sources: set[str] = set()
 
     def set_miditool_handler(self, handler: Optional[Callable[[str, dict, list], None]]) -> None:
         """Register a callback invoked on each /miditool/request packet.
@@ -524,7 +535,27 @@ class SpectralReceiver(asyncio.DatagramProtocol):
     def connection_made(self, transport: asyncio.DatagramTransport) -> None:
         self.transport = transport
 
+    # Any local process can send to this UDP port, so a stray/malicious
+    # sender could poison the spectral cache or spoof /response packets to
+    # resolve pending bridge futures. The M4L device always sends from
+    # loopback, so anything else is dropped. This is a source-address
+    # allowlist, not authentication — a full shared-token handshake between
+    # the .amxd and the bridge is a deeper fix, tracked separately.
+    _ALLOWED_HOSTS = ("127.0.0.1", "::1")
+
     def datagram_received(self, data: bytes, addr: tuple) -> None:
+        host = addr[0] if addr else None
+        if host not in self._ALLOWED_HOSTS:
+            if host not in self._warned_sources:
+                self._warned_sources.add(host)
+                import sys
+                print(
+                    f"LivePilot: dropping UDP packet from non-loopback source "
+                    f"{addr!r} on spectral bridge port (further packets from "
+                    "this source will be dropped silently)",
+                    file=sys.stderr,
+                )
+            return
         try:
             self._parse_osc(data)
         except Exception as exc:
@@ -738,7 +769,8 @@ class SpectralReceiver(asyncio.DatagramProtocol):
         """Decode a /miditool/request packet and dispatch to the configured handler.
 
         Payload: base64(JSON({request_id, context, notes})).
-        Packet arrives on the UDP receive thread; the registered handler is
+        Packet arrives on the asyncio event-loop thread (SpectralReceiver is a
+        DatagramProtocol, not a separate thread); the registered handler is
         expected to schedule work on an event loop rather than block here.
         """
         try:
@@ -886,6 +918,7 @@ class M4LBridge:
         # essentially never block; _safe_sendto guards the rare case anyway.
         self._sock.setblocking(False)
         self._m4l_addr = ("127.0.0.1", 9881)
+        self._closed = False
         self._cmd_lock: Optional[asyncio.Lock] = None
         # BUG-audit-C1: send_capture uses _capture_future, which is
         # independent of _response_callback used by send_command.
@@ -900,6 +933,16 @@ class M4LBridge:
             if self.receiver.miditool_cache is None and miditool_cache is not None:
                 self.receiver.miditool_cache = miditool_cache
 
+        # Best-effort: release this bridge's OUTGOING UDP 9881 send socket on a
+        # clean interpreter exit. NOTE: this is NOT the 9880 analyzer listener —
+        # that socket lives on the SpectralReceiver transport (created in the
+        # server lifespan) and is closed in the lifespan finally. The OS reclaims
+        # BOTH ports on any process death, INCLUDING the SIGTERM/SIGKILL the
+        # orphan-kill runbook uses (which bypasses atexit). The real defense
+        # against an orphan squatting 9880 is the §9b kill-orphan procedure +
+        # reconnect_bridge, not this handler.
+        atexit.register(self.close)
+
     def _safe_sendto(self, data: bytes) -> None:
         """Send a UDP packet without ever blocking the event loop.
 
@@ -922,7 +965,8 @@ class M4LBridge:
         """Handle a decoded /miditool/request: run the configured generator,
         send /miditool/response back with {request_id, notes}.
 
-        Invoked from SpectralReceiver on the UDP receive thread. We do not
+        Invoked from SpectralReceiver on the asyncio event-loop thread
+        (it is a DatagramProtocol, not a separate UDP thread). We do not
         await anything here — generators are synchronous, pure Python.
         If no target is configured, pass notes through unchanged (identity).
         """
@@ -1064,7 +1108,7 @@ class M4LBridge:
     async def cancel_capture_future(self) -> None:
         """Resolve any in-progress capture future with a stopped result.
 
-        Does NOT acquire _cmd_lock — send_capture holds it during recording.
+        Does NOT acquire _capture_lock — send_capture holds it during recording.
         Resolving (not cancelling) the future lets send_capture return a
         clean partial-result dict instead of raising CancelledError.
         """
@@ -1118,4 +1162,22 @@ class M4LBridge:
         return addr_bytes + tag_bytes + arg_data
 
     def close(self) -> None:
-        self._sock.close()
+        """Close the outgoing UDP socket and mark this bridge as shut down.
+
+        Idempotent — safe to call multiple times (from the server lifespan
+        finally block AND from the atexit handler registered at construction).
+
+        Releases this bridge's OUTGOING UDP 9881 send socket (M4LBridge._sock).
+        This is NOT the 9880 analyzer listener — that is the SpectralReceiver
+        transport, closed in the server lifespan finally. Called from both the
+        lifespan shutdown and an atexit handler; on abrupt process death the OS
+        reclaims the socket regardless (see the §9b orphan-kill runbook for the
+        actual stale-9880 defense).
+        """
+        if self._closed:
+            return
+        self._closed = True
+        try:
+            self._sock.close()
+        except OSError:
+            pass

package/mcp_server/memory/taste_graph.py

@@ -216,6 +216,49 @@ class TasteGraph:
                 logger.debug("update_novelty failed: %s", exc)
                 pass  # persistence is best-effort
 
+    def record_anti_preference(self, dimension: str, direction: str) -> None:
+        """Record a dimension avoidance and persist it across restarts.
+
+        Mirrors record_move_outcome's write-back contract: updates the
+        in-memory dimension_avoidances map AND persists to disk so the
+        avoidance survives a server restart. build_taste_graph hydrates
+        anti_preferences from the same persistent store, so without this
+        write-back the persisted read branch would stay dead (P2-29).
+        """
+        now = int(time.time() * 1000)
+        self.dimension_avoidances[dimension] = direction
+        self.evidence_count += 1
+        self.last_updated_ms = now
+
+        # Write-back to persistent store
+        if self._persistent_store is not None:
+            try:
+                self._persistent_store.record_anti_preference(dimension, direction)
+            except Exception as exc:
+                logger.debug("record_anti_preference failed: %s", exc)
+                pass  # persistence is best-effort
+
+    def record_dimension_weight(self, dimension: str, value: float) -> None:
+        """Set a dimension preference weight and persist it across restarts.
+
+        Mirrors record_move_outcome's write-back contract: updates the
+        in-memory dimension_weights map AND persists to disk so the weight
+        survives a server restart. build_taste_graph hydrates
+        dimension_weights from the same persistent store, so without this
+        write-back the persisted read branch would stay dead (P2-29).
+        """
+        now = int(time.time() * 1000)
+        self.dimension_weights[dimension] = value
+        self.last_updated_ms = now
+
+        # Write-back to persistent store
+        if self._persistent_store is not None:
+            try:
+                self._persistent_store.record_dimension_weight(dimension, value)
+            except Exception as exc:
+                logger.debug("record_dimension_weight failed: %s", exc)
+                pass  # persistence is best-effort
+
     # ── Ranking ──────────────────────────────────────────────────────
 
     def rank_moves(

package/mcp_server/memory/technique_store.py

@@ -2,6 +2,7 @@
 
 import copy
 import json
+import logging
 import os
 import threading
 import uuid
@@ -10,6 +11,9 @@ from pathlib import Path
 from typing import Any, Optional
 
 
+logger = logging.getLogger(__name__)
+
+
 VALID_TYPES = frozenset(
     ["beat_pattern", "device_chain", "mix_template", "browser_pin", "preference",
      "outcome", "composition_outcome", "technique_card"]
@@ -74,9 +78,28 @@ class TechniqueStore:
                     with open(self._file, "r") as f:
                         self._data = json.load(f)
                     self._loaded_sig = self._file_signature()
-                except (json.JSONDecodeError, ValueError):
-                    corrupt = self._file.with_suffix(".json.corrupt")
-                    self._file.rename(corrupt)
+                except (json.JSONDecodeError, ValueError) as exc:
+                    # Preserve the unparseable file before replacing it with
+                    # defaults, so the user's data stays recoverable. Use a
+                    # timestamped name so repeated corruptions don't clobber
+                    # an earlier backup (a plain ".json.corrupt" target would
+                    # be silently overwritten on POSIX / raise on Windows).
+                    ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S")
+                    corrupt = self._file.with_suffix(f".json.corrupt.{ts}")
+                    try:
+                        self._file.rename(corrupt)
+                        logger.warning(
+                            "techniques.json failed to parse (%s); backed up "
+                            "corrupt file to %s and reinitialized with defaults",
+                            exc, corrupt,
+                        )
+                    except OSError as rename_exc:
+                        logger.warning(
+                            "techniques.json failed to parse (%s) and the "
+                            "corrupt backup could not be written (%s); "
+                            "reinitialized with defaults — prior data lost",
+                            exc, rename_exc,
+                        )
                     self._data = {"version": 1, "techniques": []}
                     self._loaded_sig = None
             else:

package/mcp_server/memory/tools.py

@@ -6,6 +6,8 @@
 
 from __future__ import annotations
 
+import logging
+
 from fastmcp import Context
 
 from ..server import mcp
@@ -14,12 +16,29 @@ from .promotion import batch_evaluate_promotions
 from .session_memory import SessionMemoryStore
 from .taste_memory import TasteMemoryStore
 
+logger = logging.getLogger(__name__)
+
 
 def _get_anti_memory(ctx: Context) -> AntiMemoryStore:
     """Get or create the session-scoped AntiMemoryStore."""
     return ctx.lifespan_context.setdefault("anti_memory", AntiMemoryStore())
 
 
+def _get_persistent_taste_store(ctx: Context):
+    """Return the persistent taste store from the live server context, or None.
+
+    Only the live server lifespan injects "persistent_taste" (the same key the
+    wonder/runtime/preview_studio tools use, so all share ONE instance); tests
+    build their own context without it, so taste tools stay session-only
+    (hermetic) there. When present, taste write-backs and reads persist across
+    restarts (P2-29).
+    """
+    try:
+        return ctx.lifespan_context.get("persistent_taste")
+    except AttributeError:
+        return None
+
+
 def _get_session_memory(ctx: Context) -> SessionMemoryStore:
     """Get or create the session-scoped SessionMemoryStore."""
     return ctx.lifespan_context.setdefault("session_memory", SessionMemoryStore())
@@ -43,12 +62,25 @@ def record_anti_preference(
 ) -> dict:
     """Record a user dislike for a dimension+direction. direction must be 'increase' or 'decrease'."""
     if direction not in ("increase", "decrease"):
-        return {"error": "direction must be 'increase' or 'decrease'"}
+        return {"error": "direction must be 'increase' or 'decrease'",
+                "code": "INVALID_PARAM"}
     store = _get_anti_memory(ctx)
     pref = store.record_dislike(dimension, direction)
+    # P2-29: persist to the taste store so the anti-preference survives a
+    # server restart (best-effort — the session store above is authoritative
+    # for the in-session response).
+    persistent = _get_persistent_taste_store(ctx)
+    persisted = False
+    if persistent is not None:
+        try:
+            persistent.record_anti_preference(dimension, direction)
+            persisted = True
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.debug("persist anti_preference failed: %s", exc)
     return {
         "recorded": pref.to_dict(),
         "should_caution": store.should_caution(dimension, direction),
+        "persisted": persisted,
     }
 
 
@@ -143,7 +175,10 @@ def get_taste_graph(ctx: Context) -> dict:
 
     taste_store = _get_taste_memory(ctx)
     anti_store = _get_anti_memory(ctx)
-    graph = build_taste_graph(taste_store=taste_store, anti_store=anti_store)
+    graph = build_taste_graph(
+        taste_store=taste_store, anti_store=anti_store,
+        persistent_store=_get_persistent_taste_store(ctx),
+    )
     return graph.to_dict()
 
 
@@ -158,7 +193,10 @@ def explain_taste_inference(ctx: Context) -> dict:
 
     taste_store = _get_taste_memory(ctx)
     anti_store = _get_anti_memory(ctx)
-    graph = build_taste_graph(taste_store=taste_store, anti_store=anti_store)
+    graph = build_taste_graph(
+        taste_store=taste_store, anti_store=anti_store,
+        persistent_store=_get_persistent_taste_store(ctx),
+    )
     return graph.explain()
 
 
@@ -178,7 +216,10 @@ def rank_moves_by_taste(
 
     taste_store = _get_taste_memory(ctx)
     anti_store = _get_anti_memory(ctx)
-    graph = build_taste_graph(taste_store=taste_store, anti_store=anti_store)
+    graph = build_taste_graph(
+        taste_store=taste_store, anti_store=anti_store,
+        persistent_store=_get_persistent_taste_store(ctx),
+    )
 
     if isinstance(move_specs, str):
         import json
@@ -218,10 +259,27 @@ def record_positive_preference(
             matching_signals.append(sig_name)
     if matching_signals:
         taste_store.update_from_outcome({"signals": matching_signals})
+    # P2-29 (dimension-weight half): persist the updated dimension weight so it
+    # survives a server restart — symmetric with record_anti_preference. Without
+    # this the persisted dimension_weights hydration branch in build_taste_graph
+    # stayed dead (no production writer). Best-effort; session store is
+    # authoritative for the in-session response.
+    persisted = False
+    persistent = _get_persistent_taste_store(ctx)
+    if persistent is not None and matching_signals:
+        try:
+            for dim in taste_store.get_taste_dimensions():
+                if dim.name == dimension and dim.evidence_count > 0:
+                    persistent.record_dimension_weight(dimension, dim.value)
+                    persisted = True
+                    break
+        except Exception as exc:  # pragma: no cover - defensive
+            logger.debug("persist dimension_weight failed: %s", exc)
     return {
         "recorded": bool(matching_signals),
         "dimension": dimension,
         "direction": direction,
         "signals_matched": matching_signals,
         "evidence": evidence,
+        "persisted": persisted,
     }