livepilot 1.27.1 → 1.27.3

package/mcp_server/atlas/__init__.py

@@ -13,6 +13,20 @@ from typing import Any, Dict, List, Optional
 
 logger = logging.getLogger(__name__)
 
+# BUG-B39: the real atlas scanner emits "instruments" / "audio_effects" but
+# older callers and test fixtures sometimes pass the singular "instrument" /
+# "effect". Hoisted to module scope (was previously re-declared inline in
+# search()) so both search() and the truncation-warning helper below share
+# one alias table instead of drifting independently.
+_CATEGORY_ALIASES: Dict[str, set] = {
+    "instrument": {"instrument", "instruments"},
+    "instruments": {"instrument", "instruments"},
+    "effect": {"effect", "effects", "audio_effects"},
+    "effects": {"effect", "effects", "audio_effects"},
+    "audio_effect": {"effect", "effects", "audio_effects", "audio_effect"},
+    "audio_effects": {"effect", "effects", "audio_effects", "audio_effect"},
+}
+
 
 class AtlasManager:
     """In-memory device atlas with indexed lookups."""
@@ -34,15 +48,50 @@ class AtlasManager:
             self._meta["version"] = data["version"]
         self._devices: List[Dict[str, Any]] = data.get("devices", [])
 
+        # P3-47 truncation observability: scan_full_library persists
+        # stats.category_truncated (keyed by atlas device-category, e.g.
+        # "drum_kits"/"sounds"/"samples") whenever the last scan hit its
+        # max_per_category cap — or ran out of the remote script's shared
+        # iteration safety bound — mid-category. Read it once here so
+        # search()/suggest() callers can be warned their results for a
+        # truncated category are a lower bound, not the full inventory.
+        raw_stats = data.get("stats", {})
+        raw_truncated = (
+            raw_stats.get("category_truncated", {})
+            if isinstance(raw_stats, dict) else {}
+        )
+        self._category_truncated: Dict[str, bool] = {
+            str(cat): bool(hit)
+            for cat, hit in raw_truncated.items()
+        } if isinstance(raw_truncated, dict) else {}
+
         # ── Build indexes ───────────────────────────────────────────
-        self._by_id: Dict[str, Dict[str, Any]] = {}
-        self._by_name: Dict[str, Dict[str, Any]] = {}  # lowercase key
+        # P2-12: _by_id / _by_name keep a LIST of every device sharing a
+        # key, in scan order, so colliding ids/names no longer shadow each
+        # other (last-wins). 719 ids + 702 names collide in the shipped
+        # atlas while URIs are 100% unique, so every device stays reachable
+        # via lookup_all() / atlas_device_info's surfaced URIs. lookup()
+        # still returns a single Dict (the first, deterministic match) to
+        # preserve its Optional[Dict] contract for compare()/browser.py/
+        # _research_engine.py.
+        self._by_id: Dict[str, List[Dict[str, Any]]] = {}
+        self._by_name: Dict[str, List[Dict[str, Any]]] = {}  # lowercase key
         self._by_uri: Dict[str, Dict[str, Any]] = {}
         self._by_category: Dict[str, List[Dict[str, Any]]] = {}
         self._by_tag: Dict[str, List[Dict[str, Any]]] = {}
         self._by_genre: Dict[str, List[Dict[str, Any]]] = {}
         self._by_pack: Dict[str, List[Dict[str, Any]]] = {}
 
+        # Perf batch (v1.27.3): a real 40k-device user atlas can carry
+        # ~16k duplicate ids/names/uris, and warning once per duplicate
+        # produced ~16k logger.warning calls (~122ms + multi-MB of stderr)
+        # at index-build time. Cap each category at _DUP_WARN_CAP explicit
+        # warnings and emit one summary line for the rest.
+        _DUP_WARN_CAP = 20
+        _dup_id_count = 0
+        _dup_name_count = 0
+        _dup_uri_count = 0
+
         for dev in self._devices:
             dev_id = dev.get("id", "")
             dev_name = dev.get("name", "")
@@ -50,26 +99,38 @@ class AtlasManager:
             dev_category = dev.get("category", "")
 
             if dev_id:
-                if dev_id in self._by_id and self._by_id[dev_id] is not dev:
-                    logger.warning(
-                        "atlas: duplicate device id %r shadows a prior entry "
-                        "(name=%r); last-wins", dev_id, dev_name,
-                    )
-                self._by_id[dev_id] = dev
+                bucket = self._by_id.setdefault(dev_id, [])
+                if bucket and dev not in bucket:
+                    _dup_id_count += 1
+                    if _dup_id_count <= _DUP_WARN_CAP:
+                        logger.warning(
+                            "atlas: duplicate device id %r (name=%r); both kept, "
+                            "lookup() returns the first — disambiguate by uri %r",
+                            dev_id, dev_name, dev_uri,
+                        )
+                if dev not in bucket:
+                    bucket.append(dev)
             if dev_name:
                 name_key = dev_name.lower()
-                if name_key in self._by_name and self._by_name[name_key] is not dev:
-                    logger.warning(
-                        "atlas: duplicate device name %r shadows a prior entry "
-                        "(id=%r); last-wins", dev_name, dev_id,
-                    )
-                self._by_name[name_key] = dev
+                name_bucket = self._by_name.setdefault(name_key, [])
+                if name_bucket and dev not in name_bucket:
+                    _dup_name_count += 1
+                    if _dup_name_count <= _DUP_WARN_CAP:
+                        logger.warning(
+                            "atlas: duplicate device name %r (id=%r); both kept, "
+                            "lookup() returns the first — disambiguate by uri %r",
+                            dev_name, dev_id, dev_uri,
+                        )
+                if dev not in name_bucket:
+                    name_bucket.append(dev)
             if dev_uri:
                 if dev_uri in self._by_uri and self._by_uri[dev_uri] is not dev:
-                    logger.warning(
-                        "atlas: duplicate device uri %r shadows a prior entry "
-                        "(id=%r); last-wins", dev_uri, dev_id,
-                    )
+                    _dup_uri_count += 1
+                    if _dup_uri_count <= _DUP_WARN_CAP:
+                        logger.warning(
+                            "atlas: duplicate device uri %r shadows a prior entry "
+                            "(id=%r); last-wins", dev_uri, dev_id,
+                        )
                 self._by_uri[dev_uri] = dev
 
             # Category index
@@ -124,6 +185,22 @@ class AtlasManager:
             if pack_name:
                 self._by_pack.setdefault(pack_name, []).append(dev)
 
+        if _dup_id_count > _DUP_WARN_CAP:
+            logger.warning(
+                "atlas: %d more duplicate device ids suppressed",
+                _dup_id_count - _DUP_WARN_CAP,
+            )
+        if _dup_name_count > _DUP_WARN_CAP:
+            logger.warning(
+                "atlas: %d more duplicate device names suppressed",
+                _dup_name_count - _DUP_WARN_CAP,
+            )
+        if _dup_uri_count > _DUP_WARN_CAP:
+            logger.warning(
+                "atlas: %d more duplicate device uris suppressed",
+                _dup_uri_count - _DUP_WARN_CAP,
+            )
+
     # ── Properties ──────────────────────────────────────────────────
 
     @property
@@ -155,6 +232,49 @@ class AtlasManager:
             },
         }
 
+    # ── Truncation observability (P3-47) ──────────────────────────
+    def truncated_categories(self) -> List[str]:
+        """Categories where the atlas's last scan_full_library run hit
+        its max_per_category cap (or the remote script's shared iteration
+        safety bound) mid-category. Devices in these categories are a
+        lower bound, not the full inventory, until a rescan completes."""
+        return sorted(
+            cat for cat, hit in self._category_truncated.items() if hit
+        )
+
+    def truncation_warning(self, category: str = "all") -> Optional[str]:
+        """Return a warning string if `category` overlaps a category
+        flagged truncated in the loaded atlas's scan stats, else None.
+
+        category="all" (the default `atlas_search`/`atlas_suggest` filter)
+        warns if ANY category was truncated, since an unfiltered query
+        silently includes those partial results alongside complete ones.
+        A specific category only warns when it (or an alias — see
+        _CATEGORY_ALIASES) is in the truncated set.
+        """
+        truncated = self.truncated_categories()
+        if not truncated:
+            return None
+
+        if category == "all":
+            hit = truncated
+        else:
+            allowed = _CATEGORY_ALIASES.get(category, {category})
+            hit = [cat for cat in truncated if cat in allowed]
+            if not hit:
+                return None
+
+        return (
+            "Atlas categor%s %s may be incomplete — the last scan hit its "
+            "size cap for %s. Run scan_full_library(force=True) (raise "
+            "max_per_category if your library is unusually large) for a "
+            "complete inventory before trusting an exhaustive search here."
+        ) % (
+            "y" if len(hit) == 1 else "ies",
+            ", ".join(sorted(hit)),
+            "this category" if len(hit) == 1 else "these categories",
+        )
+
     # ── Pack lookup ────────────────────────────────────────────────
     def pack_info(self, pack_name: str) -> Dict[str, Any]:
         """Return summary of a pack — device list + enrichment coverage.
@@ -217,19 +337,52 @@ class AtlasManager:
     # ── Lookup ──────────────────────────────────────────────────────
 
     def lookup(self, name_or_id: str) -> Optional[Dict[str, Any]]:
-        """Exact match by ID, name (case-insensitive), or URI. Returns None on miss."""
+        """Exact match by ID, name (case-insensitive), or URI. Returns None on miss.
+
+        When an id/name collides across devices (719 ids / 702 names in the
+        shipped atlas), this returns the FIRST device in scan order
+        deterministically. Use lookup_all() to reach every colliding device,
+        or pass a unique URI (URIs never collide) to target a specific one.
+        """
         # Try ID first
-        if name_or_id in self._by_id:
-            return self._by_id[name_or_id]
+        bucket = self._by_id.get(name_or_id)
+        if bucket:
+            return bucket[0]
         # Try name (case-insensitive)
-        lower = name_or_id.lower()
-        if lower in self._by_name:
-            return self._by_name[lower]
-        # Try URI
+        name_bucket = self._by_name.get(name_or_id.lower())
+        if name_bucket:
+            return name_bucket[0]
+        # Try URI — the only key that is guaranteed unique
         if name_or_id in self._by_uri:
             return self._by_uri[name_or_id]
         return None
 
+    def lookup_all(self, name_or_id: str) -> List[Dict[str, Any]]:
+        """Return EVERY device matching an id, name, or URI — collision-aware.
+
+        P2-12: lookup() returns a single Dict (its contract), which shadows
+        the ~719 devices sharing a colliding id and ~702 sharing a name. This
+        companion returns all of them so id/name-keyed tools can disambiguate
+        (e.g. surface each candidate's unique URI). Matches are returned in
+        scan order with duplicates removed; an exact URI match alone returns
+        the single device for that URI.
+        """
+        out: List[Dict[str, Any]] = []
+        seen: set[int] = set()
+
+        def _extend(devs: List[Dict[str, Any]]) -> None:
+            for d in devs:
+                if id(d) not in seen:
+                    seen.add(id(d))
+                    out.append(d)
+
+        _extend(self._by_id.get(name_or_id, []))
+        _extend(self._by_name.get(name_or_id.lower(), []))
+        uri_hit = self._by_uri.get(name_or_id)
+        if uri_hit is not None:
+            _extend([uri_hit])
+        return out
+
     # ── Search ──────────────────────────────────────────────────────
 
     def search(
@@ -243,22 +396,10 @@ class AtlasManager:
         query_words = query_lower.split()
         results: List[Dict[str, Any]] = []
 
-        # BUG-B39: the real atlas scanner emits "instruments" /
-        # "audio_effects" but older callers and test fixtures sometimes
-        # pass the singular "instrument" / "effect". Build a tolerant
-        # category alias set so both forms work.
-        _CAT_ALIASES = {
-            "instrument": {"instrument", "instruments"},
-            "instruments": {"instrument", "instruments"},
-            "effect": {"effect", "effects", "audio_effects"},
-            "effects": {"effect", "effects", "audio_effects"},
-            "audio_effect": {"effect", "effects", "audio_effects",
-                             "audio_effect"},
-            "audio_effects": {"effect", "effects", "audio_effects",
-                              "audio_effect"},
-        }
+        # Tolerant category alias set so "instrument" and "instruments"
+        # (etc.) both work — see module-level _CATEGORY_ALIASES.
         allowed_cats = (
-            _CAT_ALIASES.get(category, {category})
+            _CATEGORY_ALIASES.get(category, {category})
             if category != "all" else None
         )
 

package/mcp_server/atlas/overlays.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 
 import logging
 import re
+import threading
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Optional
@@ -20,6 +21,15 @@ import yaml
 
 logger = logging.getLogger(__name__)
 
+# Prefer the libyaml-backed CSafeLoader (~8.5x faster than the pure-Python
+# SafeLoader) when the yaml package was built with libyaml support. Fall
+# back to SafeLoader if CSafeLoader isn't available (e.g. libyaml missing
+# from the environment) so overlay loading never hard-fails on this.
+try:
+    _YAML_LOADER = yaml.CSafeLoader
+except AttributeError:
+    _YAML_LOADER = yaml.SafeLoader
+
 
 # ─── Tokenizer (used by OverlayIndex.search) ─────────────────────────────────
 # Producer-style queries use natural language with hyphenated phrases
@@ -318,7 +328,14 @@ def load_overlays(root: Optional[Path] = None,
     if root is None:
         root = _resolve_overlay_root()
 
-    idx = get_overlay_index()
+    # Mark loaded up front (not just on success) so a call to load_overlays()
+    # — with either the default or an explicit root — always short-circuits
+    # the lazy _ensure_loaded() path on subsequent get_overlay_index() calls,
+    # including the "root doesn't exist" early-return below.
+    global _overlay_loaded
+    _overlay_loaded = True
+
+    idx = _overlay_index
     idx.clear()
 
     if not root.exists():
@@ -337,7 +354,7 @@ def load_overlays(root: Optional[Path] = None,
                 continue
             try:
                 with yaml_path.open("r") as f:
-                    parsed = yaml.safe_load(f)
+                    parsed = yaml.load(f, Loader=_YAML_LOADER)
             except yaml.YAMLError as e:
                 log.warning(f"overlays: skipped {yaml_path}: {e}")
                 continue
@@ -367,8 +384,34 @@ def load_overlays(root: Optional[Path] = None,
 # request time so they always see current state (never capture a reference).
 _overlay_index: "OverlayIndex" = OverlayIndex()
 
+# Lazy-population state (v1.27.3 perf batch). The singleton used to be
+# populated unconditionally at server import time (server.py boot hook),
+# which cost ~1.1s of a ~2s import on a machine with a populated
+# ~/.livepilot/atlas-overlays tree. Now the first call to
+# get_overlay_index() populates it under a lock; subsequent calls are a
+# cheap flag check. Explicit load_overlays() calls (tests, reload tools)
+# always force a fresh scan and mark the singleton as loaded.
+_overlay_loaded = False
+_overlay_load_lock = threading.Lock()
+
+
+def _ensure_loaded() -> None:
+    """Populate the singleton on first access only. Double-checked locking
+    avoids taking the lock on the (overwhelmingly common) already-loaded path."""
+    global _overlay_loaded
+    if _overlay_loaded:
+        return
+    with _overlay_load_lock:
+        if _overlay_loaded:
+            return
+        load_overlays()
+        _overlay_loaded = True
+
 
 def get_overlay_index() -> "OverlayIndex":
     """Accessor for the live overlay singleton. Always returns the same
-    instance — load_overlays() mutates it in place rather than replacing."""
+    instance — load_overlays() mutates it in place rather than replacing.
+
+    Triggers lazy population on first call (see _ensure_loaded)."""
+    _ensure_loaded()
     return _overlay_index