livepilot 1.23.2 → 1.23.4

package/mcp_server/user_corpus/runner.py

@@ -0,0 +1,261 @@
+"""Scan runner — orchestrates per-file scans across all configured sources.
+
+Reads a Manifest, walks each source's directory, dispatches to the registered
+Scanner for that source's type_id, and writes:
+  - <output_root>/<scanner.output_subdir>/_parses/<slug>.json   (full sidecar)
+  - <output_root>/<scanner.output_subdir>/<slug>.yaml           (search wrapper)
+
+Per-file errors are logged and counted but never abort the whole scan.
+mtime-based incremental skipping is on by default.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import os
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Iterable
+
+import yaml
+
+from .manifest import Manifest, Source, save_manifest
+from .scanner import Scanner, get_scanner
+
+logger = logging.getLogger(__name__)
+
+
+# ─── Result types ────────────────────────────────────────────────────────────
+
+
+@dataclass
+class FileScanResult:
+    path: Path
+    sidecar_path: Path | None = None
+    wrapper_path: Path | None = None
+    skipped: bool = False
+    error: str | None = None
+
+
+@dataclass
+class SourceScanResult:
+    source_id: str
+    type_id: str
+    files_scanned: int = 0
+    files_skipped: int = 0
+    files_errored: int = 0
+    errors: list[tuple[str, str]] = field(default_factory=list)  # (path, msg)
+    elapsed_sec: float = 0.0
+
+
+@dataclass
+class ScanResult:
+    sources: list[SourceScanResult] = field(default_factory=list)
+    total_scanned: int = 0
+    total_skipped: int = 0
+    total_errored: int = 0
+
+
+# ─── Entry point ─────────────────────────────────────────────────────────────
+
+
+def run_scan(
+    manifest: Manifest,
+    only_source_id: str | None = None,
+    update_manifest_path: Path | None = None,
+) -> ScanResult:
+    """Run scans for every source in the manifest (or just one if filtered).
+
+    Always writes sidecars + wrappers to the output root. Returns a ScanResult
+    aggregating per-source counts. If `update_manifest_path` is provided, the
+    runner persists `last_scanned` + `file_count` updates to disk.
+    """
+    skip_unchanged = bool(manifest.options.get("skip_unchanged", True))
+    output_root = manifest.output_root
+
+    overall = ScanResult()
+    for src in manifest.sources:
+        if only_source_id and src.id != only_source_id:
+            continue
+        try:
+            scanner = get_scanner(src.type)
+        except KeyError:
+            logger.warning(
+                "No scanner registered for type_id=%s (source %s) — skipping",
+                src.type, src.id,
+            )
+            continue
+        ssr = _scan_source(src, scanner, output_root, skip_unchanged)
+        overall.sources.append(ssr)
+        overall.total_scanned += ssr.files_scanned
+        overall.total_skipped += ssr.files_skipped
+        overall.total_errored += ssr.files_errored
+
+        # Update the source's metadata
+        src.mark_scanned(file_count=ssr.files_scanned + ssr.files_skipped)
+
+    if update_manifest_path is not None:
+        save_manifest(manifest, update_manifest_path)
+    return overall
+
+
+# ─── Per-source scan ─────────────────────────────────────────────────────────
+
+
+def _scan_source(
+    source: Source,
+    scanner: Scanner,
+    output_root: Path,
+    skip_unchanged: bool,
+) -> SourceScanResult:
+    import time
+    t0 = time.time()
+    ssr = SourceScanResult(source_id=source.id, type_id=source.type)
+
+    src_path = source.resolved_path
+    if not src_path.exists():
+        ssr.errors.append((str(src_path), "Source path does not exist"))
+        return ssr
+
+    excludes = source.exclude_globs or []
+    files = list(_iter_files(src_path, scanner, source.recursive, excludes))
+
+    sub_root = output_root / scanner.output_subdir
+    parses_dir = sub_root / "_parses"
+    parses_dir.mkdir(parents=True, exist_ok=True)
+
+    for f in files:
+        try:
+            slug = f"{source.id}__{scanner.slug(f)}"
+            sidecar_path = parses_dir / f"{slug}.json"
+            wrapper_path = sub_root / f"{slug}.yaml"
+
+            if skip_unchanged and _is_up_to_date(f, sidecar_path):
+                ssr.files_skipped += 1
+                continue
+
+            sidecar = _build_sidecar(scanner, f, source)
+            sidecar_path.write_text(
+                json.dumps(sidecar, indent=2, ensure_ascii=False),
+                encoding="utf-8",
+            )
+
+            wrapper = _build_wrapper(scanner, source, sidecar, sidecar_path)
+            wrapper_path.write_text(
+                yaml.dump(wrapper, sort_keys=False, default_flow_style=False,
+                          width=200, allow_unicode=True),
+                encoding="utf-8",
+            )
+            ssr.files_scanned += 1
+        except Exception as e:  # noqa: BLE001 — never abort over one bad file
+            ssr.files_errored += 1
+            ssr.errors.append((str(f), f"{type(e).__name__}: {e}"))
+            logger.warning("Scan error %s: %s", f, e)
+
+    ssr.elapsed_sec = round(time.time() - t0, 2)
+    return ssr
+
+
+# ─── Helpers ─────────────────────────────────────────────────────────────────
+
+
+def _iter_files(
+    root: Path, scanner: Scanner, recursive: bool, excludes: list[str],
+) -> Iterable[Path]:
+    """Yield files matching scanner extensions, honoring excludes + recursion.
+
+    Filters out:
+      - macOS AppleDouble metadata files (filenames starting with "._")
+      - Hidden files (filenames starting with ".") — usually OS/cache cruft
+      - Anything matching a caller-supplied exclude_glob
+    """
+    walker = root.rglob("*") if recursive else root.glob("*")
+    for p in walker:
+        if not p.is_file():
+            continue
+        # Skip macOS resource-fork siblings and hidden files
+        if p.name.startswith("._") or p.name.startswith("."):
+            continue
+        if not scanner.is_applicable(p):
+            continue
+        if any(p.match(g) for g in excludes):
+            continue
+        yield p
+
+
+def _is_up_to_date(source_file: Path, sidecar_path: Path) -> bool:
+    """True iff the sidecar exists AND its source mtime matches what's recorded."""
+    if not sidecar_path.exists():
+        return False
+    try:
+        sidecar = json.loads(sidecar_path.read_text(encoding="utf-8"))
+    except Exception:  # noqa: BLE001
+        return False
+    recorded_mtime = sidecar.get("source_mtime")
+    if recorded_mtime is None:
+        return False
+    try:
+        return int(recorded_mtime) == int(source_file.stat().st_mtime)
+    except OSError:
+        return False
+
+
+def _build_sidecar(scanner: Scanner, path: Path, source: Source) -> dict:
+    """Run the scanner + wrap with provenance metadata."""
+    data = scanner.scan_one(path)
+    return {
+        "schema_version": scanner.schema_version,
+        "scanner": scanner.type_id,
+        "source_id": source.id,
+        "source_path": str(path),
+        "source_mtime": int(path.stat().st_mtime),
+        "source_sha256": _sha256_short(path),
+        "scan_timestamp": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+        "data": data,
+    }
+
+
+def _sha256_short(path: Path, chunk: int = 1 << 20) -> str:
+    """First 16 hex chars of SHA-256. Cheap fingerprint for change detection."""
+    try:
+        h = hashlib.sha256()
+        with path.open("rb") as fh:
+            while True:
+                buf = fh.read(chunk)
+                if not buf:
+                    break
+                h.update(buf)
+        return h.hexdigest()[:16]
+    except OSError:
+        return ""
+
+
+def _build_wrapper(
+    scanner: Scanner, source: Source, sidecar: dict, sidecar_path: Path,
+) -> dict:
+    """Build the searchable YAML wrapper that the overlay loader indexes."""
+    data = sidecar.get("data") or {}
+    slug = sidecar_path.stem
+    namespace = f"user.{source.id}"
+    return {
+        "entity_id": slug,
+        "entity_type": f"user_{scanner.type_id}",
+        "namespace": namespace,
+        "name": data.get("name") or sidecar_path.stem,
+        "description": scanner.derive_description(data),
+        "tags": _common_tags(source, scanner) + scanner.derive_tags(data),
+        "sidecar_path": str(sidecar_path.relative_to(sidecar_path.parent.parent)),
+        "schema_version": scanner.schema_version,
+    }
+
+
+def _common_tags(source: Source, scanner: Scanner) -> list[str]:
+    """Tags every wrapper gets regardless of scanner type."""
+    return [
+        f"scanner:{scanner.type_id}",
+        f"source:{source.id}",
+        f"namespace:user.{source.id}",
+    ]

package/mcp_server/user_corpus/scanner.py

@@ -0,0 +1,115 @@
+"""Scanner abstract base class + registry.
+
+A Scanner reads ONE file of its type (e.g. one .als, one .adg, one .amxd) and
+returns a JSON-serializable sidecar dict. It also derives searchable tags +
+description from that sidecar so the corpus runner can build a YAML wrapper.
+
+Plug in your own:
+
+    from mcp_server.user_corpus.scanner import Scanner, register_scanner
+
+    @register_scanner
+    class MyScanner(Scanner):
+        type_id = "my-format"
+        file_extensions = [".myx"]
+        output_subdir = "my_format"
+
+        def scan_one(self, path):
+            return {"data": ...}
+
+        def derive_tags(self, sidecar):
+            return ["my-format", "..."]
+
+        def derive_description(self, sidecar):
+            return "..."
+
+The decorator self-registers in the global SCANNERS dict; the runner discovers
+it automatically as long as the module is imported somewhere.
+"""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import ClassVar
+
+
+class Scanner(ABC):
+    """Per-file content scanner.
+
+    Subclasses must define:
+      type_id           — unique short id, e.g. "als"
+      file_extensions   — list of lowercase extensions, e.g. [".als"]
+      output_subdir     — where sidecars land under the output root, e.g. "projects"
+      schema_version    — bump when sidecar shape changes (defaults to 1)
+
+    And implement:
+      scan_one(path)             — parse one file → dict
+      derive_tags(sidecar)       — searchable tags
+      derive_description(sidecar)— human-readable one-liner
+
+    Optional overrides:
+      slug(path)         — how to name the sidecar file (default: lowercased stem)
+      is_applicable(p)   — file-applicability predicate (default: extension match)
+    """
+
+    type_id: ClassVar[str]
+    file_extensions: ClassVar[list[str]]
+    output_subdir: ClassVar[str]
+    schema_version: ClassVar[int] = 1
+
+    @abstractmethod
+    def scan_one(self, path: Path) -> dict:
+        ...
+
+    @abstractmethod
+    def derive_tags(self, sidecar: dict) -> list[str]:
+        ...
+
+    @abstractmethod
+    def derive_description(self, sidecar: dict) -> str:
+        ...
+
+    def is_applicable(self, path: Path) -> bool:
+        return path.suffix.lower() in self.file_extensions
+
+    def slug(self, path: Path) -> str:
+        """Filesystem-safe slug derived from the file's stem.
+
+        Lowercase, spaces and underscores → hyphens, strip the suffix.
+        Override to disambiguate (e.g. include parent folder when filenames collide).
+        """
+        s = path.stem.lower()
+        s = s.replace(" ", "-").replace("_", "-")
+        return "-".join(part for part in s.split("-") if part)
+
+
+# ─── Registry ────────────────────────────────────────────────────────────────
+
+SCANNERS: dict[str, type[Scanner]] = {}
+
+
+def register_scanner(cls: type[Scanner]) -> type[Scanner]:
+    """Decorator: registers a Scanner subclass under its type_id.
+
+    Idempotent — re-registering the same type_id silently overrides the prior
+    registration (useful for hot-reload during development; do NOT depend on
+    re-registration for production correctness).
+    """
+    if not issubclass(cls, Scanner):
+        raise TypeError(f"{cls.__name__} is not a Scanner subclass")
+    if not getattr(cls, "type_id", None):
+        raise ValueError(f"{cls.__name__} must define type_id")
+    SCANNERS[cls.type_id] = cls
+    return cls
+
+
+def get_scanner(type_id: str) -> Scanner:
+    """Instantiate a registered scanner by type_id. Raises KeyError if missing."""
+    cls = SCANNERS[type_id]
+    return cls()
+
+
+def list_scanners() -> list[str]:
+    """Return all registered scanner type_ids in insertion order."""
+    return list(SCANNERS.keys())

package/mcp_server/user_corpus/scanners/__init__.py

@@ -0,0 +1,18 @@
+"""Built-in scanners for the user corpus builder.
+
+Each module in this package registers its Scanner subclass via the
+@register_scanner decorator. The package __init__ in mcp_server.user_corpus
+imports all of them eagerly so the registry is populated at import time.
+
+To add your own scanner:
+
+    # mcp_server/user_corpus/scanners/my_format.py
+    from ..scanner import Scanner, register_scanner
+
+    @register_scanner
+    class MyScanner(Scanner):
+        type_id = "my-format"
+        ...
+
+Then reference it in your manifest as `type: my-format`.
+"""

package/mcp_server/user_corpus/scanners/adg.py

@@ -0,0 +1,79 @@
+"""ADG scanner — wraps scripts/als_deep_parse.parse_adg for one-file use.
+
+Reuses the production-grade preset parser:
+  - Macro display names via KeyMidi binding resolution (BUG-PARSER#2 fix)
+  - Branch / chain extraction
+  - Device summary
+
+Same module as the .als parser; we share the lazy importer.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from ..scanner import Scanner, register_scanner
+from .als import _als_parse  # share the lazy importer
+
+
+@register_scanner
+class AdgScanner(Scanner):
+    type_id = "adg"
+    file_extensions = [".adg", ".adv"]
+    output_subdir = "racks"
+    schema_version = 1
+
+    def scan_one(self, path: Path) -> dict:
+        parser = _als_parse()
+        return parser.parse_adg(str(path))
+
+    def derive_tags(self, sidecar: dict) -> list[str]:
+        tags: list[str] = ["rack-preset"]
+        rack_class = sidecar.get("rack_class")
+        if rack_class:
+            tags.append(_class_to_tag(rack_class))
+        preset_type = sidecar.get("preset_type")
+        if preset_type:
+            tags.append(preset_type)
+
+        # Top 3 named macros (already canonicalized by the parser)
+        for m in (sidecar.get("macros") or [])[:8]:
+            name = (m.get("name") or "").strip()
+            if name and not name.startswith("Macro "):
+                slug = _slug(name)
+                if slug:
+                    tags.append(f"macro:{slug}")
+
+        if sidecar.get("chains"):
+            tags.append("has-chains")
+        bc = sidecar.get("branch_counts") or {}
+        if isinstance(bc, dict) and sum(bc.values()) > 1:
+            tags.append("multi-branch")
+        return tags
+
+    def derive_description(self, sidecar: dict) -> str:
+        rack_class = sidecar.get("rack_class") or "rack"
+        macros = sidecar.get("macros") or []
+        n_named = sum(
+            1 for m in macros
+            if (m.get("name") or "").strip() and not (m.get("name") or "").startswith("Macro ")
+        )
+        n_branches = sum((sidecar.get("branch_counts") or {}).values())
+        return (
+            f"{rack_class} preset with {len(macros)} macros "
+            f"({n_named} producer-named), {n_branches} branches"
+        )
+
+
+def _slug(s: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")
+
+
+def _class_to_tag(rack_class: str) -> str:
+    return {
+        "InstrumentGroupDevice": "instrument-rack",
+        "AudioEffectGroupDevice": "audio-effect-rack",
+        "MidiEffectGroupDevice": "midi-effect-rack",
+        "DrumGroupDevice": "drum-rack",
+    }.get(rack_class, _slug(rack_class))

package/mcp_server/user_corpus/scanners/als.py

@@ -0,0 +1,144 @@
+"""ALS scanner — wraps scripts/als_deep_parse.parse_als for one-file use.
+
+Reuses the production-grade parser already used by the factory pack atlas:
+  - Macro-value extraction (BUG-PARSER#1 fix: direct child scan, not recursive iter)
+  - Numeric scale-mode decode for Live 9/10 files (BUG-PARSER#3)
+  - Filename-key fallback for construction-kit packs (BUG-PARSER#4)
+  - Nested chain recursion (BUG-C#2 schema A)
+  - PluginDevice metadata (BUG-PARSER#5)
+
+The user corpus inherits all of these for free.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import re
+import sys
+from pathlib import Path
+from typing import Any
+
+from ..scanner import Scanner, register_scanner
+
+
+# ─── Lazy import of scripts/als_deep_parse.py ────────────────────────────────
+
+_ALS_PARSE_MODULE = None
+
+
+def _als_parse():
+    """Load scripts/als_deep_parse.py once and cache it.
+
+    Lives outside the mcp_server package, so we use importlib by path.
+    """
+    global _ALS_PARSE_MODULE
+    if _ALS_PARSE_MODULE is not None:
+        return _ALS_PARSE_MODULE
+    repo_root = Path(__file__).resolve().parents[3]
+    parser_path = repo_root / "scripts" / "als_deep_parse.py"
+    if not parser_path.exists():
+        raise ImportError(f"als_deep_parse.py not found at {parser_path}")
+    spec = importlib.util.spec_from_file_location(
+        "_als_deep_parse_user_corpus", parser_path,
+    )
+    if spec is None or spec.loader is None:
+        raise ImportError(f"Could not load {parser_path}")
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["_als_deep_parse_user_corpus"] = module
+    spec.loader.exec_module(module)
+    _ALS_PARSE_MODULE = module
+    return module
+
+
+# ─── Scanner ────────────────────────────────────────────────────────────────
+
+
+@register_scanner
+class AlsScanner(Scanner):
+    type_id = "als"
+    file_extensions = [".als"]
+    output_subdir = "projects"
+    schema_version = 1
+
+    def scan_one(self, path: Path) -> dict:
+        parser = _als_parse()
+        return parser.parse_als(str(path))
+
+    def derive_tags(self, sidecar: dict) -> list[str]:
+        tags: list[str] = ["als-project"]
+        bpm = sidecar.get("bpm")
+        if isinstance(bpm, (int, float)):
+            tags.append(f"{int(round(bpm))}bpm")
+        scale = sidecar.get("scale") or {}
+        scale_name = scale.get("name")
+        if scale_name and scale_name != "Major":  # don't add the default
+            tags.append(scale_name.lower())
+        if scale.get("source") == "filename-fallback":
+            tags.append("filename-key")
+
+        # Top device classes from track inventory (recursive into chains)
+        seen: dict[str, int] = {}
+        for cls in _walk_device_classes(sidecar.get("tracks") or []):
+            seen[cls] = seen.get(cls, 0) + 1
+        for cls, _ in sorted(seen.items(), key=lambda kv: -kv[1])[:5]:
+            tags.append(f"has-{_slug(cls)}")
+
+        # Track-count bucket
+        n_tracks = len(sidecar.get("tracks") or [])
+        if n_tracks > 0:
+            if n_tracks < 5:
+                tags.append("small-session")
+            elif n_tracks < 16:
+                tags.append("medium-session")
+            else:
+                tags.append("large-session")
+        return tags
+
+    def derive_description(self, sidecar: dict) -> str:
+        bpm = sidecar.get("bpm")
+        bpm_str = f"{int(round(bpm))} BPM" if isinstance(bpm, (int, float)) else "unknown BPM"
+        scale = sidecar.get("scale") or {}
+        root = scale.get("root_note", "")
+        name = scale.get("name", "")
+        scale_str = _format_scale(root, name)
+        n_tracks = len(sidecar.get("tracks") or [])
+        return f"{bpm_str}, {scale_str}, {n_tracks} tracks"
+
+
+# ─── Helpers ─────────────────────────────────────────────────────────────────
+
+
+def _slug(s: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", s.lower()).strip("-")
+
+
+def _format_scale(root: Any, name: str) -> str:
+    root_names = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"]
+    try:
+        i = int(str(root))
+        if 0 <= i < 12:
+            return f"{root_names[i]} {name}"
+    except (ValueError, TypeError):
+        pass
+    return name or "unknown scale"
+
+
+def _walk_device_classes(tracks: list[dict]) -> list[str]:
+    """Yield every device class across tracks AND nested rack chains."""
+    out: list[str] = []
+    for t in tracks:
+        out.extend(_walk_devices(t.get("devices") or []))
+    return out
+
+
+def _walk_devices(devices: list[dict], depth: int = 0) -> list[str]:
+    if depth > 6:  # defensive cap
+        return []
+    out: list[str] = []
+    for d in devices:
+        cls = d.get("class") or ""
+        if cls:
+            out.append(cls)
+        for chain in d.get("chains") or []:
+            out.extend(_walk_devices(chain.get("devices") or [], depth + 1))
+    return out