livepilot 1.23.2 → 1.23.4

package/mcp_server/user_corpus/plugin_engine/manual.py

@@ -0,0 +1,347 @@
+"""Phase 2.3 + 2.4 — Manual discovery + extraction.
+
+Given a DetectedPlugin, glob the standard manual locations + extract text from
+the most promising file. Supports .pdf (pypdf with pdfplumber fallback), .html
+(bs4), .md / .txt / .rtf (raw read).
+
+Section splitter recognizes common chapter heading patterns and produces a
+sections.json mapping section title → text range.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+from .detector import DetectedPlugin
+
+logger = logging.getLogger(__name__)
+
+
+_MANUAL_EXTENSIONS = {".pdf", ".html", ".htm", ".md", ".txt", ".rtf"}
+_MANUAL_NAME_HINTS = (
+    "manual", "guide", "reference", "documentation", "user", "handbook", "readme",
+)
+# Anti-hints: filenames containing these are NOT plugin manuals even if the
+# extension matches. Filters out the log/cache/preset noise that lives in
+# Application Support directories.
+_MANUAL_NAME_ANTIHINTS = (
+    "log", "cache", "settings", "prefs", "preferences", "tmp", "temp",
+    "crash", "error", "debug", "trace", "history", "registry", "license",
+    "credits", "thirdparty", "third-party", "third_party", "uninstall",
+    "lockfile", "lock-file", ".lock",
+)
+# Non-extension names we also reject (case-insensitive substring on the stem)
+_MANUAL_REJECT_STEMS = (
+    "package-info", "buildinfo", "version", "checksum",
+)
+
+
+@dataclass
+class ManualCandidate:
+    path: str
+    extension: str
+    size_kb: int
+    name_score: float          # higher = filename more clearly a manual
+    location_score: int        # higher = more authoritative location
+
+    def __lt__(self, other: "ManualCandidate") -> bool:
+        # sort: location first, then name score, then size descending
+        return (
+            self.location_score, self.name_score, self.size_kb,
+        ) > (
+            other.location_score, other.name_score, other.size_kb,
+        )
+
+
+@dataclass
+class ManualExtraction:
+    plugin_id: str
+    source_path: str
+    source_kind: str               # pdf | html | md | txt | rtf
+    text: str
+    char_count: int
+    page_count: int | None = None
+    sections: list[dict] = field(default_factory=list)  # [{title, start, end}]
+    truncated: bool = False
+    error: str | None = None
+
+
+# ─── Discovery ──────────────────────────────────────────────────────────────
+
+
+def discover_manuals_for_plugin(
+    plugin: DetectedPlugin,
+    extra_search_dirs: list[Path] | None = None,
+    max_candidates: int = 20,
+) -> list[ManualCandidate]:
+    """Glob the standard manual locations and return ranked candidates.
+
+    Search priority (high to low):
+      1. Inside the bundle: <bundle>/Contents/Resources/
+      2. /Applications/<vendor>*.app/Contents/Resources/
+      3. /Library/Audio/Plug-Ins/<format>/<vendor>/Documentation/
+      4. ~/Documents/<vendor>/, ~/Documents/<plugin>/
+      5. /Library/Application Support/<vendor>/
+    """
+    candidates: list[ManualCandidate] = []
+    bundle = Path(plugin.bundle_path)
+    vendor = plugin.vendor or ""
+    name = plugin.name
+
+    search_locations = _enumerate_search_locations(bundle, vendor, name)
+    for extra in (extra_search_dirs or []):
+        search_locations.append((extra, 1))  # extras get baseline location score
+
+    for loc, loc_score in search_locations:
+        if not loc.exists():
+            continue
+        try:
+            for path in _walk_for_manuals(loc, max_depth=4):
+                cand = _score_candidate(path, loc_score)
+                if cand:
+                    candidates.append(cand)
+                if len(candidates) >= max_candidates * 3:
+                    break
+        except (PermissionError, OSError):
+            continue
+
+    # Dedupe by path; sort
+    seen: dict[str, ManualCandidate] = {}
+    for c in candidates:
+        if c.path not in seen or c < seen[c.path]:
+            seen[c.path] = c
+    ranked = sorted(seen.values())
+    return ranked[:max_candidates]
+
+
+def _enumerate_search_locations(
+    bundle: Path, vendor: str, name: str,
+) -> list[tuple[Path, int]]:
+    """Build the (path, location_score) list to search. Higher score = more authoritative."""
+    home = Path.home()
+    out: list[tuple[Path, int]] = []
+    # Score 5: inside the bundle
+    out.append((bundle / "Contents" / "Resources", 5))
+    # Score 4: vendor app + format-specific docs folder
+    if vendor:
+        for slug in (vendor, vendor.replace(" ", ""), vendor.lower()):
+            apps = Path("/Applications").glob(f"*{slug}*.app")
+            for app in apps:
+                out.append((app / "Contents" / "Resources", 4))
+            out.append((Path(f"/Library/Audio/Plug-Ins/VST3/{slug}/Documentation"), 4))
+            out.append((Path(f"/Library/Audio/Plug-Ins/Components/{slug}/Documentation"), 4))
+    # Score 3: ~/Documents/<vendor>/, ~/Documents/<plugin>/
+    if vendor:
+        out.append((home / "Documents" / vendor, 3))
+    out.append((home / "Documents" / name, 3))
+    # Score 2: /Library/Application Support/<vendor>/
+    if vendor:
+        out.append((Path("/Library/Application Support") / vendor, 2))
+        out.append((home / "Library" / "Application Support" / vendor, 2))
+    return out
+
+
+def _walk_for_manuals(root: Path, max_depth: int = 4):
+    """Yield manual-extension files within a directory tree (depth-bounded)."""
+    if not root.exists():
+        return
+    base_depth = len(root.parts)
+    try:
+        for path in root.rglob("*"):
+            try:
+                if not path.is_file():
+                    continue
+                if (len(path.parts) - base_depth) > max_depth:
+                    continue
+                if path.suffix.lower() in _MANUAL_EXTENSIONS:
+                    yield path
+            except (PermissionError, OSError):
+                continue
+    except (PermissionError, OSError):
+        return
+
+
+def _score_candidate(path: Path, location_score: int) -> ManualCandidate | None:
+    try:
+        size_kb = int(path.stat().st_size / 1024)
+    except OSError:
+        return None
+    if size_kb < 1:
+        return None  # 0-byte files
+    name_lower = path.name.lower()
+    stem_lower = path.stem.lower()
+
+    # Reject if any anti-hint matches — these are logs/caches/settings, not manuals
+    for anti in _MANUAL_NAME_ANTIHINTS:
+        if anti in name_lower:
+            return None
+    for anti in _MANUAL_REJECT_STEMS:
+        if anti in stem_lower:
+            return None
+    # Reject any path whose immediate parent dir is named "logs", "cache", "tmp"
+    parent_lower = path.parent.name.lower()
+    if parent_lower in ("logs", "cache", "tmp", "temp", "crash", "trash"):
+        return None
+
+    name_score = 0.0
+    for hint in _MANUAL_NAME_HINTS:
+        if hint in name_lower:
+            name_score += 1.0
+    # Strong prior for "manual" specifically
+    if "manual" in name_lower:
+        name_score += 1.5
+    if path.suffix.lower() == ".pdf":
+        name_score += 0.5
+    # Require at least SOME positive signal: a name hint OR a PDF in a
+    # high-priority location. This blocks generic .txt files that aren't manuals.
+    if name_score == 0.0 and location_score < 4:
+        return None
+    return ManualCandidate(
+        path=str(path), extension=path.suffix.lower(),
+        size_kb=size_kb, name_score=name_score, location_score=location_score,
+    )
+
+
+# ─── Extraction ─────────────────────────────────────────────────────────────
+
+
+_MAX_CHAR_OUTPUT = 1_000_000   # cap so a 5000-page manual doesn't blow context
+
+
+def extract_manual_text(
+    plugin: DetectedPlugin, candidate: ManualCandidate,
+) -> ManualExtraction:
+    """Extract text from a manual candidate. Never raises."""
+    path = Path(candidate.path)
+    ext = candidate.extension
+    try:
+        if ext == ".pdf":
+            text, pages = _extract_pdf(path)
+            kind = "pdf"
+        elif ext in (".html", ".htm"):
+            text = _extract_html(path)
+            pages = None
+            kind = "html"
+        elif ext == ".rtf":
+            text = _extract_rtf(path)
+            pages = None
+            kind = "rtf"
+        else:
+            text = path.read_text(encoding="utf-8", errors="ignore")
+            pages = None
+            kind = ext.lstrip(".")
+    except Exception as e:  # noqa: BLE001
+        return ManualExtraction(
+            plugin_id=plugin.plugin_id, source_path=str(path),
+            source_kind=ext.lstrip("."), text="", char_count=0,
+            page_count=None, sections=[], truncated=False,
+            error=f"{type(e).__name__}: {e}",
+        )
+
+    truncated = False
+    if len(text) > _MAX_CHAR_OUTPUT:
+        text = text[:_MAX_CHAR_OUTPUT]
+        truncated = True
+
+    sections = _detect_sections(text)
+    return ManualExtraction(
+        plugin_id=plugin.plugin_id, source_path=str(path),
+        source_kind=kind, text=text, char_count=len(text),
+        page_count=pages, sections=sections, truncated=truncated,
+    )
+
+
+def _extract_pdf(path: Path) -> tuple[str, int]:
+    """Extract text from a PDF. Try pypdf first, fall back to pdfplumber."""
+    pages = 0
+    try:
+        import pypdf
+        reader = pypdf.PdfReader(str(path))
+        chunks: list[str] = []
+        for page in reader.pages:
+            try:
+                chunks.append(page.extract_text() or "")
+            except Exception:  # noqa: BLE001
+                continue
+            pages += 1
+        text = "\n\n".join(chunks)
+        if text.strip():
+            return text, pages
+    except Exception as e:  # noqa: BLE001
+        logger.debug("pypdf failed on %s: %s; trying pdfplumber", path, e)
+    # Fallback
+    try:
+        import pdfplumber
+        chunks: list[str] = []
+        with pdfplumber.open(str(path)) as pdf:
+            for page in pdf.pages:
+                try:
+                    chunks.append(page.extract_text() or "")
+                except Exception:  # noqa: BLE001
+                    continue
+                pages += 1
+        return "\n\n".join(chunks), pages
+    except Exception as e:  # noqa: BLE001
+        logger.warning("pdfplumber also failed on %s: %s", path, e)
+        raise
+
+
+def _extract_html(path: Path) -> str:
+    from bs4 import BeautifulSoup
+    raw = path.read_text(encoding="utf-8", errors="ignore")
+    soup = BeautifulSoup(raw, "html.parser")
+    for tag in soup(["script", "style", "nav", "footer"]):
+        tag.decompose()
+    return soup.get_text(separator="\n", strip=True)
+
+
+def _extract_rtf(path: Path) -> str:
+    """Crude RTF stripper — drops control words + groups."""
+    raw = path.read_text(encoding="latin-1", errors="ignore")
+    # Strip control words like \word123 and groups
+    text = re.sub(r"\\[a-z]+-?\d* ?", "", raw)
+    text = re.sub(r"[{}]", "", text)
+    return text
+
+
+# ─── Section detection ──────────────────────────────────────────────────────
+
+
+_HEADING_PATTERNS = [
+    re.compile(r"^\s*(?:CHAPTER\s+\d+\s*[:.\s-]+)?([A-Z][A-Z0-9 \-&,'/]{3,60})\s*$",
+               re.MULTILINE),
+    re.compile(r"^\s*\d+(?:\.\d+)?\s+([A-Z][\w\s\-&,'/]{3,60})\s*$", re.MULTILINE),
+    re.compile(r"^#{1,3}\s+(.{3,80}?)\s*$", re.MULTILINE),  # markdown
+]
+
+_KNOWN_SECTION_KEYWORDS = (
+    "parameters", "controls", "modulation", "matrix", "envelopes", "lfo",
+    "filter", "filters", "oscillator", "oscillators", "effects", "fx",
+    "presets", "tutorials", "tutorial", "introduction", "overview",
+    "installation", "getting started", "performance", "automation",
+    "midi", "macros", "global", "interface", "browser", "library",
+)
+
+
+def _detect_sections(text: str) -> list[dict]:
+    """Best-effort section splitter. Returns at most 50 sections."""
+    headings: list[tuple[int, str]] = []
+    for pat in _HEADING_PATTERNS:
+        for m in pat.finditer(text):
+            title = m.group(1).strip()
+            start = m.start()
+            if 4 <= len(title) <= 60 and title not in (h[1] for h in headings):
+                # Bias toward known section keywords
+                low = title.lower()
+                score = sum(1 for kw in _KNOWN_SECTION_KEYWORDS if kw in low)
+                if score > 0 or len(headings) < 30:
+                    headings.append((start, title))
+    headings.sort()
+    sections: list[dict] = []
+    for i, (start, title) in enumerate(headings[:50]):
+        end = headings[i + 1][0] if i + 1 < len(headings) else len(text)
+        sections.append({"title": title, "start": start, "end": end})
+    return sections

package/mcp_server/user_corpus/plugin_engine/research.py

@@ -0,0 +1,247 @@
+"""Phase 3 + 4 — Research target builder + Synthesis brief builder.
+
+These don't call WebSearch or Anthropic directly. They emit STRUCTURED TASK
+PACKETS that the Claude agent (in Claude Code) fulfills via WebSearch +
+WebFetch + Agent dispatch (sonnet subagents). This keeps the corpus engine
+portable + lets Claude Code's permission model gate every external call.
+
+See docs/PLUGIN_KNOWLEDGE_ENGINE.md §"Why the agent-driven split".
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from .detector import DetectedPlugin
+from .manual import ManualExtraction
+
+
+# ─── Phase 3: research target packet ────────────────────────────────────────
+
+
+def build_research_targets(
+    plugin: DetectedPlugin,
+    local_manual: ManualExtraction | None = None,
+    output_root: Path | None = None,
+) -> dict:
+    """Emit a structured packet of research queries for the agent to fulfill.
+
+    The packet declares:
+      - what we already know (identity + local manual presence)
+      - what queries to run (manual_alt, technique_corpus, comparison)
+      - where to cache results
+      - what tool to call next (corpus_synthesize_plugin_identity)
+    """
+    name = plugin.name
+    vendor = plugin.vendor or "unknown vendor"
+    output_dir = (
+        (output_root or _default_output_root()) / "plugins" / plugin.plugin_id / "research"
+    )
+    has_local_manual = bool(local_manual and local_manual.text and not local_manual.error)
+
+    targets: list[dict] = []
+
+    # Target 1: alternate / verified manual (always emit; even with local manual,
+    # online may have a newer version)
+    targets.append({
+        "type": "manual_alt",
+        "rationale": (
+            "Verify local manual is current OR fetch one if no local manual exists"
+            if has_local_manual else
+            "No local manual found; locate the official PDF/web manual"
+        ),
+        "queries": _manual_queries(name, vendor),
+        "priority": 1,
+        "fetch_top_n": 1 if has_local_manual else 2,
+    })
+
+    # Target 2: technique corpus (recipes, sound design walkthroughs)
+    targets.append({
+        "type": "technique_corpus",
+        "rationale": "Build a producer-oriented technique library — concrete recipes, sweet spots, common patches",
+        "queries": _technique_queries(name, vendor),
+        "priority": 2,
+        "fetch_top_n": 5,
+    })
+
+    # Target 3: comparisons (when to reach for vs alternatives)
+    targets.append({
+        "type": "comparison",
+        "rationale": "Determine when to reach for this plugin vs comparable alternatives",
+        "queries": _comparison_queries(name, vendor),
+        "priority": 3,
+        "fetch_top_n": 3,
+    })
+
+    return {
+        "plugin_id": plugin.plugin_id,
+        "plugin_identity": {
+            "name": name, "vendor": vendor, "format": plugin.format,
+            "version": plugin.version, "unique_id": plugin.unique_id,
+        },
+        "local_manual_present": has_local_manual,
+        "local_manual_path": local_manual.source_path if has_local_manual else None,
+        "research_targets": targets,
+        "cache_root": str(output_dir),
+        "instructions": (
+            "Use WebSearch + WebFetch to fulfill each target. For each query, "
+            "cache the top result(s) under cache_root/<target_type>/<n>.txt + "
+            "<n>_url.txt (the source URL). Stamp each cached file with a "
+            "retrieval timestamp comment in the first line. Failures "
+            "(no results, paywall, 404) are logged to cache_root/search_log.json "
+            "and do not abort other targets."
+        ),
+        "next_step_tool": "corpus_emit_synthesis_briefs",
+        "schema_version": 1,
+    }
+
+
+def _manual_queries(name: str, vendor: str) -> list[str]:
+    out = [
+        f'{vendor} {name} manual pdf',
+        f'{vendor} {name} user guide',
+    ]
+    if vendor and vendor != "unknown vendor":
+        out.append(f'site:{vendor.lower().split()[0]}.com {name} manual')
+    return out
+
+
+def _technique_queries(name: str, vendor: str) -> list[str]:
+    return [
+        f'{name} sound design tutorial',
+        f'{name} bass patch',
+        f'{name} pad evolving',
+        f'{name} lead sound',
+        f'{name} preset walkthrough',
+        f'how to use {name}',
+    ]
+
+
+def _comparison_queries(name: str, vendor: str) -> list[str]:
+    return [
+        f'{name} vs alternatives review',
+        f'{name} when to use',
+        f'{name} pros cons',
+    ]
+
+
+# ─── Phase 4: synthesis brief ───────────────────────────────────────────────
+
+
+def build_synthesis_brief(
+    plugin: DetectedPlugin,
+    local_manual: ManualExtraction | None = None,
+    research_cache_root: Path | None = None,
+    preset_examples: list[dict] | None = None,
+    output_root: Path | None = None,
+) -> dict:
+    """Emit a self-contained brief for a sonnet subagent to write identity.yaml.
+
+    The agent calls Anthropic-API (via Agent tool) with this brief. The brief
+    contains all source data the subagent needs. The subagent writes one YAML
+    at brief["output_path"].
+    """
+    out_root = output_root or _default_output_root()
+    plugin_dir = out_root / "plugins" / plugin.plugin_id
+    output_path = plugin_dir / "identity.yaml"
+
+    # Manual extract — either full text (capped) or a sectioned digest
+    manual_block: dict[str, Any] = {}
+    if local_manual and local_manual.text and not local_manual.error:
+        manual_block = {
+            "source_path": local_manual.source_path,
+            "source_kind": local_manual.source_kind,
+            "char_count": local_manual.char_count,
+            "page_count": local_manual.page_count,
+            "section_titles": [s["title"] for s in (local_manual.sections or [])][:30],
+            "text_preview": local_manual.text[:6000],   # first ~1000 words
+        }
+
+    # Research cache — list whatever's there as URLs + previews
+    research_block: dict[str, Any] = {"available": False}
+    if research_cache_root and Path(research_cache_root).exists():
+        research_block = _summarize_research_cache(Path(research_cache_root))
+
+    return {
+        "plugin_id": plugin.plugin_id,
+        "synthesis_inputs": {
+            "identity": {
+                "name": plugin.name, "vendor": plugin.vendor,
+                "format": plugin.format, "version": plugin.version,
+                "unique_id": plugin.unique_id,
+            },
+            "local_manual": manual_block,
+            "research_cache": research_block,
+            "preset_examples": (preset_examples or [])[:30],
+        },
+        "synthesis_schema": {
+            # ── REQUIRED overlay fields (so the file is queryable via extension_atlas_search) ──
+            "entity_id": "EXACTLY equal to plugin_id — required for overlay indexing",
+            "entity_type": 'literal string "installed_plugin" — required for overlay indexing',
+            "name": "human-readable plugin name (same as plugin's 'name' field)",
+            "description": "ONE short paragraph (≤200 chars) used as the search-result summary; can be the first sentence of sonic_fingerprint",
+            "tags": (
+                "list of search tags — MUST include: 'installed-plugin', "
+                "'vendor:<vendor-slug>', and EXACTLY ONE 'format:<primary>' tag — "
+                "VST3 preferred when available, otherwise AU, otherwise AAX/VST2/CLAP/LV2. "
+                "Do NOT emit multiple format:* tags — the full format list goes in "
+                "'formats_available' for reference, but only the primary format is queryable. "
+                "This avoids token waste and duplicate hits when a plugin ships in 4+ formats. "
+                "Plus one 'genre:<slug>' per genre_affinity entry, one 'producer:<slug>' "
+                "per producer_anchors entry, and any signature descriptors "
+                "('shimmer-capable', 'self-sustaining', 'parallel-only', etc.)."
+            ),
+            "artists": "list of producer names from producer_anchors keys (overlay search ranks artist matches +50)",
+            # ── Plugin-specific knowledge ──
+            "sonic_fingerprint": "3-5 sentences describing what the plugin SOUNDS like + its signature character. Must reference concrete sonic qualities (warmth/grit/digital/analog/etc.), not just architecture.",
+            "reach_for": "list of bullet points: when/why a producer would pick this plugin",
+            "avoid": "list of bullet points: when this plugin is the wrong tool",
+            "key_techniques": "list of producer-oriented recipes — each ideally citing specific parameter ranges or modulation routings",
+            "parameter_glossary": "dict of dial → 1-line description, capped at 12 most important",
+            "comparable_plugins": "list of {name, when_better} entries",
+            "genre_affinity": "list of genre slugs (microhouse, dub_techno, hip_hop_boom_bap_lo_fi, etc.)",
+            "producer_anchors": "list of {producer_name: rationale} entries — only cite when the research clearly supports the mapping",
+        },
+        "agent_instructions": (
+            "Read synthesis_inputs in full. Write a YAML file at output_path "
+            "matching synthesis_schema. The first 6 fields (entity_id, entity_type, "
+            "name, description, tags, artists) are REQUIRED for the overlay loader "
+            "to index the result — without them the file is unqueryable. "
+            "Be concrete, not generic — every claim should be traceable to a "
+            "specific sentence in manual or research_cache. Do not invent producer "
+            "attributions; only cite anchors when the plugin's character clearly "
+            "maps to a producer's documented sound. Cap output at ~3000 tokens."
+        ),
+        "output_path": str(output_path),
+        "schema_version": 1,
+    }
+
+
+def _summarize_research_cache(root: Path) -> dict:
+    """Walk a research/ cache dir and produce a manifest the brief can include."""
+    if not root.exists():
+        return {"available": False}
+    entries: list[dict] = []
+    for sub in sorted(root.iterdir()):
+        if not sub.is_dir():
+            continue
+        for f in sorted(sub.glob("*.txt")):
+            try:
+                preview = f.read_text(encoding="utf-8", errors="ignore")[:2000]
+            except OSError:
+                preview = ""
+            entries.append({
+                "target_type": sub.name,
+                "path": str(f),
+                "preview": preview,
+            })
+    return {
+        "available": bool(entries),
+        "entry_count": len(entries),
+        "entries": entries[:40],   # cap so the brief stays reasonable size
+    }
+
+
+def _default_output_root() -> Path:
+    return Path.home() / ".livepilot" / "atlas-overlays" / "user"