@cloudgrid-io/mcp 0.13.0 → 0.15.0

package/src/corpus/templates/semantic-search/services/backend/src/app/search.py

@@ -0,0 +1,222 @@
+"""CloudGrid-native hybrid search over Mongo.
+
+Modes blended in one query:
+  - lexical/exact : Mongo $text index over chunks.content (+ synonym expansion)
+  - semantic      : in-app cosine similarity (NumPy) over chunk embedding arrays
+  - metadata      : Mongo filters on documents (date, collection)
+
+Results aggregate to the document level; each result carries its single best
+matching passage with a highlight. Counting questions use countDocuments.
+
+Everything degrades gracefully:
+  - no embeddings / no query vector -> semantic score = 0, lexical carries it
+  - no $text index yet (empty catalog) -> lexical contributes nothing, no 500
+  - empty catalog -> empty results + a clear reason
+"""
+from __future__ import annotations
+
+import re
+
+import numpy as np
+from pymongo.errors import OperationFailure
+
+from . import config, db, embeddings
+
+
+def _synonyms() -> dict[str, list[str]]:
+    doc = db.settings().find_one({"_id": "synonyms"}) or {}
+    return doc.get("map", {})
+
+
+def _expand_query(q: str) -> str:
+    syn = _synonyms()
+    extra: list[str] = []
+    for token in q.split():
+        for alt in syn.get(token, []):
+            extra.append(alt)
+    return q if not extra else q + " " + " ".join(extra)
+
+
+def _metadata_filter(collection: str | None, date_from: str | None, date_to: str | None) -> dict:
+    f: dict = {}
+    if collection:
+        f["collection"] = collection
+    date_clause: dict = {}
+    if date_from:
+        date_clause["$gte"] = date_from
+    if date_to:
+        date_clause["$lte"] = date_to
+    if date_clause:
+        f["date"] = date_clause
+    return f
+
+
+def _cosine(query_vec: np.ndarray, mat: np.ndarray) -> np.ndarray:
+    if mat.size == 0:
+        return np.array([])
+    qn = query_vec / (np.linalg.norm(query_vec) + 1e-9)
+    norms = np.linalg.norm(mat, axis=1) + 1e-9
+    return (mat @ qn) / norms
+
+
+def _highlight(content: str, query: str) -> str:
+    terms = [t for t in re.split(r"\s+", query.strip()) if len(t) >= 2]
+    highlighted = content
+    for t in sorted(set(terms), key=len, reverse=True):
+        highlighted = re.sub(
+            f"({re.escape(t)})", r"«\1»", highlighted, flags=re.IGNORECASE
+        )
+    return highlighted
+
+
+def count_documents(collection: str | None = None,
+                    date_from: str | None = None,
+                    date_to: str | None = None) -> int:
+    """Direct count from the DB (countDocuments), not estimated from results."""
+    return db.documents().count_documents(_metadata_filter(collection, date_from, date_to))
+
+
+def search(query: str,
+           mode: str = "hybrid",
+           collection: str | None = None,
+           date_from: str | None = None,
+           date_to: str | None = None,
+           limit: int = 20) -> dict:
+    """Blended search. mode in {hybrid, lexical, semantic, metadata}."""
+    meta_filter = _metadata_filter(collection, date_from, date_to)
+
+    # Candidate documents after metadata pre-filter (shrinks the semantic set).
+    doc_ids = None
+    if meta_filter:
+        doc_ids = [d["_id"] for d in db.documents().find(meta_filter, {"_id": 1})]
+        if not doc_ids:
+            return {"results": [], "total_matching_documents": 0, "mode": mode,
+                    "note": "No documents match the metadata filter."}
+
+    # Pure metadata browse (no text query).
+    if mode == "metadata" or not query.strip():
+        return _metadata_only(meta_filter, limit)
+
+    chunk_scope: dict = {}
+    if doc_ids is not None:
+        chunk_scope["document_id"] = {"$in": doc_ids}
+
+    # ── lexical score via $text ──────────────────────────────────────────────
+    lexical: dict = {}  # chunk_id -> (score, content, document_id)
+    if mode in ("hybrid", "lexical"):
+        expanded = _expand_query(query)
+        try:
+            cursor = db.chunks().find(
+                {**chunk_scope, "$text": {"$search": expanded}},
+                {"content": 1, "document_id": 1, "score": {"$meta": "textScore"}},
+            ).sort([("score", {"$meta": "textScore"})]).limit(500)
+            for c in cursor:
+                lexical[c["_id"]] = (float(c.get("score", 0.0)), c["content"], c["document_id"])
+        except OperationFailure:
+            # No text index yet (empty catalog, not indexed) — lexical contributes
+            # nothing rather than 500ing. ensure_indexes() at startup normally
+            # prevents this; this is belt-and-suspenders.
+            pass
+
+    # ── semantic score via in-app cosine ─────────────────────────────────────
+    semantic: dict = {}  # chunk_id -> (score, content, document_id)
+    semantic_note = None
+    if mode in ("hybrid", "semantic"):
+        if not config.embeddings_configured():
+            semantic_note = "Semantic search unavailable (embeddings not configured)."
+        else:
+            try:
+                qvec = np.array(embeddings.embed_one(query), dtype=np.float32)
+                candidate = db.chunks().find(
+                    {**chunk_scope, "embedding": {"$ne": None}},
+                    {"content": 1, "document_id": 1, "embedding": 1},
+                ).limit(50000)
+                ids, contents, dids, mat = [], [], [], []
+                for c in candidate:
+                    ids.append(c["_id"])
+                    contents.append(c["content"])
+                    dids.append(c["document_id"])
+                    mat.append(c["embedding"])
+                if mat:
+                    sims = _cosine(qvec, np.array(mat, dtype=np.float32))
+                    for cid, content, did, s in zip(ids, contents, dids, sims):
+                        semantic[cid] = (float(s), content, did)
+            except Exception as exc:  # noqa: BLE001
+                semantic_note = f"Semantic search degraded: {exc}"
+
+    # ── normalize + blend at the chunk level ─────────────────────────────────
+    def _normalize(d: dict) -> dict:
+        if not d:
+            return {}
+        vals = [v[0] for v in d.values()]
+        lo, hi = min(vals), max(vals)
+        span = (hi - lo) or 1.0
+        return {k: (v[0] - lo) / span for k, v in d.items()}
+
+    lex_n = _normalize(lexical)
+    sem_n = _normalize(semantic)
+    sw, kw = config.semantic_weight(), config.keyword_weight()
+    if mode == "lexical":
+        sw, kw = 0.0, 1.0
+    elif mode == "semantic":
+        sw, kw = 1.0, 0.0
+
+    blended: dict = {}  # chunk_id -> (score, content, document_id)
+    for cid in set(lex_n) | set(sem_n):
+        content, did = (lexical.get(cid) or semantic.get(cid))[1:3]
+        score = sw * sem_n.get(cid, 0.0) + kw * lex_n.get(cid, 0.0)
+        blended[cid] = (score, content, did)
+
+    # ── aggregate to document level: keep best passage per document ──────────
+    best_per_doc: dict = {}  # document_id -> (score, content, chunk_id)
+    for cid, (score, content, did) in blended.items():
+        cur = best_per_doc.get(did)
+        if cur is None or score > cur[0]:
+            best_per_doc[did] = (score, content, cid)
+
+    ranked = sorted(best_per_doc.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
+
+    results = []
+    for did, (score, content, cid) in ranked:
+        doc = db.documents().find_one({"_id": did})
+        if not doc:
+            continue
+        results.append({
+            "document_id": str(did),
+            "title": doc.get("title"),
+            "collection": doc.get("collection"),
+            "date": doc.get("date"),
+            "has_media": doc.get("has_media", False),
+            "score": round(score, 4),
+            "best_passage": _highlight(content, query),
+            "source_filename": doc.get("source_filename"),
+        })
+
+    return {
+        "results": results,
+        "total_matching_documents": len(best_per_doc),
+        "mode": mode,
+        "note": semantic_note,
+    }
+
+
+def _metadata_only(meta_filter: dict, limit: int) -> dict:
+    cursor = db.documents().find(meta_filter).sort("date", -1).limit(limit)
+    results = []
+    for doc in cursor:
+        results.append({
+            "document_id": str(doc["_id"]),
+            "title": doc.get("title"),
+            "collection": doc.get("collection"),
+            "date": doc.get("date"),
+            "has_media": doc.get("has_media", False),
+            "score": None,
+            "best_passage": None,
+            "source_filename": doc.get("source_filename"),
+        })
+    return {
+        "results": results,
+        "total_matching_documents": db.documents().count_documents(meta_filter),
+        "mode": "metadata",
+        "note": None,
+    }

package/src/corpus/templates/semantic-search/services/backend/src/app/source.py

@@ -0,0 +1,259 @@
+"""Pluggable document-source adapter, selected by SOURCE_TYPE.
+
+The source is where the raw documents *live*, not the query database. Three
+adapters ship here:
+
+  - dropbox : read-only Dropbox via the offline refresh-token flow.
+  - local   : a directory on the container filesystem (e.g. a `needs: disk`
+              mount at DISK_PATH, or a bundled folder via LOCAL_SOURCE_PATH).
+  - url     : a manifest of document URLs (URL_SOURCE_MANIFEST) fetched over HTTP.
+
+Add your own by subclassing Source and returning it from get_source(). Never read
+source secrets at import time; a missing config surfaces as SourceUnavailable,
+handled gracefully by the indexing pipeline (it records a report and exits 0).
+"""
+from __future__ import annotations
+
+import io
+import os
+from dataclasses import dataclass
+
+import httpx
+
+from . import config
+
+
+class SourceUnavailable(RuntimeError):
+    """Raised when the configured source is not usable (missing secrets, etc)."""
+
+
+# File classes we handle.
+TEXT_EXTS = {".txt", ".md"}
+PDF_EXTS = {".pdf"}
+DOCX_EXTS = {".docx", ".doc"}
+AUDIO_EXTS = {".mp3", ".m4a", ".wav", ".aac", ".ogg"}
+
+
+@dataclass
+class SourceFile:
+    path: str            # source-internal path/URL (never exposed to clients)
+    name: str            # filename only
+    rev: str             # content revision / hash for incremental sync
+    size: int
+    is_audio: bool
+    is_text: bool        # text-extractable (txt/md/pdf/docx)
+
+
+def _classify(name: str) -> tuple[bool, bool]:
+    ext = os.path.splitext(name)[1].lower()
+    return (ext in AUDIO_EXTS, ext in (TEXT_EXTS | PDF_EXTS | DOCX_EXTS))
+
+
+class Source:
+    def list_files(self) -> list[SourceFile]:
+        raise NotImplementedError
+
+    def download(self, path: str) -> bytes:
+        raise NotImplementedError
+
+
+# ── Dropbox adapter ───────────────────────────────────────────────────────────
+
+class DropboxSource(Source):
+    """Read-only Dropbox via the offline refresh-token flow."""
+
+    def __init__(self) -> None:
+        c = config.dropbox_config()
+        if not (c["app_key"] and c["app_secret"] and c["refresh_token"]):
+            raise SourceUnavailable(
+                "Dropbox is not configured — set DROPBOX_APP_KEY, "
+                "DROPBOX_APP_SECRET and DROPBOX_REFRESH_TOKEN."
+            )
+        self._app_key = c["app_key"]
+        self._app_secret = c["app_secret"]
+        self._refresh_token = c["refresh_token"]
+        self._folder = (c["folder_path"] or "").rstrip("/")
+        self._token: str | None = None
+
+    def _access_token(self) -> str:
+        if self._token:
+            return self._token
+        resp = httpx.post(
+            "https://api.dropbox.com/oauth2/token",
+            data={
+                "grant_type": "refresh_token",
+                "refresh_token": self._refresh_token,
+            },
+            auth=(self._app_key, self._app_secret),
+            timeout=30.0,
+        )
+        resp.raise_for_status()
+        self._token = resp.json()["access_token"]
+        return self._token
+
+    def _headers(self) -> dict[str, str]:
+        return {"Authorization": f"Bearer {self._access_token()}"}
+
+    def list_files(self) -> list[SourceFile]:
+        out: list[SourceFile] = []
+        url = "https://api.dropboxapi.com/2/files/list_folder"
+        body: dict = {"path": self._folder, "recursive": True}
+        while True:
+            resp = httpx.post(url, headers=self._headers(), json=body, timeout=60.0)
+            resp.raise_for_status()
+            payload = resp.json()
+            for entry in payload.get("entries", []):
+                if entry.get(".tag") != "file":
+                    continue
+                out.append(self._to_source_file(entry))
+            if payload.get("has_more"):
+                url = "https://api.dropboxapi.com/2/files/list_folder/continue"
+                body = {"cursor": payload["cursor"]}
+            else:
+                break
+        return out
+
+    @staticmethod
+    def _to_source_file(entry: dict) -> SourceFile:
+        name = entry["name"]
+        is_audio, is_text = _classify(name)
+        return SourceFile(
+            path=entry["path_lower"],
+            name=name,
+            rev=entry.get("content_hash") or entry.get("rev", ""),
+            size=int(entry.get("size", 0)),
+            is_audio=is_audio,
+            is_text=is_text,
+        )
+
+    def download(self, path: str) -> bytes:
+        import json
+
+        resp = httpx.post(
+            "https://content.dropboxapi.com/2/files/download",
+            headers={
+                **self._headers(),
+                "Dropbox-API-Arg": json.dumps({"path": path}),
+            },
+            timeout=120.0,
+        )
+        resp.raise_for_status()
+        return resp.content
+
+
+# ── Local-filesystem adapter ──────────────────────────────────────────────────
+
+class LocalSource(Source):
+    """Read documents from a directory on the container filesystem.
+
+    Point LOCAL_SOURCE_PATH (or a `needs: disk` mount at DISK_PATH) at a folder
+    of .txt/.md/.pdf/.docx files. `rev` is the file mtime+size, so changed files
+    re-index on the next sync.
+    """
+
+    def __init__(self) -> None:
+        root = config.local_source_path()
+        if not root:
+            raise SourceUnavailable(
+                "Local source is not configured — set LOCAL_SOURCE_PATH "
+                "(or mount a disk and it will default to DISK_PATH)."
+            )
+        if not os.path.isdir(root):
+            raise SourceUnavailable(f"LOCAL_SOURCE_PATH does not exist: {root!r}")
+        self._root = root
+
+    def list_files(self) -> list[SourceFile]:
+        out: list[SourceFile] = []
+        for dirpath, _dirs, files in os.walk(self._root):
+            for name in files:
+                full = os.path.join(dirpath, name)
+                is_audio, is_text = _classify(name)
+                try:
+                    st = os.stat(full)
+                except OSError:
+                    continue
+                out.append(SourceFile(
+                    path=full,
+                    name=name,
+                    rev=f"{int(st.st_mtime)}-{st.st_size}",
+                    size=st.st_size,
+                    is_audio=is_audio,
+                    is_text=is_text,
+                ))
+        return out
+
+    def download(self, path: str) -> bytes:
+        with open(path, "rb") as fh:
+            return fh.read()
+
+
+# ── URL-manifest adapter ──────────────────────────────────────────────────────
+
+class UrlSource(Source):
+    """Fetch documents from a manifest of URLs.
+
+    URL_SOURCE_MANIFEST is a newline- or comma-separated list of document URLs.
+    The filename is the URL's last path segment; `rev` is the URL itself
+    (override by pointing the manifest at content-hashed URLs).
+    """
+
+    def __init__(self) -> None:
+        manifest = config.url_source_manifest()
+        if not manifest:
+            raise SourceUnavailable(
+                "URL source is not configured — set URL_SOURCE_MANIFEST to a "
+                "newline- or comma-separated list of document URLs."
+            )
+        raw = manifest.replace(",", "\n").splitlines()
+        self._urls = [u.strip() for u in raw if u.strip()]
+
+    def list_files(self) -> list[SourceFile]:
+        out: list[SourceFile] = []
+        for u in self._urls:
+            name = u.rstrip("/").rsplit("/", 1)[-1] or "document"
+            is_audio, is_text = _classify(name)
+            out.append(SourceFile(
+                path=u,
+                name=name,
+                rev=u,
+                size=0,
+                is_audio=is_audio,
+                is_text=is_text,
+            ))
+        return out
+
+    def download(self, path: str) -> bytes:
+        resp = httpx.get(path, timeout=120.0, follow_redirects=True)
+        resp.raise_for_status()
+        return resp.content
+
+
+def get_source() -> Source:
+    """Build the configured source adapter. Raises SourceUnavailable if unusable."""
+    st = config.source_type()
+    if st == "dropbox":
+        return DropboxSource()
+    if st == "local":
+        return LocalSource()
+    if st == "url":
+        return UrlSource()
+    raise SourceUnavailable(f"Unknown SOURCE_TYPE: {st!r}")
+
+
+# ── Text extraction (used by the pipeline) ───────────────────────────────────
+
+def extract_text(name: str, raw: bytes) -> str:
+    ext = os.path.splitext(name)[1].lower()
+    if ext in TEXT_EXTS:
+        return raw.decode("utf-8", errors="replace")
+    if ext in PDF_EXTS:
+        from pypdf import PdfReader
+
+        reader = PdfReader(io.BytesIO(raw))
+        return "\n".join((page.extract_text() or "") for page in reader.pages)
+    if ext in DOCX_EXTS:
+        from docx import Document
+
+        doc = Document(io.BytesIO(raw))
+        return "\n".join(p.text for p in doc.paragraphs)
+    return ""

package/src/corpus/templates/semantic-search/services/backend/src/main.py

@@ -0,0 +1,12 @@
+"""CloudGrid python-service entry point (services/backend/src/main.py).
+
+Launches the FastAPI app (app/main.py) with uvicorn, binding to the platform's
+injected PORT (default 8080). No secret is read here or at import time.
+"""
+import os
+
+import uvicorn
+
+if __name__ == "__main__":
+    port = int(os.environ.get("PORT", "8080"))
+    uvicorn.run("app.main:app", host="0.0.0.0", port=port, workers=1)

package/src/corpus/templates/semantic-search/services/web/index.html

@@ -0,0 +1,12 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Document Search</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.jsx"></script>
+  </body>
+</html>