@cloudgrid-io/mcp 0.14.0 → 0.15.0

package/src/corpus/templates/semantic-search/services/backend/src/app/main.py

@@ -0,0 +1,259 @@
+"""FastAPI backend for the semantic-search document app.
+
+Mounted at /backend by CloudGrid. Public search + answer endpoints, plus
+manager-only admin (Settings, Usage Dashboard, Refresh now) enforced server-side.
+
+Health is green as soon as Mongo connects — it does NOT require the document
+source or the embeddings key. No secret is read at import time.
+"""
+from __future__ import annotations
+
+import datetime as dt
+
+from fastapi import Depends, FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+
+from . import answer, auth, config, db, indexing, search
+
+app = FastAPI(title="Semantic Search API")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],       # frontend is served same-origin; open for safety
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.on_event("startup")
+def _ensure_indexes_on_startup() -> None:
+    # Create the chunks $text index + documents indexes at boot so $text search
+    # returns [] on an empty catalog instead of erroring (IndexNotFound / 500).
+    # Never block startup if Mongo is briefly unavailable — an index run recreates
+    # them.
+    try:
+        db.ensure_indexes()
+    except Exception as e:  # noqa: BLE001
+        print(f"startup: ensure_indexes skipped ({e})")
+
+
+# ── Health ────────────────────────────────────────────────────────────────────
+
+@app.get("/backend/health")
+def health():
+    """Green when Mongo connects. Independent of the source / embeddings key."""
+    try:
+        db.ping()
+        return {"status": "ok"}
+    except Exception as exc:  # noqa: BLE001
+        return {"status": "degraded", "detail": str(exc)}
+
+
+@app.get("/backend/status")
+def status():
+    """Public capability snapshot — powers the frontend's 'not indexed / missing
+    secret' states without exposing any secret value."""
+    try:
+        db.ping()
+        mongo_ok = True
+    except Exception:  # noqa: BLE001
+        mongo_ok = False
+    settings_doc = db.settings().find_one({"_id": "catalog"}) if mongo_ok else None
+    features = db.settings().find_one({"_id": "features"}) if mongo_ok else None
+    return {
+        "mongo": mongo_ok,
+        "source_type": config.source_type(),
+        "source_configured": config.source_configured(),
+        "embeddings_configured": config.embeddings_configured(),
+        "manager_auth_configured": config.manager_auth_configured(),
+        "answer_mode_enabled": bool((features or {}).get("answer_mode_enabled", True)),
+        "indexed_documents": db.documents().count_documents({}) if mongo_ok else 0,
+        "last_refresh": (settings_doc or {}).get("last_refresh"),
+    }
+
+
+# ── Search (public) ───────────────────────────────────────────────────────────
+
+@app.get("/backend/search")
+def do_search(q: str = "", mode: str = "hybrid", collection: str | None = None,
+              date_from: str | None = None, date_to: str | None = None,
+              limit: int = 20):
+    result = search.search(q, mode=mode, collection=collection,
+                           date_from=date_from, date_to=date_to, limit=limit)
+    _log_event("search", {"q": q, "mode": mode,
+                          "zero_result": len(result["results"]) == 0})
+    return result
+
+
+@app.get("/backend/count")
+def do_count(collection: str | None = None, date_from: str | None = None,
+             date_to: str | None = None):
+    """Counting answered directly from the DB (countDocuments)."""
+    return {"count": search.count_documents(collection, date_from, date_to)}
+
+
+@app.get("/backend/collections")
+def list_collections():
+    values = [c for c in db.documents().distinct("collection") if c]
+    return {"collections": sorted(values)}
+
+
+class AnswerRequest(BaseModel):
+    question: str
+
+
+@app.post("/backend/answer")
+def do_answer(req: AnswerRequest):
+    _log_event("answer", {"q": req.question})
+    return answer.compose_answer(req.question)
+
+
+@app.get("/backend/documents/{document_id}/download")
+def download_document(document_id: str):
+    """Stream a document's text through the service — source paths never exposed."""
+    from bson import ObjectId
+    from fastapi import HTTPException
+    from fastapi.responses import PlainTextResponse
+
+    try:
+        doc = db.documents().find_one({"_id": ObjectId(document_id)})
+    except Exception:  # noqa: BLE001
+        raise HTTPException(status_code=404, detail="Not found.")
+    if not doc:
+        raise HTTPException(status_code=404, detail="Not found.")
+    parts = db.chunks().find({"document_id": doc["_id"]}).sort("chunk_index", 1)
+    text = "".join(p["content"] for p in parts)
+    return PlainTextResponse(text, headers={
+        "Content-Disposition": f'attachment; filename="{doc.get("title","document")}.txt"'
+    })
+
+
+# ── Manager auth ──────────────────────────────────────────────────────────────
+
+class LoginRequest(BaseModel):
+    password: str
+
+
+@app.post("/backend/admin/login")
+def admin_login(req: LoginRequest):
+    token = auth.login(req.password)
+    return {"token": token}
+
+
+# ── Manager: Settings ─────────────────────────────────────────────────────────
+
+@app.get("/backend/admin/settings", dependencies=[Depends(auth.require_manager)])
+def get_settings():
+    features = db.settings().find_one({"_id": "features"}) or {}
+    synonyms = db.settings().find_one({"_id": "synonyms"}) or {}
+    catalog = db.settings().find_one({"_id": "catalog"}) or {}
+    return {
+        "answer_mode_enabled": bool(features.get("answer_mode_enabled", True)),
+        "synonyms": synonyms.get("map", {}),
+        "last_refresh": catalog.get("last_refresh"),
+    }
+
+
+class FeatureUpdate(BaseModel):
+    answer_mode_enabled: bool | None = None
+
+
+@app.put("/backend/admin/settings", dependencies=[Depends(auth.require_manager)])
+def update_settings(req: FeatureUpdate):
+    update = {k: v for k, v in req.dict().items() if v is not None}
+    if update:
+        db.settings().update_one({"_id": "features"}, {"$set": update}, upsert=True)
+    return {"ok": True}
+
+
+class SynonymUpdate(BaseModel):
+    map: dict[str, list[str]]
+
+
+@app.put("/backend/admin/synonyms", dependencies=[Depends(auth.require_manager)])
+def update_synonyms(req: SynonymUpdate):
+    db.settings().update_one({"_id": "synonyms"}, {"$set": {"map": req.map}}, upsert=True)
+    return {"ok": True}
+
+
+class PasswordChange(BaseModel):
+    current: str
+    new: str
+
+
+@app.post("/backend/admin/password", dependencies=[Depends(auth.require_manager)])
+def change_password(req: PasswordChange):
+    new_hash = auth.change_password(req.current, req.new)
+    # Return the hash for the manager to store as MANAGER_PASSWORD_HASH.
+    return {
+        "new_hash": new_hash,
+        "note": "Store this as the MANAGER_PASSWORD_HASH secret "
+                "(`grid secrets set MANAGER_PASSWORD_HASH=<hash>`) to persist it.",
+    }
+
+
+@app.get("/backend/admin/index-report", dependencies=[Depends(auth.require_manager)])
+def index_report():
+    reports = list(db.index_reports().find().sort("started_at", -1).limit(10))
+    for r in reports:
+        r["_id"] = str(r["_id"])
+    return {"reports": reports}
+
+
+@app.post("/backend/admin/refresh", dependencies=[Depends(auth.require_manager)])
+def refresh_now():
+    """Manager-only on-demand incremental sync.
+
+    This is the supported refresh path today. A scheduled `type: cron` refresh
+    service is a follow-up (blocked on platform issue #1585); see AGENTS.md.
+    """
+    report = indexing.run_sync(triggered_by="manager")
+    report.pop("_id", None)
+    return report
+
+
+# ── Manager: Usage Dashboard ──────────────────────────────────────────────────
+
+@app.get("/backend/admin/usage", dependencies=[Depends(auth.require_manager)])
+def usage():
+    ev = db.events()
+    total_searches = ev.count_documents({"kind": "search"})
+    by_mode = list(ev.aggregate([
+        {"$match": {"kind": "search"}},
+        {"$group": {"_id": "$meta.mode", "count": {"$sum": 1}}},
+    ]))
+    top_queries = list(ev.aggregate([
+        {"$match": {"kind": "search", "meta.q": {"$ne": ""}}},
+        {"$group": {"_id": "$meta.q", "count": {"$sum": 1}}},
+        {"$sort": {"count": -1}}, {"$limit": 10},
+    ]))
+    zero_result = list(ev.aggregate([
+        {"$match": {"kind": "search", "meta.zero_result": True}},
+        {"$group": {"_id": "$meta.q", "count": {"$sum": 1}}},
+        {"$sort": {"count": -1}}, {"$limit": 10},
+    ]))
+    unique_users = len(ev.distinct("anon_id", {"kind": "search"}))
+    return {
+        "total_searches": total_searches,
+        "unique_users": unique_users,
+        "by_mode": {row["_id"] or "unknown": row["count"] for row in by_mode},
+        "top_queries": [{"query": r["_id"], "count": r["count"]} for r in top_queries],
+        "zero_result_queries": [{"query": r["_id"], "count": r["count"]} for r in zero_result],
+        "indexed_documents": db.documents().count_documents({}),
+        "last_refresh": (db.settings().find_one({"_id": "catalog"}) or {}).get("last_refresh"),
+    }
+
+
+# ── internal ──────────────────────────────────────────────────────────────────
+
+def _log_event(kind: str, meta: dict) -> None:
+    try:
+        db.events().insert_one({
+            "kind": kind,
+            "meta": meta,
+            "anon_id": meta.get("anon_id", "anon"),
+            "at": dt.datetime.now(dt.timezone.utc),
+        })
+    except Exception:  # noqa: BLE001 - telemetry must never break a request
+        pass

package/src/corpus/templates/semantic-search/services/backend/src/app/search.py

@@ -0,0 +1,222 @@
+"""CloudGrid-native hybrid search over Mongo.
+
+Modes blended in one query:
+  - lexical/exact : Mongo $text index over chunks.content (+ synonym expansion)
+  - semantic      : in-app cosine similarity (NumPy) over chunk embedding arrays
+  - metadata      : Mongo filters on documents (date, collection)
+
+Results aggregate to the document level; each result carries its single best
+matching passage with a highlight. Counting questions use countDocuments.
+
+Everything degrades gracefully:
+  - no embeddings / no query vector -> semantic score = 0, lexical carries it
+  - no $text index yet (empty catalog) -> lexical contributes nothing, no 500
+  - empty catalog -> empty results + a clear reason
+"""
+from __future__ import annotations
+
+import re
+
+import numpy as np
+from pymongo.errors import OperationFailure
+
+from . import config, db, embeddings
+
+
+def _synonyms() -> dict[str, list[str]]:
+    doc = db.settings().find_one({"_id": "synonyms"}) or {}
+    return doc.get("map", {})
+
+
+def _expand_query(q: str) -> str:
+    syn = _synonyms()
+    extra: list[str] = []
+    for token in q.split():
+        for alt in syn.get(token, []):
+            extra.append(alt)
+    return q if not extra else q + " " + " ".join(extra)
+
+
+def _metadata_filter(collection: str | None, date_from: str | None, date_to: str | None) -> dict:
+    f: dict = {}
+    if collection:
+        f["collection"] = collection
+    date_clause: dict = {}
+    if date_from:
+        date_clause["$gte"] = date_from
+    if date_to:
+        date_clause["$lte"] = date_to
+    if date_clause:
+        f["date"] = date_clause
+    return f
+
+
+def _cosine(query_vec: np.ndarray, mat: np.ndarray) -> np.ndarray:
+    if mat.size == 0:
+        return np.array([])
+    qn = query_vec / (np.linalg.norm(query_vec) + 1e-9)
+    norms = np.linalg.norm(mat, axis=1) + 1e-9
+    return (mat @ qn) / norms
+
+
+def _highlight(content: str, query: str) -> str:
+    terms = [t for t in re.split(r"\s+", query.strip()) if len(t) >= 2]
+    highlighted = content
+    for t in sorted(set(terms), key=len, reverse=True):
+        highlighted = re.sub(
+            f"({re.escape(t)})", r"«\1»", highlighted, flags=re.IGNORECASE
+        )
+    return highlighted
+
+
+def count_documents(collection: str | None = None,
+                    date_from: str | None = None,
+                    date_to: str | None = None) -> int:
+    """Direct count from the DB (countDocuments), not estimated from results."""
+    return db.documents().count_documents(_metadata_filter(collection, date_from, date_to))
+
+
+def search(query: str,
+           mode: str = "hybrid",
+           collection: str | None = None,
+           date_from: str | None = None,
+           date_to: str | None = None,
+           limit: int = 20) -> dict:
+    """Blended search. mode in {hybrid, lexical, semantic, metadata}."""
+    meta_filter = _metadata_filter(collection, date_from, date_to)
+
+    # Candidate documents after metadata pre-filter (shrinks the semantic set).
+    doc_ids = None
+    if meta_filter:
+        doc_ids = [d["_id"] for d in db.documents().find(meta_filter, {"_id": 1})]
+        if not doc_ids:
+            return {"results": [], "total_matching_documents": 0, "mode": mode,
+                    "note": "No documents match the metadata filter."}
+
+    # Pure metadata browse (no text query).
+    if mode == "metadata" or not query.strip():
+        return _metadata_only(meta_filter, limit)
+
+    chunk_scope: dict = {}
+    if doc_ids is not None:
+        chunk_scope["document_id"] = {"$in": doc_ids}
+
+    # ── lexical score via $text ──────────────────────────────────────────────
+    lexical: dict = {}  # chunk_id -> (score, content, document_id)
+    if mode in ("hybrid", "lexical"):
+        expanded = _expand_query(query)
+        try:
+            cursor = db.chunks().find(
+                {**chunk_scope, "$text": {"$search": expanded}},
+                {"content": 1, "document_id": 1, "score": {"$meta": "textScore"}},
+            ).sort([("score", {"$meta": "textScore"})]).limit(500)
+            for c in cursor:
+                lexical[c["_id"]] = (float(c.get("score", 0.0)), c["content"], c["document_id"])
+        except OperationFailure:
+            # No text index yet (empty catalog, not indexed) — lexical contributes
+            # nothing rather than 500ing. ensure_indexes() at startup normally
+            # prevents this; this is belt-and-suspenders.
+            pass
+
+    # ── semantic score via in-app cosine ─────────────────────────────────────
+    semantic: dict = {}  # chunk_id -> (score, content, document_id)
+    semantic_note = None
+    if mode in ("hybrid", "semantic"):
+        if not config.embeddings_configured():
+            semantic_note = "Semantic search unavailable (embeddings not configured)."
+        else:
+            try:
+                qvec = np.array(embeddings.embed_one(query), dtype=np.float32)
+                candidate = db.chunks().find(
+                    {**chunk_scope, "embedding": {"$ne": None}},
+                    {"content": 1, "document_id": 1, "embedding": 1},
+                ).limit(50000)
+                ids, contents, dids, mat = [], [], [], []
+                for c in candidate:
+                    ids.append(c["_id"])
+                    contents.append(c["content"])
+                    dids.append(c["document_id"])
+                    mat.append(c["embedding"])
+                if mat:
+                    sims = _cosine(qvec, np.array(mat, dtype=np.float32))
+                    for cid, content, did, s in zip(ids, contents, dids, sims):
+                        semantic[cid] = (float(s), content, did)
+            except Exception as exc:  # noqa: BLE001
+                semantic_note = f"Semantic search degraded: {exc}"
+
+    # ── normalize + blend at the chunk level ─────────────────────────────────
+    def _normalize(d: dict) -> dict:
+        if not d:
+            return {}
+        vals = [v[0] for v in d.values()]
+        lo, hi = min(vals), max(vals)
+        span = (hi - lo) or 1.0
+        return {k: (v[0] - lo) / span for k, v in d.items()}
+
+    lex_n = _normalize(lexical)
+    sem_n = _normalize(semantic)
+    sw, kw = config.semantic_weight(), config.keyword_weight()
+    if mode == "lexical":
+        sw, kw = 0.0, 1.0
+    elif mode == "semantic":
+        sw, kw = 1.0, 0.0
+
+    blended: dict = {}  # chunk_id -> (score, content, document_id)
+    for cid in set(lex_n) | set(sem_n):
+        content, did = (lexical.get(cid) or semantic.get(cid))[1:3]
+        score = sw * sem_n.get(cid, 0.0) + kw * lex_n.get(cid, 0.0)
+        blended[cid] = (score, content, did)
+
+    # ── aggregate to document level: keep best passage per document ──────────
+    best_per_doc: dict = {}  # document_id -> (score, content, chunk_id)
+    for cid, (score, content, did) in blended.items():
+        cur = best_per_doc.get(did)
+        if cur is None or score > cur[0]:
+            best_per_doc[did] = (score, content, cid)
+
+    ranked = sorted(best_per_doc.items(), key=lambda kv: kv[1][0], reverse=True)[:limit]
+
+    results = []
+    for did, (score, content, cid) in ranked:
+        doc = db.documents().find_one({"_id": did})
+        if not doc:
+            continue
+        results.append({
+            "document_id": str(did),
+            "title": doc.get("title"),
+            "collection": doc.get("collection"),
+            "date": doc.get("date"),
+            "has_media": doc.get("has_media", False),
+            "score": round(score, 4),
+            "best_passage": _highlight(content, query),
+            "source_filename": doc.get("source_filename"),
+        })
+
+    return {
+        "results": results,
+        "total_matching_documents": len(best_per_doc),
+        "mode": mode,
+        "note": semantic_note,
+    }
+
+
+def _metadata_only(meta_filter: dict, limit: int) -> dict:
+    cursor = db.documents().find(meta_filter).sort("date", -1).limit(limit)
+    results = []
+    for doc in cursor:
+        results.append({
+            "document_id": str(doc["_id"]),
+            "title": doc.get("title"),
+            "collection": doc.get("collection"),
+            "date": doc.get("date"),
+            "has_media": doc.get("has_media", False),
+            "score": None,
+            "best_passage": None,
+            "source_filename": doc.get("source_filename"),
+        })
+    return {
+        "results": results,
+        "total_matching_documents": db.documents().count_documents(meta_filter),
+        "mode": "metadata",
+        "note": None,
+    }