@cloudgrid-io/mcp 0.14.0 → 0.16.0

package/src/corpus/templates/semantic-search/services/backend/src/app/indexing.py

@@ -0,0 +1,223 @@
+"""Incremental indexing pipeline: discover -> extract -> chunk -> embed -> store.
+
+Shared by the manager 'Refresh now' endpoint AND the scheduled `type: cron`
+refresh service (services/refresh, which vendors this module). It is resilient:
+if the source or
+embeddings are not configured it records a clear, non-crashing report and
+returns. Only new/changed files (by rev / content-hash) are reprocessed.
+"""
+from __future__ import annotations
+
+import datetime as dt
+import os
+import re
+
+from . import config, db, embeddings, source
+
+CHUNK_CHARS = 1200
+CHUNK_OVERLAP = 150
+
+# Generic date detection from filename or document head (best effort, nullable).
+_DATE_RE = re.compile(r"(\d{4})[./-](\d{1,2})[./-](\d{1,2})")        # 2026-07-05
+_DATE_RE_ALT = re.compile(r"(\d{1,2})[./-](\d{1,2})[./-](\d{2,4})")  # 05/07/2026
+
+
+def _utcnow() -> dt.datetime:
+    return dt.datetime.now(dt.timezone.utc)
+
+
+def _chunk_text(text: str) -> list[str]:
+    text = text.strip()
+    if not text:
+        return []
+    out: list[str] = []
+    start = 0
+    n = len(text)
+    while start < n:
+        end = min(start + CHUNK_CHARS, n)
+        out.append(text[start:end])
+        if end == n:
+            break
+        start = end - CHUNK_OVERLAP
+    return out
+
+
+def _parse_metadata(name: str, text: str) -> dict:
+    """Best-effort title / date / collection extraction. All fields nullable.
+
+    Generalize this for your domain: `collection` is a free-form grouping tag
+    (folder, series, category), `date` is an ISO date if one can be found.
+    """
+    title = os.path.splitext(name)[0].strip()
+    # Collection tag = the leading non-numeric words of the title.
+    collection = re.sub(r"[\d].*$", "", title).strip() or None
+
+    date_iso = None
+    haystack = f"{name}\n{text[:2000]}"
+    m = _DATE_RE.search(haystack)
+    if m:
+        y, mo, d = m.groups()
+        try:
+            date_iso = dt.datetime(int(y), int(mo), int(d)).date().isoformat()
+        except ValueError:
+            date_iso = None
+    if date_iso is None:
+        m = _DATE_RE_ALT.search(haystack)
+        if m:
+            d, mo, y = m.groups()
+            if len(y) == 2:
+                y = "20" + y
+            try:
+                date_iso = dt.datetime(int(y), int(mo), int(d)).date().isoformat()
+            except ValueError:
+                date_iso = None
+
+    return {"title": title, "collection": collection, "date": date_iso}
+
+
+def run_sync(triggered_by: str = "manual") -> dict:
+    """Run one incremental sync. Never raises on missing config — returns a report."""
+    started = _utcnow()
+    report: dict = {
+        "started_at": started,
+        "triggered_by": triggered_by,
+        "files": [],
+        "status": "ok",
+        "note": None,
+    }
+
+    # Mongo is required for indexing (and always injected). Fail loud only here.
+    db.ensure_indexes()
+
+    if not config.source_configured():
+        report["status"] = "skipped"
+        report["note"] = (
+            f"Source not configured (SOURCE_TYPE={config.source_type()!r} — "
+            "missing its secrets/config)."
+        )
+        _save_report(report)
+        return report
+
+    try:
+        src = source.get_source()
+        files = src.list_files()
+    except Exception as exc:  # noqa: BLE001 - resilient by design
+        report["status"] = "error"
+        report["note"] = f"Source listing failed: {exc}"
+        _save_report(report)
+        return report
+
+    embeddings_ok = config.embeddings_configured()
+    if not embeddings_ok:
+        report["note"] = (
+            "Embeddings not configured — documents indexed for lexical/metadata "
+            "search only (no semantic vectors)."
+        )
+
+    for f in files:
+        outcome = _process_file(src, f, embeddings_ok)
+        report["files"].append(outcome)
+
+    report["finished_at"] = _utcnow()
+    _save_report(report)
+    return report
+
+
+def _process_file(src: source.Source, f: source.SourceFile, embeddings_ok: bool) -> dict:
+    existing = db.documents().find_one({"source_filename": f.name})
+    if existing and existing.get("source_rev") == f.rev:
+        return {"file": f.name, "result": "unchanged"}
+
+    if f.is_audio and not f.is_text:
+        # Catalogue audio/media with a flag; do not search inside it.
+        doc = {
+            "title": os.path.splitext(f.name)[0],
+            "source_filename": f.name,
+            "source_rev": f.rev,
+            "file_path": f.path,
+            "collection": None,
+            "date": None,
+            "has_media": True,
+            "media_path": f.path,
+            "word_count": 0,
+            "indexed_at": _utcnow(),
+        }
+        doc_id = _upsert_document(doc)
+        db.chunks().delete_many({"document_id": doc_id})
+        return {"file": f.name, "result": "catalogued_media"}
+
+    if not f.is_text:
+        return {"file": f.name, "result": "skipped_unsupported"}
+
+    try:
+        raw = src.download(f.path)
+        text = source.extract_text(f.name, raw)
+    except Exception as exc:  # noqa: BLE001
+        return {"file": f.name, "result": "error", "detail": str(exc)}
+
+    meta = _parse_metadata(f.name, text)
+    doc = {
+        **meta,
+        "source_filename": f.name,
+        "source_rev": f.rev,
+        "file_path": f.path,
+        "has_media": False,
+        "media_path": None,
+        "word_count": len(text.split()),
+        "indexed_at": _utcnow(),
+    }
+    doc_id = _upsert_document(doc)
+
+    # Rebuild chunks for this document.
+    db.chunks().delete_many({"document_id": doc_id})
+    parts = _chunk_text(text)
+    vectors: list[list[float] | None] = [None] * len(parts)
+    if embeddings_ok and parts:
+        try:
+            vectors = embeddings.embed(parts)  # type: ignore[assignment]
+        except Exception as exc:  # noqa: BLE001
+            return {
+                "file": f.name,
+                "result": "indexed_no_embeddings",
+                "detail": f"embedding failed: {exc}",
+                "chunks": len(parts),
+            }
+
+    docs = []
+    for i, content in enumerate(parts):
+        docs.append({
+            "document_id": doc_id,
+            "chunk_index": i,
+            "content": content,
+            "embedding": vectors[i] if i < len(vectors) else None,
+            "token_count": len(content.split()),
+        })
+    if docs:
+        db.chunks().insert_many(docs)
+
+    return {
+        "file": f.name,
+        "result": "indexed" if embeddings_ok else "indexed_no_embeddings",
+        "chunks": len(docs),
+    }
+
+
+def _upsert_document(doc: dict):
+    db.documents().update_one(
+        {"source_filename": doc["source_filename"]},
+        {"$set": doc},
+        upsert=True,
+    )
+    return db.documents().find_one({"source_filename": doc["source_filename"]})["_id"]
+
+
+def _save_report(report: dict) -> None:
+    try:
+        db.index_reports().insert_one(dict(report))
+        db.settings().update_one(
+            {"_id": "catalog"},
+            {"$set": {"last_refresh": report.get("finished_at") or report["started_at"]}},
+            upsert=True,
+        )
+    except Exception:  # noqa: BLE001 - reporting must never crash the caller
+        pass

package/src/corpus/templates/semantic-search/services/backend/src/app/main.py

@@ -0,0 +1,259 @@
+"""FastAPI backend for the semantic-search document app.
+
+Mounted at /backend by CloudGrid. Public search + answer endpoints, plus
+manager-only admin (Settings, Usage Dashboard, Refresh now) enforced server-side.
+
+Health is green as soon as Mongo connects — it does NOT require the document
+source or the embeddings key. No secret is read at import time.
+"""
+from __future__ import annotations
+
+import datetime as dt
+
+from fastapi import Depends, FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+
+from . import answer, auth, config, db, indexing, search
+
+app = FastAPI(title="Semantic Search API")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],       # frontend is served same-origin; open for safety
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.on_event("startup")
+def _ensure_indexes_on_startup() -> None:
+    # Create the chunks $text index + documents indexes at boot so $text search
+    # returns [] on an empty catalog instead of erroring (IndexNotFound / 500).
+    # Never block startup if Mongo is briefly unavailable — an index run recreates
+    # them.
+    try:
+        db.ensure_indexes()
+    except Exception as e:  # noqa: BLE001
+        print(f"startup: ensure_indexes skipped ({e})")
+
+
+# ── Health ────────────────────────────────────────────────────────────────────
+
+@app.get("/backend/health")
+def health():
+    """Green when Mongo connects. Independent of the source / embeddings key."""
+    try:
+        db.ping()
+        return {"status": "ok"}
+    except Exception as exc:  # noqa: BLE001
+        return {"status": "degraded", "detail": str(exc)}
+
+
+@app.get("/backend/status")
+def status():
+    """Public capability snapshot — powers the frontend's 'not indexed / missing
+    secret' states without exposing any secret value."""
+    try:
+        db.ping()
+        mongo_ok = True
+    except Exception:  # noqa: BLE001
+        mongo_ok = False
+    settings_doc = db.settings().find_one({"_id": "catalog"}) if mongo_ok else None
+    features = db.settings().find_one({"_id": "features"}) if mongo_ok else None
+    return {
+        "mongo": mongo_ok,
+        "source_type": config.source_type(),
+        "source_configured": config.source_configured(),
+        "embeddings_configured": config.embeddings_configured(),
+        "manager_auth_configured": config.manager_auth_configured(),
+        "answer_mode_enabled": bool((features or {}).get("answer_mode_enabled", True)),
+        "indexed_documents": db.documents().count_documents({}) if mongo_ok else 0,
+        "last_refresh": (settings_doc or {}).get("last_refresh"),
+    }
+
+
+# ── Search (public) ───────────────────────────────────────────────────────────
+
+@app.get("/backend/search")
+def do_search(q: str = "", mode: str = "hybrid", collection: str | None = None,
+              date_from: str | None = None, date_to: str | None = None,
+              limit: int = 20):
+    result = search.search(q, mode=mode, collection=collection,
+                           date_from=date_from, date_to=date_to, limit=limit)
+    _log_event("search", {"q": q, "mode": mode,
+                          "zero_result": len(result["results"]) == 0})
+    return result
+
+
+@app.get("/backend/count")
+def do_count(collection: str | None = None, date_from: str | None = None,
+             date_to: str | None = None):
+    """Counting answered directly from the DB (countDocuments)."""
+    return {"count": search.count_documents(collection, date_from, date_to)}
+
+
+@app.get("/backend/collections")
+def list_collections():
+    values = [c for c in db.documents().distinct("collection") if c]
+    return {"collections": sorted(values)}
+
+
+class AnswerRequest(BaseModel):
+    question: str
+
+
+@app.post("/backend/answer")
+def do_answer(req: AnswerRequest):
+    _log_event("answer", {"q": req.question})
+    return answer.compose_answer(req.question)
+
+
+@app.get("/backend/documents/{document_id}/download")
+def download_document(document_id: str):
+    """Stream a document's text through the service — source paths never exposed."""
+    from bson import ObjectId
+    from fastapi import HTTPException
+    from fastapi.responses import PlainTextResponse
+
+    try:
+        doc = db.documents().find_one({"_id": ObjectId(document_id)})
+    except Exception:  # noqa: BLE001
+        raise HTTPException(status_code=404, detail="Not found.")
+    if not doc:
+        raise HTTPException(status_code=404, detail="Not found.")
+    parts = db.chunks().find({"document_id": doc["_id"]}).sort("chunk_index", 1)
+    text = "".join(p["content"] for p in parts)
+    return PlainTextResponse(text, headers={
+        "Content-Disposition": f'attachment; filename="{doc.get("title","document")}.txt"'
+    })
+
+
+# ── Manager auth ──────────────────────────────────────────────────────────────
+
+class LoginRequest(BaseModel):
+    password: str
+
+
+@app.post("/backend/admin/login")
+def admin_login(req: LoginRequest):
+    token = auth.login(req.password)
+    return {"token": token}
+
+
+# ── Manager: Settings ─────────────────────────────────────────────────────────
+
+@app.get("/backend/admin/settings", dependencies=[Depends(auth.require_manager)])
+def get_settings():
+    features = db.settings().find_one({"_id": "features"}) or {}
+    synonyms = db.settings().find_one({"_id": "synonyms"}) or {}
+    catalog = db.settings().find_one({"_id": "catalog"}) or {}
+    return {
+        "answer_mode_enabled": bool(features.get("answer_mode_enabled", True)),
+        "synonyms": synonyms.get("map", {}),
+        "last_refresh": catalog.get("last_refresh"),
+    }
+
+
+class FeatureUpdate(BaseModel):
+    answer_mode_enabled: bool | None = None
+
+
+@app.put("/backend/admin/settings", dependencies=[Depends(auth.require_manager)])
+def update_settings(req: FeatureUpdate):
+    update = {k: v for k, v in req.dict().items() if v is not None}
+    if update:
+        db.settings().update_one({"_id": "features"}, {"$set": update}, upsert=True)
+    return {"ok": True}
+
+
+class SynonymUpdate(BaseModel):
+    map: dict[str, list[str]]
+
+
+@app.put("/backend/admin/synonyms", dependencies=[Depends(auth.require_manager)])
+def update_synonyms(req: SynonymUpdate):
+    db.settings().update_one({"_id": "synonyms"}, {"$set": {"map": req.map}}, upsert=True)
+    return {"ok": True}
+
+
+class PasswordChange(BaseModel):
+    current: str
+    new: str
+
+
+@app.post("/backend/admin/password", dependencies=[Depends(auth.require_manager)])
+def change_password(req: PasswordChange):
+    new_hash = auth.change_password(req.current, req.new)
+    # Return the hash for the manager to store as MANAGER_PASSWORD_HASH.
+    return {
+        "new_hash": new_hash,
+        "note": "Store this as the MANAGER_PASSWORD_HASH secret "
+                "(`grid secrets set MANAGER_PASSWORD_HASH=<hash>`) to persist it.",
+    }
+
+
+@app.get("/backend/admin/index-report", dependencies=[Depends(auth.require_manager)])
+def index_report():
+    reports = list(db.index_reports().find().sort("started_at", -1).limit(10))
+    for r in reports:
+        r["_id"] = str(r["_id"])
+    return {"reports": reports}
+
+
+@app.post("/backend/admin/refresh", dependencies=[Depends(auth.require_manager)])
+def refresh_now():
+    """Manager-only on-demand incremental sync.
+
+    Runs the same pipeline as the scheduled `type: cron` refresh service
+    (services/refresh); this is the on-demand path. See AGENTS.md §8.
+    """
+    report = indexing.run_sync(triggered_by="manager")
+    report.pop("_id", None)
+    return report
+
+
+# ── Manager: Usage Dashboard ──────────────────────────────────────────────────
+
+@app.get("/backend/admin/usage", dependencies=[Depends(auth.require_manager)])
+def usage():
+    ev = db.events()
+    total_searches = ev.count_documents({"kind": "search"})
+    by_mode = list(ev.aggregate([
+        {"$match": {"kind": "search"}},
+        {"$group": {"_id": "$meta.mode", "count": {"$sum": 1}}},
+    ]))
+    top_queries = list(ev.aggregate([
+        {"$match": {"kind": "search", "meta.q": {"$ne": ""}}},
+        {"$group": {"_id": "$meta.q", "count": {"$sum": 1}}},
+        {"$sort": {"count": -1}}, {"$limit": 10},
+    ]))
+    zero_result = list(ev.aggregate([
+        {"$match": {"kind": "search", "meta.zero_result": True}},
+        {"$group": {"_id": "$meta.q", "count": {"$sum": 1}}},
+        {"$sort": {"count": -1}}, {"$limit": 10},
+    ]))
+    unique_users = len(ev.distinct("anon_id", {"kind": "search"}))
+    return {
+        "total_searches": total_searches,
+        "unique_users": unique_users,
+        "by_mode": {row["_id"] or "unknown": row["count"] for row in by_mode},
+        "top_queries": [{"query": r["_id"], "count": r["count"]} for r in top_queries],
+        "zero_result_queries": [{"query": r["_id"], "count": r["count"]} for r in zero_result],
+        "indexed_documents": db.documents().count_documents({}),
+        "last_refresh": (db.settings().find_one({"_id": "catalog"}) or {}).get("last_refresh"),
+    }
+
+
+# ── internal ──────────────────────────────────────────────────────────────────
+
+def _log_event(kind: str, meta: dict) -> None:
+    try:
+        db.events().insert_one({
+            "kind": kind,
+            "meta": meta,
+            "anon_id": meta.get("anon_id", "anon"),
+            "at": dt.datetime.now(dt.timezone.utc),
+        })
+    except Exception:  # noqa: BLE001 - telemetry must never break a request
+        pass