@cloudgrid-io/mcp 0.14.0 → 0.15.0

package/src/corpus/templates/semantic-search/services/backend/src/app/answer.py

@@ -0,0 +1,90 @@
+"""Grounded answer mode.
+
+Composes an answer from the indexed documents ONLY, with citations back to the
+documents used. No outside knowledge. Toggleable from Settings. If the answer LLM
+is not configured, it falls back to an extractive answer (the top passages
+themselves), still grounded and cited — never crashes.
+"""
+from __future__ import annotations
+
+import httpx
+
+from . import config, search
+
+
+def _answer_enabled_from_db() -> bool:
+    from . import db
+
+    doc = db.settings().find_one({"_id": "features"}) or {}
+    return bool(doc.get("answer_mode_enabled", True))
+
+
+def compose_answer(question: str) -> dict:
+    """Return a grounded answer + the documents cited. Extractive fallback if no LLM."""
+    if not _answer_enabled_from_db():
+        return {"answer": None, "citations": [], "note": "Answer mode is disabled."}
+
+    hits = search.search(question, mode="hybrid", limit=5)
+    passages = hits["results"]
+    if not passages:
+        return {
+            "answer": None,
+            "citations": [],
+            "note": "No indexed material matches this question.",
+        }
+
+    citations = [
+        {"document_id": p["document_id"], "title": p["title"], "collection": p["collection"]}
+        for p in passages
+    ]
+
+    # Grounded generation only if the LLM key is present; otherwise return an
+    # extractive answer built from the retrieved passages.
+    key = config.embeddings_api_key()
+    if not key:
+        extractive = "\n\n".join(
+            f"- {p['title']}: {p['best_passage']}" for p in passages[:3]
+        )
+        return {
+            "answer": extractive,
+            "citations": citations,
+            "mode": "extractive",
+            "note": "Grounded generation unavailable (no API key) — showing "
+                    "the most relevant source passages instead.",
+        }
+
+    context = "\n\n".join(
+        f"[{i+1}] {p['title']}:\n{p['best_passage']}" for i, p in enumerate(passages)
+    )
+    system = (
+        "You answer strictly and only from the provided document passages. "
+        "Cite sources as [n]. If the passages do not contain the answer, say so. "
+        "Never use outside knowledge."
+    )
+    try:
+        resp = httpx.post(
+            f"{config.embeddings_base_url()}/chat/completions",
+            headers={"Authorization": f"Bearer {key}"},
+            json={
+                "model": config.answer_model(),
+                "messages": [
+                    {"role": "system", "content": system},
+                    {"role": "user", "content": f"Question: {question}\n\nSources:\n{context}"},
+                ],
+                "temperature": 0.2,
+            },
+            timeout=60.0,
+        )
+        resp.raise_for_status()
+        text = resp.json()["choices"][0]["message"]["content"]
+        return {"answer": text, "citations": citations, "mode": "generated", "note": None}
+    except Exception as exc:  # noqa: BLE001 - degrade to extractive, never crash
+        extractive = "\n\n".join(
+            f"- {p['title']}: {p['best_passage']}" for p in passages[:3]
+        )
+        return {
+            "answer": extractive,
+            "citations": citations,
+            "mode": "extractive",
+            "note": f"Generation failed ({exc}); showing source passages.",
+        }

package/src/corpus/templates/semantic-search/services/backend/src/app/auth.py

@@ -0,0 +1,65 @@
+"""Server-side manager session auth.
+
+Admin functions are enforced on the server, not just hidden in the UI. Sessions
+are opaque tokens stored in Mongo. The password is compared against
+MANAGER_PASSWORD_HASH (sha256 hex). If no hash is configured, admin login is
+refused (never open by default) — but the app still starts and search stays
+public.
+"""
+from __future__ import annotations
+
+import datetime as dt
+import hashlib
+import secrets
+
+from fastapi import Header, HTTPException
+
+from . import config, db
+
+SESSION_TTL_HOURS = 12
+
+
+def _hash(password: str) -> str:
+    return hashlib.sha256(password.encode("utf-8")).hexdigest()
+
+
+def login(password: str) -> str:
+    configured = config.manager_password_hash()
+    if not configured:
+        raise HTTPException(
+            status_code=503,
+            detail="Manager auth is not configured (MANAGER_PASSWORD_HASH unset).",
+        )
+    if not secrets.compare_digest(_hash(password), configured):
+        raise HTTPException(status_code=401, detail="Invalid password.")
+    token = secrets.token_urlsafe(32)
+    db.get_db().sessions.insert_one({
+        "_id": token,
+        "created_at": dt.datetime.now(dt.timezone.utc),
+        "expires_at": dt.datetime.now(dt.timezone.utc) + dt.timedelta(hours=SESSION_TTL_HOURS),
+    })
+    return token
+
+
+def require_manager(authorization: str | None = Header(default=None)) -> str:
+    """FastAPI dependency: validate the Bearer session token."""
+    if not authorization or not authorization.lower().startswith("bearer "):
+        raise HTTPException(status_code=401, detail="Manager session required.")
+    token = authorization.split(" ", 1)[1].strip()
+    session = db.get_db().sessions.find_one({"_id": token})
+    if not session:
+        raise HTTPException(status_code=401, detail="Invalid session.")
+    if session["expires_at"] < dt.datetime.now(dt.timezone.utc):
+        db.get_db().sessions.delete_one({"_id": token})
+        raise HTTPException(status_code=401, detail="Session expired.")
+    return token
+
+
+def change_password(current: str, new: str) -> str:
+    """Self-contained password change. Returns the NEW hash for the manager to
+    store as the MANAGER_PASSWORD_HASH secret (we never persist the plaintext or
+    the secret ourselves)."""
+    configured = config.manager_password_hash()
+    if configured and not secrets.compare_digest(_hash(current), configured):
+        raise HTTPException(status_code=401, detail="Current password is incorrect.")
+    return _hash(new)

package/src/corpus/templates/semantic-search/services/backend/src/app/config.py

@@ -0,0 +1,110 @@
+"""Central configuration and secret gating.
+
+CRITICAL CloudGrid rule: NEVER read a secret at module import time. Every value
+here is resolved lazily inside a function, so the process starts and passes
+`GET /backend/health` even when NO secret is set. Search and indexing degrade
+gracefully (empty results + a clear "not indexed / missing secret" note) rather
+than crashing.
+
+The Mongo connection is the ONLY hard dependency for health, and it is injected
+automatically by CloudGrid (`needs: database`) as DATABASE_MONGODB_URL.
+
+Two pluggable config points:
+  - SOURCE_TYPE  selects the document-source adapter (dropbox | local | url).
+  - EMBEDDINGS_* selects the embeddings provider (OpenAI-compatible by default).
+"""
+from __future__ import annotations
+
+import os
+
+
+# ── Mongo (injected by the grid — the only health-critical config) ──────────
+
+def mongo_url() -> str | None:
+    """Injected by CloudGrid `needs: database`. Never set by hand."""
+    return os.environ.get("DATABASE_MONGODB_URL") or os.environ.get("MONGODB_URL")
+
+
+# ── Embeddings (one config point, gated) ─────────────────────────────────────
+
+def embeddings_api_key() -> str | None:
+    return os.environ.get("EMBEDDINGS_API_KEY")
+
+
+def embeddings_model() -> str:
+    # Changing the model changes vector dimensions and requires a full reindex.
+    return os.environ.get("EMBEDDINGS_MODEL", "text-embedding-3-small")
+
+
+def embeddings_base_url() -> str:
+    # Any OpenAI-compatible endpoint. Default is OpenAI.
+    return os.environ.get("EMBEDDINGS_BASE_URL", "https://api.openai.com/v1")
+
+
+def embeddings_configured() -> bool:
+    return bool(embeddings_api_key())
+
+
+def answer_model() -> str:
+    return os.environ.get("ANSWER_MODEL", "gpt-4o-mini")
+
+
+# ── Source adapter (one config point, gated) ─────────────────────────────────
+
+def source_type() -> str:
+    """dropbox | local | url. Selects the document-source adapter."""
+    return os.environ.get("SOURCE_TYPE", "dropbox")
+
+
+def dropbox_config() -> dict[str, str | None]:
+    return {
+        "app_key": os.environ.get("DROPBOX_APP_KEY"),
+        "app_secret": os.environ.get("DROPBOX_APP_SECRET"),
+        "refresh_token": os.environ.get("DROPBOX_REFRESH_TOKEN"),
+        "folder_path": os.environ.get("DROPBOX_FOLDER_PATH", ""),
+    }
+
+
+def local_source_path() -> str:
+    """Directory the local adapter reads. Point at a `needs: disk` mount
+    (DISK_PATH) or a bundled folder."""
+    return os.environ.get("LOCAL_SOURCE_PATH", os.environ.get("DISK_PATH", ""))
+
+
+def url_source_manifest() -> str | None:
+    """A newline- or comma-separated list of document URLs the url adapter
+    fetches. Set as a secret or plain env var."""
+    return os.environ.get("URL_SOURCE_MANIFEST")
+
+
+def source_configured() -> bool:
+    st = source_type()
+    if st == "dropbox":
+        c = dropbox_config()
+        return bool(c["app_key"] and c["app_secret"] and c["refresh_token"])
+    if st == "local":
+        return bool(local_source_path())
+    if st == "url":
+        return bool(url_source_manifest())
+    return False
+
+
+# ── Manager auth (gated) ──────────────────────────────────────────────────────
+
+def manager_password_hash() -> str | None:
+    """A sha256 hex digest of the manager password. Compared server-side."""
+    return os.environ.get("MANAGER_PASSWORD_HASH")
+
+
+def manager_auth_configured() -> bool:
+    return bool(manager_password_hash())
+
+
+# ── Search blend weights ─────────────────────────────────────────────────────
+
+def semantic_weight() -> float:
+    return float(os.environ.get("SEMANTIC_WEIGHT", "0.55"))
+
+
+def keyword_weight() -> float:
+    return float(os.environ.get("KEYWORD_WEIGHT", "0.45"))

package/src/corpus/templates/semantic-search/services/backend/src/app/db.py

@@ -0,0 +1,89 @@
+"""Lazy MongoDB access.
+
+The client and connection string are resolved LAZILY (inside get_client), never
+at module top level — so importing this module never touches the network and
+never requires the env var to exist. CloudGrid injects DATABASE_MONGODB_URL at
+runtime from `needs: database`.
+
+Collections:
+  documents  — one row per source document (title, collection tag, date, ...).
+  chunks     — one embeddable passage per document (content + embedding array).
+  settings   — feature flags, synonyms, catalog metadata (last_refresh).
+  usage_events, index_reports, sessions — telemetry / admin.
+"""
+from __future__ import annotations
+
+from typing import Optional
+
+from pymongo import ASCENDING, MongoClient, TEXT
+from pymongo.database import Database
+
+from . import config
+
+_client: Optional[MongoClient] = None
+
+
+def get_client() -> MongoClient:
+    global _client
+    if _client is None:
+        uri = config.mongo_url()
+        if not uri:
+            raise RuntimeError(
+                "DATABASE_MONGODB_URL is not set. CloudGrid injects it "
+                "automatically from `needs: database` — run with `grid dev` "
+                "locally or deploy with `grid plug`. Do not set it by hand."
+            )
+        # Short server-selection timeout so health checks fail fast, not hang.
+        _client = MongoClient(uri, serverSelectionTimeoutMS=3000)
+    return _client
+
+
+def get_db() -> Database:
+    # Default DB name comes from the connection-string path segment the grid injects.
+    return get_client().get_default_database()
+
+
+def ping() -> bool:
+    """True if Mongo is reachable. Used by /backend/health."""
+    get_client().admin.command("ping")
+    return True
+
+
+def ensure_indexes() -> None:
+    """Create the indexes the search relies on. Safe to call repeatedly.
+
+    Called at FastAPI startup so the chunks $text index exists before any query
+    — without it, `$text` raises IndexNotFound (a 500) on an empty catalog.
+    """
+    db = get_db()
+    # Text index on chunk content for lexical / $text search.
+    db.chunks.create_index([("content", TEXT)], name="chunks_content_text")
+    db.chunks.create_index([("document_id", ASCENDING)], name="chunks_document_id")
+    # Metadata filter indexes on documents.
+    db.documents.create_index([("date", ASCENDING)], name="documents_date")
+    db.documents.create_index([("collection", ASCENDING)], name="documents_collection")
+    db.documents.create_index(
+        [("source_filename", ASCENDING)], name="documents_source_filename", unique=True
+    )
+
+
+# ── Collection accessors ─────────────────────────────────────────────────────
+
+def documents():
+    return get_db().documents
+
+
+def chunks():
+    return get_db().chunks
+
+
+def settings():
+    return get_db().settings
+
+
+def events():
+    return get_db().usage_events
+
+
+def index_reports():
+    return get_db().index_reports

package/src/corpus/templates/semantic-search/services/backend/src/app/embeddings.py

@@ -0,0 +1,44 @@
+"""Single config point for embeddings (OpenAI-compatible).
+
+Isolated behind one module so the provider can be swapped by env var
+(EMBEDDINGS_BASE_URL / EMBEDDINGS_MODEL / EMBEDDINGS_API_KEY). NEVER reads the
+API key at import time — only inside embed(). If the key is missing, embed()
+raises EmbeddingsUnavailable, which callers translate into a graceful degraded
+state (lexical + metadata search still work).
+"""
+from __future__ import annotations
+
+import httpx
+
+from . import config
+
+
+class EmbeddingsUnavailable(RuntimeError):
+    """Raised when embeddings are requested but not configured/reachable."""
+
+
+def embed(texts: list[str]) -> list[list[float]]:
+    """Return one embedding vector per input text.
+
+    Raises EmbeddingsUnavailable if EMBEDDINGS_API_KEY is not set.
+    """
+    if not texts:
+        return []
+    key = config.embeddings_api_key()
+    if not key:
+        raise EmbeddingsUnavailable(
+            "EMBEDDINGS_API_KEY is not set — semantic embedding is unavailable."
+        )
+    resp = httpx.post(
+        f"{config.embeddings_base_url()}/embeddings",
+        headers={"Authorization": f"Bearer {key}"},
+        json={"model": config.embeddings_model(), "input": texts},
+        timeout=60.0,
+    )
+    resp.raise_for_status()
+    data = resp.json()["data"]
+    return [row["embedding"] for row in data]
+
+
+def embed_one(text: str) -> list[float]:
+    return embed([text])[0]

package/src/corpus/templates/semantic-search/services/backend/src/app/indexing.py

@@ -0,0 +1,222 @@
+"""Incremental indexing pipeline: discover -> extract -> chunk -> embed -> store.
+
+Shared by the manager 'Refresh now' endpoint (and, once platform issue #1585
+lands, a `type: cron` refresh service). It is resilient: if the source or
+embeddings are not configured it records a clear, non-crashing report and
+returns. Only new/changed files (by rev / content-hash) are reprocessed.
+"""
+from __future__ import annotations
+
+import datetime as dt
+import os
+import re
+
+from . import config, db, embeddings, source
+
+CHUNK_CHARS = 1200
+CHUNK_OVERLAP = 150
+
+# Generic date detection from filename or document head (best effort, nullable).
+_DATE_RE = re.compile(r"(\d{4})[./-](\d{1,2})[./-](\d{1,2})")        # 2026-07-05
+_DATE_RE_ALT = re.compile(r"(\d{1,2})[./-](\d{1,2})[./-](\d{2,4})")  # 05/07/2026
+
+
+def _utcnow() -> dt.datetime:
+    return dt.datetime.now(dt.timezone.utc)
+
+
+def _chunk_text(text: str) -> list[str]:
+    text = text.strip()
+    if not text:
+        return []
+    out: list[str] = []
+    start = 0
+    n = len(text)
+    while start < n:
+        end = min(start + CHUNK_CHARS, n)
+        out.append(text[start:end])
+        if end == n:
+            break
+        start = end - CHUNK_OVERLAP
+    return out
+
+
+def _parse_metadata(name: str, text: str) -> dict:
+    """Best-effort title / date / collection extraction. All fields nullable.
+
+    Generalize this for your domain: `collection` is a free-form grouping tag
+    (folder, series, category), `date` is an ISO date if one can be found.
+    """
+    title = os.path.splitext(name)[0].strip()
+    # Collection tag = the leading non-numeric words of the title.
+    collection = re.sub(r"[\d].*$", "", title).strip() or None
+
+    date_iso = None
+    haystack = f"{name}\n{text[:2000]}"
+    m = _DATE_RE.search(haystack)
+    if m:
+        y, mo, d = m.groups()
+        try:
+            date_iso = dt.datetime(int(y), int(mo), int(d)).date().isoformat()
+        except ValueError:
+            date_iso = None
+    if date_iso is None:
+        m = _DATE_RE_ALT.search(haystack)
+        if m:
+            d, mo, y = m.groups()
+            if len(y) == 2:
+                y = "20" + y
+            try:
+                date_iso = dt.datetime(int(y), int(mo), int(d)).date().isoformat()
+            except ValueError:
+                date_iso = None
+
+    return {"title": title, "collection": collection, "date": date_iso}
+
+
+def run_sync(triggered_by: str = "manual") -> dict:
+    """Run one incremental sync. Never raises on missing config — returns a report."""
+    started = _utcnow()
+    report: dict = {
+        "started_at": started,
+        "triggered_by": triggered_by,
+        "files": [],
+        "status": "ok",
+        "note": None,
+    }
+
+    # Mongo is required for indexing (and always injected). Fail loud only here.
+    db.ensure_indexes()
+
+    if not config.source_configured():
+        report["status"] = "skipped"
+        report["note"] = (
+            f"Source not configured (SOURCE_TYPE={config.source_type()!r} — "
+            "missing its secrets/config)."
+        )
+        _save_report(report)
+        return report
+
+    try:
+        src = source.get_source()
+        files = src.list_files()
+    except Exception as exc:  # noqa: BLE001 - resilient by design
+        report["status"] = "error"
+        report["note"] = f"Source listing failed: {exc}"
+        _save_report(report)
+        return report
+
+    embeddings_ok = config.embeddings_configured()
+    if not embeddings_ok:
+        report["note"] = (
+            "Embeddings not configured — documents indexed for lexical/metadata "
+            "search only (no semantic vectors)."
+        )
+
+    for f in files:
+        outcome = _process_file(src, f, embeddings_ok)
+        report["files"].append(outcome)
+
+    report["finished_at"] = _utcnow()
+    _save_report(report)
+    return report
+
+
+def _process_file(src: source.Source, f: source.SourceFile, embeddings_ok: bool) -> dict:
+    existing = db.documents().find_one({"source_filename": f.name})
+    if existing and existing.get("source_rev") == f.rev:
+        return {"file": f.name, "result": "unchanged"}
+
+    if f.is_audio and not f.is_text:
+        # Catalogue audio/media with a flag; do not search inside it.
+        doc = {
+            "title": os.path.splitext(f.name)[0],
+            "source_filename": f.name,
+            "source_rev": f.rev,
+            "file_path": f.path,
+            "collection": None,
+            "date": None,
+            "has_media": True,
+            "media_path": f.path,
+            "word_count": 0,
+            "indexed_at": _utcnow(),
+        }
+        doc_id = _upsert_document(doc)
+        db.chunks().delete_many({"document_id": doc_id})
+        return {"file": f.name, "result": "catalogued_media"}
+
+    if not f.is_text:
+        return {"file": f.name, "result": "skipped_unsupported"}
+
+    try:
+        raw = src.download(f.path)
+        text = source.extract_text(f.name, raw)
+    except Exception as exc:  # noqa: BLE001
+        return {"file": f.name, "result": "error", "detail": str(exc)}
+
+    meta = _parse_metadata(f.name, text)
+    doc = {
+        **meta,
+        "source_filename": f.name,
+        "source_rev": f.rev,
+        "file_path": f.path,
+        "has_media": False,
+        "media_path": None,
+        "word_count": len(text.split()),
+        "indexed_at": _utcnow(),
+    }
+    doc_id = _upsert_document(doc)
+
+    # Rebuild chunks for this document.
+    db.chunks().delete_many({"document_id": doc_id})
+    parts = _chunk_text(text)
+    vectors: list[list[float] | None] = [None] * len(parts)
+    if embeddings_ok and parts:
+        try:
+            vectors = embeddings.embed(parts)  # type: ignore[assignment]
+        except Exception as exc:  # noqa: BLE001
+            return {
+                "file": f.name,
+                "result": "indexed_no_embeddings",
+                "detail": f"embedding failed: {exc}",
+                "chunks": len(parts),
+            }
+
+    docs = []
+    for i, content in enumerate(parts):
+        docs.append({
+            "document_id": doc_id,
+            "chunk_index": i,
+            "content": content,
+            "embedding": vectors[i] if i < len(vectors) else None,
+            "token_count": len(content.split()),
+        })
+    if docs:
+        db.chunks().insert_many(docs)
+
+    return {
+        "file": f.name,
+        "result": "indexed" if embeddings_ok else "indexed_no_embeddings",
+        "chunks": len(docs),
+    }
+
+
+def _upsert_document(doc: dict):
+    db.documents().update_one(
+        {"source_filename": doc["source_filename"]},
+        {"$set": doc},
+        upsert=True,
+    )
+    return db.documents().find_one({"source_filename": doc["source_filename"]})["_id"]
+
+
+def _save_report(report: dict) -> None:
+    try:
+        db.index_reports().insert_one(dict(report))
+        db.settings().update_one(
+            {"_id": "catalog"},
+            {"$set": {"last_refresh": report.get("finished_at") or report["started_at"]}},
+            upsert=True,
+        )
+    except Exception:  # noqa: BLE001 - reporting must never crash the caller
+        pass