ltcai 9.6.0 → 9.8.0

package/lattice_brain/graph/retrieval.py

@@ -5,6 +5,40 @@ from __future__ import annotations
 from ._kg_common import *  # noqa: F403,F401
 
 
+def context_quality_signal(
+    mode: str,
+    nodes: int,
+    *,
+    reason: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Honest RAG context-quality signal (v9.8.0, additive contract).
+
+    Shape consumed by the chat metadata channel:
+    ``{"mode": "hybrid"|"lexical_only"|"none", "nodes": int, "limited": bool,
+    "reason": str|None}``. ``nodes == 0`` always collapses ``mode`` to
+    ``"none"``; ``limited`` is true whenever the context is thin (0–1 nodes)
+    or the vector side fell back to lexical-only retrieval. ``reason`` is a
+    short human-readable Korean phrase, only present when limited.
+    """
+    nodes = max(0, int(nodes or 0))
+    mode = str(mode or "none")
+    if nodes == 0:
+        mode = "none"
+    if mode not in ("hybrid", "lexical_only", "none"):
+        mode = "lexical_only"
+    limited = nodes <= 1 or mode != "hybrid"
+    if reason is None and limited:
+        if nodes == 0:
+            reason = "그래프에서 관련 지식을 찾지 못했습니다"
+        elif mode == "lexical_only":
+            reason = "벡터 검색을 사용할 수 없어 키워드 검색 결과만 사용했습니다"
+        else:
+            reason = "그래프 기반 컨텍스트가 제한적입니다"
+    if not limited:
+        reason = None
+    return {"mode": mode, "nodes": nodes, "limited": limited, "reason": reason}
+
+
 class KnowledgeGraphRetrievalMixin:
     _GRAPH_VISIBLE_TYPES = (
         "Computer",  # 내 컴퓨터
@@ -411,24 +445,249 @@ class KnowledgeGraphRetrievalMixin:
             )
         return {"query": query, "matches": matches}
 
-    def context_for_query(
+    def hybrid_search(
         self,
         query: str,
-        limit: int = 6,
         *,
+        top_k: int = 20,
+        alpha: float = 0.6,
+        workspace_id: Optional[str] = None,
         allowed_workspaces=None,
         include_legacy_global: bool = False,
-    ) -> str:
-        """Return compact graph-backed RAG context for chat generation."""
+        lexical_limit: Optional[int] = None,
+        vector_limit: Optional[int] = None,
+        min_vector_score: float = 0.0,
+    ) -> Dict[str, Any]:
+        """Unified lexical + vector retrieval with alpha-weighted linear fusion.
+
+        Runs the SQLite lexical :meth:`search` and the embedding-backed
+        ``vector_search`` (sibling mixin via the store MRO), normalizes both
+        score spaces to ``[0, 1]``, fuses them as
+        ``alpha * vector + (1 - alpha) * lexical`` (the same shape as
+        ``lattice_brain.quality.HybridFusion`` — reimplemented here without
+        importing that module), and dedupes by ``node_id`` (chunk hits roll up
+        to their parent node).
+
+        Degrades gracefully: when the vector side is unavailable (mixin not
+        composed, embedder/index failure) the result falls back to
+        lexical-only ranking and reports ``mode: "lexical_only"`` with a
+        ``detail`` explaining why. Each match carries per-source ``scores``
+        and a ``fusion`` field (``lexical`` / ``vector`` / ``both``).
+
+        ``workspace_id`` is a convenience for single-workspace callers; the
+        richer ``allowed_workspaces`` set wins when both are provided.
+        """
         query = str(query or "").strip()
+        try:
+            top_k = int(top_k)
+        except (TypeError, ValueError):
+            top_k = 20
+        top_k = max(1, min(top_k, 100))
+        try:
+            alpha = float(alpha)
+        except (TypeError, ValueError):
+            alpha = 0.6
+        alpha = max(0.0, min(alpha, 1.0))
+        if allowed_workspaces is None and workspace_id:
+            allowed_workspaces = {str(workspace_id)}
+
         if not query:
-            return ""
-        matches = self.search(
+            return {
+                "query": query,
+                "mode": "hybrid",
+                "alpha": alpha,
+                "top_k": top_k,
+                "sources": {"lexical": 0, "vector": 0},
+                "matches": [],
+                "detail": None,
+            }
+
+        lex_fetch = max(1, min(int(lexical_limit or max(top_k * 2, 20)), 100))
+        vec_fetch = max(1, min(int(vector_limit or max(top_k * 2, 20)), 100))
+
+        lexical_matches = self.search(
             query,
-            limit,
+            lex_fetch,
             allowed_workspaces=allowed_workspaces,
             include_legacy_global=include_legacy_global,
         ).get("matches", [])
+
+        mode = "hybrid"
+        detail: Optional[str] = None
+        vector_matches: List[Dict[str, Any]] = []
+        vector_fn = getattr(self, "vector_search", None)
+        if not callable(vector_fn):
+            mode = "lexical_only"
+            detail = "vector search is not available on this store"
+        else:
+            try:
+                vector_matches = list(
+                    (vector_fn(query, limit=vec_fetch, min_score=min_vector_score) or {}).get(
+                        "matches", []
+                    )
+                )
+            except Exception as exc:  # noqa: BLE001 — degrade, never fail the search
+                mode = "lexical_only"
+                detail = f"vector index unavailable: {exc}"
+                vector_matches = []
+        if vector_matches and allowed_workspaces is not None:
+            vector_matches = self.filter_scoped_nodes(
+                vector_matches,
+                allowed_workspaces,
+                id_key="node_id",
+                include_legacy_global=include_legacy_global,
+            )
+
+        def _parent_node_id(match: Dict[str, Any]) -> str:
+            # Chunk-level hits dedupe to their parent content node.
+            if match.get("type") == "Chunk":
+                meta = match.get("metadata") or {}
+                parent = meta.get("source_node") or meta.get("parent_source_node")
+                if parent:
+                    return str(parent)
+            return str(match.get("node_id") or match.get("id") or "")
+
+        entries: Dict[str, Dict[str, Any]] = {}
+
+        def _entry_for(node_id: str, match: Dict[str, Any]) -> Dict[str, Any]:
+            entry = entries.get(node_id)
+            if entry is None:
+                entry = {
+                    "node_id": node_id,
+                    "id": match.get("id") or node_id,
+                    "type": match.get("type"),
+                    "title": match.get("title"),
+                    "summary": match.get("summary"),
+                    "metadata": match.get("metadata") or {},
+                    "updated_at": match.get("updated_at"),
+                    "scores": {"lexical": 0.0, "vector": 0.0},
+                    "_lexical": False,
+                    "_vector": False,
+                }
+                entries[node_id] = entry
+            return entry
+
+        for rank, match in enumerate(lexical_matches, start=1):
+            node_id = _parent_node_id(match)
+            if not node_id:
+                continue
+            entry = _entry_for(node_id, match)
+            entry["scores"]["lexical"] = max(
+                entry["scores"]["lexical"], round(1.0 / rank, 6)
+            )
+            entry["_lexical"] = True
+
+        # Max-normalize cosine scores into [0, 1] (guard the score-0 falsy trap
+        # by comparing explicitly, never with truthiness).
+        max_vec = 0.0
+        for match in vector_matches:
+            raw = match.get("score")
+            if raw is not None and float(raw) > max_vec:
+                max_vec = float(raw)
+        for match in vector_matches:
+            node_id = _parent_node_id(match)
+            if not node_id:
+                continue
+            raw = float(match.get("score") or 0.0)
+            vec_norm = max(0.0, raw) / max_vec if max_vec > 0 else 0.0
+            entry = _entry_for(node_id, match)
+            entry["scores"]["vector"] = max(entry["scores"]["vector"], round(vec_norm, 6))
+            entry["_vector"] = True
+            # Prefer a real snippet when the lexical row had no summary.
+            if not entry.get("summary") and match.get("summary"):
+                entry["summary"] = match.get("summary")
+
+        matches: List[Dict[str, Any]] = []
+        for entry in entries.values():
+            lex_score = float(entry["scores"]["lexical"])
+            vec_score = float(entry["scores"]["vector"])
+            if mode == "lexical_only":
+                fused = lex_score
+            else:
+                fused = alpha * vec_score + (1.0 - alpha) * lex_score
+            entry["score"] = round(fused, 6)
+            from_lexical = bool(entry.pop("_lexical", False))
+            from_vector = bool(entry.pop("_vector", False))
+            if from_lexical and from_vector:
+                entry["fusion"] = "both"
+            elif from_vector:
+                entry["fusion"] = "vector"
+            else:
+                entry["fusion"] = "lexical"
+            matches.append(entry)
+
+        matches.sort(key=lambda item: (-item["score"], item["node_id"]))
+        matches = matches[:top_k]
+        for rank, match in enumerate(matches, start=1):
+            match["rank"] = rank
+        return {
+            "query": query,
+            "mode": mode,
+            "alpha": alpha,
+            "top_k": top_k,
+            "sources": {"lexical": len(lexical_matches), "vector": len(vector_matches)},
+            "matches": matches,
+            "detail": detail,
+        }
+
+    def context_for_query(
+        self,
+        query: str,
+        limit: int = 6,
+        *,
+        allowed_workspaces=None,
+        include_legacy_global: bool = False,
+        use_hybrid: bool = False,
+        with_meta: bool = False,
+    ):
+        """Return compact graph-backed RAG context for chat generation.
+
+        ``use_hybrid=True`` sources the matches from :meth:`hybrid_search`
+        (lexical + vector fusion) instead of the lexical-only :meth:`search`.
+        Default behavior is unchanged, and any hybrid failure silently falls
+        back to the legacy lexical path.
+
+        ``with_meta=True`` (additive, v9.8.0) returns
+        ``{"context": str, "quality": {...}}`` instead of the bare string.
+        ``quality`` follows :func:`context_quality_signal` and honestly
+        reports how the context was retrieved (hybrid vs lexical-only
+        fallback vs nothing). The ``context`` value is byte-identical to the
+        default ``with_meta=False`` return for the same arguments.
+        """
+        query = str(query or "").strip()
+        if not query:
+            if with_meta:
+                return {
+                    "context": "",
+                    "quality": context_quality_signal(
+                        "none", 0, reason="질의가 비어 있습니다"
+                    ),
+                }
+            return ""
+        matches: List[Dict[str, Any]] = []
+        retrieval_mode = "none"
+        if use_hybrid:
+            try:
+                hybrid = self.hybrid_search(
+                    query,
+                    top_k=limit,
+                    allowed_workspaces=allowed_workspaces,
+                    include_legacy_global=include_legacy_global,
+                )
+                matches = hybrid.get("matches", [])
+                if matches:
+                    retrieval_mode = str(hybrid.get("mode") or "hybrid")
+            except Exception:  # noqa: BLE001 — context building must never fail
+                matches = []
+        if not matches:
+            matches = self.search(
+                query,
+                limit,
+                allowed_workspaces=allowed_workspaces,
+                include_legacy_global=include_legacy_global,
+            ).get("matches", [])
+            if matches:
+                retrieval_mode = "lexical_only"
         if not matches:
             topics = _topic_candidates(query, limit=4)
             if topics:
@@ -471,6 +730,8 @@ class KnowledgeGraphRetrievalMixin:
                         allowed_workspaces,
                         include_legacy_global=include_legacy_global,
                     )
+                if matches:
+                    retrieval_mode = "lexical_only"
         lines = []
         for match in matches[:limit]:
             meta = match.get("metadata") or {}
@@ -485,7 +746,39 @@ class KnowledgeGraphRetrievalMixin:
             lines.append(
                 f"- [{match['type']}] {match['title']} | source={source} | {summary}"
             )
-        return "\n".join(lines)
+        context = "\n".join(lines)
+        if not with_meta:
+            return context
+        return {
+            "context": context,
+            "quality": context_quality_signal(retrieval_mode, len(matches[:limit])),
+        }
+
+    def context_for_query_with_meta(
+        self,
+        query: str,
+        limit: int = 6,
+        *,
+        allowed_workspaces=None,
+        include_legacy_global: bool = False,
+        use_hybrid: bool = True,
+    ) -> Dict[str, Any]:
+        """Additive companion to :meth:`context_for_query` (v9.8.0).
+
+        Always returns ``{"context": str, "quality": {...}}`` so chat callers
+        can surface an honest retrieval signal without changing the legacy
+        string-returning contract. Defaults to hybrid retrieval because meta
+        consumers want the vector-fallback signal; pass ``use_hybrid=False``
+        for the legacy lexical-only sourcing.
+        """
+        return self.context_for_query(
+            query,
+            limit,
+            allowed_workspaces=allowed_workspaces,
+            include_legacy_global=include_legacy_global,
+            use_hybrid=use_hybrid,
+            with_meta=True,
+        )
 
     def neighbors(
         self,

package/lattice_brain/graph/retrieval_vector.py

@@ -72,6 +72,108 @@ class KnowledgeGraphVectorMixin:
                     )
         return items
 
+    def index_node_incremental(self, node_id: str) -> Dict[str, Any]:
+        """Embed/index only ``node_id`` and its chunks (incremental sync).
+
+        The item construction mirrors :meth:`_iter_vector_source_items` exactly
+        (same ids, same ``source_node``/``parent_source_node`` semantics), so
+        anything this method indexes is indistinguishable from a full
+        :meth:`rebuild_vector_index` pass — and anything it *fails* to index
+        stays visible as ``missing``/``stale`` backlog in :meth:`index_status`,
+        where a later rebuild picks it up.
+
+        Never raises: embedding-provider or storage failures are reported as
+        ``{"status": "failed", ...}`` so ingestion callers can degrade instead
+        of losing an already-persisted write.
+        """
+        node_id = str(node_id or "").strip()
+        started = time.perf_counter()
+        summary: Dict[str, Any] = {
+            "node_id": node_id,
+            "items_total": 0,
+            "items_indexed": 0,
+            "items_skipped": 0,
+        }
+        if not node_id:
+            return {**summary, "status": "skipped", "detail": "node_id required"}
+        try:
+            with self._connect() as conn:
+                row = conn.execute(
+                    "SELECT id, type, title, summary, metadata_json FROM nodes WHERE id=?",
+                    (node_id,),
+                ).fetchone()
+                if row is None:
+                    return {**summary, "status": "skipped", "detail": "node not found"}
+                items: List[Dict[str, Any]] = []
+                if row["type"] != "Chunk":
+                    metadata = _safe_loads(row["metadata_json"])
+                    text = self._vector_text_for_node(
+                        title=row["title"],
+                        summary=row["summary"] or "",
+                        metadata=metadata,
+                    )
+                    if text:
+                        items.append(
+                            {
+                                "item_id": row["id"],
+                                "item_type": "node",
+                                "source_node": row["id"],
+                                "text": text,
+                                "metadata": {"node_type": row["type"], **metadata},
+                            }
+                        )
+                for chunk_row in conn.execute(
+                    """
+                        SELECT c.id, c.source_node AS parent_source_node, c.text, c.metadata_json
+                        FROM chunks c
+                        JOIN nodes n ON n.id=c.id
+                        WHERE c.source_node=?
+                        ORDER BY c.created_at ASC, c.id ASC
+                        """,
+                    (node_id,),
+                ).fetchall():
+                    metadata = _safe_loads(chunk_row["metadata_json"])
+                    text = _clean_text(chunk_row["text"] or "")
+                    if text:
+                        items.append(
+                            {
+                                "item_id": chunk_row["id"],
+                                "item_type": "chunk",
+                                "source_node": chunk_row["id"],
+                                "text": text,
+                                "metadata": {
+                                    **metadata,
+                                    "parent_source_node": chunk_row["parent_source_node"],
+                                },
+                            }
+                        )
+                indexed = skipped = 0
+                for item in items:
+                    if self._upsert_vector_item(conn, **item):
+                        indexed += 1
+                    else:
+                        skipped += 1
+            summary.update(
+                {
+                    "items_total": len(items),
+                    "items_indexed": indexed,
+                    "items_skipped": skipped,
+                }
+            )
+            return {
+                **summary,
+                "status": "indexed" if indexed else "noop",
+                "duration_ms": round((time.perf_counter() - started) * 1000, 2),
+                "embedding_model": self._embedding_model.model_id,
+            }
+        except Exception as exc:  # noqa: BLE001 — incremental sync must never raise
+            return {
+                **summary,
+                "status": "failed",
+                "detail": str(exc),
+                "duration_ms": round((time.perf_counter() - started) * 1000, 2),
+            }
+
     def rebuild_vector_index(
         self,
         *,
@@ -368,6 +470,49 @@ class KnowledgeGraphVectorMixin:
             ],
         }
 
+    def vector_freshness(self) -> Dict[str, Any]:
+        """Compact vector-index freshness summary for API surfaces (v9.8.0).
+
+        Reduces :meth:`index_status` (``pending = missing + stale``) to the
+        fixed contract ``{"status", "pending_items", "total_items", "detail"}``
+        with ``status`` in ``ready`` / ``pending`` / ``unavailable``.
+
+        Never raises: environments where the embedding provider or index
+        storage cannot be used report ``"unavailable"`` with the cause in
+        ``detail`` instead of surfacing an exception to the API layer.
+        """
+        try:
+            status = self.index_status()
+        except Exception as exc:  # noqa: BLE001 — freshness must degrade, not fail
+            return {
+                "status": "unavailable",
+                "pending_items": 0,
+                "total_items": 0,
+                "detail": f"vector index status unavailable: {exc}",
+            }
+        pending = int(status.get("pending_items") or 0)
+        total = int(status.get("source_items") or 0)
+        if pending > 0:
+            return {
+                "status": "pending",
+                "pending_items": pending,
+                "total_items": total,
+                "detail": (
+                    f"{pending} of {total} items are missing or stale in the vector index"
+                ),
+            }
+        detail = (
+            "vector index is up to date"
+            if total
+            else "vector index is empty (no indexable items yet)"
+        )
+        return {
+            "status": "ready",
+            "pending_items": 0,
+            "total_items": total,
+            "detail": detail,
+        }
+
     def vector_search(
         self,
         query: str,