ltcai 11.2.0 → 11.4.0

package/lattice_brain/graph/retrieval/signals.py

@@ -0,0 +1,95 @@
+"""Honest context-quality and multimodal signals for a result set.
+
+Plain functions over an already-computed match list — no store, no I/O — so
+every mixin in the package (and the chat surfaces outside it) can build the
+same signal the same way.
+"""
+
+from __future__ import annotations
+
+# ruff: noqa: F403,F405
+from .._kg_common import *  # noqa: F403,F401
+
+#: Node types that are a *thing you can look at or listen to*, not prose. A
+#: match of one of these means the answer rests on more than text.
+MULTIMODAL_NODE_TYPES = ("Image", "ImageText")
+
+
+def multimodal_signal(matches: Iterable[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """``{"images": n, "types": [...]}`` when a result set includes pictures.
+
+    ``None`` when it does not: the context-quality contract stays four keys
+    wide for the ordinary all-text case, and a caller that sees the key knows
+    it means something rather than having to compare a zero.
+    """
+    images = 0
+    seen: List[str] = []
+    for match in matches:
+        node_type = str(match.get("type") or "")
+        if node_type in MULTIMODAL_NODE_TYPES:
+            images += 1
+            if node_type not in seen:
+                seen.append(node_type)
+    if not images:
+        return None
+    return {"images": images, "types": seen}
+
+
+def context_quality_signal(
+    mode: str,
+    nodes: int,
+    *,
+    reason: Optional[str] = None,
+    vector: Optional[Dict[str, Any]] = None,
+    multimodal: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    """Honest RAG context-quality signal (v9.8.0, additive contract).
+
+    Shape consumed by the chat metadata channel:
+    ``{"mode": "hybrid"|"lexical_only"|"none", "nodes": int, "limited": bool,
+    "reason": str|None}``. ``nodes == 0`` always collapses ``mode`` to
+    ``"none"``; ``limited`` is true whenever the context is thin (0–1 nodes)
+    or the vector side fell back to lexical-only retrieval. ``reason`` is a
+    short human-readable Korean phrase, only present when limited.
+
+    ``vector`` (v11.1.0) carries the vector channel's own honesty block —
+    which backend scored, whether it was approximate, whether the candidate
+    scan was truncated. "hybrid, 6 nodes" describes two different answers
+    depending on those bits, and the caller that has to say "I did not find
+    it" deserves to know which one it got. The key is present **only when
+    there is a caveat to report**: an exact, complete vector scan is the
+    contract's baseline assumption, so annotating it would be noise, and the
+    four-key shape stays exactly what existing consumers pin.
+
+    ``multimodal`` (v11.1.0) follows the same present-only-when-true rule and
+    says that part of this context is a picture. "6 nodes" reads differently
+    when two of them are screenshots whose text came out of OCR, and the
+    surface that has to explain the answer deserves to know.
+    """
+    nodes = max(0, int(nodes or 0))
+    mode = str(mode or "none")
+    if nodes == 0:
+        mode = "none"
+    if mode not in ("hybrid", "lexical_only", "none"):
+        mode = "lexical_only"
+    limited = nodes <= 1 or mode != "hybrid"
+    if reason is None and limited:
+        if nodes == 0:
+            reason = "그래프에서 관련 지식을 찾지 못했습니다"
+        elif mode == "lexical_only":
+            reason = "벡터 검색을 사용할 수 없어 키워드 검색 결과만 사용했습니다"
+        else:
+            reason = "그래프 기반 컨텍스트가 제한적입니다"
+    if not limited:
+        reason = None
+    signal: Dict[str, Any] = {
+        "mode": mode,
+        "nodes": nodes,
+        "limited": limited,
+        "reason": reason,
+    }
+    if vector is not None:
+        signal["vector"] = dict(vector)
+    if multimodal is not None:
+        signal["multimodal"] = dict(multimodal)
+    return signal

package/lattice_brain/graph/retrieval_vector/__init__.py

@@ -0,0 +1,42 @@
+"""The derived vector index: build it, report on it, search it.
+
+v11.3.0 turned this module into a package. ``KnowledgeGraphVectorMixin`` is
+now composed from four cohesive sub-mixins — embedder fingerprint, index
+build, index status, and search — each moved here verbatim. Every name this
+module exported before still resolves from
+``lattice_brain.graph.retrieval_vector``.
+"""
+
+from __future__ import annotations
+
+# ruff: noqa: F403,F405
+from .._kg_common import *  # noqa: F403,F401
+from .fingerprint import _VectorFingerprintMixin
+from .indexing import _VectorIndexingMixin
+from .search import (  # noqa: F401
+    DEFAULT_VECTOR_MAX_CANDIDATES,
+    VECTOR_MAX_CANDIDATES_CEILING,
+    VECTOR_MAX_CANDIDATES_ENV,
+    VECTOR_SCAN_BATCH,
+    _configured_vector_max_candidates,
+    _VectorSearchMixin,
+)
+from .status import _VectorStatusMixin
+
+
+# Base order is most-composed-first (status → indexing → fingerprint, then
+# search): each half names the ones it calls as its typing-only base, and C3
+# needs a subclass ahead of the class it extends. The method sets are disjoint,
+# so at runtime the order changes nothing.
+class KnowledgeGraphVectorMixin(
+    _VectorStatusMixin,
+    _VectorIndexingMixin,
+    _VectorFingerprintMixin,
+    _VectorSearchMixin,
+):
+    """Vector-embedding index build/status/search, split out of retrieval.
+
+    Composed into KnowledgeGraphStore alongside KnowledgeGraphRetrievalMixin;
+    both mixins share the same instance, so vector methods still reach sibling
+    retrieval/write helpers (e.g. self._vector_text_for_node) through the MRO.
+    """

package/lattice_brain/graph/retrieval_vector/fingerprint.py

@@ -0,0 +1,97 @@
+"""Which embedder actually built the index, recorded in ``graph_meta``.
+
+``vector_search`` filters on the current model/dim, so swapping the embedder
+silently yields zero vector rows; the fingerprint turns that into the honest
+``stale_embedder`` signal. Moved verbatim out of ``retrieval_vector.py``.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+# ruff: noqa: F403,F405
+from .._kg_common import *  # noqa: F403,F401
+
+# The cross-mixin surface (`_connect`, `_upsert_node`, …) is declared in
+# `_kg_contract.KnowledgeGraphCore`. It is a typing-only base: at runtime this
+# is `object`, so the MRO of `KnowledgeGraphStore` is unchanged.
+if TYPE_CHECKING:
+    from .._kg_contract import KnowledgeGraphCore as _Core
+else:
+    _Core = object
+
+
+class _VectorFingerprintMixin(_Core):
+    """Embedder-fingerprint read/write. Composed into the public mixin."""
+
+    # ── embedder fingerprint (review Wave 2.2 — stale_embedder) ──────────────
+    # vector_search filters on the CURRENT model/dim, so swapping the embedder
+    # silently yields zero vector rows. The fingerprint persisted in graph_meta
+    # records which embedder actually built the index; a mismatch is surfaced
+    # as the honest ``stale_embedder`` signal instead of a silent degradation.
+
+    _EMBEDDER_FINGERPRINT_KEY = "embedder_fingerprint"
+
+    def _embedder_fingerprint_record(
+        self, conn: sqlite3.Connection
+    ) -> Optional[Dict[str, Any]]:
+        """Read the recorded embedder fingerprint from graph_meta (or None)."""
+        row = conn.execute(
+            "SELECT value FROM graph_meta WHERE key=?",
+            (self._EMBEDDER_FINGERPRINT_KEY,),
+        ).fetchone()
+        if not row:
+            return None
+        payload = _safe_loads(row["value"])
+        if not isinstance(payload, dict) or not payload.get("model_id"):
+            return None
+        try:
+            dim = int(payload.get("dim") or 0)
+        except (TypeError, ValueError):
+            dim = 0
+        return {"model_id": str(payload["model_id"]), "dim": dim}
+
+    def _write_embedder_fingerprint(self, conn: sqlite3.Connection) -> Dict[str, Any]:
+        """Persist the CURRENT embedder identity (same transaction as caller)."""
+        fingerprint = {
+            "model_id": self._embedding_model.model_id,
+            "dim": int(self._embedding_model.dim),
+        }
+        conn.execute(
+            "INSERT OR REPLACE INTO graph_meta(key, value) VALUES (?, ?)",
+            (self._EMBEDDER_FINGERPRINT_KEY, _json(fingerprint)),
+        )
+        return fingerprint
+
+    def record_embedder_fingerprint(self) -> Dict[str, Any]:
+        """Record the current embedder (model_id + dim) as the index builder."""
+        with self._connect() as conn:
+            return self._write_embedder_fingerprint(conn)
+
+    def embedder_fingerprint_status(self) -> Dict[str, Any]:
+        """Compare the current embedder against the recorded index fingerprint.
+
+        Returns ``{"current": {model_id, dim}, "recorded": {...} | None,
+        "stale_embedder": bool}``. ``stale_embedder`` is True only when a
+        fingerprint was recorded AND it differs from the current embedder —
+        an unrecorded index (legacy DBs, nothing indexed yet) is honestly
+        "unknown", never reported stale. Never raises.
+        """
+        current = {
+            "model_id": self._embedding_model.model_id,
+            "dim": int(self._embedding_model.dim),
+        }
+        recorded: Optional[Dict[str, Any]] = None
+        try:
+            with self._connect() as conn:
+                recorded = self._embedder_fingerprint_record(conn)
+        except Exception:  # noqa: BLE001 — status must degrade, never raise
+            recorded = None
+        stale = bool(
+            recorded is not None
+            and (
+                recorded.get("model_id") != current["model_id"]
+                or recorded.get("dim") != current["dim"]
+            )
+        )
+        return {"current": current, "recorded": recorded, "stale_embedder": stale}

package/lattice_brain/graph/retrieval_vector/indexing.py

@@ -0,0 +1,347 @@
+"""Building the vector index: incremental upserts and full rebuilds.
+
+Moved verbatim out of ``retrieval_vector.py`` (v11.3.0 decomposition).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+# ruff: noqa: F403,F405
+from .._kg_common import *  # noqa: F403,F401
+
+# Typing-only base (runtime value is `object`, so the store's MRO is
+# unchanged). Every build path records which embedder produced the rows, so
+# this half calls the fingerprint half through `self`; naming it as the base
+# states that assumption and carries the store contract
+# (`_connect`, `_upsert_node`, …) along with it.
+if TYPE_CHECKING:
+    from .fingerprint import _VectorFingerprintMixin as _Core
+else:
+    _Core = object
+
+
+class _VectorIndexingMixin(_Core):
+    """Vector index build/refresh. Composed into the public mixin."""
+
+    def _vector_text_hashes(self, conn: sqlite3.Connection) -> Dict[str, str]:
+        """``item_id -> text_hash`` for rows already embedded by *this* embedder.
+
+        The incremental rebuild's job is mostly deciding what it does *not*
+        have to do, and it used to ask that question with one ``SELECT`` per
+        candidate item — a round trip per node and per chunk on every run,
+        almost all of which answer "unchanged". One query returning two short
+        columns replaces all of them.
+
+        Rows written by a different embedder are left out, so they compare as
+        missing and get re-embedded, which is what an embedder swap requires.
+        """
+        return {
+            row["item_id"]: row["text_hash"]
+            for row in conn.execute(
+                """
+                    SELECT item_id, text_hash
+                    FROM vector_embeddings
+                    WHERE embedding_model=? AND embedding_dim=?
+                    """,
+                (self._embedding_model.model_id, self._embedding_model.dim),
+            ).fetchall()
+        }
+
+    def _iter_vector_source_items(
+        self,
+        conn: sqlite3.Connection,
+        *,
+        include_nodes: bool = True,
+        include_chunks: bool = True,
+    ) -> Iterator[Dict[str, Any]]:
+        """Stream the graph's embeddable text, one item at a time.
+
+        Yields rather than returns a list. Every caller consumes this exactly
+        once in a ``for``, and building the list first meant a rebuild held
+        the full text of every node and chunk in memory simultaneously — the
+        one shape guaranteed to fail on precisely the large graph that most
+        needs the index.
+        """
+        if include_nodes:
+            for row in conn.execute(
+                """
+                    SELECT id, type, title, summary, metadata_json
+                    FROM nodes
+                    WHERE type <> 'Chunk'
+                    ORDER BY updated_at DESC, id ASC
+                    """
+            ).fetchall():
+                metadata = _safe_loads(row["metadata_json"])
+                text = self._vector_text_for_node(
+                    title=row["title"],
+                    summary=row["summary"] or "",
+                    metadata=metadata,
+                )
+                if text:
+                    yield {
+                        "item_id": row["id"],
+                        "item_type": "node",
+                        "source_node": row["id"],
+                        "text": text,
+                        "metadata": {"node_type": row["type"], **metadata},
+                    }
+        if include_chunks:
+            for row in conn.execute(
+                """
+                    SELECT c.id, c.source_node AS parent_source_node, c.text, c.metadata_json
+                    FROM chunks c
+                    JOIN nodes n ON n.id=c.id
+                    ORDER BY c.created_at DESC, c.id ASC
+                    """
+            ).fetchall():
+                metadata = _safe_loads(row["metadata_json"])
+                text = _clean_text(row["text"] or "")
+                if text:
+                    yield {
+                        "item_id": row["id"],
+                        "item_type": "chunk",
+                        "source_node": row["id"],
+                        "text": text,
+                        "metadata": {
+                            **metadata,
+                            "parent_source_node": row["parent_source_node"],
+                        },
+                    }
+
+    def index_node_incremental(self, node_id: str) -> Dict[str, Any]:
+        """Embed/index only ``node_id`` and its chunks (incremental sync).
+
+        The item construction mirrors :meth:`_iter_vector_source_items` exactly
+        (same ids, same ``source_node``/``parent_source_node`` semantics), so
+        anything this method indexes is indistinguishable from a full
+        :meth:`rebuild_vector_index` pass — and anything it *fails* to index
+        stays visible as ``missing``/``stale`` backlog in :meth:`index_status`,
+        where a later rebuild picks it up.
+
+        Never raises: embedding-provider or storage failures are reported as
+        ``{"status": "failed", ...}`` so ingestion callers can degrade instead
+        of losing an already-persisted write.
+        """
+        node_id = str(node_id or "").strip()
+        started = time.perf_counter()
+        summary: Dict[str, Any] = {
+            "node_id": node_id,
+            "items_total": 0,
+            "items_indexed": 0,
+            "items_skipped": 0,
+        }
+        if not node_id:
+            return {**summary, "status": "skipped", "detail": "node_id required"}
+        try:
+            with self._connect() as conn:
+                row = conn.execute(
+                    "SELECT id, type, title, summary, metadata_json FROM nodes WHERE id=?",
+                    (node_id,),
+                ).fetchone()
+                if row is None:
+                    return {**summary, "status": "skipped", "detail": "node not found"}
+                items: List[Dict[str, Any]] = []
+                if row["type"] != "Chunk":
+                    metadata = _safe_loads(row["metadata_json"])
+                    text = self._vector_text_for_node(
+                        title=row["title"],
+                        summary=row["summary"] or "",
+                        metadata=metadata,
+                    )
+                    if text:
+                        items.append(
+                            {
+                                "item_id": row["id"],
+                                "item_type": "node",
+                                "source_node": row["id"],
+                                "text": text,
+                                "metadata": {"node_type": row["type"], **metadata},
+                            }
+                        )
+                for chunk_row in conn.execute(
+                    """
+                        SELECT c.id, c.source_node AS parent_source_node, c.text, c.metadata_json
+                        FROM chunks c
+                        JOIN nodes n ON n.id=c.id
+                        WHERE c.source_node=?
+                        ORDER BY c.created_at ASC, c.id ASC
+                        """,
+                    (node_id,),
+                ).fetchall():
+                    metadata = _safe_loads(chunk_row["metadata_json"])
+                    text = _clean_text(chunk_row["text"] or "")
+                    if text:
+                        items.append(
+                            {
+                                "item_id": chunk_row["id"],
+                                "item_type": "chunk",
+                                "source_node": chunk_row["id"],
+                                "text": text,
+                                "metadata": {
+                                    **metadata,
+                                    "parent_source_node": chunk_row["parent_source_node"],
+                                },
+                            }
+                        )
+                indexed = skipped = 0
+                for item in items:
+                    if self._upsert_vector_item(conn, **item):
+                        indexed += 1
+                    else:
+                        skipped += 1
+                if indexed and self._embedder_fingerprint_record(conn) is None:
+                    # First successful vector write establishes the fingerprint;
+                    # later incremental writes never overwrite it (only a full
+                    # rebuild may flip it after an embedder swap).
+                    self._write_embedder_fingerprint(conn)
+            summary.update(
+                {
+                    "items_total": len(items),
+                    "items_indexed": indexed,
+                    "items_skipped": skipped,
+                }
+            )
+            return {
+                **summary,
+                "status": "indexed" if indexed else "noop",
+                "duration_ms": round((time.perf_counter() - started) * 1000, 2),
+                "embedding_model": self._embedding_model.model_id,
+            }
+        except Exception as exc:  # noqa: BLE001 — incremental sync must never raise
+            return {
+                **summary,
+                "status": "failed",
+                "detail": str(exc),
+                "duration_ms": round((time.perf_counter() - started) * 1000, 2),
+            }
+
+    def rebuild_vector_index(
+        self,
+        *,
+        full: bool = False,
+        include_nodes: bool = True,
+        include_chunks: bool = True,
+    ) -> Dict[str, Any]:
+        """Rebuild the derived vector index without mutating graph content."""
+        op_id = f"vector-op:{_sha256_text(f'{time.time()}:{os.getpid()}')[:24]}"
+        requested_at = _now()
+        started = time.perf_counter()
+        try:
+            with self._connect() as conn:
+                conn.execute(
+                    """
+                        INSERT INTO vector_index_operations(
+                          id, operation, status, requested_at, started_at, metadata_json
+                        )
+                        VALUES (?, ?, 'running', ?, ?, ?)
+                        """,
+                    (
+                        op_id,
+                        "rebuild_full" if full else "rebuild_incremental",
+                        requested_at,
+                        requested_at,
+                        _json(
+                            {
+                                "include_nodes": include_nodes,
+                                "include_chunks": include_chunks,
+                            }
+                        ),
+                    ),
+                )
+                if full:
+                    filters = []
+                    if include_nodes:
+                        filters.append("'node'")
+                    if include_chunks:
+                        filters.append("'chunk'")
+                    if filters:
+                        conn.execute(
+                            f"DELETE FROM vector_embeddings WHERE item_type IN ({','.join(filters)})"
+                        )
+                # After a full wipe nothing is current by definition, so the
+                # prefetch would only be a wasted scan of a table we just
+                # emptied. Incremental is where it pays: it turns "one SELECT
+                # per item, nearly all of which say unchanged" into one query.
+                known = {} if full else self._vector_text_hashes(conn)
+                total = indexed = skipped = 0
+                for item in self._iter_vector_source_items(
+                    conn,
+                    include_nodes=include_nodes,
+                    include_chunks=include_chunks,
+                ):
+                    total += 1
+                    if known.get(item["item_id"]) == _sha256_text(_clean_text(item["text"])):
+                        skipped += 1
+                        continue
+                    if self._upsert_vector_item(conn, **item):
+                        indexed += 1
+                    else:
+                        skipped += 1
+                duration_ms = round((time.perf_counter() - started) * 1000, 2)
+                conn.execute(
+                    """
+                        UPDATE vector_index_operations
+                        SET status='completed', completed_at=?, items_total=?,
+                            items_indexed=?, items_skipped=?, metadata_json=?
+                        WHERE id=?
+                        """,
+                    (
+                        _now(),
+                        total,
+                        indexed,
+                        skipped,
+                        _json(
+                            {
+                                "include_nodes": include_nodes,
+                                "include_chunks": include_chunks,
+                                "duration_ms": duration_ms,
+                                "embedding_model": self._embedding_model.model_id,
+                                "embedding_dim": self._embedding_model.dim,
+                            }
+                        ),
+                        op_id,
+                    ),
+                )
+                # A successful rebuild (re)establishes which embedder built
+                # the index — this is the only path that may flip a recorded
+                # fingerprint after an embedder swap.
+                self._write_embedder_fingerprint(conn)
+            return {
+                "status": "completed",
+                "operation_id": op_id,
+                "full": bool(full),
+                "items_total": total,
+                "items_indexed": indexed,
+                "items_skipped": skipped,
+                "duration_ms": duration_ms,
+                "embedding_model": self._embedding_model.model_id,
+                "embedding_dim": self._embedding_model.dim,
+            }
+        except Exception as exc:
+            duration_ms = round((time.perf_counter() - started) * 1000, 2)
+            with self._connect() as conn:
+                conn.execute(
+                    """
+                        INSERT INTO vector_index_operations(
+                          id, operation, status, requested_at, started_at, completed_at,
+                          error_message, metadata_json
+                        )
+                        VALUES (?, ?, 'failed', ?, ?, ?, ?, ?)
+                        ON CONFLICT(id) DO UPDATE SET
+                          status='failed',
+                          completed_at=excluded.completed_at,
+                          error_message=excluded.error_message,
+                          metadata_json=excluded.metadata_json
+                        """,
+                    (
+                        op_id,
+                        "rebuild_full" if full else "rebuild_incremental",
+                        requested_at,
+                        requested_at,
+                        _now(),
+                        str(exc),
+                        _json({"duration_ms": duration_ms}),
+                    ),
+                )
+            raise