ltcai 9.9.3 → 9.9.5

package/lattice_brain/graph/retrieval_vector.py

@@ -13,6 +13,78 @@ class KnowledgeGraphVectorMixin:
     retrieval/write helpers (e.g. self._vector_text_for_node) through the MRO.
     """
 
+    # ── embedder fingerprint (review Wave 2.2 — stale_embedder) ──────────────
+    # vector_search filters on the CURRENT model/dim, so swapping the embedder
+    # silently yields zero vector rows. The fingerprint persisted in graph_meta
+    # records which embedder actually built the index; a mismatch is surfaced
+    # as the honest ``stale_embedder`` signal instead of a silent degradation.
+
+    _EMBEDDER_FINGERPRINT_KEY = "embedder_fingerprint"
+
+    def _embedder_fingerprint_record(
+        self, conn: sqlite3.Connection
+    ) -> Optional[Dict[str, Any]]:
+        """Read the recorded embedder fingerprint from graph_meta (or None)."""
+        row = conn.execute(
+            "SELECT value FROM graph_meta WHERE key=?",
+            (self._EMBEDDER_FINGERPRINT_KEY,),
+        ).fetchone()
+        if not row:
+            return None
+        payload = _safe_loads(row["value"])
+        if not isinstance(payload, dict) or not payload.get("model_id"):
+            return None
+        try:
+            dim = int(payload.get("dim") or 0)
+        except (TypeError, ValueError):
+            dim = 0
+        return {"model_id": str(payload["model_id"]), "dim": dim}
+
+    def _write_embedder_fingerprint(self, conn: sqlite3.Connection) -> Dict[str, Any]:
+        """Persist the CURRENT embedder identity (same transaction as caller)."""
+        fingerprint = {
+            "model_id": self._embedding_model.model_id,
+            "dim": int(self._embedding_model.dim),
+        }
+        conn.execute(
+            "INSERT OR REPLACE INTO graph_meta(key, value) VALUES (?, ?)",
+            (self._EMBEDDER_FINGERPRINT_KEY, _json(fingerprint)),
+        )
+        return fingerprint
+
+    def record_embedder_fingerprint(self) -> Dict[str, Any]:
+        """Record the current embedder (model_id + dim) as the index builder."""
+        with self._connect() as conn:
+            return self._write_embedder_fingerprint(conn)
+
+    def embedder_fingerprint_status(self) -> Dict[str, Any]:
+        """Compare the current embedder against the recorded index fingerprint.
+
+        Returns ``{"current": {model_id, dim}, "recorded": {...} | None,
+        "stale_embedder": bool}``. ``stale_embedder`` is True only when a
+        fingerprint was recorded AND it differs from the current embedder —
+        an unrecorded index (legacy DBs, nothing indexed yet) is honestly
+        "unknown", never reported stale. Never raises.
+        """
+        current = {
+            "model_id": self._embedding_model.model_id,
+            "dim": int(self._embedding_model.dim),
+        }
+        recorded: Optional[Dict[str, Any]] = None
+        try:
+            with self._connect() as conn:
+                recorded = self._embedder_fingerprint_record(conn)
+        except Exception:  # noqa: BLE001 — status must degrade, never raise
+            recorded = None
+        stale = bool(
+            recorded is not None
+            and (
+                recorded.get("model_id") != current["model_id"]
+                or recorded.get("dim") != current["dim"]
+            )
+        )
+        return {"current": current, "recorded": recorded, "stale_embedder": stale}
+
     def _iter_vector_source_items(
         self,
         conn: sqlite3.Connection,
@@ -153,6 +225,11 @@ class KnowledgeGraphVectorMixin:
                         indexed += 1
                     else:
                         skipped += 1
+                if indexed and self._embedder_fingerprint_record(conn) is None:
+                    # First successful vector write establishes the fingerprint;
+                    # later incremental writes never overwrite it (only a full
+                    # rebuild may flip it after an embedder swap).
+                    self._write_embedder_fingerprint(conn)
             summary.update(
                 {
                     "items_total": len(items),
@@ -254,6 +331,10 @@ class KnowledgeGraphVectorMixin:
                         op_id,
                     ),
                 )
+                # A successful rebuild (re)establishes which embedder built
+                # the index — this is the only path that may flip a recorded
+                # fingerprint after an embedder swap.
+                self._write_embedder_fingerprint(conn)
             return {
                 "status": "completed",
                 "operation_id": op_id,
@@ -401,8 +482,10 @@ class KnowledgeGraphVectorMixin:
                         "within_target": float(duration_ms) <= 10_000,
                     }
                 )
+        embedder_status = self.embedder_fingerprint_status()
         return {
             "status": "ready" if pending == 0 else "needs_reindex",
+            "embedder": embedder_status,
             "storage": {
                 "db_path": str(self.db_path),
                 "backend": "sqlite",
@@ -449,7 +532,11 @@ class KnowledgeGraphVectorMixin:
                 "backlog_reasons": backlog_reasons,
                 "backlog_samples": backlog_samples,
                 "incremental_reindex_recommended": pending > 0,
-                "full_rebuild_recommended": orphaned_items > 0,
+                # A stale embedder means every old-model row must be re-embedded;
+                # only a full rebuild (which re-records the fingerprint) heals it.
+                "full_rebuild_recommended": bool(
+                    orphaned_items > 0 or embedder_status["stale_embedder"]
+                ),
                 "latency_budget": latency_budget,
             },
             "operations": [
@@ -475,7 +562,11 @@ class KnowledgeGraphVectorMixin:
 
         Reduces :meth:`index_status` (``pending = missing + stale``) to the
         fixed contract ``{"status", "pending_items", "total_items", "detail"}``
-        with ``status`` in ``ready`` / ``pending`` / ``unavailable``.
+        with ``status`` in ``ready`` / ``pending`` / ``stale_embedder`` /
+        ``unavailable``. ``stale_embedder`` (review Wave 2.2) is reported only
+        when the recorded embedder fingerprint differs from the current
+        embedder AND rows indexed under the old model still exist — the index
+        needs a full rebuild, not an incremental sync.
 
         Never raises: environments where the embedding provider or index
         storage cannot be used report ``"unavailable"`` with the cause in
@@ -492,6 +583,35 @@ class KnowledgeGraphVectorMixin:
             }
         pending = int(status.get("pending_items") or 0)
         total = int(status.get("source_items") or 0)
+        embedder = status.get("embedder") or {}
+        if embedder.get("stale_embedder"):
+            old_model_rows = 0
+            try:
+                with self._connect() as conn:
+                    old_model_rows = int(
+                        conn.execute(
+                            "SELECT COUNT(*) AS c FROM vector_embeddings "
+                            "WHERE embedding_model<>? OR embedding_dim<>?",
+                            (
+                                self._embedding_model.model_id,
+                                int(self._embedding_model.dim),
+                            ),
+                        ).fetchone()["c"]
+                    )
+            except Exception:  # noqa: BLE001 — keep the existing statuses on failure
+                old_model_rows = 0
+            if old_model_rows > 0:
+                recorded = embedder.get("recorded") or {}
+                return {
+                    "status": "stale_embedder",
+                    "pending_items": pending,
+                    "total_items": total,
+                    "detail": (
+                        f"embedding model changed ({recorded.get('model_id')} → "
+                        f"{self._embedding_model.model_id}); {old_model_rows} indexed "
+                        "rows still use the previous model — run a full vector index rebuild"
+                    ),
+                }
         if pending > 0:
             return {
                 "status": "pending",

package/lattice_brain/graph/store.py

@@ -11,6 +11,7 @@ from .projection import KnowledgeGraphProjectionMixin
 from .provenance import KnowledgeGraphProvenanceMixin
 from .retrieval import KnowledgeGraphRetrievalMixin
 from .retrieval_docgen import KnowledgeGraphDocGenMixin
+from .retrieval_reads import KnowledgeGraphReadsMixin
 from .retrieval_vector import KnowledgeGraphVectorMixin
 from .write_master import KnowledgeGraphWriteMixin
 
@@ -24,6 +25,7 @@ class KnowledgeGraphStore(
     KnowledgeGraphProvenanceMixin,
     KnowledgeGraphDocumentsMixin,
     KnowledgeGraphRetrievalMixin,
+    KnowledgeGraphReadsMixin,
     KnowledgeGraphVectorMixin,
     KnowledgeGraphDocGenMixin,
 ):

package/lattice_brain/portability.py

@@ -14,6 +14,7 @@ any cloud service. Two complementary mechanisms, both fully local:
 from __future__ import annotations
 
 import json
+import logging
 import os
 import shutil
 import sqlite3
@@ -74,9 +75,12 @@ def _checkpoint_sqlite(db_path: Path) -> None:
 
 
 def _restore_sibling(path: Path, backup: Path) -> None:
-    if backup.exists():
+    try:
         shutil.copy2(backup, path)
-    else:
+    except FileNotFoundError:
+        # The backup vanished (or never existed — transient -wal/-shm): the
+        # honest reconstruction is "no such sibling", never a crash that
+        # masks the error that triggered the rollback.
         path.unlink(missing_ok=True)
 
 
@@ -101,10 +105,22 @@ def _replace_sqlite_atomically(src: Path, dest: Path, backup_dir: Path) -> None:
             sibling.unlink(missing_ok=True)
         os.replace(tmp, dest)
     except Exception:
-        if tmp.exists():
-            tmp.unlink()
+        # Recovery I/O must never replace the swap error being reported —
+        # an exception raised here would mask it (the CI-observed
+        # "[Errno 2]" over the real failure). Best-effort restore, then
+        # always re-raise the original.
+        try:
+            tmp.unlink(missing_ok=True)
+        except OSError as cleanup_exc:
+            logging.warning("restore tmp cleanup failed: %s", cleanup_exc)
         for sibling in _sqlite_siblings(dest):
-            _restore_sibling(sibling, backups.get(sibling, backup_dir / sibling.name))
+            try:
+                _restore_sibling(sibling, backups.get(sibling, backup_dir / sibling.name))
+            except OSError as rollback_exc:
+                logging.warning(
+                    "restore sibling rollback incomplete for %s: %s",
+                    sibling, rollback_exc,
+                )
         raise
 
 
@@ -127,12 +143,17 @@ def _replace_tree_with_backup(src: Optional[Path], dest: Path, backup_dir: Path)
             shutil.rmtree(dest)
         os.replace(staged, dest)
     except Exception:
-        if staged.exists():
-            shutil.rmtree(staged)
-        if dest.exists():
-            shutil.rmtree(dest)
-        if backup.exists():
-            shutil.copytree(backup, dest)
+        # Same masking guard as the sqlite swap: rollback I/O is
+        # best-effort and the original failure always propagates.
+        try:
+            if staged.exists():
+                shutil.rmtree(staged)
+            if dest.exists():
+                shutil.rmtree(dest)
+            if backup.exists():
+                shutil.copytree(backup, dest)
+        except OSError as rollback_exc:
+            logging.warning("blob tree rollback incomplete: %s", rollback_exc)
         raise
 
 
@@ -307,7 +328,15 @@ class KGPortabilityService:
                     blob_src = tmp / "blobs"
                     _replace_tree_with_backup(blob_src if blob_src.exists() else None, blob_dest, backup_dir)
                 except Exception:
-                    _rollback_sqlite_from_backup(db_dest, backup_dir)
+                    # The rollback is best-effort recovery; an I/O slip inside
+                    # it must never mask the restore failure being reported.
+                    try:
+                        _rollback_sqlite_from_backup(db_dest, backup_dir)
+                    except OSError as rollback_exc:
+                        logging.warning(
+                            "pre-restore rollback incomplete (backup kept at %s): %s",
+                            backup_dir, rollback_exc,
+                        )
                     raise
         stats = self._kg.stats()
         return {

package/lattice_brain/quality.py

@@ -131,13 +131,33 @@ class HybridFusion:
         return sorted(results, key=lambda x: x["fused_score"], reverse=True)
 
 class RerankerInterface:
-    """Pluggable reranker interface"""
+    """Pluggable reranker interface.
+
+    Default is identity (fused score). When
+    ``LATTICEAI_CROSS_ENCODER_RERANK=1`` and a CrossEncoder is importable,
+    delegates to :func:`lattice_brain.graph.rerank.rerank_matches`. Failures
+    never raise — always returns a ranked list.
+    """
+
     def rerank(self, query: str, candidates: List[Dict], top_k: int = 5) -> List[Dict]:
-        # Default local-safe fallback: preserve fused ordering without making
-        # unearned cross-encoder claims.
-        for c in candidates:
-            c["rerank_score"] = c.get("fused_score", 0.0)
-        return sorted(candidates, key=lambda x: x.get("rerank_score", 0), reverse=True)[:top_k]
+        try:
+            from lattice_brain.graph.rerank import rerank_matches
+
+            # Map fused_score → score so the shared helper ranks consistently.
+            prepared = []
+            for c in candidates:
+                item = dict(c)
+                if "score" not in item:
+                    item["score"] = item.get("fused_score", 0.0)
+                prepared.append(item)
+            result = rerank_matches(query, prepared, top_k=top_k)
+            return list(result.get("matches") or [])
+        except Exception:  # noqa: BLE001 — quality path must never raise
+            for c in candidates:
+                c["rerank_score"] = c.get("fused_score", 0.0)
+            return sorted(
+                candidates, key=lambda x: x.get("rerank_score", 0), reverse=True
+            )[:top_k]
 
 # -----------------------------
 # 3. Memory Candidate Extraction / Scoring / Dedupe / Merge / Conflict / Retention

package/lattice_brain/runtime/multi_agent.py

@@ -48,7 +48,7 @@ from .contracts import multi_agent_contract
 from ..utils import now_iso as _now
 
 
-MULTI_AGENT_VERSION = "9.9.3"
+MULTI_AGENT_VERSION = "9.9.5"
 
 AGENT_ROLES = ("researcher", "planner", "executor", "reviewer", "release")
 CORE_PIPELINE = ("planner", "executor", "reviewer")

package/latticeai/__init__.py

@@ -1,3 +1,3 @@
 """Lattice AI - modular server package."""
 
-__version__ = "9.9.3"
+__version__ = "9.9.5"

package/latticeai/api/chat.py

@@ -45,6 +45,7 @@ from latticeai.api.chat_helpers import (
     workspace_scope_from_request,
 )
 from latticeai.api.chat_history import HistoryRouteDependencies, register_history_routes
+from latticeai.core.run_store import AgentRunStore
 from latticeai.api.chat_intents import ChatIntentController
 from latticeai.api.chat_stream import stream_chat
 from latticeai.services.app_context import AppContext
@@ -191,6 +192,11 @@ def create_chat_router(context: AppContext) -> APIRouter:
         agent_root=AGENT_ROOT,
         ensure_agent_root=ensure_agent_root,
         funnel_metrics=context.funnel_metrics,
+        run_store=(
+            AgentRunStore(Path(context.data_dir) / "agent_runs")
+            if context.data_dir
+            else None
+        ),
     )
     intent_controller = ChatIntentController(
         model_router=model_router,