ltcai 9.9.2 → 9.9.4

package/lattice_brain/graph/retrieval_policy.py

@@ -0,0 +1,174 @@
+"""Single retrieval policy consulted by both hybrid fusion layers.
+
+Review 2026-07-25 (UX/harness/loop/KG-RAG), Wave 0.2 — "RetrievalPolicy
+단일화" + "query rewrite" + "recency age decay". Before this module the two
+hybrid entry points (``KnowledgeGraphStore.hybrid_search`` two-channel
+alpha fusion and ``SearchService.hybrid_search`` three-channel weight
+fusion) each resolved :func:`lattice_brain.graph.fusion.fusion_profile`
+independently and had no shared place to hang retrieval-wide behavior.
+This module is that single place:
+
+* :func:`rewrite_query` — deterministic, rule-based ko/en normalization
+  (NO LLM). Strips a conservative fixed list of leading/trailing politeness
+  fillers ("~좀 알려줘", "궁금해", "뭐였지", "what is", "please", …) and
+  collapses repeated whitespace. Code-class queries pass through untouched
+  (except whitespace collapse) because exact identifiers/filenames ARE the
+  retrieval signal. ``LATTICEAI_QUERY_REWRITE=0`` disables rewriting.
+* :func:`resolve_policy` — composes the rewrite with
+  :func:`~lattice_brain.graph.fusion.fusion_profile` (the weight table is
+  NOT duplicated here) and adds ``recency_half_life_days``, which is
+  non-``None`` ONLY for the ``recency`` query class — the honest signal
+  that age decay applies exactly where the fusion layers wire it.
+
+Both functions are pure, deterministic, and never raise; the ``fact``
+query class resolves to the historical default weights, so fact-class
+behavior stays byte-compatible. Scores are never combined with ``or``
+(score ``0.0`` is falsy but valid).
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any, Dict, List, Mapping, Optional
+
+from .fusion import classify_query, fusion_profile
+
+QUERY_REWRITE_ENV = "LATTICEAI_QUERY_REWRITE"
+
+# Half-life (days) for the recency-class age decay. Matches the default of
+# lattice_brain.graph._kg_fsutil._recency_score, which the fusion layers use
+# to compute the actual multiplier — this module only declares the policy.
+RECENCY_HALF_LIFE_DAYS = 14.0
+
+# A filler strip is applied only when the remaining query keeps at least this
+# many characters — never rewrite a query down to (near-)nothing.
+_MIN_REMAINDER_CHARS = 4
+
+_WS_RE = re.compile(r"\s+")
+
+# Conservative trailing Korean politeness/filler phrases ("… 좀 알려줘",
+# "… 궁금해", "… 뭐였지"). Anchored at end-of-string, must start the query or
+# follow whitespace so word interiors are never clipped.
+_KO_FILLER_TAIL_RE = re.compile(
+    r"(?:^|(?<=\s))(?:좀\s+)?"
+    r"(?:알려\s*줘요?|알려\s*주세요|알려\s*줄래요?"
+    r"|말해\s*줘요?|말해\s*주세요"
+    r"|설명해\s*줘요?|설명해\s*주세요"
+    r"|궁금해요?|궁금합니다"
+    r"|뭐였지|뭐였더라|뭐더라|뭐지|뭐야"
+    r")\s*[?!.…~]*\s*$"
+)
+
+# Conservative leading English filler ("what is …", "tell me about …",
+# optionally prefixed with "please").
+_EN_FILLER_LEAD_RE = re.compile(
+    r"^(?:please\s+)?(?:tell\s+me\s+about|what\s+is|what\s+was|what\s+are|what's)\s+",
+    re.IGNORECASE,
+)
+
+# Conservative trailing English politeness ("…, please?").
+_EN_FILLER_TAIL_RE = re.compile(r"[,\s]*\bplease\b\s*[?!.]*\s*$", re.IGNORECASE)
+
+_FILLER_RULES = (
+    ("strip_filler_ko", _KO_FILLER_TAIL_RE),
+    ("strip_filler_en_leading", _EN_FILLER_LEAD_RE),
+    ("strip_filler_en_trailing", _EN_FILLER_TAIL_RE),
+)
+
+
+def _rewrite_enabled() -> bool:
+    raw = os.getenv(QUERY_REWRITE_ENV, "").strip().lower()
+    return raw not in {"0", "false", "no", "off"}
+
+
+def rewrite_query(query: Any) -> Dict[str, Any]:
+    """Deterministic rule-based ko/en query normalization (never raises).
+
+    Returns ``{"original": str, "rewritten": str, "rules": [rule names]}``.
+    Only three conservative rules exist: whitespace collapse, trailing
+    Korean filler strip, and leading/trailing English filler strip — each
+    filler strip applies only when the remainder keeps >= 4 characters.
+    Code-class queries (exact identifiers/filenames are the signal) are
+    returned unchanged apart from whitespace collapse. The
+    ``LATTICEAI_QUERY_REWRITE=0`` kill-switch disables rewriting entirely
+    (``rewritten == original``). Empty input yields empty output.
+    """
+    try:
+        text = str(query or "").strip()
+    except Exception:  # noqa: BLE001 — rewrite must never raise
+        return {"original": "", "rewritten": "", "rules": []}
+    if not text:
+        return {"original": "", "rewritten": "", "rules": []}
+    original = text
+    if not _rewrite_enabled():
+        return {"original": original, "rewritten": original, "rules": []}
+
+    rules: List[str] = []
+    rewritten = _WS_RE.sub(" ", text).strip()
+    if rewritten != original:
+        rules.append("collapse_whitespace")
+
+    if classify_query(original) == "code":
+        # Exact identifiers/filenames are the retrieval signal — never touch
+        # them (whitespace collapse is the only permitted normalization).
+        return {"original": original, "rewritten": rewritten, "rules": rules}
+
+    for name, pattern in _FILLER_RULES:
+        candidate = pattern.sub("", rewritten).strip()
+        if candidate != rewritten and len(candidate) >= _MIN_REMAINDER_CHARS:
+            rewritten = candidate
+            rules.append(name)
+    return {"original": original, "rewritten": rewritten, "rules": rules}
+
+
+def resolve_policy(
+    query: Any,
+    *,
+    overrides: Optional[Mapping[str, Mapping[str, float]]] = None,
+) -> Dict[str, Any]:
+    """Resolve the full retrieval policy for ``query`` (never raises).
+
+    Composes :func:`rewrite_query` with
+    :func:`~lattice_brain.graph.fusion.fusion_profile` (classification runs
+    on the ORIGINAL query, so ``query_class`` / ``weights`` / ``alpha`` are
+    byte-identical to calling ``fusion_profile`` directly, as both hybrid
+    layers did before this module). Returns::
+
+        {
+          "query_class": "fact" | "code" | "person" | "recency",
+          "weights": {"keyword", "vector", "graph"},   # service fusion
+          "alpha": float,                              # graph-layer fusion
+          "original_query": str,
+          "search_query": str,       # the rewritten form to search with
+          "rewrite_rules": [str],
+          "recency_half_life_days": float | None,      # 14.0 only for recency
+        }
+
+    ``recency_half_life_days`` is non-``None`` only for the ``recency``
+    class — the honest contract that age decay applies exactly where the
+    fusion layers wire it, and nowhere else.
+    """
+    rewrite = rewrite_query(query)
+    profile = fusion_profile(rewrite["original"], overrides=overrides)
+    query_class = profile["query_class"]
+    search_query = rewrite["rewritten"] if rewrite["rewritten"] else rewrite["original"]
+    return {
+        "query_class": query_class,
+        "weights": dict(profile["weights"]),
+        "alpha": float(profile["alpha"]),
+        "original_query": rewrite["original"],
+        "search_query": search_query,
+        "rewrite_rules": list(rewrite["rules"]),
+        "recency_half_life_days": (
+            RECENCY_HALF_LIFE_DAYS if query_class == "recency" else None
+        ),
+    }
+
+
+__all__ = [
+    "QUERY_REWRITE_ENV",
+    "RECENCY_HALF_LIFE_DAYS",
+    "resolve_policy",
+    "rewrite_query",
+]

package/lattice_brain/graph/retrieval_vector.py

@@ -13,6 +13,78 @@ class KnowledgeGraphVectorMixin:
     retrieval/write helpers (e.g. self._vector_text_for_node) through the MRO.
     """
 
+    # ── embedder fingerprint (review Wave 2.2 — stale_embedder) ──────────────
+    # vector_search filters on the CURRENT model/dim, so swapping the embedder
+    # silently yields zero vector rows. The fingerprint persisted in graph_meta
+    # records which embedder actually built the index; a mismatch is surfaced
+    # as the honest ``stale_embedder`` signal instead of a silent degradation.
+
+    _EMBEDDER_FINGERPRINT_KEY = "embedder_fingerprint"
+
+    def _embedder_fingerprint_record(
+        self, conn: sqlite3.Connection
+    ) -> Optional[Dict[str, Any]]:
+        """Read the recorded embedder fingerprint from graph_meta (or None)."""
+        row = conn.execute(
+            "SELECT value FROM graph_meta WHERE key=?",
+            (self._EMBEDDER_FINGERPRINT_KEY,),
+        ).fetchone()
+        if not row:
+            return None
+        payload = _safe_loads(row["value"])
+        if not isinstance(payload, dict) or not payload.get("model_id"):
+            return None
+        try:
+            dim = int(payload.get("dim") or 0)
+        except (TypeError, ValueError):
+            dim = 0
+        return {"model_id": str(payload["model_id"]), "dim": dim}
+
+    def _write_embedder_fingerprint(self, conn: sqlite3.Connection) -> Dict[str, Any]:
+        """Persist the CURRENT embedder identity (same transaction as caller)."""
+        fingerprint = {
+            "model_id": self._embedding_model.model_id,
+            "dim": int(self._embedding_model.dim),
+        }
+        conn.execute(
+            "INSERT OR REPLACE INTO graph_meta(key, value) VALUES (?, ?)",
+            (self._EMBEDDER_FINGERPRINT_KEY, _json(fingerprint)),
+        )
+        return fingerprint
+
+    def record_embedder_fingerprint(self) -> Dict[str, Any]:
+        """Record the current embedder (model_id + dim) as the index builder."""
+        with self._connect() as conn:
+            return self._write_embedder_fingerprint(conn)
+
+    def embedder_fingerprint_status(self) -> Dict[str, Any]:
+        """Compare the current embedder against the recorded index fingerprint.
+
+        Returns ``{"current": {model_id, dim}, "recorded": {...} | None,
+        "stale_embedder": bool}``. ``stale_embedder`` is True only when a
+        fingerprint was recorded AND it differs from the current embedder —
+        an unrecorded index (legacy DBs, nothing indexed yet) is honestly
+        "unknown", never reported stale. Never raises.
+        """
+        current = {
+            "model_id": self._embedding_model.model_id,
+            "dim": int(self._embedding_model.dim),
+        }
+        recorded: Optional[Dict[str, Any]] = None
+        try:
+            with self._connect() as conn:
+                recorded = self._embedder_fingerprint_record(conn)
+        except Exception:  # noqa: BLE001 — status must degrade, never raise
+            recorded = None
+        stale = bool(
+            recorded is not None
+            and (
+                recorded.get("model_id") != current["model_id"]
+                or recorded.get("dim") != current["dim"]
+            )
+        )
+        return {"current": current, "recorded": recorded, "stale_embedder": stale}
+
     def _iter_vector_source_items(
         self,
         conn: sqlite3.Connection,
@@ -153,6 +225,11 @@ class KnowledgeGraphVectorMixin:
                         indexed += 1
                     else:
                         skipped += 1
+                if indexed and self._embedder_fingerprint_record(conn) is None:
+                    # First successful vector write establishes the fingerprint;
+                    # later incremental writes never overwrite it (only a full
+                    # rebuild may flip it after an embedder swap).
+                    self._write_embedder_fingerprint(conn)
             summary.update(
                 {
                     "items_total": len(items),
@@ -254,6 +331,10 @@ class KnowledgeGraphVectorMixin:
                         op_id,
                     ),
                 )
+                # A successful rebuild (re)establishes which embedder built
+                # the index — this is the only path that may flip a recorded
+                # fingerprint after an embedder swap.
+                self._write_embedder_fingerprint(conn)
             return {
                 "status": "completed",
                 "operation_id": op_id,
@@ -401,8 +482,10 @@ class KnowledgeGraphVectorMixin:
                         "within_target": float(duration_ms) <= 10_000,
                     }
                 )
+        embedder_status = self.embedder_fingerprint_status()
         return {
             "status": "ready" if pending == 0 else "needs_reindex",
+            "embedder": embedder_status,
             "storage": {
                 "db_path": str(self.db_path),
                 "backend": "sqlite",
@@ -449,7 +532,11 @@ class KnowledgeGraphVectorMixin:
                 "backlog_reasons": backlog_reasons,
                 "backlog_samples": backlog_samples,
                 "incremental_reindex_recommended": pending > 0,
-                "full_rebuild_recommended": orphaned_items > 0,
+                # A stale embedder means every old-model row must be re-embedded;
+                # only a full rebuild (which re-records the fingerprint) heals it.
+                "full_rebuild_recommended": bool(
+                    orphaned_items > 0 or embedder_status["stale_embedder"]
+                ),
                 "latency_budget": latency_budget,
             },
             "operations": [
@@ -475,7 +562,11 @@ class KnowledgeGraphVectorMixin:
 
         Reduces :meth:`index_status` (``pending = missing + stale``) to the
         fixed contract ``{"status", "pending_items", "total_items", "detail"}``
-        with ``status`` in ``ready`` / ``pending`` / ``unavailable``.
+        with ``status`` in ``ready`` / ``pending`` / ``stale_embedder`` /
+        ``unavailable``. ``stale_embedder`` (review Wave 2.2) is reported only
+        when the recorded embedder fingerprint differs from the current
+        embedder AND rows indexed under the old model still exist — the index
+        needs a full rebuild, not an incremental sync.
 
         Never raises: environments where the embedding provider or index
         storage cannot be used report ``"unavailable"`` with the cause in
@@ -492,6 +583,35 @@ class KnowledgeGraphVectorMixin:
             }
         pending = int(status.get("pending_items") or 0)
         total = int(status.get("source_items") or 0)
+        embedder = status.get("embedder") or {}
+        if embedder.get("stale_embedder"):
+            old_model_rows = 0
+            try:
+                with self._connect() as conn:
+                    old_model_rows = int(
+                        conn.execute(
+                            "SELECT COUNT(*) AS c FROM vector_embeddings "
+                            "WHERE embedding_model<>? OR embedding_dim<>?",
+                            (
+                                self._embedding_model.model_id,
+                                int(self._embedding_model.dim),
+                            ),
+                        ).fetchone()["c"]
+                    )
+            except Exception:  # noqa: BLE001 — keep the existing statuses on failure
+                old_model_rows = 0
+            if old_model_rows > 0:
+                recorded = embedder.get("recorded") or {}
+                return {
+                    "status": "stale_embedder",
+                    "pending_items": pending,
+                    "total_items": total,
+                    "detail": (
+                        f"embedding model changed ({recorded.get('model_id')} → "
+                        f"{self._embedding_model.model_id}); {old_model_rows} indexed "
+                        "rows still use the previous model — run a full vector index rebuild"
+                    ),
+                }
         if pending > 0:
             return {
                 "status": "pending",

package/lattice_brain/ingestion.py

@@ -245,6 +245,72 @@ def assess_extraction_quality(
     return {"score": round(score, 4), "level": _quality_level(score), "reasons": reasons}
 
 
+# ── capture quality CTA (backlog #9, review §7.2 C) ──────────────────────────
+# Structured verdict over the same extraction-quality schema the rest of the
+# pipeline uses, so capture surfaces (browser extension, read-url) can render
+# an honest "this capture is thin" CTA instead of silently storing junk.
+CAPTURE_SUGGESTIONS_THIN = ["recapture", "paste_manually", "highlight_source"]
+_CAPTURE_REASON_LABELS = {
+    "empty_text": "추출된 본문이 비어 있습니다",
+    "very_short_text": "추출된 본문이 매우 짧습니다",
+    "short_text": "추출된 본문이 짧습니다",
+    "no_sentence_structure": "문장 구조가 거의 없습니다",
+    "low_character_diversity": "반복 문자가 대부분입니다",
+    "repetitive_lines": "같은 줄이 반복됩니다",
+    "repetitive_words": "같은 단어가 반복됩니다",
+    "high_whitespace_ratio": "공백이 지나치게 많습니다",
+    "fragmented_lines": "줄이 잘게 조각나 있습니다",
+    "nav_menu_remnants": "메뉴/내비게이션 잔여물이 많습니다",
+    "boilerplate_markers": "상용구 텍스트가 많습니다",
+    "no_extracted_text": "추출된 텍스트가 없습니다",
+}
+
+
+def capture_quality_verdict(
+    extraction_quality: Optional[Dict[str, Any]],
+    *,
+    source_type: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Structured CTA verdict from a pipeline ``extraction_quality`` dict.
+
+    ``{"status": "thin"|"ok", "reason": str|None, "suggestions": [...],
+    "score": float|None, "level": str|None}``. ``thin`` (level == "low", the
+    same threshold as the ingest warning) carries actionable suggestions —
+    ``recapture`` / ``paste_manually`` / ``highlight_source`` — so the UI can
+    offer the user a way to fix the capture instead of hiding the problem.
+    Deterministic and never raises; ``None`` input yields an honest ``thin``.
+    """
+    if not isinstance(extraction_quality, dict):
+        return {
+            "status": "thin",
+            "reason": _CAPTURE_REASON_LABELS["no_extracted_text"],
+            "reason_codes": ["no_extracted_text"],
+            "suggestions": list(CAPTURE_SUGGESTIONS_THIN),
+            "score": None,
+            "level": None,
+        }
+    level = str(extraction_quality.get("level") or "")
+    score = extraction_quality.get("score")
+    reasons = [str(item) for item in (extraction_quality.get("reasons") or [])]
+    thin = level == "low"
+    reason = None
+    if thin:
+        labeled = [
+            _CAPTURE_REASON_LABELS[code]
+            for code in reasons
+            if code in _CAPTURE_REASON_LABELS
+        ]
+        reason = "; ".join(labeled) if labeled else QUALITY_LOW_WARNING
+    return {
+        "status": "thin" if thin else "ok",
+        "reason": reason,
+        "reason_codes": reasons if thin else [],
+        "suggestions": list(CAPTURE_SUGGESTIONS_THIN) if thin else [],
+        "score": score,
+        "level": level or None,
+    }
+
+
 def _load_latticeignore(root: Path) -> List[str]:
     """Parse ``root/.latticeignore`` → glob patterns (gitignore-like subset)."""
     ignore_file = root / LATTICEIGNORE_FILENAME

package/lattice_brain/portability.py

@@ -14,6 +14,7 @@ any cloud service. Two complementary mechanisms, both fully local:
 from __future__ import annotations
 
 import json
+import logging
 import os
 import shutil
 import sqlite3
@@ -74,9 +75,12 @@ def _checkpoint_sqlite(db_path: Path) -> None:
 
 
 def _restore_sibling(path: Path, backup: Path) -> None:
-    if backup.exists():
+    try:
         shutil.copy2(backup, path)
-    else:
+    except FileNotFoundError:
+        # The backup vanished (or never existed — transient -wal/-shm): the
+        # honest reconstruction is "no such sibling", never a crash that
+        # masks the error that triggered the rollback.
         path.unlink(missing_ok=True)
 
 
@@ -101,10 +105,22 @@ def _replace_sqlite_atomically(src: Path, dest: Path, backup_dir: Path) -> None:
             sibling.unlink(missing_ok=True)
         os.replace(tmp, dest)
     except Exception:
-        if tmp.exists():
-            tmp.unlink()
+        # Recovery I/O must never replace the swap error being reported —
+        # an exception raised here would mask it (the CI-observed
+        # "[Errno 2]" over the real failure). Best-effort restore, then
+        # always re-raise the original.
+        try:
+            tmp.unlink(missing_ok=True)
+        except OSError as cleanup_exc:
+            logging.warning("restore tmp cleanup failed: %s", cleanup_exc)
         for sibling in _sqlite_siblings(dest):
-            _restore_sibling(sibling, backups.get(sibling, backup_dir / sibling.name))
+            try:
+                _restore_sibling(sibling, backups.get(sibling, backup_dir / sibling.name))
+            except OSError as rollback_exc:
+                logging.warning(
+                    "restore sibling rollback incomplete for %s: %s",
+                    sibling, rollback_exc,
+                )
         raise
 
 
@@ -127,12 +143,17 @@ def _replace_tree_with_backup(src: Optional[Path], dest: Path, backup_dir: Path)
             shutil.rmtree(dest)
         os.replace(staged, dest)
     except Exception:
-        if staged.exists():
-            shutil.rmtree(staged)
-        if dest.exists():
-            shutil.rmtree(dest)
-        if backup.exists():
-            shutil.copytree(backup, dest)
+        # Same masking guard as the sqlite swap: rollback I/O is
+        # best-effort and the original failure always propagates.
+        try:
+            if staged.exists():
+                shutil.rmtree(staged)
+            if dest.exists():
+                shutil.rmtree(dest)
+            if backup.exists():
+                shutil.copytree(backup, dest)
+        except OSError as rollback_exc:
+            logging.warning("blob tree rollback incomplete: %s", rollback_exc)
         raise
 
 
@@ -307,7 +328,15 @@ class KGPortabilityService:
                     blob_src = tmp / "blobs"
                     _replace_tree_with_backup(blob_src if blob_src.exists() else None, blob_dest, backup_dir)
                 except Exception:
-                    _rollback_sqlite_from_backup(db_dest, backup_dir)
+                    # The rollback is best-effort recovery; an I/O slip inside
+                    # it must never mask the restore failure being reported.
+                    try:
+                        _rollback_sqlite_from_backup(db_dest, backup_dir)
+                    except OSError as rollback_exc:
+                        logging.warning(
+                            "pre-restore rollback incomplete (backup kept at %s): %s",
+                            backup_dir, rollback_exc,
+                        )
                     raise
         stats = self._kg.stats()
         return {

package/lattice_brain/runtime/multi_agent.py

@@ -48,7 +48,7 @@ from .contracts import multi_agent_contract
 from ..utils import now_iso as _now
 
 
-MULTI_AGENT_VERSION = "9.9.2"
+MULTI_AGENT_VERSION = "9.9.4"
 
 AGENT_ROLES = ("researcher", "planner", "executor", "reviewer", "release")
 CORE_PIPELINE = ("planner", "executor", "reviewer")

package/latticeai/__init__.py

@@ -1,3 +1,3 @@
 """Lattice AI - modular server package."""
 
-__version__ = "9.9.2"
+__version__ = "9.9.4"