ltcai 9.9.3 → 9.9.5

package/lattice_brain/graph/projection.py

@@ -4,6 +4,22 @@ from __future__ import annotations
 
 from ._kg_common import *  # noqa: F403,F401
 
+# ── promotion review mode (review 2026-07-25 Wave 4) ─────────────────────────
+# When enabled, curate() parks would-be Topic promotions in graph_meta for a
+# human decision instead of writing them immediately. Explicit review_mode=
+# argument wins; otherwise this env opt-in decides; default stays auto-promote.
+_PROMOTION_REVIEW_ENV = "LATTICEAI_GRAPH_PROMOTION_REVIEW"
+_PENDING_PROMOTIONS_KEY = "pending_promotions"
+_PENDING_PROMOTIONS_CAP = 100
+
+# graph_meta stamp written by an applied (dry_run=False) noise-curate run; the
+# Command Center hygiene advisory reads it to pace its suggestion (Wave 2.5).
+_LAST_NOISE_CURATE_KEY = "last_noise_curate_at"
+
+
+def _promotion_review_default() -> bool:
+    return os.getenv(_PROMOTION_REVIEW_ENV, "").strip().lower() in ("1", "true", "yes")
+
 
 class KnowledgeGraphProjectionMixin:
     _FTS_SQL = """
@@ -408,7 +424,11 @@ class KnowledgeGraphProjectionMixin:
             )
 
     def curate(
-        self, *, max_documents: int = 200, max_new_nodes: int = 8
+        self,
+        *,
+        max_documents: int = 200,
+        max_new_nodes: int = 8,
+        review_mode: Optional[bool] = None,
     ) -> Dict[str, Any]:
         """On-demand graph curation (T4.4 — graph_curator goes live).
 
@@ -418,6 +438,14 @@ class KnowledgeGraphProjectionMixin:
         Topic nodes (with MENTIONS edges back to their sources and a real
         importance_score in nodes_v2). Explicit and observable — the result
         reports everything promoted AND everything skipped, with reasons.
+
+        ``review_mode`` (review 2026-07-25 Wave 4): when True, nothing is
+        written — the would-be promotions are parked in ``graph_meta`` as
+        ``pending_promotions`` for a human decision via
+        :meth:`apply_pending_promotions` / :meth:`reject_pending_promotions`.
+        Explicit argument wins; ``None`` falls back to the
+        ``LATTICEAI_GRAPH_PROMOTION_REVIEW`` env opt-in; default stays the
+        historical auto-promote behavior.
         """
         from .curator import auto_build_graph_overlay
 
@@ -464,46 +492,37 @@ class KnowledgeGraphProjectionMixin:
             existing_node_labels=existing_labels,
             max_new_nodes=max(1, min(int(max_new_nodes), 50)),
         )
+        valid_ids = {row["id"] for row in rows}
+        review = review_mode if review_mode is not None else _promotion_review_default()
+        if review:
+            proposed_at = _now()
+            proposed = [
+                {
+                    "id": f"topic:{_slug(promo['label'])}",
+                    "label": promo["label"],
+                    "importance": promo["importance"],
+                    "aliases": promo["aliases"],
+                    "sources": [s for s in promo["sources"][:10] if s in valid_ids],
+                    "proposed_at": proposed_at,
+                }
+                for promo in overlay["promotions"]
+            ]
+            with self._connect() as conn:
+                merged = self._merge_pending_promotions(conn, proposed)
+            return {
+                "status": "pending_review",
+                "documents_scanned": len(documents),
+                "candidates_total": overlay["candidates_total"],
+                "pending": proposed,
+                "pending_total": len(merged),
+                "skipped": overlay["skipped"][:50],
+                "skipped_total": len(overlay["skipped"]),
+            }
         promoted: List[Dict[str, Any]] = []
         with self._connect() as conn:
-            valid_ids = {row["id"] for row in rows}
             for promo in overlay["promotions"]:
-                topic_id = f"topic:{_slug(promo['label'])}"
-                self._upsert_node(
-                    conn,
-                    topic_id,
-                    "Topic",
-                    promo["label"],
-                    metadata={
-                        "curated": True,
-                        "importance": promo["importance"],
-                        "aliases": promo["aliases"],
-                        "source": "graph_curator",
-                    },
-                )
-                conn.execute(
-                    "UPDATE nodes_v2 SET importance_score=? WHERE id=?",
-                    (float(promo["importance"]), topic_id),
-                )
-                linked = 0
-                for source_id in promo["sources"][:10]:
-                    if source_id in valid_ids:
-                        self._upsert_edge(
-                            conn,
-                            source_id,
-                            topic_id,
-                            "MENTIONS",
-                            weight=0.6,
-                            metadata={"source": "graph_curator"},
-                        )
-                        linked += 1
                 promoted.append(
-                    {
-                        "node_id": topic_id,
-                        "label": promo["label"],
-                        "importance": promo["importance"],
-                        "linked_sources": linked,
-                    }
+                    self._write_promotion(conn, promo, valid_source_ids=valid_ids)
                 )
         return {
             "status": "ok",
@@ -514,6 +533,148 @@ class KnowledgeGraphProjectionMixin:
             "skipped_total": len(overlay["skipped"]),
         }
 
+    def _write_promotion(
+        self,
+        conn: sqlite3.Connection,
+        promo: Dict[str, Any],
+        *,
+        valid_source_ids: Optional[set] = None,
+    ) -> Dict[str, Any]:
+        """Write one curator promotion: Topic node + importance + MENTIONS edges.
+
+        Single write path shared by direct ``curate()`` and
+        :meth:`apply_pending_promotions`, so a human-approved promotion lands
+        exactly like an auto-promoted one. ``valid_source_ids`` restricts the
+        linkable sources to this curate run's scanned rows; when ``None``
+        (apply-after-review), each stored source is checked for existence so a
+        node deleted between propose and apply is skipped, not an error.
+        """
+        topic_id = str(promo.get("id") or f"topic:{_slug(str(promo['label']))}")
+        self._upsert_node(
+            conn,
+            topic_id,
+            "Topic",
+            str(promo["label"]),
+            metadata={
+                "curated": True,
+                "importance": promo["importance"],
+                "aliases": list(promo.get("aliases") or []),
+                "source": "graph_curator",
+            },
+        )
+        conn.execute(
+            "UPDATE nodes_v2 SET importance_score=? WHERE id=?",
+            (float(promo["importance"]), topic_id),
+        )
+        linked = 0
+        for source_id in list(promo.get("sources") or [])[:10]:
+            if valid_source_ids is not None:
+                if source_id not in valid_source_ids:
+                    continue
+            elif not conn.execute(
+                "SELECT 1 FROM nodes WHERE id=?", (source_id,)
+            ).fetchone():
+                continue
+            self._upsert_edge(
+                conn,
+                source_id,
+                topic_id,
+                "MENTIONS",
+                weight=0.6,
+                metadata={"source": "graph_curator"},
+            )
+            linked += 1
+        return {
+            "node_id": topic_id,
+            "label": promo["label"],
+            "importance": promo["importance"],
+            "linked_sources": linked,
+        }
+
+    # ── pending promotion queue (review 2026-07-25 Wave 4) ───────────────────
+
+    def _read_pending_promotions(
+        self, conn: sqlite3.Connection
+    ) -> List[Dict[str, Any]]:
+        try:
+            row = conn.execute(
+                "SELECT value FROM graph_meta WHERE key=?",
+                (_PENDING_PROMOTIONS_KEY,),
+            ).fetchone()
+        except sqlite3.Error:
+            return []
+        if not row or not row["value"]:
+            return []
+        try:
+            parsed = json.loads(row["value"])
+        except (TypeError, ValueError):
+            return []
+        if not isinstance(parsed, list):
+            return []
+        return [
+            item for item in parsed if isinstance(item, dict) and item.get("id")
+        ]
+
+    def _store_pending_promotions(
+        self, conn: sqlite3.Connection, entries: List[Dict[str, Any]]
+    ) -> None:
+        conn.execute(
+            "INSERT OR REPLACE INTO graph_meta(key, value) VALUES (?, ?)",
+            (_PENDING_PROMOTIONS_KEY, json.dumps(entries, ensure_ascii=False)),
+        )
+
+    def _merge_pending_promotions(
+        self, conn: sqlite3.Connection, proposed: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        """Merge new proposals into the stored queue (dedupe by id, cap 100)."""
+        merged: Dict[str, Dict[str, Any]] = {}
+        for item in self._read_pending_promotions(conn) + list(proposed):
+            merged[str(item["id"])] = item  # newest proposal wins per id
+        entries = list(merged.values())[-_PENDING_PROMOTIONS_CAP:]
+        self._store_pending_promotions(conn, entries)
+        return entries
+
+    def pending_promotions(self) -> List[Dict[str, Any]]:
+        """List promotions waiting for a human decision (review mode)."""
+        with self._connect() as conn:
+            return self._read_pending_promotions(conn)
+
+    def apply_pending_promotions(
+        self, ids: Optional[List[str]] = None
+    ) -> Dict[str, Any]:
+        """Apply stored pending promotions (all of them when ``ids`` is None).
+
+        Uses the exact node-writing path as direct ``curate()`` via
+        :meth:`_write_promotion`; applied entries leave the queue.
+        """
+        wanted = None if ids is None else {str(item) for item in ids}
+        applied: List[Dict[str, Any]] = []
+        remaining: List[Dict[str, Any]] = []
+        with self._connect() as conn:
+            for promo in self._read_pending_promotions(conn):
+                if wanted is not None and str(promo.get("id")) not in wanted:
+                    remaining.append(promo)
+                    continue
+                applied.append(self._write_promotion(conn, promo))
+            self._store_pending_promotions(conn, remaining)
+        return {"status": "ok", "applied": applied, "remaining": len(remaining)}
+
+    def reject_pending_promotions(
+        self, ids: Optional[List[str]] = None
+    ) -> Dict[str, Any]:
+        """Drop pending promotions without writing (all when ``ids`` is None)."""
+        wanted = None if ids is None else {str(item) for item in ids}
+        rejected: List[str] = []
+        remaining: List[Dict[str, Any]] = []
+        with self._connect() as conn:
+            for promo in self._read_pending_promotions(conn):
+                if wanted is not None and str(promo.get("id")) not in wanted:
+                    remaining.append(promo)
+                    continue
+                rejected.append(str(promo.get("id")))
+            self._store_pending_promotions(conn, remaining)
+        return {"status": "ok", "rejected": rejected, "remaining": len(remaining)}
+
     _NOISE_CONTENT_TYPES = (
         "Document",
         "File",
@@ -651,6 +812,13 @@ class KnowledgeGraphProjectionMixin:
                         (canonical, original),
                     )
                     conn.execute("DELETE FROM edges WHERE type=?", (original,))
+                # Stamp every applied run — even a no-op one means the graph
+                # was inspected, so the Command Center hygiene advisory
+                # (review 2026-07-25 Wave 2.5) stops re-suggesting for a while.
+                conn.execute(
+                    "INSERT OR REPLACE INTO graph_meta(key, value) VALUES (?, ?)",
+                    (_LAST_NOISE_CURATE_KEY, _now()),
+                )
 
         return {
             "status": "ok",
@@ -675,6 +843,22 @@ class KnowledgeGraphProjectionMixin:
             },
         }
 
+    def last_noise_curate_at(self) -> Optional[str]:
+        """Timestamp of the last applied (dry_run=False) noise-curate run.
+
+        ``None`` when the job never ran or the meta table is unreadable —
+        advisory readers treat both as "curation is due" (fail-open).
+        """
+        try:
+            with self._connect() as conn:
+                row = conn.execute(
+                    "SELECT value FROM graph_meta WHERE key=?",
+                    (_LAST_NOISE_CURATE_KEY,),
+                ).fetchone()
+        except sqlite3.Error:
+            return None
+        return str(row["value"]) if row and row["value"] else None
+
     def mark_superseded(self, old_node_id: str, new_node_id: str) -> Dict[str, Any]:
         """Record that ``old_node_id`` was replaced by ``new_node_id``.
 

package/lattice_brain/graph/rerank.py

@@ -0,0 +1,160 @@
+"""Optional cross-encoder rerank for hybrid retrieval (v9.9.5).
+
+Default path is identity (fused score preserved) — no model download, no
+latency tax, no unearned claims. Opt in with::
+
+    LATTICEAI_CROSS_ENCODER_RERANK=1
+    # optional model id (sentence-transformers CrossEncoder):
+    LATTICEAI_CROSS_ENCODER_MODEL=cross-encoder/ms-marco-MiniLM-L-6-v2
+
+When the env kill-switch is off, or ``sentence_transformers`` / the model is
+unavailable, :func:`rerank_matches` returns the candidates unchanged and
+reports ``mode="identity"``. Failures never raise into the search path.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+CROSS_ENCODER_RERANK_ENV = "LATTICEAI_CROSS_ENCODER_RERANK"
+CROSS_ENCODER_MODEL_ENV = "LATTICEAI_CROSS_ENCODER_MODEL"
+DEFAULT_CROSS_ENCODER_MODEL = "cross-encoder/ms-marco-MiniLM-L-6-v2"
+
+_model_cache: Dict[str, Any] = {}
+
+
+def _rerank_enabled() -> bool:
+    raw = os.getenv(CROSS_ENCODER_RERANK_ENV, "").strip().lower()
+    return raw in {"1", "true", "yes", "on"}
+
+
+def _model_id() -> str:
+    raw = os.getenv(CROSS_ENCODER_MODEL_ENV, "").strip()
+    return raw or DEFAULT_CROSS_ENCODER_MODEL
+
+
+def _candidate_text(match: Dict[str, Any]) -> str:
+    parts = [
+        str(match.get("title") or ""),
+        str(match.get("summary") or ""),
+        str((match.get("metadata") or {}).get("snippet") or ""),
+    ]
+    return " ".join(p for p in parts if p).strip() or str(match.get("node_id") or "")
+
+
+def _load_cross_encoder(model_id: str) -> Any:
+    if model_id in _model_cache:
+        return _model_cache[model_id]
+    from sentence_transformers import CrossEncoder  # type: ignore
+
+    model = CrossEncoder(model_id)
+    _model_cache[model_id] = model
+    return model
+
+
+def identity_rerank(
+    query: str,
+    candidates: List[Dict[str, Any]],
+    *,
+    top_k: Optional[int] = None,
+) -> Dict[str, Any]:
+    """Preserve fused ordering; stamp identity scores for an honest contract."""
+    del query  # identity path does not use the query text
+    ranked = list(candidates)
+    for item in ranked:
+        item["rerank_score"] = float(item.get("score") or item.get("fused_score") or 0.0)
+    if top_k is not None:
+        ranked = ranked[: max(1, int(top_k))]
+    return {
+        "matches": ranked,
+        "mode": "identity",
+        "model": None,
+        "detail": None,
+    }
+
+
+def cross_encoder_rerank(
+    query: str,
+    candidates: List[Dict[str, Any]],
+    *,
+    top_k: Optional[int] = None,
+    model_id: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Score (query, candidate) pairs with a CrossEncoder when available."""
+    if not candidates:
+        return {
+            "matches": [],
+            "mode": "cross_encoder",
+            "model": model_id or _model_id(),
+            "detail": None,
+        }
+    mid = model_id or _model_id()
+    try:
+        model = _load_cross_encoder(mid)
+    except Exception as exc:  # noqa: BLE001 — never break search
+        logger.info("cross-encoder unavailable (%s); falling back to identity", exc)
+        result = identity_rerank(query, candidates, top_k=top_k)
+        result["detail"] = f"cross_encoder_unavailable: {exc}"
+        return result
+
+    pairs = [[str(query or ""), _candidate_text(c)] for c in candidates]
+    try:
+        scores = model.predict(pairs)
+    except Exception as exc:  # noqa: BLE001
+        logger.warning("cross-encoder predict failed: %s", exc)
+        result = identity_rerank(query, candidates, top_k=top_k)
+        result["detail"] = f"cross_encoder_predict_failed: {exc}"
+        return result
+
+    ranked = list(candidates)
+    for item, score in zip(ranked, scores):
+        item["rerank_score"] = float(score)
+        # Surface the rerank score as the primary ranking key while keeping
+        # the pre-rerank fused score under scores.fused for audit.
+        scores_map = item.setdefault("scores", {})
+        if isinstance(scores_map, dict):
+            scores_map.setdefault("fused", float(item.get("score") or 0.0))
+            scores_map["rerank"] = float(score)
+        item["score"] = float(score)
+    ranked.sort(key=lambda m: (-float(m.get("rerank_score") or 0.0), str(m.get("node_id") or "")))
+    if top_k is not None:
+        ranked = ranked[: max(1, int(top_k))]
+    for rank, match in enumerate(ranked, start=1):
+        match["rank"] = rank
+    return {
+        "matches": ranked,
+        "mode": "cross_encoder",
+        "model": mid,
+        "detail": None,
+    }
+
+
+def rerank_matches(
+    query: str,
+    candidates: List[Dict[str, Any]],
+    *,
+    top_k: Optional[int] = None,
+    force: Optional[bool] = None,
+) -> Dict[str, Any]:
+    """Public entry: cross-encoder when enabled, else identity.
+
+    ``force=True/False`` overrides the env kill-switch (tests only).
+    """
+    enabled = _rerank_enabled() if force is None else bool(force)
+    if not enabled:
+        return identity_rerank(query, candidates, top_k=top_k)
+    return cross_encoder_rerank(query, candidates, top_k=top_k)
+
+
+__all__ = [
+    "CROSS_ENCODER_MODEL_ENV",
+    "CROSS_ENCODER_RERANK_ENV",
+    "DEFAULT_CROSS_ENCODER_MODEL",
+    "cross_encoder_rerank",
+    "identity_rerank",
+    "rerank_matches",
+]