ltcai 11.1.0 → 11.4.0

package/latticeai/services/brain_intelligence/digest.py

@@ -0,0 +1,258 @@
+"""What the Brain would tell you if you asked "what's going on in there?"
+
+Three read-only views over the same graph slice, each answering a different
+shape of that question:
+
+* :meth:`insights` — the proactive digest: recent growth, trending types, stale
+  knowledge, orphans, and questions grounded in real node titles;
+* :meth:`garden_overview` — the same knowledge as four beds a gardener tends
+  (new growth, weeds, neglect, load-bearing), because "how healthy is my
+  knowledge?" in aggregate is not a question anyone acts on;
+* :meth:`graph_duplicates` / :meth:`quality_report` — the graph layer's own
+  proactive scans, surfaced without applying anything.
+
+Honest when empty: an unavailable graph produces empty beds and
+``available: False``, never invented plants.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict, List, Optional
+
+from latticeai.core.timeutil import now_iso as _now
+
+from ._contract import BrainIntelligenceCore as _Core
+from .constants import _RECENT_DAYS, _STALE_DAYS, LOGGER, _parse_ts
+
+
+class BrainDigestMixin(_Core):
+    """Insights, garden, and the graph-layer quality reads."""
+
+    # ── insights digest ──────────────────────────────────────────────────
+
+    def insights(
+        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        sample = self._graph_sample(workspace_id=workspace_id)
+        nodes, edges = sample["nodes"], sample["edges"]
+        now = datetime.now(timezone.utc)
+        recent_cutoff = now - timedelta(days=_RECENT_DAYS)
+        stale_cutoff = now - timedelta(days=_STALE_DAYS)
+
+        recent_nodes: List[Dict[str, Any]] = []
+        stale_nodes: List[Dict[str, Any]] = []
+        type_counts: Dict[str, int] = {}
+        for node in nodes:
+            node_type = str(node.get("type") or "node")
+            ts = _parse_ts(node.get("updated_at"))
+            if ts is not None and ts >= recent_cutoff:
+                recent_nodes.append(node)
+                type_counts[node_type] = type_counts.get(node_type, 0) + 1
+            elif ts is not None and ts < stale_cutoff:
+                stale_nodes.append(node)
+
+        connected = set()
+        for edge in edges:
+            connected.add(str(edge.get("source") or edge.get("from_node") or ""))
+            connected.add(str(edge.get("target") or edge.get("to_node") or ""))
+        orphans = [n for n in nodes if str(n.get("id")) not in connected]
+
+        def _slim(node: Dict[str, Any]) -> Dict[str, Any]:
+            return {
+                "id": node.get("id"),
+                "type": node.get("type"),
+                "title": str(node.get("title") or "")[:120],
+                "updated_at": node.get("updated_at"),
+            }
+
+        trending = sorted(type_counts.items(), key=lambda kv: kv[1], reverse=True)[:5]
+        suggested_questions = [
+            f"{str(node.get('title') or '').strip()[:60]}에 대해 지금까지 알고 있는 것을 정리해줘"
+            for node in recent_nodes[:3]
+            if str(node.get("title") or "").strip()
+        ]
+
+        return {
+            "window_days": _RECENT_DAYS,
+            "activity": {
+                "recent_nodes": len(recent_nodes),
+                "recent_samples": [_slim(n) for n in recent_nodes[:8]],
+                "trending_types": [{"type": t, "count": c} for t, c in trending],
+            },
+            "attention": {
+                "stale_nodes": len(stale_nodes),
+                "stale_samples": [_slim(n) for n in stale_nodes[:8]],
+                "orphan_nodes": len(orphans),
+                "orphan_samples": [_slim(n) for n in orphans[:8]],
+            },
+            "suggested_questions": suggested_questions,
+            "graph_available": sample["available"],
+            "generated_at": _now(),
+        }
+
+    def garden_overview(
+        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None,
+        limit: int = 8,
+    ) -> Dict[str, Any]:
+        """The knowledge garden in four beds (v9.9.7).
+
+        Living Brain answers "how healthy is my knowledge?" in aggregate.
+        A gardener asks four concrete questions instead, and this answers all
+        four from one workspace-scoped graph sample plus the memory tier:
+
+        * **recent** — what came in lately (the garden's new growth);
+        * **contradictions** — what disagrees with itself (needs weeding);
+        * **stale** — what has not been touched in a long time;
+        * **frequent** — what the rest of the graph leans on most (by degree).
+
+        Read-only and honest: when the graph is unavailable every bed is empty
+        and ``available`` is false — the view never invents plants.
+        """
+        # Explicit 0 clamps to 1 — `limit or 8` would silently re-expand it.
+        try:
+            limit = max(1, min(int(limit), 50))
+        except (TypeError, ValueError):
+            limit = 8
+        sample = self._graph_sample(workspace_id=workspace_id)
+        nodes, edges = sample["nodes"], sample["edges"]
+        now = datetime.now(timezone.utc)
+        recent_cutoff = now - timedelta(days=_RECENT_DAYS)
+        stale_cutoff = now - timedelta(days=_STALE_DAYS)
+
+        def _slim(node: Dict[str, Any], **extra: Any) -> Dict[str, Any]:
+            return {
+                "id": node.get("id"),
+                "type": node.get("type"),
+                "title": str(node.get("title") or "")[:120],
+                "updated_at": node.get("updated_at"),
+                **extra,
+            }
+
+        recent: List[Dict[str, Any]] = []
+        stale: List[Dict[str, Any]] = []
+        for node in nodes:
+            # Chunks are retrieval plumbing, not knowledge a gardener tends.
+            if str(node.get("type") or "") == "Chunk":
+                continue
+            ts = _parse_ts(node.get("updated_at"))
+            if ts is None:
+                continue
+            if ts >= recent_cutoff:
+                recent.append(node)
+            elif ts < stale_cutoff:
+                stale.append(node)
+        recent.sort(key=lambda n: str(n.get("updated_at") or ""), reverse=True)
+        stale.sort(key=lambda n: str(n.get("updated_at") or ""))
+
+        # "Frequent" is degree, not a guess: how many relations actually point
+        # at a node. Chunks are retrieval plumbing, never garden plants.
+        degree: Dict[str, int] = {}
+        for edge in edges:
+            for key in ("source", "target"):
+                node_id = str(edge.get(key) or "")
+                if node_id:
+                    degree[node_id] = degree.get(node_id, 0) + 1
+        by_id = {str(node.get("id")): node for node in nodes}
+        frequent = [
+            _slim(by_id[node_id], degree=count)
+            for node_id, count in sorted(degree.items(), key=lambda kv: kv[1], reverse=True)
+            if node_id in by_id and str(by_id[node_id].get("type") or "") != "Chunk"
+        ][:limit]
+
+        contradiction_items: List[Dict[str, Any]] = []
+        contradiction_count = 0
+        try:
+            found = self.contradictions(user_email=user_email, workspace_id=workspace_id)
+            items = found.get("items") if isinstance(found, dict) else None
+            if isinstance(items, list):
+                contradiction_count = len(items)
+                contradiction_items = items[:limit]
+        except Exception:  # noqa: BLE001 — one empty bed, never a broken view
+            LOGGER.exception("garden overview contradictions failed")
+
+        return {
+            "available": sample["available"],
+            "window_days": _RECENT_DAYS,
+            "stale_threshold_days": _STALE_DAYS,
+            "beds": {
+                "recent": {
+                    "count": len(recent),
+                    "items": [_slim(node) for node in recent[:limit]],
+                },
+                "contradictions": {
+                    "count": contradiction_count,
+                    "items": contradiction_items,
+                },
+                "stale": {
+                    "count": len(stale),
+                    "items": [_slim(node) for node in stale[:limit]],
+                },
+                "frequent": {
+                    "count": len(frequent),
+                    "items": frequent,
+                },
+            },
+            "generated_at": _now(),
+        }
+
+    # ── graph-layer proactive quality (v9.6.x) ───────────────────────────
+
+    def graph_duplicates(
+        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Duplicate graph nodes (exact groups + near pairs) — read only."""
+        proactive = self._proactive()
+        if proactive is None:
+            return {
+                "available": False,
+                "exact_groups": [],
+                "near_pairs": [],
+                "exact_duplicate_nodes": 0,
+                "nodes_scanned": 0,
+                "generated_at": _now(),
+            }
+        try:
+            result = dict(proactive.find_duplicates(workspace_id=workspace_id))
+        except Exception as exc:
+            LOGGER.exception("graph duplicates scan failed")
+            return {
+                "available": False,
+                "error": str(exc),
+                "exact_groups": [],
+                "near_pairs": [],
+                "exact_duplicate_nodes": 0,
+                "nodes_scanned": 0,
+                "generated_at": _now(),
+            }
+        result["available"] = True
+        result["generated_at"] = _now()
+        return result
+
+    def quality_report(
+        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Combined graph quality report: duplicates, contradictions, stale
+        nodes, edge quality — one workspace-scoped graph sample."""
+        proactive = self._proactive()
+        if proactive is None:
+            return {"available": False, "generated_at": _now()}
+        try:
+            result = dict(proactive.quality_report(workspace_id=workspace_id))
+        except Exception as exc:
+            LOGGER.exception("graph quality report failed")
+            return {"available": False, "error": str(exc), "generated_at": _now()}
+        result["available"] = True
+        # v11.1.0: decay is part of quality, and "the Brain is tidying up" is a
+        # state the user is entitled to see rather than a background surprise.
+        importance = self.importance_report(
+            user_email=user_email, workspace_id=workspace_id
+        )
+        candidates = len(importance.get("candidates") or [])
+        result["importance"] = importance
+        result["tidying"] = bool(importance.get("available")) and candidates > 0
+        summary = dict(result.get("summary") or {})
+        summary["consolidation_candidates"] = candidates
+        result["summary"] = summary
+        result["generated_at"] = _now()
+        return result

package/latticeai/services/brain_intelligence/health.py

@@ -0,0 +1,331 @@
+"""The scored diagnosis, and the vector index's freshness.
+
+:meth:`health_report` grades four dimensions — freshness, connectivity,
+embedding coverage, consistency — from the live stores, and is deliberate about
+what it refuses to grade. A dimension it cannot measure scores ``None`` with a
+stated reason rather than a flattering number (an empty index covers 100% of
+nothing; scoring that as a perfect 100 is how a brand-new Brain used to grade
+itself "excellent"). The composite reports its own ``coverage``, and when
+nothing could be measured it says why — the 9.9.7 rule that a "—" always states
+its reason.
+
+:meth:`vector_freshness` is a fixed four-key contract that never raises, plus
+an additive ``breakdown`` when the store can split "never embedded" from
+"embedded but stale".
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from typing import Any, Dict, List, Optional
+
+from latticeai.core.timeutil import now_iso as _now
+
+from ._contract import BrainIntelligenceCore as _Core
+from .constants import _STALE_DAYS, LOGGER, _no_graph_reason, _parse_ts
+
+
+class BrainHealthMixin(_Core):
+    """Health report + vector freshness. Mixed into the service."""
+
+    # ── health report ─────────────────────────────────────────────────────
+
+    def health_report(
+        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        sample = self._graph_sample(workspace_id=workspace_id)
+        nodes, edges = sample["nodes"], sample["edges"]
+        now = datetime.now(timezone.utc)
+        dimensions: Dict[str, Dict[str, Any]] = {}
+
+        # Freshness — how much of the sampled knowledge saw recent updates.
+        if sample["available"] and nodes:
+            stale_cutoff = now - timedelta(days=_STALE_DAYS)
+            dated = [(_parse_ts(n.get("updated_at")), n) for n in nodes]
+            known = [pair for pair in dated if pair[0] is not None]
+            stale = [n for ts, n in known if ts is not None and ts < stale_cutoff]
+            fresh_ratio = 1.0 - (len(stale) / len(known)) if known else 0.0
+            dimensions["freshness"] = {
+                "status": "ok",
+                "score": round(fresh_ratio * 100),
+                "sampled": len(nodes),
+                "stale_nodes": len(stale),
+                "stale_threshold_days": _STALE_DAYS,
+            }
+        else:
+            dimensions["freshness"] = {
+                "status": "unavailable",
+                "score": None,
+                "reason": _no_graph_reason(sample["available"]),
+            }
+
+        # Connectivity — orphan nodes are knowledge the Brain cannot reason
+        # across; a well-tended graph keeps them rare.
+        if sample["available"] and nodes:
+            connected = set()
+            for edge in edges:
+                connected.add(str(edge.get("source") or edge.get("from_node") or ""))
+                connected.add(str(edge.get("target") or edge.get("to_node") or ""))
+            orphans = [n for n in nodes if str(n.get("id")) not in connected]
+            ratio = 1.0 - (len(orphans) / len(nodes))
+            dimensions["connectivity"] = {
+                "status": "ok",
+                "score": round(ratio * 100),
+                "sampled": len(nodes),
+                "orphan_nodes": len(orphans),
+                "edges": len(edges),
+            }
+        else:
+            dimensions["connectivity"] = {
+                "status": "unavailable",
+                "score": None,
+                "reason": _no_graph_reason(sample["available"]),
+            }
+
+        # Embedding coverage — semantic recall only works for indexed items.
+        index_status: Dict[str, Any] = {}
+        if self._enable_graph and hasattr(self._kg, "index_status"):
+            try:
+                index_status = self._kg.index_status()
+            except Exception as exc:
+                LOGGER.exception("brain intelligence index status failed")
+                index_status = {"error": str(exc)}
+        scale = index_status.get("scale") or {}
+        indexable = scale.get("source_items", index_status.get("source_items"))
+        if "coverage_ratio" not in scale:
+            dimensions["embedding_coverage"] = {
+                "status": "unavailable",
+                "score": None,
+                "reason": "this knowledge store does not report vector index coverage",
+            }
+        elif indexable == 0:
+            # An empty index covers 100% of nothing. Scoring that as a perfect
+            # 100 is how a brand-new Brain used to grade itself "excellent"
+            # off its only measurable dimension (audit v11.2.0, Finding 3).
+            dimensions["embedding_coverage"] = {
+                "status": "unavailable",
+                "score": None,
+                "reason": "no indexable items yet",
+            }
+        else:
+            dimensions["embedding_coverage"] = {
+                "status": "ok",
+                "score": round(float(scale["coverage_ratio"]) * 100),
+                "ready_items": scale.get("ready_items"),
+                "pending_items": scale.get("pending_items"),
+                "needs_reindex": index_status.get("status") == "needs_reindex",
+            }
+
+        # Edge quality + contradiction pressure — reuses the quality layer.
+        if sample["available"] and edges:
+            metrics = self._edge_quality.compute_quality_metrics(edges)
+            contradiction_edges = [
+                e for e in edges if "CONTRADICT" in str(e.get("type") or "").upper()
+            ]
+            pressure = min(1.0, metrics.get("dup_rate", 0.0) + len(contradiction_edges) / max(len(edges), 1))
+            dimensions["consistency"] = {
+                "status": "ok",
+                "score": round((1.0 - pressure) * 100),
+                "edge_metrics": metrics,
+                "contradiction_edges": len(contradiction_edges),
+            }
+        else:
+            dimensions["consistency"] = {
+                "status": "unavailable",
+                "score": None,
+                "reason": (
+                    _no_graph_reason(sample["available"])
+                    if not (sample["available"] and nodes)
+                    else "no relationships recorded yet"
+                ),
+            }
+
+        scores = [d["score"] for d in dimensions.values() if d.get("score") is not None]
+        overall = round(sum(scores) / len(scores)) if scores else None
+        grade = (
+            None if overall is None
+            else "excellent" if overall >= 85
+            else "good" if overall >= 70
+            else "attention" if overall >= 50
+            else "critical"
+        )
+        # What the verdict rests on. A composite averages only what could be
+        # measured, so the count of measured dimensions is part of the answer
+        # rather than a footnote — and when nothing could be measured the
+        # report says so instead of leaving a bare null (the 9.9.7 rule: a
+        # "—" always states why).
+        unmeasured = sorted(
+            name for name, dim in dimensions.items() if dim.get("score") is None
+        )
+        coverage: Dict[str, Any] = {
+            "measured": len(scores),
+            "total": len(dimensions),
+            "unavailable": unmeasured,
+            "partial": bool(unmeasured),
+        }
+        reason: Optional[str] = None
+        if overall is None:
+            reason = (
+                "no health dimension could be measured yet — "
+                + "; ".join(
+                    f"{name}: {dimensions[name].get('reason') or 'unavailable'}"
+                    for name in unmeasured
+                )
+            )
+
+        actions: List[Dict[str, str]] = []
+        emb = dimensions["embedding_coverage"]
+        if emb.get("needs_reindex"):
+            actions.append({
+                "id": "rebuild_vector_index",
+                "reason": f"{emb.get('pending_items', 0)} items are missing or stale in the vector index.",
+            })
+        conn_dim = dimensions["connectivity"]
+        conn_score = conn_dim.get("score")
+        if conn_score is not None and conn_score < 70:
+            actions.append({
+                "id": "review_orphans",
+                "reason": f"{conn_dim.get('orphan_nodes', 0)} nodes have no relationships.",
+            })
+        fresh_dim = dimensions["freshness"]
+        fresh_score = fresh_dim.get("score")
+        if fresh_score is not None and fresh_score < 60:
+            actions.append({
+                "id": "refresh_stale_knowledge",
+                "reason": f"{fresh_dim.get('stale_nodes', 0)} nodes untouched for over {_STALE_DAYS} days.",
+            })
+        cons_dim = dimensions["consistency"]
+        if cons_dim.get("contradiction_edges"):
+            actions.append({
+                "id": "resolve_contradictions",
+                "reason": f"{cons_dim['contradiction_edges']} contradiction edges recorded in the graph.",
+            })
+
+        report: Dict[str, Any] = {
+            "overall_score": overall,
+            "grade": grade,
+            "dimensions": dimensions,
+            "coverage": coverage,
+            "recommended_actions": actions,
+            "graph_available": sample["available"],
+            "generated_at": _now(),
+        }
+        if reason is not None:
+            report["reason"] = reason
+        return report
+
+    # ── vector freshness (v9.8.0) ────────────────────────────────────────
+
+    def vector_freshness(
+        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
+    ) -> Dict[str, Any]:
+        """Fixed-contract vector index freshness for ``/api/brain/vector-freshness``.
+
+        Always returns ``{"status": "ready"|"pending"|"unavailable",
+        "pending_items": int, "total_items": int, "detail": str}`` and never
+        raises. The vector index is store-global (not workspace-partitioned);
+        scope arguments are accepted for router symmetry but do not narrow
+        the report.
+
+        Since v11.2.0 a store that can split its backlog also gets a
+        ``breakdown`` key (see :meth:`_freshness_breakdown`). It is additive:
+        the four keys above keep their meaning and their types, so the
+        freshness chip that reads ``pending_items`` is untouched, and a store
+        without the split simply has no ``breakdown``.
+        """
+        payload = self._vector_freshness_contract()
+        breakdown = self._freshness_breakdown()
+        if breakdown is not None:
+            payload["breakdown"] = breakdown
+        return payload
+
+    def _freshness_breakdown(self) -> Optional[Dict[str, Any]]:
+        """``vector_freshness_breakdown()`` from the store, or ``None``.
+
+        "12 pending" hides two different situations — twelve items never
+        embedded (a new import) and twelve whose text changed under an
+        existing embedding (edits, where current answers are quietly wrong).
+        The store has always known the difference; until now nothing asked it.
+
+        ``None`` covers every reason the split is not available (graph off,
+        an older store, an unreadable index), because a caller can only act on
+        numbers that were really measured — never on zeros standing in for
+        them.
+        """
+        if not self._enable_graph or self._kg is None:
+            return None
+        breakdown_fn = getattr(self._kg, "vector_freshness_breakdown", None)
+        if not callable(breakdown_fn):
+            return None
+        try:
+            raw = breakdown_fn()
+        except Exception:
+            LOGGER.exception("vector freshness breakdown read failed")
+            return None
+        # An empty or unrecognisable answer is "not measured", which is what
+        # ``None`` already means here — publishing an empty block would claim
+        # a split nobody computed.
+        if not isinstance(raw, dict) or not raw:
+            return None
+        return dict(raw)
+
+    def _vector_freshness_contract(self) -> Dict[str, Any]:
+        """The four-key freshness payload, unchanged since v9.8.0."""
+
+        def _unavailable(detail: str) -> Dict[str, Any]:
+            return {
+                "status": "unavailable",
+                "pending_items": 0,
+                "total_items": 0,
+                "detail": detail,
+            }
+
+        if not self._enable_graph or self._kg is None:
+            return _unavailable("knowledge graph is disabled; no vector index is configured")
+
+        freshness_fn = getattr(self._kg, "vector_freshness", None)
+        if callable(freshness_fn):
+            try:
+                raw = freshness_fn() or {}
+            except Exception as exc:
+                LOGGER.exception("vector freshness read failed")
+                return _unavailable(f"vector freshness read failed: {exc}")
+            status = str(raw.get("status") or "unavailable")
+            if status == "needs_reindex":
+                status = "pending"
+            if status not in {"ready", "pending", "unavailable"}:
+                status = "unavailable"
+            return {
+                "status": status,
+                "pending_items": int(raw.get("pending_items") or 0),
+                "total_items": int(raw.get("total_items") or 0),
+                "detail": str(raw.get("detail") or ""),
+            }
+
+        # Older/lighter stores: summarize index_status directly.
+        status_fn = getattr(self._kg, "index_status", None)
+        if callable(status_fn):
+            try:
+                raw = status_fn() or {}
+            except Exception as exc:
+                LOGGER.exception("vector index status read failed")
+                return _unavailable(f"vector index status unavailable: {exc}")
+            pending = int(raw.get("pending_items") or 0)
+            total = int(raw.get("source_items") or 0)
+            if pending > 0:
+                return {
+                    "status": "pending",
+                    "pending_items": pending,
+                    "total_items": total,
+                    "detail": (
+                        f"{pending} of {total} items are missing or stale in the vector index"
+                    ),
+                }
+            return {
+                "status": "ready",
+                "pending_items": 0,
+                "total_items": total,
+                "detail": "vector index is up to date",
+            }
+
+        return _unavailable("this knowledge store does not expose a vector index")