ltcai 11.5.2 → 11.6.0

package/latticeai/services/brain_intelligence/digest.py

@@ -1,258 +0,0 @@
-"""What the Brain would tell you if you asked "what's going on in there?"
-
-Three read-only views over the same graph slice, each answering a different
-shape of that question:
-
-* :meth:`insights` — the proactive digest: recent growth, trending types, stale
-  knowledge, orphans, and questions grounded in real node titles;
-* :meth:`garden_overview` — the same knowledge as four beds a gardener tends
-  (new growth, weeds, neglect, load-bearing), because "how healthy is my
-  knowledge?" in aggregate is not a question anyone acts on;
-* :meth:`graph_duplicates` / :meth:`quality_report` — the graph layer's own
-  proactive scans, surfaced without applying anything.
-
-Honest when empty: an unavailable graph produces empty beds and
-``available: False``, never invented plants.
-"""
-
-from __future__ import annotations
-
-from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, List, Optional
-
-from latticeai.core.timeutil import now_iso as _now
-
-from ._contract import BrainIntelligenceCore as _Core
-from .constants import _RECENT_DAYS, _STALE_DAYS, LOGGER, _parse_ts
-
-
-class BrainDigestMixin(_Core):
-    """Insights, garden, and the graph-layer quality reads."""
-
-    # ── insights digest ──────────────────────────────────────────────────
-
-    def insights(
-        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
-    ) -> Dict[str, Any]:
-        sample = self._graph_sample(workspace_id=workspace_id)
-        nodes, edges = sample["nodes"], sample["edges"]
-        now = datetime.now(timezone.utc)
-        recent_cutoff = now - timedelta(days=_RECENT_DAYS)
-        stale_cutoff = now - timedelta(days=_STALE_DAYS)
-
-        recent_nodes: List[Dict[str, Any]] = []
-        stale_nodes: List[Dict[str, Any]] = []
-        type_counts: Dict[str, int] = {}
-        for node in nodes:
-            node_type = str(node.get("type") or "node")
-            ts = _parse_ts(node.get("updated_at"))
-            if ts is not None and ts >= recent_cutoff:
-                recent_nodes.append(node)
-                type_counts[node_type] = type_counts.get(node_type, 0) + 1
-            elif ts is not None and ts < stale_cutoff:
-                stale_nodes.append(node)
-
-        connected = set()
-        for edge in edges:
-            connected.add(str(edge.get("source") or edge.get("from_node") or ""))
-            connected.add(str(edge.get("target") or edge.get("to_node") or ""))
-        orphans = [n for n in nodes if str(n.get("id")) not in connected]
-
-        def _slim(node: Dict[str, Any]) -> Dict[str, Any]:
-            return {
-                "id": node.get("id"),
-                "type": node.get("type"),
-                "title": str(node.get("title") or "")[:120],
-                "updated_at": node.get("updated_at"),
-            }
-
-        trending = sorted(type_counts.items(), key=lambda kv: kv[1], reverse=True)[:5]
-        suggested_questions = [
-            f"{str(node.get('title') or '').strip()[:60]}에 대해 지금까지 알고 있는 것을 정리해줘"
-            for node in recent_nodes[:3]
-            if str(node.get("title") or "").strip()
-        ]
-
-        return {
-            "window_days": _RECENT_DAYS,
-            "activity": {
-                "recent_nodes": len(recent_nodes),
-                "recent_samples": [_slim(n) for n in recent_nodes[:8]],
-                "trending_types": [{"type": t, "count": c} for t, c in trending],
-            },
-            "attention": {
-                "stale_nodes": len(stale_nodes),
-                "stale_samples": [_slim(n) for n in stale_nodes[:8]],
-                "orphan_nodes": len(orphans),
-                "orphan_samples": [_slim(n) for n in orphans[:8]],
-            },
-            "suggested_questions": suggested_questions,
-            "graph_available": sample["available"],
-            "generated_at": _now(),
-        }
-
-    def garden_overview(
-        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None,
-        limit: int = 8,
-    ) -> Dict[str, Any]:
-        """The knowledge garden in four beds (v9.9.7).
-
-        Living Brain answers "how healthy is my knowledge?" in aggregate.
-        A gardener asks four concrete questions instead, and this answers all
-        four from one workspace-scoped graph sample plus the memory tier:
-
-        * **recent** — what came in lately (the garden's new growth);
-        * **contradictions** — what disagrees with itself (needs weeding);
-        * **stale** — what has not been touched in a long time;
-        * **frequent** — what the rest of the graph leans on most (by degree).
-
-        Read-only and honest: when the graph is unavailable every bed is empty
-        and ``available`` is false — the view never invents plants.
-        """
-        # Explicit 0 clamps to 1 — `limit or 8` would silently re-expand it.
-        try:
-            limit = max(1, min(int(limit), 50))
-        except (TypeError, ValueError):
-            limit = 8
-        sample = self._graph_sample(workspace_id=workspace_id)
-        nodes, edges = sample["nodes"], sample["edges"]
-        now = datetime.now(timezone.utc)
-        recent_cutoff = now - timedelta(days=_RECENT_DAYS)
-        stale_cutoff = now - timedelta(days=_STALE_DAYS)
-
-        def _slim(node: Dict[str, Any], **extra: Any) -> Dict[str, Any]:
-            return {
-                "id": node.get("id"),
-                "type": node.get("type"),
-                "title": str(node.get("title") or "")[:120],
-                "updated_at": node.get("updated_at"),
-                **extra,
-            }
-
-        recent: List[Dict[str, Any]] = []
-        stale: List[Dict[str, Any]] = []
-        for node in nodes:
-            # Chunks are retrieval plumbing, not knowledge a gardener tends.
-            if str(node.get("type") or "") == "Chunk":
-                continue
-            ts = _parse_ts(node.get("updated_at"))
-            if ts is None:
-                continue
-            if ts >= recent_cutoff:
-                recent.append(node)
-            elif ts < stale_cutoff:
-                stale.append(node)
-        recent.sort(key=lambda n: str(n.get("updated_at") or ""), reverse=True)
-        stale.sort(key=lambda n: str(n.get("updated_at") or ""))
-
-        # "Frequent" is degree, not a guess: how many relations actually point
-        # at a node. Chunks are retrieval plumbing, never garden plants.
-        degree: Dict[str, int] = {}
-        for edge in edges:
-            for key in ("source", "target"):
-                node_id = str(edge.get(key) or "")
-                if node_id:
-                    degree[node_id] = degree.get(node_id, 0) + 1
-        by_id = {str(node.get("id")): node for node in nodes}
-        frequent = [
-            _slim(by_id[node_id], degree=count)
-            for node_id, count in sorted(degree.items(), key=lambda kv: kv[1], reverse=True)
-            if node_id in by_id and str(by_id[node_id].get("type") or "") != "Chunk"
-        ][:limit]
-
-        contradiction_items: List[Dict[str, Any]] = []
-        contradiction_count = 0
-        try:
-            found = self.contradictions(user_email=user_email, workspace_id=workspace_id)
-            items = found.get("items") if isinstance(found, dict) else None
-            if isinstance(items, list):
-                contradiction_count = len(items)
-                contradiction_items = items[:limit]
-        except Exception:  # noqa: BLE001 — one empty bed, never a broken view
-            LOGGER.exception("garden overview contradictions failed")
-
-        return {
-            "available": sample["available"],
-            "window_days": _RECENT_DAYS,
-            "stale_threshold_days": _STALE_DAYS,
-            "beds": {
-                "recent": {
-                    "count": len(recent),
-                    "items": [_slim(node) for node in recent[:limit]],
-                },
-                "contradictions": {
-                    "count": contradiction_count,
-                    "items": contradiction_items,
-                },
-                "stale": {
-                    "count": len(stale),
-                    "items": [_slim(node) for node in stale[:limit]],
-                },
-                "frequent": {
-                    "count": len(frequent),
-                    "items": frequent,
-                },
-            },
-            "generated_at": _now(),
-        }
-
-    # ── graph-layer proactive quality (v9.6.x) ───────────────────────────
-
-    def graph_duplicates(
-        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """Duplicate graph nodes (exact groups + near pairs) — read only."""
-        proactive = self._proactive()
-        if proactive is None:
-            return {
-                "available": False,
-                "exact_groups": [],
-                "near_pairs": [],
-                "exact_duplicate_nodes": 0,
-                "nodes_scanned": 0,
-                "generated_at": _now(),
-            }
-        try:
-            result = dict(proactive.find_duplicates(workspace_id=workspace_id))
-        except Exception as exc:
-            LOGGER.exception("graph duplicates scan failed")
-            return {
-                "available": False,
-                "error": str(exc),
-                "exact_groups": [],
-                "near_pairs": [],
-                "exact_duplicate_nodes": 0,
-                "nodes_scanned": 0,
-                "generated_at": _now(),
-            }
-        result["available"] = True
-        result["generated_at"] = _now()
-        return result
-
-    def quality_report(
-        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """Combined graph quality report: duplicates, contradictions, stale
-        nodes, edge quality — one workspace-scoped graph sample."""
-        proactive = self._proactive()
-        if proactive is None:
-            return {"available": False, "generated_at": _now()}
-        try:
-            result = dict(proactive.quality_report(workspace_id=workspace_id))
-        except Exception as exc:
-            LOGGER.exception("graph quality report failed")
-            return {"available": False, "error": str(exc), "generated_at": _now()}
-        result["available"] = True
-        # v11.1.0: decay is part of quality, and "the Brain is tidying up" is a
-        # state the user is entitled to see rather than a background surprise.
-        importance = self.importance_report(
-            user_email=user_email, workspace_id=workspace_id
-        )
-        candidates = len(importance.get("candidates") or [])
-        result["importance"] = importance
-        result["tidying"] = bool(importance.get("available")) and candidates > 0
-        summary = dict(result.get("summary") or {})
-        summary["consolidation_candidates"] = candidates
-        result["summary"] = summary
-        result["generated_at"] = _now()
-        return result

package/latticeai/services/brain_intelligence/health.py

@@ -1,331 +0,0 @@
-"""The scored diagnosis, and the vector index's freshness.
-
-:meth:`health_report` grades four dimensions — freshness, connectivity,
-embedding coverage, consistency — from the live stores, and is deliberate about
-what it refuses to grade. A dimension it cannot measure scores ``None`` with a
-stated reason rather than a flattering number (an empty index covers 100% of
-nothing; scoring that as a perfect 100 is how a brand-new Brain used to grade
-itself "excellent"). The composite reports its own ``coverage``, and when
-nothing could be measured it says why — the 9.9.7 rule that a "—" always states
-its reason.
-
-:meth:`vector_freshness` is a fixed four-key contract that never raises, plus
-an additive ``breakdown`` when the store can split "never embedded" from
-"embedded but stale".
-"""
-
-from __future__ import annotations
-
-from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, List, Optional
-
-from latticeai.core.timeutil import now_iso as _now
-
-from ._contract import BrainIntelligenceCore as _Core
-from .constants import _STALE_DAYS, LOGGER, _no_graph_reason, _parse_ts
-
-
-class BrainHealthMixin(_Core):
-    """Health report + vector freshness. Mixed into the service."""
-
-    # ── health report ─────────────────────────────────────────────────────
-
-    def health_report(
-        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
-    ) -> Dict[str, Any]:
-        sample = self._graph_sample(workspace_id=workspace_id)
-        nodes, edges = sample["nodes"], sample["edges"]
-        now = datetime.now(timezone.utc)
-        dimensions: Dict[str, Dict[str, Any]] = {}
-
-        # Freshness — how much of the sampled knowledge saw recent updates.
-        if sample["available"] and nodes:
-            stale_cutoff = now - timedelta(days=_STALE_DAYS)
-            dated = [(_parse_ts(n.get("updated_at")), n) for n in nodes]
-            known = [pair for pair in dated if pair[0] is not None]
-            stale = [n for ts, n in known if ts is not None and ts < stale_cutoff]
-            fresh_ratio = 1.0 - (len(stale) / len(known)) if known else 0.0
-            dimensions["freshness"] = {
-                "status": "ok",
-                "score": round(fresh_ratio * 100),
-                "sampled": len(nodes),
-                "stale_nodes": len(stale),
-                "stale_threshold_days": _STALE_DAYS,
-            }
-        else:
-            dimensions["freshness"] = {
-                "status": "unavailable",
-                "score": None,
-                "reason": _no_graph_reason(sample["available"]),
-            }
-
-        # Connectivity — orphan nodes are knowledge the Brain cannot reason
-        # across; a well-tended graph keeps them rare.
-        if sample["available"] and nodes:
-            connected = set()
-            for edge in edges:
-                connected.add(str(edge.get("source") or edge.get("from_node") or ""))
-                connected.add(str(edge.get("target") or edge.get("to_node") or ""))
-            orphans = [n for n in nodes if str(n.get("id")) not in connected]
-            ratio = 1.0 - (len(orphans) / len(nodes))
-            dimensions["connectivity"] = {
-                "status": "ok",
-                "score": round(ratio * 100),
-                "sampled": len(nodes),
-                "orphan_nodes": len(orphans),
-                "edges": len(edges),
-            }
-        else:
-            dimensions["connectivity"] = {
-                "status": "unavailable",
-                "score": None,
-                "reason": _no_graph_reason(sample["available"]),
-            }
-
-        # Embedding coverage — semantic recall only works for indexed items.
-        index_status: Dict[str, Any] = {}
-        if self._enable_graph and hasattr(self._kg, "index_status"):
-            try:
-                index_status = self._kg.index_status()
-            except Exception as exc:
-                LOGGER.exception("brain intelligence index status failed")
-                index_status = {"error": str(exc)}
-        scale = index_status.get("scale") or {}
-        indexable = scale.get("source_items", index_status.get("source_items"))
-        if "coverage_ratio" not in scale:
-            dimensions["embedding_coverage"] = {
-                "status": "unavailable",
-                "score": None,
-                "reason": "this knowledge store does not report vector index coverage",
-            }
-        elif indexable == 0:
-            # An empty index covers 100% of nothing. Scoring that as a perfect
-            # 100 is how a brand-new Brain used to grade itself "excellent"
-            # off its only measurable dimension (audit v11.2.0, Finding 3).
-            dimensions["embedding_coverage"] = {
-                "status": "unavailable",
-                "score": None,
-                "reason": "no indexable items yet",
-            }
-        else:
-            dimensions["embedding_coverage"] = {
-                "status": "ok",
-                "score": round(float(scale["coverage_ratio"]) * 100),
-                "ready_items": scale.get("ready_items"),
-                "pending_items": scale.get("pending_items"),
-                "needs_reindex": index_status.get("status") == "needs_reindex",
-            }
-
-        # Edge quality + contradiction pressure — reuses the quality layer.
-        if sample["available"] and edges:
-            metrics = self._edge_quality.compute_quality_metrics(edges)
-            contradiction_edges = [
-                e for e in edges if "CONTRADICT" in str(e.get("type") or "").upper()
-            ]
-            pressure = min(1.0, metrics.get("dup_rate", 0.0) + len(contradiction_edges) / max(len(edges), 1))
-            dimensions["consistency"] = {
-                "status": "ok",
-                "score": round((1.0 - pressure) * 100),
-                "edge_metrics": metrics,
-                "contradiction_edges": len(contradiction_edges),
-            }
-        else:
-            dimensions["consistency"] = {
-                "status": "unavailable",
-                "score": None,
-                "reason": (
-                    _no_graph_reason(sample["available"])
-                    if not (sample["available"] and nodes)
-                    else "no relationships recorded yet"
-                ),
-            }
-
-        scores = [d["score"] for d in dimensions.values() if d.get("score") is not None]
-        overall = round(sum(scores) / len(scores)) if scores else None
-        grade = (
-            None if overall is None
-            else "excellent" if overall >= 85
-            else "good" if overall >= 70
-            else "attention" if overall >= 50
-            else "critical"
-        )
-        # What the verdict rests on. A composite averages only what could be
-        # measured, so the count of measured dimensions is part of the answer
-        # rather than a footnote — and when nothing could be measured the
-        # report says so instead of leaving a bare null (the 9.9.7 rule: a
-        # "—" always states why).
-        unmeasured = sorted(
-            name for name, dim in dimensions.items() if dim.get("score") is None
-        )
-        coverage: Dict[str, Any] = {
-            "measured": len(scores),
-            "total": len(dimensions),
-            "unavailable": unmeasured,
-            "partial": bool(unmeasured),
-        }
-        reason: Optional[str] = None
-        if overall is None:
-            reason = (
-                "no health dimension could be measured yet — "
-                + "; ".join(
-                    f"{name}: {dimensions[name].get('reason') or 'unavailable'}"
-                    for name in unmeasured
-                )
-            )
-
-        actions: List[Dict[str, str]] = []
-        emb = dimensions["embedding_coverage"]
-        if emb.get("needs_reindex"):
-            actions.append({
-                "id": "rebuild_vector_index",
-                "reason": f"{emb.get('pending_items', 0)} items are missing or stale in the vector index.",
-            })
-        conn_dim = dimensions["connectivity"]
-        conn_score = conn_dim.get("score")
-        if conn_score is not None and conn_score < 70:
-            actions.append({
-                "id": "review_orphans",
-                "reason": f"{conn_dim.get('orphan_nodes', 0)} nodes have no relationships.",
-            })
-        fresh_dim = dimensions["freshness"]
-        fresh_score = fresh_dim.get("score")
-        if fresh_score is not None and fresh_score < 60:
-            actions.append({
-                "id": "refresh_stale_knowledge",
-                "reason": f"{fresh_dim.get('stale_nodes', 0)} nodes untouched for over {_STALE_DAYS} days.",
-            })
-        cons_dim = dimensions["consistency"]
-        if cons_dim.get("contradiction_edges"):
-            actions.append({
-                "id": "resolve_contradictions",
-                "reason": f"{cons_dim['contradiction_edges']} contradiction edges recorded in the graph.",
-            })
-
-        report: Dict[str, Any] = {
-            "overall_score": overall,
-            "grade": grade,
-            "dimensions": dimensions,
-            "coverage": coverage,
-            "recommended_actions": actions,
-            "graph_available": sample["available"],
-            "generated_at": _now(),
-        }
-        if reason is not None:
-            report["reason"] = reason
-        return report
-
-    # ── vector freshness (v9.8.0) ────────────────────────────────────────
-
-    def vector_freshness(
-        self, *, user_email: Optional[str] = None, workspace_id: Optional[str] = None
-    ) -> Dict[str, Any]:
-        """Fixed-contract vector index freshness for ``/api/brain/vector-freshness``.
-
-        Always returns ``{"status": "ready"|"pending"|"unavailable",
-        "pending_items": int, "total_items": int, "detail": str}`` and never
-        raises. The vector index is store-global (not workspace-partitioned);
-        scope arguments are accepted for router symmetry but do not narrow
-        the report.
-
-        Since v11.2.0 a store that can split its backlog also gets a
-        ``breakdown`` key (see :meth:`_freshness_breakdown`). It is additive:
-        the four keys above keep their meaning and their types, so the
-        freshness chip that reads ``pending_items`` is untouched, and a store
-        without the split simply has no ``breakdown``.
-        """
-        payload = self._vector_freshness_contract()
-        breakdown = self._freshness_breakdown()
-        if breakdown is not None:
-            payload["breakdown"] = breakdown
-        return payload
-
-    def _freshness_breakdown(self) -> Optional[Dict[str, Any]]:
-        """``vector_freshness_breakdown()`` from the store, or ``None``.
-
-        "12 pending" hides two different situations — twelve items never
-        embedded (a new import) and twelve whose text changed under an
-        existing embedding (edits, where current answers are quietly wrong).
-        The store has always known the difference; until now nothing asked it.
-
-        ``None`` covers every reason the split is not available (graph off,
-        an older store, an unreadable index), because a caller can only act on
-        numbers that were really measured — never on zeros standing in for
-        them.
-        """
-        if not self._enable_graph or self._kg is None:
-            return None
-        breakdown_fn = getattr(self._kg, "vector_freshness_breakdown", None)
-        if not callable(breakdown_fn):
-            return None
-        try:
-            raw = breakdown_fn()
-        except Exception:
-            LOGGER.exception("vector freshness breakdown read failed")
-            return None
-        # An empty or unrecognisable answer is "not measured", which is what
-        # ``None`` already means here — publishing an empty block would claim
-        # a split nobody computed.
-        if not isinstance(raw, dict) or not raw:
-            return None
-        return dict(raw)
-
-    def _vector_freshness_contract(self) -> Dict[str, Any]:
-        """The four-key freshness payload, unchanged since v9.8.0."""
-
-        def _unavailable(detail: str) -> Dict[str, Any]:
-            return {
-                "status": "unavailable",
-                "pending_items": 0,
-                "total_items": 0,
-                "detail": detail,
-            }
-
-        if not self._enable_graph or self._kg is None:
-            return _unavailable("knowledge graph is disabled; no vector index is configured")
-
-        freshness_fn = getattr(self._kg, "vector_freshness", None)
-        if callable(freshness_fn):
-            try:
-                raw = freshness_fn() or {}
-            except Exception as exc:
-                LOGGER.exception("vector freshness read failed")
-                return _unavailable(f"vector freshness read failed: {exc}")
-            status = str(raw.get("status") or "unavailable")
-            if status == "needs_reindex":
-                status = "pending"
-            if status not in {"ready", "pending", "unavailable"}:
-                status = "unavailable"
-            return {
-                "status": status,
-                "pending_items": int(raw.get("pending_items") or 0),
-                "total_items": int(raw.get("total_items") or 0),
-                "detail": str(raw.get("detail") or ""),
-            }
-
-        # Older/lighter stores: summarize index_status directly.
-        status_fn = getattr(self._kg, "index_status", None)
-        if callable(status_fn):
-            try:
-                raw = status_fn() or {}
-            except Exception as exc:
-                LOGGER.exception("vector index status read failed")
-                return _unavailable(f"vector index status unavailable: {exc}")
-            pending = int(raw.get("pending_items") or 0)
-            total = int(raw.get("source_items") or 0)
-            if pending > 0:
-                return {
-                    "status": "pending",
-                    "pending_items": pending,
-                    "total_items": total,
-                    "detail": (
-                        f"{pending} of {total} items are missing or stale in the vector index"
-                    ),
-                }
-            return {
-                "status": "ready",
-                "pending_items": 0,
-                "total_items": total,
-                "detail": "vector index is up to date",
-            }
-
-        return _unavailable("this knowledge store does not expose a vector index")