ltcai 6.3.1 → 6.5.0

package/frontend/src/styles.css

@@ -1421,7 +1421,65 @@ body {
   line-height: 1.35;
 }
 
+.brain-readiness-strip {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) minmax(5.6rem, 0.6fr) auto;
+  align-items: center;
+  gap: 0.7rem;
+  margin-top: 0.75rem;
+  border: 1px solid hsl(var(--brain-core) / 0.28);
+  border-radius: 8px;
+  background: hsl(var(--brain-core) / 0.06);
+  padding: 0.66rem;
+}
+
+.brain-readiness-strip span {
+  display: block;
+  color: hsl(var(--fg-muted));
+  font-size: 0.64rem;
+  font-weight: 820;
+  text-transform: uppercase;
+}
+
+.brain-readiness-strip strong {
+  display: block;
+  margin-top: 0.12rem;
+  font-size: 0.82rem;
+  line-height: 1.2;
+}
+
+.brain-readiness-strip button {
+  min-height: 2rem;
+  border: 1px solid hsl(var(--brain-core) / 0.34);
+  border-radius: 999px;
+  background: hsl(var(--surface) / 0.78);
+  color: hsl(var(--fg));
+  padding: 0 0.72rem;
+  font-size: 0.76rem;
+  font-weight: 760;
+  white-space: nowrap;
+}
+
+.brain-readiness-meter {
+  height: 0.42rem;
+  overflow: hidden;
+  border-radius: 999px;
+  background: hsl(var(--bg) / 0.55);
+}
+
+.brain-readiness-meter i {
+  display: block;
+  height: 100%;
+  border-radius: inherit;
+  background: linear-gradient(90deg, hsl(var(--memory)), hsl(var(--knowledge)), hsl(var(--brain-core)));
+  transition: width 260ms ease;
+}
+
 .brain-save-feedback {
+  display: grid;
+  grid-template-columns: auto minmax(0, 1fr);
+  align-items: center;
+  gap: 0.22rem 0.48rem;
   border: 1px solid hsl(var(--memory) / 0.38);
   border-radius: 8px;
   background: hsl(var(--memory) / 0.08);
@@ -1431,6 +1489,13 @@ body {
   font-weight: 720;
 }
 
+.brain-save-feedback small {
+  grid-column: 2;
+  color: hsl(var(--fg-muted));
+  font-size: 0.72rem;
+  font-weight: 560;
+}
+
 .brain-message {
   max-width: 78%;
   margin-bottom: 1.1rem;
@@ -1911,6 +1976,59 @@ body {
   outline: none;
 }
 
+.brain-depth-rail {
+  position: absolute;
+  left: 50%;
+  top: 0.85rem;
+  z-index: 6;
+  display: flex;
+  width: min(92vw, 35rem);
+  transform: translateX(-50%);
+  gap: 0.28rem;
+  border: 1px solid hsl(var(--border) / 0.48);
+  border-radius: 999px;
+  background: hsl(var(--surface-glass));
+  padding: 0.32rem;
+  box-shadow: 0 12px 30px hsl(200 30% 3% / 0.18);
+  backdrop-filter: blur(18px);
+}
+
+.brain-depth-rail button {
+  display: inline-grid;
+  min-width: 0;
+  flex: 1 1 0;
+  align-items: center;
+  gap: 0.1rem;
+  border: 0;
+  border-radius: 999px;
+  background: transparent;
+  color: hsl(var(--fg-muted));
+  padding: 0.36rem 0.42rem;
+  text-align: center;
+}
+
+.brain-depth-rail span {
+  font-size: 0.58rem;
+  font-weight: 840;
+}
+
+.brain-depth-rail strong {
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+  font-size: 0.66rem;
+  line-height: 1.05;
+}
+
+.brain-depth-rail button.is-revealed {
+  background: hsl(var(--brain-core) / 0.1);
+  color: hsl(var(--fg));
+}
+
+.brain-depth-rail button[aria-current="step"] {
+  box-shadow: 0 0 0 1px hsl(var(--brain-core) / 0.38) inset;
+}
+
 .memory-fragment,
 .concept-signal,
 .graph-node {
@@ -2833,6 +2951,16 @@ body {
   .brain-depth-actions button {
     flex: 0 0 auto;
   }
+  .brain-depth-rail {
+    top: 0.35rem;
+    width: min(100vw - 1rem, 25rem);
+  }
+  .brain-depth-rail strong {
+    display: none;
+  }
+  .brain-readiness-strip {
+    grid-template-columns: 1fr;
+  }
   .brain-overview-grid {
     grid-template-columns: 1fr;
   }

package/lattice_brain/__init__.py

@@ -26,7 +26,7 @@ from .storage import (
     storage_from_env,
 )
 
-__version__ = "6.3.1"
+__version__ = "6.5.0"
 
 __all__ = [
     "AgentRuntime",
@@ -45,6 +45,7 @@ __all__ = [
     "IngestionPipeline",
     "KGPortabilityService",
     "KnowledgeGraphStore",
+    "LatticeBrainQuality",
     "MultiAgentOrchestrator",
     "PostgresConfig",
     "PostgresEngine",
@@ -65,6 +66,7 @@ _LAZY = {
     "ConversationStore": ("conversations", "ConversationStore"),
     "BrainMemory": ("memory", "BrainMemory"),
     "KnowledgeGraphStore": ("graph.store", "KnowledgeGraphStore"),
+    "LatticeBrainQuality": ("quality", "LatticeBrainQuality"),
     "IngestionItem": ("ingestion", "IngestionItem"),
     "IngestionPipeline": ("ingestion", "IngestionPipeline"),
     "KGPortabilityService": ("portability", "KGPortabilityService"),

package/lattice_brain/quality.py

@@ -0,0 +1,409 @@
+#!/usr/bin/env python3
+"""
+lattice_brain/quality.py
+Pure Python Quality Layer for Lattice Brain (v6.4+ hardening)
+- Does not modify any DB schema or existing APIs
+- Self-contained, dependency-free core (stdlib only)
+"""
+
+from __future__ import annotations
+import math
+import time
+import hashlib
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+from collections import defaultdict
+import re
+
+# -----------------------------
+# 1. Embedding Fallback Labelling + Drift/Reindex Plan
+# -----------------------------
+@dataclass
+class EmbeddingLabel:
+    vector_id: str
+    label: str
+    confidence: float
+    drift_score: float = 0.0
+    needs_reindex: bool = False
+
+class EmbeddingFallbackLabeller:
+    """Fallback labelling when primary embedding unavailable + drift detection"""
+
+    def __init__(self, drift_threshold: float = 0.15):
+        self.drift_threshold = drift_threshold
+        self.label_cache: Dict[str, EmbeddingLabel] = {}
+        self.vector_cache: Dict[str, List[float]] = {}
+
+    def label(self, vector_id: str, embedding: Optional[List[float]] = None,
+              metadata: Optional[Dict] = None) -> EmbeddingLabel:
+        if embedding is None:
+            label = "unembedded_fallback"
+            conf = 0.3
+        else:
+            # Simple hash-based pseudo-label for determinism
+            h = hashlib.md5(str(embedding[:4]).encode()).hexdigest()[:8]
+            label = f"emb_cluster_{h}"
+            conf = 0.85
+
+        drift = self._compute_drift(vector_id, embedding)
+        needs_reindex = drift > self.drift_threshold
+
+        el = EmbeddingLabel(vector_id, label, conf, drift, needs_reindex)
+        self.label_cache[vector_id] = el
+        if embedding is not None:
+            self.vector_cache[vector_id] = [float(value) for value in embedding]
+        return el
+
+    def _compute_drift(self, vector_id: str, new_emb: Optional[List[float]]) -> float:
+        old = self.vector_cache.get(vector_id)
+        if old is None or new_emb is None:
+            return 0.0
+        return self._cosine_distance(old, new_emb)
+
+    def _cosine_distance(self, a: List[float], b: List[float]) -> float:
+        if not a or not b or len(a) != len(b):
+            return 1.0
+        dot = sum(x*y for x,y in zip(a,b))
+        na = math.sqrt(sum(x*x for x in a))
+        nb = math.sqrt(sum(x*x for x in b))
+        return 1.0 - (dot / (na*nb + 1e-9))
+
+    def generate_reindex_plan(self) -> List[str]:
+        return [vid for vid, lab in self.label_cache.items() if lab.needs_reindex]
+
+# -----------------------------
+# 2. BM25 Lexical Scoring + Hybrid Fusion + Reranker Interface
+# -----------------------------
+class BM25Scorer:
+    def __init__(self, k1: float = 1.5, b: float = 0.75):
+        self.k1 = k1
+        self.b = b
+        self.doc_freqs: Dict[str, int] = defaultdict(int)
+        self.doc_lens: Dict[str, int] = {}
+        self.corpus_size = 0
+        self.avgdl = 0.0
+
+    def fit(self, corpus: Dict[str, str]):
+        self.corpus_size = len(corpus)
+        total_len = 0
+        for did, text in corpus.items():
+            tokens = self._tokenize(text)
+            self.doc_lens[did] = len(tokens)
+            total_len += len(tokens)
+            for t in set(tokens):
+                self.doc_freqs[t] += 1
+        self.avgdl = total_len / max(1, self.corpus_size)
+
+    def score(self, query: str, doc_id: str, doc_text: str) -> float:
+        tokens = self._tokenize(query)
+        doc_tokens = self._tokenize(doc_text)
+        score = 0.0
+        for t in tokens:
+            if t not in doc_tokens:
+                continue
+            tf = doc_tokens.count(t)
+            df = self.doc_freqs.get(t, 0)
+            idf = math.log((self.corpus_size - df + 0.5) / (df + 0.5) + 1)
+            denom = tf + self.k1 * (1 - self.b + self.b * self.doc_lens.get(doc_id, 0) / self.avgdl)
+            score += idf * tf * (self.k1 + 1) / (denom + 1e-9)
+        return score
+
+    def _tokenize(self, text: str) -> List[str]:
+        return re.findall(r'\w+', text.lower())
+
+class HybridFusion:
+    """Hybrid fusion of lexical (BM25) + vector scores"""
+    def __init__(self, alpha: float = 0.6):
+        self.alpha = alpha  # weight for vector score
+        self.bm25 = BM25Scorer()
+
+    def fuse(self, query: str, candidates: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        # candidates: [{"id": , "text": , "vector_score": }]
+        corpus = {c["id"]: c.get("text", "") for c in candidates}
+        self.bm25.fit(corpus)
+        results = []
+        for c in candidates:
+            lex = self.bm25.score(query, c["id"], c.get("text", ""))
+            vec = c.get("vector_score", 0.5)
+            fused = self.alpha * vec + (1 - self.alpha) * (lex / 10.0)  # normalize rough
+            c["fused_score"] = round(fused, 4)
+            results.append(c)
+        return sorted(results, key=lambda x: x["fused_score"], reverse=True)
+
+class RerankerInterface:
+    """Pluggable reranker interface"""
+    def rerank(self, query: str, candidates: List[Dict], top_k: int = 5) -> List[Dict]:
+        # Default local-safe fallback: preserve fused ordering without making
+        # unearned cross-encoder claims.
+        for c in candidates:
+            c["rerank_score"] = c.get("fused_score", 0.0)
+        return sorted(candidates, key=lambda x: x.get("rerank_score", 0), reverse=True)[:top_k]
+
+# -----------------------------
+# 3. Memory Candidate Extraction / Scoring / Dedupe / Merge / Conflict / Retention
+# -----------------------------
+@dataclass
+class MemoryCandidate:
+    id: str
+    content: str
+    score: float = 0.0
+    source: str = "unknown"
+    timestamp: float = field(default_factory=time.time)
+    conflicts: List[str] = field(default_factory=list)
+
+class MemoryQualityManager:
+    def extract_candidates(self, memories: List[Dict]) -> List[MemoryCandidate]:
+        return [MemoryCandidate(m["id"], m["content"], m.get("score", 0.6), m.get("source", "mem")) for m in memories]
+
+    def score_candidates(self, cands: List[MemoryCandidate], query: str) -> List[MemoryCandidate]:
+        for c in cands:
+            # simple lexical overlap score
+            overlap = sum(1 for w in query.lower().split() if w in c.content.lower())
+            c.score = min(1.0, 0.4 + overlap * 0.15)
+        return sorted(cands, key=lambda x: x.score, reverse=True)
+
+    def dedupe(self, cands: List[MemoryCandidate], threshold: float = 0.85) -> List[MemoryCandidate]:
+        kept = []
+        seen = set()
+        for c in cands:
+            h = hashlib.md5(c.content.encode()).hexdigest()[:16]
+            if h not in seen:
+                seen.add(h)
+                kept.append(c)
+        return kept
+
+    def merge(self, cands: List[MemoryCandidate]) -> List[MemoryCandidate]:
+        # naive merge by content prefix
+        merged = {}
+        for c in cands:
+            key = c.content[:30]
+            if key not in merged or c.score > merged[key].score:
+                merged[key] = c
+        return list(merged.values())
+
+    def detect_conflicts(self, cands: List[MemoryCandidate]) -> List[MemoryCandidate]:
+        # Lightweight local heuristic; LLM conflict classifiers can replace it.
+        for i, c in enumerate(cands):
+            if "not" in c.content.lower() or "반대" in c.content:
+                c.conflicts.append("conflict:possible_negation")
+        return cands
+
+    def apply_retention(self, cands: List[MemoryCandidate], max_age_days: int = 90) -> List[MemoryCandidate]:
+        now = time.time()
+        return [c for c in cands if (now - c.timestamp) < max_age_days * 86400]
+
+# -----------------------------
+# 4. Graph Edge Validation / Confidence / Evidence / Duplicate Merge / Quality Metrics
+# -----------------------------
+@dataclass
+class GraphEdgeQuality:
+    edge_id: str
+    confidence: float
+    evidence_count: int
+    is_duplicate: bool = False
+    quality_score: float = 0.0
+
+class GraphEdgeQualityManager:
+    def validate_edge(self, edge: Dict[str, Any]) -> GraphEdgeQuality:
+        conf = edge.get("confidence", 0.7)
+        ev = len(edge.get("evidence", []))
+        q = min(1.0, conf * 0.75 + min(ev, 5) / 5 * 0.3)
+        return GraphEdgeQuality(edge.get("id", "e0"), conf, ev, quality_score=q)
+
+    def detect_duplicate_edges(self, edges: List[Dict]) -> List[str]:
+        seen = {}
+        dups = []
+        for e in edges:
+            key = (e.get("source"), e.get("target"), e.get("type"))
+            if key in seen:
+                dups.append(e.get("id"))
+            else:
+                seen[key] = e.get("id")
+        return dups
+
+    def merge_duplicate_edges(self, edges: List[Dict]) -> List[Dict]:
+        # keep highest confidence
+        best = {}
+        for e in edges:
+            key = (e.get("source"), e.get("target"), e.get("type"))
+            if key not in best or e.get("confidence", 0) > best[key].get("confidence", 0):
+                best[key] = e
+        return list(best.values())
+
+    def compute_quality_metrics(self, edges: List[Dict]) -> Dict[str, float]:
+        if not edges:
+            return {"avg_conf": 0.0, "avg_evidence": 0.0, "dup_rate": 0.0}
+        confs = [e.get("confidence", 0.5) for e in edges]
+        evs = [len(e.get("evidence", [])) for e in edges]
+        dups = len(self.detect_duplicate_edges(edges))
+        return {
+            "avg_conf": round(sum(confs)/len(confs), 3),
+            "avg_evidence": round(sum(evs)/len(evs), 2),
+            "dup_rate": round(dups / len(edges), 3)
+        }
+
+# -----------------------------
+# 5. Structured Context Assembly + Guardrails
+# -----------------------------
+@dataclass
+class ContextGuardrails:
+    known: bool = True
+    inferred: bool = False
+    stale: bool = False
+    unknown: bool = False
+    confidence: float = 0.8
+    timestamp: float = field(default_factory=time.time)
+    attribution: str = "system"
+
+class StructuredContextAssembler:
+    SECTIONS = ["Facts", "Decisions", "Preferences", "Relationships", "Projects", "Recent Events"]
+
+    def assemble(self, items: List[Dict[str, Any]]) -> Dict[str, List[Dict]]:
+        ctx: Dict[str, List[Dict]] = {s: [] for s in self.SECTIONS}
+        for item in items:
+            section = item.get("section", "Facts")
+            if section not in ctx:
+                section = "Facts"
+            guard = ContextGuardrails(
+                known=item.get("known", True),
+                inferred=item.get("inferred", False),
+                stale=item.get("stale", False),
+                unknown=item.get("unknown", False),
+                confidence=item.get("confidence", 0.75),
+                attribution=item.get("attribution", "user")
+            )
+            item["guardrails"] = guard.__dict__
+            ctx[section].append(item)
+        return ctx
+
+    def apply_guardrails(self, ctx: Dict[str, List[Dict]]) -> Dict[str, List[Dict]]:
+        cleaned = {}
+        for sec, items in ctx.items():
+            cleaned[sec] = [i for i in items if not i.get("guardrails", {}).get("unknown", False)]
+        return cleaned
+
+# -----------------------------
+# 6. Retrieval Benchmark Fixture Runner
+# -----------------------------
+class RetrievalBenchmarkRunner:
+    def __init__(self):
+        self.results: List[Dict] = []
+
+    def run_fixture(self, fixture_name: str, queries: List[Any], top_k: int = 5) -> Dict[str, Any]:
+        start = time.time()
+        judged = [q for q in queries if isinstance(q, dict)]
+        if judged:
+            recalls = []
+            precisions = []
+            ndcgs = []
+            for query in judged:
+                relevant = set(query.get("relevant") or [])
+                retrieved = list(query.get("retrieved") or [])[:top_k]
+                if not relevant:
+                    continue
+                hits = [doc_id for doc_id in retrieved if doc_id in relevant]
+                recalls.append(len(hits) / len(relevant))
+                precisions.append(len(hits) / max(1, len(retrieved)))
+                dcg = sum(
+                    1.0 / math.log2(rank + 2)
+                    for rank, doc_id in enumerate(retrieved)
+                    if doc_id in relevant
+                )
+                ideal = sum(1.0 / math.log2(rank + 2) for rank in range(min(len(relevant), top_k)))
+                ndcgs.append(dcg / ideal if ideal else 0.0)
+            recall = sum(recalls) / len(recalls) if recalls else 0.0
+            precision = sum(precisions) / len(precisions) if precisions else 0.0
+            ndcg = sum(ndcgs) / len(ndcgs) if ndcgs else 0.0
+        else:
+            recall = precision = ndcg = 0.0
+        metrics = {
+            "fixture": fixture_name,
+            "queries": len(queries),
+            "avg_latency_ms": round((time.time() - start) * 1000 / max(1, len(queries)), 2),
+            "recall@5": round(recall, 4),
+            "precision@5": round(precision, 4),
+            "ndcg@5": round(ndcg, 4),
+            "top_k": top_k,
+            "judged": len(judged),
+        }
+        self.results.append(metrics)
+        return metrics
+
+    def summary(self) -> Dict[str, Any]:
+        if not self.results:
+            return {"status": "no runs"}
+        return {
+            "total_runs": len(self.results),
+            "avg_recall": round(sum(r["recall@5"] for r in self.results) / len(self.results), 3)
+        }
+
+# -----------------------------
+# Main Quality Layer Facade
+# -----------------------------
+class LatticeBrainQuality:
+    """Main entry point - pure Python quality hardening layer"""
+    def __init__(self):
+        self.embed_labeller = EmbeddingFallbackLabeller()
+        self.hybrid = HybridFusion()
+        self.reranker = RerankerInterface()
+        self.memory_mgr = MemoryQualityManager()
+        self.graph_mgr = GraphEdgeQualityManager()
+        self.context_assembler = StructuredContextAssembler()
+        self.benchmark = RetrievalBenchmarkRunner()
+
+    def full_quality_pass(self, payload: Dict[str, Any]) -> Dict[str, Any]:
+        """End-to-end quality pipeline (non-destructive)"""
+        result = {"status": "ok", "timestamp": time.time()}
+
+        # embedding
+        if "embeddings" in payload:
+            labels = [self.embed_labeller.label(e["id"], e.get("vector")) for e in payload["embeddings"]]
+            result["embedding_labels"] = [label.__dict__ for label in labels]
+            result["reindex_plan"] = self.embed_labeller.generate_reindex_plan()
+
+        # hybrid retrieval
+        if "retrieval" in payload:
+            fused = self.hybrid.fuse(payload["retrieval"]["query"], payload["retrieval"]["candidates"])
+            reranked = self.reranker.rerank(payload["retrieval"]["query"], fused)
+            result["retrieval"] = {"fused": fused, "reranked": reranked}
+
+        # memory
+        if "memories" in payload:
+            cands = self.memory_mgr.extract_candidates(payload["memories"])
+            cands = self.memory_mgr.score_candidates(cands, payload.get("query", ""))
+            cands = self.memory_mgr.dedupe(cands)
+            cands = self.memory_mgr.merge(cands)
+            cands = self.memory_mgr.detect_conflicts(cands)
+            cands = self.memory_mgr.apply_retention(cands)
+            result["memory_candidates"] = [c.__dict__ for c in cands]
+
+        # graph
+        if "graph_edges" in payload:
+            eqs = [self.graph_mgr.validate_edge(e) for e in payload["graph_edges"]]
+            result["graph_quality"] = [q.__dict__ for q in eqs]
+            result["graph_metrics"] = self.graph_mgr.compute_quality_metrics(payload["graph_edges"])
+
+        # context
+        if "context_items" in payload:
+            ctx = self.context_assembler.assemble(payload["context_items"])
+            ctx = self.context_assembler.apply_guardrails(ctx)
+            result["structured_context"] = ctx
+
+        return result
+
+
+__all__ = [
+    "BM25Scorer",
+    "ContextGuardrails",
+    "EmbeddingFallbackLabeller",
+    "EmbeddingLabel",
+    "GraphEdgeQuality",
+    "GraphEdgeQualityManager",
+    "HybridFusion",
+    "LatticeBrainQuality",
+    "MemoryCandidate",
+    "MemoryQualityManager",
+    "RetrievalBenchmarkRunner",
+    "RerankerInterface",
+    "StructuredContextAssembler",
+]

package/lattice_brain/runtime/multi_agent.py

@@ -19,7 +19,7 @@ from datetime import datetime
 from typing import Any, Callable, Dict, List, Optional
 
 
-MULTI_AGENT_VERSION = "6.3.1"
+MULTI_AGENT_VERSION = "6.5.0"
 
 AGENT_ROLES = ("researcher", "planner", "executor", "reviewer", "release")
 CORE_PIPELINE = ("planner", "executor", "reviewer")

package/latticeai/__init__.py

@@ -1,3 +1,3 @@
 """Lattice AI - modular server package."""
 
-__version__ = "6.3.1"
+__version__ = "6.5.0"