ltcai 8.9.0 → 9.0.0

package/lattice_brain/graph/store.py

@@ -5,10 +5,13 @@ from __future__ import annotations
 from ._kg_common import *  # noqa: F403,F401
 from .documents import KnowledgeGraphDocumentsMixin
 from .discovery import KnowledgeGraphDiscoveryMixin
+from .discovery_index import KnowledgeGraphLocalIndexMixin
 from .ingest import KnowledgeGraphIngestMixin
 from .projection import KnowledgeGraphProjectionMixin
 from .provenance import KnowledgeGraphProvenanceMixin
 from .retrieval import KnowledgeGraphRetrievalMixin
+from .retrieval_docgen import KnowledgeGraphDocGenMixin
+from .retrieval_vector import KnowledgeGraphVectorMixin
 from .write_master import KnowledgeGraphWriteMixin
 
 
@@ -16,10 +19,13 @@ class KnowledgeGraphStore(
     KnowledgeGraphProjectionMixin,
     KnowledgeGraphWriteMixin,
     KnowledgeGraphDiscoveryMixin,
+    KnowledgeGraphLocalIndexMixin,
     KnowledgeGraphIngestMixin,
     KnowledgeGraphProvenanceMixin,
     KnowledgeGraphDocumentsMixin,
     KnowledgeGraphRetrievalMixin,
+    KnowledgeGraphVectorMixin,
+    KnowledgeGraphDocGenMixin,
 ):
     def __init__(
         self,

package/lattice_brain/ingestion.py

@@ -50,6 +50,52 @@ def _now_iso() -> str:
     return datetime.now(timezone.utc).isoformat()
 
 
+# --- Large candidate 1 slice: incremental / background ingestion support ---
+@dataclass
+class BackgroundIngestionJob:
+    """Job descriptor for background/incremental indexing (KG scale-up slice)."""
+    job_id: str
+    items: List[IngestionItem]
+    status: str = "pending"  # pending | running | done | failed
+    created_at: str = field(default_factory=_now_iso)
+    processed: int = 0
+    total: int = 0
+    errors: List[str] = field(default_factory=list)
+
+
+class BackgroundIngestionQueue:
+    """Simple in-memory queue for background incremental ingestion.
+
+    For large corpus: this is the seam where a real scheduler / worker pool
+    (celery, rq, or internal thread) can be plugged later without changing callers.
+    Supports incremental (skip duplicates) vs force reindex.
+    """
+    def __init__(self) -> None:
+        self._jobs: Dict[str, BackgroundIngestionJob] = {}
+        self._counter = 0
+
+    def schedule(self, items: List[IngestionItem], *, incremental: bool = True) -> BackgroundIngestionJob:
+        self._counter += 1
+        job_id = f"bg_ingest_{self._counter:04d}"
+        job = BackgroundIngestionJob(
+            job_id=job_id,
+            items=items,
+            total=len(items),
+        )
+        # annotate items for downstream
+        for it in job.items:
+            # attach flag without breaking dataclass defaults (use metadata)
+            it.metadata = {**it.metadata, "incremental": incremental, "bg_job": job_id}
+        self._jobs[job_id] = job
+        return job
+
+    def get(self, job_id: str) -> Optional[BackgroundIngestionJob]:
+        return self._jobs.get(job_id)
+
+    def list_pending(self) -> List[BackgroundIngestionJob]:
+        return [j for j in self._jobs.values() if j.status == "pending"]
+
+
 @dataclass
 class IngestionItem:
     """A single thing to ingest, normalized across every source type."""
@@ -118,6 +164,7 @@ class IngestionPipeline:
         audit: Optional[Any] = None,
         max_text_bytes: int = DEFAULT_MAX_TEXT_BYTES,
         pipeline_name: str = "unified-ingestion",
+        bg_queue: Optional[BackgroundIngestionQueue] = None,
     ) -> None:
         self._kg = knowledge_graph
         self._hooks = hooks
@@ -125,6 +172,7 @@ class IngestionPipeline:
         self._audit = audit
         self._max_text_bytes = int(max_text_bytes)
         self._pipeline_name = pipeline_name
+        self._bg_queue = bg_queue or BackgroundIngestionQueue()
 
     def available(self) -> bool:
         return self._enable and self._kg is not None
@@ -243,6 +291,26 @@ class IngestionPipeline:
             detail=provenance_detail,
         )
 
+    # --- Large candidate #1: background / incremental scheduling (slice) ---
+    def schedule_background(
+        self,
+        items: List[IngestionItem],
+        *,
+        incremental: bool = True,
+    ) -> BackgroundIngestionJob:
+        """Schedule items for background incremental indexing.
+
+        Returns a job handle. Actual execution can be driven by caller
+        (or future worker) calling pipeline.ingest on each. This seam enables
+        large-corpus scale without blocking user requests.
+        """
+        job = self._bg_queue.schedule(items, incremental=incremental)
+        # mark initial status on results concept (jobs track)
+        return job
+
+    def get_background_job(self, job_id: str) -> Optional[BackgroundIngestionJob]:
+        return self._bg_queue.get(job_id)
+
     # ── routing helpers ──────────────────────────────────────────────────────
     def _ingest_text(self, item, *, source_type, owner, captured_at) -> Dict[str, Any]:
         text = item.text or ""

package/lattice_brain/portability.py

@@ -13,7 +13,6 @@ any cloud service. Two complementary mechanisms, both fully local:
 
 from __future__ import annotations
 
-import hashlib
 import json
 import os
 import shutil
@@ -30,6 +29,7 @@ from .storage import (
     PostgresEngine,
     SQLiteToPostgresMigrator,
 )
+from .utils import sha256_file as _sha256_file
 
 FORMAT = "latticeai.kg.export"
 FORMAT_VERSION = 1
@@ -44,14 +44,6 @@ def _stamp() -> str:
     return _now_iso().replace(":", "").replace("-", "").replace(".", "")[:15]
 
 
-def _sha256_file(path: Path) -> str:
-    h = hashlib.sha256()
-    with open(path, "rb") as fh:
-        for block in iter(lambda: fh.read(65536), b""):
-            h.update(block)
-    return h.hexdigest()
-
-
 def _safe_zip_names(names) -> None:
     for name in names:
         path = PurePosixPath(name)

package/lattice_brain/quality.py

@@ -152,6 +152,28 @@ class MemoryCandidate:
     conflicts: List[str] = field(default_factory=list)
 
 class MemoryQualityManager:
+    _NEGATION_PATTERNS = (
+        "not",
+        "does not",
+        "don't",
+        "do not",
+        "싫어",
+        "원하지 않",
+        "하지 않",
+        "반대",
+    )
+    _POSITIVE_PATTERNS = (
+        "prefers",
+        "prefer",
+        "likes",
+        "like",
+        "wants",
+        "want",
+        "좋아",
+        "선호",
+        "원해",
+    )
+
     def extract_candidates(self, memories: List[Dict]) -> List[MemoryCandidate]:
         return [MemoryCandidate(m["id"], m["content"], m.get("score", 0.6), m.get("source", "mem")) for m in memories]
 
@@ -163,10 +185,16 @@ class MemoryQualityManager:
         return sorted(cands, key=lambda x: x.score, reverse=True)
 
     def dedupe(self, cands: List[MemoryCandidate], threshold: float = 0.85) -> List[MemoryCandidate]:
+        """Advanced content dedup (KG scale slice).
+
+        Uses sha256 prefix + normalized length bucket for better collision resistance
+        than plain md5. Future: plug embedding cosine or simhash here.
+        """
         kept = []
         seen = set()
         for c in cands:
-            h = hashlib.md5(c.content.encode()).hexdigest()[:16]
+            norm = " ".join(c.content.lower().split())[:200]
+            h = hashlib.sha256((norm + f"|{len(c.content)//50}").encode()).hexdigest()[:16]
             if h not in seen:
                 seen.add(h)
                 kept.append(c)
@@ -182,12 +210,78 @@ class MemoryQualityManager:
         return list(merged.values())
 
     def detect_conflicts(self, cands: List[MemoryCandidate]) -> List[MemoryCandidate]:
-        # Lightweight local heuristic; LLM conflict classifiers can replace it.
-        for i, c in enumerate(cands):
-            if "not" in c.content.lower() or "반대" in c.content:
+        """Flag local contradiction signals before proactive synthesis.
+
+        The heuristic stays deterministic/offline: direct negation marks a row
+        as risky, and pairwise token overlap catches common preference conflicts
+        such as "prefers light mode" vs "does not like light mode".
+        """
+        for c in cands:
+            lowered = c.content.lower()
+            if any(pattern in lowered for pattern in self._NEGATION_PATTERNS):
                 c.conflicts.append("conflict:possible_negation")
+
+        signatures = [(c, self._content_signature(c.content)) for c in cands]
+        for left_index, (left, left_sig) in enumerate(signatures):
+            for right, right_sig in signatures[left_index + 1:]:
+                if not left_sig or len(left_sig & right_sig) < 2:
+                    continue
+                left_negative = self._is_negative(left.content)
+                right_negative = self._is_negative(right.content)
+                left_positive = self._is_positive(left.content)
+                right_positive = self._is_positive(right.content)
+                if left_negative == right_negative:
+                    continue
+                if not (left_positive or right_positive):
+                    continue
+                left.conflicts.append(f"conflict:contradicts:{right.id}")
+                right.conflicts.append(f"conflict:contradicts:{left.id}")
         return cands
 
+    def _is_negative(self, content: str) -> bool:
+        lowered = content.lower()
+        return any(pattern in lowered for pattern in self._NEGATION_PATTERNS)
+
+    def _is_positive(self, content: str) -> bool:
+        lowered = content.lower()
+        return any(pattern in lowered for pattern in self._POSITIVE_PATTERNS)
+
+    def _content_signature(self, content: str) -> set[str]:
+        stopwords = {
+            "a", "an", "and", "for", "i", "is", "it", "mode", "the", "to",
+            "user", "users", "does", "do", "not", "like", "likes", "prefer",
+            "prefers", "want", "wants",
+        }
+        tokens = set(re.findall(r"\w+", content.lower()))
+        return {token for token in tokens if len(token) > 2 and token not in stopwords}
+
+    # --- Large candidate #4 slice: proactive / temporal contradiction detection ---
+    def detect_temporal_contradictions(self, memories: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+        """Simple temporal + contradiction detector over memory list (proactive synthesis extension).
+
+        Looks at negation keywords across similar contents + differing timestamps.
+        Real version would walk graph historical projections + LLM judge.
+        Returns flagged items with 'contradiction' tag.
+        """
+        flagged = []
+        neg_tokens = ("not ", "반대", "거짓", "no longer", "never", "중단", "사용하지")
+        pos_group = []
+        neg_group = []
+        for m in memories:
+            c = (m.get("content") or "").lower()
+            if any(t in c for t in neg_tokens):
+                neg_group.append(m)
+            else:
+                pos_group.append(m)
+        # cross check: if both pos and neg exist with time spread
+        all_t = [m.get("timestamp") or m.get("created_at") or 0 for m in memories]
+        if neg_group and pos_group and len(set(all_t)) > 1:
+            for g in (pos_group + neg_group):
+                gg = dict(g)
+                gg["proactive_flag"] = "contradiction:temporal_negation"
+                flagged.append(gg)
+        return flagged
+
     def apply_retention(self, cands: List[MemoryCandidate], max_age_days: int = 90) -> List[MemoryCandidate]:
         now = time.time()
         return [c for c in cands if (now - c.timestamp) < max_age_days * 86400]

package/lattice_brain/runtime/agent_runtime.py

@@ -65,6 +65,10 @@ def _now() -> str:
     return datetime.now().isoformat(timespec="seconds")
 
 
+def _compact_text(value: Any, *, limit: int) -> str:
+    return " ".join(str(value or "").split())[:limit]
+
+
 class AgentRuntimeUnavailable(RuntimeError):
     """Raised when a product run would otherwise persist simulation output."""
 
@@ -80,6 +84,8 @@ class AgentRuntime:
         max_retries_cap: int = 5,
         hooks: Any = None,
         allow_simulation_runs: bool = False,
+        memory_ingest: Optional[Callable[..., Dict[str, Any]]] = None,
+        review_sink: Any = None,
     ):
         self._store = store
         self._orchestrator_factory = orchestrator_factory
@@ -91,6 +97,11 @@ class AgentRuntime:
         self._hooks = hooks
         self._allow_simulation_runs = bool(allow_simulation_runs)
         self._run_executor: Any = None
+        # Optional memory synthesis: successful agent runs produce durable
+        # Brain memories (long_term / workspace tier) so users *feel* the results
+        # in BrainBrief, MemoryRings, search and graph immediately.
+        self._memory_ingest = memory_ingest
+        self._review_sink = review_sink
 
     def attach_executor(self, executor: Any) -> None:
         self._run_executor = executor
@@ -415,6 +426,155 @@ class AgentRuntime:
             "current_role": None,
         }
 
+    def _synthesize_brain_memory(self, *, goal: str, result: Any, user_email: Optional[str], scope: Optional[str]) -> None:
+        """Turn a successful agent run into durable Brain memory + graph nodes.
+
+        This is the key to users *strongly feeling* the scale of agent work:
+        delegated goals don't disappear into Act tab; they become part of the
+        Living Brain (searchable, visible in rings/brief, connected in graph).
+        """
+        if not self._memory_ingest:
+            return
+        if getattr(result, "status", None) not in ("ok", "retried_ok"):
+            return
+        try:
+            output = _compact_text(getattr(result, "output", ""), limit=2200)
+            plan_steps = getattr(result, "plan", None) or []
+            sections = self._agent_synthesis_sections(goal=goal, output=output, plan_steps=plan_steps, result=result)
+            plan_summary = "; ".join(sections["plan_steps"][:4])
+            content = "\n\n".join([
+                f"[Agent synthesis] Goal: {_compact_text(goal, limit=240)}",
+                f"Outcome: {output}",
+                self._format_synthesis_section("Key facts", sections["facts"]),
+                self._format_synthesis_section("Decisions", sections["decisions"]),
+                self._format_synthesis_section("Follow-ups", sections["followups"]),
+                f"Plan: {plan_summary}" if plan_summary else "",
+            ]).strip()
+            if not content or len(content) < 20:
+                return
+            tags = ["agent-synthesis", "delegated", "auto"]
+            self._memory_ingest(
+                kind="long_term",
+                content=content,
+                user_email=user_email,
+                tags=tags,
+                metadata={
+                    "source": "agent_runtime",
+                    "synthesis_version": 2,
+                    "goal": _compact_text(goal, limit=200),
+                    "roles": getattr(result, "roles_run", None),
+                    "facts": sections["facts"],
+                    "decisions": sections["decisions"],
+                    "followups": sections["followups"],
+                },
+                graph=self._workspace_graph() if hasattr(self, "_workspace_graph") else None,
+                workspace_id=scope,
+            )
+            if sections["decisions"]:
+                self._memory_ingest(
+                    kind="decisions",
+                    content=f"Agent decision for {_compact_text(goal, limit=120)}: {'; '.join(sections['decisions'][:3])}",
+                    user_email=user_email,
+                    tags=["agent", "outcome", "decision"],
+                    metadata={"source": "agent_runtime_synthesis", "synthesis_version": 2, "goal": _compact_text(goal, limit=200)},
+                    workspace_id=scope,
+                )
+            if sections["followups"]:
+                self._memory_ingest(
+                    kind="workspace",
+                    content=f"Agent follow-ups for {_compact_text(goal, limit=120)}: {'; '.join(sections['followups'][:5])}",
+                    user_email=user_email,
+                    tags=["agent", "follow-up", "next-action"],
+                    metadata={"source": "agent_runtime_followups", "synthesis_version": 2, "goal": _compact_text(goal, limit=200)},
+                    workspace_id=scope,
+                )
+                self._enqueue_agent_followups(
+                    goal=goal,
+                    followups=sections["followups"],
+                    result=result,
+                    user_email=user_email,
+                    scope=scope,
+                    output=output,
+                )
+        except Exception:
+            # Synthesis must never break the run record.
+            pass
+
+    @staticmethod
+    def _format_synthesis_section(title: str, items: List[str]) -> str:
+        if not items:
+            return ""
+        return f"{title}:\n" + "\n".join(f"- {item}" for item in items[:5])
+
+    @staticmethod
+    def _agent_synthesis_sections(*, goal: str, output: str, plan_steps: List[Dict[str, Any]], result: Any) -> Dict[str, List[str]]:
+        plan_descriptions = [
+            _compact_text(step.get("description") or step.get("name") or step.get("id"), limit=120)
+            for step in plan_steps
+            if isinstance(step, dict) and (step.get("description") or step.get("name") or step.get("id"))
+        ]
+        sentences = [
+            sentence.strip(" -•\t")
+            for sentence in output.replace("\n", ". ").split(".")
+            if sentence.strip(" -•\t")
+        ]
+        facts = [_compact_text(sentence, limit=180) for sentence in sentences[:4]]
+        review = getattr(result, "review", None) if isinstance(getattr(result, "review", None), dict) else {}
+        plan_review = getattr(result, "plan_review", None) if isinstance(getattr(result, "plan_review", None), dict) else {}
+        decision_seed = review.get("decision") or review.get("status") or plan_review.get("decision") or getattr(result, "status", "")
+        decisions = [
+            _compact_text(f"Run finished with {decision_seed}", limit=160),
+            _compact_text(f"Goal accepted: {goal}", limit=180),
+        ]
+        followups = [
+            item for item in plan_descriptions
+            if any(token in item.lower() for token in ("next", "follow", "review", "verify", "test", "ship", "publish", "document", "implement"))
+        ][:5]
+        if not followups:
+            followups = plan_descriptions[:3]
+        return {
+            "facts": [item for item in facts if item],
+            "decisions": [item for item in decisions if item],
+            "followups": [item for item in followups if item],
+            "plan_steps": plan_descriptions,
+        }
+
+    def _enqueue_agent_followups(
+        self,
+        *,
+        goal: str,
+        followups: List[str],
+        result: Any,
+        user_email: Optional[str],
+        scope: Optional[str],
+        output: str,
+    ) -> None:
+        if self._review_sink is None:
+            return
+        for index, followup in enumerate(followups[:5], start=1):
+            try:
+                self._review_sink.create(
+                    title=_compact_text(followup, limit=96) or f"Agent follow-up {index}",
+                    summary=_compact_text(f"From goal: {goal}", limit=420),
+                    source="agent_followup",
+                    kind="task_draft",
+                    payload={
+                        "goal": _compact_text(goal, limit=300),
+                        "followup": _compact_text(followup, limit=300),
+                        "output_preview": _compact_text(output, limit=800),
+                        "roles": getattr(result, "roles_run", None),
+                    },
+                    provenance={
+                        "agent_id": getattr(result, "agent_id", ""),
+                        "source_detail": "agent_runtime_followup",
+                        "status": getattr(result, "status", ""),
+                    },
+                    user_email=user_email,
+                    workspace_id=scope,
+                )
+            except Exception:
+                continue
+
     def _post_run_hooks(
         self,
         *,
@@ -572,6 +732,9 @@ class AgentRuntime:
             status=updated.get("status") or result.status,
             retries=result.retries,
         )
+        # Large-scale user-visible impact: successful runs enrich the Brain permanently.
+        if (updated.get("status") or result.status) in ("ok", "retried_ok"):
+            self._synthesize_brain_memory(goal=goal, result=result, user_email=user_email, scope=scope)
         post_dispatch = self._post_run_hooks(
             run_id=run_id,
             result=result,
@@ -651,6 +814,9 @@ class AgentRuntime:
             status=result.status,
             retries=result.retries,
         )
+        # Large-scale user-visible impact: successful runs enrich the Brain permanently.
+        if result.status in ("ok", "retried_ok"):
+            self._synthesize_brain_memory(goal=goal, result=result, user_email=user_email, scope=scope)
 
         run_id = run.get("id") or run.get("run_id") if isinstance(run, dict) else None
         post_dispatch = self._post_run_hooks(

package/lattice_brain/runtime/multi_agent.py

@@ -21,7 +21,7 @@ from typing import Any, Callable, Dict, List, Optional
 from .contracts import multi_agent_contract
 
 
-MULTI_AGENT_VERSION = "8.9.0"
+MULTI_AGENT_VERSION = "9.0.0"
 
 AGENT_ROLES = ("researcher", "planner", "executor", "reviewer", "release")
 CORE_PIPELINE = ("planner", "executor", "reviewer")

package/lattice_brain/utils.py

@@ -0,0 +1,28 @@
+"""Shared Brain Core utility helpers."""
+
+from __future__ import annotations
+
+import hashlib
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+
+def parse_iso(value: Optional[str]) -> Optional[datetime]:
+    if not value:
+        return None
+    try:
+        return datetime.fromisoformat(str(value))
+    except (TypeError, ValueError):
+        return None
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with open(path, "rb") as fh:
+        for block in iter(lambda: fh.read(65536), b""):
+            digest.update(block)
+    return digest.hexdigest()
+
+
+__all__ = ["parse_iso", "sha256_file"]

package/latticeai/__init__.py

@@ -1,3 +1,3 @@
 """Lattice AI - modular server package."""
 
-__version__ = "8.9.0"
+__version__ = "9.0.0"