ltcai 4.3.1 → 4.4.0

package/lattice_brain/context.py

@@ -1,3 +1,213 @@
-from latticeai.brain.context import AssembledContext, ContextAssembler, ContextSection, approx_tokens
+"""Context System — one budgeted, provenance-carrying assembly pipeline.
 
-__all__ = ["AssembledContext", "ContextAssembler", "ContextSection", "approx_tokens"]
+Replaces the ad-hoc string concatenation that built chat context (language
+hint + vault scan + KG LIKE search + recent chat appended in fixed order
+with no size control). Every section the assembler emits records WHY it is
+in the prompt (source, ids, scores), so "why is this in my context?" is
+answerable, and the whole assembly respects a token budget.
+
+Token counts are an explicit approximation: ``approx_tokens = ceil(len/4)``
+(the stack ships no model-agnostic tokenizer; the field name says so —
+design-review amendment T5).
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+def approx_tokens(text: str) -> int:
+    """Documented chars/4 approximation — NOT a real tokenizer count."""
+    return max(0, (len(text or "") + 3) // 4)
+
+
+@dataclass
+class ContextSection:
+    name: str
+    content: str
+    source: str                      # memory | knowledge | notes | recent_chat | system
+    provenance: List[Dict[str, Any]] = field(default_factory=list)
+    truncated: bool = False
+
+    @property
+    def approx_tokens(self) -> int:
+        return approx_tokens(self.content)
+
+    def as_trace(self) -> Dict[str, Any]:
+        return {
+            "name": self.name,
+            "source": self.source,
+            "approx_tokens": self.approx_tokens,
+            "truncated": self.truncated,
+            "provenance": self.provenance,
+        }
+
+
+@dataclass
+class AssembledContext:
+    sections: List[ContextSection]
+    budget_approx_tokens: int
+
+    @property
+    def text(self) -> str:
+        parts = []
+        for section in self.sections:
+            if section.content.strip():
+                parts.append(f"[{section.name}]\n{section.content.strip()}")
+        return "\n\n".join(parts)
+
+    @property
+    def approx_tokens(self) -> int:
+        return sum(s.approx_tokens for s in self.sections)
+
+    def trace(self) -> Dict[str, Any]:
+        return {
+            "budget_approx_tokens": self.budget_approx_tokens,
+            "used_approx_tokens": self.approx_tokens,
+            "sections": [s.as_trace() for s in self.sections],
+        }
+
+
+class ContextAssembler:
+    """Ordered, budgeted context assembly over injected retrieval seams.
+
+    Section priority (kept under budget in this order — semantic memories
+    first because they are cheap and durable; recency last):
+      1. memories  — workspace semantic memories (preferences/decisions/…)
+      2. knowledge — hybrid search over the brain (the product's own engine)
+      3. notes     — garden-note context
+      4. recent    — the user's recent exchange
+    Every seam is optional; an absent seam contributes nothing (honest
+    absence), never a fabricated section.
+    """
+
+    def __init__(
+        self,
+        *,
+        memory_recall: Optional[Callable[..., Dict[str, Any]]] = None,
+        hybrid_search: Optional[Callable[..., Dict[str, Any]]] = None,
+        notes_context: Optional[Callable[..., str]] = None,
+        recent_chat: Optional[Callable[..., str]] = None,
+    ) -> None:
+        self._memory_recall = memory_recall
+        self._hybrid_search = hybrid_search
+        self._notes_context = notes_context
+        self._recent_chat = recent_chat
+
+    def assemble(
+        self,
+        query: str,
+        *,
+        user_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        conversation_id: Optional[str] = None,
+        budget: int = 2000,
+        memory_limit: int = 5,
+        knowledge_limit: int = 5,
+    ) -> AssembledContext:
+        sections: List[ContextSection] = []
+
+        if self._memory_recall is not None:
+            sections.append(self._memories_section(query, user_email, workspace_id, memory_limit))
+        if self._hybrid_search is not None:
+            sections.append(self._knowledge_section(query, knowledge_limit, user_email))
+        if self._notes_context is not None:
+            sections.append(self._notes_section(query))
+        if self._recent_chat is not None:
+            sections.append(self._recent_section(user_email, conversation_id))
+
+        sections = [s for s in sections if s.content.strip()]
+        self._apply_budget(sections, budget)
+        return AssembledContext(sections=sections, budget_approx_tokens=budget)
+
+    # ── section builders (each failure-isolated and honest) ───────────────
+    def _memories_section(self, query, user_email, workspace_id, limit) -> ContextSection:
+        try:
+            recall = self._memory_recall(query, user_email=user_email, workspace_id=workspace_id, limit=limit)
+            results = [r for r in recall.get("results", []) if r.get("source") == "workspace"][:limit]
+        except Exception as exc:
+            logging.debug("context: memory recall failed: %s", exc)
+            results = []
+        lines = [f"- ({r.get('kind') or 'memory'}) {r.get('snippet') or ''}" for r in results]
+        return ContextSection(
+            name="User memories",
+            content="\n".join(lines),
+            source="memory",
+            provenance=[
+                {"id": r.get("id"), "kind": r.get("kind"), "score": r.get("score")}
+                for r in results
+            ],
+        )
+
+    def _knowledge_section(self, query, limit, user_email=None) -> ContextSection:
+        try:
+            hybrid = self._hybrid_search(query, limit=limit, user_email=user_email)
+            matches = hybrid.get("matches", [])[:limit]
+        except Exception as exc:
+            logging.debug("context: hybrid search failed: %s", exc)
+            matches = []
+        lines = []
+        provenance = []
+        for m in matches:
+            title = m.get("title") or m.get("id") or "item"
+            body = (m.get("summary") or m.get("snippet") or "")[:400]
+            lines.append(f"- {title}: {body}" if body else f"- {title}")
+            provenance.append({
+                "id": m.get("id"),
+                "score": m.get("score"),
+                "sources": m.get("sources"),
+            })
+        return ContextSection(
+            name="Knowledge",
+            content="\n".join(lines),
+            source="knowledge",
+            provenance=provenance,
+        )
+
+    def _notes_section(self, query) -> ContextSection:
+        try:
+            content = self._notes_context(query) or ""
+        except Exception as exc:
+            logging.debug("context: notes context failed: %s", exc)
+            content = ""
+        return ContextSection(
+            name="Garden notes",
+            content=content,
+            source="notes",
+            provenance=[{"source": "garden", "included": bool(content)}],
+        )
+
+    def _recent_section(self, user_email, conversation_id) -> ContextSection:
+        try:
+            content = self._recent_chat(user_email=user_email, conversation_id=conversation_id) or ""
+        except Exception as exc:
+            logging.debug("context: recent chat failed: %s", exc)
+            content = ""
+        return ContextSection(
+            name="Recent conversation",
+            content=content,
+            source="recent_chat",
+            provenance=[{"conversation_id": conversation_id, "user_email": user_email}],
+        )
+
+    # ── budget ─────────────────────────────────────────────────────────────
+    @staticmethod
+    def _apply_budget(sections: List[ContextSection], budget: int) -> None:
+        """Trim from the END (lowest priority) until under budget."""
+        budget = max(1, int(budget))
+        used = 0
+        for section in sections:
+            remaining = budget - used
+            if remaining <= 0:
+                section.content = ""
+                section.truncated = True
+                continue
+            if section.approx_tokens > remaining:
+                section.content = section.content[: remaining * 4]
+                section.truncated = True
+            used += section.approx_tokens
+
+
+__all__ = ["ContextAssembler", "ContextSection", "AssembledContext", "approx_tokens"]

package/lattice_brain/conversations.py

@@ -1,3 +1,236 @@
-from latticeai.brain.conversations import ConversationStore
+"""Durable conversation store — kills the 50-message chat_history.json cap.
+
+Conversations are episodic memory, the most valuable raw input a Digital
+Brain has; truncating them contradicted "knowledge is durable". This store
+keeps every message in SQLite — by default in the same database file as the
+knowledge graph, so the existing kg_portability backup/restore covers
+conversations with no manifest changes.
+
+The public item shape is exactly the legacy chat_history.json entry
+(role/content/timestamp + optional user_email/user_nickname/source/
+conversation_id), so every existing consumer of ``get_history()`` keeps
+working. Legacy history is imported once, idempotently (content-hash dedup).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import sqlite3
+import threading
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+def _message_hash(item: Dict[str, Any]) -> str:
+    basis = "|".join(
+        str(item.get(key) or "")
+        for key in ("role", "content", "timestamp", "user_email", "conversation_id", "source")
+    )
+    return hashlib.sha256(basis.encode("utf-8", "ignore")).hexdigest()
+
+
+class ConversationStore:
+    """Unbounded, per-conversation chat history in SQLite."""
+
+    def __init__(self, db_path: Path):
+        self.db_path = Path(db_path)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.RLock()
+        self._init_db()
+
+    def _connect(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(str(self.db_path))
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA journal_mode=WAL")
+        return conn
+
+    def _init_db(self) -> None:
+        with self._lock, self._connect() as conn:
+            conn.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS conversation_messages (
+                  id INTEGER PRIMARY KEY AUTOINCREMENT,
+                  message_hash TEXT NOT NULL UNIQUE,
+                  conversation_id TEXT,
+                  role TEXT NOT NULL,
+                  content TEXT NOT NULL,
+                  user_email TEXT,
+                  user_nickname TEXT,
+                  source TEXT,
+                  timestamp TEXT NOT NULL,
+                  metadata_json TEXT NOT NULL DEFAULT '{}'
+                );
+                CREATE INDEX IF NOT EXISTS idx_conv_messages_conv
+                  ON conversation_messages(conversation_id);
+                CREATE INDEX IF NOT EXISTS idx_conv_messages_time
+                  ON conversation_messages(timestamp);
+                """
+            )
+
+    # ── writes ────────────────────────────────────────────────────────────
+    def append(self, item: Dict[str, Any]) -> Dict[str, Any]:
+        """Persist one chat item (the legacy chat_history.json entry shape)."""
+        known = {"role", "content", "timestamp", "user_email", "user_nickname", "source", "conversation_id"}
+        extra = {k: v for k, v in item.items() if k not in known}
+        with self._lock, self._connect() as conn:
+            conn.execute(
+                """
+                INSERT OR IGNORE INTO conversation_messages
+                  (message_hash, conversation_id, role, content, user_email,
+                   user_nickname, source, timestamp, metadata_json)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    _message_hash(item),
+                    item.get("conversation_id"),
+                    str(item.get("role") or "user"),
+                    str(item.get("content") or ""),
+                    item.get("user_email"),
+                    item.get("user_nickname"),
+                    item.get("source"),
+                    str(item.get("timestamp") or ""),
+                    json.dumps(extra, ensure_ascii=False) if extra else "{}",
+                ),
+            )
+        return item
+
+    def import_legacy_json(self, history_file: Path) -> int:
+        """One-time, idempotent import of a chat_history.json file.
+
+        Re-running never duplicates (message_hash UNIQUE + INSERT OR IGNORE);
+        the source file is left untouched on disk.
+        """
+        path = Path(history_file)
+        if not path.exists():
+            return 0
+        try:
+            with open(path, "r", encoding="utf-8") as fh:
+                items = json.load(fh)
+        except Exception as exc:
+            logging.warning("conversation store: legacy import failed to read %s: %s", path, exc)
+            return 0
+        if not isinstance(items, list):
+            return 0
+        imported = 0
+        with self._lock, self._connect() as conn:
+            for item in items:
+                if not isinstance(item, dict):
+                    continue
+                cur = conn.execute(
+                    """
+                    INSERT OR IGNORE INTO conversation_messages
+                      (message_hash, conversation_id, role, content, user_email,
+                       user_nickname, source, timestamp, metadata_json)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, '{}')
+                    """,
+                    (
+                        _message_hash(item),
+                        item.get("conversation_id"),
+                        str(item.get("role") or "user"),
+                        str(item.get("content") or ""),
+                        item.get("user_email"),
+                        item.get("user_nickname"),
+                        item.get("source"),
+                        str(item.get("timestamp") or ""),
+                    ),
+                )
+                imported += cur.rowcount if cur.rowcount > 0 else 0
+        if imported:
+            logging.info("conversation store: imported %d legacy chat messages from %s", imported, path)
+        return imported
+
+    # ── reads (legacy item shape) ─────────────────────────────────────────
+    @staticmethod
+    def _row_to_item(row: sqlite3.Row) -> Dict[str, Any]:
+        item: Dict[str, Any] = {
+            "role": row["role"],
+            "content": row["content"],
+            "timestamp": row["timestamp"],
+        }
+        for key in ("user_email", "user_nickname", "source", "conversation_id"):
+            if row[key]:
+                item[key] = row[key]
+        try:
+            extra = json.loads(row["metadata_json"] or "{}")
+        except Exception:
+            extra = {}
+        item.update(extra)
+        return item
+
+    def history(self, *, conversation_id: Optional[str] = None, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Chronological items; the unbounded successor of get_history()."""
+        query = "SELECT * FROM conversation_messages"
+        params: List[Any] = []
+        if conversation_id is not None:
+            query += " WHERE conversation_id IS ?" if conversation_id == "" else " WHERE conversation_id = ?"
+            params.append(None if conversation_id == "" else conversation_id)
+        query += " ORDER BY id ASC"
+        if limit is not None:
+            query += " LIMIT ?"
+            params.append(max(1, int(limit)))
+        with self._connect() as conn:
+            rows = conn.execute(query, params).fetchall()
+        return [self._row_to_item(row) for row in rows]
+
+    def count(self) -> int:
+        with self._connect() as conn:
+            return conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
+
+    def size_bytes(self) -> int:
+        try:
+            return self.db_path.stat().st_size if self.db_path.exists() else 0
+        except OSError:
+            return 0
+
+    # ── clears (legacy semantics preserved) ───────────────────────────────
+    def clear_all(self, keep_last: int = 0) -> Dict[str, Any]:
+        keep_last = max(0, min(int(keep_last or 0), 20))
+        with self._lock, self._connect() as conn:
+            total = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
+            if keep_last:
+                conn.execute(
+                    """
+                    DELETE FROM conversation_messages WHERE id NOT IN (
+                      SELECT id FROM conversation_messages ORDER BY id DESC LIMIT ?
+                    )
+                    """,
+                    (keep_last,),
+                )
+            else:
+                conn.execute("DELETE FROM conversation_messages")
+            kept = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
+        return {"status": "cleared", "removed": max(0, total - kept), "kept": kept}
+
+    def clear_conversation(self, conversation_id: str, started_at: Optional[str] = None) -> Dict[str, Any]:
+        """Remove one conversation.
+
+        ``legacy-previous-history`` targets unattributed messages; when
+        ``started_at`` is given, unattributed messages from that point on are
+        removed too (mirrors the pre-v4 JSON behaviour exactly).
+        """
+        with self._lock, self._connect() as conn:
+            total = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
+            if conversation_id == "legacy-previous-history":
+                conn.execute("DELETE FROM conversation_messages WHERE conversation_id IS NULL")
+            else:
+                conn.execute(
+                    "DELETE FROM conversation_messages WHERE conversation_id = ?",
+                    (conversation_id,),
+                )
+                if started_at:
+                    conn.execute(
+                        "DELETE FROM conversation_messages WHERE conversation_id IS NULL AND timestamp >= ?",
+                        (str(started_at),),
+                    )
+            kept = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
+        return {
+            "status": "cleared",
+            "conversation_id": conversation_id,
+            "removed": max(0, total - kept),
+            "kept": kept,
+        }
+
 
 __all__ = ["ConversationStore"]

package/lattice_brain/discovery.py

@@ -1 +1,11 @@
-from latticeai.brain.discovery import *  # noqa: F401,F403
+"""Compatibility shim: implementation moved to lattice_brain.graph.discovery.
+
+This module aliases itself to the physical module so identity, singletons,
+and monkeypatching behave as if the old flat path were the real module.
+"""
+
+import sys
+
+from .graph import discovery as _impl
+
+sys.modules[__name__] = _impl

package/lattice_brain/documents.py

@@ -1 +1,11 @@
-from latticeai.brain.documents import *  # noqa: F401,F403
+"""Compatibility shim: implementation moved to lattice_brain.graph.documents.
+
+This module aliases itself to the physical module so identity, singletons,
+and monkeypatching behave as if the old flat path were the real module.
+"""
+
+import sys
+
+from .graph import documents as _impl
+
+sys.modules[__name__] = _impl

package/lattice_brain/graph/__init__.py

@@ -0,0 +1,28 @@
+"""Knowledge graph subsystem of the Brain Core.
+
+Physically hosts the graph schema, store, mixins (write/retrieval/discovery/
+documents/ingest/projection/provenance), device identity, brain network, and
+the graph curator. Heavy modules are lazy-loaded so importing
+``lattice_brain.graph`` stays cheap.
+"""
+
+from __future__ import annotations
+
+__all__ = [
+    "KnowledgeGraphStore",
+    "KGStoreV2",
+    "NodeType",
+    "EdgeType",
+]
+
+
+def __getattr__(name: str):
+    if name == "KnowledgeGraphStore":
+        from .store import KnowledgeGraphStore
+
+        return KnowledgeGraphStore
+    if name in {"KGStoreV2", "NodeType", "EdgeType"}:
+        from . import schema
+
+        return getattr(schema, name)
+    raise AttributeError(name)