ltcai 4.3.1 → 4.4.0

package/latticeai/brain/context.py

@@ -1,213 +1,17 @@
-"""Context System — one budgeted, provenance-carrying assembly pipeline.
+"""Deprecated shim: physically moved to lattice_brain.context.
 
-Replaces the ad-hoc string concatenation that built chat context (language
-hint + vault scan + KG LIKE search + recent chat appended in fixed order
-with no size control). Every section the assembler emits records WHY it is
-in the prompt (source, ids, scores), so "why is this in my context?" is
-answerable, and the whole assembly respects a token budget.
-
-Token counts are an explicit approximation: ``approx_tokens = ceil(len/4)``
-(the stack ships no model-agnostic tokenizer; the field name says so —
-design-review amendment T5).
+Kept only for the compatibility window. The module aliases itself to the
+physical module so identity, singletons, and monkeypatching are preserved.
 """
 
-from __future__ import annotations
-
-import logging
-from dataclasses import dataclass, field
-from typing import Any, Callable, Dict, List, Optional
-
-
-def approx_tokens(text: str) -> int:
-    """Documented chars/4 approximation — NOT a real tokenizer count."""
-    return max(0, (len(text or "") + 3) // 4)
-
-
-@dataclass
-class ContextSection:
-    name: str
-    content: str
-    source: str                      # memory | knowledge | notes | recent_chat | system
-    provenance: List[Dict[str, Any]] = field(default_factory=list)
-    truncated: bool = False
-
-    @property
-    def approx_tokens(self) -> int:
-        return approx_tokens(self.content)
-
-    def as_trace(self) -> Dict[str, Any]:
-        return {
-            "name": self.name,
-            "source": self.source,
-            "approx_tokens": self.approx_tokens,
-            "truncated": self.truncated,
-            "provenance": self.provenance,
-        }
-
-
-@dataclass
-class AssembledContext:
-    sections: List[ContextSection]
-    budget_approx_tokens: int
-
-    @property
-    def text(self) -> str:
-        parts = []
-        for section in self.sections:
-            if section.content.strip():
-                parts.append(f"[{section.name}]\n{section.content.strip()}")
-        return "\n\n".join(parts)
-
-    @property
-    def approx_tokens(self) -> int:
-        return sum(s.approx_tokens for s in self.sections)
-
-    def trace(self) -> Dict[str, Any]:
-        return {
-            "budget_approx_tokens": self.budget_approx_tokens,
-            "used_approx_tokens": self.approx_tokens,
-            "sections": [s.as_trace() for s in self.sections],
-        }
-
-
-class ContextAssembler:
-    """Ordered, budgeted context assembly over injected retrieval seams.
-
-    Section priority (kept under budget in this order — semantic memories
-    first because they are cheap and durable; recency last):
-      1. memories  — workspace semantic memories (preferences/decisions/…)
-      2. knowledge — hybrid search over the brain (the product's own engine)
-      3. notes     — garden-note context
-      4. recent    — the user's recent exchange
-    Every seam is optional; an absent seam contributes nothing (honest
-    absence), never a fabricated section.
-    """
-
-    def __init__(
-        self,
-        *,
-        memory_recall: Optional[Callable[..., Dict[str, Any]]] = None,
-        hybrid_search: Optional[Callable[..., Dict[str, Any]]] = None,
-        notes_context: Optional[Callable[..., str]] = None,
-        recent_chat: Optional[Callable[..., str]] = None,
-    ) -> None:
-        self._memory_recall = memory_recall
-        self._hybrid_search = hybrid_search
-        self._notes_context = notes_context
-        self._recent_chat = recent_chat
-
-    def assemble(
-        self,
-        query: str,
-        *,
-        user_email: Optional[str] = None,
-        workspace_id: Optional[str] = None,
-        conversation_id: Optional[str] = None,
-        budget: int = 2000,
-        memory_limit: int = 5,
-        knowledge_limit: int = 5,
-    ) -> AssembledContext:
-        sections: List[ContextSection] = []
-
-        if self._memory_recall is not None:
-            sections.append(self._memories_section(query, user_email, workspace_id, memory_limit))
-        if self._hybrid_search is not None:
-            sections.append(self._knowledge_section(query, knowledge_limit, user_email))
-        if self._notes_context is not None:
-            sections.append(self._notes_section(query))
-        if self._recent_chat is not None:
-            sections.append(self._recent_section(user_email, conversation_id))
-
-        sections = [s for s in sections if s.content.strip()]
-        self._apply_budget(sections, budget)
-        return AssembledContext(sections=sections, budget_approx_tokens=budget)
-
-    # ── section builders (each failure-isolated and honest) ───────────────
-    def _memories_section(self, query, user_email, workspace_id, limit) -> ContextSection:
-        try:
-            recall = self._memory_recall(query, user_email=user_email, workspace_id=workspace_id, limit=limit)
-            results = [r for r in recall.get("results", []) if r.get("source") == "workspace"][:limit]
-        except Exception as exc:
-            logging.debug("context: memory recall failed: %s", exc)
-            results = []
-        lines = [f"- ({r.get('kind') or 'memory'}) {r.get('snippet') or ''}" for r in results]
-        return ContextSection(
-            name="User memories",
-            content="\n".join(lines),
-            source="memory",
-            provenance=[
-                {"id": r.get("id"), "kind": r.get("kind"), "score": r.get("score")}
-                for r in results
-            ],
-        )
-
-    def _knowledge_section(self, query, limit, user_email=None) -> ContextSection:
-        try:
-            hybrid = self._hybrid_search(query, limit=limit, user_email=user_email)
-            matches = hybrid.get("matches", [])[:limit]
-        except Exception as exc:
-            logging.debug("context: hybrid search failed: %s", exc)
-            matches = []
-        lines = []
-        provenance = []
-        for m in matches:
-            title = m.get("title") or m.get("id") or "item"
-            body = (m.get("summary") or m.get("snippet") or "")[:400]
-            lines.append(f"- {title}: {body}" if body else f"- {title}")
-            provenance.append({
-                "id": m.get("id"),
-                "score": m.get("score"),
-                "sources": m.get("sources"),
-            })
-        return ContextSection(
-            name="Knowledge",
-            content="\n".join(lines),
-            source="knowledge",
-            provenance=provenance,
-        )
-
-    def _notes_section(self, query) -> ContextSection:
-        try:
-            content = self._notes_context(query) or ""
-        except Exception as exc:
-            logging.debug("context: notes context failed: %s", exc)
-            content = ""
-        return ContextSection(
-            name="Garden notes",
-            content=content,
-            source="notes",
-            provenance=[{"source": "garden", "included": bool(content)}],
-        )
-
-    def _recent_section(self, user_email, conversation_id) -> ContextSection:
-        try:
-            content = self._recent_chat(user_email=user_email, conversation_id=conversation_id) or ""
-        except Exception as exc:
-            logging.debug("context: recent chat failed: %s", exc)
-            content = ""
-        return ContextSection(
-            name="Recent conversation",
-            content=content,
-            source="recent_chat",
-            provenance=[{"conversation_id": conversation_id, "user_email": user_email}],
-        )
-
-    # ── budget ─────────────────────────────────────────────────────────────
-    @staticmethod
-    def _apply_budget(sections: List[ContextSection], budget: int) -> None:
-        """Trim from the END (lowest priority) until under budget."""
-        budget = max(1, int(budget))
-        used = 0
-        for section in sections:
-            remaining = budget - used
-            if remaining <= 0:
-                section.content = ""
-                section.truncated = True
-                continue
-            if section.approx_tokens > remaining:
-                section.content = section.content[: remaining * 4]
-                section.truncated = True
-            used += section.approx_tokens
+import sys
+import warnings
 
+import lattice_brain.context as _impl
 
-__all__ = ["ContextAssembler", "ContextSection", "AssembledContext", "approx_tokens"]
+warnings.warn(
+    "latticeai.brain.context is deprecated; import lattice_brain.context instead",
+    DeprecationWarning,
+    stacklevel=2,
+)
+sys.modules[__name__] = _impl

package/latticeai/brain/conversations.py

@@ -1,236 +1,17 @@
-"""Durable conversation store — kills the 50-message chat_history.json cap.
+"""Deprecated shim: physically moved to lattice_brain.conversations.
 
-Conversations are episodic memory, the most valuable raw input a Digital
-Brain has; truncating them contradicted "knowledge is durable". This store
-keeps every message in SQLite — by default in the same database file as the
-knowledge graph, so the existing kg_portability backup/restore covers
-conversations with no manifest changes.
-
-The public item shape is exactly the legacy chat_history.json entry
-(role/content/timestamp + optional user_email/user_nickname/source/
-conversation_id), so every existing consumer of ``get_history()`` keeps
-working. Legacy history is imported once, idempotently (content-hash dedup).
+Kept only for the compatibility window. The module aliases itself to the
+physical module so identity, singletons, and monkeypatching are preserved.
 """
 
-from __future__ import annotations
-
-import hashlib
-import json
-import logging
-import sqlite3
-import threading
-from pathlib import Path
-from typing import Any, Dict, List, Optional
-
-
-def _message_hash(item: Dict[str, Any]) -> str:
-    basis = "|".join(
-        str(item.get(key) or "")
-        for key in ("role", "content", "timestamp", "user_email", "conversation_id", "source")
-    )
-    return hashlib.sha256(basis.encode("utf-8", "ignore")).hexdigest()
-
-
-class ConversationStore:
-    """Unbounded, per-conversation chat history in SQLite."""
-
-    def __init__(self, db_path: Path):
-        self.db_path = Path(db_path)
-        self.db_path.parent.mkdir(parents=True, exist_ok=True)
-        self._lock = threading.RLock()
-        self._init_db()
-
-    def _connect(self) -> sqlite3.Connection:
-        conn = sqlite3.connect(str(self.db_path))
-        conn.row_factory = sqlite3.Row
-        conn.execute("PRAGMA journal_mode=WAL")
-        return conn
-
-    def _init_db(self) -> None:
-        with self._lock, self._connect() as conn:
-            conn.executescript(
-                """
-                CREATE TABLE IF NOT EXISTS conversation_messages (
-                  id INTEGER PRIMARY KEY AUTOINCREMENT,
-                  message_hash TEXT NOT NULL UNIQUE,
-                  conversation_id TEXT,
-                  role TEXT NOT NULL,
-                  content TEXT NOT NULL,
-                  user_email TEXT,
-                  user_nickname TEXT,
-                  source TEXT,
-                  timestamp TEXT NOT NULL,
-                  metadata_json TEXT NOT NULL DEFAULT '{}'
-                );
-                CREATE INDEX IF NOT EXISTS idx_conv_messages_conv
-                  ON conversation_messages(conversation_id);
-                CREATE INDEX IF NOT EXISTS idx_conv_messages_time
-                  ON conversation_messages(timestamp);
-                """
-            )
-
-    # ── writes ────────────────────────────────────────────────────────────
-    def append(self, item: Dict[str, Any]) -> Dict[str, Any]:
-        """Persist one chat item (the legacy chat_history.json entry shape)."""
-        known = {"role", "content", "timestamp", "user_email", "user_nickname", "source", "conversation_id"}
-        extra = {k: v for k, v in item.items() if k not in known}
-        with self._lock, self._connect() as conn:
-            conn.execute(
-                """
-                INSERT OR IGNORE INTO conversation_messages
-                  (message_hash, conversation_id, role, content, user_email,
-                   user_nickname, source, timestamp, metadata_json)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-                """,
-                (
-                    _message_hash(item),
-                    item.get("conversation_id"),
-                    str(item.get("role") or "user"),
-                    str(item.get("content") or ""),
-                    item.get("user_email"),
-                    item.get("user_nickname"),
-                    item.get("source"),
-                    str(item.get("timestamp") or ""),
-                    json.dumps(extra, ensure_ascii=False) if extra else "{}",
-                ),
-            )
-        return item
-
-    def import_legacy_json(self, history_file: Path) -> int:
-        """One-time, idempotent import of a chat_history.json file.
-
-        Re-running never duplicates (message_hash UNIQUE + INSERT OR IGNORE);
-        the source file is left untouched on disk.
-        """
-        path = Path(history_file)
-        if not path.exists():
-            return 0
-        try:
-            with open(path, "r", encoding="utf-8") as fh:
-                items = json.load(fh)
-        except Exception as exc:
-            logging.warning("conversation store: legacy import failed to read %s: %s", path, exc)
-            return 0
-        if not isinstance(items, list):
-            return 0
-        imported = 0
-        with self._lock, self._connect() as conn:
-            for item in items:
-                if not isinstance(item, dict):
-                    continue
-                cur = conn.execute(
-                    """
-                    INSERT OR IGNORE INTO conversation_messages
-                      (message_hash, conversation_id, role, content, user_email,
-                       user_nickname, source, timestamp, metadata_json)
-                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, '{}')
-                    """,
-                    (
-                        _message_hash(item),
-                        item.get("conversation_id"),
-                        str(item.get("role") or "user"),
-                        str(item.get("content") or ""),
-                        item.get("user_email"),
-                        item.get("user_nickname"),
-                        item.get("source"),
-                        str(item.get("timestamp") or ""),
-                    ),
-                )
-                imported += cur.rowcount if cur.rowcount > 0 else 0
-        if imported:
-            logging.info("conversation store: imported %d legacy chat messages from %s", imported, path)
-        return imported
-
-    # ── reads (legacy item shape) ─────────────────────────────────────────
-    @staticmethod
-    def _row_to_item(row: sqlite3.Row) -> Dict[str, Any]:
-        item: Dict[str, Any] = {
-            "role": row["role"],
-            "content": row["content"],
-            "timestamp": row["timestamp"],
-        }
-        for key in ("user_email", "user_nickname", "source", "conversation_id"):
-            if row[key]:
-                item[key] = row[key]
-        try:
-            extra = json.loads(row["metadata_json"] or "{}")
-        except Exception:
-            extra = {}
-        item.update(extra)
-        return item
-
-    def history(self, *, conversation_id: Optional[str] = None, limit: Optional[int] = None) -> List[Dict[str, Any]]:
-        """Chronological items; the unbounded successor of get_history()."""
-        query = "SELECT * FROM conversation_messages"
-        params: List[Any] = []
-        if conversation_id is not None:
-            query += " WHERE conversation_id IS ?" if conversation_id == "" else " WHERE conversation_id = ?"
-            params.append(None if conversation_id == "" else conversation_id)
-        query += " ORDER BY id ASC"
-        if limit is not None:
-            query += " LIMIT ?"
-            params.append(max(1, int(limit)))
-        with self._connect() as conn:
-            rows = conn.execute(query, params).fetchall()
-        return [self._row_to_item(row) for row in rows]
-
-    def count(self) -> int:
-        with self._connect() as conn:
-            return conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
-
-    def size_bytes(self) -> int:
-        try:
-            return self.db_path.stat().st_size if self.db_path.exists() else 0
-        except OSError:
-            return 0
-
-    # ── clears (legacy semantics preserved) ───────────────────────────────
-    def clear_all(self, keep_last: int = 0) -> Dict[str, Any]:
-        keep_last = max(0, min(int(keep_last or 0), 20))
-        with self._lock, self._connect() as conn:
-            total = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
-            if keep_last:
-                conn.execute(
-                    """
-                    DELETE FROM conversation_messages WHERE id NOT IN (
-                      SELECT id FROM conversation_messages ORDER BY id DESC LIMIT ?
-                    )
-                    """,
-                    (keep_last,),
-                )
-            else:
-                conn.execute("DELETE FROM conversation_messages")
-            kept = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
-        return {"status": "cleared", "removed": max(0, total - kept), "kept": kept}
-
-    def clear_conversation(self, conversation_id: str, started_at: Optional[str] = None) -> Dict[str, Any]:
-        """Remove one conversation.
-
-        ``legacy-previous-history`` targets unattributed messages; when
-        ``started_at`` is given, unattributed messages from that point on are
-        removed too (mirrors the pre-v4 JSON behaviour exactly).
-        """
-        with self._lock, self._connect() as conn:
-            total = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
-            if conversation_id == "legacy-previous-history":
-                conn.execute("DELETE FROM conversation_messages WHERE conversation_id IS NULL")
-            else:
-                conn.execute(
-                    "DELETE FROM conversation_messages WHERE conversation_id = ?",
-                    (conversation_id,),
-                )
-                if started_at:
-                    conn.execute(
-                        "DELETE FROM conversation_messages WHERE conversation_id IS NULL AND timestamp >= ?",
-                        (str(started_at),),
-                    )
-            kept = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
-        return {
-            "status": "cleared",
-            "conversation_id": conversation_id,
-            "removed": max(0, total - kept),
-            "kept": kept,
-        }
+import sys
+import warnings
 
+import lattice_brain.conversations as _impl
 
-__all__ = ["ConversationStore"]
+warnings.warn(
+    "latticeai.brain.conversations is deprecated; import lattice_brain.conversations instead",
+    DeprecationWarning,
+    stacklevel=2,
+)
+sys.modules[__name__] = _impl