ltcai 3.5.0 → 4.0.0

package/latticeai/brain/__init__.py

@@ -0,0 +1,18 @@
+"""latticeai.brain — the durable substrate of the Digital Brain.
+
+v4 home for the brain's storage modules. The knowledge-graph store itself
+still lives in the root ``knowledge_graph`` module pending its decomposition
+(T3d); new brain components land here first.
+"""
+
+from latticeai.brain.context import AssembledContext, ContextAssembler, ContextSection
+from latticeai.brain.conversations import ConversationStore
+from latticeai.brain.memory import BrainMemory
+
+__all__ = [
+    "AssembledContext",
+    "BrainMemory",
+    "ContextAssembler",
+    "ContextSection",
+    "ConversationStore",
+]

package/latticeai/brain/context.py

@@ -0,0 +1,213 @@
+"""Context System — one budgeted, provenance-carrying assembly pipeline.
+
+Replaces the ad-hoc string concatenation that built chat context (language
+hint + vault scan + KG LIKE search + recent chat appended in fixed order
+with no size control). Every section the assembler emits records WHY it is
+in the prompt (source, ids, scores), so "why is this in my context?" is
+answerable, and the whole assembly respects a token budget.
+
+Token counts are an explicit approximation: ``approx_tokens = ceil(len/4)``
+(the stack ships no model-agnostic tokenizer; the field name says so —
+design-review amendment T5).
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional
+
+
+def approx_tokens(text: str) -> int:
+    """Documented chars/4 approximation — NOT a real tokenizer count."""
+    return max(0, (len(text or "") + 3) // 4)
+
+
+@dataclass
+class ContextSection:
+    name: str
+    content: str
+    source: str                      # memory | knowledge | notes | recent_chat | system
+    provenance: List[Dict[str, Any]] = field(default_factory=list)
+    truncated: bool = False
+
+    @property
+    def approx_tokens(self) -> int:
+        return approx_tokens(self.content)
+
+    def as_trace(self) -> Dict[str, Any]:
+        return {
+            "name": self.name,
+            "source": self.source,
+            "approx_tokens": self.approx_tokens,
+            "truncated": self.truncated,
+            "provenance": self.provenance,
+        }
+
+
+@dataclass
+class AssembledContext:
+    sections: List[ContextSection]
+    budget_approx_tokens: int
+
+    @property
+    def text(self) -> str:
+        parts = []
+        for section in self.sections:
+            if section.content.strip():
+                parts.append(f"[{section.name}]\n{section.content.strip()}")
+        return "\n\n".join(parts)
+
+    @property
+    def approx_tokens(self) -> int:
+        return sum(s.approx_tokens for s in self.sections)
+
+    def trace(self) -> Dict[str, Any]:
+        return {
+            "budget_approx_tokens": self.budget_approx_tokens,
+            "used_approx_tokens": self.approx_tokens,
+            "sections": [s.as_trace() for s in self.sections],
+        }
+
+
+class ContextAssembler:
+    """Ordered, budgeted context assembly over injected retrieval seams.
+
+    Section priority (kept under budget in this order — semantic memories
+    first because they are cheap and durable; recency last):
+      1. memories  — workspace semantic memories (preferences/decisions/…)
+      2. knowledge — hybrid search over the brain (the product's own engine)
+      3. notes     — garden-note context
+      4. recent    — the user's recent exchange
+    Every seam is optional; an absent seam contributes nothing (honest
+    absence), never a fabricated section.
+    """
+
+    def __init__(
+        self,
+        *,
+        memory_recall: Optional[Callable[..., Dict[str, Any]]] = None,
+        hybrid_search: Optional[Callable[..., Dict[str, Any]]] = None,
+        notes_context: Optional[Callable[..., str]] = None,
+        recent_chat: Optional[Callable[..., str]] = None,
+    ) -> None:
+        self._memory_recall = memory_recall
+        self._hybrid_search = hybrid_search
+        self._notes_context = notes_context
+        self._recent_chat = recent_chat
+
+    def assemble(
+        self,
+        query: str,
+        *,
+        user_email: Optional[str] = None,
+        workspace_id: Optional[str] = None,
+        conversation_id: Optional[str] = None,
+        budget: int = 2000,
+        memory_limit: int = 5,
+        knowledge_limit: int = 5,
+    ) -> AssembledContext:
+        sections: List[ContextSection] = []
+
+        if self._memory_recall is not None:
+            sections.append(self._memories_section(query, user_email, workspace_id, memory_limit))
+        if self._hybrid_search is not None:
+            sections.append(self._knowledge_section(query, knowledge_limit, user_email))
+        if self._notes_context is not None:
+            sections.append(self._notes_section(query))
+        if self._recent_chat is not None:
+            sections.append(self._recent_section(user_email, conversation_id))
+
+        sections = [s for s in sections if s.content.strip()]
+        self._apply_budget(sections, budget)
+        return AssembledContext(sections=sections, budget_approx_tokens=budget)
+
+    # ── section builders (each failure-isolated and honest) ───────────────
+    def _memories_section(self, query, user_email, workspace_id, limit) -> ContextSection:
+        try:
+            recall = self._memory_recall(query, user_email=user_email, workspace_id=workspace_id, limit=limit)
+            results = [r for r in recall.get("results", []) if r.get("source") == "workspace"][:limit]
+        except Exception as exc:
+            logging.debug("context: memory recall failed: %s", exc)
+            results = []
+        lines = [f"- ({r.get('kind') or 'memory'}) {r.get('snippet') or ''}" for r in results]
+        return ContextSection(
+            name="User memories",
+            content="\n".join(lines),
+            source="memory",
+            provenance=[
+                {"id": r.get("id"), "kind": r.get("kind"), "score": r.get("score")}
+                for r in results
+            ],
+        )
+
+    def _knowledge_section(self, query, limit, user_email=None) -> ContextSection:
+        try:
+            hybrid = self._hybrid_search(query, limit=limit, user_email=user_email)
+            matches = hybrid.get("matches", [])[:limit]
+        except Exception as exc:
+            logging.debug("context: hybrid search failed: %s", exc)
+            matches = []
+        lines = []
+        provenance = []
+        for m in matches:
+            title = m.get("title") or m.get("id") or "item"
+            body = (m.get("summary") or m.get("snippet") or "")[:400]
+            lines.append(f"- {title}: {body}" if body else f"- {title}")
+            provenance.append({
+                "id": m.get("id"),
+                "score": m.get("score"),
+                "sources": m.get("sources"),
+            })
+        return ContextSection(
+            name="Knowledge",
+            content="\n".join(lines),
+            source="knowledge",
+            provenance=provenance,
+        )
+
+    def _notes_section(self, query) -> ContextSection:
+        try:
+            content = self._notes_context(query) or ""
+        except Exception as exc:
+            logging.debug("context: notes context failed: %s", exc)
+            content = ""
+        return ContextSection(
+            name="Garden notes",
+            content=content,
+            source="notes",
+            provenance=[{"source": "garden", "included": bool(content)}],
+        )
+
+    def _recent_section(self, user_email, conversation_id) -> ContextSection:
+        try:
+            content = self._recent_chat(user_email=user_email, conversation_id=conversation_id) or ""
+        except Exception as exc:
+            logging.debug("context: recent chat failed: %s", exc)
+            content = ""
+        return ContextSection(
+            name="Recent conversation",
+            content=content,
+            source="recent_chat",
+            provenance=[{"conversation_id": conversation_id, "user_email": user_email}],
+        )
+
+    # ── budget ─────────────────────────────────────────────────────────────
+    @staticmethod
+    def _apply_budget(sections: List[ContextSection], budget: int) -> None:
+        """Trim from the END (lowest priority) until under budget."""
+        budget = max(1, int(budget))
+        used = 0
+        for section in sections:
+            remaining = budget - used
+            if remaining <= 0:
+                section.content = ""
+                section.truncated = True
+                continue
+            if section.approx_tokens > remaining:
+                section.content = section.content[: remaining * 4]
+                section.truncated = True
+            used += section.approx_tokens
+
+
+__all__ = ["ContextAssembler", "ContextSection", "AssembledContext", "approx_tokens"]

package/latticeai/brain/conversations.py

@@ -0,0 +1,236 @@
+"""Durable conversation store — kills the 50-message chat_history.json cap.
+
+Conversations are episodic memory, the most valuable raw input a Digital
+Brain has; truncating them contradicted "knowledge is durable". This store
+keeps every message in SQLite — by default in the same database file as the
+knowledge graph, so the existing kg_portability backup/restore covers
+conversations with no manifest changes.
+
+The public item shape is exactly the legacy chat_history.json entry
+(role/content/timestamp + optional user_email/user_nickname/source/
+conversation_id), so every existing consumer of ``get_history()`` keeps
+working. Legacy history is imported once, idempotently (content-hash dedup).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import sqlite3
+import threading
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+def _message_hash(item: Dict[str, Any]) -> str:
+    basis = "|".join(
+        str(item.get(key) or "")
+        for key in ("role", "content", "timestamp", "user_email", "conversation_id", "source")
+    )
+    return hashlib.sha256(basis.encode("utf-8", "ignore")).hexdigest()
+
+
+class ConversationStore:
+    """Unbounded, per-conversation chat history in SQLite."""
+
+    def __init__(self, db_path: Path):
+        self.db_path = Path(db_path)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.RLock()
+        self._init_db()
+
+    def _connect(self) -> sqlite3.Connection:
+        conn = sqlite3.connect(str(self.db_path))
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA journal_mode=WAL")
+        return conn
+
+    def _init_db(self) -> None:
+        with self._lock, self._connect() as conn:
+            conn.executescript(
+                """
+                CREATE TABLE IF NOT EXISTS conversation_messages (
+                  id INTEGER PRIMARY KEY AUTOINCREMENT,
+                  message_hash TEXT NOT NULL UNIQUE,
+                  conversation_id TEXT,
+                  role TEXT NOT NULL,
+                  content TEXT NOT NULL,
+                  user_email TEXT,
+                  user_nickname TEXT,
+                  source TEXT,
+                  timestamp TEXT NOT NULL,
+                  metadata_json TEXT NOT NULL DEFAULT '{}'
+                );
+                CREATE INDEX IF NOT EXISTS idx_conv_messages_conv
+                  ON conversation_messages(conversation_id);
+                CREATE INDEX IF NOT EXISTS idx_conv_messages_time
+                  ON conversation_messages(timestamp);
+                """
+            )
+
+    # ── writes ────────────────────────────────────────────────────────────
+    def append(self, item: Dict[str, Any]) -> Dict[str, Any]:
+        """Persist one chat item (the legacy chat_history.json entry shape)."""
+        known = {"role", "content", "timestamp", "user_email", "user_nickname", "source", "conversation_id"}
+        extra = {k: v for k, v in item.items() if k not in known}
+        with self._lock, self._connect() as conn:
+            conn.execute(
+                """
+                INSERT OR IGNORE INTO conversation_messages
+                  (message_hash, conversation_id, role, content, user_email,
+                   user_nickname, source, timestamp, metadata_json)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    _message_hash(item),
+                    item.get("conversation_id"),
+                    str(item.get("role") or "user"),
+                    str(item.get("content") or ""),
+                    item.get("user_email"),
+                    item.get("user_nickname"),
+                    item.get("source"),
+                    str(item.get("timestamp") or ""),
+                    json.dumps(extra, ensure_ascii=False) if extra else "{}",
+                ),
+            )
+        return item
+
+    def import_legacy_json(self, history_file: Path) -> int:
+        """One-time, idempotent import of a chat_history.json file.
+
+        Re-running never duplicates (message_hash UNIQUE + INSERT OR IGNORE);
+        the source file is left untouched on disk.
+        """
+        path = Path(history_file)
+        if not path.exists():
+            return 0
+        try:
+            with open(path, "r", encoding="utf-8") as fh:
+                items = json.load(fh)
+        except Exception as exc:
+            logging.warning("conversation store: legacy import failed to read %s: %s", path, exc)
+            return 0
+        if not isinstance(items, list):
+            return 0
+        imported = 0
+        with self._lock, self._connect() as conn:
+            for item in items:
+                if not isinstance(item, dict):
+                    continue
+                cur = conn.execute(
+                    """
+                    INSERT OR IGNORE INTO conversation_messages
+                      (message_hash, conversation_id, role, content, user_email,
+                       user_nickname, source, timestamp, metadata_json)
+                    VALUES (?, ?, ?, ?, ?, ?, ?, ?, '{}')
+                    """,
+                    (
+                        _message_hash(item),
+                        item.get("conversation_id"),
+                        str(item.get("role") or "user"),
+                        str(item.get("content") or ""),
+                        item.get("user_email"),
+                        item.get("user_nickname"),
+                        item.get("source"),
+                        str(item.get("timestamp") or ""),
+                    ),
+                )
+                imported += cur.rowcount if cur.rowcount > 0 else 0
+        if imported:
+            logging.info("conversation store: imported %d legacy chat messages from %s", imported, path)
+        return imported
+
+    # ── reads (legacy item shape) ─────────────────────────────────────────
+    @staticmethod
+    def _row_to_item(row: sqlite3.Row) -> Dict[str, Any]:
+        item: Dict[str, Any] = {
+            "role": row["role"],
+            "content": row["content"],
+            "timestamp": row["timestamp"],
+        }
+        for key in ("user_email", "user_nickname", "source", "conversation_id"):
+            if row[key]:
+                item[key] = row[key]
+        try:
+            extra = json.loads(row["metadata_json"] or "{}")
+        except Exception:
+            extra = {}
+        item.update(extra)
+        return item
+
+    def history(self, *, conversation_id: Optional[str] = None, limit: Optional[int] = None) -> List[Dict[str, Any]]:
+        """Chronological items; the unbounded successor of get_history()."""
+        query = "SELECT * FROM conversation_messages"
+        params: List[Any] = []
+        if conversation_id is not None:
+            query += " WHERE conversation_id IS ?" if conversation_id == "" else " WHERE conversation_id = ?"
+            params.append(None if conversation_id == "" else conversation_id)
+        query += " ORDER BY id ASC"
+        if limit is not None:
+            query += " LIMIT ?"
+            params.append(max(1, int(limit)))
+        with self._connect() as conn:
+            rows = conn.execute(query, params).fetchall()
+        return [self._row_to_item(row) for row in rows]
+
+    def count(self) -> int:
+        with self._connect() as conn:
+            return conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
+
+    def size_bytes(self) -> int:
+        try:
+            return self.db_path.stat().st_size if self.db_path.exists() else 0
+        except OSError:
+            return 0
+
+    # ── clears (legacy semantics preserved) ───────────────────────────────
+    def clear_all(self, keep_last: int = 0) -> Dict[str, Any]:
+        keep_last = max(0, min(int(keep_last or 0), 20))
+        with self._lock, self._connect() as conn:
+            total = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
+            if keep_last:
+                conn.execute(
+                    """
+                    DELETE FROM conversation_messages WHERE id NOT IN (
+                      SELECT id FROM conversation_messages ORDER BY id DESC LIMIT ?
+                    )
+                    """,
+                    (keep_last,),
+                )
+            else:
+                conn.execute("DELETE FROM conversation_messages")
+            kept = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
+        return {"status": "cleared", "removed": max(0, total - kept), "kept": kept}
+
+    def clear_conversation(self, conversation_id: str, started_at: Optional[str] = None) -> Dict[str, Any]:
+        """Remove one conversation.
+
+        ``legacy-previous-history`` targets unattributed messages; when
+        ``started_at`` is given, unattributed messages from that point on are
+        removed too (mirrors the pre-v4 JSON behaviour exactly).
+        """
+        with self._lock, self._connect() as conn:
+            total = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
+            if conversation_id == "legacy-previous-history":
+                conn.execute("DELETE FROM conversation_messages WHERE conversation_id IS NULL")
+            else:
+                conn.execute(
+                    "DELETE FROM conversation_messages WHERE conversation_id = ?",
+                    (conversation_id,),
+                )
+                if started_at:
+                    conn.execute(
+                        "DELETE FROM conversation_messages WHERE conversation_id IS NULL AND timestamp >= ?",
+                        (str(started_at),),
+                    )
+            kept = conn.execute("SELECT COUNT(*) FROM conversation_messages").fetchone()[0]
+        return {
+            "status": "cleared",
+            "conversation_id": conversation_id,
+            "removed": max(0, total - kept),
+            "kept": kept,
+        }
+
+
+__all__ = ["ConversationStore"]

package/latticeai/brain/identity.py

@@ -0,0 +1,175 @@
+"""Device identity — the sovereignty primitive.
+
+Every Lattice installation owns an Ed25519 keypair. Exports are signed by
+it, peers pair against its public key, and imported knowledge records which
+device it came from. The private key never leaves the machine: it lives in
+the OS keyring when one is available, otherwise in a 0600 file under the
+data directory (the storage backend is reported honestly).
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import (
+    Ed25519PrivateKey,
+    Ed25519PublicKey,
+)
+
+_KEYRING_SERVICE = "lattice-ai-device-identity"
+_KEYRING_ENTRY = "ed25519-private-key"
+
+
+def _b64(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).decode("ascii").rstrip("=")
+
+
+def _unb64(text: str) -> bytes:
+    padded = text + "=" * (-len(text) % 4)
+    return base64.urlsafe_b64decode(padded.encode("ascii"))
+
+
+def _keyring_opt_in() -> bool:
+    """Keyring storage is opt-in (LATTICEAI_DEVICE_KEY_KEYRING=1).
+
+    OS keychain access can block or prompt during startup/tests; the default
+    is a 0600 file under the data dir, and ``describe()`` reports which
+    backend holds the key — no silent security theater either way.
+    """
+    return os.getenv("LATTICEAI_DEVICE_KEY_KEYRING", "").strip() in {"1", "true", "yes"}
+
+
+class DeviceIdentity:
+    """Loads-or-creates the installation's Ed25519 keypair."""
+
+    def __init__(self, data_dir: Path, *, use_keyring: Optional[bool] = None):
+        if use_keyring is None:
+            use_keyring = _keyring_opt_in()
+        self._data_dir = Path(data_dir)
+        self._key_file = self._data_dir / "device_identity.key"
+        self._private: Ed25519PrivateKey
+        self.storage: str  # "keyring" | "file"
+        self._load_or_create(use_keyring)
+
+    # ── key material ───────────────────────────────────────────────────────
+    def _load_or_create(self, use_keyring: bool) -> None:
+        raw: Optional[bytes] = None
+        backend = "file"
+        if use_keyring:
+            try:
+                import keyring
+
+                stored = keyring.get_password(_KEYRING_SERVICE, _KEYRING_ENTRY)
+                if stored:
+                    raw = _unb64(stored)
+                    backend = "keyring"
+            except Exception as exc:
+                logging.debug("device identity: keyring unavailable (%s)", exc)
+        if raw is None and self._key_file.exists():
+            raw = _unb64(self._key_file.read_text().strip())
+            backend = "file"
+        if raw is None:
+            key = Ed25519PrivateKey.generate()
+            raw = key.private_bytes(
+                serialization.Encoding.Raw,
+                serialization.PrivateFormat.Raw,
+                serialization.NoEncryption(),
+            )
+            backend = self._persist_new(raw, use_keyring)
+        self._private = Ed25519PrivateKey.from_private_bytes(raw)
+        self.storage = backend
+
+    def _persist_new(self, raw: bytes, use_keyring: bool) -> str:
+        if use_keyring:
+            try:
+                import keyring
+
+                keyring.set_password(_KEYRING_SERVICE, _KEYRING_ENTRY, _b64(raw))
+                return "keyring"
+            except Exception as exc:
+                logging.debug("device identity: keyring store failed (%s); using file", exc)
+        self._data_dir.mkdir(parents=True, exist_ok=True)
+        self._key_file.write_text(_b64(raw))
+        os.chmod(self._key_file, 0o600)
+        return "file"
+
+    # ── public surface ─────────────────────────────────────────────────────
+    @property
+    def public_key_b64(self) -> str:
+        return _b64(
+            self._private.public_key().public_bytes(
+                serialization.Encoding.Raw, serialization.PublicFormat.Raw
+            )
+        )
+
+    @property
+    def fingerprint(self) -> str:
+        """Short human-comparable id: sha256 of the raw public key."""
+        raw = self._private.public_key().public_bytes(
+            serialization.Encoding.Raw, serialization.PublicFormat.Raw
+        )
+        digest = hashlib.sha256(raw).hexdigest()
+        return ":".join(digest[i : i + 4] for i in range(0, 16, 4))
+
+    def describe(self) -> Dict[str, Any]:
+        return {
+            "fingerprint": self.fingerprint,
+            "public_key": self.public_key_b64,
+            "algorithm": "ed25519",
+            "storage": self.storage,
+        }
+
+    # ── signing ────────────────────────────────────────────────────────────
+    def sign(self, payload: bytes) -> str:
+        return _b64(self._private.sign(payload))
+
+    def sign_manifest(self, manifest: Dict[str, Any]) -> Dict[str, Any]:
+        """Detached signature over the canonical JSON of a manifest."""
+        canonical = json.dumps(manifest, sort_keys=True, ensure_ascii=False).encode("utf-8")
+        return {
+            "algorithm": "ed25519",
+            "public_key": self.public_key_b64,
+            "fingerprint": self.fingerprint,
+            "signature": self.sign(canonical),
+        }
+
+
+def fingerprint_of(public_key_b64: str) -> str:
+    """Human-comparable fingerprint of an Ed25519 public key.
+
+    Raises ValueError when the input is not a valid key — the pairing flow
+    uses this as its validation gate.
+    """
+    raw = _unb64(public_key_b64)
+    Ed25519PublicKey.from_public_bytes(raw)  # validates; raises on garbage
+    digest = hashlib.sha256(raw).hexdigest()
+    return ":".join(digest[i : i + 4] for i in range(0, 16, 4))
+
+
+def verify_signature(public_key_b64: str, payload: bytes, signature_b64: str) -> bool:
+    """True iff ``signature`` is valid for ``payload`` under the given key."""
+    try:
+        key = Ed25519PublicKey.from_public_bytes(_unb64(public_key_b64))
+        key.verify(_unb64(signature_b64), payload)
+        return True
+    except Exception:
+        return False
+
+
+def verify_manifest(manifest: Dict[str, Any], signature_block: Dict[str, Any]) -> bool:
+    canonical = json.dumps(manifest, sort_keys=True, ensure_ascii=False).encode("utf-8")
+    return verify_signature(
+        str(signature_block.get("public_key") or ""),
+        canonical,
+        str(signature_block.get("signature") or ""),
+    )
+
+
+__all__ = ["DeviceIdentity", "verify_signature", "verify_manifest"]