ltcai 4.1.0 → 4.2.0

package/lattice_brain/__init__.py

@@ -0,0 +1,70 @@
+"""lattice-brain — independent Brain Core package for Lattice AI.
+
+Heavy graph modules are lazy-loaded so storage and archive utilities remain
+usable without importing the FastAPI application or creating runtime globals.
+"""
+
+from .archive import BrainArchivePaths, EncryptedBrainArchive
+from .core import BrainCore, BrainCoreConfig
+from .storage import (
+    DockerPostgresPlan,
+    DockerPostgresWizard,
+    PostgresConfig,
+    PostgresEngine,
+    SQLiteEngine,
+    SQLiteToPostgresMigrator,
+    StorageCapabilities,
+    StorageEngine,
+    StorageUnavailable,
+    storage_from_env,
+)
+
+__version__ = "4.2.0"
+
+__all__ = [
+    "AssembledContext",
+    "BrainArchivePaths",
+    "BrainCore",
+    "BrainCoreConfig",
+    "BrainMemory",
+    "ContextAssembler",
+    "ContextSection",
+    "ConversationStore",
+    "DockerPostgresPlan",
+    "DockerPostgresWizard",
+    "EncryptedBrainArchive",
+    "KnowledgeGraphStore",
+    "PostgresConfig",
+    "PostgresEngine",
+    "SQLiteEngine",
+    "SQLiteToPostgresMigrator",
+    "StorageCapabilities",
+    "StorageEngine",
+    "StorageUnavailable",
+    "storage_from_env",
+    "__version__",
+]
+
+
+def __getattr__(name: str):
+    if name in {"AssembledContext", "ContextAssembler", "ContextSection"}:
+        from .context import AssembledContext, ContextAssembler, ContextSection
+
+        return {
+            "AssembledContext": AssembledContext,
+            "ContextAssembler": ContextAssembler,
+            "ContextSection": ContextSection,
+        }[name]
+    if name == "ConversationStore":
+        from .conversations import ConversationStore
+
+        return ConversationStore
+    if name == "BrainMemory":
+        from .memory import BrainMemory
+
+        return BrainMemory
+    if name == "KnowledgeGraphStore":
+        from .store import KnowledgeGraphStore
+
+        return KnowledgeGraphStore
+    raise AttributeError(name)

package/lattice_brain/_kg_common.py

@@ -0,0 +1 @@
+from latticeai.brain._kg_common import *  # noqa: F401,F403

package/lattice_brain/archive.py

@@ -0,0 +1,133 @@
+"""Encrypted .latticebrain archive support."""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import shutil
+import tempfile
+import zipfile
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, Optional
+
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+
+
+ARCHIVE_FORMAT = "latticebrain.encrypted"
+ARCHIVE_VERSION = 1
+KDF_ITERATIONS = 390_000
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def _derive_key(passphrase: str, salt: bytes) -> bytes:
+    if not passphrase:
+        raise ValueError("A passphrase is required for encrypted .latticebrain archives.")
+    kdf = PBKDF2HMAC(
+        algorithm=hashes.SHA256(),
+        length=32,
+        salt=salt,
+        iterations=KDF_ITERATIONS,
+    )
+    return kdf.derive(passphrase.encode("utf-8"))
+
+
+@dataclass(frozen=True)
+class BrainArchivePaths:
+    db_path: Path
+    blob_dir: Optional[Path] = None
+
+
+class EncryptedBrainArchive:
+    """Create and restore encrypted local Brain Core archives."""
+
+    def __init__(self, paths: BrainArchivePaths) -> None:
+        self.paths = paths
+
+    def create(self, destination: Path, *, passphrase: str) -> Dict[str, object]:
+        dest = Path(destination)
+        if dest.suffix != ".latticebrain":
+            dest = dest.with_suffix(".latticebrain")
+        if not self.paths.db_path.exists():
+            raise FileNotFoundError(f"Brain database not found: {self.paths.db_path}")
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        with tempfile.TemporaryDirectory() as tmp_s:
+            tmp = Path(tmp_s)
+            payload = tmp / "payload.zip"
+            with zipfile.ZipFile(payload, "w", zipfile.ZIP_DEFLATED) as zf:
+                zf.write(self.paths.db_path, "knowledge_graph.sqlite")
+                if self.paths.blob_dir and self.paths.blob_dir.exists():
+                    for file in self.paths.blob_dir.rglob("*"):
+                        if file.is_file():
+                            zf.write(file, f"blobs/{file.relative_to(self.paths.blob_dir)}")
+            salt = os.urandom(16)
+            nonce = os.urandom(12)
+            key = _derive_key(passphrase, salt)
+            ciphertext = AESGCM(key).encrypt(nonce, payload.read_bytes(), None)
+            envelope = {
+                "format": ARCHIVE_FORMAT,
+                "format_version": ARCHIVE_VERSION,
+                "created_at": _now(),
+                "kdf": {
+                    "name": "PBKDF2HMAC-SHA256",
+                    "iterations": KDF_ITERATIONS,
+                    "salt": base64.b64encode(salt).decode("ascii"),
+                },
+                "cipher": {
+                    "name": "AES-256-GCM",
+                    "nonce": base64.b64encode(nonce).decode("ascii"),
+                },
+                "payload": base64.b64encode(ciphertext).decode("ascii"),
+            }
+            dest.write_text(json.dumps(envelope, indent=2), encoding="utf-8")
+        return {"path": str(dest), "bytes": dest.stat().st_size, "encrypted": True}
+
+    def restore(self, source: Path, *, passphrase: str, target: BrainArchivePaths) -> Dict[str, object]:
+        src = Path(source)
+        if not src.exists():
+            raise FileNotFoundError(f"Brain archive not found: {src}")
+        envelope = json.loads(src.read_text(encoding="utf-8"))
+        if envelope.get("format") != ARCHIVE_FORMAT:
+            raise ValueError("Not a .latticebrain encrypted archive.")
+        salt = base64.b64decode(envelope["kdf"]["salt"])
+        nonce = base64.b64decode(envelope["cipher"]["nonce"])
+        ciphertext = base64.b64decode(envelope["payload"])
+        key = _derive_key(passphrase, salt)
+        try:
+            plaintext = AESGCM(key).decrypt(nonce, ciphertext, None)
+        except InvalidTag as exc:
+            raise ValueError("Archive decryption failed; passphrase or archive data is invalid.") from exc
+        with tempfile.TemporaryDirectory() as tmp_s:
+            tmp = Path(tmp_s)
+            payload = tmp / "payload.zip"
+            payload.write_bytes(plaintext)
+            with zipfile.ZipFile(payload) as zf:
+                zf.extractall(tmp / "out")
+            db_src = tmp / "out" / "knowledge_graph.sqlite"
+            if not db_src.exists():
+                raise ValueError("Archive payload is missing knowledge_graph.sqlite.")
+            target.db_path.parent.mkdir(parents=True, exist_ok=True)
+            for sibling in (target.db_path, Path(str(target.db_path) + "-wal"), Path(str(target.db_path) + "-shm")):
+                if sibling.exists():
+                    sibling.unlink()
+            shutil.copyfile(db_src, target.db_path)
+            blobs_src = tmp / "out" / "blobs"
+            if target.blob_dir:
+                if target.blob_dir.exists():
+                    shutil.rmtree(target.blob_dir)
+                if blobs_src.exists():
+                    shutil.copytree(blobs_src, target.blob_dir)
+                else:
+                    target.blob_dir.mkdir(parents=True, exist_ok=True)
+        return {"restored": True, "path": str(target.db_path), "encrypted": True}
+
+
+__all__ = ["BrainArchivePaths", "EncryptedBrainArchive"]

package/lattice_brain/context.py

@@ -0,0 +1,3 @@
+from latticeai.brain.context import AssembledContext, ContextAssembler, ContextSection, approx_tokens
+
+__all__ = ["AssembledContext", "ContextAssembler", "ContextSection", "approx_tokens"]

package/lattice_brain/conversations.py

@@ -0,0 +1,3 @@
+from latticeai.brain.conversations import ConversationStore
+
+__all__ = ["ConversationStore"]

package/lattice_brain/core.py

@@ -0,0 +1,82 @@
+"""Independent Brain Core package facade."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from .archive import BrainArchivePaths, EncryptedBrainArchive
+from .storage import SQLiteEngine, StorageEngine, StorageUnavailable
+
+
+@dataclass(frozen=True)
+class BrainCoreConfig:
+    data_dir: Path
+    blob_dir: Optional[Path] = None
+    storage_engine: Optional[StorageEngine] = None
+
+
+class BrainCore:
+    """Stable application boundary for the local Digital Brain.
+
+    FastAPI, CLI, tests, and future tools should depend on this package-level
+    facade instead of constructing scattered storage objects directly.
+    """
+
+    def __init__(self, config: BrainCoreConfig, *, embedder: Any = None) -> None:
+        self.config = config
+        self.data_dir = Path(config.data_dir)
+        self.db_path = self.data_dir / "knowledge_graph.sqlite"
+        self.blob_dir = Path(config.blob_dir) if config.blob_dir else self.data_dir / "knowledge_graph_blobs"
+        self.storage_engine = config.storage_engine or SQLiteEngine(self.db_path)
+        caps = self.storage_engine.capabilities()
+        if not caps.available:
+            raise StorageUnavailable(caps.reason or f"{caps.engine} storage is unavailable")
+        if caps.engine != "sqlite":
+            raise StorageUnavailable(
+                "The active FastAPI Brain Core runtime currently requires SQLiteEngine. "
+                "Use PostgresEngine through the explicit migration/scale tooling; no SQLite fallback was attempted."
+            )
+
+        from .conversations import ConversationStore
+        from .store import KnowledgeGraphStore
+
+        self.knowledge = KnowledgeGraphStore(
+            self.db_path,
+            self.blob_dir,
+            embedder=embedder,
+            storage_engine=self.storage_engine,
+        )
+        self.conversations = ConversationStore(self.db_path)
+        self.archive = EncryptedBrainArchive(
+            BrainArchivePaths(db_path=self.db_path, blob_dir=self.blob_dir)
+        )
+
+    @classmethod
+    def from_paths(
+        cls,
+        data_dir: Path,
+        *,
+        blob_dir: Optional[Path] = None,
+        embedder: Any = None,
+        storage_engine: Optional[StorageEngine] = None,
+    ) -> "BrainCore":
+        return cls(
+            BrainCoreConfig(
+                data_dir=Path(data_dir),
+                blob_dir=blob_dir,
+                storage_engine=storage_engine,
+            ),
+            embedder=embedder,
+        )
+
+    def status(self) -> dict:
+        return {
+            "storage": self.storage_engine.capabilities().as_dict(),
+            "db_path": str(self.db_path),
+            "blob_dir": str(self.blob_dir),
+        }
+
+
+__all__ = ["BrainCore", "BrainCoreConfig"]

package/lattice_brain/discovery.py

@@ -0,0 +1 @@
+from latticeai.brain.discovery import *  # noqa: F401,F403

package/lattice_brain/documents.py

@@ -0,0 +1 @@
+from latticeai.brain.documents import *  # noqa: F401,F403

package/lattice_brain/embeddings.py

@@ -0,0 +1,82 @@
+"""Local deterministic embeddings used by the standalone Brain Core package."""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import os
+import re
+import struct
+from dataclasses import dataclass
+from typing import Iterable, List
+
+
+DEFAULT_EMBEDDING_DIM = int(os.getenv("LATTICEAI_VECTOR_DIM", "384"))
+EMBEDDING_MODEL_ID = f"lattice-local-hash-v1:{DEFAULT_EMBEDDING_DIM}"
+
+
+def _tokenize(text: str) -> List[str]:
+    raw = str(text or "").lower()
+    tokens = re.findall(r"[a-z0-9][a-z0-9_.:/+-]{1,}|[가-힣]{2,}", raw)
+    features: List[str] = []
+    for token in tokens:
+        features.append(f"tok:{token}")
+        if len(token) >= 5 and re.search(r"[a-z]", token):
+            for i in range(0, len(token) - 2):
+                features.append(f"tri:{token[i:i+3]}")
+        if re.search(r"[가-힣]", token) and len(token) >= 3:
+            for i in range(0, len(token) - 1):
+                features.append(f"ko:{token[i:i+2]}")
+    return features
+
+
+def _hash_to_index(feature: str, dim: int) -> tuple[int, float]:
+    digest = hashlib.blake2b(feature.encode("utf-8"), digest_size=8).digest()
+    value = int.from_bytes(digest, "big", signed=False)
+    sign = 1.0 if (value & 1) == 0 else -1.0
+    return value % dim, sign
+
+
+@dataclass(frozen=True)
+class LocalEmbeddingModel:
+    """Deterministic local embedder.
+
+    This is intentionally not presented as a production semantic model. It is
+    a real, offline cosine signal for local-first operation and tests; setup
+    wizard provisioning can replace it with a user-consented model/provider.
+    """
+
+    dim: int = DEFAULT_EMBEDDING_DIM
+    model_id: str = EMBEDDING_MODEL_ID
+
+    def embed(self, text: str) -> List[float]:
+        vector = [0.0] * self.dim
+        features = _tokenize(text)
+        if not features:
+            return vector
+        for feature in features:
+            index, sign = _hash_to_index(feature, self.dim)
+            vector[index] += sign
+        norm = math.sqrt(sum(value * value for value in vector))
+        if norm <= 0:
+            return vector
+        return [value / norm for value in vector]
+
+    def similarity(self, left: Iterable[float], right: Iterable[float]) -> float:
+        return float(sum(a * b for a, b in zip(left, right)))
+
+    def encode(self, vector: Iterable[float]) -> bytes:
+        values = list(vector)
+        return struct.pack(f"<{len(values)}f", *values)
+
+    def decode(self, payload: bytes, dim: int | None = None) -> List[float]:
+        if not payload:
+            return []
+        count = int(dim or self.dim)
+        expected = count * 4
+        if len(payload) != expected:
+            count = len(payload) // 4
+        return list(struct.unpack(f"<{count}f", payload[: count * 4]))
+
+
+__all__ = ["DEFAULT_EMBEDDING_DIM", "EMBEDDING_MODEL_ID", "LocalEmbeddingModel"]

package/lattice_brain/identity.py

@@ -0,0 +1,13 @@
+from latticeai.brain.identity import (
+    DeviceIdentity,
+    fingerprint_of,
+    verify_manifest,
+    verify_signature,
+)
+
+__all__ = [
+    "DeviceIdentity",
+    "fingerprint_of",
+    "verify_manifest",
+    "verify_signature",
+]

package/lattice_brain/ingest.py

@@ -0,0 +1 @@
+from latticeai.brain.ingest import *  # noqa: F401,F403

package/lattice_brain/memory.py

@@ -0,0 +1,3 @@
+from latticeai.brain.memory import BrainMemory
+
+__all__ = ["BrainMemory"]

package/lattice_brain/network.py

@@ -0,0 +1 @@
+from latticeai.brain.network import *  # noqa: F401,F403

package/lattice_brain/projection.py

@@ -0,0 +1 @@
+from latticeai.brain.projection import *  # noqa: F401,F403

package/lattice_brain/provenance.py

@@ -0,0 +1 @@
+from latticeai.brain.provenance import *  # noqa: F401,F403

package/lattice_brain/retrieval.py

@@ -0,0 +1 @@
+from latticeai.brain.retrieval import *  # noqa: F401,F403

package/lattice_brain/schema.py

@@ -0,0 +1 @@
+from latticeai.brain.schema import *  # noqa: F401,F403

package/lattice_brain/storage/__init__.py

@@ -0,0 +1,22 @@
+"""Pluggable storage layer for lattice-brain."""
+
+from .base import StorageCapabilities, StorageEngine, StorageUnavailable
+from .docker import DockerPostgresPlan, DockerPostgresWizard
+from .factory import storage_from_env
+from .migration import SQLiteToPostgresMigrator, TablePlan
+from .postgres import PostgresConfig, PostgresEngine
+from .sqlite import SQLiteEngine
+
+__all__ = [
+    "DockerPostgresPlan",
+    "DockerPostgresWizard",
+    "PostgresConfig",
+    "PostgresEngine",
+    "SQLiteEngine",
+    "SQLiteToPostgresMigrator",
+    "StorageCapabilities",
+    "StorageEngine",
+    "StorageUnavailable",
+    "TablePlan",
+    "storage_from_env",
+]

package/lattice_brain/storage/base.py

@@ -0,0 +1,72 @@
+"""StorageEngine contracts for the independent Brain Core package."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+class StorageUnavailable(RuntimeError):
+    """Raised when an explicitly requested storage engine cannot be used."""
+
+
+@dataclass(frozen=True)
+class StorageCapabilities:
+    engine: str
+    available: bool
+    reason: Optional[str] = None
+    vector_backend: str = "none"
+    vector_available: bool = False
+    backup_restore: bool = False
+    migrations: bool = False
+    encrypted_archives: bool = False
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def as_dict(self) -> Dict[str, Any]:
+        return {
+            "engine": self.engine,
+            "available": self.available,
+            "reason": self.reason,
+            "vector_backend": self.vector_backend,
+            "vector_available": self.vector_available,
+            "backup_restore": self.backup_restore,
+            "migrations": self.migrations,
+            "encrypted_archives": self.encrypted_archives,
+            "metadata": self.metadata,
+        }
+
+
+class StorageEngine(ABC):
+    """Unified storage interface used by Brain Core.
+
+    The knowledge graph currently uses SQL directly, so ``connect`` is part of
+    the contract. Engines must fail loudly when unavailable; callers must not
+    silently fall back to SQLite after an explicit Postgres selection.
+    """
+
+    name: str
+
+    @abstractmethod
+    def capabilities(self) -> StorageCapabilities:
+        """Return an honest capability report."""
+
+    @abstractmethod
+    def initialize(self) -> Dict[str, Any]:
+        """Create required storage structures or raise ``StorageUnavailable``."""
+
+    @abstractmethod
+    def connect(self) -> Any:
+        """Return a DB-API-like connection for this engine."""
+
+    @abstractmethod
+    def backup(self, destination: Path) -> Dict[str, Any]:
+        """Create a faithful engine backup at ``destination``."""
+
+    @abstractmethod
+    def restore(self, source: Path) -> Dict[str, Any]:
+        """Restore a faithful engine backup from ``source``."""
+
+
+__all__ = ["StorageCapabilities", "StorageEngine", "StorageUnavailable"]

package/lattice_brain/storage/docker.py

@@ -0,0 +1,105 @@
+"""Explicit-consent Docker setup wizard for Postgres/pgvector."""
+
+from __future__ import annotations
+
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List
+
+
+@dataclass(frozen=True)
+class DockerPostgresPlan:
+    compose_path: Path
+    project_name: str
+    service_name: str = "postgres"
+    port: int = 5432
+
+    def command(self) -> List[str]:
+        return [
+            "docker",
+            "compose",
+            "-p",
+            self.project_name,
+            "-f",
+            str(self.compose_path),
+            "up",
+            "-d",
+            self.service_name,
+        ]
+
+
+class DockerPostgresWizard:
+    """Creates and starts a local Postgres container only after consent."""
+
+    def __init__(self, data_dir: Path, *, port: int = 5432) -> None:
+        self.data_dir = Path(data_dir)
+        self.port = int(port)
+
+    def write_compose(self) -> DockerPostgresPlan:
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+        compose = self.data_dir / "postgres.compose.yml"
+        compose.write_text(
+            f"""services:
+  postgres:
+    image: pgvector/pgvector:pg16
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: lattice_brain
+      POSTGRES_USER: lattice
+      POSTGRES_PASSWORD: lattice-local-only
+    ports:
+      - "127.0.0.1:{self.port}:5432"
+    volumes:
+      - ./postgres-data:/var/lib/postgresql/data
+""",
+            encoding="utf-8",
+        )
+        return DockerPostgresPlan(
+            compose_path=compose,
+            project_name="lattice-brain",
+            port=self.port,
+        )
+
+    def start(
+        self,
+        *,
+        consent: bool,
+        dry_run: bool = False,
+        runner=subprocess.run,
+    ) -> Dict[str, object]:
+        plan = self.write_compose()
+        if not consent:
+            return {
+                "status": "consent_required",
+                "started": False,
+                "compose_path": str(plan.compose_path),
+                "command": plan.command(),
+            }
+        if dry_run:
+            return {
+                "status": "dry_run",
+                "started": False,
+                "compose_path": str(plan.compose_path),
+                "command": plan.command(),
+            }
+        completed = runner(plan.command(), check=False, capture_output=True, text=True)
+        if completed.returncode != 0:
+            return {
+                "status": "failed",
+                "started": False,
+                "compose_path": str(plan.compose_path),
+                "returncode": completed.returncode,
+                "stdout": completed.stdout,
+                "stderr": completed.stderr,
+            }
+        return {
+            "status": "started",
+            "started": True,
+            "compose_path": str(plan.compose_path),
+            "stdout": completed.stdout,
+            "stderr": completed.stderr,
+        }
+
+
+__all__ = ["DockerPostgresPlan", "DockerPostgresWizard"]

package/lattice_brain/storage/factory.py

@@ -0,0 +1,31 @@
+"""StorageEngine construction from environment/config values."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Mapping
+
+from .base import StorageEngine, StorageUnavailable
+from .postgres import PostgresEngine
+from .sqlite import SQLiteEngine
+
+
+def storage_from_env(env: Mapping[str, str], *, data_dir: Path) -> StorageEngine:
+    engine = (env.get("LATTICEAI_STORAGE_ENGINE") or "sqlite").strip().lower()
+    if engine in {"", "sqlite"}:
+        return SQLiteEngine(Path(data_dir) / "knowledge_graph.sqlite")
+    if engine in {"postgres", "pg", "pgvector"}:
+        dsn = env.get("LATTICEAI_POSTGRES_DSN") or ""
+        if not dsn:
+            raise StorageUnavailable(
+                "LATTICEAI_STORAGE_ENGINE=postgres requires LATTICEAI_POSTGRES_DSN; "
+                "SQLite fallback is disabled for explicit Postgres selection."
+            )
+        return PostgresEngine(
+            dsn,
+            schema=env.get("LATTICEAI_POSTGRES_SCHEMA") or "lattice_brain",
+        )
+    raise StorageUnavailable(f"Unknown Brain Core storage engine: {engine}")
+
+
+__all__ = ["storage_from_env"]