ltcai 4.1.0 → 4.3.0

package/lattice_brain/conversations.py

@@ -0,0 +1,3 @@
+from latticeai.brain.conversations import ConversationStore
+
+__all__ = ["ConversationStore"]

package/lattice_brain/core.py

@@ -0,0 +1,82 @@
+"""Independent Brain Core package facade."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Optional
+
+from .archive import BrainArchivePaths, EncryptedBrainArchive
+from .storage import SQLiteEngine, StorageEngine, StorageUnavailable
+
+
+@dataclass(frozen=True)
+class BrainCoreConfig:
+    data_dir: Path
+    blob_dir: Optional[Path] = None
+    storage_engine: Optional[StorageEngine] = None
+
+
+class BrainCore:
+    """Stable application boundary for the local Digital Brain.
+
+    FastAPI, CLI, tests, and future tools should depend on this package-level
+    facade instead of constructing scattered storage objects directly.
+    """
+
+    def __init__(self, config: BrainCoreConfig, *, embedder: Any = None) -> None:
+        self.config = config
+        self.data_dir = Path(config.data_dir)
+        self.db_path = self.data_dir / "knowledge_graph.sqlite"
+        self.blob_dir = Path(config.blob_dir) if config.blob_dir else self.data_dir / "knowledge_graph_blobs"
+        self.storage_engine = config.storage_engine or SQLiteEngine(self.db_path)
+        caps = self.storage_engine.capabilities()
+        if not caps.available:
+            raise StorageUnavailable(caps.reason or f"{caps.engine} storage is unavailable")
+        if caps.engine != "sqlite":
+            raise StorageUnavailable(
+                "The active FastAPI Brain Core runtime currently requires SQLiteEngine. "
+                "Use PostgresEngine through the explicit migration/scale tooling; no SQLite fallback was attempted."
+            )
+
+        from .conversations import ConversationStore
+        from .store import KnowledgeGraphStore
+
+        self.knowledge = KnowledgeGraphStore(
+            self.db_path,
+            self.blob_dir,
+            embedder=embedder,
+            storage_engine=self.storage_engine,
+        )
+        self.conversations = ConversationStore(self.db_path)
+        self.archive = EncryptedBrainArchive(
+            BrainArchivePaths(db_path=self.db_path, blob_dir=self.blob_dir)
+        )
+
+    @classmethod
+    def from_paths(
+        cls,
+        data_dir: Path,
+        *,
+        blob_dir: Optional[Path] = None,
+        embedder: Any = None,
+        storage_engine: Optional[StorageEngine] = None,
+    ) -> "BrainCore":
+        return cls(
+            BrainCoreConfig(
+                data_dir=Path(data_dir),
+                blob_dir=blob_dir,
+                storage_engine=storage_engine,
+            ),
+            embedder=embedder,
+        )
+
+    def status(self) -> dict:
+        return {
+            "storage": self.storage_engine.capabilities().as_dict(),
+            "db_path": str(self.db_path),
+            "blob_dir": str(self.blob_dir),
+        }
+
+
+__all__ = ["BrainCore", "BrainCoreConfig"]

package/lattice_brain/discovery.py

@@ -0,0 +1 @@
+from latticeai.brain.discovery import *  # noqa: F401,F403

package/lattice_brain/documents.py

@@ -0,0 +1 @@
+from latticeai.brain.documents import *  # noqa: F401,F403

package/lattice_brain/embeddings.py

@@ -0,0 +1,82 @@
+"""Local deterministic embeddings used by the standalone Brain Core package."""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import os
+import re
+import struct
+from dataclasses import dataclass
+from typing import Iterable, List
+
+
+DEFAULT_EMBEDDING_DIM = int(os.getenv("LATTICEAI_VECTOR_DIM", "384"))
+EMBEDDING_MODEL_ID = f"lattice-local-hash-v1:{DEFAULT_EMBEDDING_DIM}"
+
+
+def _tokenize(text: str) -> List[str]:
+    raw = str(text or "").lower()
+    tokens = re.findall(r"[a-z0-9][a-z0-9_.:/+-]{1,}|[가-힣]{2,}", raw)
+    features: List[str] = []
+    for token in tokens:
+        features.append(f"tok:{token}")
+        if len(token) >= 5 and re.search(r"[a-z]", token):
+            for i in range(0, len(token) - 2):
+                features.append(f"tri:{token[i:i+3]}")
+        if re.search(r"[가-힣]", token) and len(token) >= 3:
+            for i in range(0, len(token) - 1):
+                features.append(f"ko:{token[i:i+2]}")
+    return features
+
+
+def _hash_to_index(feature: str, dim: int) -> tuple[int, float]:
+    digest = hashlib.blake2b(feature.encode("utf-8"), digest_size=8).digest()
+    value = int.from_bytes(digest, "big", signed=False)
+    sign = 1.0 if (value & 1) == 0 else -1.0
+    return value % dim, sign
+
+
+@dataclass(frozen=True)
+class LocalEmbeddingModel:
+    """Deterministic local embedder.
+
+    This is intentionally not presented as a production semantic model. It is
+    a real, offline cosine signal for local-first operation and tests; setup
+    wizard provisioning can replace it with a user-consented model/provider.
+    """
+
+    dim: int = DEFAULT_EMBEDDING_DIM
+    model_id: str = EMBEDDING_MODEL_ID
+
+    def embed(self, text: str) -> List[float]:
+        vector = [0.0] * self.dim
+        features = _tokenize(text)
+        if not features:
+            return vector
+        for feature in features:
+            index, sign = _hash_to_index(feature, self.dim)
+            vector[index] += sign
+        norm = math.sqrt(sum(value * value for value in vector))
+        if norm <= 0:
+            return vector
+        return [value / norm for value in vector]
+
+    def similarity(self, left: Iterable[float], right: Iterable[float]) -> float:
+        return float(sum(a * b for a, b in zip(left, right)))
+
+    def encode(self, vector: Iterable[float]) -> bytes:
+        values = list(vector)
+        return struct.pack(f"<{len(values)}f", *values)
+
+    def decode(self, payload: bytes, dim: int | None = None) -> List[float]:
+        if not payload:
+            return []
+        count = int(dim or self.dim)
+        expected = count * 4
+        if len(payload) != expected:
+            count = len(payload) // 4
+        return list(struct.unpack(f"<{count}f", payload[: count * 4]))
+
+
+__all__ = ["DEFAULT_EMBEDDING_DIM", "EMBEDDING_MODEL_ID", "LocalEmbeddingModel"]

package/lattice_brain/identity.py

@@ -0,0 +1,13 @@
+from latticeai.brain.identity import (
+    DeviceIdentity,
+    fingerprint_of,
+    verify_manifest,
+    verify_signature,
+)
+
+__all__ = [
+    "DeviceIdentity",
+    "fingerprint_of",
+    "verify_manifest",
+    "verify_signature",
+]

package/lattice_brain/ingest.py

@@ -0,0 +1 @@
+from latticeai.brain.ingest import *  # noqa: F401,F403

package/lattice_brain/memory.py

@@ -0,0 +1,3 @@
+from latticeai.brain.memory import BrainMemory
+
+__all__ = ["BrainMemory"]

package/lattice_brain/network.py

@@ -0,0 +1 @@
+from latticeai.brain.network import *  # noqa: F401,F403

package/lattice_brain/projection.py

@@ -0,0 +1 @@
+from latticeai.brain.projection import *  # noqa: F401,F403

package/lattice_brain/provenance.py

@@ -0,0 +1 @@
+from latticeai.brain.provenance import *  # noqa: F401,F403

package/lattice_brain/retrieval.py

@@ -0,0 +1 @@
+from latticeai.brain.retrieval import *  # noqa: F401,F403

package/lattice_brain/schema.py

@@ -0,0 +1 @@
+from latticeai.brain.schema import *  # noqa: F401,F403

package/lattice_brain/storage/__init__.py

@@ -0,0 +1,22 @@
+"""Pluggable storage layer for lattice-brain."""
+
+from .base import StorageCapabilities, StorageEngine, StorageUnavailable
+from .docker import DockerPostgresPlan, DockerPostgresWizard
+from .factory import storage_from_env
+from .migration import SQLiteToPostgresMigrator, TablePlan
+from .postgres import PostgresConfig, PostgresEngine
+from .sqlite import SQLiteEngine
+
+__all__ = [
+    "DockerPostgresPlan",
+    "DockerPostgresWizard",
+    "PostgresConfig",
+    "PostgresEngine",
+    "SQLiteEngine",
+    "SQLiteToPostgresMigrator",
+    "StorageCapabilities",
+    "StorageEngine",
+    "StorageUnavailable",
+    "TablePlan",
+    "storage_from_env",
+]

package/lattice_brain/storage/base.py

@@ -0,0 +1,72 @@
+"""StorageEngine contracts for the independent Brain Core package."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+class StorageUnavailable(RuntimeError):
+    """Raised when an explicitly requested storage engine cannot be used."""
+
+
+@dataclass(frozen=True)
+class StorageCapabilities:
+    engine: str
+    available: bool
+    reason: Optional[str] = None
+    vector_backend: str = "none"
+    vector_available: bool = False
+    backup_restore: bool = False
+    migrations: bool = False
+    encrypted_archives: bool = False
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def as_dict(self) -> Dict[str, Any]:
+        return {
+            "engine": self.engine,
+            "available": self.available,
+            "reason": self.reason,
+            "vector_backend": self.vector_backend,
+            "vector_available": self.vector_available,
+            "backup_restore": self.backup_restore,
+            "migrations": self.migrations,
+            "encrypted_archives": self.encrypted_archives,
+            "metadata": self.metadata,
+        }
+
+
+class StorageEngine(ABC):
+    """Unified storage interface used by Brain Core.
+
+    The knowledge graph currently uses SQL directly, so ``connect`` is part of
+    the contract. Engines must fail loudly when unavailable; callers must not
+    silently fall back to SQLite after an explicit Postgres selection.
+    """
+
+    name: str
+
+    @abstractmethod
+    def capabilities(self) -> StorageCapabilities:
+        """Return an honest capability report."""
+
+    @abstractmethod
+    def initialize(self) -> Dict[str, Any]:
+        """Create required storage structures or raise ``StorageUnavailable``."""
+
+    @abstractmethod
+    def connect(self) -> Any:
+        """Return a DB-API-like connection for this engine."""
+
+    @abstractmethod
+    def backup(self, destination: Path) -> Dict[str, Any]:
+        """Create a faithful engine backup at ``destination``."""
+
+    @abstractmethod
+    def restore(self, source: Path) -> Dict[str, Any]:
+        """Restore a faithful engine backup from ``source``."""
+
+
+__all__ = ["StorageCapabilities", "StorageEngine", "StorageUnavailable"]

package/lattice_brain/storage/docker.py

@@ -0,0 +1,105 @@
+"""Explicit-consent Docker setup wizard for Postgres/pgvector."""
+
+from __future__ import annotations
+
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List
+
+
+@dataclass(frozen=True)
+class DockerPostgresPlan:
+    compose_path: Path
+    project_name: str
+    service_name: str = "postgres"
+    port: int = 5432
+
+    def command(self) -> List[str]:
+        return [
+            "docker",
+            "compose",
+            "-p",
+            self.project_name,
+            "-f",
+            str(self.compose_path),
+            "up",
+            "-d",
+            self.service_name,
+        ]
+
+
+class DockerPostgresWizard:
+    """Creates and starts a local Postgres container only after consent."""
+
+    def __init__(self, data_dir: Path, *, port: int = 5432) -> None:
+        self.data_dir = Path(data_dir)
+        self.port = int(port)
+
+    def write_compose(self) -> DockerPostgresPlan:
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+        compose = self.data_dir / "postgres.compose.yml"
+        compose.write_text(
+            f"""services:
+  postgres:
+    image: pgvector/pgvector:pg16
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: lattice_brain
+      POSTGRES_USER: lattice
+      POSTGRES_PASSWORD: lattice-local-only
+    ports:
+      - "127.0.0.1:{self.port}:5432"
+    volumes:
+      - ./postgres-data:/var/lib/postgresql/data
+""",
+            encoding="utf-8",
+        )
+        return DockerPostgresPlan(
+            compose_path=compose,
+            project_name="lattice-brain",
+            port=self.port,
+        )
+
+    def start(
+        self,
+        *,
+        consent: bool,
+        dry_run: bool = False,
+        runner=subprocess.run,
+    ) -> Dict[str, object]:
+        plan = self.write_compose()
+        if not consent:
+            return {
+                "status": "consent_required",
+                "started": False,
+                "compose_path": str(plan.compose_path),
+                "command": plan.command(),
+            }
+        if dry_run:
+            return {
+                "status": "dry_run",
+                "started": False,
+                "compose_path": str(plan.compose_path),
+                "command": plan.command(),
+            }
+        completed = runner(plan.command(), check=False, capture_output=True, text=True)
+        if completed.returncode != 0:
+            return {
+                "status": "failed",
+                "started": False,
+                "compose_path": str(plan.compose_path),
+                "returncode": completed.returncode,
+                "stdout": completed.stdout,
+                "stderr": completed.stderr,
+            }
+        return {
+            "status": "started",
+            "started": True,
+            "compose_path": str(plan.compose_path),
+            "stdout": completed.stdout,
+            "stderr": completed.stderr,
+        }
+
+
+__all__ = ["DockerPostgresPlan", "DockerPostgresWizard"]

package/lattice_brain/storage/factory.py

@@ -0,0 +1,31 @@
+"""StorageEngine construction from environment/config values."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Mapping
+
+from .base import StorageEngine, StorageUnavailable
+from .postgres import PostgresEngine
+from .sqlite import SQLiteEngine
+
+
+def storage_from_env(env: Mapping[str, str], *, data_dir: Path) -> StorageEngine:
+    engine = (env.get("LATTICEAI_STORAGE_ENGINE") or "sqlite").strip().lower()
+    if engine in {"", "sqlite"}:
+        return SQLiteEngine(Path(data_dir) / "knowledge_graph.sqlite")
+    if engine in {"postgres", "pg", "pgvector"}:
+        dsn = env.get("LATTICEAI_POSTGRES_DSN") or ""
+        if not dsn:
+            raise StorageUnavailable(
+                "LATTICEAI_STORAGE_ENGINE=postgres requires LATTICEAI_POSTGRES_DSN; "
+                "SQLite fallback is disabled for explicit Postgres selection."
+            )
+        return PostgresEngine(
+            dsn,
+            schema=env.get("LATTICEAI_POSTGRES_SCHEMA") or "lattice_brain",
+        )
+    raise StorageUnavailable(f"Unknown Brain Core storage engine: {engine}")
+
+
+__all__ = ["storage_from_env"]

package/lattice_brain/storage/migration.py

@@ -0,0 +1,190 @@
+"""Safe SQLite to Postgres migration tooling."""
+
+from __future__ import annotations
+
+import sqlite3
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List
+
+from .postgres import PostgresEngine, _quote_ident
+
+
+def _pg_type(sqlite_type: str) -> str:
+    t = str(sqlite_type or "").upper()
+    if "INT" in t:
+        return "bigint"
+    if any(token in t for token in ("REAL", "FLOA", "DOUB")):
+        return "double precision"
+    if "BLOB" in t:
+        return "bytea"
+    return "text"
+
+
+def _adapt_value(value: Any) -> Any:
+    if isinstance(value, memoryview):
+        return bytes(value)
+    return value
+
+
+@dataclass(frozen=True)
+class TablePlan:
+    name: str
+    columns: List[Dict[str, str]]
+    rows: int
+    conflict_key: str
+    conflict_columns: List[str]
+    rowid_available: bool
+
+
+class SQLiteToPostgresMigrator:
+    """Copies every user table from a Lattice SQLite brain into Postgres.
+
+    The migration is idempotent: tables with an ``id`` column upsert on ``id``;
+    tables without one use the preserved SQLite rowid in ``__source_rowid``.
+    SQLite remains untouched throughout.
+    """
+
+    def __init__(self, sqlite_path: Path, target: PostgresEngine) -> None:
+        self.sqlite_path = Path(sqlite_path)
+        self.target = target
+
+    def plan(self) -> Dict[str, Any]:
+        if not self.sqlite_path.exists():
+            raise FileNotFoundError(f"SQLite brain database not found: {self.sqlite_path}")
+        with sqlite3.connect(str(self.sqlite_path)) as conn:
+            conn.row_factory = sqlite3.Row
+            table_names = [
+                row["name"]
+                for row in conn.execute(
+                    """
+                    SELECT name FROM sqlite_master
+                    WHERE type='table' AND name NOT LIKE 'sqlite_%'
+                    ORDER BY name
+                    """
+                )
+            ]
+            tables = []
+            for table in table_names:
+                cols = [
+                    {"name": row["name"], "type": row["type"] or "TEXT"}
+                    for row in conn.execute(f"PRAGMA table_info({_quote_sqlite_ident(table)})")
+                ]
+                row_count = conn.execute(f"SELECT COUNT(*) FROM {_quote_sqlite_ident(table)}").fetchone()[0]
+                names = {c["name"] for c in cols}
+                rowid_available = _rowid_available(conn, table)
+                pk_columns = [
+                    row["name"]
+                    for row in sorted(
+                        conn.execute(f"PRAGMA table_info({_quote_sqlite_ident(table)})"),
+                        key=lambda item: int(item["pk"] or 0),
+                    )
+                    if int(row["pk"] or 0) > 0
+                ]
+                conflict_columns = (
+                    ["id"]
+                    if "id" in names
+                    else pk_columns
+                    if pk_columns
+                    else ["__source_rowid"]
+                    if rowid_available
+                    else []
+                )
+                if not conflict_columns:
+                    raise RuntimeError(
+                        f"Cannot safely migrate rowid-less SQLite table without a primary key: {table}"
+                    )
+                tables.append(
+                    TablePlan(
+                        name=table,
+                        columns=cols,
+                        rows=int(row_count),
+                        conflict_key=conflict_columns[0],
+                        conflict_columns=conflict_columns,
+                        rowid_available=rowid_available,
+                    )
+                )
+        return {
+            "source": str(self.sqlite_path),
+            "target_engine": self.target.name,
+            "target_schema": self.target.config.schema,
+            "tables": [table.__dict__ for table in tables],
+            "total_rows": sum(table.rows for table in tables),
+        }
+
+    def migrate(self, *, dry_run: bool = False) -> Dict[str, Any]:
+        plan = self.plan()
+        if dry_run:
+            return {"status": "planned", **plan}
+        schema = _quote_ident(self.target.config.schema)
+        copied: Dict[str, int] = {}
+        self.target.initialize()
+        with sqlite3.connect(str(self.sqlite_path)) as src, self.target.connect() as dst:
+            src.row_factory = sqlite3.Row
+            with dst.cursor() as cur:
+                for table in plan["tables"]:
+                    name = str(table["name"])
+                    cols = list(table["columns"])
+                    conflict_columns = list(table.get("conflict_columns") or [table["conflict_key"]])
+                    rowid_available = bool(table.get("rowid_available", True))
+                    pg_table = f"{schema}.{_quote_ident(name)}"
+                    defs = ["__source_rowid bigint NOT NULL"] if rowid_available else []
+                    for col in cols:
+                        defs.append(f"{_quote_ident(col['name'])} {_pg_type(col['type'])}")
+                    pk = ", ".join(_quote_ident(c) for c in conflict_columns)
+                    cur.execute(
+                        f"CREATE TABLE IF NOT EXISTS {pg_table} ({', '.join(defs)}, PRIMARY KEY ({pk}))"
+                    )
+                    if rowid_available:
+                        select_sql = (
+                            f"SELECT rowid AS __source_rowid, * FROM {_quote_sqlite_ident(name)} ORDER BY rowid"
+                        )
+                    else:
+                        order_by = ", ".join(_quote_sqlite_ident(c) for c in conflict_columns)
+                        select_sql = f"SELECT * FROM {_quote_sqlite_ident(name)} ORDER BY {order_by}"
+                    rows = src.execute(select_sql).fetchall()
+                    if not rows:
+                        copied[name] = 0
+                        continue
+                    columns = (["__source_rowid"] if rowid_available else []) + [c["name"] for c in cols]
+                    placeholders = ", ".join(["%s"] * len(columns))
+                    quoted_columns = ", ".join(_quote_ident(c) for c in columns)
+                    updates = ", ".join(
+                        f"{_quote_ident(c)} = EXCLUDED.{_quote_ident(c)}"
+                        for c in columns
+                        if c not in conflict_columns
+                    )
+                    conflict_action = f"DO UPDATE SET {updates}" if updates else "DO NOTHING"
+                    sql = (
+                        f"INSERT INTO {pg_table} ({quoted_columns}) VALUES ({placeholders}) "
+                        f"ON CONFLICT ({pk}) {conflict_action}"
+                    )
+                    cur.executemany(
+                        sql,
+                        [
+                            tuple(_adapt_value(row[col]) for col in columns)
+                            for row in rows
+                        ],
+                    )
+                    copied[name] = len(rows)
+        return {
+            "status": "migrated",
+            **plan,
+            "copied_rows": copied,
+            "total_copied_rows": sum(copied.values()),
+        }
+
+
+def _quote_sqlite_ident(value: str) -> str:
+    return '"' + str(value).replace('"', '""') + '"'
+
+
+def _rowid_available(conn: sqlite3.Connection, table: str) -> bool:
+    try:
+        conn.execute(f"SELECT rowid FROM {_quote_sqlite_ident(table)} LIMIT 1").fetchall()
+        return True
+    except sqlite3.OperationalError:
+        return False
+
+
+__all__ = ["SQLiteToPostgresMigrator", "TablePlan"]