ltcai 4.0.1 → 4.2.0

package/lattice_brain/storage/base.py

@@ -0,0 +1,72 @@
+"""StorageEngine contracts for the independent Brain Core package."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+
+class StorageUnavailable(RuntimeError):
+    """Raised when an explicitly requested storage engine cannot be used."""
+
+
+@dataclass(frozen=True)
+class StorageCapabilities:
+    engine: str
+    available: bool
+    reason: Optional[str] = None
+    vector_backend: str = "none"
+    vector_available: bool = False
+    backup_restore: bool = False
+    migrations: bool = False
+    encrypted_archives: bool = False
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def as_dict(self) -> Dict[str, Any]:
+        return {
+            "engine": self.engine,
+            "available": self.available,
+            "reason": self.reason,
+            "vector_backend": self.vector_backend,
+            "vector_available": self.vector_available,
+            "backup_restore": self.backup_restore,
+            "migrations": self.migrations,
+            "encrypted_archives": self.encrypted_archives,
+            "metadata": self.metadata,
+        }
+
+
+class StorageEngine(ABC):
+    """Unified storage interface used by Brain Core.
+
+    The knowledge graph currently uses SQL directly, so ``connect`` is part of
+    the contract. Engines must fail loudly when unavailable; callers must not
+    silently fall back to SQLite after an explicit Postgres selection.
+    """
+
+    name: str
+
+    @abstractmethod
+    def capabilities(self) -> StorageCapabilities:
+        """Return an honest capability report."""
+
+    @abstractmethod
+    def initialize(self) -> Dict[str, Any]:
+        """Create required storage structures or raise ``StorageUnavailable``."""
+
+    @abstractmethod
+    def connect(self) -> Any:
+        """Return a DB-API-like connection for this engine."""
+
+    @abstractmethod
+    def backup(self, destination: Path) -> Dict[str, Any]:
+        """Create a faithful engine backup at ``destination``."""
+
+    @abstractmethod
+    def restore(self, source: Path) -> Dict[str, Any]:
+        """Restore a faithful engine backup from ``source``."""
+
+
+__all__ = ["StorageCapabilities", "StorageEngine", "StorageUnavailable"]

package/lattice_brain/storage/docker.py

@@ -0,0 +1,105 @@
+"""Explicit-consent Docker setup wizard for Postgres/pgvector."""
+
+from __future__ import annotations
+
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List
+
+
+@dataclass(frozen=True)
+class DockerPostgresPlan:
+    compose_path: Path
+    project_name: str
+    service_name: str = "postgres"
+    port: int = 5432
+
+    def command(self) -> List[str]:
+        return [
+            "docker",
+            "compose",
+            "-p",
+            self.project_name,
+            "-f",
+            str(self.compose_path),
+            "up",
+            "-d",
+            self.service_name,
+        ]
+
+
+class DockerPostgresWizard:
+    """Creates and starts a local Postgres container only after consent."""
+
+    def __init__(self, data_dir: Path, *, port: int = 5432) -> None:
+        self.data_dir = Path(data_dir)
+        self.port = int(port)
+
+    def write_compose(self) -> DockerPostgresPlan:
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+        compose = self.data_dir / "postgres.compose.yml"
+        compose.write_text(
+            f"""services:
+  postgres:
+    image: pgvector/pgvector:pg16
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: lattice_brain
+      POSTGRES_USER: lattice
+      POSTGRES_PASSWORD: lattice-local-only
+    ports:
+      - "127.0.0.1:{self.port}:5432"
+    volumes:
+      - ./postgres-data:/var/lib/postgresql/data
+""",
+            encoding="utf-8",
+        )
+        return DockerPostgresPlan(
+            compose_path=compose,
+            project_name="lattice-brain",
+            port=self.port,
+        )
+
+    def start(
+        self,
+        *,
+        consent: bool,
+        dry_run: bool = False,
+        runner=subprocess.run,
+    ) -> Dict[str, object]:
+        plan = self.write_compose()
+        if not consent:
+            return {
+                "status": "consent_required",
+                "started": False,
+                "compose_path": str(plan.compose_path),
+                "command": plan.command(),
+            }
+        if dry_run:
+            return {
+                "status": "dry_run",
+                "started": False,
+                "compose_path": str(plan.compose_path),
+                "command": plan.command(),
+            }
+        completed = runner(plan.command(), check=False, capture_output=True, text=True)
+        if completed.returncode != 0:
+            return {
+                "status": "failed",
+                "started": False,
+                "compose_path": str(plan.compose_path),
+                "returncode": completed.returncode,
+                "stdout": completed.stdout,
+                "stderr": completed.stderr,
+            }
+        return {
+            "status": "started",
+            "started": True,
+            "compose_path": str(plan.compose_path),
+            "stdout": completed.stdout,
+            "stderr": completed.stderr,
+        }
+
+
+__all__ = ["DockerPostgresPlan", "DockerPostgresWizard"]

package/lattice_brain/storage/factory.py

@@ -0,0 +1,31 @@
+"""StorageEngine construction from environment/config values."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Mapping
+
+from .base import StorageEngine, StorageUnavailable
+from .postgres import PostgresEngine
+from .sqlite import SQLiteEngine
+
+
+def storage_from_env(env: Mapping[str, str], *, data_dir: Path) -> StorageEngine:
+    engine = (env.get("LATTICEAI_STORAGE_ENGINE") or "sqlite").strip().lower()
+    if engine in {"", "sqlite"}:
+        return SQLiteEngine(Path(data_dir) / "knowledge_graph.sqlite")
+    if engine in {"postgres", "pg", "pgvector"}:
+        dsn = env.get("LATTICEAI_POSTGRES_DSN") or ""
+        if not dsn:
+            raise StorageUnavailable(
+                "LATTICEAI_STORAGE_ENGINE=postgres requires LATTICEAI_POSTGRES_DSN; "
+                "SQLite fallback is disabled for explicit Postgres selection."
+            )
+        return PostgresEngine(
+            dsn,
+            schema=env.get("LATTICEAI_POSTGRES_SCHEMA") or "lattice_brain",
+        )
+    raise StorageUnavailable(f"Unknown Brain Core storage engine: {engine}")
+
+
+__all__ = ["storage_from_env"]

package/lattice_brain/storage/migration.py

@@ -0,0 +1,190 @@
+"""Safe SQLite to Postgres migration tooling."""
+
+from __future__ import annotations
+
+import sqlite3
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List
+
+from .postgres import PostgresEngine, _quote_ident
+
+
+def _pg_type(sqlite_type: str) -> str:
+    t = str(sqlite_type or "").upper()
+    if "INT" in t:
+        return "bigint"
+    if any(token in t for token in ("REAL", "FLOA", "DOUB")):
+        return "double precision"
+    if "BLOB" in t:
+        return "bytea"
+    return "text"
+
+
+def _adapt_value(value: Any) -> Any:
+    if isinstance(value, memoryview):
+        return bytes(value)
+    return value
+
+
+@dataclass(frozen=True)
+class TablePlan:
+    name: str
+    columns: List[Dict[str, str]]
+    rows: int
+    conflict_key: str
+    conflict_columns: List[str]
+    rowid_available: bool
+
+
+class SQLiteToPostgresMigrator:
+    """Copies every user table from a Lattice SQLite brain into Postgres.
+
+    The migration is idempotent: tables with an ``id`` column upsert on ``id``;
+    tables without one use the preserved SQLite rowid in ``__source_rowid``.
+    SQLite remains untouched throughout.
+    """
+
+    def __init__(self, sqlite_path: Path, target: PostgresEngine) -> None:
+        self.sqlite_path = Path(sqlite_path)
+        self.target = target
+
+    def plan(self) -> Dict[str, Any]:
+        if not self.sqlite_path.exists():
+            raise FileNotFoundError(f"SQLite brain database not found: {self.sqlite_path}")
+        with sqlite3.connect(str(self.sqlite_path)) as conn:
+            conn.row_factory = sqlite3.Row
+            table_names = [
+                row["name"]
+                for row in conn.execute(
+                    """
+                    SELECT name FROM sqlite_master
+                    WHERE type='table' AND name NOT LIKE 'sqlite_%'
+                    ORDER BY name
+                    """
+                )
+            ]
+            tables = []
+            for table in table_names:
+                cols = [
+                    {"name": row["name"], "type": row["type"] or "TEXT"}
+                    for row in conn.execute(f"PRAGMA table_info({_quote_sqlite_ident(table)})")
+                ]
+                row_count = conn.execute(f"SELECT COUNT(*) FROM {_quote_sqlite_ident(table)}").fetchone()[0]
+                names = {c["name"] for c in cols}
+                rowid_available = _rowid_available(conn, table)
+                pk_columns = [
+                    row["name"]
+                    for row in sorted(
+                        conn.execute(f"PRAGMA table_info({_quote_sqlite_ident(table)})"),
+                        key=lambda item: int(item["pk"] or 0),
+                    )
+                    if int(row["pk"] or 0) > 0
+                ]
+                conflict_columns = (
+                    ["id"]
+                    if "id" in names
+                    else pk_columns
+                    if pk_columns
+                    else ["__source_rowid"]
+                    if rowid_available
+                    else []
+                )
+                if not conflict_columns:
+                    raise RuntimeError(
+                        f"Cannot safely migrate rowid-less SQLite table without a primary key: {table}"
+                    )
+                tables.append(
+                    TablePlan(
+                        name=table,
+                        columns=cols,
+                        rows=int(row_count),
+                        conflict_key=conflict_columns[0],
+                        conflict_columns=conflict_columns,
+                        rowid_available=rowid_available,
+                    )
+                )
+        return {
+            "source": str(self.sqlite_path),
+            "target_engine": self.target.name,
+            "target_schema": self.target.config.schema,
+            "tables": [table.__dict__ for table in tables],
+            "total_rows": sum(table.rows for table in tables),
+        }
+
+    def migrate(self, *, dry_run: bool = False) -> Dict[str, Any]:
+        plan = self.plan()
+        if dry_run:
+            return {"status": "planned", **plan}
+        schema = _quote_ident(self.target.config.schema)
+        copied: Dict[str, int] = {}
+        self.target.initialize()
+        with sqlite3.connect(str(self.sqlite_path)) as src, self.target.connect() as dst:
+            src.row_factory = sqlite3.Row
+            with dst.cursor() as cur:
+                for table in plan["tables"]:
+                    name = str(table["name"])
+                    cols = list(table["columns"])
+                    conflict_columns = list(table.get("conflict_columns") or [table["conflict_key"]])
+                    rowid_available = bool(table.get("rowid_available", True))
+                    pg_table = f"{schema}.{_quote_ident(name)}"
+                    defs = ["__source_rowid bigint NOT NULL"] if rowid_available else []
+                    for col in cols:
+                        defs.append(f"{_quote_ident(col['name'])} {_pg_type(col['type'])}")
+                    pk = ", ".join(_quote_ident(c) for c in conflict_columns)
+                    cur.execute(
+                        f"CREATE TABLE IF NOT EXISTS {pg_table} ({', '.join(defs)}, PRIMARY KEY ({pk}))"
+                    )
+                    if rowid_available:
+                        select_sql = (
+                            f"SELECT rowid AS __source_rowid, * FROM {_quote_sqlite_ident(name)} ORDER BY rowid"
+                        )
+                    else:
+                        order_by = ", ".join(_quote_sqlite_ident(c) for c in conflict_columns)
+                        select_sql = f"SELECT * FROM {_quote_sqlite_ident(name)} ORDER BY {order_by}"
+                    rows = src.execute(select_sql).fetchall()
+                    if not rows:
+                        copied[name] = 0
+                        continue
+                    columns = (["__source_rowid"] if rowid_available else []) + [c["name"] for c in cols]
+                    placeholders = ", ".join(["%s"] * len(columns))
+                    quoted_columns = ", ".join(_quote_ident(c) for c in columns)
+                    updates = ", ".join(
+                        f"{_quote_ident(c)} = EXCLUDED.{_quote_ident(c)}"
+                        for c in columns
+                        if c not in conflict_columns
+                    )
+                    conflict_action = f"DO UPDATE SET {updates}" if updates else "DO NOTHING"
+                    sql = (
+                        f"INSERT INTO {pg_table} ({quoted_columns}) VALUES ({placeholders}) "
+                        f"ON CONFLICT ({pk}) {conflict_action}"
+                    )
+                    cur.executemany(
+                        sql,
+                        [
+                            tuple(_adapt_value(row[col]) for col in columns)
+                            for row in rows
+                        ],
+                    )
+                    copied[name] = len(rows)
+        return {
+            "status": "migrated",
+            **plan,
+            "copied_rows": copied,
+            "total_copied_rows": sum(copied.values()),
+        }
+
+
+def _quote_sqlite_ident(value: str) -> str:
+    return '"' + str(value).replace('"', '""') + '"'
+
+
+def _rowid_available(conn: sqlite3.Connection, table: str) -> bool:
+    try:
+        conn.execute(f"SELECT rowid FROM {_quote_sqlite_ident(table)} LIMIT 1").fetchall()
+        return True
+    except sqlite3.OperationalError:
+        return False
+
+
+__all__ = ["SQLiteToPostgresMigrator", "TablePlan"]

package/lattice_brain/storage/postgres.py

@@ -0,0 +1,123 @@
+"""Opt-in Postgres/pgvector storage engine for Brain Core scale mode."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict
+
+from .base import StorageCapabilities, StorageEngine, StorageUnavailable
+
+
+def _quote_ident(value: str) -> str:
+    return '"' + str(value).replace('"', '""') + '"'
+
+
+@dataclass(frozen=True)
+class PostgresConfig:
+    dsn: str
+    schema: str = "lattice_brain"
+
+
+class PostgresEngine(StorageEngine):
+    name = "postgres"
+
+    def __init__(self, dsn: str, *, schema: str = "lattice_brain") -> None:
+        self.config = PostgresConfig(dsn=dsn, schema=schema)
+
+    def _psycopg(self):
+        try:
+            import psycopg  # type: ignore
+        except Exception as exc:
+            raise StorageUnavailable(
+                "Postgres storage requires optional dependency 'psycopg'. "
+                "Install the postgres extra before selecting LATTICEAI_STORAGE_ENGINE=postgres."
+            ) from exc
+        return psycopg
+
+    def connect(self) -> Any:
+        if not self.config.dsn:
+            raise StorageUnavailable(
+                "Postgres storage requires LATTICEAI_POSTGRES_DSN; no SQLite fallback is attempted."
+            )
+        psycopg = self._psycopg()
+        return psycopg.connect(self.config.dsn)
+
+    def initialize(self) -> Dict[str, Any]:
+        schema = _quote_ident(self.config.schema)
+        with self.connect() as conn:
+            with conn.cursor() as cur:
+                cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
+                cur.execute(f"CREATE SCHEMA IF NOT EXISTS {schema}")
+                cur.execute(
+                    f"""
+                    CREATE TABLE IF NOT EXISTS {schema}.storage_meta (
+                      key text PRIMARY KEY,
+                      value text NOT NULL,
+                      updated_at timestamptz NOT NULL DEFAULT now()
+                    )
+                    """
+                )
+                cur.execute(
+                    f"""
+                    CREATE TABLE IF NOT EXISTS {schema}.brain_vectors (
+                      item_id text PRIMARY KEY,
+                      item_type text NOT NULL,
+                      source_node text NOT NULL,
+                      text_hash text NOT NULL,
+                      embedding vector,
+                      embedding_dim integer NOT NULL,
+                      embedding_model text NOT NULL,
+                      metadata_json jsonb NOT NULL DEFAULT '{{}}'::jsonb,
+                      indexed_at timestamptz NOT NULL DEFAULT now()
+                    )
+                    """
+                )
+                cur.execute(
+                    f"""
+                    INSERT INTO {schema}.storage_meta(key, value)
+                    VALUES ('engine', 'postgres')
+                    ON CONFLICT (key) DO UPDATE
+                      SET value = EXCLUDED.value, updated_at = now()
+                    """
+                )
+        return {"engine": self.name, "schema": self.config.schema}
+
+    def capabilities(self) -> StorageCapabilities:
+        try:
+            with self.connect() as conn:
+                with conn.cursor() as cur:
+                    cur.execute("SELECT extname FROM pg_extension WHERE extname='vector'")
+                    pgvector = cur.fetchone() is not None
+        except Exception as exc:
+            return StorageCapabilities(
+                engine=self.name,
+                available=False,
+                reason=str(exc),
+                vector_backend="pgvector",
+                metadata={"schema": self.config.schema},
+            )
+        return StorageCapabilities(
+            engine=self.name,
+            available=True,
+            reason=None if pgvector else "pgvector extension is not installed",
+            vector_backend="pgvector",
+            vector_available=pgvector,
+            backup_restore=False,
+            migrations=True,
+            encrypted_archives=False,
+            metadata={"schema": self.config.schema},
+        )
+
+    def backup(self, destination: Path) -> Dict[str, Any]:
+        raise StorageUnavailable(
+            "Postgres logical backup is not implemented inside the app; use pg_dump for this engine."
+        )
+
+    def restore(self, source: Path) -> Dict[str, Any]:
+        raise StorageUnavailable(
+            "Postgres restore is not implemented inside the app; use pg_restore/psql for this engine."
+        )
+
+
+__all__ = ["PostgresConfig", "PostgresEngine", "_quote_ident"]

package/lattice_brain/storage/sqlite.py

@@ -0,0 +1,128 @@
+"""SQLite storage engine for Brain Core.
+
+SQLite is the default and remains fully local-first. sqlite-vec is detected and
+loaded when present; otherwise the existing brute-force cosine path remains the
+honest, real fallback and is surfaced in capability reports.
+"""
+
+from __future__ import annotations
+
+import shutil
+import sqlite3
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from .base import StorageCapabilities, StorageEngine
+
+
+def _load_sqlite_vec(conn: sqlite3.Connection) -> tuple[bool, Optional[str]]:
+    try:
+        import sqlite_vec  # type: ignore
+    except Exception as exc:
+        return False, f"sqlite-vec Python package not installed: {exc}"
+    try:
+        conn.enable_load_extension(True)
+    except Exception:
+        pass
+    try:
+        sqlite_vec.load(conn)
+    except Exception as exc:
+        return False, f"sqlite-vec extension failed to load: {exc}"
+    return True, None
+
+
+class SQLiteEngine(StorageEngine):
+    name = "sqlite"
+
+    def __init__(self, db_path: Path, *, load_vec: bool = True) -> None:
+        self.db_path = Path(db_path)
+        self.load_vec = bool(load_vec)
+        self._sqlite_vec_loaded = False
+        self._sqlite_vec_reason: Optional[str] = None
+
+    def connect(self) -> sqlite3.Connection:
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        conn = sqlite3.connect(str(self.db_path))
+        conn.row_factory = sqlite3.Row
+        conn.execute("PRAGMA journal_mode=WAL")
+        conn.execute("PRAGMA foreign_keys=ON")
+        if self.load_vec:
+            loaded, reason = _load_sqlite_vec(conn)
+            self._sqlite_vec_loaded = loaded
+            self._sqlite_vec_reason = reason
+        return conn
+
+    def initialize(self) -> Dict[str, Any]:
+        with self.connect() as conn:
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS storage_meta (
+                  key TEXT PRIMARY KEY,
+                  value TEXT NOT NULL
+                )
+                """
+            )
+            conn.execute(
+                "INSERT OR REPLACE INTO storage_meta(key, value) VALUES ('engine', 'sqlite')"
+            )
+        return {"engine": self.name, "db_path": str(self.db_path)}
+
+    def capabilities(self) -> StorageCapabilities:
+        if not self.db_path.parent.exists():
+            return StorageCapabilities(
+                engine=self.name,
+                available=True,
+                vector_backend="bruteforce-cosine",
+                vector_available=True,
+                backup_restore=True,
+                migrations=True,
+                encrypted_archives=True,
+                metadata={"db_path": str(self.db_path), "sqlite_vec_loaded": False},
+            )
+        # Probe on demand so status is accurate even before the graph opens.
+        try:
+            with self.connect():
+                pass
+        except Exception as exc:
+            return StorageCapabilities(
+                engine=self.name,
+                available=False,
+                reason=str(exc),
+                metadata={"db_path": str(self.db_path)},
+            )
+        vector_backend = "sqlite-vec" if self._sqlite_vec_loaded else "bruteforce-cosine"
+        return StorageCapabilities(
+            engine=self.name,
+            available=True,
+            reason=None if self._sqlite_vec_loaded else self._sqlite_vec_reason,
+            vector_backend=vector_backend,
+            vector_available=True,
+            backup_restore=True,
+            migrations=True,
+            encrypted_archives=True,
+            metadata={
+                "db_path": str(self.db_path),
+                "sqlite_vec_loaded": self._sqlite_vec_loaded,
+            },
+        )
+
+    def backup(self, destination: Path) -> Dict[str, Any]:
+        dest = Path(destination)
+        dest.parent.mkdir(parents=True, exist_ok=True)
+        with self.connect() as src, sqlite3.connect(str(dest)) as dst:
+            src.backup(dst)
+        return {"engine": self.name, "path": str(dest), "bytes": dest.stat().st_size}
+
+    def restore(self, source: Path) -> Dict[str, Any]:
+        src = Path(source)
+        if not src.exists():
+            raise FileNotFoundError(f"SQLite backup not found: {src}")
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        for sibling in (self.db_path, Path(str(self.db_path) + "-wal"), Path(str(self.db_path) + "-shm")):
+            if sibling.exists():
+                sibling.unlink()
+        shutil.copyfile(src, self.db_path)
+        return {"engine": self.name, "restored": True, "path": str(self.db_path)}
+
+
+__all__ = ["SQLiteEngine"]

package/lattice_brain/store.py

@@ -0,0 +1,3 @@
+from latticeai.brain.store import KnowledgeGraphStore
+
+__all__ = ["KnowledgeGraphStore"]

package/lattice_brain/write_master.py

@@ -0,0 +1 @@
+from latticeai.brain.write_master import *  # noqa: F401,F403

package/latticeai/__init__.py

@@ -1,3 +1,3 @@
 """Lattice AI - modular server package."""
 
-__version__ = "4.0.1"
+__version__ = "4.2.0"