ltcai 11.5.2 → 11.7.0

package/lattice_brain/sensitivity.py

@@ -1,94 +0,0 @@
-"""Which local sources are never allowed to leave the machine.
-
-This lives in Brain Core rather than in the app layer because it is a property
-of the *data*, not of any transport: a node stamped never-leaves stays that way
-whether it is read by the cloud path, an export, or a future surface nobody has
-written yet. Brain Core cannot import from ``latticeai`` (see
-``tests/unit/test_import_guard.py``), so the app-layer network boundary imports
-this, not the other way round.
-
-Deliberately narrow. These are locations whose *entire purpose* is to hold
-secrets, so a match is a fact about the path rather than a guess about content.
-A broader heuristic — scanning text for things that look like keys — would
-produce false confidence in both directions: it would miss secrets in unusual
-shapes and quarantine ordinary notes that merely discuss credentials.
-"""
-
-from __future__ import annotations
-
-from typing import Any, Optional
-
-# Path fragments checked case-insensitively against the POSIX form of the path.
-SENSITIVE_PATH_FRAGMENTS: tuple = (
-    "/.ssh/",
-    "/.gnupg/",
-    "/.aws/",
-    "/.kube/",
-    "/.docker/config.json",
-    "/.netrc",
-    "/.npmrc",
-    "/.pypirc",
-    "/.git-credentials",
-    "id_rsa",
-    "id_ed25519",
-    ".pem",
-    ".p12",
-    ".pfx",
-    ".keystore",
-)
-
-# Exact filenames (in any directory) that are secret-bearing by convention.
-SENSITIVE_FILENAMES: frozenset = frozenset({
-    ".env",
-    ".env.local",
-    ".env.production",
-    ".env.development",
-    "credentials",
-    "secrets.yaml",
-    "secrets.yml",
-    "secrets.json",
-})
-
-#: Metadata key stamped on a node that must never leave the machine.
-LOCAL_ONLY_FLAG = "local_only"
-#: Companion key holding the human-readable reason.
-LOCAL_ONLY_REASON = "local_only_reason"
-
-
-def sensitive_reason_for_path(path: Any) -> Optional[str]:
-    """Return why ``path`` is never-leaves, or ``None`` if it is ordinary."""
-    if not path:
-        return None
-    lowered = str(path).replace("\\", "/").lower()
-    name = lowered.rsplit("/", 1)[-1]
-    if name in SENSITIVE_FILENAMES:
-        return f"{name!r} is a secret-bearing filename"
-    for fragment in SENSITIVE_PATH_FRAGMENTS:
-        if fragment in lowered:
-            return f"path contains {fragment!r}"
-    return None
-
-
-def stamp_sensitivity(metadata: dict, path: Any) -> Optional[str]:
-    """Stamp never-leaves onto ``metadata`` when ``path`` warrants it.
-
-    Returns the reason it stamped, or ``None``. Never clears an existing flag —
-    a user or another rule may have set it for a reason this function cannot
-    see, and downgrading a never-leaves marker is not a decision code should
-    make on its own.
-    """
-    reason = sensitive_reason_for_path(path)
-    if reason:
-        metadata[LOCAL_ONLY_FLAG] = True
-        metadata.setdefault(LOCAL_ONLY_REASON, reason)
-    return reason
-
-
-__all__ = [
-    "SENSITIVE_PATH_FRAGMENTS",
-    "SENSITIVE_FILENAMES",
-    "LOCAL_ONLY_FLAG",
-    "LOCAL_ONLY_REASON",
-    "sensitive_reason_for_path",
-    "stamp_sensitivity",
-]

package/lattice_brain/storage/__init__.py

@@ -1,22 +0,0 @@
-"""Pluggable storage layer for lattice-brain."""
-
-from .base import StorageCapabilities, StorageEngine, StorageUnavailable
-from .docker import DockerPostgresPlan, DockerPostgresWizard
-from .factory import storage_from_env
-from .migration import SQLiteToPostgresMigrator, TablePlan
-from .postgres import PostgresConfig, PostgresEngine
-from .sqlite import SQLiteEngine
-
-__all__ = [
-    "DockerPostgresPlan",
-    "DockerPostgresWizard",
-    "PostgresConfig",
-    "PostgresEngine",
-    "SQLiteEngine",
-    "SQLiteToPostgresMigrator",
-    "StorageCapabilities",
-    "StorageEngine",
-    "StorageUnavailable",
-    "TablePlan",
-    "storage_from_env",
-]

package/lattice_brain/storage/base.py

@@ -1,100 +0,0 @@
-"""StorageEngine contracts for the independent Brain Core package."""
-
-from __future__ import annotations
-
-from abc import ABC, abstractmethod
-from contextlib import contextmanager
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Any, Dict, Iterator, Optional
-
-
-class StorageUnavailable(RuntimeError):
-    """Raised when an explicitly requested storage engine cannot be used."""
-
-
-@dataclass(frozen=True)
-class StorageCapabilities:
-    engine: str
-    available: bool
-    reason: Optional[str] = None
-    vector_backend: str = "none"
-    vector_available: bool = False
-    backup_restore: bool = False
-    migrations: bool = False
-    encrypted_archives: bool = False
-    metadata: Dict[str, Any] = field(default_factory=dict)
-
-    def as_dict(self) -> Dict[str, Any]:
-        return {
-            "engine": self.engine,
-            "available": self.available,
-            "reason": self.reason,
-            "vector_backend": self.vector_backend,
-            "vector_available": self.vector_available,
-            "backup_restore": self.backup_restore,
-            "migrations": self.migrations,
-            "encrypted_archives": self.encrypted_archives,
-            "metadata": self.metadata,
-        }
-
-
-class StorageEngine(ABC):
-    """Unified storage interface used by Brain Core.
-
-    The knowledge graph currently uses SQL directly, so ``connect`` is part of
-    the contract. Engines must fail loudly when unavailable; callers must not
-    silently fall back to SQLite after an explicit Postgres selection.
-    """
-
-    name: str
-
-    @abstractmethod
-    def capabilities(self) -> StorageCapabilities:
-        """Return an honest capability report."""
-
-    @abstractmethod
-    def initialize(self) -> Dict[str, Any]:
-        """Create required storage structures or raise ``StorageUnavailable``."""
-
-    @abstractmethod
-    def connect(self) -> Any:
-        """Return a DB-API-like connection for this engine.
-
-        The caller owns the connection and must close it. Prefer
-        :meth:`session`, which does that for you — a bare ``connect()`` is only
-        correct when the statement cannot run inside a transaction (``VACUUM``)
-        or when the connection must outlive one block.
-        """
-
-    @contextmanager
-    def session(self) -> Iterator[Any]:
-        """Transactional connection that is always closed.
-
-        ``with sqlite3.connect(...) as conn`` commits or rolls back — it does
-        **not** close. Every site that relied on that leaked a file descriptor
-        until the connection was garbage collected, which CPython usually did
-        promptly enough to hide it. Anything holding a frame alive (a coverage
-        tracer, a profiler, a logged traceback) delays that collection and the
-        descriptors accumulate until the process hits ``EMFILE``.
-
-        This yields inside ``with conn`` so commit/rollback semantics are
-        unchanged, then closes in ``finally``.
-        """
-        conn = self.connect()
-        try:
-            with conn:
-                yield conn
-        finally:
-            conn.close()
-
-    @abstractmethod
-    def backup(self, destination: Path) -> Dict[str, Any]:
-        """Create a faithful engine backup at ``destination``."""
-
-    @abstractmethod
-    def restore(self, source: Path) -> Dict[str, Any]:
-        """Restore a faithful engine backup from ``source``."""
-
-
-__all__ = ["StorageCapabilities", "StorageEngine", "StorageUnavailable"]

package/lattice_brain/storage/docker.py

@@ -1,105 +0,0 @@
-"""Explicit-consent Docker setup wizard for Postgres/pgvector."""
-
-from __future__ import annotations
-
-import subprocess
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Dict, List
-
-
-@dataclass(frozen=True)
-class DockerPostgresPlan:
-    compose_path: Path
-    project_name: str
-    service_name: str = "postgres"
-    port: int = 5432
-
-    def command(self) -> List[str]:
-        return [
-            "docker",
-            "compose",
-            "-p",
-            self.project_name,
-            "-f",
-            str(self.compose_path),
-            "up",
-            "-d",
-            self.service_name,
-        ]
-
-
-class DockerPostgresWizard:
-    """Creates and starts a local Postgres container only after consent."""
-
-    def __init__(self, data_dir: Path, *, port: int = 5432) -> None:
-        self.data_dir = Path(data_dir)
-        self.port = int(port)
-
-    def write_compose(self) -> DockerPostgresPlan:
-        self.data_dir.mkdir(parents=True, exist_ok=True)
-        compose = self.data_dir / "postgres.compose.yml"
-        compose.write_text(
-            f"""services:
-  postgres:
-    image: pgvector/pgvector:pg16
-    restart: unless-stopped
-    environment:
-      POSTGRES_DB: lattice_brain
-      POSTGRES_USER: lattice
-      POSTGRES_PASSWORD: lattice-local-only
-    ports:
-      - "127.0.0.1:{self.port}:5432"
-    volumes:
-      - ./postgres-data:/var/lib/postgresql/data
-""",
-            encoding="utf-8",
-        )
-        return DockerPostgresPlan(
-            compose_path=compose,
-            project_name="lattice-brain",
-            port=self.port,
-        )
-
-    def start(
-        self,
-        *,
-        consent: bool,
-        dry_run: bool = False,
-        runner=subprocess.run,
-    ) -> Dict[str, object]:
-        plan = self.write_compose()
-        if not consent:
-            return {
-                "status": "consent_required",
-                "started": False,
-                "compose_path": str(plan.compose_path),
-                "command": plan.command(),
-            }
-        if dry_run:
-            return {
-                "status": "dry_run",
-                "started": False,
-                "compose_path": str(plan.compose_path),
-                "command": plan.command(),
-            }
-        completed = runner(plan.command(), check=False, capture_output=True, text=True)
-        if completed.returncode != 0:
-            return {
-                "status": "failed",
-                "started": False,
-                "compose_path": str(plan.compose_path),
-                "returncode": completed.returncode,
-                "stdout": completed.stdout,
-                "stderr": completed.stderr,
-            }
-        return {
-            "status": "started",
-            "started": True,
-            "compose_path": str(plan.compose_path),
-            "stdout": completed.stdout,
-            "stderr": completed.stderr,
-        }
-
-
-__all__ = ["DockerPostgresPlan", "DockerPostgresWizard"]

package/lattice_brain/storage/factory.py

@@ -1,31 +0,0 @@
-"""StorageEngine construction from environment/config values."""
-
-from __future__ import annotations
-
-from pathlib import Path
-from typing import Mapping
-
-from .base import StorageEngine, StorageUnavailable
-from .postgres import PostgresEngine
-from .sqlite import SQLiteEngine
-
-
-def storage_from_env(env: Mapping[str, str], *, data_dir: Path) -> StorageEngine:
-    engine = (env.get("LATTICEAI_STORAGE_ENGINE") or "sqlite").strip().lower()
-    if engine in {"", "sqlite"}:
-        return SQLiteEngine(Path(data_dir) / "knowledge_graph.sqlite")
-    if engine in {"postgres", "pg", "pgvector"}:
-        dsn = env.get("LATTICEAI_POSTGRES_DSN") or ""
-        if not dsn:
-            raise StorageUnavailable(
-                "LATTICEAI_STORAGE_ENGINE=postgres requires LATTICEAI_POSTGRES_DSN; "
-                "SQLite fallback is disabled for explicit Postgres selection."
-            )
-        return PostgresEngine(
-            dsn,
-            schema=env.get("LATTICEAI_POSTGRES_SCHEMA") or "lattice_brain",
-        )
-    raise StorageUnavailable(f"Unknown Brain Core storage engine: {engine}")
-
-
-__all__ = ["storage_from_env"]

package/lattice_brain/storage/migration.py

@@ -1,191 +0,0 @@
-"""Safe SQLite to Postgres migration tooling."""
-
-from __future__ import annotations
-
-import sqlite3
-from contextlib import closing
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Dict, List
-
-from .postgres import PostgresEngine, _quote_ident
-
-
-def _pg_type(sqlite_type: str) -> str:
-    t = str(sqlite_type or "").upper()
-    if "INT" in t:
-        return "bigint"
-    if any(token in t for token in ("REAL", "FLOA", "DOUB")):
-        return "double precision"
-    if "BLOB" in t:
-        return "bytea"
-    return "text"
-
-
-def _adapt_value(value: Any) -> Any:
-    if isinstance(value, memoryview):
-        return bytes(value)
-    return value
-
-
-@dataclass(frozen=True)
-class TablePlan:
-    name: str
-    columns: List[Dict[str, str]]
-    rows: int
-    conflict_key: str
-    conflict_columns: List[str]
-    rowid_available: bool
-
-
-class SQLiteToPostgresMigrator:
-    """Copies every user table from a Lattice SQLite brain into Postgres.
-
-    The migration is idempotent: tables with an ``id`` column upsert on ``id``;
-    tables without one use the preserved SQLite rowid in ``__source_rowid``.
-    SQLite remains untouched throughout.
-    """
-
-    def __init__(self, sqlite_path: Path, target: PostgresEngine) -> None:
-        self.sqlite_path = Path(sqlite_path)
-        self.target = target
-
-    def plan(self) -> Dict[str, Any]:
-        if not self.sqlite_path.exists():
-            raise FileNotFoundError(f"SQLite brain database not found: {self.sqlite_path}")
-        with closing(sqlite3.connect(str(self.sqlite_path))) as conn, conn:
-            conn.row_factory = sqlite3.Row
-            table_names = [
-                row["name"]
-                for row in conn.execute(
-                    """
-                    SELECT name FROM sqlite_master
-                    WHERE type='table' AND name NOT LIKE 'sqlite_%'
-                    ORDER BY name
-                    """
-                )
-            ]
-            tables = []
-            for table in table_names:
-                cols = [
-                    {"name": row["name"], "type": row["type"] or "TEXT"}
-                    for row in conn.execute(f"PRAGMA table_info({_quote_sqlite_ident(table)})")
-                ]
-                row_count = conn.execute(f"SELECT COUNT(*) FROM {_quote_sqlite_ident(table)}").fetchone()[0]
-                names = {c["name"] for c in cols}
-                rowid_available = _rowid_available(conn, table)
-                pk_columns = [
-                    row["name"]
-                    for row in sorted(
-                        conn.execute(f"PRAGMA table_info({_quote_sqlite_ident(table)})"),
-                        key=lambda item: int(item["pk"] or 0),
-                    )
-                    if int(row["pk"] or 0) > 0
-                ]
-                conflict_columns = (
-                    ["id"]
-                    if "id" in names
-                    else pk_columns
-                    if pk_columns
-                    else ["__source_rowid"]
-                    if rowid_available
-                    else []
-                )
-                if not conflict_columns:
-                    raise RuntimeError(
-                        f"Cannot safely migrate rowid-less SQLite table without a primary key: {table}"
-                    )
-                tables.append(
-                    TablePlan(
-                        name=table,
-                        columns=cols,
-                        rows=int(row_count),
-                        conflict_key=conflict_columns[0],
-                        conflict_columns=conflict_columns,
-                        rowid_available=rowid_available,
-                    )
-                )
-        return {
-            "source": str(self.sqlite_path),
-            "target_engine": self.target.name,
-            "target_schema": self.target.config.schema,
-            "tables": [table.__dict__ for table in tables],
-            "total_rows": sum(table.rows for table in tables),
-        }
-
-    def migrate(self, *, dry_run: bool = False) -> Dict[str, Any]:
-        plan = self.plan()
-        if dry_run:
-            return {"status": "planned", **plan}
-        schema = _quote_ident(self.target.config.schema)
-        copied: Dict[str, int] = {}
-        self.target.initialize()
-        with closing(sqlite3.connect(str(self.sqlite_path))) as src, self.target.session() as dst:
-            src.row_factory = sqlite3.Row
-            with dst.cursor() as cur:
-                for table in plan["tables"]:
-                    name = str(table["name"])
-                    cols = list(table["columns"])
-                    conflict_columns = list(table.get("conflict_columns") or [table["conflict_key"]])
-                    rowid_available = bool(table.get("rowid_available", True))
-                    pg_table = f"{schema}.{_quote_ident(name)}"
-                    defs = ["__source_rowid bigint NOT NULL"] if rowid_available else []
-                    for col in cols:
-                        defs.append(f"{_quote_ident(col['name'])} {_pg_type(col['type'])}")
-                    pk = ", ".join(_quote_ident(c) for c in conflict_columns)
-                    cur.execute(
-                        f"CREATE TABLE IF NOT EXISTS {pg_table} ({', '.join(defs)}, PRIMARY KEY ({pk}))"
-                    )
-                    if rowid_available:
-                        select_sql = (
-                            f"SELECT rowid AS __source_rowid, * FROM {_quote_sqlite_ident(name)} ORDER BY rowid"
-                        )
-                    else:
-                        order_by = ", ".join(_quote_sqlite_ident(c) for c in conflict_columns)
-                        select_sql = f"SELECT * FROM {_quote_sqlite_ident(name)} ORDER BY {order_by}"
-                    rows = src.execute(select_sql).fetchall()
-                    if not rows:
-                        copied[name] = 0
-                        continue
-                    columns = (["__source_rowid"] if rowid_available else []) + [c["name"] for c in cols]
-                    placeholders = ", ".join(["%s"] * len(columns))
-                    quoted_columns = ", ".join(_quote_ident(c) for c in columns)
-                    updates = ", ".join(
-                        f"{_quote_ident(c)} = EXCLUDED.{_quote_ident(c)}"
-                        for c in columns
-                        if c not in conflict_columns
-                    )
-                    conflict_action = f"DO UPDATE SET {updates}" if updates else "DO NOTHING"
-                    sql = (
-                        f"INSERT INTO {pg_table} ({quoted_columns}) VALUES ({placeholders}) "
-                        f"ON CONFLICT ({pk}) {conflict_action}"
-                    )
-                    cur.executemany(
-                        sql,
-                        [
-                            tuple(_adapt_value(row[col]) for col in columns)
-                            for row in rows
-                        ],
-                    )
-                    copied[name] = len(rows)
-        return {
-            "status": "migrated",
-            **plan,
-            "copied_rows": copied,
-            "total_copied_rows": sum(copied.values()),
-        }
-
-
-def _quote_sqlite_ident(value: str) -> str:
-    return '"' + str(value).replace('"', '""') + '"'
-
-
-def _rowid_available(conn: sqlite3.Connection, table: str) -> bool:
-    try:
-        conn.execute(f"SELECT rowid FROM {_quote_sqlite_ident(table)} LIMIT 1").fetchall()
-        return True
-    except sqlite3.OperationalError:
-        return False
-
-
-__all__ = ["SQLiteToPostgresMigrator", "TablePlan"]

package/lattice_brain/storage/postgres.py

@@ -1,123 +0,0 @@
-"""Opt-in Postgres/pgvector storage engine for Brain Core scale mode."""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-from pathlib import Path
-from typing import Any, Dict
-
-from .base import StorageCapabilities, StorageEngine, StorageUnavailable
-
-
-def _quote_ident(value: str) -> str:
-    return '"' + str(value).replace('"', '""') + '"'
-
-
-@dataclass(frozen=True)
-class PostgresConfig:
-    dsn: str
-    schema: str = "lattice_brain"
-
-
-class PostgresEngine(StorageEngine):
-    name = "postgres"
-
-    def __init__(self, dsn: str, *, schema: str = "lattice_brain") -> None:
-        self.config = PostgresConfig(dsn=dsn, schema=schema)
-
-    def _psycopg(self):
-        try:
-            import psycopg  # type: ignore
-        except Exception as exc:
-            raise StorageUnavailable(
-                "Postgres storage requires optional dependency 'psycopg'. "
-                "Install the postgres extra before selecting LATTICEAI_STORAGE_ENGINE=postgres."
-            ) from exc
-        return psycopg
-
-    def connect(self) -> Any:
-        if not self.config.dsn:
-            raise StorageUnavailable(
-                "Postgres storage requires LATTICEAI_POSTGRES_DSN; no SQLite fallback is attempted."
-            )
-        psycopg = self._psycopg()
-        return psycopg.connect(self.config.dsn)
-
-    def initialize(self) -> Dict[str, Any]:
-        schema = _quote_ident(self.config.schema)
-        with self.session() as conn:
-            with conn.cursor() as cur:
-                cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
-                cur.execute(f"CREATE SCHEMA IF NOT EXISTS {schema}")
-                cur.execute(
-                    f"""
-                    CREATE TABLE IF NOT EXISTS {schema}.storage_meta (
-                      key text PRIMARY KEY,
-                      value text NOT NULL,
-                      updated_at timestamptz NOT NULL DEFAULT now()
-                    )
-                    """
-                )
-                cur.execute(
-                    f"""
-                    CREATE TABLE IF NOT EXISTS {schema}.brain_vectors (
-                      item_id text PRIMARY KEY,
-                      item_type text NOT NULL,
-                      source_node text NOT NULL,
-                      text_hash text NOT NULL,
-                      embedding vector,
-                      embedding_dim integer NOT NULL,
-                      embedding_model text NOT NULL,
-                      metadata_json jsonb NOT NULL DEFAULT '{{}}'::jsonb,
-                      indexed_at timestamptz NOT NULL DEFAULT now()
-                    )
-                    """
-                )
-                cur.execute(
-                    f"""
-                    INSERT INTO {schema}.storage_meta(key, value)
-                    VALUES ('engine', 'postgres')
-                    ON CONFLICT (key) DO UPDATE
-                      SET value = EXCLUDED.value, updated_at = now()
-                    """
-                )
-        return {"engine": self.name, "schema": self.config.schema}
-
-    def capabilities(self) -> StorageCapabilities:
-        try:
-            with self.session() as conn:
-                with conn.cursor() as cur:
-                    cur.execute("SELECT extname FROM pg_extension WHERE extname='vector'")
-                    pgvector = cur.fetchone() is not None
-        except Exception as exc:
-            return StorageCapabilities(
-                engine=self.name,
-                available=False,
-                reason=str(exc),
-                vector_backend="pgvector",
-                metadata={"schema": self.config.schema},
-            )
-        return StorageCapabilities(
-            engine=self.name,
-            available=True,
-            reason=None if pgvector else "pgvector extension is not installed",
-            vector_backend="pgvector",
-            vector_available=pgvector,
-            backup_restore=False,
-            migrations=True,
-            encrypted_archives=False,
-            metadata={"schema": self.config.schema},
-        )
-
-    def backup(self, destination: Path) -> Dict[str, Any]:
-        raise StorageUnavailable(
-            "Postgres logical backup is not implemented inside the app; use pg_dump for this engine."
-        )
-
-    def restore(self, source: Path) -> Dict[str, Any]:
-        raise StorageUnavailable(
-            "Postgres restore is not implemented inside the app; use pg_restore/psql for this engine."
-        )
-
-
-__all__ = ["PostgresConfig", "PostgresEngine", "_quote_ident"]