ltcai 11.5.2 → 11.7.0

package/lattice_brain/graph/vector_index/hnsw.py

@@ -1,290 +0,0 @@
-"""Approximate nearest-neighbour index backed by optional ``hnswlib``.
-
-``hnswlib`` is a compiled extension. It is **not** a core dependency: the
-install stays pure-Python-plus-wheels for everyone who never asked for ANN,
-and this backend is reachable only through the ``hnsw`` optional extra
-(``pip install "ltcai[hnsw]"``) plus an explicit
-``LATTICEAI_VECTOR_INDEX=hnsw``. When the module is missing the import is
-caught here and reported as a reason string; the selector then falls back to
-the exact scan and says so, rather than failing a search.
-
-The graph search index is a **derivative**. SQLite remains the source of
-truth, so the ``.hnsw`` sidecar next to the brain database is disposable: if
-it is deleted, corrupt, or built for a different embedder/row-set, the
-fingerprint check rejects it and the caller rebuilds from
-``vector_embeddings``.
-
-Approximation is reported, never hidden: this backend sets ``approx = True``
-and ``exhaustive = False``, and ``scripts/bench_vector_index.py`` measures
-recall@k against the brute-force backend so the cost of the speed is a
-number, not a promise.
-"""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
-
-from .base import IndexItem, IndexStats, ScoredId, score_floor
-
-HNSW_BACKEND = "hnsw"
-HNSWLIB_MODULE = "hnswlib"
-#: Sidecar suffix for the persisted graph (a derivative of the SQLite rows).
-HNSW_SUFFIX = ".hnsw"
-#: Companion JSON: fingerprint + label→id mapping. Without it the binary
-#: graph is unreadable, because hnswlib only stores integer labels.
-HNSW_META_SUFFIX = ".hnsw.meta.json"
-
-
-def load_hnswlib() -> Tuple[Optional[Any], Optional[str]]:
-    """Import ``hnswlib`` or explain why it is unavailable (never raises)."""
-    try:
-        import hnswlib  # noqa: PLC0415 — guarded, optional, import-on-demand
-    except Exception as exc:  # noqa: BLE001 — any import failure is "no ANN"
-        return None, (
-            f"{HNSWLIB_MODULE} is not available ({exc}); install the optional "
-            'extra with `pip install "ltcai[hnsw]"`'
-        )
-    return hnswlib, None
-
-
-def hnswlib_available() -> bool:
-    """True when the optional ANN engine can actually be imported."""
-    module, _ = load_hnswlib()
-    return module is not None
-
-
-def sidecar_paths(db_path: Any) -> Tuple[Path, Path]:
-    """``(index file, meta file)`` for a brain database path."""
-    base = Path(db_path)
-    return (
-        base.with_suffix(HNSW_SUFFIX),
-        base.with_suffix(HNSW_META_SUFFIX),
-    )
-
-
-class HnswIndex:
-    """Small-M HNSW graph over the brain's embeddings."""
-
-    def __init__(
-        self,
-        *,
-        dim: int,
-        space: str = "cosine",
-        ef_construction: int = 200,
-        m: int = 16,
-        ef_search: int = 64,
-    ) -> None:
-        self._dim = int(dim)
-        self._space = str(space)
-        self._ef_construction = int(ef_construction)
-        self._m = int(m)
-        self._ef_search = int(ef_search)
-        self._module, self._detail = load_hnswlib()
-        self._vectors: Dict[str, List[float]] = {}
-        self._metadata: Dict[str, Dict[str, Any]] = {}
-        self._labels: List[str] = []
-        self._ann: Any = None
-        self._dirty = True
-        self._loaded = False
-
-    # ── identity ─────────────────────────────────────────────────────────────
-    @property
-    def backend(self) -> str:
-        return HNSW_BACKEND
-
-    @property
-    def approx(self) -> bool:
-        return True
-
-    @property
-    def exhaustive(self) -> bool:
-        return False
-
-    @property
-    def available(self) -> bool:
-        return self._module is not None
-
-    @property
-    def unavailable_detail(self) -> Optional[str]:
-        return self._detail
-
-    @property
-    def loaded_from_sidecar(self) -> bool:
-        """True when the graph came off disk instead of being rebuilt."""
-        return self._loaded
-
-    # ── mutation ─────────────────────────────────────────────────────────────
-    def _reject_mutation_after_load(self) -> None:
-        if self._loaded:
-            raise RuntimeError(
-                "this HnswIndex was loaded from a sidecar and holds no source "
-                "vectors; rebuild it from vector_embeddings before mutating"
-            )
-
-    def add(
-        self,
-        id: str,
-        vector: Sequence[float],
-        metadata: Optional[Mapping[str, Any]] = None,
-    ) -> None:
-        self._reject_mutation_after_load()
-        key = str(id)
-        self._vectors[key] = [float(value) for value in vector]
-        self._metadata[key] = dict(metadata or {})
-        self._dirty = True
-
-    def remove(self, id: str) -> None:
-        self._reject_mutation_after_load()
-        key = str(id)
-        self._vectors.pop(key, None)
-        self._metadata.pop(key, None)
-        self._dirty = True
-
-    def rebuild(self, items: Iterable[IndexItem]) -> None:
-        self._vectors.clear()
-        self._metadata.clear()
-        self._loaded = False
-        for item_id, vector, metadata in items:
-            self.add(item_id, vector, metadata)
-        self._dirty = True
-
-    # ── graph construction ───────────────────────────────────────────────────
-    def _new_graph(self, module: Any, capacity: int) -> Any:
-        graph = module.Index(space=self._space, dim=self._dim)
-        graph.init_index(
-            max_elements=max(1, capacity),
-            ef_construction=self._ef_construction,
-            M=self._m,
-        )
-        return graph
-
-    def _ensure_graph(self) -> Any:
-        """Build the graph if the held vectors changed (None when disabled)."""
-        module = self._module
-        if module is None:
-            return None
-        if self._ann is not None and not self._dirty:
-            return self._ann
-        labels = list(self._vectors)
-        graph = self._new_graph(module, len(labels))
-        if labels:
-            graph.add_items(
-                [self._vectors[key] for key in labels],
-                list(range(len(labels))),
-            )
-        self._ann = graph
-        self._labels = labels
-        self._dirty = False
-        return graph
-
-    # ── search ───────────────────────────────────────────────────────────────
-    def search(
-        self,
-        query: Sequence[float],
-        top_k: int,
-        filter: Optional[Mapping[str, Any]] = None,
-    ) -> List[ScoredId]:
-        graph = self._ensure_graph()
-        if graph is None or not self._labels:
-            return []
-        floor = score_floor(filter)
-        wanted = max(1, min(int(top_k), len(self._labels)))
-        graph.set_ef(max(self._ef_search, wanted))
-        labels, distances = graph.knn_query([list(query)], k=wanted)
-        scored: List[ScoredId] = []
-        for label, distance in zip(labels[0], distances[0], strict=True):
-            # cosine/ip space: hnswlib returns 1 - similarity as the distance.
-            score = 1.0 - float(distance)
-            if score < floor:
-                continue
-            scored.append((self._labels[int(label)], score))
-        return scored
-
-    def stats(self) -> IndexStats:
-        return IndexStats(
-            backend=self.backend,
-            size=len(self._labels) if self._loaded else len(self._vectors),
-            dim=self._dim,
-            approx=True,
-            exhaustive=False,
-            available=self.available,
-            detail=self._detail,
-        )
-
-    # ── sidecar persistence (a derivative — safe to delete) ──────────────────
-    def save(self, db_path: Any, *, fingerprint: str) -> bool:
-        """Write the graph + label map beside the brain database.
-
-        Returns False (never raises) when ANN is unavailable, the index is
-        empty, or the filesystem refuses the write: a missing sidecar only
-        costs a rebuild on the next search.
-        """
-        graph = self._ensure_graph()
-        if graph is None or not self._labels:
-            return False
-        index_path, meta_path = sidecar_paths(db_path)
-        try:
-            index_path.parent.mkdir(parents=True, exist_ok=True)
-            graph.save_index(str(index_path))
-            meta_path.write_text(
-                json.dumps(
-                    {
-                        "fingerprint": str(fingerprint),
-                        "dim": self._dim,
-                        "space": self._space,
-                        "labels": self._labels,
-                    },
-                    ensure_ascii=False,
-                ),
-                encoding="utf-8",
-            )
-        except Exception:  # noqa: BLE001 — persistence is best-effort
-            return False
-        return True
-
-    def load(self, db_path: Any, *, fingerprint: str) -> bool:
-        """Adopt a sidecar graph when it provably matches ``fingerprint``.
-
-        Any mismatch — missing file, unreadable JSON, different embedder or
-        row-set, wrong dimension — returns False so the caller rebuilds. The
-        sidecar is never trusted over SQLite.
-        """
-        module = self._module
-        if module is None:
-            return False
-        index_path, meta_path = sidecar_paths(db_path)
-        try:
-            meta = json.loads(meta_path.read_text(encoding="utf-8"))
-            matches = str(meta["fingerprint"]) == str(fingerprint)
-            matches = matches and int(meta["dim"]) == self._dim
-            labels = [str(label) for label in meta["labels"]]
-        except Exception:  # noqa: BLE001 — absent/corrupt sidecar = rebuild
-            return False
-        if not (matches and labels):
-            return False
-        try:
-            graph = module.Index(space=self._space, dim=self._dim)
-            graph.load_index(str(index_path), max_elements=len(labels))
-        except Exception:  # noqa: BLE001 — corrupt binary = rebuild
-            return False
-        self._ann = graph
-        self._labels = labels
-        self._vectors.clear()
-        self._metadata.clear()
-        self._dirty = False
-        self._loaded = True
-        return True
-
-
-__all__ = [
-    "HNSW_BACKEND",
-    "HNSW_META_SUFFIX",
-    "HNSW_SUFFIX",
-    "HNSWLIB_MODULE",
-    "HnswIndex",
-    "hnswlib_available",
-    "load_hnswlib",
-    "sidecar_paths",
-]

package/lattice_brain/graph/vector_index/jobs.py

@@ -1,287 +0,0 @@
-"""Durable pending-embed queue: ingest now, embed a moment later.
-
-The write path already degrades honestly — when the incremental vector sync
-fails, ``IngestionResult.indexing_status`` becomes ``"pending"`` and
-``index_status()`` keeps the node visible as backlog. What was missing is
-anyone whose job it is to *come back for it*. Until a human ran a rebuild,
-"pending" meant "unsearchable, indefinitely".
-
-This is that worker's memory, and it is on disk for the same reason
-``ingestion_jobs`` is: a queue that lives in one process's heap forgets its
-backlog on restart, which is exactly when a backlog exists. The table is
-created lazily inside whichever SQLite file the brain already uses, so the
-existing backup/restore covers it with no manifest change.
-
-The queue is a pure service — schedule + tick, no threads, no server, no
-event loop. :meth:`VectorEmbedQueue.tick_async` exists for callers that live
-on the event loop and hands the blocking work to a thread.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import logging
-import sqlite3
-from contextlib import contextmanager
-from pathlib import Path
-from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple
-
-from ...utils import utc_now_iso
-
-#: Job lifecycle. ``failed`` is terminal — the retry budget is exhausted.
-VECTOR_JOB_STATUSES = ("pending", "running", "done", "failed")
-#: How many times one node may fail before the queue stops retrying it.
-DEFAULT_MAX_ATTEMPTS = 3
-#: Nodes claimed per :meth:`VectorEmbedQueue.tick`.
-DEFAULT_TICK_LIMIT = 25
-
-LOGGER = logging.getLogger(__name__)
-
-
-class VectorJobStore:
-    """SQLite persistence for the pending-embed backlog.
-
-    Own connection per operation, always closed — ``with sqlite3.connect(...)``
-    commits but never closes, which is how 70 leaks accumulated before 10.2.0.
-    """
-
-    def __init__(self, db_path: Any) -> None:
-        self.db_path = Path(db_path)
-        self.db_path.parent.mkdir(parents=True, exist_ok=True)
-        self._init_db()
-
-    @contextmanager
-    def _connect(self) -> Iterator[sqlite3.Connection]:
-        conn = sqlite3.connect(str(self.db_path))
-        conn.row_factory = sqlite3.Row
-        try:
-            with conn:
-                yield conn
-        finally:
-            conn.close()
-
-    def _init_db(self) -> None:
-        with self._connect() as conn:
-            conn.executescript(
-                """
-                CREATE TABLE IF NOT EXISTS vector_jobs (
-                  node_id TEXT PRIMARY KEY,
-                  status TEXT NOT NULL DEFAULT 'pending',
-                  attempts INTEGER NOT NULL DEFAULT 0,
-                  detail TEXT,
-                  created_at TEXT NOT NULL,
-                  updated_at TEXT NOT NULL
-                );
-                CREATE INDEX IF NOT EXISTS idx_vector_jobs_status
-                  ON vector_jobs(status, created_at);
-                """
-            )
-
-    def enqueue(self, node_id: str, *, detail: Optional[str] = None) -> bool:
-        """Mark ``node_id`` pending. Re-queuing an existing row is expected."""
-        node_id = str(node_id or "").strip()
-        if not node_id:
-            return False
-        now = utc_now_iso()
-        with self._connect() as conn:
-            conn.execute(
-                """
-                INSERT INTO vector_jobs(node_id, status, attempts, detail,
-                                        created_at, updated_at)
-                VALUES (?, 'pending', 0, ?, ?, ?)
-                ON CONFLICT(node_id) DO UPDATE SET
-                  status='pending',
-                  detail=excluded.detail,
-                  updated_at=excluded.updated_at
-                """,
-                (node_id, detail, now, now),
-            )
-        return True
-
-    def claim(self, limit: int) -> List[str]:
-        """Take up to ``limit`` pending node ids and mark them running."""
-        limit = max(1, int(limit))
-        now = utc_now_iso()
-        with self._connect() as conn:
-            rows = conn.execute(
-                """
-                SELECT node_id FROM vector_jobs
-                WHERE status='pending'
-                ORDER BY created_at ASC, node_id ASC
-                LIMIT ?
-                """,
-                (limit,),
-            ).fetchall()
-            claimed = [str(row["node_id"]) for row in rows]
-            for node_id in claimed:
-                conn.execute(
-                    """
-                    UPDATE vector_jobs
-                    SET status='running', attempts=attempts+1, updated_at=?
-                    WHERE node_id=?
-                    """,
-                    (now, node_id),
-                )
-        return claimed
-
-    def finish(self, node_id: str, *, status: str, detail: Optional[str]) -> None:
-        with self._connect() as conn:
-            conn.execute(
-                "UPDATE vector_jobs SET status=?, detail=?, updated_at=? "
-                "WHERE node_id=?",
-                (status, detail, utc_now_iso(), str(node_id)),
-            )
-
-    def attempts_for(self, node_id: str) -> int:
-        with self._connect() as conn:
-            row = conn.execute(
-                "SELECT attempts FROM vector_jobs WHERE node_id=?", (str(node_id),)
-            ).fetchone()
-        return int(row["attempts"]) if row is not None else 0
-
-    def counts(self) -> Dict[str, int]:
-        """``{status: count}`` with every known status present (zero-filled)."""
-        counts = dict.fromkeys(VECTOR_JOB_STATUSES, 0)
-        with self._connect() as conn:
-            for row in conn.execute(
-                "SELECT status, COUNT(*) AS total FROM vector_jobs GROUP BY status"
-            ):
-                counts[str(row["status"])] = int(row["total"])
-        return counts
-
-
-class VectorEmbedQueue:
-    """Schedule nodes for background embedding, then drain them a tick at a time.
-
-    ``indexer`` is normally ``KnowledgeGraphStore.index_node_incremental``: it
-    returns a status dict and never raises. A queue built without a usable
-    database is *disabled*, not silently in-memory — :meth:`describe` says so
-    and :meth:`schedule` returns False, because a backlog that evaporates on
-    restart is worse than an honest "not tracked".
-    """
-
-    def __init__(
-        self,
-        *,
-        db_path: Optional[Any] = None,
-        indexer: Optional[Callable[[str], Any]] = None,
-        max_attempts: int = DEFAULT_MAX_ATTEMPTS,
-    ) -> None:
-        self._indexer = indexer
-        self._max_attempts = max(1, int(max_attempts))
-        self._store: Optional[VectorJobStore] = None
-        self._detail: Optional[str] = None
-        if db_path is None:
-            self._detail = "no database configured; the embed backlog is not tracked"
-        else:
-            try:
-                self._store = VectorJobStore(db_path)
-            except Exception as exc:  # noqa: BLE001 — never block ingestion
-                self._detail = f"vector job persistence unavailable: {exc}"
-                LOGGER.warning("vector job persistence unavailable: %s", exc)
-
-    @property
-    def available(self) -> bool:
-        return self._store is not None
-
-    def describe(self) -> Dict[str, Any]:
-        """Whether the backlog survives a restart, and where it lives."""
-        return {
-            "persistent": self._store is not None,
-            "db_path": str(self._store.db_path) if self._store is not None else None,
-            "detail": self._detail,
-        }
-
-    def schedule(self, node_id: str, *, detail: Optional[str] = None) -> bool:
-        if self._store is None:
-            return False
-        try:
-            return self._store.enqueue(node_id, detail=detail)
-        except Exception as exc:  # noqa: BLE001 — queueing is best-effort
-            LOGGER.warning("vector job %s could not be queued: %s", node_id, exc)
-            return False
-
-    def snapshot(self) -> Dict[str, int]:
-        """Backlog counts by status (all zero when the queue is disabled)."""
-        if self._store is None:
-            return dict.fromkeys(VECTOR_JOB_STATUSES, 0)
-        try:
-            return self._store.counts()
-        except Exception:  # noqa: BLE001 — a status read must never raise
-            return dict.fromkeys(VECTOR_JOB_STATUSES, 0)
-
-    def pending_count(self) -> int:
-        """Nodes still owed an embedding (queued plus in flight)."""
-        counts = self.snapshot()
-        return int(counts["pending"]) + int(counts["running"])
-
-    def tick(self, limit: int = DEFAULT_TICK_LIMIT) -> Dict[str, Any]:
-        """Drain up to ``limit`` queued nodes. Never raises.
-
-        A node whose indexing fails goes back to ``pending`` until its retry
-        budget runs out, then stays ``failed`` with the last reason attached —
-        visible backlog, not a silent retry loop.
-        """
-        summary: Dict[str, Any] = {
-            "claimed": 0,
-            "indexed": 0,
-            "retried": 0,
-            "failed": 0,
-            "detail": self._detail,
-        }
-        store = self._store
-        if store is None:
-            return summary
-        indexer = self._indexer
-        if indexer is None:
-            summary["detail"] = "no indexer configured; queued nodes were left pending"
-            return summary
-        claimed = store.claim(limit)
-        summary["claimed"] = len(claimed)
-        for node_id in claimed:
-            status, detail = self._run_one(store, indexer, node_id)
-            if status == "done":
-                summary["indexed"] += 1
-            elif status == "pending":
-                summary["retried"] += 1
-            else:
-                summary["failed"] += 1
-            store.finish(node_id, status=status, detail=detail)
-        return summary
-
-    async def tick_async(self, limit: int = DEFAULT_TICK_LIMIT) -> Dict[str, Any]:
-        """:meth:`tick` off the event loop — embedding and SQLite both block."""
-        return await asyncio.to_thread(self.tick, limit)
-
-    def _run_one(
-        self,
-        store: VectorJobStore,
-        indexer: Callable[[str], Any],
-        node_id: str,
-    ) -> Tuple[str, Optional[str]]:
-        """Index one node → the status to persist and the reason for it."""
-        try:
-            outcome = indexer(node_id) or {}
-        except Exception as exc:  # noqa: BLE001 — one bad node must not stop the drain
-            return self._retry_or_fail(store, node_id, str(exc))
-        if str(outcome.get("status") or "") == "failed":
-            detail = str(outcome.get("detail") or "unknown error")
-            return self._retry_or_fail(store, node_id, detail)
-        return "done", None
-
-    def _retry_or_fail(
-        self, store: VectorJobStore, node_id: str, detail: str
-    ) -> Tuple[str, Optional[str]]:
-        attempts = store.attempts_for(node_id)
-        if attempts >= self._max_attempts:
-            return "failed", f"{detail} (gave up after {attempts} attempts)"
-        return "pending", detail
-
-
-__all__ = [
-    "DEFAULT_MAX_ATTEMPTS",
-    "DEFAULT_TICK_LIMIT",
-    "VECTOR_JOB_STATUSES",
-    "VectorEmbedQueue",
-    "VectorJobStore",
-]