ltcai 10.9.0 → 11.0.1

package/lattice_brain/graph/vector_index/quantized.py

@@ -0,0 +1,151 @@
+"""int8-quantized exhaustive index — same recall shape, a fraction of the RAM.
+
+A 384-dimension embedding held as a Python ``list[float]`` costs ~3 KB (each
+float is a boxed object plus a pointer); the same vector as an ``array("b")``
+of int8 codes costs 384 bytes plus one header. On a 100 000-vector brain that
+is the difference between ~300 MB and ~40 MB of resident scoring state, which
+is what decides whether a large index can be held at all.
+
+The trade is precision, not coverage: every vector is still compared (the
+index is *exhaustive*), but each score is reconstructed from 8-bit codes, so
+it lands within roughly ±1% of the exact cosine. Near-ties can therefore swap
+places — which is why this backend reports ``approx = True`` and why the
+default stays :class:`~.brute_force.BruteForceIndex`.
+
+Symmetric per-vector scaling: each vector is divided by its own peak absolute
+value before rounding, so a short vector keeps its full 8-bit resolution
+instead of being crushed by the largest vector in the corpus.
+
+Measured caveat (docs/PERFORMANCE.md, 10k vectors): the RAM saving above is a
+property of the *data structure*, and the current search path does not cash it
+in. ``retrieval_vector`` already feeds the index in bounded batches, so
+resident vectors were never the dominant term — the fetched SQLite rows are —
+and peak memory moves by ~1 MB while latency roughly doubles. This backend is
+therefore honest, exhaustive, and currently not the one to choose; it exists
+as the representation a *held* (cross-query) index would need.
+"""
+
+from __future__ import annotations
+
+from array import array
+from operator import mul
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
+
+from .base import IndexItem, IndexStats, ScoredId, score_floor, take_top
+
+QUANTIZED_BACKEND = "quantized-int8"
+#: int8 keeps ``-128``; the symmetric range stops at ``-127`` so ``+peak`` and
+#: ``-peak`` quantize to codes of equal magnitude.
+INT8_MAX = 127
+
+
+def quantize(vector: Sequence[float]) -> Tuple[array, float]:
+    """``vector`` → (int8 codes, scale) with ``value ≈ code * scale``.
+
+    An all-zero vector has no peak to scale against; it quantizes to all-zero
+    codes with scale ``0.0``, which scores 0 against everything — the same
+    answer the exact scan gives for a zero vector.
+    """
+    values = [float(value) for value in vector]
+    peak = max((abs(value) for value in values), default=0.0)
+    if peak <= 0.0:
+        return array("b", bytes(len(values))), 0.0
+    scale = peak / INT8_MAX
+    codes = array(
+        "b",
+        (
+            max(-INT8_MAX, min(INT8_MAX, int(round(value / scale))))
+            for value in values
+        ),
+    )
+    return codes, scale
+
+
+class QuantizedIndex:
+    """Exhaustive int8 index: full coverage, approximate scores."""
+
+    def __init__(self, *, dim: int = 0) -> None:
+        self._dim = int(dim)
+        self._codes: Dict[str, array] = {}
+        self._scales: Dict[str, float] = {}
+        self._metadata: Dict[str, Dict[str, Any]] = {}
+
+    @property
+    def backend(self) -> str:
+        return QUANTIZED_BACKEND
+
+    @property
+    def approx(self) -> bool:
+        return True
+
+    @property
+    def exhaustive(self) -> bool:
+        return True
+
+    def add(
+        self,
+        id: str,
+        vector: Sequence[float],
+        metadata: Optional[Mapping[str, Any]] = None,
+    ) -> None:
+        codes, scale = quantize(vector)
+        if not self._dim:
+            self._dim = len(codes)
+        key = str(id)
+        self._codes[key] = codes
+        self._scales[key] = scale
+        self._metadata[key] = dict(metadata or {})
+
+    def remove(self, id: str) -> None:
+        key = str(id)
+        self._codes.pop(key, None)
+        self._scales.pop(key, None)
+        self._metadata.pop(key, None)
+
+    def rebuild(self, items: Iterable[IndexItem]) -> None:
+        self._codes.clear()
+        self._scales.clear()
+        self._metadata.clear()
+        for item_id, vector, metadata in items:
+            self.add(item_id, vector, metadata)
+
+    def metadata_for(self, id: str) -> Dict[str, Any]:
+        return dict(self._metadata.get(str(id), {}))
+
+    def search(
+        self,
+        query: Sequence[float],
+        top_k: int,
+        filter: Optional[Mapping[str, Any]] = None,
+    ) -> List[ScoredId]:
+        floor = score_floor(filter)
+        query_codes, query_scale = quantize(query)
+        scored: List[ScoredId] = []
+        for item_id, codes in self._codes.items():
+            if len(codes) != len(query_codes):
+                raise ValueError(
+                    f"embedding dimension mismatch: {len(query_codes)} vs "
+                    f"{len(codes)}; the vector index was built with a "
+                    "different model"
+                )
+            # Integer dot product, then one float multiply for the two scales:
+            # all the arithmetic that decides the ranking stays in ints.
+            raw = sum(map(mul, query_codes, codes))
+            score = float(raw) * query_scale * self._scales[item_id]
+            if score < floor:
+                continue
+            scored.append((item_id, score))
+        return take_top(scored, top_k)
+
+    def stats(self) -> IndexStats:
+        return IndexStats(
+            backend=self.backend,
+            size=len(self._codes),
+            dim=self._dim,
+            approx=True,
+            exhaustive=True,
+            detail="scores are reconstructed from 8-bit codes (~1% error)",
+        )
+
+
+__all__ = ["INT8_MAX", "QUANTIZED_BACKEND", "QuantizedIndex", "quantize"]

package/lattice_brain/graph/vector_index/selector.py

@@ -0,0 +1,131 @@
+"""Which backend scores this search, and — when it is not the one you asked
+for — why.
+
+``LATTICEAI_VECTOR_INDEX`` selects ``brute`` (default) / ``quantized`` /
+``hnsw``. Two failure modes have to stay loud, because both otherwise look
+exactly like "search is a bit worse today":
+
+* an unknown name (typo, stale config) must not silently mean "the default";
+* asking for ``hnsw`` without the optional extra installed must not silently
+  mean "you have ANN".
+
+Both resolve to the exact brute-force scan and carry a ``detail`` string
+naming the cause, which ``index_status()`` and the search result surface.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import Any, Dict, Optional
+
+from .base import Similarity, VectorIndex
+from .brute_force import BRUTE_FORCE_BACKEND, BruteForceIndex
+from .hnsw import HNSW_BACKEND, HnswIndex, load_hnswlib
+from .quantized import QUANTIZED_BACKEND, QuantizedIndex
+
+VECTOR_INDEX_ENV = "LATTICEAI_VECTOR_INDEX"
+DEFAULT_VECTOR_INDEX = "brute"
+#: Accepted values of ``LATTICEAI_VECTOR_INDEX``.
+VECTOR_INDEX_CHOICES = ("brute", "quantized", "hnsw")
+
+_BACKEND_LABELS = {
+    "brute": BRUTE_FORCE_BACKEND,
+    "quantized": QUANTIZED_BACKEND,
+    "hnsw": HNSW_BACKEND,
+}
+_APPROX = {"brute": False, "quantized": True, "hnsw": True}
+_EXHAUSTIVE = {"brute": True, "quantized": True, "hnsw": False}
+
+
+@dataclass(frozen=True)
+class BackendSelection:
+    """The resolved backend plus an honest account of any substitution."""
+
+    requested: str
+    name: str
+    backend: str
+    approx: bool
+    exhaustive: bool
+    detail: Optional[str] = None
+
+    @property
+    def honored(self) -> bool:
+        """True when the caller got the backend they asked for."""
+        return self.requested == self.name
+
+    def as_dict(self) -> Dict[str, Any]:
+        return {
+            "requested": self.requested,
+            "backend": self.backend,
+            "name": self.name,
+            "approx": self.approx,
+            "exhaustive": self.exhaustive,
+            "honored": self.honored,
+            "detail": self.detail,
+        }
+
+
+def _selection(name: str, *, requested: str, detail: Optional[str]) -> BackendSelection:
+    return BackendSelection(
+        requested=requested,
+        name=name,
+        backend=_BACKEND_LABELS[name],
+        approx=_APPROX[name],
+        exhaustive=_EXHAUSTIVE[name],
+        detail=detail,
+    )
+
+
+def resolve_vector_index(requested: Optional[str] = None) -> BackendSelection:
+    """Resolve the configured backend (never raises, always falls back safe)."""
+    raw = requested if requested is not None else os.getenv(VECTOR_INDEX_ENV, "")
+    name = str(raw or "").strip().lower() or DEFAULT_VECTOR_INDEX
+    if name not in VECTOR_INDEX_CHOICES:
+        return _selection(
+            DEFAULT_VECTOR_INDEX,
+            requested=name,
+            detail=(
+                f"unknown vector index backend {name!r}; expected one of "
+                f"{', '.join(VECTOR_INDEX_CHOICES)} — using the exact "
+                "brute-force scan"
+            ),
+        )
+    if name == "hnsw":
+        module, reason = load_hnswlib()
+        if module is None:
+            return _selection(
+                DEFAULT_VECTOR_INDEX,
+                requested=name,
+                detail=f"{reason} — using the exact brute-force scan",
+            )
+    return _selection(name, requested=name, detail=None)
+
+
+def build_index(
+    selection: BackendSelection,
+    *,
+    dim: int = 0,
+    similarity: Optional[Similarity] = None,
+) -> VectorIndex:
+    """Instantiate the backend named by ``selection``.
+
+    ``similarity`` is threaded into the exact backend only: quantized and
+    HNSW score in their own representation, and pretending otherwise would
+    make an injected function look respected when it is not.
+    """
+    if selection.name == "quantized":
+        return QuantizedIndex(dim=dim)
+    if selection.name == HNSW_BACKEND:
+        return HnswIndex(dim=dim)
+    return BruteForceIndex(dim=dim, similarity=similarity)
+
+
+__all__ = [
+    "DEFAULT_VECTOR_INDEX",
+    "VECTOR_INDEX_CHOICES",
+    "VECTOR_INDEX_ENV",
+    "BackendSelection",
+    "build_index",
+    "resolve_vector_index",
+]

package/lattice_brain/ingestion.py

@@ -46,6 +46,7 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Dict, Iterable, List, Optional, Tuple
 
+from .graph.vector_index import DEFAULT_TICK_LIMIT as VECTOR_TICK_LIMIT
 from .runtime.hooks import dispatch_tool
 from .utils import utc_now_iso
 
@@ -715,12 +716,32 @@ class IngestionPipeline:
             "detail": "; ".join(p for p in parts if p),
         }
 
+    def _queue_pending_embed(self, node_id: str, detail: str) -> bool:
+        """Hand a node the inline sync could not embed to the background queue.
+
+        Before v11.1.0 ``indexing_status="pending"`` was the end of the story:
+        honest, but nobody was coming back for it, so the node stayed
+        unsearchable until a human ran a rebuild. The durable queue is who
+        comes back. A store without one (older stores, mocks) just keeps the
+        old behaviour — the node is still visible as ``index_status`` backlog.
+        """
+        queue = getattr(self._kg, "vector_queue", None)
+        if queue is None:
+            return False
+        try:
+            return bool(queue.schedule(node_id, detail=detail))
+        except Exception:  # noqa: BLE001 — queueing must never fail an ingest
+            quiet()
+            return False
+
     def _sync_vector_index(self, node_id: str) -> Tuple[str, Optional[str]]:
         """Best-effort incremental vector sync → (indexing_status, detail).
 
         Any failure — missing method on older stores, embedding provider down,
         storage error — yields ``("pending", detail)`` so a later
-        ``rebuild_vector_index`` run picks the node up from the backlog.
+        ``rebuild_vector_index`` run picks the node up from the backlog, and
+        the node is queued for background embedding so that pickup happens on
+        its own.
         """
         sync = getattr(self._kg, "index_node_incremental", None)
         if not callable(sync):
@@ -730,12 +751,38 @@ class IngestionPipeline:
         try:
             outcome = sync(node_id) or {}
         except Exception as exc:  # noqa: BLE001 — vector sync must never fail the ingest
-            return "pending", f"vector index sync failed: {exc}"
+            return "pending", self._pending_detail(node_id, f"vector index sync failed: {exc}")
         if str(outcome.get("status") or "") == "failed":
             reason = outcome.get("detail") or "unknown error"
-            return "pending", f"vector index sync failed: {reason}"
+            return "pending", self._pending_detail(
+                node_id, f"vector index sync failed: {reason}"
+            )
         return "indexed", None
 
+    def _pending_detail(self, node_id: str, reason: str) -> str:
+        """``reason``, plus whether a background retry was actually scheduled."""
+        if self._queue_pending_embed(node_id, reason):
+            return f"{reason}; queued for background embedding"
+        return reason
+
+    def drain_vector_queue(self, limit: int = VECTOR_TICK_LIMIT) -> Dict[str, Any]:
+        """Run one background-embedding tick over the store's pending backlog.
+
+        Deliberately caller-driven (a scheduler, a CLI, a test) rather than a
+        thread this pipeline owns: the queue is durable, so "who runs it" is a
+        deployment decision, not a property of having ingested something.
+        """
+        queue = getattr(self._kg, "vector_queue", None)
+        if queue is None:
+            return {
+                "claimed": 0,
+                "indexed": 0,
+                "retried": 0,
+                "failed": 0,
+                "detail": "this store has no background vector queue",
+            }
+        return dict(queue.tick(limit))
+
     # --- Large candidate #1: background / incremental scheduling (slice) ---
     def schedule_background(
         self,