ltcai 11.5.2 → 11.6.0

package/lattice_brain/graph/vector_index/quantized.py

@@ -1,148 +0,0 @@
-"""int8-quantized exhaustive index — same recall shape, a fraction of the RAM.
-
-A 384-dimension embedding held as a Python ``list[float]`` costs ~3 KB (each
-float is a boxed object plus a pointer); the same vector as an ``array("b")``
-of int8 codes costs 384 bytes plus one header. On a 100 000-vector brain that
-is the difference between ~300 MB and ~40 MB of resident scoring state, which
-is what decides whether a large index can be held at all.
-
-The trade is precision, not coverage: every vector is still compared (the
-index is *exhaustive*), but each score is reconstructed from 8-bit codes, so
-it lands within roughly ±1% of the exact cosine. Near-ties can therefore swap
-places — which is why this backend reports ``approx = True`` and why the
-default stays :class:`~.brute_force.BruteForceIndex`.
-
-Symmetric per-vector scaling: each vector is divided by its own peak absolute
-value before rounding, so a short vector keeps its full 8-bit resolution
-instead of being crushed by the largest vector in the corpus.
-
-Measured caveat (docs/PERFORMANCE.md, 10k vectors): the RAM saving above is a
-property of the *data structure*, and the current search path does not cash it
-in. ``retrieval_vector`` already feeds the index in bounded batches, so
-resident vectors were never the dominant term — the fetched SQLite rows are —
-and peak memory moves by ~1 MB while latency roughly doubles. This backend is
-therefore honest, exhaustive, and currently not the one to choose; it exists
-as the representation a *held* (cross-query) index would need.
-"""
-
-from __future__ import annotations
-
-from array import array
-from operator import mul
-from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple
-
-from .base import IndexItem, IndexStats, ScoredId, score_floor, take_top
-
-QUANTIZED_BACKEND = "quantized-int8"
-#: int8 keeps ``-128``; the symmetric range stops at ``-127`` so ``+peak`` and
-#: ``-peak`` quantize to codes of equal magnitude.
-INT8_MAX = 127
-
-
-def quantize(vector: Sequence[float]) -> Tuple[array, float]:
-    """``vector`` → (int8 codes, scale) with ``value ≈ code * scale``.
-
-    An all-zero vector has no peak to scale against; it quantizes to all-zero
-    codes with scale ``0.0``, which scores 0 against everything — the same
-    answer the exact scan gives for a zero vector.
-    """
-    values = [float(value) for value in vector]
-    peak = max((abs(value) for value in values), default=0.0)
-    if peak <= 0.0:
-        return array("b", bytes(len(values))), 0.0
-    scale = peak / INT8_MAX
-    codes = array(
-        "b",
-        (
-            max(-INT8_MAX, min(INT8_MAX, int(round(value / scale))))
-            for value in values
-        ),
-    )
-    return codes, scale
-
-
-class QuantizedIndex:
-    """Exhaustive int8 index: full coverage, approximate scores."""
-
-    def __init__(self, *, dim: int = 0) -> None:
-        self._dim = int(dim)
-        self._codes: Dict[str, array] = {}
-        self._scales: Dict[str, float] = {}
-        self._metadata: Dict[str, Dict[str, Any]] = {}
-
-    @property
-    def backend(self) -> str:
-        return QUANTIZED_BACKEND
-
-    @property
-    def approx(self) -> bool:
-        return True
-
-    @property
-    def exhaustive(self) -> bool:
-        return True
-
-    def add(
-        self,
-        id: str,
-        vector: Sequence[float],
-        metadata: Optional[Mapping[str, Any]] = None,
-    ) -> None:
-        codes, scale = quantize(vector)
-        if not self._dim:
-            self._dim = len(codes)
-        key = str(id)
-        self._codes[key] = codes
-        self._scales[key] = scale
-        self._metadata[key] = dict(metadata or {})
-
-    def remove(self, id: str) -> None:
-        key = str(id)
-        self._codes.pop(key, None)
-        self._scales.pop(key, None)
-        self._metadata.pop(key, None)
-
-    def rebuild(self, items: Iterable[IndexItem]) -> None:
-        self._codes.clear()
-        self._scales.clear()
-        self._metadata.clear()
-        for item_id, vector, metadata in items:
-            self.add(item_id, vector, metadata)
-
-    def search(
-        self,
-        query: Sequence[float],
-        top_k: int,
-        filter: Optional[Mapping[str, Any]] = None,
-    ) -> List[ScoredId]:
-        floor = score_floor(filter)
-        query_codes, query_scale = quantize(query)
-        scored: List[ScoredId] = []
-        for item_id, codes in self._codes.items():
-            if len(codes) != len(query_codes):
-                raise ValueError(
-                    f"embedding dimension mismatch: {len(query_codes)} vs "
-                    f"{len(codes)}; the vector index was built with a "
-                    "different model"
-                )
-            # Integer dot product, then one float multiply for the two scales:
-            # all the arithmetic that decides the ranking stays in ints.
-            raw = sum(map(mul, query_codes, codes))
-            score = float(raw) * query_scale * self._scales[item_id]
-            if score < floor:
-                continue
-            scored.append((item_id, score))
-        return take_top(scored, top_k)
-
-    def stats(self) -> IndexStats:
-        return IndexStats(
-            backend=self.backend,
-            size=len(self._codes),
-            dim=self._dim,
-            approx=True,
-            exhaustive=True,
-            detail="scores are reconstructed from 8-bit codes (~1% error)",
-        )
-
-
-__all__ = ["INT8_MAX", "QUANTIZED_BACKEND", "QuantizedIndex", "quantize"]

package/lattice_brain/graph/vector_index/selector.py

@@ -1,161 +0,0 @@
-"""Which backend scores this search, and — when it is not the one you asked
-for — why.
-
-``LATTICEAI_VECTOR_INDEX`` selects ``brute`` (default) / ``quantized`` /
-``hnsw``. Two failure modes have to stay loud, because both otherwise look
-exactly like "search is a bit worse today":
-
-* an unknown name (typo, stale config) must not silently mean "the default";
-* asking for ``hnsw`` without the optional extra installed must not silently
-  mean "you have ANN".
-
-Both resolve to the exact brute-force scan and carry a ``detail`` string
-naming the cause, which ``index_status()`` and the search result surface.
-
-The backend is a *choice*, not a switch, so it cannot ride a
-:class:`~lattice_brain.gates.FeatureGate` (which answers booleans). It gets the
-same shape of seam instead: :func:`bind_vector_index_resolver` installs a
-caller-supplied resolver that is consulted ahead of the environment, which is
-what lets the settings panel change the backend without a restart. With nothing
-bound — the default — the environment variable is still the whole control
-surface, exactly as before.
-"""
-
-from __future__ import annotations
-
-import os
-from dataclasses import dataclass
-from typing import Any, Callable, Dict, Optional
-
-from .base import Similarity, VectorIndex
-from .brute_force import BRUTE_FORCE_BACKEND, BruteForceIndex
-from .hnsw import HNSW_BACKEND, HnswIndex, load_hnswlib
-from .quantized import QUANTIZED_BACKEND, QuantizedIndex
-
-VECTOR_INDEX_ENV = "LATTICEAI_VECTOR_INDEX"
-DEFAULT_VECTOR_INDEX = "brute"
-#: Accepted values of ``LATTICEAI_VECTOR_INDEX``.
-VECTOR_INDEX_CHOICES = ("brute", "quantized", "hnsw")
-
-_BACKEND_LABELS = {
-    "brute": BRUTE_FORCE_BACKEND,
-    "quantized": QUANTIZED_BACKEND,
-    "hnsw": HNSW_BACKEND,
-}
-_APPROX = {"brute": False, "quantized": True, "hnsw": True}
-_EXHAUSTIVE = {"brute": True, "quantized": True, "hnsw": False}
-
-#: App-layer resolver, consulted before the environment. ``None`` (the default)
-#: leaves this module reading exactly the env var it always read.
-_RESOLVER: Optional[Callable[[], Optional[str]]] = None
-
-
-def bind_vector_index_resolver(
-    resolver: Optional[Callable[[], Optional[str]]],
-) -> None:
-    """Delegate backend selection to a callable (``None`` hands it back to env).
-
-    A resolver that returns ``None`` also falls through to the environment, so
-    "the settings service has no opinion" and "there is no settings service"
-    reach the same answer instead of two.
-    """
-    global _RESOLVER
-    _RESOLVER = resolver
-
-
-@dataclass(frozen=True)
-class BackendSelection:
-    """The resolved backend plus an honest account of any substitution."""
-
-    requested: str
-    name: str
-    backend: str
-    approx: bool
-    exhaustive: bool
-    detail: Optional[str] = None
-
-    @property
-    def honored(self) -> bool:
-        """True when the caller got the backend they asked for."""
-        return self.requested == self.name
-
-    def as_dict(self) -> Dict[str, Any]:
-        return {
-            "requested": self.requested,
-            "backend": self.backend,
-            "name": self.name,
-            "approx": self.approx,
-            "exhaustive": self.exhaustive,
-            "honored": self.honored,
-            "detail": self.detail,
-        }
-
-
-def _selection(name: str, *, requested: str, detail: Optional[str]) -> BackendSelection:
-    return BackendSelection(
-        requested=requested,
-        name=name,
-        backend=_BACKEND_LABELS[name],
-        approx=_APPROX[name],
-        exhaustive=_EXHAUSTIVE[name],
-        detail=detail,
-    )
-
-
-def resolve_vector_index(requested: Optional[str] = None) -> BackendSelection:
-    """Resolve the configured backend (never raises, always falls back safe)."""
-    raw = requested
-    if raw is None and _RESOLVER is not None:
-        raw = _RESOLVER()
-    if raw is None:
-        raw = os.getenv(VECTOR_INDEX_ENV, "")
-    name = str(raw or "").strip().lower() or DEFAULT_VECTOR_INDEX
-    if name not in VECTOR_INDEX_CHOICES:
-        return _selection(
-            DEFAULT_VECTOR_INDEX,
-            requested=name,
-            detail=(
-                f"unknown vector index backend {name!r}; expected one of "
-                f"{', '.join(VECTOR_INDEX_CHOICES)} — using the exact "
-                "brute-force scan"
-            ),
-        )
-    if name == "hnsw":
-        module, reason = load_hnswlib()
-        if module is None:
-            return _selection(
-                DEFAULT_VECTOR_INDEX,
-                requested=name,
-                detail=f"{reason} — using the exact brute-force scan",
-            )
-    return _selection(name, requested=name, detail=None)
-
-
-def build_index(
-    selection: BackendSelection,
-    *,
-    dim: int = 0,
-    similarity: Optional[Similarity] = None,
-) -> VectorIndex:
-    """Instantiate the backend named by ``selection``.
-
-    ``similarity`` is threaded into the exact backend only: quantized and
-    HNSW score in their own representation, and pretending otherwise would
-    make an injected function look respected when it is not.
-    """
-    if selection.name == "quantized":
-        return QuantizedIndex(dim=dim)
-    if selection.name == HNSW_BACKEND:
-        return HnswIndex(dim=dim)
-    return BruteForceIndex(dim=dim, similarity=similarity)
-
-
-__all__ = [
-    "DEFAULT_VECTOR_INDEX",
-    "VECTOR_INDEX_CHOICES",
-    "VECTOR_INDEX_ENV",
-    "BackendSelection",
-    "bind_vector_index_resolver",
-    "build_index",
-    "resolve_vector_index",
-]

package/lattice_brain/graph/write_master.py

@@ -1,308 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-# ruff: noqa: F403,F405
-from ._kg_common import *  # noqa: F403,F401
-
-# The cross-mixin surface (`_connect`, `_upsert_node`, …) is declared in
-# `_kg_contract.KnowledgeGraphCore`. It is a typing-only base: at runtime this
-# is `object`, so the MRO of `KnowledgeGraphStore` is unchanged.
-if TYPE_CHECKING:
-    from ._kg_contract import KnowledgeGraphCore as _Core
-else:
-    _Core = object
-
-
-
-class KnowledgeGraphWriteMixin(_Core):
-    def _upsert_node(
-        self,
-        conn: sqlite3.Connection,
-        node_id: str,
-        node_type: str,
-        title: str,
-        summary: str = "",
-        metadata: Optional[Dict[str, Any]] = None,
-        raw: Optional[Dict[str, Any]] = None,
-        owner: Optional[str] = None,
-        workspace_id: Optional[str] = None,
-        visibility: Optional[str] = None,
-    ) -> str:
-        now = _now()
-        # v4 write-mastering: nodes_v2 is authoritative; the legacy nodes
-        # table is maintained as the compatibility projection.
-        title_s = title[:240]
-        summary_s = summary[:1000]
-        meta_json = _json(metadata)
-        self._v2_project_node(
-            conn,
-            node_id,
-            node_type,
-            title_s,
-            summary_s,
-            meta_json,
-            created_at=now,
-            updated_at=now,
-            owner=owner,
-            workspace_id=workspace_id,
-            visibility=visibility,
-            strict=True,
-        )
-        conn.execute(
-            """
-                INSERT INTO nodes(id, type, title, summary, metadata_json, raw_json, created_at, updated_at)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
-                ON CONFLICT(id) DO UPDATE SET
-                  title=excluded.title,
-                  summary=excluded.summary,
-                  metadata_json=excluded.metadata_json,
-                  raw_json=excluded.raw_json,
-                  updated_at=excluded.updated_at
-                """,
-            (node_id, node_type, title_s, summary_s, meta_json, _json(raw), now, now),
-        )
-        if node_type != "Chunk":
-            self._upsert_vector_item(
-                conn,
-                item_id=node_id,
-                item_type="node",
-                source_node=node_id,
-                text=self._vector_text_for_node(
-                    title=title_s, summary=summary_s, metadata=metadata
-                ),
-                metadata={"node_type": node_type, **(metadata or {})},
-            )
-        return node_id
-
-    def _upsert_edge(
-        self,
-        conn: sqlite3.Connection,
-        from_node: str,
-        to_node: str,
-        edge_type: str,
-        weight: float = 1.0,
-        metadata: Optional[Dict[str, Any]] = None,
-        *,
-        legacy_label: Optional[str] = None,
-    ) -> str:
-        # v4 write door: every new edge stores the canonical EdgeType value —
-        # free-string types (e.g. '포함함', '언급함') are normalized here, so no
-        # caller can mint new legacy taxonomy. The original label survives in
-        # metadata.legacy_label for traceability.
-        #
-        # legacy_type in edges_v2:
-        #   - normal write door: synonyms always dedupe to ONE row with legacy_type=''
-        #   - import (passes legacy_label=) : preserves distinct legacy labels as
-        #     separate v2 rows (lossless collision for old artifacts / backfill)
-        passed_for_legacy = legacy_label or edge_type
-        if EdgeType is not None:
-            canonical = EdgeType.from_legacy(edge_type).value
-            if canonical != edge_type or (legacy_label and legacy_label != canonical):
-                metadata = dict(metadata or {})
-                ll = legacy_label or edge_type
-                if ll != canonical:
-                    metadata.setdefault("legacy_label", ll)
-            edge_type = canonical
-        edge_id = f"edge:{_sha256_text(f'{from_node}|{edge_type}|{to_node}')[:24]}"
-        now = _now()
-        meta_json = _json(metadata)  # canonical string shared with the projection
-        v2_legacy = ""
-        if passed_for_legacy and passed_for_legacy != edge_type:
-            if legacy_label is not None:
-                # explicit legacy_label from import path forces distinct legacy_type
-                # rows in v2 for collision preservation
-                v2_legacy = passed_for_legacy
-            else:
-                # normal write-door dedupes even for synonym labels (legacy_type='')
-                v2_legacy = ""
-        # v2 may use distinct eid when preserving legacy collisions (different from
-        # the canon-based edge_id used for legacy edges table which always dedupes)
-        v2_eid = (
-            f"edge:{_sha256_text(f'{from_node}|{edge_type}|{to_node}|{v2_legacy}')[:24]}"
-            if v2_legacy
-            else edge_id
-        )
-        self._v2_project_edge(
-            conn,
-            from_node,
-            to_node,
-            edge_type,
-            float(weight),
-            meta_json,
-            edge_id=v2_eid,
-            created_at=now,
-            strict=True,
-            legacy_type=v2_legacy,
-        )
-        conn.execute(
-            """
-                INSERT INTO edges(id, from_node, to_node, type, weight, metadata_json, created_at)
-                VALUES (?, ?, ?, ?, ?, ?, ?)
-                ON CONFLICT(from_node, to_node, type) DO UPDATE SET
-                  weight=max(edges.weight, excluded.weight),
-                  metadata_json=excluded.metadata_json
-                """,
-            (edge_id, from_node, to_node, edge_type, float(weight), meta_json, now),
-        )
-        return edge_id
-
-    def _vector_text_for_node(
-        self,
-        *,
-        title: str,
-        summary: str = "",
-        metadata: Optional[Dict[str, Any]] = None,
-    ) -> str:
-        metadata = metadata or {}
-        meta_parts = []
-        for key in (
-            "filename",
-            "relative_path",
-            "file_path",
-            "conversation_id",
-            "source",
-            "category",
-            "ext",
-            "role",
-        ):
-            value = metadata.get(key)
-            if value:
-                meta_parts.append(str(value))
-        return _clean_text(
-            "\n".join([str(title or ""), str(summary or ""), " ".join(meta_parts)])
-        )
-
-    def _upsert_vector_item(
-        self,
-        conn: sqlite3.Connection,
-        *,
-        item_id: str,
-        item_type: str,
-        source_node: str,
-        text: str,
-        metadata: Optional[Dict[str, Any]] = None,
-    ) -> bool:
-        text = _clean_text(text)
-        if len(text) < 2:
-            conn.execute("DELETE FROM vector_embeddings WHERE item_id=?", (item_id,))
-            return False
-        text_hash = _sha256_text(text)
-        existing = conn.execute(
-            """
-                SELECT text_hash, embedding_dim, embedding_model
-                FROM vector_embeddings
-                WHERE item_id=?
-                """,
-            (item_id,),
-        ).fetchone()
-        if (
-            existing
-            and existing["text_hash"] == text_hash
-            and existing["embedding_dim"] == self._embedding_model.dim
-            and existing["embedding_model"] == self._embedding_model.model_id
-        ):
-            return False
-        embedding = self._embedding_model.encode(
-            self._embedding_model.embed(text[:50_000])
-        )
-        conn.execute(
-            """
-                INSERT INTO vector_embeddings(
-                  item_id, item_type, source_node, text_hash, embedding,
-                  embedding_dim, embedding_model, metadata_json, indexed_at
-                )
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
-                ON CONFLICT(item_id) DO UPDATE SET
-                  item_type=excluded.item_type,
-                  source_node=excluded.source_node,
-                  text_hash=excluded.text_hash,
-                  embedding=excluded.embedding,
-                  embedding_dim=excluded.embedding_dim,
-                  embedding_model=excluded.embedding_model,
-                  metadata_json=excluded.metadata_json,
-                  indexed_at=excluded.indexed_at
-                """,
-            (
-                item_id,
-                item_type,
-                source_node,
-                text_hash,
-                embedding,
-                self._embedding_model.dim,
-                self._embedding_model.model_id,
-                _json(metadata),
-                _now(),
-            ),
-        )
-        return True
-
-    def _upsert_chunk(
-        self,
-        conn: sqlite3.Connection,
-        *,
-        chunk_id: str,
-        source_node: str,
-        text: str,
-        metadata: Optional[Dict[str, Any]] = None,
-    ) -> None:
-        metadata = metadata or {}
-        conn.execute(
-            "INSERT OR REPLACE INTO chunks(id, source_node, text, metadata_json, created_at) "
-            "VALUES (?, ?, ?, ?, ?)",
-            (chunk_id, source_node, text, _json(metadata), _now()),
-        )
-        self._upsert_vector_item(
-            conn,
-            item_id=chunk_id,
-            item_type="chunk",
-            source_node=chunk_id,
-            text=text,
-            metadata={**metadata, "parent_source_node": source_node},
-        )
-
-    def set_node_sensitivity(
-        self,
-        node_id: str,
-        *,
-        local_only: bool,
-        reason: Optional[str] = None,
-    ) -> Dict[str, Any]:
-        """Mark (or unmark) one node as never-leaving.
-
-        The cloud filter has always looked for this flag; until 10.2.0 nothing
-        could set it, so the guard was unreachable. This is the user-driven
-        half — ingestion stamps secret-bearing paths automatically, and this
-        covers everything a path cannot tell you, like a note whose *content*
-        is private.
-
-        Unmarking is allowed and audited by the caller: a user who flagged
-        something by mistake must be able to undo it, but the reason is cleared
-        with the flag so a stale justification cannot linger.
-        """
-        with self._connect() as conn:
-            row = conn.execute(
-                "SELECT metadata_json FROM nodes WHERE id=?", (node_id,)
-            ).fetchone()
-            if row is None:
-                return {"ok": False, "node_id": node_id, "reason": "node not found"}
-            try:
-                metadata = json.loads(row[0] or "{}")
-            except Exception:
-                metadata = {}
-            if local_only:
-                metadata["local_only"] = True
-                metadata["local_only_reason"] = reason or "marked by the user"
-            else:
-                metadata.pop("local_only", None)
-                metadata.pop("local_only_reason", None)
-            conn.execute(
-                "UPDATE nodes SET metadata_json=?, updated_at=? WHERE id=?",
-                (json.dumps(metadata, ensure_ascii=False), _now(), node_id),
-            )
-        return {
-            "ok": True,
-            "node_id": node_id,
-            "local_only": bool(local_only),
-            "reason": metadata.get("local_only_reason"),
-        }