ltcai 10.9.0 → 11.0.1

package/lattice_brain/graph/schema.py

@@ -52,6 +52,7 @@ from pathlib import Path
 from typing import Any, Dict, Optional
 
 from ..quiet import quiet
+from ..utils import now_iso as _stamp_now
 
 # ── Schema version ──────────────────────────────────────────────────────────
 KG_SCHEMA_V2_VERSION = 2
@@ -117,6 +118,16 @@ class NodeType(str, Enum):
     ORGANIZATION = "ORGANIZATION"  # 조직 / 회사 / 팀
     WORKFLOW = "WORKFLOW"  # 워크플로우 정의/실행
     AGENT = "AGENT"  # 에이전트(역할/실행 주체)
+    # v11.1.0 Personal Ontology (Track 4) — the Self-Model subgraph. These are
+    # not "another kind of document": they are what the Brain has been told
+    # about *its owner*, so they are read, edited and deleted as a unit
+    # (``lattice_brain.self_model``). DECISION already exists above and is
+    # reused rather than duplicated — a decision is a decision whether it came
+    # from a meeting note or from "결정: 로컬 모델을 기본값으로 쓴다".
+    SELF = "SELF"  # 사용자 자신(Self-Model 루트)
+    PREFERENCE = "PREFERENCE"  # 선호 (좋아함/싫어함/선호 방식)
+    HABIT = "HABIT"  # 습관 (반복되는 행동)
+    RELATIONSHIP = "RELATIONSHIP"  # 관계 (동료/친구/멘토 등)
 
     @classmethod
     def from_legacy(cls, label: str) -> "NodeType":
@@ -252,6 +263,19 @@ _LEGACY_NODE_MAP: Dict[str, NodeType] = {
     "team": NodeType.ORGANIZATION,
     "workflow": NodeType.WORKFLOW,
     "agent": NodeType.AGENT,
+    # v11.1.0 Personal Ontology — the Self-Model vocabulary, including the
+    # Korean labels a person actually types.
+    "self": NodeType.SELF,
+    "selfmodel": NodeType.SELF,
+    "me": NodeType.SELF,
+    "나": NodeType.SELF,
+    "preference": NodeType.PREFERENCE,
+    "선호": NodeType.PREFERENCE,
+    "habit": NodeType.HABIT,
+    "습관": NodeType.HABIT,
+    "relationship": NodeType.RELATIONSHIP,
+    "관계": NodeType.RELATIONSHIP,
+    "결정": NodeType.DECISION,
 }
 
 _LEGACY_EDGE_MAP: Dict[str, EdgeType] = {
@@ -346,6 +370,13 @@ CREATE TABLE IF NOT EXISTS nodes_v2 (
   visibility       TEXT NOT NULL DEFAULT 'private',
   -- Revision chain: a node replaced by a newer one points at its successor.
   superseded_by    TEXT,
+  -- Bitemporal validity (v11.1.0). NULL is the convention, not '':
+  --   valid_from IS NULL  → the row has been valid since created_at
+  --   valid_to   IS NULL  → the row is still valid now
+  -- An empty string would read as "valid since the beginning of time" in a
+  -- string comparison, which is why the columns are nullable with no default.
+  valid_from       TEXT,
+  valid_to         TEXT,
   created_at       TEXT NOT NULL,
   updated_at       TEXT NOT NULL,
   style            TEXT,
@@ -366,6 +397,12 @@ CREATE TABLE IF NOT EXISTS edges_v2 (
   metadata     TEXT NOT NULL DEFAULT '{}',
   created_by   TEXT NOT NULL DEFAULT 'user',
   created_at   TEXT NOT NULL,
+  -- Bitemporal validity + revision chain (v11.1.0), same NULL convention as
+  -- nodes_v2: NULL valid_from falls back to created_at, NULL valid_to means
+  -- "still holds". Relationships go stale exactly like facts do.
+  valid_from   TEXT,
+  valid_to     TEXT,
+  superseded_by TEXT,
   -- Edge identity (v4): the normalized type AND the raw legacy type.
   -- Migrated rows keep their legacy_type discriminator, so two distinct
   -- legacy strings between one pair (e.g. "mentions" / "관련됨") stay
@@ -403,6 +440,20 @@ CREATE INDEX IF NOT EXISTS idx_edges_v2_legacy   ON edges_v2(legacy_type);
 """
 
 
+# ── Temporal read convention (v11.1.0) ──────────────────────────────────────
+# One predicate, shared by every ``as_of`` read so the fallback rule cannot
+# drift between call sites:
+#   * ``valid_from IS NULL``  → the row has been valid since ``created_at``
+#     (plan §10 mitigation: existing Brains keep working with no backfill)
+#   * ``valid_to IS NULL``    → the row is still valid
+# The window is half-open ``[valid_from, valid_to)``: the instant a fact is
+# superseded belongs to its successor, so an ``as_of`` at exactly that instant
+# returns exactly one of the two.
+TEMPORAL_PREDICATE_SQL = (
+    "COALESCE(valid_from, created_at) <= ? AND (valid_to IS NULL OR valid_to > ?)"
+)
+
+
 def _exec_script(conn: sqlite3.Connection, script: str) -> None:
     """Run a multi-statement SQL script on ``conn`` statement-by-statement.
 
@@ -456,6 +507,9 @@ class KGStoreV2:
             "metadata",
             "created_by",
             "created_at",
+            "valid_from",
+            "valid_to",
+            "superseded_by",
         },
         "nodes_v2": {
             "id",
@@ -469,6 +523,8 @@ class KGStoreV2:
             "workspace_id",
             "visibility",
             "superseded_by",
+            "valid_from",
+            "valid_to",
             "created_at",
             "updated_at",
             "style",
@@ -481,8 +537,17 @@ class KGStoreV2:
     # Columns added after a table's first release that can be healed in place
     # with ALTER TABLE ADD COLUMN (nullable / defaulted only).
     _V2_ADDABLE_COLUMNS = {
-        "nodes_v2": {"workspace_id": "TEXT", "superseded_by": "TEXT"},
-        "edges_v2": {},
+        "nodes_v2": {
+            "workspace_id": "TEXT",
+            "superseded_by": "TEXT",
+            "valid_from": "TEXT",
+            "valid_to": "TEXT",
+        },
+        "edges_v2": {
+            "valid_from": "TEXT",
+            "valid_to": "TEXT",
+            "superseded_by": "TEXT",
+        },
     }
 
     def _drop_stale_empty_v2_tables(self, conn: sqlite3.Connection) -> None:
@@ -586,6 +651,64 @@ class KGStoreV2:
             ("embed_dim", str(EMBED_DIM)),
         )
 
+    # ── Temporal stamping (v11.1.0) ──────────────────────────
+    #
+    # These are *primitives*, not a policy: the only caller in the product is
+    # the review-approval path in :mod:`lattice_brain.synthesis`. Background
+    # jobs and agents never call them directly — they raise a proposal and the
+    # user approves it. See ``ARCHITECTURE.md`` → "Temporal knowledge".
+    def stamp_node_validity(
+        self,
+        node_id: str,
+        *,
+        valid_from: Optional[str] = None,
+        valid_to: Optional[str] = None,
+        superseded_by: Optional[str] = None,
+    ) -> bool:
+        """Stamp a node's validity window. ``True`` when a row was updated.
+
+        Only the arguments actually supplied are written; ``None`` leaves the
+        stored value alone (it never clears a column back to NULL, so a second
+        resolution cannot silently un-supersede an earlier one).
+        """
+        return self._stamp_row(
+            "nodes_v2",
+            node_id,
+            {
+                "valid_from": valid_from,
+                "valid_to": valid_to,
+                "superseded_by": superseded_by,
+            },
+        )
+
+    def touch_node(self, node_id: str, *, at: Optional[str] = None) -> bool:
+        """Record one access: bump ``importance_score`` and set ``last_used``.
+
+        Read-path bookkeeping for the importance/decay report. Returns whether
+        a row existed; callers treat a ``False``/failure as harmless.
+        """
+        with self._conn() as conn:
+            cursor = conn.execute(
+                "UPDATE nodes_v2 SET importance_score = importance_score + 1.0, "
+                "last_used = ? WHERE id = ?",
+                (at or _stamp_now(), str(node_id)),
+            )
+            return int(cursor.rowcount or 0) > 0
+
+    def _stamp_row(
+        self, table: str, row_id: str, updates: Dict[str, Optional[str]]
+    ) -> bool:
+        fields = {key: value for key, value in updates.items() if value is not None}
+        if not fields:
+            return False
+        assignments = ", ".join(f"{key}=?" for key in fields)
+        with self._conn() as conn:
+            cursor = conn.execute(
+                f"UPDATE {table} SET {assignments} WHERE id=?",
+                (*fields.values(), str(row_id)),
+            )
+            return int(cursor.rowcount or 0) > 0
+
     # ── Maintenance ──────────────────────────────────────────
     def stats(self) -> Dict[str, Any]:
         with self._conn() as conn:

package/lattice_brain/graph/vector_index/__init__.py

@@ -0,0 +1,85 @@
+"""Pluggable vector-index backends + the durable pending-embed queue.
+
+Public surface (v11.1.0, Track 1):
+
+* :class:`VectorIndex` — the Protocol every backend satisfies.
+* :class:`BruteForceIndex` — exact, exhaustive, the default.
+* :class:`QuantizedIndex` — int8 storage, exhaustive, approximate scores.
+* :class:`HnswIndex` — approximate nearest neighbour, optional ``hnswlib``.
+* :func:`resolve_vector_index` / :func:`build_index` — ``LATTICEAI_VECTOR_INDEX``
+  resolution with an honest fallback reason.
+* :class:`VectorEmbedQueue` — SQLite-backed background embedding backlog.
+"""
+
+from __future__ import annotations
+
+from .base import (
+    IndexItem,
+    IndexStats,
+    ScoredId,
+    Similarity,
+    VectorIndex,
+    dot_similarity,
+    score_floor,
+    take_top,
+)
+from .brute_force import BRUTE_FORCE_BACKEND, BruteForceIndex
+from .hnsw import (
+    HNSW_BACKEND,
+    HNSW_META_SUFFIX,
+    HNSW_SUFFIX,
+    HnswIndex,
+    hnswlib_available,
+    load_hnswlib,
+    sidecar_paths,
+)
+from .jobs import (
+    DEFAULT_MAX_ATTEMPTS,
+    DEFAULT_TICK_LIMIT,
+    VECTOR_JOB_STATUSES,
+    VectorEmbedQueue,
+    VectorJobStore,
+)
+from .quantized import QUANTIZED_BACKEND, QuantizedIndex, quantize
+from .selector import (
+    DEFAULT_VECTOR_INDEX,
+    VECTOR_INDEX_CHOICES,
+    VECTOR_INDEX_ENV,
+    BackendSelection,
+    build_index,
+    resolve_vector_index,
+)
+
+__all__ = [
+    "BRUTE_FORCE_BACKEND",
+    "DEFAULT_MAX_ATTEMPTS",
+    "DEFAULT_TICK_LIMIT",
+    "DEFAULT_VECTOR_INDEX",
+    "HNSW_BACKEND",
+    "HNSW_META_SUFFIX",
+    "HNSW_SUFFIX",
+    "QUANTIZED_BACKEND",
+    "VECTOR_INDEX_CHOICES",
+    "VECTOR_INDEX_ENV",
+    "VECTOR_JOB_STATUSES",
+    "BackendSelection",
+    "BruteForceIndex",
+    "HnswIndex",
+    "IndexItem",
+    "IndexStats",
+    "QuantizedIndex",
+    "ScoredId",
+    "Similarity",
+    "VectorEmbedQueue",
+    "VectorIndex",
+    "VectorJobStore",
+    "build_index",
+    "dot_similarity",
+    "hnswlib_available",
+    "load_hnswlib",
+    "quantize",
+    "resolve_vector_index",
+    "score_floor",
+    "sidecar_paths",
+    "take_top",
+]

package/lattice_brain/graph/vector_index/base.py

@@ -0,0 +1,170 @@
+"""The vector-index contract: what it means to score embeddings (v11.1.0).
+
+Before this package the only vector "index" was the scan loop inside
+``retrieval_vector.vector_search``: fetch every embedding row, decode it,
+score it, sort. That is a perfectly good *exact* index and it stays the
+default — but it was welded to SQL row handling, so there was nowhere to put
+a second strategy without forking the search.
+
+This module defines the seam. A backend owns exactly one question — *given a
+query vector, which stored ids score highest* — and nothing about SQLite,
+chunk metadata, workspaces, or citations. The vectors are ``Sequence[float]``
+(the repo's embedders return ``list[float]``; numpy is not a dependency).
+
+Honesty contract: every backend reports whether it is ``approx`` (scores are
+estimates) and whether it is ``exhaustive`` (every stored vector was
+compared). Those two bits travel with the search result so a caller never has
+to guess whether "no match" means "not in your brain" or "the index did not
+look".
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Protocol,
+    Sequence,
+    Tuple,
+)
+
+#: ``(item_id, score)`` — the one shape every backend returns.
+ScoredId = Tuple[str, float]
+#: ``(item_id, vector, metadata)`` — one item as handed to :meth:`rebuild`.
+IndexItem = Tuple[str, Sequence[float], Mapping[str, Any]]
+#: Pluggable similarity so a backend can borrow the embedder's own function.
+Similarity = Callable[[Sequence[float], Sequence[float]], float]
+
+#: "no score floor" — ``0.0`` is a meaningful floor, so it cannot be the
+#: sentinel (the score-0-is-falsy trap this codebase keeps re-learning).
+NO_FLOOR = float("-inf")
+
+
+def dot_similarity(left: Sequence[float], right: Sequence[float]) -> float:
+    """Dot product, which is cosine similarity for unit-length vectors.
+
+    Mirrors :meth:`lattice_brain.embeddings.LocalEmbeddingModel.similarity`,
+    including its refusal to compare different dimensions: truncating to the
+    shorter vector produces a plausible-looking number that means nothing.
+    """
+    left_v, right_v = list(left), list(right)
+    if len(left_v) != len(right_v):
+        raise ValueError(
+            f"embedding dimension mismatch: {len(left_v)} vs {len(right_v)}; "
+            "the vector index was built with a different model"
+        )
+    return float(sum(a * b for a, b in zip(left_v, right_v, strict=True)))
+
+
+def score_floor(filter: Optional[Mapping[str, Any]]) -> float:
+    """Resolve ``filter["min_score"]`` (absent/unparseable → no floor)."""
+    if not filter:
+        return NO_FLOOR
+    raw = filter.get("min_score")
+    if raw is None:
+        return NO_FLOOR
+    try:
+        return float(raw)
+    except (TypeError, ValueError):
+        return NO_FLOOR
+
+
+def take_top(scored: List[ScoredId], top_k: int) -> List[ScoredId]:
+    """Sort by score descending (stable → insertion order breaks ties), cut."""
+    scored.sort(key=lambda pair: -pair[1])
+    return scored[: max(0, int(top_k))]
+
+
+@dataclass(frozen=True)
+class IndexStats:
+    """What a backend is willing to claim about itself."""
+
+    backend: str
+    size: int
+    dim: int
+    approx: bool
+    exhaustive: bool
+    available: bool = True
+    detail: Optional[str] = None
+
+    def as_dict(self) -> Dict[str, Any]:
+        return {
+            "backend": self.backend,
+            "size": self.size,
+            "dim": self.dim,
+            "approx": self.approx,
+            "exhaustive": self.exhaustive,
+            "available": self.available,
+            "detail": self.detail,
+        }
+
+
+class VectorIndex(Protocol):
+    """Structural contract implemented by every backend in this package.
+
+    Declared, never implemented — same convention as
+    :mod:`lattice_brain.graph._kg_contract`: the bodies raise so a stub can
+    never be mistaken for a working index.
+    """
+
+    @property
+    def backend(self) -> str:
+        """Stable id of the scoring strategy (goes into ``recall.backend``)."""
+        raise NotImplementedError
+
+    @property
+    def approx(self) -> bool:
+        """True when scores are estimates rather than exact similarities."""
+        raise NotImplementedError
+
+    @property
+    def exhaustive(self) -> bool:
+        """True when every stored vector is compared on every search."""
+        raise NotImplementedError
+
+    def add(
+        self,
+        id: str,
+        vector: Sequence[float],
+        metadata: Optional[Mapping[str, Any]] = None,
+    ) -> None:
+        raise NotImplementedError
+
+    def remove(self, id: str) -> None:
+        raise NotImplementedError
+
+    def rebuild(self, items: Iterable[IndexItem]) -> None:
+        raise NotImplementedError
+
+    def metadata_for(self, id: str) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    def search(
+        self,
+        query: Sequence[float],
+        top_k: int,
+        filter: Optional[Mapping[str, Any]] = None,
+    ) -> List[ScoredId]:
+        raise NotImplementedError
+
+    def stats(self) -> IndexStats:
+        raise NotImplementedError
+
+
+__all__ = [
+    "NO_FLOOR",
+    "IndexItem",
+    "IndexStats",
+    "ScoredId",
+    "Similarity",
+    "VectorIndex",
+    "dot_similarity",
+    "score_floor",
+    "take_top",
+]

package/lattice_brain/graph/vector_index/brute_force.py

@@ -0,0 +1,114 @@
+"""Exact cosine scan — the historical ``vector_search`` scoring loop, moved.
+
+This backend is deliberately boring and deliberately the default. It compares
+the query against every vector it holds, so its answer is the ground truth the
+other backends are measured against (``scripts/bench_vector_index.py`` reports
+recall@k relative to this one).
+
+Behaviour parity with the pre-11.1.0 inline loop is the whole point of the
+move: same similarity function (injected, so it is the embedder's own), same
+``score < min_score`` drop, same stable ordering. ``retrieval_vector`` feeds
+it in bounded batches so a scan never holds every decoded vector at once.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Mapping, Optional, Sequence
+
+from .base import (
+    IndexItem,
+    IndexStats,
+    ScoredId,
+    Similarity,
+    dot_similarity,
+    score_floor,
+    take_top,
+)
+
+BRUTE_FORCE_BACKEND = "bruteforce-cosine"
+
+
+class BruteForceIndex:
+    """Exhaustive, exact, in-memory cosine index."""
+
+    def __init__(
+        self,
+        *,
+        dim: int = 0,
+        similarity: Optional[Similarity] = None,
+    ) -> None:
+        self._dim = int(dim)
+        self._similarity: Similarity = (
+            dot_similarity if similarity is None else similarity
+        )
+        self._vectors: Dict[str, List[float]] = {}
+        self._metadata: Dict[str, Dict[str, Any]] = {}
+
+    @property
+    def backend(self) -> str:
+        return BRUTE_FORCE_BACKEND
+
+    @property
+    def approx(self) -> bool:
+        return False
+
+    @property
+    def exhaustive(self) -> bool:
+        return True
+
+    def add(
+        self,
+        id: str,
+        vector: Sequence[float],
+        metadata: Optional[Mapping[str, Any]] = None,
+    ) -> None:
+        values = [float(value) for value in vector]
+        if not self._dim:
+            self._dim = len(values)
+        key = str(id)
+        self._vectors[key] = values
+        self._metadata[key] = dict(metadata or {})
+
+    def remove(self, id: str) -> None:
+        key = str(id)
+        self._vectors.pop(key, None)
+        self._metadata.pop(key, None)
+
+    def rebuild(self, items: Iterable[IndexItem]) -> None:
+        self._vectors.clear()
+        self._metadata.clear()
+        for item_id, vector, metadata in items:
+            self.add(item_id, vector, metadata)
+
+    def metadata_for(self, id: str) -> Dict[str, Any]:
+        """Metadata stored alongside ``id`` (empty dict when unknown)."""
+        return dict(self._metadata.get(str(id), {}))
+
+    def search(
+        self,
+        query: Sequence[float],
+        top_k: int,
+        filter: Optional[Mapping[str, Any]] = None,
+    ) -> List[ScoredId]:
+        floor = score_floor(filter)
+        query_vector = list(query)
+        scored: List[ScoredId] = []
+        for item_id, vector in self._vectors.items():
+            score = float(self._similarity(query_vector, vector))
+            if score < floor:
+                continue
+            scored.append((item_id, score))
+        return take_top(scored, top_k)
+
+    def stats(self) -> IndexStats:
+        return IndexStats(
+            backend=self.backend,
+            size=len(self._vectors),
+            dim=self._dim,
+            approx=False,
+            exhaustive=True,
+            detail=None,
+        )
+
+
+__all__ = ["BRUTE_FORCE_BACKEND", "BruteForceIndex"]