ltcai 11.0.1 → 11.1.0

package/lattice_brain/graph/image_vectors.py

@@ -0,0 +1,230 @@
+"""The image vector space — stored apart, joined by late fusion (v11.1.0).
+
+A CLIP image vector and a BGE text vector are both "1024 floats", and that is
+the entire extent of what they have in common. Filing them in one index means
+every cosine between them is a number with no meaning, and the ranking that
+comes out looks exactly like a working search.
+
+So image vectors get their own table (``image_embeddings``) keyed by the vision
+model that produced them, and their own search. Text queries never touch this
+index: they reach pictures through OCR text and captions, which are text and
+live in the text index like everything else. This index answers one question —
+*which stored images look like this vector* — and its scores enter
+``hybrid_search`` by late fusion, as a separate channel with its own weight,
+rather than by pretending to be comparable up front.
+
+The table is created on demand, is derivable from the images themselves, and is
+never the source of truth: dropping it costs a re-embed, not a memory.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+import struct
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
+
+from .vector_index import build_index, resolve_vector_index
+
+IMAGE_VECTOR_TABLE = "image_embeddings"
+#: Default weight the image channel carries when fused with text scores.
+DEFAULT_IMAGE_FUSION_WEIGHT = 0.5
+
+
+def ensure_image_vector_table(conn: sqlite3.Connection) -> None:
+    """Create the image-vector table if this graph has never held one."""
+    conn.execute(
+        f"""
+        CREATE TABLE IF NOT EXISTS {IMAGE_VECTOR_TABLE} (
+            node_id TEXT PRIMARY KEY,
+            model_id TEXT NOT NULL,
+            dim INTEGER NOT NULL,
+            space TEXT NOT NULL DEFAULT 'image',
+            vector BLOB NOT NULL,
+            updated_at TEXT
+        )
+        """
+    )
+
+
+def encode_vector(vector: Sequence[float]) -> bytes:
+    values = [float(value) for value in vector]
+    return struct.pack(f"<{len(values)}f", *values)
+
+
+def decode_vector(payload: bytes) -> List[float]:
+    count = len(payload) // 4
+    return list(struct.unpack(f"<{count}f", payload[: count * 4]))
+
+
+def record_image_vector(
+    store: Any,
+    *,
+    node_id: str,
+    vector: Sequence[float],
+    model_id: str,
+    space: str = "image",
+    updated_at: Optional[str] = None,
+) -> bool:
+    """Persist one image vector. Returns False when there is nothing to store.
+
+    Never raises: an image whose vector could not be filed is still a memory,
+    and :func:`image_index_status` reports it as backlog.
+    """
+    values = [float(value) for value in (vector or [])]
+    if not node_id or not values or not model_id:
+        return False
+    try:
+        with store._connect() as conn:
+            ensure_image_vector_table(conn)
+            conn.execute(
+                f"""
+                INSERT INTO {IMAGE_VECTOR_TABLE}
+                    (node_id, model_id, dim, space, vector, updated_at)
+                VALUES (?, ?, ?, ?, ?, ?)
+                ON CONFLICT(node_id) DO UPDATE SET
+                    model_id=excluded.model_id,
+                    dim=excluded.dim,
+                    space=excluded.space,
+                    vector=excluded.vector,
+                    updated_at=excluded.updated_at
+                """,
+                (
+                    str(node_id),
+                    str(model_id),
+                    len(values),
+                    str(space or "image"),
+                    encode_vector(values),
+                    updated_at,
+                ),
+            )
+        return True
+    except Exception:  # noqa: BLE001 — a vector is derivable; the image is not
+        return False
+
+
+def _rows(
+    store: Any, model_id: Optional[str], dim: Optional[int]
+) -> List[Tuple[str, List[float]]]:
+    with store._connect() as conn:
+        ensure_image_vector_table(conn)
+        if model_id:
+            cursor = conn.execute(
+                f"SELECT node_id, vector, dim FROM {IMAGE_VECTOR_TABLE} WHERE model_id=? ORDER BY node_id ASC",
+                (str(model_id),),
+            )
+        else:
+            cursor = conn.execute(
+                f"SELECT node_id, vector, dim FROM {IMAGE_VECTOR_TABLE} ORDER BY node_id ASC"
+            )
+        rows = cursor.fetchall()
+    out: List[Tuple[str, List[float]]] = []
+    for row in rows:
+        if dim is not None and int(row["dim"]) != dim:
+            # A different width means a different model. Comparing across it is
+            # the exact silent wrongness this whole module exists to prevent.
+            continue
+        out.append((str(row["node_id"]), decode_vector(row["vector"])))
+    return out
+
+
+def image_similarity_search(
+    store: Any,
+    vector: Sequence[float],
+    *,
+    top_k: int = 10,
+    model_id: Optional[str] = None,
+    min_score: float = 0.0,
+) -> Dict[str, Any]:
+    """Rank stored images against a query *image* vector.
+
+    Scoring is delegated to the pluggable backend layer (Track 1), so this
+    index picks up ``LATTICEAI_VECTOR_INDEX`` and reports the same
+    ``approx``/``exhaustive`` honesty bits as the text index.
+    """
+    query = [float(value) for value in (vector or [])]
+    result: Dict[str, Any] = {
+        "matches": [],
+        "count": 0,
+        "candidates": 0,
+        "model_id": model_id,
+        "detail": None,
+    }
+    if not query:
+        result["detail"] = "an image query needs an image vector"
+        return result
+    try:
+        rows = _rows(store, model_id, len(query))
+    except Exception as exc:  # noqa: BLE001 — degrade, never fail the search
+        result["detail"] = f"image vector index unavailable: {exc}"
+        return result
+    result["candidates"] = len(rows)
+    selection = resolve_vector_index()
+    index = build_index(selection, dim=len(query))
+    index.rebuild((node_id, values, {}) for node_id, values in rows)
+    scored = index.search(query, max(1, int(top_k)), {"min_score": min_score})
+    result["matches"] = [
+        {"node_id": node_id, "score": round(float(score), 6)} for node_id, score in scored
+    ]
+    result["count"] = len(result["matches"])
+    result["index"] = selection.as_dict()
+    return result
+
+
+def image_index_status(store: Any) -> Dict[str, Any]:
+    """How many images carry a vector, and under which model."""
+    status: Dict[str, Any] = {"vectors": 0, "models": {}, "detail": None}
+    try:
+        with store._connect() as conn:
+            ensure_image_vector_table(conn)
+            rows = conn.execute(
+                f"SELECT model_id, COUNT(*) AS total FROM {IMAGE_VECTOR_TABLE} GROUP BY model_id"
+            ).fetchall()
+    except Exception as exc:  # noqa: BLE001 — status must never raise
+        status["detail"] = str(exc)
+        return status
+    models = {str(row["model_id"]): int(row["total"]) for row in rows}
+    status["models"] = models
+    status["vectors"] = sum(models.values())
+    return status
+
+
+def fuse_image_scores(
+    matches: Iterable[Dict[str, Any]],
+    image_scores: Dict[str, float],
+    *,
+    weight: float = DEFAULT_IMAGE_FUSION_WEIGHT,
+) -> int:
+    """Late-fuse image-space scores into an already-fused match list.
+
+    Mutates each match in place — ``scores.image`` records the raw image-space
+    similarity and ``score`` blends it in — and returns how many matches the
+    image channel actually touched. Late fusion is the point: the two spaces
+    are ranked separately and combined at the end, so neither is ever asked to
+    interpret the other's numbers.
+    """
+    weight = max(0.0, min(1.0, float(weight)))
+    touched = 0
+    for match in matches:
+        node_id = str(match.get("node_id") or "")
+        raw = image_scores.get(node_id)
+        if raw is None:
+            continue
+        touched += 1
+        scores = match.setdefault("scores", {})
+        scores["image"] = round(float(raw), 6)
+        blended = (1.0 - weight) * float(match.get("score") or 0.0) + weight * float(raw)
+        match["score"] = round(blended, 6)
+    return touched
+
+
+__all__ = [
+    "DEFAULT_IMAGE_FUSION_WEIGHT",
+    "IMAGE_VECTOR_TABLE",
+    "decode_vector",
+    "encode_vector",
+    "ensure_image_vector_table",
+    "fuse_image_scores",
+    "image_index_status",
+    "image_similarity_search",
+    "record_image_vector",
+]

package/lattice_brain/graph/ingest.py

@@ -636,14 +636,20 @@ class KnowledgeGraphIngestMixin(_Core):
         modified_at: Optional[str] = None,
         conversation_id: Optional[str] = None,
         metadata: Optional[Dict[str, Any]] = None,
+        node_type: str = "Document",
     ) -> Dict[str, Any]:
         """Unified text/web ingestion: one shape for URL, browser tab, note, text.
 
-        Creates a content ``Document`` node (idempotent by content hash), a
-        ``Source`` node linked via ``indexed_from``, RAG chunks, and extracted
+        Creates a content node (idempotent by content hash), a ``Source`` node
+        linked via ``indexed_from``, RAG chunks, and extracted
         Concept/Task/Decision nodes — mirroring ingest_document for non-file
         sources. Returns the full set of ids the caller needs to record
         provenance, including ``duplicate`` (was the content already indexed).
+
+        ``node_type`` names the content node. It defaults to ``Document``, so
+        every historical caller is unchanged; a door that knows its material is
+        something else (``Audio`` for a recording) says so here rather than
+        writing a second ingest path to say it.
         """
         source_type = str(source_type or "text")
         text = str(text or "")
@@ -674,11 +680,11 @@ class KnowledgeGraphIngestMixin(_Core):
 
         with self._connect() as conn:
             duplicate = self._node_exists(conn, content_id)
-            # ── 콘텐츠 노드 (점: 명사 — 문서) ────────────────────────────────
+            # ── 콘텐츠 노드 (점: 명사 — 기본은 문서) ─────────────────────────
             self._upsert_node(
                 conn,
                 content_id,
-                "Document",
+                node_type,
                 title,
                 summary=(text or title)[:500],
                 metadata=node_meta,
@@ -813,7 +819,7 @@ class KnowledgeGraphIngestMixin(_Core):
 
         return {
             "node_id": content_id,
-            "type": "Document",
+            "type": node_type,
             "source_node_id": source_node_id,
             "content_hash": content_hash,
             "chunk_ids": chunk_ids,

package/lattice_brain/graph/provenance.py

@@ -37,9 +37,34 @@ class KnowledgeGraphProvenanceMixin(_Core):
         permissions: Optional[Dict[str, Any]] = None,
         metadata: Optional[Dict[str, Any]] = None,
     ) -> Dict[str, Any]:
-        """Append a provenance record for an ingested node (audit trail)."""
+        """Record where an ingested node came from (upsert by origin).
+
+        Row identity is ``(node, content, source_type, source_uri, pipeline)``
+        — deliberately *not* the wall clock. Through 11.0.x the basis included
+        a second-resolution timestamp, which made the record's identity depend
+        on when it happened: re-ingesting unchanged content twice inside the
+        same second collapsed onto one row, and one second later appended a
+        duplicate. That is not an audit trail, it is a race — the same class of
+        defect as the 11.0.0 review-item ids — and it grew this table (and the
+        "recent ingestions" list built from it) without bound on every re-scan
+        of an unchanged folder or vault.
+
+        With the clock out of the basis, re-ingesting the same content from the
+        same origin *updates* one record (``created_at`` moves to the latest
+        sighting, so "recently seen" stays true), while genuinely new content or
+        a genuinely different origin — another source URI, another pipeline —
+        still appends its own record. The timestamp is data on the row, never
+        part of its identity. Every individual ingest event remains visible in
+        the audit log (``kg_ingest``), which is where per-event history belongs.
+        """
         now = _now()
-        prov_basis = f"{node_id}|{content_hash or ''}|{now}"
+        prov_basis = "|".join([
+            node_id,
+            content_hash or "",
+            source_type,
+            source_uri or "",
+            pipeline,
+        ])
         prov_id = f"prov:{_sha256_text(prov_basis)[:24]}"
         with self._connect() as conn:
             conn.execute(

package/lattice_brain/graph/retrieval.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Sequence
 
 # ruff: noqa: F403,F405
 from ._kg_common import *  # noqa: F403,F401
@@ -29,6 +29,30 @@ from .fusion import (
 )
 from .retrieval_reads import KnowledgeGraphReadsMixin  # noqa: F401
 
+#: Node types that are a *thing you can look at or listen to*, not prose. A
+#: match of one of these means the answer rests on more than text.
+MULTIMODAL_NODE_TYPES = ("Image", "ImageText")
+
+
+def multimodal_signal(matches: Iterable[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """``{"images": n, "types": [...]}`` when a result set includes pictures.
+
+    ``None`` when it does not: the context-quality contract stays four keys
+    wide for the ordinary all-text case, and a caller that sees the key knows
+    it means something rather than having to compare a zero.
+    """
+    images = 0
+    seen: List[str] = []
+    for match in matches:
+        node_type = str(match.get("type") or "")
+        if node_type in MULTIMODAL_NODE_TYPES:
+            images += 1
+            if node_type not in seen:
+                seen.append(node_type)
+    if not images:
+        return None
+    return {"images": images, "types": seen}
+
 
 def context_quality_signal(
     mode: str,
@@ -36,6 +60,7 @@ def context_quality_signal(
     *,
     reason: Optional[str] = None,
     vector: Optional[Dict[str, Any]] = None,
+    multimodal: Optional[Dict[str, Any]] = None,
 ) -> Dict[str, Any]:
     """Honest RAG context-quality signal (v9.8.0, additive contract).
 
@@ -54,6 +79,11 @@ def context_quality_signal(
     there is a caveat to report**: an exact, complete vector scan is the
     contract's baseline assumption, so annotating it would be noise, and the
     four-key shape stays exactly what existing consumers pin.
+
+    ``multimodal`` (v11.1.0) follows the same present-only-when-true rule and
+    says that part of this context is a picture. "6 nodes" reads differently
+    when two of them are screenshots whose text came out of OCR, and the
+    surface that has to explain the answer deserves to know.
     """
     nodes = max(0, int(nodes or 0))
     mode = str(mode or "none")
@@ -79,6 +109,8 @@ def context_quality_signal(
     }
     if vector is not None:
         signal["vector"] = dict(vector)
+    if multimodal is not None:
+        signal["multimodal"] = dict(multimodal)
     return signal
 
 
@@ -95,6 +127,7 @@ class KnowledgeGraphRetrievalMixin(_Core):
         "SlideDeck",  # 프레젠테이션
         "Image",  # 이미지
         "ImageText",  # OCR 텍스트
+        "Audio",  # 녹음 / 음성 메모 (11.1.0)
         "Concept",  # 개념 / 아이디어 / 기술 용어
         "Person",  # 사람
         "Error",  # 오류 / 버그
@@ -344,6 +377,7 @@ class KnowledgeGraphRetrievalMixin(_Core):
                         "SlideDeck",
                         "Image",
                         "ImageText",
+                        "Audio",
                         "Page",
                         "Slide",
                     }
@@ -387,6 +421,8 @@ class KnowledgeGraphRetrievalMixin(_Core):
         lexical_limit: Optional[int] = None,
         vector_limit: Optional[int] = None,
         min_vector_score: float = 0.0,
+        image_vector: Optional[Sequence[float]] = None,
+        image_fusion_weight: Optional[float] = None,
     ) -> Dict[str, Any]:
         """Unified lexical + vector retrieval with alpha-weighted linear fusion.
 
@@ -418,6 +454,15 @@ class KnowledgeGraphRetrievalMixin(_Core):
         fused score into the ``[0.5, 1.0]`` band (``scores.age_decay``).
         Passing an explicit ``alpha`` pins it exactly as before and disables
         rewrite + decay.
+
+        ``image_vector`` (v11.1.0) is the *late fusion* seam for the separate
+        image space: the caller supplies a query vector from the same vision
+        model that embedded the pictures, its own index is ranked
+        independently, and only then are the two rankings blended
+        (``image_fusion_weight``, default 0.5). A text query never produces
+        one — it reaches images through their OCR text and captions — which is
+        exactly why the image channel has to enter at the end rather than
+        pretending to share the text index.
         """
         query = str(query or "").strip()
         try:
@@ -705,6 +750,16 @@ class KnowledgeGraphRetrievalMixin(_Core):
                 match["scores"]["age_decay"] = round(multiplier, 6)
                 match["score"] = round(float(match["score"]) * multiplier, 6)
 
+        # Late fusion of the image space (v11.1.0). Runs after the text
+        # channels have produced a ranking and before the cut, so image
+        # evidence can lift a picture into the answer without ever having been
+        # compared against a text vector.
+        image_fusion: Optional[Dict[str, Any]] = None
+        if image_vector is not None:
+            image_fusion = self._fuse_image_channel(
+                matches, image_vector, top_k=top_k, weight=image_fusion_weight
+            )
+
         matches.sort(key=lambda item: (-item["score"], item["node_id"]))
         # Optional cross-encoder rerank (v9.9.5). Off by default; when the
         # env kill-switch is set and the model loads, pair scores reorder the
@@ -749,8 +804,57 @@ class KnowledgeGraphRetrievalMixin(_Core):
                 result["vector_degraded"] = "partial_recall"
         vector_meta["degraded"] = result.get("vector_degraded")
         result["vector"] = vector_meta
+        multimodal = multimodal_signal(matches)
+        if multimodal is not None or image_fusion is not None:
+            result["multimodal"] = {
+                **(multimodal or {"images": 0, "types": []}),
+                **({"image_fusion": image_fusion} if image_fusion is not None else {}),
+            }
         return result
 
+    def _fuse_image_channel(
+        self,
+        matches: List[Dict[str, Any]],
+        image_vector: Sequence[float],
+        *,
+        top_k: int,
+        weight: Optional[float],
+    ) -> Dict[str, Any]:
+        """Rank the image index separately, then blend it into ``matches``.
+
+        Any failure degrades to "the image channel contributed nothing" with
+        the reason attached — an image index that cannot be read is not a
+        reason to lose the text answer.
+        """
+        from .image_vectors import (
+            DEFAULT_IMAGE_FUSION_WEIGHT,
+            fuse_image_scores,
+            image_similarity_search,
+        )
+
+        share = DEFAULT_IMAGE_FUSION_WEIGHT if weight is None else float(weight)
+        report: Dict[str, Any] = {
+            "weight": round(max(0.0, min(1.0, share)), 4),
+            "candidates": 0,
+            "fused": 0,
+            "detail": None,
+        }
+        try:
+            found = image_similarity_search(
+                self, image_vector, top_k=max(1, int(top_k) * 2)
+            )
+        except Exception as exc:  # noqa: BLE001 — never fail the text answer
+            report["detail"] = f"image index unavailable: {exc}"
+            return report
+        report["candidates"] = int(found.get("candidates") or 0)
+        report["detail"] = found.get("detail")
+        scores = {
+            str(row.get("node_id")): float(row.get("score") or 0.0)
+            for row in found.get("matches") or []
+        }
+        report["fused"] = fuse_image_scores(matches, scores, weight=share)
+        return report
+
     def context_for_query(
         self,
         query: str,
@@ -788,6 +892,7 @@ class KnowledgeGraphRetrievalMixin(_Core):
         matches: List[Dict[str, Any]] = []
         retrieval_mode = "none"
         vector_meta: Optional[Dict[str, Any]] = None
+        multimodal_meta: Optional[Dict[str, Any]] = None
         if use_hybrid:
             try:
                 hybrid = self.hybrid_search(
@@ -880,10 +985,16 @@ class KnowledgeGraphRetrievalMixin(_Core):
         context = "\n".join(lines)
         if not with_meta:
             return context
+        # Only the context that actually reached the model counts as
+        # multimodal — matches trimmed by ``limit`` are not in the answer.
+        multimodal_meta = multimodal_signal(matches[:limit])
         return {
             "context": context,
             "quality": context_quality_signal(
-                retrieval_mode, len(matches[:limit]), vector=vector_meta
+                retrieval_mode,
+                len(matches[:limit]),
+                vector=vector_meta,
+                multimodal=multimodal_meta,
             ),
         }
 

package/lattice_brain/graph/retrieval_docgen.py

@@ -54,9 +54,9 @@ class KnowledgeGraphDocGenMixin(_Core):
                     FROM {nt}
                     WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
                       AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
-                                   'Spreadsheet', 'Image', 'ImageText', 'Chat',
-                                   'Decision', 'Task', 'Concept', 'Feature',
-                                   'Page', 'Slide')
+                                   'Spreadsheet', 'Image', 'ImageText', 'Audio',
+                                   'Chat', 'Decision', 'Task', 'Concept',
+                                   'Feature', 'Page', 'Slide')
                     ORDER BY updated_at DESC, id ASC
                     LIMIT ?
                     """,
@@ -75,9 +75,9 @@ class KnowledgeGraphDocGenMixin(_Core):
                         FROM {nt}
                         WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
                           AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
-                                       'Spreadsheet', 'Image', 'ImageText', 'Chat',
-                                       'Decision', 'Task', 'Concept', 'Feature',
-                                       'Page', 'Slide')
+                                       'Spreadsheet', 'Image', 'ImageText', 'Audio',
+                                       'Chat', 'Decision', 'Task', 'Concept',
+                                       'Feature', 'Page', 'Slide')
                         ORDER BY updated_at DESC, id ASC
                         LIMIT ?
                         """,

package/lattice_brain/graph/schema.py

@@ -128,6 +128,12 @@ class NodeType(str, Enum):
     PREFERENCE = "PREFERENCE"  # 선호 (좋아함/싫어함/선호 방식)
     HABIT = "HABIT"  # 습관 (반복되는 행동)
     RELATIONSHIP = "RELATIONSHIP"  # 관계 (동료/친구/멘토 등)
+    # v11.1.0 Multi-modal (Track 3) — a recording is its own noun. It used to
+    # enter through the text door as a DOCUMENT because "a transcript *is*
+    # text"; that is true of the transcript and false of the recording, which
+    # exists whether or not anyone could hear it. IMAGE has been first-class
+    # since 3.6.0 for the same reason, and AUDIO now sits beside it.
+    AUDIO = "AUDIO"  # 녹음 / 음성 메모
 
     @classmethod
     def from_legacy(cls, label: str) -> "NodeType":
@@ -276,6 +282,9 @@ _LEGACY_NODE_MAP: Dict[str, NodeType] = {
     "relationship": NodeType.RELATIONSHIP,
     "관계": NodeType.RELATIONSHIP,
     "결정": NodeType.DECISION,
+    # v11.1.0 Multi-modal — recordings as a first-class noun.
+    "audio": NodeType.AUDIO,
+    "오디오": NodeType.AUDIO,
 }
 
 _LEGACY_EDGE_MAP: Dict[str, EdgeType] = {