ltcai 8.9.0 → 9.0.0

package/lattice_brain/graph/retrieval_docgen.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+# ruff: noqa: F403,F405
+
+from ._kg_common import *  # noqa: F403,F401
+
+
+class KnowledgeGraphDocGenMixin:
+    """Multi-hop retrieval specialised for document generation, split out
+    of retrieval. Composed into KnowledgeGraphStore alongside the other
+    retrieval mixins; shared instance means sibling helpers resolve via MRO.
+    """
+
+    def search_for_document_generation(
+        self, query: str, limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """Hybrid retrieval optimized for document generation.
+
+        Scoring: 0.5*text_relevance + 0.3*graph_relationship + 0.2*recency
+        Returns nodes with rich context for document generation prompts.
+        """
+        query = str(query or "").strip()
+        if not query:
+            return []
+        limit = max(1, min(int(limit or 10), 50))
+        terms = _topic_candidates(query, limit=12)
+        now = datetime.now()
+        nt, et = self._read_tables()
+
+        with self._connect() as conn:
+            candidate_rows = []
+            seen_ids = set()
+
+            if query:
+                q = f"%{query}%"
+                rows = conn.execute(
+                    f"""
+                        SELECT id, type, title, summary, metadata_json, updated_at
+                        FROM {nt}
+                        WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
+                          AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
+                                       'Spreadsheet', 'Image', 'ImageText', 'Chat',
+                                       'Decision', 'Task', 'Concept', 'Feature',
+                                       'Page', 'Slide')
+                        ORDER BY updated_at DESC, id ASC
+                        LIMIT ?
+                        """,
+                    (q, q, q, limit * 5),
+                ).fetchall()
+                for row in rows:
+                    if row["id"] not in seen_ids:
+                        seen_ids.add(row["id"])
+                        candidate_rows.append(row)
+
+            for term in terms:
+                t = f"%{term}%"
+                rows = conn.execute(
+                    f"""
+                        SELECT id, type, title, summary, metadata_json, updated_at
+                        FROM {nt}
+                        WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
+                          AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
+                                       'Spreadsheet', 'Image', 'ImageText', 'Chat',
+                                       'Decision', 'Task', 'Concept', 'Feature',
+                                       'Page', 'Slide')
+                        ORDER BY updated_at DESC, id ASC
+                        LIMIT ?
+                        """,
+                    (t, t, t, limit * 3),
+                ).fetchall()
+                for row in rows:
+                    if row["id"] not in seen_ids:
+                        seen_ids.add(row["id"])
+                        candidate_rows.append(row)
+
+            scored_results = []
+            for row in candidate_rows:
+                haystack = (
+                    f"{row['title']} {row['summary']} {row['metadata_json']}".lower()
+                )
+
+                text_hits = sum(1 for term in terms if term.lower() in haystack)
+                text_score = min(1.0, text_hits / max(len(terms), 1))
+
+                edge_count = conn.execute(
+                    f"SELECT COUNT(*) AS c FROM {et} WHERE from_node=? OR to_node=?",
+                    (row["id"], row["id"]),
+                ).fetchone()["c"]
+                graph_score = min(1.0, math.log1p(edge_count) / 4.0)
+
+                recency = _recency_score(
+                    row["updated_at"], now=now, half_life_days=14.0
+                )
+
+                doc_type_boost = (
+                    1.2
+                    if row["type"]
+                    in (
+                        "Document",
+                        "File",
+                        "SlideDeck",
+                        "Decision",
+                    )
+                    else 1.0
+                )
+
+                hybrid_score = (
+                    0.5 * text_score + 0.3 * graph_score + 0.2 * recency
+                ) * doc_type_boost
+
+                meta = _safe_loads(row["metadata_json"])
+                neighbor_concepts = []
+                neighbor_rows = conn.execute(
+                    f"""
+                        SELECT n.title, n.type FROM {et} e
+                        JOIN {nt} n ON n.id = CASE WHEN e.from_node = ? THEN e.to_node ELSE e.from_node END
+                        WHERE (e.from_node = ? OR e.to_node = ?)
+                          AND n.type IN ('Concept', 'Feature', 'Decision', 'Task')
+                        LIMIT 8
+                        """,
+                    (row["id"], row["id"], row["id"]),
+                ).fetchall()
+                for nr in neighbor_rows:
+                    neighbor_concepts.append({"title": nr["title"], "type": nr["type"]})
+
+                scored_results.append(
+                    {
+                        "id": row["id"],
+                        "type": row["type"],
+                        "title": row["title"],
+                        "summary": row["summary"],
+                        "metadata": meta,
+                        "updated_at": row["updated_at"],
+                        "hybrid_score": round(hybrid_score, 4),
+                        "scores": {
+                            "text": round(text_score, 4),
+                            "graph": round(graph_score, 4),
+                            "recency": round(recency, 4),
+                        },
+                        "related_concepts": neighbor_concepts,
+                    }
+                )
+
+            scored_results.sort(key=lambda x: x["hybrid_score"], reverse=True)
+            return scored_results[:limit]
+
+    def multi_hop_context(
+        self, node_ids: List[str], max_hops: int = 2
+    ) -> Dict[str, Any]:
+        """Multi-hop graph traversal from seed nodes for richer context."""
+        visited_nodes = set()
+        visited_edges = set()
+        all_nodes = []
+        all_edges = []
+        frontier = set(node_ids)
+        nt, et = self._read_tables()
+
+        with self._connect() as conn:
+            for hop in range(max_hops):
+                if not frontier:
+                    break
+                next_frontier = set()
+                for nid in frontier:
+                    if nid in visited_nodes:
+                        continue
+                    visited_nodes.add(nid)
+                    row = conn.execute(
+                        f"SELECT id, type, title, summary, metadata_json, updated_at FROM {nt} WHERE id=?",
+                        (nid,),
+                    ).fetchone()
+                    if row:
+                        all_nodes.append(
+                            {
+                                "id": row["id"],
+                                "type": row["type"],
+                                "title": row["title"],
+                                "summary": row["summary"],
+                                "metadata": _safe_loads(row["metadata_json"]),
+                                "hop": hop,
+                            }
+                        )
+                    edge_rows = conn.execute(
+                        f"""
+                            SELECT id, from_node, to_node, type, weight
+                            FROM {et} WHERE from_node=? OR to_node=?
+                            ORDER BY id ASC
+                            """,
+                        (nid, nid),
+                    ).fetchall()
+                    for er in edge_rows:
+                        if er["id"] not in visited_edges:
+                            visited_edges.add(er["id"])
+                            all_edges.append(
+                                {
+                                    "from": er["from_node"],
+                                    "to": er["to_node"],
+                                    "type": er["type"],
+                                    "weight": er["weight"],
+                                }
+                            )
+                            other = (
+                                er["to_node"]
+                                if er["from_node"] == nid
+                                else er["from_node"]
+                            )
+                            if other not in visited_nodes:
+                                next_frontier.add(other)
+                frontier = next_frontier
+
+        return {"nodes": all_nodes, "edges": all_edges}

package/lattice_brain/graph/retrieval_vector.py

@@ -0,0 +1,460 @@
+from __future__ import annotations
+
+# ruff: noqa: F403,F405
+
+from ._kg_common import *  # noqa: F403,F401
+
+
+class KnowledgeGraphVectorMixin:
+    """Vector-embedding index build/status/search, split out of retrieval.
+
+    Composed into KnowledgeGraphStore alongside KnowledgeGraphRetrievalMixin;
+    both mixins share the same instance, so vector methods still reach sibling
+    retrieval/write helpers (e.g. self._vector_text_for_node) through the MRO.
+    """
+
+    def _iter_vector_source_items(
+        self,
+        conn: sqlite3.Connection,
+        *,
+        include_nodes: bool = True,
+        include_chunks: bool = True,
+    ) -> List[Dict[str, Any]]:
+        items: List[Dict[str, Any]] = []
+        if include_nodes:
+            for row in conn.execute(
+                """
+                    SELECT id, type, title, summary, metadata_json
+                    FROM nodes
+                    WHERE type <> 'Chunk'
+                    ORDER BY updated_at DESC, id ASC
+                    """
+            ).fetchall():
+                metadata = _safe_loads(row["metadata_json"])
+                text = self._vector_text_for_node(
+                    title=row["title"],
+                    summary=row["summary"] or "",
+                    metadata=metadata,
+                )
+                if text:
+                    items.append(
+                        {
+                            "item_id": row["id"],
+                            "item_type": "node",
+                            "source_node": row["id"],
+                            "text": text,
+                            "metadata": {"node_type": row["type"], **metadata},
+                        }
+                    )
+        if include_chunks:
+            for row in conn.execute(
+                """
+                    SELECT c.id, c.source_node AS parent_source_node, c.text, c.metadata_json
+                    FROM chunks c
+                    JOIN nodes n ON n.id=c.id
+                    ORDER BY c.created_at DESC, c.id ASC
+                    """
+            ).fetchall():
+                metadata = _safe_loads(row["metadata_json"])
+                text = _clean_text(row["text"] or "")
+                if text:
+                    items.append(
+                        {
+                            "item_id": row["id"],
+                            "item_type": "chunk",
+                            "source_node": row["id"],
+                            "text": text,
+                            "metadata": {
+                                **metadata,
+                                "parent_source_node": row["parent_source_node"],
+                            },
+                        }
+                    )
+        return items
+
+    def rebuild_vector_index(
+        self,
+        *,
+        full: bool = False,
+        include_nodes: bool = True,
+        include_chunks: bool = True,
+    ) -> Dict[str, Any]:
+        """Rebuild the derived vector index without mutating graph content."""
+        op_id = f"vector-op:{_sha256_text(f'{time.time()}:{os.getpid()}')[:24]}"
+        requested_at = _now()
+        started = time.perf_counter()
+        try:
+            with self._connect() as conn:
+                conn.execute(
+                    """
+                        INSERT INTO vector_index_operations(
+                          id, operation, status, requested_at, started_at, metadata_json
+                        )
+                        VALUES (?, ?, 'running', ?, ?, ?)
+                        """,
+                    (
+                        op_id,
+                        "rebuild_full" if full else "rebuild_incremental",
+                        requested_at,
+                        requested_at,
+                        _json(
+                            {
+                                "include_nodes": include_nodes,
+                                "include_chunks": include_chunks,
+                            }
+                        ),
+                    ),
+                )
+                if full:
+                    filters = []
+                    if include_nodes:
+                        filters.append("'node'")
+                    if include_chunks:
+                        filters.append("'chunk'")
+                    if filters:
+                        conn.execute(
+                            f"DELETE FROM vector_embeddings WHERE item_type IN ({','.join(filters)})"
+                        )
+                items = self._iter_vector_source_items(
+                    conn,
+                    include_nodes=include_nodes,
+                    include_chunks=include_chunks,
+                )
+                indexed = skipped = 0
+                for item in items:
+                    changed = self._upsert_vector_item(conn, **item)
+                    if changed:
+                        indexed += 1
+                    else:
+                        skipped += 1
+                duration_ms = round((time.perf_counter() - started) * 1000, 2)
+                conn.execute(
+                    """
+                        UPDATE vector_index_operations
+                        SET status='completed', completed_at=?, items_total=?,
+                            items_indexed=?, items_skipped=?, metadata_json=?
+                        WHERE id=?
+                        """,
+                    (
+                        _now(),
+                        len(items),
+                        indexed,
+                        skipped,
+                        _json(
+                            {
+                                "include_nodes": include_nodes,
+                                "include_chunks": include_chunks,
+                                "duration_ms": duration_ms,
+                                "embedding_model": self._embedding_model.model_id,
+                                "embedding_dim": self._embedding_model.dim,
+                            }
+                        ),
+                        op_id,
+                    ),
+                )
+            return {
+                "status": "completed",
+                "operation_id": op_id,
+                "full": bool(full),
+                "items_total": len(items),
+                "items_indexed": indexed,
+                "items_skipped": skipped,
+                "duration_ms": duration_ms,
+                "embedding_model": self._embedding_model.model_id,
+                "embedding_dim": self._embedding_model.dim,
+            }
+        except Exception as exc:
+            duration_ms = round((time.perf_counter() - started) * 1000, 2)
+            with self._connect() as conn:
+                conn.execute(
+                    """
+                        INSERT INTO vector_index_operations(
+                          id, operation, status, requested_at, started_at, completed_at,
+                          error_message, metadata_json
+                        )
+                        VALUES (?, ?, 'failed', ?, ?, ?, ?, ?)
+                        ON CONFLICT(id) DO UPDATE SET
+                          status='failed',
+                          completed_at=excluded.completed_at,
+                          error_message=excluded.error_message,
+                          metadata_json=excluded.metadata_json
+                        """,
+                    (
+                        op_id,
+                        "rebuild_full" if full else "rebuild_incremental",
+                        requested_at,
+                        requested_at,
+                        _now(),
+                        str(exc),
+                        _json({"duration_ms": duration_ms}),
+                    ),
+                )
+            raise
+
+    def index_status(self) -> Dict[str, Any]:
+        storage_capabilities = None
+        try:
+            storage_capabilities = self.storage_engine.capabilities().as_dict()
+        except Exception as exc:
+            storage_capabilities = {
+                "engine": "sqlite",
+                "available": False,
+                "reason": str(exc),
+            }
+        with self._connect() as conn:
+            vector_counts = {
+                row["item_type"]: row["count"]
+                for row in conn.execute(
+                    "SELECT item_type, COUNT(*) AS count FROM vector_embeddings GROUP BY item_type"
+                )
+            }
+            source_items = self._iter_vector_source_items(conn)
+            vector_rows = {
+                row["item_id"]: row
+                for row in conn.execute(
+                    """
+                        SELECT item_id, text_hash, embedding_dim, embedding_model, indexed_at
+                        FROM vector_embeddings
+                        """
+                ).fetchall()
+            }
+            latest_rows = conn.execute(
+                """
+                    SELECT id, operation, status, requested_at, started_at, completed_at,
+                           items_total, items_indexed, items_skipped, error_message, metadata_json
+                    FROM vector_index_operations
+                    ORDER BY requested_at DESC, id DESC
+                    LIMIT 5
+                    """
+            ).fetchall()
+        missing = stale = ready = 0
+        source_item_ids = {str(item["item_id"]) for item in source_items}
+        backlog_by_type: Dict[str, int] = {}
+        backlog_reasons: Dict[str, int] = {}
+        backlog_samples: List[Dict[str, Any]] = []
+
+        def add_backlog(item: Dict[str, Any], reason: str) -> None:
+            item_type = str(item.get("item_type") or "unknown")
+            backlog_by_type[item_type] = backlog_by_type.get(item_type, 0) + 1
+            backlog_reasons[reason] = backlog_reasons.get(reason, 0) + 1
+            if len(backlog_samples) >= 20:
+                return
+            backlog_samples.append(
+                {
+                    "item_id": item.get("item_id"),
+                    "item_type": item_type,
+                    "source_node": item.get("source_node"),
+                    "reason": reason,
+                    "metadata": {
+                        key: value
+                        for key, value in dict(item.get("metadata") or {}).items()
+                        if key in {"node_type", "source", "conversation_id", "parent_source_node"}
+                    },
+                }
+            )
+
+        for item in source_items:
+            vector_row = vector_rows.get(item["item_id"])
+            expected_hash = _sha256_text(_clean_text(item["text"]))
+            if not vector_row:
+                missing += 1
+                add_backlog(item, "missing_vector")
+            elif (
+                vector_row["text_hash"] != expected_hash
+                or vector_row["embedding_dim"] != self._embedding_model.dim
+                or vector_row["embedding_model"] != self._embedding_model.model_id
+            ):
+                stale += 1
+                reason = "text_changed"
+                if vector_row["embedding_model"] != self._embedding_model.model_id:
+                    reason = "model_changed"
+                elif vector_row["embedding_dim"] != self._embedding_model.dim:
+                    reason = "dimension_changed"
+                add_backlog(item, reason)
+            else:
+                ready += 1
+        pending = missing + stale
+        orphaned_items = max(0, len(set(vector_rows) - source_item_ids))
+        coverage_ratio = round(ready / len(source_items), 6) if source_items else 1.0
+        latest_completed = None
+        for row in latest_rows:
+            if row["status"] == "completed":
+                latest_completed = row
+                break
+        latency_budget: Dict[str, Any] = {
+            "target_rebuild_ms": 10_000,
+            "last_rebuild_duration_ms": None,
+            "last_items_per_second": None,
+            "within_target": None,
+        }
+        if latest_completed is not None:
+            metadata = _safe_loads(latest_completed["metadata_json"])
+            duration_ms = metadata.get("duration_ms")
+            items_total = int(latest_completed["items_total"] or 0)
+            if isinstance(duration_ms, (int, float)) and duration_ms > 0:
+                latency_budget.update(
+                    {
+                        "last_rebuild_duration_ms": round(float(duration_ms), 2),
+                        "last_items_per_second": round(items_total / (float(duration_ms) / 1000.0), 2),
+                        "within_target": float(duration_ms) <= 10_000,
+                    }
+                )
+        return {
+            "status": "ready" if pending == 0 else "needs_reindex",
+            "storage": {
+                "db_path": str(self.db_path),
+                "backend": "sqlite",
+                "embedding_model": self._embedding_model.model_id,
+                "embedding_dim": self._embedding_model.dim,
+                # Honest capability report: trigram FTS5 keyword index, or
+                # LIKE-scan fallback when this SQLite build lacks it.
+                "fts_enabled": bool(getattr(self, "_fts_enabled", False)),
+                "engine": storage_capabilities,
+                "vector_search_backend": (
+                    storage_capabilities.get("vector_backend")
+                    if isinstance(storage_capabilities, dict)
+                    else "bruteforce-cosine"
+                ),
+                "vector_search_mode": (
+                    (storage_capabilities.get("metadata") or {}).get("vector_mode")
+                    if isinstance(storage_capabilities, dict)
+                    else "fallback"
+                ),
+                "sqlite_vec_ann_available": (
+                    bool((storage_capabilities.get("metadata") or {}).get("sqlite_vec_ann_available"))
+                    if isinstance(storage_capabilities, dict)
+                    else False
+                ),
+            },
+            "source_items": len(source_items),
+            "indexed_items": sum(vector_counts.values()),
+            "ready_items": ready,
+            "missing_items": missing,
+            "stale_items": stale,
+            "pending_items": pending,
+            "by_item_type": vector_counts,
+            "scale": {
+                "version": 1,
+                "coverage_ratio": coverage_ratio,
+                "coverage_percent": round(coverage_ratio * 100.0, 2),
+                "source_items": len(source_items),
+                "ready_items": ready,
+                "pending_items": pending,
+                "missing_items": missing,
+                "stale_items": stale,
+                "orphaned_items": orphaned_items,
+                "backlog_by_item_type": backlog_by_type,
+                "backlog_reasons": backlog_reasons,
+                "backlog_samples": backlog_samples,
+                "incremental_reindex_recommended": pending > 0,
+                "full_rebuild_recommended": orphaned_items > 0,
+                "latency_budget": latency_budget,
+            },
+            "operations": [
+                {
+                    "id": row["id"],
+                    "operation": row["operation"],
+                    "status": row["status"],
+                    "requested_at": row["requested_at"],
+                    "started_at": row["started_at"],
+                    "completed_at": row["completed_at"],
+                    "items_total": row["items_total"],
+                    "items_indexed": row["items_indexed"],
+                    "items_skipped": row["items_skipped"],
+                    "error_message": row["error_message"],
+                    "metadata": _safe_loads(row["metadata_json"]),
+                }
+                for row in latest_rows
+            ],
+        }
+
+    def vector_search(
+        self,
+        query: str,
+        *,
+        limit: int = 30,
+        min_score: float = 0.0,
+        max_candidates: int = 10_000,
+    ) -> Dict[str, Any]:
+        query = str(query or "").strip()
+        limit = max(1, min(int(limit or 30), 100))
+        min_score = float(min_score or 0.0)
+        if not query:
+            return {"query": query, "matches": []}
+        query_vector = self._embedding_model.embed(query)
+        max_candidates = max(limit, min(int(max_candidates or 10_000), 50_000))
+        with self._connect() as conn:
+            rows = conn.execute(
+                """
+                    SELECT
+                      ve.item_id, ve.item_type, ve.source_node, ve.embedding,
+                      ve.embedding_dim, ve.embedding_model, ve.metadata_json AS vector_metadata,
+                      n.type AS node_type, n.title AS node_title, n.summary AS node_summary,
+                      n.metadata_json AS node_metadata, n.updated_at AS node_updated_at,
+                      c.text AS chunk_text, c.source_node AS parent_node_id,
+                      pn.type AS parent_type, pn.title AS parent_title,
+                      pn.summary AS parent_summary, pn.metadata_json AS parent_metadata,
+                      pn.updated_at AS parent_updated_at
+                    FROM vector_embeddings ve
+                    LEFT JOIN nodes n ON n.id=ve.source_node
+                    LEFT JOIN chunks c ON c.id=ve.item_id
+                    LEFT JOIN nodes pn ON pn.id=c.source_node
+                    WHERE ve.embedding_model=? AND ve.embedding_dim=?
+                    ORDER BY ve.indexed_at DESC
+                    LIMIT ?
+                    """,
+                (
+                    self._embedding_model.model_id,
+                    self._embedding_model.dim,
+                    max_candidates,
+                ),
+            ).fetchall()
+        scored = []
+        for row in rows:
+            vector = self._embedding_model.decode(
+                row["embedding"], row["embedding_dim"]
+            )
+            score = self._embedding_model.similarity(query_vector, vector)
+            if score < min_score:
+                continue
+            is_chunk = row["item_type"] == "chunk"
+            summary = (
+                row["chunk_text"]
+                if is_chunk and row["chunk_text"]
+                else row["node_summary"]
+            )
+            parent_metadata = _safe_loads(row["parent_metadata"])
+            node_metadata = _safe_loads(row["node_metadata"])
+            scored.append(
+                {
+                    "id": row["item_id"],
+                    "node_id": row["parent_node_id"]
+                    if is_chunk and row["parent_node_id"]
+                    else row["source_node"],
+                    "item_type": row["item_type"],
+                    "type": "Chunk" if is_chunk else row["node_type"],
+                    "title": row["parent_title"]
+                    if is_chunk and row["parent_title"]
+                    else row["node_title"],
+                    "summary": _clean_text(summary or "")[:1000],
+                    "score": round(float(score), 6),
+                    "metadata": {
+                        **(parent_metadata if is_chunk else node_metadata),
+                        "vector": _safe_loads(row["vector_metadata"]),
+                        "parent_node_id": row["parent_node_id"],
+                        "parent_type": row["parent_type"],
+                    },
+                    "updated_at": row["parent_updated_at"]
+                    if is_chunk and row["parent_updated_at"]
+                    else row["node_updated_at"],
+                }
+            )
+        scored.sort(
+            key=lambda item: (item["score"], item.get("updated_at") or ""), reverse=True
+        )
+        return {
+            "query": query,
+            "embedding_model": self._embedding_model.model_id,
+            "embedding_dim": self._embedding_model.dim,
+            "matches": scored[:limit],
+        }