ltcai 8.9.0 → 9.0.0

package/lattice_brain/graph/retrieval.py

@@ -673,378 +673,6 @@ class KnowledgeGraphRetrievalMixin:
             edges = [edge for edge in edges if edge.get("from") in kept and edge.get("to") in kept]
         return {"root": node_id, "depth": depth, "nodes": nodes, "edges": edges}
 
-    def _iter_vector_source_items(
-        self,
-        conn: sqlite3.Connection,
-        *,
-        include_nodes: bool = True,
-        include_chunks: bool = True,
-    ) -> List[Dict[str, Any]]:
-        items: List[Dict[str, Any]] = []
-        if include_nodes:
-            for row in conn.execute(
-                """
-                    SELECT id, type, title, summary, metadata_json
-                    FROM nodes
-                    WHERE type <> 'Chunk'
-                    ORDER BY updated_at DESC, id ASC
-                    """
-            ).fetchall():
-                metadata = _safe_loads(row["metadata_json"])
-                text = self._vector_text_for_node(
-                    title=row["title"],
-                    summary=row["summary"] or "",
-                    metadata=metadata,
-                )
-                if text:
-                    items.append(
-                        {
-                            "item_id": row["id"],
-                            "item_type": "node",
-                            "source_node": row["id"],
-                            "text": text,
-                            "metadata": {"node_type": row["type"], **metadata},
-                        }
-                    )
-        if include_chunks:
-            for row in conn.execute(
-                """
-                    SELECT c.id, c.source_node AS parent_source_node, c.text, c.metadata_json
-                    FROM chunks c
-                    JOIN nodes n ON n.id=c.id
-                    ORDER BY c.created_at DESC, c.id ASC
-                    """
-            ).fetchall():
-                metadata = _safe_loads(row["metadata_json"])
-                text = _clean_text(row["text"] or "")
-                if text:
-                    items.append(
-                        {
-                            "item_id": row["id"],
-                            "item_type": "chunk",
-                            "source_node": row["id"],
-                            "text": text,
-                            "metadata": {
-                                **metadata,
-                                "parent_source_node": row["parent_source_node"],
-                            },
-                        }
-                    )
-        return items
-
-    def rebuild_vector_index(
-        self,
-        *,
-        full: bool = False,
-        include_nodes: bool = True,
-        include_chunks: bool = True,
-    ) -> Dict[str, Any]:
-        """Rebuild the derived vector index without mutating graph content."""
-        op_id = f"vector-op:{_sha256_text(f'{time.time()}:{os.getpid()}')[:24]}"
-        requested_at = _now()
-        started = time.perf_counter()
-        try:
-            with self._connect() as conn:
-                conn.execute(
-                    """
-                        INSERT INTO vector_index_operations(
-                          id, operation, status, requested_at, started_at, metadata_json
-                        )
-                        VALUES (?, ?, 'running', ?, ?, ?)
-                        """,
-                    (
-                        op_id,
-                        "rebuild_full" if full else "rebuild_incremental",
-                        requested_at,
-                        requested_at,
-                        _json(
-                            {
-                                "include_nodes": include_nodes,
-                                "include_chunks": include_chunks,
-                            }
-                        ),
-                    ),
-                )
-                if full:
-                    filters = []
-                    if include_nodes:
-                        filters.append("'node'")
-                    if include_chunks:
-                        filters.append("'chunk'")
-                    if filters:
-                        conn.execute(
-                            f"DELETE FROM vector_embeddings WHERE item_type IN ({','.join(filters)})"
-                        )
-                items = self._iter_vector_source_items(
-                    conn,
-                    include_nodes=include_nodes,
-                    include_chunks=include_chunks,
-                )
-                indexed = skipped = 0
-                for item in items:
-                    changed = self._upsert_vector_item(conn, **item)
-                    if changed:
-                        indexed += 1
-                    else:
-                        skipped += 1
-                duration_ms = round((time.perf_counter() - started) * 1000, 2)
-                conn.execute(
-                    """
-                        UPDATE vector_index_operations
-                        SET status='completed', completed_at=?, items_total=?,
-                            items_indexed=?, items_skipped=?, metadata_json=?
-                        WHERE id=?
-                        """,
-                    (
-                        _now(),
-                        len(items),
-                        indexed,
-                        skipped,
-                        _json(
-                            {
-                                "include_nodes": include_nodes,
-                                "include_chunks": include_chunks,
-                                "duration_ms": duration_ms,
-                                "embedding_model": self._embedding_model.model_id,
-                                "embedding_dim": self._embedding_model.dim,
-                            }
-                        ),
-                        op_id,
-                    ),
-                )
-            return {
-                "status": "completed",
-                "operation_id": op_id,
-                "full": bool(full),
-                "items_total": len(items),
-                "items_indexed": indexed,
-                "items_skipped": skipped,
-                "duration_ms": duration_ms,
-                "embedding_model": self._embedding_model.model_id,
-                "embedding_dim": self._embedding_model.dim,
-            }
-        except Exception as exc:
-            duration_ms = round((time.perf_counter() - started) * 1000, 2)
-            with self._connect() as conn:
-                conn.execute(
-                    """
-                        INSERT INTO vector_index_operations(
-                          id, operation, status, requested_at, started_at, completed_at,
-                          error_message, metadata_json
-                        )
-                        VALUES (?, ?, 'failed', ?, ?, ?, ?, ?)
-                        ON CONFLICT(id) DO UPDATE SET
-                          status='failed',
-                          completed_at=excluded.completed_at,
-                          error_message=excluded.error_message,
-                          metadata_json=excluded.metadata_json
-                        """,
-                    (
-                        op_id,
-                        "rebuild_full" if full else "rebuild_incremental",
-                        requested_at,
-                        requested_at,
-                        _now(),
-                        str(exc),
-                        _json({"duration_ms": duration_ms}),
-                    ),
-                )
-            raise
-
-    def index_status(self) -> Dict[str, Any]:
-        storage_capabilities = None
-        try:
-            storage_capabilities = self.storage_engine.capabilities().as_dict()
-        except Exception as exc:
-            storage_capabilities = {
-                "engine": "sqlite",
-                "available": False,
-                "reason": str(exc),
-            }
-        with self._connect() as conn:
-            vector_counts = {
-                row["item_type"]: row["count"]
-                for row in conn.execute(
-                    "SELECT item_type, COUNT(*) AS count FROM vector_embeddings GROUP BY item_type"
-                )
-            }
-            source_items = self._iter_vector_source_items(conn)
-            vector_rows = {
-                row["item_id"]: row
-                for row in conn.execute(
-                    """
-                        SELECT item_id, text_hash, embedding_dim, embedding_model, indexed_at
-                        FROM vector_embeddings
-                        """
-                ).fetchall()
-            }
-            latest_rows = conn.execute(
-                """
-                    SELECT id, operation, status, requested_at, started_at, completed_at,
-                           items_total, items_indexed, items_skipped, error_message, metadata_json
-                    FROM vector_index_operations
-                    ORDER BY requested_at DESC, id DESC
-                    LIMIT 5
-                    """
-            ).fetchall()
-        missing = stale = ready = 0
-        for item in source_items:
-            vector_row = vector_rows.get(item["item_id"])
-            expected_hash = _sha256_text(_clean_text(item["text"]))
-            if not vector_row:
-                missing += 1
-            elif (
-                vector_row["text_hash"] != expected_hash
-                or vector_row["embedding_dim"] != self._embedding_model.dim
-                or vector_row["embedding_model"] != self._embedding_model.model_id
-            ):
-                stale += 1
-            else:
-                ready += 1
-        pending = missing + stale
-        return {
-            "status": "ready" if pending == 0 else "needs_reindex",
-            "storage": {
-                "db_path": str(self.db_path),
-                "backend": "sqlite",
-                "embedding_model": self._embedding_model.model_id,
-                "embedding_dim": self._embedding_model.dim,
-                # Honest capability report: trigram FTS5 keyword index, or
-                # LIKE-scan fallback when this SQLite build lacks it.
-                "fts_enabled": bool(getattr(self, "_fts_enabled", False)),
-                "engine": storage_capabilities,
-                "vector_search_backend": (
-                    storage_capabilities.get("vector_backend")
-                    if isinstance(storage_capabilities, dict)
-                    else "bruteforce-cosine"
-                ),
-                "vector_search_mode": (
-                    (storage_capabilities.get("metadata") or {}).get("vector_mode")
-                    if isinstance(storage_capabilities, dict)
-                    else "fallback"
-                ),
-                "sqlite_vec_ann_available": (
-                    bool((storage_capabilities.get("metadata") or {}).get("sqlite_vec_ann_available"))
-                    if isinstance(storage_capabilities, dict)
-                    else False
-                ),
-            },
-            "source_items": len(source_items),
-            "indexed_items": sum(vector_counts.values()),
-            "ready_items": ready,
-            "missing_items": missing,
-            "stale_items": stale,
-            "pending_items": pending,
-            "by_item_type": vector_counts,
-            "operations": [
-                {
-                    "id": row["id"],
-                    "operation": row["operation"],
-                    "status": row["status"],
-                    "requested_at": row["requested_at"],
-                    "started_at": row["started_at"],
-                    "completed_at": row["completed_at"],
-                    "items_total": row["items_total"],
-                    "items_indexed": row["items_indexed"],
-                    "items_skipped": row["items_skipped"],
-                    "error_message": row["error_message"],
-                    "metadata": _safe_loads(row["metadata_json"]),
-                }
-                for row in latest_rows
-            ],
-        }
-
-    def vector_search(
-        self,
-        query: str,
-        *,
-        limit: int = 30,
-        min_score: float = 0.0,
-        max_candidates: int = 10_000,
-    ) -> Dict[str, Any]:
-        query = str(query or "").strip()
-        limit = max(1, min(int(limit or 30), 100))
-        min_score = float(min_score or 0.0)
-        if not query:
-            return {"query": query, "matches": []}
-        query_vector = self._embedding_model.embed(query)
-        max_candidates = max(limit, min(int(max_candidates or 10_000), 50_000))
-        with self._connect() as conn:
-            rows = conn.execute(
-                """
-                    SELECT
-                      ve.item_id, ve.item_type, ve.source_node, ve.embedding,
-                      ve.embedding_dim, ve.embedding_model, ve.metadata_json AS vector_metadata,
-                      n.type AS node_type, n.title AS node_title, n.summary AS node_summary,
-                      n.metadata_json AS node_metadata, n.updated_at AS node_updated_at,
-                      c.text AS chunk_text, c.source_node AS parent_node_id,
-                      pn.type AS parent_type, pn.title AS parent_title,
-                      pn.summary AS parent_summary, pn.metadata_json AS parent_metadata,
-                      pn.updated_at AS parent_updated_at
-                    FROM vector_embeddings ve
-                    LEFT JOIN nodes n ON n.id=ve.source_node
-                    LEFT JOIN chunks c ON c.id=ve.item_id
-                    LEFT JOIN nodes pn ON pn.id=c.source_node
-                    WHERE ve.embedding_model=? AND ve.embedding_dim=?
-                    ORDER BY ve.indexed_at DESC
-                    LIMIT ?
-                    """,
-                (
-                    self._embedding_model.model_id,
-                    self._embedding_model.dim,
-                    max_candidates,
-                ),
-            ).fetchall()
-        scored = []
-        for row in rows:
-            vector = self._embedding_model.decode(
-                row["embedding"], row["embedding_dim"]
-            )
-            score = self._embedding_model.similarity(query_vector, vector)
-            if score < min_score:
-                continue
-            is_chunk = row["item_type"] == "chunk"
-            summary = (
-                row["chunk_text"]
-                if is_chunk and row["chunk_text"]
-                else row["node_summary"]
-            )
-            parent_metadata = _safe_loads(row["parent_metadata"])
-            node_metadata = _safe_loads(row["node_metadata"])
-            scored.append(
-                {
-                    "id": row["item_id"],
-                    "node_id": row["parent_node_id"]
-                    if is_chunk and row["parent_node_id"]
-                    else row["source_node"],
-                    "item_type": row["item_type"],
-                    "type": "Chunk" if is_chunk else row["node_type"],
-                    "title": row["parent_title"]
-                    if is_chunk and row["parent_title"]
-                    else row["node_title"],
-                    "summary": _clean_text(summary or "")[:1000],
-                    "score": round(float(score), 6),
-                    "metadata": {
-                        **(parent_metadata if is_chunk else node_metadata),
-                        "vector": _safe_loads(row["vector_metadata"]),
-                        "parent_node_id": row["parent_node_id"],
-                        "parent_type": row["parent_type"],
-                    },
-                    "updated_at": row["parent_updated_at"]
-                    if is_chunk and row["parent_updated_at"]
-                    else row["node_updated_at"],
-                }
-            )
-        scored.sort(
-            key=lambda item: (item["score"], item.get("updated_at") or ""), reverse=True
-        )
-        return {
-            "query": query,
-            "embedding_model": self._embedding_model.model_id,
-            "embedding_dim": self._embedding_model.dim,
-            "matches": scored[:limit],
-        }
-
     def delete_conversation(self, conversation_id: str) -> Dict[str, Any]:
         conversation_id = str(conversation_id or "").strip()
         if not conversation_id:
@@ -1180,201 +808,3 @@ class KnowledgeGraphRetrievalMixin:
             "local_file_status": local_file_status,
             "v2": v2,
         }
-
-    def search_for_document_generation(
-        self, query: str, limit: int = 10
-    ) -> List[Dict[str, Any]]:
-        """Hybrid retrieval optimized for document generation.
-
-        Scoring: 0.5*text_relevance + 0.3*graph_relationship + 0.2*recency
-        Returns nodes with rich context for document generation prompts.
-        """
-        query = str(query or "").strip()
-        if not query:
-            return []
-        limit = max(1, min(int(limit or 10), 50))
-        terms = _topic_candidates(query, limit=12)
-        now = datetime.now()
-        nt, et = self._read_tables()
-
-        with self._connect() as conn:
-            candidate_rows = []
-            seen_ids = set()
-
-            if query:
-                q = f"%{query}%"
-                rows = conn.execute(
-                    f"""
-                        SELECT id, type, title, summary, metadata_json, updated_at
-                        FROM {nt}
-                        WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
-                          AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
-                                       'Spreadsheet', 'Image', 'ImageText', 'Chat',
-                                       'Decision', 'Task', 'Concept', 'Feature',
-                                       'Page', 'Slide')
-                        ORDER BY updated_at DESC, id ASC
-                        LIMIT ?
-                        """,
-                    (q, q, q, limit * 5),
-                ).fetchall()
-                for row in rows:
-                    if row["id"] not in seen_ids:
-                        seen_ids.add(row["id"])
-                        candidate_rows.append(row)
-
-            for term in terms:
-                t = f"%{term}%"
-                rows = conn.execute(
-                    f"""
-                        SELECT id, type, title, summary, metadata_json, updated_at
-                        FROM {nt}
-                        WHERE (title LIKE ? OR summary LIKE ? OR metadata_json LIKE ?)
-                          AND type IN ('Document', 'File', 'CodeFile', 'SlideDeck',
-                                       'Spreadsheet', 'Image', 'ImageText', 'Chat',
-                                       'Decision', 'Task', 'Concept', 'Feature',
-                                       'Page', 'Slide')
-                        ORDER BY updated_at DESC, id ASC
-                        LIMIT ?
-                        """,
-                    (t, t, t, limit * 3),
-                ).fetchall()
-                for row in rows:
-                    if row["id"] not in seen_ids:
-                        seen_ids.add(row["id"])
-                        candidate_rows.append(row)
-
-            scored_results = []
-            for row in candidate_rows:
-                haystack = (
-                    f"{row['title']} {row['summary']} {row['metadata_json']}".lower()
-                )
-
-                text_hits = sum(1 for term in terms if term.lower() in haystack)
-                text_score = min(1.0, text_hits / max(len(terms), 1))
-
-                edge_count = conn.execute(
-                    f"SELECT COUNT(*) AS c FROM {et} WHERE from_node=? OR to_node=?",
-                    (row["id"], row["id"]),
-                ).fetchone()["c"]
-                graph_score = min(1.0, math.log1p(edge_count) / 4.0)
-
-                recency = _recency_score(
-                    row["updated_at"], now=now, half_life_days=14.0
-                )
-
-                doc_type_boost = (
-                    1.2
-                    if row["type"]
-                    in (
-                        "Document",
-                        "File",
-                        "SlideDeck",
-                        "Decision",
-                    )
-                    else 1.0
-                )
-
-                hybrid_score = (
-                    0.5 * text_score + 0.3 * graph_score + 0.2 * recency
-                ) * doc_type_boost
-
-                meta = _safe_loads(row["metadata_json"])
-                neighbor_concepts = []
-                neighbor_rows = conn.execute(
-                    f"""
-                        SELECT n.title, n.type FROM {et} e
-                        JOIN {nt} n ON n.id = CASE WHEN e.from_node = ? THEN e.to_node ELSE e.from_node END
-                        WHERE (e.from_node = ? OR e.to_node = ?)
-                          AND n.type IN ('Concept', 'Feature', 'Decision', 'Task')
-                        LIMIT 8
-                        """,
-                    (row["id"], row["id"], row["id"]),
-                ).fetchall()
-                for nr in neighbor_rows:
-                    neighbor_concepts.append({"title": nr["title"], "type": nr["type"]})
-
-                scored_results.append(
-                    {
-                        "id": row["id"],
-                        "type": row["type"],
-                        "title": row["title"],
-                        "summary": row["summary"],
-                        "metadata": meta,
-                        "updated_at": row["updated_at"],
-                        "hybrid_score": round(hybrid_score, 4),
-                        "scores": {
-                            "text": round(text_score, 4),
-                            "graph": round(graph_score, 4),
-                            "recency": round(recency, 4),
-                        },
-                        "related_concepts": neighbor_concepts,
-                    }
-                )
-
-            scored_results.sort(key=lambda x: x["hybrid_score"], reverse=True)
-            return scored_results[:limit]
-
-    def multi_hop_context(
-        self, node_ids: List[str], max_hops: int = 2
-    ) -> Dict[str, Any]:
-        """Multi-hop graph traversal from seed nodes for richer context."""
-        visited_nodes = set()
-        visited_edges = set()
-        all_nodes = []
-        all_edges = []
-        frontier = set(node_ids)
-        nt, et = self._read_tables()
-
-        with self._connect() as conn:
-            for hop in range(max_hops):
-                if not frontier:
-                    break
-                next_frontier = set()
-                for nid in frontier:
-                    if nid in visited_nodes:
-                        continue
-                    visited_nodes.add(nid)
-                    row = conn.execute(
-                        f"SELECT id, type, title, summary, metadata_json, updated_at FROM {nt} WHERE id=?",
-                        (nid,),
-                    ).fetchone()
-                    if row:
-                        all_nodes.append(
-                            {
-                                "id": row["id"],
-                                "type": row["type"],
-                                "title": row["title"],
-                                "summary": row["summary"],
-                                "metadata": _safe_loads(row["metadata_json"]),
-                                "hop": hop,
-                            }
-                        )
-                    edge_rows = conn.execute(
-                        f"""
-                            SELECT id, from_node, to_node, type, weight
-                            FROM {et} WHERE from_node=? OR to_node=?
-                            ORDER BY id ASC
-                            """,
-                        (nid, nid),
-                    ).fetchall()
-                    for er in edge_rows:
-                        if er["id"] not in visited_edges:
-                            visited_edges.add(er["id"])
-                            all_edges.append(
-                                {
-                                    "from": er["from_node"],
-                                    "to": er["to_node"],
-                                    "type": er["type"],
-                                    "weight": er["weight"],
-                                }
-                            )
-                            other = (
-                                er["to_node"]
-                                if er["from_node"] == nid
-                                else er["from_node"]
-                            )
-                            if other not in visited_nodes:
-                                next_frontier.add(other)
-                frontier = next_frontier
-
-        return {"nodes": all_nodes, "edges": all_edges}