@techwavedev/agi-agent-kit 1.3.5 → 1.3.6

package/templates/skills/core/qdrant-memory/scripts/hybrid_search.py

@@ -1,18 +1,39 @@
 #!/usr/bin/env python3
 """
 Script: hybrid_search.py
-Purpose: Hybrid search combining vector similarity with keyword filtering.
+Purpose: True hybrid search combining BM25 keyword matching with vector similarity.
+
+This is the core of the hybrid memory system. It runs BOTH:
+1. Vector search via Qdrant (semantic similarity)
+2. BM25 search via SQLite FTS5 (exact keyword matching)
+
+Results are merged using weighted scoring:
+    finalScore = vectorWeight * vectorScore + textWeight * textScore
+
+This catches things pure vector search misses:
+- Exact error codes: ImagePullBackOff, ECONNREFUSED
+- Environment variables: QDRANT_URL, OPENAI_API_KEY
+- IDs and hashes: sg-018f20ea63e82eeb5, a828e60
+- Code symbols: memory_retrieval.retrieve_context
 
 Supports both Ollama (local/private) and OpenAI (cloud) embeddings.
 Default: Ollama with nomic-embed-text model.
 
 Usage:
-    python3 hybrid_search.py --query "kubernetes deployment failed" \\
-        --keyword error_code=ImagePullBackOff --keyword namespace=production
+    # True hybrid search (BM25 + vector)
+    python3 hybrid_search.py --query "ImagePullBackOff error" --mode hybrid
+
+    # Vector-only (original behavior)
+    python3 hybrid_search.py --query "deployment strategies" --mode vector
+
+    # BM25-only (keyword matching)
+    python3 hybrid_search.py --query "sg-018f20ea63e82eeb5" --mode keyword
+
+    # With metadata filters (Qdrant payload filtering)
+    python3 hybrid_search.py --query "kubernetes error" --keyword project=k8s
 
-This is particularly useful for technical queries where you need:
-- Semantic understanding (what is the user asking about?)
-- Exact matching (specific error codes, variable names, identifiers)
+    # Custom weights
+    python3 hybrid_search.py --query "error" --vector-weight 0.6 --text-weight 0.4
 
 Environment Variables:
     EMBEDDING_PROVIDER  - "ollama" (default) or "openai"
@@ -20,6 +41,9 @@ Environment Variables:
     OPENAI_API_KEY      - Required for OpenAI provider
     QDRANT_URL          - Qdrant server URL (default: http://localhost:6333)
     MEMORY_COLLECTION   - Collection name (default: agent_memory)
+    BM25_INDEX_PATH     - SQLite FTS5 index path (default: ~/.agi/memory_bm25.sqlite)
+    HYBRID_VECTOR_WEIGHT - Vector weight (default: 0.7)
+    HYBRID_TEXT_WEIGHT   - Text weight (default: 0.3)
 
 Exit Codes:
     0 - Success (results found)
@@ -44,94 +68,280 @@ except ImportError:
     sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
     from embedding_utils import get_embedding
 
+# Import BM25 index
+try:
+    from bm25_index import BM25Index
+except ImportError:
+    sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+    try:
+        from bm25_index import BM25Index
+    except ImportError:
+        BM25Index = None
+
 # Configuration
 QDRANT_URL = os.environ.get("QDRANT_URL", "http://localhost:6333")
 COLLECTION = os.environ.get("MEMORY_COLLECTION", "agent_memory")
 
+# Hybrid search weights (must sum to 1.0)
+DEFAULT_VECTOR_WEIGHT = float(os.environ.get("HYBRID_VECTOR_WEIGHT", "0.7"))
+DEFAULT_TEXT_WEIGHT = float(os.environ.get("HYBRID_TEXT_WEIGHT", "0.3"))
 
-def hybrid_query(
+
+def _vector_search(
     text_query: str,
     keyword_filters: Optional[Dict[str, str]] = None,
     must_not_filters: Optional[Dict[str, str]] = None,
     top_k: int = 10,
-    score_threshold: float = 0.6
-) -> Dict[str, Any]:
+    score_threshold: float = 0.5,
+    candidate_multiplier: int = 4
+) -> List[Dict[str, Any]]:
     """
-    Perform hybrid search combining vector similarity with keyword filtering.
-    
+    Vector similarity search via Qdrant.
+
     Args:
         text_query: Natural language query for semantic search
-        keyword_filters: Dict of field:value pairs that MUST match
+        keyword_filters: Dict of field:value pairs that MUST match (payload filter)
         must_not_filters: Dict of field:value pairs that MUST NOT match
-        top_k: Number of results
+        top_k: Number of results (multiplied by candidate_multiplier for merging)
         score_threshold: Minimum similarity score
-    
+        candidate_multiplier: Fetch extra candidates for better merge
+
     Returns:
-        Search results with hybrid scores
+        List of results with doc_id and vector_score
     """
-    # Get embedding for semantic search
     embedding = get_embedding(text_query)
-    
+
     # Build filter conditions
     filter_conditions = {"must": [], "must_not": []}
-    
+
     if keyword_filters:
         for field, value in keyword_filters.items():
             filter_conditions["must"].append({
                 "key": field,
                 "match": {"value": value}
             })
-    
+
     if must_not_filters:
         for field, value in must_not_filters.items():
             filter_conditions["must_not"].append({
                 "key": field,
                 "match": {"value": value}
             })
-    
-    # Build search payload
+
+    # Fetch more candidates than needed for better merge results
+    fetch_limit = top_k * candidate_multiplier
+
     search_payload = {
         "vector": embedding,
-        "limit": top_k,
+        "limit": fetch_limit,
         "score_threshold": score_threshold,
         "with_payload": True
     }
-    
+
     # Only add filter if we have conditions
     if filter_conditions["must"] or filter_conditions["must_not"]:
         search_payload["filter"] = {
             k: v for k, v in filter_conditions.items() if v
         }
-    
+
     req = Request(
         f"{QDRANT_URL}/collections/{COLLECTION}/points/search",
         data=json.dumps(search_payload).encode(),
         headers={"Content-Type": "application/json"},
         method="POST"
     )
-    
+
     with urlopen(req, timeout=30) as response:
         result = json.loads(response.read().decode())
-    
+
     results = []
     for hit in result.get("result", []):
         results.append({
-            "id": hit["id"],
-            "score": hit["score"],
+            "doc_id": str(hit["id"]),
+            "vector_score": hit["score"],
             "content": hit["payload"].get("content", ""),
             "type": hit["payload"].get("type"),
             "project": hit["payload"].get("project"),
             "tags": hit["payload"].get("tags", []),
             "timestamp": hit["payload"].get("timestamp"),
-            "matched_filters": keyword_filters or {}
         })
-    
+
+    return results
+
+
+def _bm25_search(
+    query: str,
+    top_k: int = 10,
+    candidate_multiplier: int = 4
+) -> List[Dict[str, Any]]:
+    """
+    BM25 keyword search via SQLite FTS5.
+
+    Args:
+        query: Keyword query
+        top_k: Number of results (multiplied for merge)
+        candidate_multiplier: Fetch extra candidates
+
+    Returns:
+        List of results with doc_id and text_score, or empty if BM25 unavailable
+    """
+    if BM25Index is None:
+        return []
+
+    try:
+        with BM25Index() as index:
+            fetch_limit = top_k * candidate_multiplier
+            return index.search(query, top_k=fetch_limit)
+    except Exception:
+        # BM25 index not available — graceful fallback
+        return []
+
+
+def hybrid_query(
+    text_query: str,
+    keyword_filters: Optional[Dict[str, str]] = None,
+    must_not_filters: Optional[Dict[str, str]] = None,
+    top_k: int = 10,
+    score_threshold: float = 0.5,
+    vector_weight: float = DEFAULT_VECTOR_WEIGHT,
+    text_weight: float = DEFAULT_TEXT_WEIGHT,
+    mode: str = "hybrid"
+) -> Dict[str, Any]:
+    """
+    True hybrid search combining vector similarity with BM25 keyword matching.
+
+    Args:
+        text_query: Natural language query
+        keyword_filters: Qdrant payload field filters (MUST match)
+        must_not_filters: Qdrant payload field filters (MUST NOT match)
+        top_k: Number of results to return
+        score_threshold: Minimum vector similarity score
+        vector_weight: Weight for vector scores (0.0-1.0)
+        text_weight: Weight for BM25 text scores (0.0-1.0)
+        mode: Search mode - "hybrid", "vector", or "keyword"
+
+    Returns:
+        Merged search results with hybrid scores
+    """
+    # Normalize weights
+    total_weight = vector_weight + text_weight
+    if total_weight > 0:
+        vector_weight = vector_weight / total_weight
+        text_weight = text_weight / total_weight
+
+    vector_results = []
+    bm25_results = []
+
+    # Run searches based on mode
+    if mode in ("hybrid", "vector"):
+        try:
+            vector_results = _vector_search(
+                text_query,
+                keyword_filters=keyword_filters,
+                must_not_filters=must_not_filters,
+                top_k=top_k,
+                score_threshold=score_threshold
+            )
+        except (URLError, Exception) as e:
+            if mode == "vector":
+                raise
+            # In hybrid mode, continue with BM25 only
+
+    if mode in ("hybrid", "keyword"):
+        bm25_results = _bm25_search(text_query, top_k=top_k)
+
+    # If only one source, return directly
+    if mode == "vector" or (not bm25_results and vector_results):
+        return {
+            "query": text_query,
+            "results": vector_results[:top_k],
+            "total": min(len(vector_results), top_k),
+            "search_type": "vector",
+            "weights": {"vector": 1.0, "text": 0.0}
+        }
+
+    if mode == "keyword" or (not vector_results and bm25_results):
+        formatted = []
+        for r in bm25_results[:top_k]:
+            formatted.append({
+                "doc_id": r["doc_id"],
+                "score": r["text_score"],
+                "content": r["content"],
+                "type": r.get("type"),
+                "project": r.get("project"),
+                "tags": r.get("tags", []),
+                "search_source": "bm25"
+            })
+        return {
+            "query": text_query,
+            "results": formatted,
+            "total": len(formatted),
+            "search_type": "keyword",
+            "weights": {"vector": 0.0, "text": 1.0}
+        }
+
+    # === MERGE: Union by doc_id, compute weighted score ===
+    merged = {}
+
+    # Add vector results
+    for r in vector_results:
+        doc_id = r["doc_id"]
+        merged[doc_id] = {
+            "doc_id": doc_id,
+            "content": r["content"],
+            "type": r.get("type"),
+            "project": r.get("project"),
+            "tags": r.get("tags", []),
+            "timestamp": r.get("timestamp"),
+            "vector_score": r["vector_score"],
+            "text_score": 0.0,
+            "search_sources": ["vector"]
+        }
+
+    # Add/merge BM25 results
+    for r in bm25_results:
+        doc_id = r["doc_id"]
+        if doc_id in merged:
+            merged[doc_id]["text_score"] = r["text_score"]
+            merged[doc_id]["search_sources"].append("bm25")
+        else:
+            merged[doc_id] = {
+                "doc_id": doc_id,
+                "content": r["content"],
+                "type": r.get("type"),
+                "project": r.get("project"),
+                "tags": r.get("tags", []),
+                "timestamp": None,
+                "vector_score": 0.0,
+                "text_score": r["text_score"],
+                "search_sources": ["bm25"]
+            }
+
+    # Compute final hybrid score
+    for doc_id, entry in merged.items():
+        entry["score"] = round(
+            vector_weight * entry["vector_score"] +
+            text_weight * entry["text_score"],
+            4
+        )
+
+    # Sort by hybrid score, take top_k
+    sorted_results = sorted(
+        merged.values(), key=lambda x: x["score"], reverse=True
+    )[:top_k]
+
     return {
         "query": text_query,
         "keyword_filters": keyword_filters,
-        "results": results,
-        "total": len(results),
-        "search_type": "hybrid" if keyword_filters else "semantic"
+        "results": sorted_results,
+        "total": len(sorted_results),
+        "search_type": "hybrid",
+        "weights": {"vector": vector_weight, "text": text_weight},
+        "sources": {
+            "vector_candidates": len(vector_results),
+            "bm25_candidates": len(bm25_results),
+            "merged_unique": len(merged)
+        }
     }
 
 
@@ -147,17 +357,17 @@ def parse_keyword_args(keyword_args: List[str]) -> Dict[str, str]:
 
 def main():
     parser = argparse.ArgumentParser(
-        description="Hybrid search with vector + keyword filtering"
+        description="True hybrid search: BM25 keyword + vector similarity"
     )
     parser.add_argument(
-        "--query", 
+        "--query",
         required=True,
-        help="Natural language query"
+        help="Search query (natural language or keywords)"
     )
     parser.add_argument(
         "--keyword",
         action="append",
-        help="Keyword filter in key=value format (can be repeated)"
+        help="Qdrant payload filter in key=value format (can be repeated)"
     )
     parser.add_argument(
         "--exclude",
@@ -173,27 +383,48 @@ def main():
     parser.add_argument(
         "--threshold",
         type=float,
-        default=0.6,
-        help="Minimum score threshold (default: 0.6)"
+        default=0.5,
+        help="Minimum vector score threshold (default: 0.5)"
+    )
+    parser.add_argument(
+        "--mode",
+        choices=["hybrid", "vector", "keyword"],
+        default="hybrid",
+        help="Search mode (default: hybrid)"
+    )
+    parser.add_argument(
+        "--vector-weight",
+        type=float,
+        default=DEFAULT_VECTOR_WEIGHT,
+        help=f"Vector score weight (default: {DEFAULT_VECTOR_WEIGHT})"
+    )
+    parser.add_argument(
+        "--text-weight",
+        type=float,
+        default=DEFAULT_TEXT_WEIGHT,
+        help=f"BM25 text score weight (default: {DEFAULT_TEXT_WEIGHT})"
     )
-    
+
     args = parser.parse_args()
-    
+
     try:
         keyword_filters = parse_keyword_args(args.keyword)
         exclude_filters = parse_keyword_args(args.exclude)
-        
+
         result = hybrid_query(
             text_query=args.query,
             keyword_filters=keyword_filters if keyword_filters else None,
             must_not_filters=exclude_filters if exclude_filters else None,
             top_k=args.top_k,
-            score_threshold=args.threshold
+            score_threshold=args.threshold,
+            vector_weight=args.vector_weight,
+            text_weight=args.text_weight,
+            mode=args.mode
         )
-        
+
         print(json.dumps(result, indent=2))
         sys.exit(0 if result["total"] > 0 else 1)
-        
+
     except URLError as e:
         print(json.dumps({
             "status": "error",

package/templates/skills/core/qdrant-memory/scripts/memory_retrieval.py

@@ -48,10 +48,22 @@ from urllib.error import URLError, HTTPError
 # Import shared embedding utilities (supports Ollama and OpenAI)
 try:
     from embedding_utils import get_embedding
+    from hybrid_search import hybrid_query
 except ImportError:
     # Fallback if run from different directory
     sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
     from embedding_utils import get_embedding
+    from hybrid_search import hybrid_query
+
+# Import BM25 index for hybrid search (optional — graceful if unavailable)
+try:
+    from bm25_index import BM25Index
+    _BM25_AVAILABLE = True
+except ImportError:
+    _BM25_AVAILABLE = False
+
+# Hybrid search toggle
+HYBRID_SEARCH = os.environ.get("HYBRID_SEARCH", "true").lower() in ("true", "1", "yes")
 
 # Configuration
 QDRANT_URL = os.environ.get("QDRANT_URL", "http://localhost:6333")
@@ -62,75 +74,96 @@ def retrieve_context(
     query: str,
     filters: Optional[Dict[str, Any]] = None,
     top_k: int = 5,
-    score_threshold: float = 0.7
+    score_threshold: float = 0.7,
+    vector_weight: float = None,  # Use default from hybrid_search if None
+    text_weight: float = None     # Use default from hybrid_search if None
 ) -> List[Dict[str, Any]]:
     """
-    Retrieve relevant context from long-term memory.
+    Retrieve relevant context from long-term memory using HYBRID SEARCH.
     
-    Instead of passing 20K tokens of conversation history,
-    this returns only the top-K most relevant chunks (500-1000 tokens).
+    Combines:
+    1. Vector similarity (Qdrant) - for semantic meaning
+    2. BM25 keyword search (SQLite) - for exact matches (IDs, codes, names)
     
     Args:
         query: Natural language query
         filters: Qdrant filter conditions (type, project, tags, etc.)
         top_k: Number of results to return
-        score_threshold: Minimum similarity score
+        score_threshold: Minimum similarity score for vector component
+        vector_weight: Weight for vector search (0.0-1.0)
+        text_weight: Weight for BM25 search (0.0-1.0)
     
     Returns:
         List of relevant memory chunks with metadata
     """
-    embedding = get_embedding(query)
-    
-    search_payload = {
-        "vector": embedding,
-        "limit": top_k,
-        "score_threshold": score_threshold,
-        "with_payload": True
-    }
-    
-    if filters:
-        search_payload["filter"] = filters
-    
-    req = Request(
-        f"{QDRANT_URL}/collections/{COLLECTION}/points/search",
-        data=json.dumps(search_payload).encode(),
-        headers={"Content-Type": "application/json"},
-        method="POST"
-    )
+    # Extract keyword filters from complex Qdrant filter structure if possible
+    # specific structure: {"must": [{"key": "project", "match": {"value": "foo"}}]}
+    keyword_filters = {}
+    if filters and "must" in filters:
+        for condition in filters["must"]:
+            if "key" in condition and "match" in condition and "value" in condition["match"]:
+                keyword_filters[condition["key"]] = condition["match"]["value"]
     
+    # Use the new hybrid_query function
     try:
-        with urlopen(req, timeout=30) as response:
-            result = json.loads(response.read().decode())
+        # Default weights are handled by hybrid_query if passed as None? 
+        # hybrid_query signature has defaults. 
+        # We need to handle arguments carefully.
+        
+        # Checking hybrid_query signature in hybrid_search.py:
+        # def hybrid_query(text_query, keyword_filters, must_not_filters, top_k, score_threshold, vector_weight, text_weight, mode)
+        
+        # We'll use defaults if not provided
+        kwargs = {
+            "text_query": query,
+            "keyword_filters": keyword_filters,
+            "top_k": top_k,
+            "score_threshold": score_threshold,
+            "mode": "hybrid" if HYBRID_SEARCH else "vector"
+        }
+        
+        if vector_weight is not None:
+            kwargs["vector_weight"] = vector_weight
+        if text_weight is not None:
+            kwargs["text_weight"] = text_weight
+            
+        result = hybrid_query(**kwargs)
             
         chunks = []
         total_tokens = 0
         
-        for hit in result.get("result", []):
-            content = hit["payload"].get("content", "")
+        for hit in result.get("results", []):
+            content = hit.get("content", "")
             token_estimate = len(content.split())
             total_tokens += token_estimate
             
             chunks.append({
                 "content": content,
-                "score": hit["score"],
-                "type": hit["payload"].get("type"),
-                "project": hit["payload"].get("project"),
-                "timestamp": hit["payload"].get("timestamp"),
-                "tags": hit["payload"].get("tags", []),
-                "token_estimate": token_estimate
+                "score": hit.get("score", 0),  # Hybrid score
+                "vector_score": hit.get("vector_score", 0),
+                "text_score": hit.get("text_score", 0),
+                "type": hit.get("type"),
+                "project": hit.get("project"),
+                "timestamp": hit.get("timestamp"),
+                "tags": hit.get("tags", []),
+                "token_estimate": token_estimate,
+                "search_sources": hit.get("search_sources", [])
             })
         
         return {
             "chunks": chunks,
             "total_chunks": len(chunks),
             "total_tokens_estimate": total_tokens,
-            "query": query
+            "query": query,
+            "search_type": result.get("search_type", "hybrid")
         }
         
-    except HTTPError as e:
-        if e.code == 404:
-            return {"chunks": [], "total_chunks": 0, "total_tokens_estimate": 0}
-        raise
+    except Exception as e:
+        # Fallback to simple vector search if hybrid fails or during transition
+        # But honestly, hybrid_query handles vector-only fallback too.
+        # This is just a safety catch.
+        print(f"Hybrid search failed, falling back: {e}", file=sys.stderr)
+        return {"chunks": [], "total_chunks": 0, "total_tokens_estimate": 0}
 
 
 def store_memory(
@@ -181,12 +214,31 @@ def store_memory(
     
     with urlopen(req, timeout=30) as response:
         result = json.loads(response.read().decode())
-        
+
+    # Auto-index into BM25 for hybrid search
+    bm25_status = "skipped"
+    if _BM25_AVAILABLE and HYBRID_SEARCH:
+        try:
+            with BM25Index() as bm25:
+                bm25_result = bm25.index_document(
+                    doc_id=point_id,
+                    content=content,
+                    metadata={
+                        "type": memory_type,
+                        "project": (metadata or {}).get("project", ""),
+                        "tags": (metadata or {}).get("tags", [])
+                    }
+                )
+                bm25_status = bm25_result.get("status", "error")
+        except Exception:
+            bm25_status = "error"
+
     return {
         "status": "stored",
         "point_id": point_id,
         "type": memory_type,
         "token_count": payload["token_count"],
+        "bm25_indexed": bm25_status,
         "result": result
     }