claude-self-reflect 3.2.4 → 3.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/claude-self-reflect-test.md +992 -510
- package/.claude/agents/reflection-specialist.md +59 -3
- package/README.md +14 -5
- package/installer/cli.js +16 -0
- package/installer/postinstall.js +14 -0
- package/installer/statusline-setup.js +289 -0
- package/mcp-server/run-mcp.sh +73 -5
- package/mcp-server/src/app_context.py +64 -0
- package/mcp-server/src/config.py +57 -0
- package/mcp-server/src/connection_pool.py +286 -0
- package/mcp-server/src/decay_manager.py +106 -0
- package/mcp-server/src/embedding_manager.py +64 -40
- package/mcp-server/src/embeddings_old.py +141 -0
- package/mcp-server/src/models.py +64 -0
- package/mcp-server/src/parallel_search.py +305 -0
- package/mcp-server/src/project_resolver.py +5 -0
- package/mcp-server/src/reflection_tools.py +211 -0
- package/mcp-server/src/rich_formatting.py +196 -0
- package/mcp-server/src/search_tools.py +874 -0
- package/mcp-server/src/server.py +127 -1720
- package/mcp-server/src/temporal_design.py +132 -0
- package/mcp-server/src/temporal_tools.py +604 -0
- package/mcp-server/src/temporal_utils.py +384 -0
- package/mcp-server/src/utils.py +150 -67
- package/package.json +15 -1
- package/scripts/add-timestamp-indexes.py +134 -0
- package/scripts/ast_grep_final_analyzer.py +325 -0
- package/scripts/ast_grep_unified_registry.py +556 -0
- package/scripts/check-collections.py +29 -0
- package/scripts/csr-status +366 -0
- package/scripts/debug-august-parsing.py +76 -0
- package/scripts/debug-import-single.py +91 -0
- package/scripts/debug-project-resolver.py +82 -0
- package/scripts/debug-temporal-tools.py +135 -0
- package/scripts/delta-metadata-update.py +547 -0
- package/scripts/import-conversations-unified.py +157 -25
- package/scripts/precompact-hook.sh +33 -0
- package/scripts/session_quality_tracker.py +481 -0
- package/scripts/streaming-watcher.py +1578 -0
- package/scripts/update_patterns.py +334 -0
- package/scripts/utils.py +39 -0
package/mcp-server/src/server.py
CHANGED
|
@@ -4,9 +4,8 @@ import os
|
|
|
4
4
|
import asyncio
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
from typing import Any, Optional, List, Dict, Union
|
|
7
|
-
from datetime import datetime, timezone
|
|
7
|
+
from datetime import datetime, timezone, timedelta
|
|
8
8
|
import json
|
|
9
|
-
import numpy as np
|
|
10
9
|
import hashlib
|
|
11
10
|
import time
|
|
12
11
|
import logging
|
|
@@ -27,10 +26,14 @@ except ImportError:
|
|
|
27
26
|
logging.warning("Using legacy utils.normalize_project_name - shared module not found")
|
|
28
27
|
|
|
29
28
|
from .project_resolver import ProjectResolver
|
|
29
|
+
from .temporal_utils import SessionDetector, TemporalParser, WorkSession, group_by_time_period
|
|
30
|
+
from .temporal_tools import register_temporal_tools
|
|
31
|
+
from .search_tools import register_search_tools
|
|
32
|
+
from .reflection_tools import register_reflection_tools
|
|
30
33
|
from pydantic import BaseModel, Field
|
|
31
34
|
from qdrant_client import AsyncQdrantClient, models
|
|
32
35
|
from qdrant_client.models import (
|
|
33
|
-
PointStruct, VectorParams, Distance
|
|
36
|
+
PointStruct, VectorParams, Distance, OrderBy
|
|
34
37
|
)
|
|
35
38
|
|
|
36
39
|
# Try to import newer Qdrant API for native decay
|
|
@@ -85,11 +88,49 @@ EMBEDDING_MODEL = os.getenv('EMBEDDING_MODEL', 'sentence-transformers/all-MiniLM
|
|
|
85
88
|
# Import the robust embedding manager
|
|
86
89
|
from .embedding_manager import get_embedding_manager
|
|
87
90
|
|
|
91
|
+
# Import new performance modules with fallback
|
|
92
|
+
try:
|
|
93
|
+
from .connection_pool import QdrantConnectionPool, CircuitBreaker
|
|
94
|
+
CONNECTION_POOL_AVAILABLE = True
|
|
95
|
+
except ImportError:
|
|
96
|
+
CONNECTION_POOL_AVAILABLE = False
|
|
97
|
+
logging.warning("Connection pool module not available")
|
|
98
|
+
|
|
99
|
+
try:
|
|
100
|
+
from .parallel_search import parallel_search_collections
|
|
101
|
+
PARALLEL_SEARCH_AVAILABLE = True
|
|
102
|
+
except ImportError:
|
|
103
|
+
PARALLEL_SEARCH_AVAILABLE = False
|
|
104
|
+
logging.warning("Parallel search module not available")
|
|
105
|
+
|
|
106
|
+
# Set default configuration values
|
|
107
|
+
MAX_RESULTS_PER_COLLECTION = 10
|
|
108
|
+
MAX_TOTAL_RESULTS = 1000
|
|
109
|
+
MAX_MEMORY_MB = 500
|
|
110
|
+
POOL_SIZE = 10
|
|
111
|
+
POOL_MAX_OVERFLOW = 5
|
|
112
|
+
POOL_TIMEOUT = 30.0
|
|
113
|
+
RETRY_ATTEMPTS = 3
|
|
114
|
+
RETRY_DELAY = 1.0
|
|
115
|
+
MAX_CONCURRENT_SEARCHES = 10
|
|
116
|
+
ENABLE_PARALLEL_SEARCH = True
|
|
117
|
+
|
|
118
|
+
try:
|
|
119
|
+
from .decay_manager import DecayManager
|
|
120
|
+
DECAY_MANAGER_AVAILABLE = True
|
|
121
|
+
except ImportError:
|
|
122
|
+
DECAY_MANAGER_AVAILABLE = False
|
|
123
|
+
logging.warning("Decay manager module not available")
|
|
124
|
+
|
|
88
125
|
# Lazy initialization - models will be loaded on first use
|
|
89
126
|
embedding_manager = None
|
|
90
127
|
voyage_client = None # Keep for backward compatibility
|
|
91
128
|
local_embedding_model = None # Keep for backward compatibility
|
|
92
129
|
|
|
130
|
+
# Initialize connection pool
|
|
131
|
+
qdrant_pool = None
|
|
132
|
+
circuit_breaker = None
|
|
133
|
+
|
|
93
134
|
def initialize_embeddings():
|
|
94
135
|
"""Initialize embedding models with robust fallback."""
|
|
95
136
|
global embedding_manager, voyage_client, local_embedding_model
|
|
@@ -101,7 +142,7 @@ def initialize_embeddings():
|
|
|
101
142
|
if embedding_manager.model_type == 'voyage':
|
|
102
143
|
voyage_client = embedding_manager.voyage_client
|
|
103
144
|
elif embedding_manager.model_type == 'local':
|
|
104
|
-
local_embedding_model = embedding_manager.
|
|
145
|
+
local_embedding_model = embedding_manager.local_model
|
|
105
146
|
|
|
106
147
|
return True
|
|
107
148
|
except Exception as e:
|
|
@@ -109,9 +150,8 @@ def initialize_embeddings():
|
|
|
109
150
|
return False
|
|
110
151
|
|
|
111
152
|
# Debug environment loading and startup
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
startup_time = dt.datetime.now().isoformat()
|
|
153
|
+
# Debug environment loading and startup
|
|
154
|
+
startup_time = datetime.now(timezone.utc).isoformat()
|
|
115
155
|
print(f"[STARTUP] MCP Server starting at {startup_time}", file=sys.stderr)
|
|
116
156
|
print(f"[STARTUP] Python: {sys.version}", file=sys.stderr)
|
|
117
157
|
print(f"[STARTUP] Working directory: {os.getcwd()}", file=sys.stderr)
|
|
@@ -152,8 +192,32 @@ mcp = FastMCP(
|
|
|
152
192
|
instructions="Search past conversations and store reflections with time-based memory decay"
|
|
153
193
|
)
|
|
154
194
|
|
|
155
|
-
#
|
|
156
|
-
|
|
195
|
+
# Initialize Qdrant client with connection pooling if available
|
|
196
|
+
if CONNECTION_POOL_AVAILABLE and ENABLE_PARALLEL_SEARCH:
|
|
197
|
+
qdrant_pool = QdrantConnectionPool(
|
|
198
|
+
url=QDRANT_URL,
|
|
199
|
+
pool_size=POOL_SIZE,
|
|
200
|
+
max_overflow=POOL_MAX_OVERFLOW,
|
|
201
|
+
timeout=POOL_TIMEOUT,
|
|
202
|
+
retry_attempts=RETRY_ATTEMPTS,
|
|
203
|
+
retry_delay=RETRY_DELAY
|
|
204
|
+
)
|
|
205
|
+
# Create a wrapper for backward compatibility
|
|
206
|
+
qdrant_client = AsyncQdrantClient(url=QDRANT_URL)
|
|
207
|
+
circuit_breaker = CircuitBreaker(failure_threshold=5, recovery_timeout=60.0)
|
|
208
|
+
print(f"[INFO] Connection pool initialized with size {POOL_SIZE}", file=sys.stderr)
|
|
209
|
+
else:
|
|
210
|
+
# Fallback to single client
|
|
211
|
+
qdrant_client = AsyncQdrantClient(url=QDRANT_URL)
|
|
212
|
+
qdrant_pool = None
|
|
213
|
+
circuit_breaker = None
|
|
214
|
+
print(f"[INFO] Using single Qdrant client (no pooling)", file=sys.stderr)
|
|
215
|
+
|
|
216
|
+
# Initialize decay manager if available
|
|
217
|
+
decay_manager = None
|
|
218
|
+
if DECAY_MANAGER_AVAILABLE:
|
|
219
|
+
decay_manager = DecayManager()
|
|
220
|
+
print(f"[INFO] Decay manager initialized", file=sys.stderr)
|
|
157
221
|
|
|
158
222
|
# Add MCP Resources for system status
|
|
159
223
|
@mcp.resource("status://import-stats")
|
|
@@ -492,7 +556,14 @@ def get_embedding_dimension() -> int:
|
|
|
492
556
|
|
|
493
557
|
def get_collection_suffix() -> str:
|
|
494
558
|
"""Get the collection suffix based on embedding provider."""
|
|
495
|
-
|
|
559
|
+
# Use embedding_manager's model type if available
|
|
560
|
+
if embedding_manager and hasattr(embedding_manager, 'model_type'):
|
|
561
|
+
if embedding_manager.model_type == 'voyage':
|
|
562
|
+
return "_voyage"
|
|
563
|
+
else:
|
|
564
|
+
return "_local"
|
|
565
|
+
# Fallback to environment variable
|
|
566
|
+
elif PREFER_LOCAL_EMBEDDINGS:
|
|
496
567
|
return "_local"
|
|
497
568
|
else:
|
|
498
569
|
return "_voyage"
|
|
@@ -578,1718 +649,54 @@ def aggregate_pattern_intelligence(results: List[SearchResult]) -> Dict[str, Any
|
|
|
578
649
|
return intelligence
|
|
579
650
|
|
|
580
651
|
# Register tools
|
|
581
|
-
@mcp.tool()
|
|
582
|
-
async def reflect_on_past(
|
|
583
|
-
ctx: Context,
|
|
584
|
-
query: str = Field(description="The search query to find semantically similar conversations"),
|
|
585
|
-
limit: int = Field(default=5, description="Maximum number of results to return"),
|
|
586
|
-
min_score: float = Field(default=0.3, description="Minimum similarity score (0-1)"),
|
|
587
|
-
use_decay: Union[int, str] = Field(default=-1, description="Apply time-based decay: 1=enable, 0=disable, -1=use environment default (accepts int or str)"),
|
|
588
|
-
project: Optional[str] = Field(default=None, description="Search specific project only. If not provided, searches current project based on working directory. Use 'all' to search across all projects."),
|
|
589
|
-
include_raw: bool = Field(default=False, description="Include raw Qdrant payload data for debugging (increases response size)"),
|
|
590
|
-
response_format: str = Field(default="xml", description="Response format: 'xml' or 'markdown'"),
|
|
591
|
-
brief: bool = Field(default=False, description="Brief mode: returns minimal information for faster response"),
|
|
592
|
-
mode: str = Field(default="full", description="Search mode: 'full' (all results with details), 'quick' (count + top result only), 'summary' (aggregated insights without individual results)")
|
|
593
|
-
) -> str:
|
|
594
|
-
"""Search for relevant past conversations using semantic search with optional time decay."""
|
|
595
|
-
|
|
596
|
-
logger.info(f"=== SEARCH START === Query: '{query}', Project: '{project}', Limit: {limit}")
|
|
597
|
-
|
|
598
|
-
# Validate mode parameter
|
|
599
|
-
if mode not in ['full', 'quick', 'summary']:
|
|
600
|
-
return f"<error>Invalid mode '{mode}'. Must be 'full', 'quick', or 'summary'</error>"
|
|
601
|
-
|
|
602
|
-
# Start timing
|
|
603
|
-
start_time = time.time()
|
|
604
|
-
timing_info = {}
|
|
605
|
-
|
|
606
|
-
# Normalize use_decay to integer
|
|
607
|
-
timing_info['param_parsing_start'] = time.time()
|
|
608
|
-
if isinstance(use_decay, str):
|
|
609
|
-
try:
|
|
610
|
-
use_decay = int(use_decay)
|
|
611
|
-
except ValueError:
|
|
612
|
-
raise ValueError("use_decay must be '1', '0', or '-1'")
|
|
613
|
-
timing_info['param_parsing_end'] = time.time()
|
|
614
|
-
|
|
615
|
-
# Parse decay parameter using integer approach
|
|
616
|
-
should_use_decay = (
|
|
617
|
-
True if use_decay == 1
|
|
618
|
-
else False if use_decay == 0
|
|
619
|
-
else ENABLE_MEMORY_DECAY # -1 or any other value
|
|
620
|
-
)
|
|
621
|
-
|
|
622
|
-
# Determine project scope
|
|
623
|
-
target_project = project
|
|
624
|
-
|
|
625
|
-
# Always get the working directory for logging purposes
|
|
626
|
-
cwd = os.environ.get('MCP_CLIENT_CWD', os.getcwd())
|
|
627
|
-
await ctx.debug(f"CWD: {cwd}, Project param: {project}")
|
|
628
|
-
|
|
629
|
-
if project is None:
|
|
630
|
-
# Use MCP_CLIENT_CWD environment variable set by run-mcp.sh
|
|
631
|
-
# This contains the actual working directory where Claude Code is running
|
|
632
|
-
|
|
633
|
-
# Extract project name from path (e.g., /Users/.../projects/project-name)
|
|
634
|
-
path_parts = Path(cwd).parts
|
|
635
|
-
if 'projects' in path_parts:
|
|
636
|
-
idx = path_parts.index('projects')
|
|
637
|
-
if idx + 1 < len(path_parts):
|
|
638
|
-
target_project = path_parts[idx + 1]
|
|
639
|
-
elif '.claude' in path_parts:
|
|
640
|
-
# If we're in a .claude directory, go up to find project
|
|
641
|
-
for i, part in enumerate(path_parts):
|
|
642
|
-
if part == '.claude' and i > 0:
|
|
643
|
-
target_project = path_parts[i - 1]
|
|
644
|
-
break
|
|
645
|
-
|
|
646
|
-
# If still no project detected, use the last directory name
|
|
647
|
-
if target_project is None:
|
|
648
|
-
target_project = Path(cwd).name
|
|
649
|
-
|
|
650
|
-
await ctx.debug(f"Auto-detected project from path: {target_project}")
|
|
651
|
-
|
|
652
|
-
# For project matching, we need to handle the dash-encoded format
|
|
653
|
-
# Convert folder name to the format used in stored data
|
|
654
|
-
if target_project != 'all':
|
|
655
|
-
# The stored format uses full path with dashes, so we need to construct it
|
|
656
|
-
# For now, let's try to match based on the end of the project name
|
|
657
|
-
pass # We'll handle this differently in the filtering logic
|
|
658
|
-
|
|
659
|
-
await ctx.debug(f"Searching for: {query}")
|
|
660
|
-
await ctx.debug(f"Client working directory: {cwd}")
|
|
661
|
-
await ctx.debug(f"Project scope: {target_project if target_project != 'all' else 'all projects'}")
|
|
662
|
-
await ctx.debug(f"Decay enabled: {should_use_decay}")
|
|
663
|
-
await ctx.debug(f"Native decay mode: {USE_NATIVE_DECAY}")
|
|
664
|
-
await ctx.debug(f"ENABLE_MEMORY_DECAY env: {ENABLE_MEMORY_DECAY}")
|
|
665
|
-
await ctx.debug(f"DECAY_WEIGHT: {DECAY_WEIGHT}, DECAY_SCALE_DAYS: {DECAY_SCALE_DAYS}")
|
|
666
|
-
|
|
667
|
-
try:
|
|
668
|
-
# We'll generate embeddings on-demand per collection type
|
|
669
|
-
timing_info['embedding_prep_start'] = time.time()
|
|
670
|
-
query_embeddings = {} # Cache embeddings by type
|
|
671
|
-
timing_info['embedding_prep_end'] = time.time()
|
|
672
|
-
|
|
673
|
-
# Get all collections
|
|
674
|
-
timing_info['get_collections_start'] = time.time()
|
|
675
|
-
all_collections = await get_all_collections()
|
|
676
|
-
timing_info['get_collections_end'] = time.time()
|
|
677
|
-
|
|
678
|
-
if not all_collections:
|
|
679
|
-
return "No conversation collections found. Please import conversations first."
|
|
680
|
-
|
|
681
|
-
# Filter collections by project if not searching all
|
|
682
|
-
project_collections = [] # Define at this scope for later use
|
|
683
|
-
if target_project != 'all':
|
|
684
|
-
# Use ProjectResolver with sync client (resolver expects sync operations)
|
|
685
|
-
from qdrant_client import QdrantClient as SyncQdrantClient
|
|
686
|
-
sync_client = SyncQdrantClient(url=QDRANT_URL)
|
|
687
|
-
resolver = ProjectResolver(sync_client)
|
|
688
|
-
project_collections = resolver.find_collections_for_project(target_project)
|
|
689
|
-
|
|
690
|
-
if not project_collections:
|
|
691
|
-
# Fall back to old method for backward compatibility
|
|
692
|
-
normalized_name = normalize_project_name(target_project)
|
|
693
|
-
project_hash = hashlib.md5(normalized_name.encode()).hexdigest()[:8]
|
|
694
|
-
project_collections = [
|
|
695
|
-
c for c in all_collections
|
|
696
|
-
if c.startswith(f"conv_{project_hash}_")
|
|
697
|
-
]
|
|
698
|
-
|
|
699
|
-
# Always include reflections collections when searching a specific project
|
|
700
|
-
reflections_collections = [c for c in all_collections if c.startswith('reflections')]
|
|
701
|
-
|
|
702
|
-
if not project_collections:
|
|
703
|
-
# Fall back to searching all collections but filtering by project metadata
|
|
704
|
-
await ctx.debug(f"No collections found for project {target_project}, will filter by metadata")
|
|
705
|
-
collections_to_search = all_collections
|
|
706
|
-
else:
|
|
707
|
-
await ctx.debug(f"Found {len(project_collections)} collections for project {target_project}")
|
|
708
|
-
# Include both project collections and reflections
|
|
709
|
-
collections_to_search = project_collections + reflections_collections
|
|
710
|
-
# Remove duplicates
|
|
711
|
-
collections_to_search = list(set(collections_to_search))
|
|
712
|
-
else:
|
|
713
|
-
collections_to_search = all_collections
|
|
714
|
-
|
|
715
|
-
await ctx.debug(f"Searching across {len(collections_to_search)} collections")
|
|
716
|
-
await ctx.debug(f"Using {'local' if PREFER_LOCAL_EMBEDDINGS or not voyage_client else 'Voyage AI'} embeddings")
|
|
717
|
-
|
|
718
|
-
all_results = []
|
|
719
|
-
|
|
720
|
-
# Search each collection
|
|
721
|
-
timing_info['search_all_start'] = time.time()
|
|
722
|
-
collection_timings = []
|
|
723
|
-
|
|
724
|
-
# Report initial progress
|
|
725
|
-
await ctx.report_progress(progress=0, total=len(collections_to_search))
|
|
726
|
-
|
|
727
|
-
for idx, collection_name in enumerate(collections_to_search):
|
|
728
|
-
collection_timing = {'name': collection_name, 'start': time.time()}
|
|
729
|
-
|
|
730
|
-
# Report progress before searching each collection
|
|
731
|
-
await ctx.report_progress(
|
|
732
|
-
progress=idx,
|
|
733
|
-
total=len(collections_to_search),
|
|
734
|
-
message=f"Searching {collection_name}"
|
|
735
|
-
)
|
|
736
|
-
|
|
737
|
-
try:
|
|
738
|
-
# Determine embedding type for this collection
|
|
739
|
-
embedding_type_for_collection = 'voyage' if collection_name.endswith('_voyage') else 'local'
|
|
740
|
-
|
|
741
|
-
# Generate or retrieve cached embedding for this type
|
|
742
|
-
if embedding_type_for_collection not in query_embeddings:
|
|
743
|
-
try:
|
|
744
|
-
query_embeddings[embedding_type_for_collection] = await generate_embedding(query, force_type=embedding_type_for_collection)
|
|
745
|
-
except Exception as e:
|
|
746
|
-
await ctx.debug(f"Failed to generate {embedding_type_for_collection} embedding: {e}")
|
|
747
|
-
continue
|
|
748
|
-
|
|
749
|
-
query_embedding = query_embeddings[embedding_type_for_collection]
|
|
750
|
-
|
|
751
|
-
if should_use_decay and USE_NATIVE_DECAY and NATIVE_DECAY_AVAILABLE:
|
|
752
|
-
# Use native Qdrant decay with newer API
|
|
753
|
-
await ctx.debug(f"Using NATIVE Qdrant decay (new API) for {collection_name}")
|
|
754
|
-
|
|
755
|
-
# Build the query with native Qdrant decay formula using newer API
|
|
756
|
-
# Convert half-life to seconds (Qdrant uses seconds for datetime)
|
|
757
|
-
half_life_seconds = DECAY_SCALE_DAYS * 24 * 60 * 60
|
|
758
|
-
|
|
759
|
-
# Build query using proper Python models as per Qdrant docs
|
|
760
|
-
from qdrant_client import models
|
|
761
|
-
|
|
762
|
-
query_obj = models.FormulaQuery(
|
|
763
|
-
formula=models.SumExpression(
|
|
764
|
-
sum=[
|
|
765
|
-
"$score", # Original similarity score
|
|
766
|
-
models.MultExpression(
|
|
767
|
-
mult=[
|
|
768
|
-
DECAY_WEIGHT, # Weight multiplier
|
|
769
|
-
models.ExpDecayExpression(
|
|
770
|
-
exp_decay=models.DecayParamsExpression(
|
|
771
|
-
x=models.DatetimeKeyExpression(
|
|
772
|
-
datetime_key="timestamp" # Payload field with datetime
|
|
773
|
-
),
|
|
774
|
-
target=models.DatetimeExpression(
|
|
775
|
-
datetime="now" # Current time on server
|
|
776
|
-
),
|
|
777
|
-
scale=half_life_seconds, # Scale in seconds
|
|
778
|
-
midpoint=0.5 # Half-life semantics
|
|
779
|
-
)
|
|
780
|
-
)
|
|
781
|
-
]
|
|
782
|
-
)
|
|
783
|
-
]
|
|
784
|
-
)
|
|
785
|
-
)
|
|
786
|
-
|
|
787
|
-
# Execute query with native decay (new API)
|
|
788
|
-
results = await qdrant_client.query_points(
|
|
789
|
-
collection_name=collection_name,
|
|
790
|
-
query=query_obj,
|
|
791
|
-
limit=limit,
|
|
792
|
-
with_payload=True
|
|
793
|
-
# No score_threshold - let Qdrant's decay formula handle relevance
|
|
794
|
-
)
|
|
795
|
-
elif should_use_decay and USE_NATIVE_DECAY and not NATIVE_DECAY_AVAILABLE:
|
|
796
|
-
# Use native Qdrant decay with older API
|
|
797
|
-
await ctx.debug(f"Using NATIVE Qdrant decay (legacy API) for {collection_name}")
|
|
798
|
-
|
|
799
|
-
# Convert half-life to seconds (Qdrant uses seconds for datetime)
|
|
800
|
-
half_life_seconds = DECAY_SCALE_DAYS * 24 * 60 * 60
|
|
801
|
-
|
|
802
|
-
# Build the query with native Qdrant decay formula using older API
|
|
803
|
-
# Use the same models but with FormulaQuery
|
|
804
|
-
query_obj = FormulaQuery(
|
|
805
|
-
nearest=query_embedding,
|
|
806
|
-
formula=SumExpression(
|
|
807
|
-
sum=[
|
|
808
|
-
"$score", # Original similarity score
|
|
809
|
-
{
|
|
810
|
-
"mult": [
|
|
811
|
-
DECAY_WEIGHT, # Weight multiplier
|
|
812
|
-
{
|
|
813
|
-
"exp_decay": DecayParamsExpression(
|
|
814
|
-
x=DatetimeKeyExpression(datetime_key="timestamp"),
|
|
815
|
-
target=DatetimeExpression(datetime="now"),
|
|
816
|
-
scale=half_life_seconds, # Scale in seconds
|
|
817
|
-
midpoint=0.5 # Half-life semantics
|
|
818
|
-
)
|
|
819
|
-
}
|
|
820
|
-
]
|
|
821
|
-
}
|
|
822
|
-
]
|
|
823
|
-
)
|
|
824
|
-
)
|
|
825
|
-
|
|
826
|
-
# Execute query with native decay
|
|
827
|
-
results = await qdrant_client.query_points(
|
|
828
|
-
collection_name=collection_name,
|
|
829
|
-
query=query_obj,
|
|
830
|
-
limit=limit,
|
|
831
|
-
with_payload=True
|
|
832
|
-
# No score_threshold - let Qdrant's decay formula handle relevance
|
|
833
|
-
)
|
|
834
|
-
|
|
835
|
-
# Process results from native decay search
|
|
836
|
-
for point in results.points:
|
|
837
|
-
# Clean timestamp for proper parsing
|
|
838
|
-
raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
|
|
839
|
-
clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
|
|
840
|
-
|
|
841
|
-
# Check project filter if we're searching all collections but want specific project
|
|
842
|
-
point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
|
|
843
|
-
|
|
844
|
-
# Special handling for reflections - they're global by default but can have project context
|
|
845
|
-
is_reflection_collection = collection_name.startswith('reflections')
|
|
846
|
-
|
|
847
|
-
# Handle project matching - check if the target project name appears at the end of the stored project path
|
|
848
|
-
if target_project != 'all' and not project_collections and not is_reflection_collection:
|
|
849
|
-
# The stored project name is like "-Users-username-projects-ShopifyMCPMockShop"
|
|
850
|
-
# We want to match just "ShopifyMCPMockShop"
|
|
851
|
-
# Also handle underscore/dash variations (procsolve-website vs procsolve_website)
|
|
852
|
-
normalized_target = target_project.replace('-', '_')
|
|
853
|
-
normalized_stored = point_project.replace('-', '_')
|
|
854
|
-
if not (normalized_stored.endswith(f"_{normalized_target}") or
|
|
855
|
-
normalized_stored == normalized_target or
|
|
856
|
-
point_project.endswith(f"-{target_project}") or
|
|
857
|
-
point_project == target_project):
|
|
858
|
-
continue # Skip results from other projects
|
|
859
|
-
|
|
860
|
-
# For reflections with project context, optionally filter by project
|
|
861
|
-
if is_reflection_collection and target_project != 'all' and 'project' in point.payload:
|
|
862
|
-
# Only filter if the reflection has project metadata
|
|
863
|
-
reflection_project = point.payload.get('project', '')
|
|
864
|
-
if reflection_project:
|
|
865
|
-
# Normalize both for comparison (handle underscore/dash variations)
|
|
866
|
-
normalized_target = target_project.replace('-', '_')
|
|
867
|
-
normalized_reflection = reflection_project.replace('-', '_')
|
|
868
|
-
if not (
|
|
869
|
-
reflection_project == target_project or
|
|
870
|
-
normalized_reflection == normalized_target or
|
|
871
|
-
reflection_project.endswith(f"/{target_project}") or
|
|
872
|
-
reflection_project.endswith(f"-{target_project}") or
|
|
873
|
-
normalized_reflection.endswith(f"_{normalized_target}") or
|
|
874
|
-
normalized_reflection.endswith(f"/{normalized_target}")
|
|
875
|
-
):
|
|
876
|
-
continue # Skip reflections from other projects
|
|
877
|
-
|
|
878
|
-
# Log pattern data
|
|
879
|
-
patterns = point.payload.get('code_patterns')
|
|
880
|
-
logger.info(f"DEBUG: Creating SearchResult for point {point.id} from {collection_name}: has_patterns={bool(patterns)}, pattern_keys={list(patterns.keys()) if patterns else None}")
|
|
881
|
-
|
|
882
|
-
all_results.append(SearchResult(
|
|
883
|
-
id=str(point.id),
|
|
884
|
-
score=point.score, # Score already includes decay
|
|
885
|
-
timestamp=clean_timestamp,
|
|
886
|
-
role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
|
|
887
|
-
excerpt=(point.payload.get('text', '')[:350] + '...' if len(point.payload.get('text', '')) > 350 else point.payload.get('text', '')),
|
|
888
|
-
project_name=point_project,
|
|
889
|
-
conversation_id=point.payload.get('conversation_id'),
|
|
890
|
-
base_conversation_id=point.payload.get('base_conversation_id'),
|
|
891
|
-
collection_name=collection_name,
|
|
892
|
-
raw_payload=point.payload, # Always include payload for metadata extraction
|
|
893
|
-
# Pattern intelligence metadata
|
|
894
|
-
code_patterns=point.payload.get('code_patterns'),
|
|
895
|
-
files_analyzed=point.payload.get('files_analyzed'),
|
|
896
|
-
tools_used=list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
|
|
897
|
-
concepts=point.payload.get('concepts')
|
|
898
|
-
))
|
|
899
|
-
|
|
900
|
-
elif should_use_decay:
|
|
901
|
-
# Use client-side decay (existing implementation)
|
|
902
|
-
await ctx.debug(f"Using CLIENT-SIDE decay for {collection_name}")
|
|
903
|
-
|
|
904
|
-
# Search without score threshold to get all candidates
|
|
905
|
-
results = await qdrant_client.search(
|
|
906
|
-
collection_name=collection_name,
|
|
907
|
-
query_vector=query_embedding,
|
|
908
|
-
limit=limit * 3, # Get more candidates for decay filtering
|
|
909
|
-
with_payload=True
|
|
910
|
-
)
|
|
911
|
-
|
|
912
|
-
# Apply decay scoring manually
|
|
913
|
-
now = datetime.now(timezone.utc)
|
|
914
|
-
scale_ms = DECAY_SCALE_DAYS * 24 * 60 * 60 * 1000
|
|
915
|
-
|
|
916
|
-
decay_results = []
|
|
917
|
-
for point in results:
|
|
918
|
-
try:
|
|
919
|
-
# Get timestamp from payload
|
|
920
|
-
timestamp_str = point.payload.get('timestamp')
|
|
921
|
-
if timestamp_str:
|
|
922
|
-
timestamp = datetime.fromisoformat(timestamp_str.replace('Z', '+00:00'))
|
|
923
|
-
# Ensure timestamp is timezone-aware
|
|
924
|
-
if timestamp.tzinfo is None:
|
|
925
|
-
timestamp = timestamp.replace(tzinfo=timezone.utc)
|
|
926
|
-
age_ms = (now - timestamp).total_seconds() * 1000
|
|
927
|
-
|
|
928
|
-
# Calculate decay factor using proper half-life formula
|
|
929
|
-
# For half-life H: decay = exp(-ln(2) * age / H)
|
|
930
|
-
ln2 = math.log(2)
|
|
931
|
-
decay_factor = math.exp(-ln2 * age_ms / scale_ms)
|
|
932
|
-
|
|
933
|
-
# Apply multiplicative decay formula to keep scores bounded [0, 1]
|
|
934
|
-
# adjusted = score * ((1 - weight) + weight * decay)
|
|
935
|
-
adjusted_score = point.score * ((1 - DECAY_WEIGHT) + DECAY_WEIGHT * decay_factor)
|
|
936
|
-
|
|
937
|
-
# Debug: show the calculation
|
|
938
|
-
age_days = age_ms / (24 * 60 * 60 * 1000)
|
|
939
|
-
await ctx.debug(f"Point: age={age_days:.1f} days, original_score={point.score:.3f}, decay_factor={decay_factor:.3f}, adjusted_score={adjusted_score:.3f}")
|
|
940
|
-
else:
|
|
941
|
-
adjusted_score = point.score
|
|
942
|
-
|
|
943
|
-
# Only include if above min_score after decay
|
|
944
|
-
if adjusted_score >= min_score:
|
|
945
|
-
decay_results.append((adjusted_score, point))
|
|
946
|
-
|
|
947
|
-
except Exception as e:
|
|
948
|
-
await ctx.debug(f"Error applying decay to point: {e}")
|
|
949
|
-
decay_results.append((point.score, point))
|
|
950
|
-
|
|
951
|
-
# Sort by adjusted score and take top results
|
|
952
|
-
decay_results.sort(key=lambda x: x[0], reverse=True)
|
|
953
|
-
|
|
954
|
-
# Convert to SearchResult format
|
|
955
|
-
for adjusted_score, point in decay_results[:limit]:
|
|
956
|
-
# Clean timestamp for proper parsing
|
|
957
|
-
raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
|
|
958
|
-
clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
|
|
959
|
-
|
|
960
|
-
# Check project filter if we're searching all collections but want specific project
|
|
961
|
-
point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
|
|
962
|
-
|
|
963
|
-
# Special handling for reflections - they're global by default but can have project context
|
|
964
|
-
is_reflection_collection = collection_name.startswith('reflections')
|
|
965
|
-
|
|
966
|
-
# Handle project matching - check if the target project name appears at the end of the stored project path
|
|
967
|
-
if target_project != 'all' and not project_collections and not is_reflection_collection:
|
|
968
|
-
# The stored project name is like "-Users-username-projects-ShopifyMCPMockShop"
|
|
969
|
-
# We want to match just "ShopifyMCPMockShop"
|
|
970
|
-
# Also handle underscore/dash variations (procsolve-website vs procsolve_website)
|
|
971
|
-
normalized_target = target_project.replace('-', '_')
|
|
972
|
-
normalized_stored = point_project.replace('-', '_')
|
|
973
|
-
if not (normalized_stored.endswith(f"_{normalized_target}") or
|
|
974
|
-
normalized_stored == normalized_target or
|
|
975
|
-
point_project.endswith(f"-{target_project}") or
|
|
976
|
-
point_project == target_project):
|
|
977
|
-
continue # Skip results from other projects
|
|
978
|
-
|
|
979
|
-
# For reflections with project context, optionally filter by project
|
|
980
|
-
if is_reflection_collection and target_project != 'all' and 'project' in point.payload:
|
|
981
|
-
# Only filter if the reflection has project metadata
|
|
982
|
-
reflection_project = point.payload.get('project', '')
|
|
983
|
-
if reflection_project:
|
|
984
|
-
# Normalize both for comparison (handle underscore/dash variations)
|
|
985
|
-
normalized_target = target_project.replace('-', '_')
|
|
986
|
-
normalized_reflection = reflection_project.replace('-', '_')
|
|
987
|
-
if not (
|
|
988
|
-
reflection_project == target_project or
|
|
989
|
-
normalized_reflection == normalized_target or
|
|
990
|
-
reflection_project.endswith(f"/{target_project}") or
|
|
991
|
-
reflection_project.endswith(f"-{target_project}") or
|
|
992
|
-
normalized_reflection.endswith(f"_{normalized_target}") or
|
|
993
|
-
normalized_reflection.endswith(f"/{normalized_target}")
|
|
994
|
-
):
|
|
995
|
-
continue # Skip reflections from other projects
|
|
996
|
-
|
|
997
|
-
all_results.append(SearchResult(
|
|
998
|
-
id=str(point.id),
|
|
999
|
-
score=adjusted_score, # Use adjusted score
|
|
1000
|
-
timestamp=clean_timestamp,
|
|
1001
|
-
role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
|
|
1002
|
-
excerpt=(point.payload.get('text', '')[:350] + '...' if len(point.payload.get('text', '')) > 350 else point.payload.get('text', '')),
|
|
1003
|
-
project_name=point_project,
|
|
1004
|
-
conversation_id=point.payload.get('conversation_id'),
|
|
1005
|
-
base_conversation_id=point.payload.get('base_conversation_id'),
|
|
1006
|
-
collection_name=collection_name,
|
|
1007
|
-
raw_payload=point.payload, # Always include payload for metadata extraction
|
|
1008
|
-
# Pattern intelligence metadata
|
|
1009
|
-
code_patterns=point.payload.get('code_patterns'),
|
|
1010
|
-
files_analyzed=point.payload.get('files_analyzed'),
|
|
1011
|
-
tools_used=list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
|
|
1012
|
-
concepts=point.payload.get('concepts')
|
|
1013
|
-
))
|
|
1014
|
-
else:
|
|
1015
|
-
# Standard search without decay
|
|
1016
|
-
# Let Qdrant handle scoring natively
|
|
1017
|
-
results = await qdrant_client.search(
|
|
1018
|
-
collection_name=collection_name,
|
|
1019
|
-
query_vector=query_embedding,
|
|
1020
|
-
limit=limit * 2, # Get more results to account for filtering
|
|
1021
|
-
with_payload=True
|
|
1022
|
-
# No score_threshold - let Qdrant decide what's relevant
|
|
1023
|
-
)
|
|
1024
|
-
|
|
1025
|
-
for point in results:
|
|
1026
|
-
# Clean timestamp for proper parsing
|
|
1027
|
-
raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
|
|
1028
|
-
clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
|
|
1029
|
-
|
|
1030
|
-
# Check project filter if we're searching all collections but want specific project
|
|
1031
|
-
point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
|
|
1032
|
-
|
|
1033
|
-
# Special handling for reflections - they're global by default but can have project context
|
|
1034
|
-
is_reflection_collection = collection_name.startswith('reflections')
|
|
1035
|
-
|
|
1036
|
-
# Handle project matching - check if the target project name appears at the end of the stored project path
|
|
1037
|
-
if target_project != 'all' and not project_collections and not is_reflection_collection:
|
|
1038
|
-
# The stored project name is like "-Users-username-projects-ShopifyMCPMockShop"
|
|
1039
|
-
# We want to match just "ShopifyMCPMockShop"
|
|
1040
|
-
# Also handle underscore/dash variations (procsolve-website vs procsolve_website)
|
|
1041
|
-
normalized_target = target_project.replace('-', '_')
|
|
1042
|
-
normalized_stored = point_project.replace('-', '_')
|
|
1043
|
-
if not (normalized_stored.endswith(f"_{normalized_target}") or
|
|
1044
|
-
normalized_stored == normalized_target or
|
|
1045
|
-
point_project.endswith(f"-{target_project}") or
|
|
1046
|
-
point_project == target_project):
|
|
1047
|
-
continue # Skip results from other projects
|
|
1048
|
-
|
|
1049
|
-
# For reflections with project context, optionally filter by project
|
|
1050
|
-
if is_reflection_collection and target_project != 'all' and 'project' in point.payload:
|
|
1051
|
-
# Only filter if the reflection has project metadata
|
|
1052
|
-
reflection_project = point.payload.get('project', '')
|
|
1053
|
-
if reflection_project:
|
|
1054
|
-
# Normalize both for comparison (handle underscore/dash variations)
|
|
1055
|
-
normalized_target = target_project.replace('-', '_')
|
|
1056
|
-
normalized_reflection = reflection_project.replace('-', '_')
|
|
1057
|
-
if not (
|
|
1058
|
-
reflection_project == target_project or
|
|
1059
|
-
normalized_reflection == normalized_target or
|
|
1060
|
-
reflection_project.endswith(f"/{target_project}") or
|
|
1061
|
-
reflection_project.endswith(f"-{target_project}") or
|
|
1062
|
-
normalized_reflection.endswith(f"_{normalized_target}") or
|
|
1063
|
-
normalized_reflection.endswith(f"/{normalized_target}")
|
|
1064
|
-
):
|
|
1065
|
-
continue # Skip reflections from other projects
|
|
1066
|
-
|
|
1067
|
-
# BOOST V2 CHUNKS: Apply score boost for v2 chunks (better quality)
|
|
1068
|
-
original_score = point.score
|
|
1069
|
-
final_score = original_score
|
|
1070
|
-
chunking_version = point.payload.get('chunking_version', 'v1')
|
|
1071
|
-
|
|
1072
|
-
if chunking_version == 'v2':
|
|
1073
|
-
# Boost v2 chunks by 20% (configurable)
|
|
1074
|
-
boost_factor = 1.2 # From migration config
|
|
1075
|
-
final_score = min(1.0, original_score * boost_factor)
|
|
1076
|
-
await ctx.debug(f"Boosted v2 chunk: {original_score:.3f} -> {final_score:.3f}")
|
|
1077
|
-
|
|
1078
|
-
# Apply minimum score threshold after boosting
|
|
1079
|
-
if final_score < min_score:
|
|
1080
|
-
continue
|
|
1081
|
-
|
|
1082
|
-
search_result = SearchResult(
|
|
1083
|
-
id=str(point.id),
|
|
1084
|
-
score=final_score,
|
|
1085
|
-
timestamp=clean_timestamp,
|
|
1086
|
-
role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
|
|
1087
|
-
excerpt=(point.payload.get('text', '')[:350] + '...' if len(point.payload.get('text', '')) > 350 else point.payload.get('text', '')),
|
|
1088
|
-
project_name=point_project,
|
|
1089
|
-
conversation_id=point.payload.get('conversation_id'),
|
|
1090
|
-
base_conversation_id=point.payload.get('base_conversation_id'),
|
|
1091
|
-
collection_name=collection_name,
|
|
1092
|
-
raw_payload=point.payload, # Always include payload for metadata extraction
|
|
1093
|
-
# Pattern intelligence metadata
|
|
1094
|
-
code_patterns=point.payload.get('code_patterns'),
|
|
1095
|
-
files_analyzed=point.payload.get('files_analyzed'),
|
|
1096
|
-
tools_used=list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
|
|
1097
|
-
concepts=point.payload.get('concepts')
|
|
1098
|
-
)
|
|
1099
|
-
|
|
1100
|
-
all_results.append(search_result)
|
|
1101
|
-
|
|
1102
|
-
except Exception as e:
|
|
1103
|
-
await ctx.debug(f"Error searching {collection_name}: {str(e)}")
|
|
1104
|
-
collection_timing['error'] = str(e)
|
|
1105
|
-
|
|
1106
|
-
collection_timing['end'] = time.time()
|
|
1107
|
-
collection_timings.append(collection_timing)
|
|
1108
|
-
|
|
1109
|
-
timing_info['search_all_end'] = time.time()
|
|
1110
|
-
|
|
1111
|
-
# Report completion of search phase
|
|
1112
|
-
await ctx.report_progress(
|
|
1113
|
-
progress=len(collections_to_search),
|
|
1114
|
-
total=len(collections_to_search),
|
|
1115
|
-
message="Search complete, processing results"
|
|
1116
|
-
)
|
|
1117
|
-
|
|
1118
|
-
# Apply base_conversation_id boosting before sorting
|
|
1119
|
-
timing_info['boost_start'] = time.time()
|
|
1120
|
-
|
|
1121
|
-
# Group results by base_conversation_id to identify related chunks
|
|
1122
|
-
base_conversation_groups = {}
|
|
1123
|
-
for result in all_results:
|
|
1124
|
-
base_id = result.base_conversation_id
|
|
1125
|
-
if base_id:
|
|
1126
|
-
if base_id not in base_conversation_groups:
|
|
1127
|
-
base_conversation_groups[base_id] = []
|
|
1128
|
-
base_conversation_groups[base_id].append(result)
|
|
1129
|
-
|
|
1130
|
-
# Apply boost to results from base conversations with multiple high-scoring chunks
|
|
1131
|
-
base_conversation_boost = 0.1 # Boost factor for base conversation matching
|
|
1132
|
-
for base_id, group_results in base_conversation_groups.items():
|
|
1133
|
-
if len(group_results) > 1: # Multiple chunks from same base conversation
|
|
1134
|
-
avg_score = sum(r.score for r in group_results) / len(group_results)
|
|
1135
|
-
if avg_score > 0.8: # Only boost high-quality base conversations
|
|
1136
|
-
for result in group_results:
|
|
1137
|
-
result.score += base_conversation_boost
|
|
1138
|
-
await ctx.debug(f"Boosted result from base_conversation_id {base_id}: {result.score:.3f}")
|
|
1139
|
-
|
|
1140
|
-
timing_info['boost_end'] = time.time()
|
|
1141
|
-
|
|
1142
|
-
# Sort by score and limit
|
|
1143
|
-
timing_info['sort_start'] = time.time()
|
|
1144
|
-
all_results.sort(key=lambda x: x.score, reverse=True)
|
|
1145
|
-
|
|
1146
|
-
# Apply mode-specific limits
|
|
1147
|
-
if mode == "quick":
|
|
1148
|
-
# For quick mode, only keep the top result
|
|
1149
|
-
all_results = all_results[:1]
|
|
1150
|
-
elif mode == "summary":
|
|
1151
|
-
# For summary mode, we'll process all results but not return individual ones
|
|
1152
|
-
pass # Keep all for aggregation
|
|
1153
|
-
else:
|
|
1154
|
-
# For full mode, apply the normal limit
|
|
1155
|
-
all_results = all_results[:limit]
|
|
1156
|
-
|
|
1157
|
-
timing_info['sort_end'] = time.time()
|
|
1158
|
-
|
|
1159
|
-
logger.info(f"Total results: {len(all_results)}, Mode: {mode}, Returning: {len(all_results[:limit])}")
|
|
1160
|
-
for r in all_results[:3]: # Log first 3
|
|
1161
|
-
logger.debug(f"Result: id={r.id}, has_patterns={bool(r.code_patterns)}, pattern_keys={list(r.code_patterns.keys()) if r.code_patterns else None}")
|
|
1162
|
-
|
|
1163
|
-
if not all_results:
|
|
1164
|
-
return f"No conversations found matching '{query}'. Try different keywords or check if conversations have been imported."
|
|
1165
|
-
|
|
1166
|
-
# Aggregate pattern intelligence across results
|
|
1167
|
-
pattern_intelligence = aggregate_pattern_intelligence(all_results)
|
|
1168
|
-
|
|
1169
|
-
# Update indexing status before returning results
|
|
1170
|
-
await update_indexing_status()
|
|
1171
|
-
|
|
1172
|
-
# Format results based on response_format and mode
|
|
1173
|
-
timing_info['format_start'] = time.time()
|
|
1174
|
-
|
|
1175
|
-
# Handle mode-specific responses
|
|
1176
|
-
if mode == "quick":
|
|
1177
|
-
# Quick mode: return just count and top result
|
|
1178
|
-
total_count = len(all_results) # Before we limited to 1
|
|
1179
|
-
if response_format == "xml":
|
|
1180
|
-
result_text = f"<quick_search>\n"
|
|
1181
|
-
result_text += f" <count>{total_count}</count>\n"
|
|
1182
|
-
if all_results:
|
|
1183
|
-
top_result = all_results[0]
|
|
1184
|
-
result_text += f" <top_result>\n"
|
|
1185
|
-
result_text += f" <score>{top_result.score:.3f}</score>\n"
|
|
1186
|
-
result_text += f" <excerpt>{escape(top_result.excerpt[:200])}</excerpt>\n"
|
|
1187
|
-
result_text += f" <project>{escape(top_result.project_name)}</project>\n"
|
|
1188
|
-
result_text += f" <conversation_id>{escape(top_result.conversation_id or '')}</conversation_id>\n"
|
|
1189
|
-
result_text += f" </top_result>\n"
|
|
1190
|
-
result_text += f"</quick_search>"
|
|
1191
|
-
return result_text
|
|
1192
|
-
else:
|
|
1193
|
-
# Markdown format for quick mode
|
|
1194
|
-
if all_results:
|
|
1195
|
-
return f"**Found {total_count} matches**\n\nTop result (score: {all_results[0].score:.3f}):\n{all_results[0].excerpt[:200]}"
|
|
1196
|
-
else:
|
|
1197
|
-
return f"No matches found for '{query}'"
|
|
1198
|
-
|
|
1199
|
-
elif mode == "summary":
|
|
1200
|
-
# Summary mode: return aggregated insights without individual results
|
|
1201
|
-
if not all_results:
|
|
1202
|
-
return f"No conversations found to summarize for '{query}'"
|
|
1203
|
-
|
|
1204
|
-
# Aggregate data
|
|
1205
|
-
total_count = len(all_results)
|
|
1206
|
-
avg_score = sum(r.score for r in all_results) / total_count
|
|
1207
|
-
|
|
1208
|
-
# Extract common concepts and tools
|
|
1209
|
-
all_concepts = []
|
|
1210
|
-
all_tools = []
|
|
1211
|
-
all_files = []
|
|
1212
|
-
projects = set()
|
|
1213
|
-
|
|
1214
|
-
for result in all_results:
|
|
1215
|
-
if result.concepts:
|
|
1216
|
-
all_concepts.extend(result.concepts)
|
|
1217
|
-
if result.tools_used:
|
|
1218
|
-
all_tools.extend(result.tools_used)
|
|
1219
|
-
if result.files_analyzed:
|
|
1220
|
-
all_files.extend(result.files_analyzed)
|
|
1221
|
-
projects.add(result.project_name)
|
|
1222
|
-
|
|
1223
|
-
# Count frequencies
|
|
1224
|
-
from collections import Counter
|
|
1225
|
-
concept_counts = Counter(all_concepts).most_common(5)
|
|
1226
|
-
tool_counts = Counter(all_tools).most_common(5)
|
|
1227
|
-
|
|
1228
|
-
if response_format == "xml":
|
|
1229
|
-
result_text = f"<search_summary>\n"
|
|
1230
|
-
result_text += f" <query>{escape(query)}</query>\n"
|
|
1231
|
-
result_text += f" <total_matches>{total_count}</total_matches>\n"
|
|
1232
|
-
result_text += f" <average_score>{avg_score:.3f}</average_score>\n"
|
|
1233
|
-
result_text += f" <projects_involved>{len(projects)}</projects_involved>\n"
|
|
1234
|
-
if concept_counts:
|
|
1235
|
-
result_text += f" <common_concepts>\n"
|
|
1236
|
-
for concept, count in concept_counts:
|
|
1237
|
-
result_text += f" <concept count=\"{count}\">{escape(concept)}</concept>\n"
|
|
1238
|
-
result_text += f" </common_concepts>\n"
|
|
1239
|
-
if tool_counts:
|
|
1240
|
-
result_text += f" <common_tools>\n"
|
|
1241
|
-
for tool, count in tool_counts:
|
|
1242
|
-
result_text += f" <tool count=\"{count}\">{escape(tool)}</tool>\n"
|
|
1243
|
-
result_text += f" </common_tools>\n"
|
|
1244
|
-
result_text += f"</search_summary>"
|
|
1245
|
-
return result_text
|
|
1246
|
-
else:
|
|
1247
|
-
# Markdown format for summary
|
|
1248
|
-
result_text = f"## Summary for: {query}\n\n"
|
|
1249
|
-
result_text += f"- **Total matches**: {total_count}\n"
|
|
1250
|
-
result_text += f"- **Average relevance**: {avg_score:.3f}\n"
|
|
1251
|
-
result_text += f"- **Projects involved**: {len(projects)}\n\n"
|
|
1252
|
-
if concept_counts:
|
|
1253
|
-
result_text += "**Common concepts**:\n"
|
|
1254
|
-
for concept, count in concept_counts:
|
|
1255
|
-
result_text += f"- {concept} ({count} occurrences)\n"
|
|
1256
|
-
if tool_counts:
|
|
1257
|
-
result_text += "\n**Common tools**:\n"
|
|
1258
|
-
for tool, count in tool_counts:
|
|
1259
|
-
result_text += f"- {tool} ({count} uses)\n"
|
|
1260
|
-
return result_text
|
|
1261
|
-
|
|
1262
|
-
# Continue with normal formatting for full mode
|
|
1263
|
-
if response_format == "xml":
|
|
1264
|
-
# Add upfront summary for immediate visibility (before collapsible XML)
|
|
1265
|
-
upfront_summary = ""
|
|
1266
|
-
|
|
1267
|
-
# Show result summary
|
|
1268
|
-
if all_results:
|
|
1269
|
-
score_info = "high" if all_results[0].score >= 0.85 else "good" if all_results[0].score >= 0.75 else "partial"
|
|
1270
|
-
upfront_summary += f"🎯 RESULTS: {len(all_results)} matches ({score_info} relevance, top score: {all_results[0].score:.3f})\n"
|
|
1271
|
-
|
|
1272
|
-
# Show performance with indexing status inline
|
|
1273
|
-
total_time = time.time() - start_time
|
|
1274
|
-
indexing_info = f" | 📊 {indexing_status['indexed_conversations']}/{indexing_status['total_conversations']} indexed" if indexing_status["percentage"] < 100.0 else ""
|
|
1275
|
-
upfront_summary += f"⚡ PERFORMANCE: {int(total_time * 1000)}ms ({len(collections_to_search)} collections searched{indexing_info})\n"
|
|
1276
|
-
else:
|
|
1277
|
-
upfront_summary += f"❌ NO RESULTS: No conversations found matching '{query}'\n"
|
|
1278
|
-
|
|
1279
|
-
# XML format (compact tags for performance)
|
|
1280
|
-
result_text = upfront_summary + "\n<search>\n"
|
|
1281
|
-
|
|
1282
|
-
# Add indexing status if not fully baselined - put key stats in opening tag for immediate visibility
|
|
1283
|
-
if indexing_status["percentage"] < 95.0:
|
|
1284
|
-
result_text += f' <info status="indexing" progress="{indexing_status["percentage"]:.1f}%" backlog="{indexing_status["backlog_count"]}">\n'
|
|
1285
|
-
result_text += f' <message>📊 Indexing: {indexing_status["indexed_conversations"]}/{indexing_status["total_conversations"]} conversations ({indexing_status["percentage"]:.1f}% complete, {indexing_status["backlog_count"]} pending)</message>\n'
|
|
1286
|
-
result_text += f" </info>\n"
|
|
1287
|
-
|
|
1288
|
-
# Add high-level result summary
|
|
1289
|
-
if all_results:
|
|
1290
|
-
# Count today's results
|
|
1291
|
-
now = datetime.now(timezone.utc)
|
|
1292
|
-
today_count = 0
|
|
1293
|
-
yesterday_count = 0
|
|
1294
|
-
week_count = 0
|
|
1295
|
-
|
|
1296
|
-
for result in all_results:
|
|
1297
|
-
timestamp_clean = result.timestamp.replace('Z', '+00:00') if result.timestamp.endswith('Z') else result.timestamp
|
|
1298
|
-
timestamp_dt = datetime.fromisoformat(timestamp_clean)
|
|
1299
|
-
if timestamp_dt.tzinfo is None:
|
|
1300
|
-
timestamp_dt = timestamp_dt.replace(tzinfo=timezone.utc)
|
|
1301
|
-
|
|
1302
|
-
days_ago = (now - timestamp_dt).days
|
|
1303
|
-
if days_ago == 0:
|
|
1304
|
-
today_count += 1
|
|
1305
|
-
elif days_ago == 1:
|
|
1306
|
-
yesterday_count += 1
|
|
1307
|
-
if days_ago <= 7:
|
|
1308
|
-
week_count += 1
|
|
1309
|
-
|
|
1310
|
-
# Compact summary with key info in opening tag
|
|
1311
|
-
time_info = ""
|
|
1312
|
-
if today_count > 0:
|
|
1313
|
-
time_info = f"{today_count} today"
|
|
1314
|
-
elif yesterday_count > 0:
|
|
1315
|
-
time_info = f"{yesterday_count} yesterday"
|
|
1316
|
-
elif week_count > 0:
|
|
1317
|
-
time_info = f"{week_count} this week"
|
|
1318
|
-
else:
|
|
1319
|
-
time_info = "older results"
|
|
1320
|
-
|
|
1321
|
-
score_info = "high" if all_results[0].score >= 0.85 else "good" if all_results[0].score >= 0.75 else "partial"
|
|
1322
|
-
|
|
1323
|
-
result_text += f' <summary count="{len(all_results)}" relevance="{score_info}" recency="{time_info}" top-score="{all_results[0].score:.3f}">\n'
|
|
1324
|
-
|
|
1325
|
-
# Short preview of top result
|
|
1326
|
-
top_excerpt = all_results[0].excerpt[:100].strip()
|
|
1327
|
-
if '...' not in top_excerpt:
|
|
1328
|
-
top_excerpt += "..."
|
|
1329
|
-
result_text += f' <preview>{top_excerpt}</preview>\n'
|
|
1330
|
-
result_text += f" </summary>\n"
|
|
1331
|
-
else:
|
|
1332
|
-
result_text += f" <result-summary>\n"
|
|
1333
|
-
result_text += f" <headline>No matches found</headline>\n"
|
|
1334
|
-
result_text += f" <relevance>No conversations matched your query</relevance>\n"
|
|
1335
|
-
result_text += f" </result-summary>\n"
|
|
1336
|
-
|
|
1337
|
-
result_text += f" <meta>\n"
|
|
1338
|
-
result_text += f" <q>{query}</q>\n"
|
|
1339
|
-
result_text += f" <scope>{target_project if target_project != 'all' else 'all'}</scope>\n"
|
|
1340
|
-
result_text += f" <count>{len(all_results)}</count>\n"
|
|
1341
|
-
if all_results:
|
|
1342
|
-
result_text += f" <range>{all_results[-1].score:.3f}-{all_results[0].score:.3f}</range>\n"
|
|
1343
|
-
result_text += f" <embed>{'local' if PREFER_LOCAL_EMBEDDINGS or not voyage_client else 'voyage'}</embed>\n"
|
|
1344
|
-
|
|
1345
|
-
# Add timing metadata
|
|
1346
|
-
total_time = time.time() - start_time
|
|
1347
|
-
result_text += f" <perf>\n"
|
|
1348
|
-
result_text += f" <ttl>{int(total_time * 1000)}</ttl>\n"
|
|
1349
|
-
result_text += f" <emb>{int((timing_info.get('embedding_end', 0) - timing_info.get('embedding_start', 0)) * 1000)}</emb>\n"
|
|
1350
|
-
result_text += f" <srch>{int((timing_info.get('search_all_end', 0) - timing_info.get('search_all_start', 0)) * 1000)}</srch>\n"
|
|
1351
|
-
result_text += f" <cols>{len(collections_to_search)}</cols>\n"
|
|
1352
|
-
result_text += f" </perf>\n"
|
|
1353
|
-
result_text += f" </meta>\n"
|
|
1354
|
-
|
|
1355
|
-
result_text += " <results>\n"
|
|
1356
|
-
for i, result in enumerate(all_results):
|
|
1357
|
-
result_text += f' <r rank="{i+1}">\n'
|
|
1358
|
-
result_text += f" <s>{result.score:.3f}</s>\n"
|
|
1359
|
-
result_text += f" <p>{result.project_name}</p>\n"
|
|
1360
|
-
|
|
1361
|
-
# Calculate relative time
|
|
1362
|
-
timestamp_clean = result.timestamp.replace('Z', '+00:00') if result.timestamp.endswith('Z') else result.timestamp
|
|
1363
|
-
timestamp_dt = datetime.fromisoformat(timestamp_clean)
|
|
1364
|
-
# Ensure both datetimes are timezone-aware
|
|
1365
|
-
if timestamp_dt.tzinfo is None:
|
|
1366
|
-
timestamp_dt = timestamp_dt.replace(tzinfo=timezone.utc)
|
|
1367
|
-
now = datetime.now(timezone.utc)
|
|
1368
|
-
days_ago = (now - timestamp_dt).days
|
|
1369
|
-
if days_ago == 0:
|
|
1370
|
-
time_str = "today"
|
|
1371
|
-
elif days_ago == 1:
|
|
1372
|
-
time_str = "yesterday"
|
|
1373
|
-
else:
|
|
1374
|
-
time_str = f"{days_ago}d"
|
|
1375
|
-
result_text += f" <t>{time_str}</t>\n"
|
|
1376
|
-
|
|
1377
|
-
if not brief:
|
|
1378
|
-
# Extract title from first line of excerpt
|
|
1379
|
-
excerpt_lines = result.excerpt.split('\n')
|
|
1380
|
-
title = excerpt_lines[0][:80] + "..." if len(excerpt_lines[0]) > 80 else excerpt_lines[0]
|
|
1381
|
-
result_text += f" <title>{title}</title>\n"
|
|
1382
|
-
|
|
1383
|
-
# Key finding - summarize the main point
|
|
1384
|
-
key_finding = result.excerpt[:100] + "..." if len(result.excerpt) > 100 else result.excerpt
|
|
1385
|
-
result_text += f" <key-finding>{key_finding.strip()}</key-finding>\n"
|
|
1386
|
-
|
|
1387
|
-
# Always include excerpt, but shorter in brief mode
|
|
1388
|
-
if brief:
|
|
1389
|
-
brief_excerpt = result.excerpt[:100] + "..." if len(result.excerpt) > 100 else result.excerpt
|
|
1390
|
-
result_text += f" <excerpt>{brief_excerpt.strip()}</excerpt>\n"
|
|
1391
|
-
else:
|
|
1392
|
-
result_text += f" <excerpt><![CDATA[{result.excerpt}]]></excerpt>\n"
|
|
1393
|
-
|
|
1394
|
-
if result.conversation_id:
|
|
1395
|
-
result_text += f" <cid>{result.conversation_id}</cid>\n"
|
|
1396
|
-
|
|
1397
|
-
# Include raw data if requested
|
|
1398
|
-
if include_raw and result.raw_payload:
|
|
1399
|
-
result_text += " <raw>\n"
|
|
1400
|
-
result_text += f" <txt><![CDATA[{result.raw_payload.get('text', '')}]]></txt>\n"
|
|
1401
|
-
result_text += f" <id>{result.id}</id>\n"
|
|
1402
|
-
result_text += f" <dist>{1 - result.score:.3f}</dist>\n"
|
|
1403
|
-
result_text += " <meta>\n"
|
|
1404
|
-
for key, value in result.raw_payload.items():
|
|
1405
|
-
if key != 'text':
|
|
1406
|
-
result_text += f" <{key}>{value}</{key}>\n"
|
|
1407
|
-
result_text += " </meta>\n"
|
|
1408
|
-
result_text += " </raw>\n"
|
|
1409
|
-
|
|
1410
|
-
# Add patterns if they exist - with detailed logging
|
|
1411
|
-
if result.code_patterns and isinstance(result.code_patterns, dict):
|
|
1412
|
-
logger.info(f"DEBUG: Point {result.id} has code_patterns dict with keys: {list(result.code_patterns.keys())}")
|
|
1413
|
-
patterns_to_show = []
|
|
1414
|
-
for category, patterns in result.code_patterns.items():
|
|
1415
|
-
if patterns and isinstance(patterns, list) and len(patterns) > 0:
|
|
1416
|
-
# Take up to 5 patterns from each category
|
|
1417
|
-
patterns_to_show.append((category, patterns[:5]))
|
|
1418
|
-
logger.info(f"DEBUG: Added category '{category}' with {len(patterns)} patterns")
|
|
1419
|
-
|
|
1420
|
-
if patterns_to_show:
|
|
1421
|
-
logger.info(f"DEBUG: Adding patterns XML for point {result.id}")
|
|
1422
|
-
result_text += " <patterns>\n"
|
|
1423
|
-
for category, patterns in patterns_to_show:
|
|
1424
|
-
# Escape both category name and pattern content for XML safety
|
|
1425
|
-
safe_patterns = ', '.join(escape(str(p)) for p in patterns)
|
|
1426
|
-
result_text += f" <cat name=\"{escape(category)}\">{safe_patterns}</cat>\n"
|
|
1427
|
-
result_text += " </patterns>\n"
|
|
1428
|
-
else:
|
|
1429
|
-
logger.info(f"DEBUG: Point {result.id} has code_patterns but no valid patterns to show")
|
|
1430
|
-
else:
|
|
1431
|
-
logger.info(f"DEBUG: Point {result.id} has no patterns. code_patterns={result.code_patterns}, type={type(result.code_patterns)}")
|
|
1432
|
-
|
|
1433
|
-
if result.files_analyzed and len(result.files_analyzed) > 0:
|
|
1434
|
-
result_text += f" <files>{', '.join(result.files_analyzed[:5])}</files>\n"
|
|
1435
|
-
if result.concepts and len(result.concepts) > 0:
|
|
1436
|
-
result_text += f" <concepts>{', '.join(result.concepts[:5])}</concepts>\n"
|
|
1437
|
-
|
|
1438
|
-
# Include structured metadata for agent consumption
|
|
1439
|
-
# This provides clean, parsed fields that agents can easily use
|
|
1440
|
-
if hasattr(result, 'raw_payload') and result.raw_payload:
|
|
1441
|
-
import json
|
|
1442
|
-
payload = result.raw_payload
|
|
1443
|
-
|
|
1444
|
-
# Files section - structured for easy agent parsing
|
|
1445
|
-
files_analyzed = payload.get('files_analyzed', [])
|
|
1446
|
-
files_edited = payload.get('files_edited', [])
|
|
1447
|
-
if files_analyzed or files_edited:
|
|
1448
|
-
result_text += " <files>\n"
|
|
1449
|
-
if files_analyzed:
|
|
1450
|
-
result_text += f" <analyzed count=\"{len(files_analyzed)}\">"
|
|
1451
|
-
result_text += ", ".join(files_analyzed[:5]) # First 5 files
|
|
1452
|
-
if len(files_analyzed) > 5:
|
|
1453
|
-
result_text += f" ... and {len(files_analyzed)-5} more"
|
|
1454
|
-
result_text += "</analyzed>\n"
|
|
1455
|
-
if files_edited:
|
|
1456
|
-
result_text += f" <edited count=\"{len(files_edited)}\">"
|
|
1457
|
-
result_text += ", ".join(files_edited[:5]) # First 5 files
|
|
1458
|
-
if len(files_edited) > 5:
|
|
1459
|
-
result_text += f" ... and {len(files_edited)-5} more"
|
|
1460
|
-
result_text += "</edited>\n"
|
|
1461
|
-
result_text += " </files>\n"
|
|
1462
|
-
|
|
1463
|
-
# Concepts section - clean list for agents
|
|
1464
|
-
concepts = payload.get('concepts', [])
|
|
1465
|
-
if concepts:
|
|
1466
|
-
result_text += f" <concepts>{', '.join(concepts)}</concepts>\n"
|
|
1467
|
-
|
|
1468
|
-
# Tools section - summarized with counts
|
|
1469
|
-
tools_used = payload.get('tools_used', [])
|
|
1470
|
-
if tools_used:
|
|
1471
|
-
# Count tool usage
|
|
1472
|
-
tool_counts = {}
|
|
1473
|
-
for tool in tools_used:
|
|
1474
|
-
tool_counts[tool] = tool_counts.get(tool, 0) + 1
|
|
1475
|
-
# Sort by frequency
|
|
1476
|
-
sorted_tools = sorted(tool_counts.items(), key=lambda x: x[1], reverse=True)
|
|
1477
|
-
tool_summary = ", ".join(f"{tool}({count})" for tool, count in sorted_tools[:5])
|
|
1478
|
-
if len(sorted_tools) > 5:
|
|
1479
|
-
tool_summary += f" ... and {len(sorted_tools)-5} more"
|
|
1480
|
-
result_text += f" <tools>{tool_summary}</tools>\n"
|
|
1481
|
-
|
|
1482
|
-
# Code patterns section - structured by category
|
|
1483
|
-
code_patterns = payload.get('code_patterns', {})
|
|
1484
|
-
if code_patterns:
|
|
1485
|
-
result_text += " <code_patterns>\n"
|
|
1486
|
-
for category, patterns in code_patterns.items():
|
|
1487
|
-
if patterns:
|
|
1488
|
-
pattern_list = patterns if isinstance(patterns, list) else [patterns]
|
|
1489
|
-
# Clean up pattern names
|
|
1490
|
-
clean_patterns = []
|
|
1491
|
-
for p in pattern_list[:5]:
|
|
1492
|
-
# Remove common prefixes like $FUNC, $VAR
|
|
1493
|
-
clean_p = str(p).replace('$FUNC', '').replace('$VAR', '').strip()
|
|
1494
|
-
if clean_p:
|
|
1495
|
-
clean_patterns.append(clean_p)
|
|
1496
|
-
if clean_patterns:
|
|
1497
|
-
result_text += f" <{category}>{', '.join(clean_patterns)}</{category}>\n"
|
|
1498
|
-
result_text += " </code_patterns>\n"
|
|
1499
|
-
|
|
1500
|
-
# Pattern inheritance info - shows propagation details
|
|
1501
|
-
pattern_inheritance = payload.get('pattern_inheritance', {})
|
|
1502
|
-
if pattern_inheritance:
|
|
1503
|
-
source_chunk = pattern_inheritance.get('source_chunk', '')
|
|
1504
|
-
confidence = pattern_inheritance.get('confidence', 0)
|
|
1505
|
-
distance = pattern_inheritance.get('distance', 0)
|
|
1506
|
-
if source_chunk:
|
|
1507
|
-
result_text += f" <pattern_source chunk=\"{source_chunk}\" confidence=\"{confidence:.2f}\" distance=\"{distance}\"/>\n"
|
|
1508
|
-
|
|
1509
|
-
# Message stats for context
|
|
1510
|
-
msg_count = payload.get('message_count')
|
|
1511
|
-
total_length = payload.get('total_length')
|
|
1512
|
-
if msg_count or total_length:
|
|
1513
|
-
stats_attrs = []
|
|
1514
|
-
if msg_count:
|
|
1515
|
-
stats_attrs.append(f'messages="{msg_count}"')
|
|
1516
|
-
if total_length:
|
|
1517
|
-
stats_attrs.append(f'length="{total_length}"')
|
|
1518
|
-
result_text += f" <stats {' '.join(stats_attrs)}/>\n"
|
|
1519
|
-
|
|
1520
|
-
# Raw metadata dump for backwards compatibility
|
|
1521
|
-
# Kept minimal - only truly unique fields
|
|
1522
|
-
remaining_metadata = {}
|
|
1523
|
-
excluded_keys = {'text', 'conversation_id', 'timestamp', 'role', 'project', 'chunk_index',
|
|
1524
|
-
'files_analyzed', 'files_edited', 'concepts', 'tools_used',
|
|
1525
|
-
'code_patterns', 'pattern_inheritance', 'message_count', 'total_length',
|
|
1526
|
-
'chunking_version', 'chunk_method', 'chunk_overlap', 'migration_type'}
|
|
1527
|
-
for key, value in payload.items():
|
|
1528
|
-
if key not in excluded_keys and value is not None:
|
|
1529
|
-
if isinstance(value, set):
|
|
1530
|
-
value = list(value)
|
|
1531
|
-
remaining_metadata[key] = value
|
|
1532
|
-
|
|
1533
|
-
if remaining_metadata:
|
|
1534
|
-
try:
|
|
1535
|
-
# Only include if there's actually extra data
|
|
1536
|
-
result_text += f" <metadata_extra><![CDATA[{json.dumps(remaining_metadata, default=str)}]]></metadata_extra>\n"
|
|
1537
|
-
except:
|
|
1538
|
-
pass
|
|
1539
|
-
|
|
1540
|
-
result_text += " </r>\n"
|
|
1541
|
-
result_text += " </results>\n"
|
|
1542
|
-
|
|
1543
|
-
# Add aggregated pattern intelligence section
|
|
1544
|
-
if pattern_intelligence and pattern_intelligence.get('total_unique_patterns', 0) > 0:
|
|
1545
|
-
result_text += " <pattern_intelligence>\n"
|
|
1546
|
-
|
|
1547
|
-
# Summary statistics
|
|
1548
|
-
result_text += f" <summary>\n"
|
|
1549
|
-
result_text += f" <unique_patterns>{pattern_intelligence['total_unique_patterns']}</unique_patterns>\n"
|
|
1550
|
-
result_text += f" <pattern_diversity>{pattern_intelligence['pattern_diversity_score']:.2f}</pattern_diversity>\n"
|
|
1551
|
-
result_text += f" </summary>\n"
|
|
1552
|
-
|
|
1553
|
-
# Most common patterns
|
|
1554
|
-
if pattern_intelligence.get('most_common_patterns'):
|
|
1555
|
-
result_text += " <common_patterns>\n"
|
|
1556
|
-
for pattern, count in pattern_intelligence['most_common_patterns'][:5]:
|
|
1557
|
-
result_text += f" <pattern count=\"{count}\">{pattern}</pattern>\n"
|
|
1558
|
-
result_text += " </common_patterns>\n"
|
|
1559
|
-
|
|
1560
|
-
# Pattern categories
|
|
1561
|
-
if pattern_intelligence.get('category_coverage'):
|
|
1562
|
-
result_text += " <categories>\n"
|
|
1563
|
-
for category, count in pattern_intelligence['category_coverage'].items():
|
|
1564
|
-
result_text += f" <cat name=\"{category}\" count=\"{count}\"/>\n"
|
|
1565
|
-
result_text += " </categories>\n"
|
|
1566
|
-
|
|
1567
|
-
# Pattern combinations insight
|
|
1568
|
-
if pattern_intelligence.get('pattern_combinations'):
|
|
1569
|
-
combos = pattern_intelligence['pattern_combinations']
|
|
1570
|
-
if combos.get('async_with_error_handling'):
|
|
1571
|
-
result_text += " <insight>Async patterns combined with error handling detected</insight>\n"
|
|
1572
|
-
if combos.get('react_with_state'):
|
|
1573
|
-
result_text += " <insight>React hooks with state management patterns detected</insight>\n"
|
|
1574
|
-
|
|
1575
|
-
# Files referenced across results
|
|
1576
|
-
if pattern_intelligence.get('files_referenced') and len(pattern_intelligence['files_referenced']) > 0:
|
|
1577
|
-
result_text += f" <files_across_results>{', '.join(pattern_intelligence['files_referenced'][:10])}</files_across_results>\n"
|
|
1578
|
-
|
|
1579
|
-
# Concepts discussed
|
|
1580
|
-
if pattern_intelligence.get('concepts_discussed') and len(pattern_intelligence['concepts_discussed']) > 0:
|
|
1581
|
-
result_text += f" <concepts_discussed>{', '.join(pattern_intelligence['concepts_discussed'][:10])}</concepts_discussed>\n"
|
|
1582
|
-
|
|
1583
|
-
result_text += " </pattern_intelligence>\n"
|
|
1584
|
-
|
|
1585
|
-
result_text += "</search>"
|
|
1586
|
-
|
|
1587
|
-
else:
|
|
1588
|
-
# Markdown format (original)
|
|
1589
|
-
result_text = f"Found {len(all_results)} relevant conversation(s) for '{query}':\n\n"
|
|
1590
|
-
for i, result in enumerate(all_results):
|
|
1591
|
-
result_text += f"**Result {i+1}** (Score: {result.score:.3f})\n"
|
|
1592
|
-
# Handle timezone suffix 'Z' properly
|
|
1593
|
-
timestamp_clean = result.timestamp.replace('Z', '+00:00') if result.timestamp.endswith('Z') else result.timestamp
|
|
1594
|
-
result_text += f"Time: {datetime.fromisoformat(timestamp_clean).strftime('%Y-%m-%d %H:%M:%S')}\n"
|
|
1595
|
-
result_text += f"Project: {result.project_name}\n"
|
|
1596
|
-
result_text += f"Role: {result.role}\n"
|
|
1597
|
-
result_text += f"Excerpt: {result.excerpt}\n"
|
|
1598
|
-
result_text += "---\n\n"
|
|
1599
|
-
|
|
1600
|
-
timing_info['format_end'] = time.time()
|
|
1601
|
-
|
|
1602
|
-
# Log detailed timing breakdown
|
|
1603
|
-
await ctx.debug(f"\n=== TIMING BREAKDOWN ===")
|
|
1604
|
-
await ctx.debug(f"Total time: {(time.time() - start_time) * 1000:.1f}ms")
|
|
1605
|
-
await ctx.debug(f"Embedding generation: {(timing_info.get('embedding_end', 0) - timing_info.get('embedding_start', 0)) * 1000:.1f}ms")
|
|
1606
|
-
await ctx.debug(f"Get collections: {(timing_info.get('get_collections_end', 0) - timing_info.get('get_collections_start', 0)) * 1000:.1f}ms")
|
|
1607
|
-
await ctx.debug(f"Search all collections: {(timing_info.get('search_all_end', 0) - timing_info.get('search_all_start', 0)) * 1000:.1f}ms")
|
|
1608
|
-
await ctx.debug(f"Sorting results: {(timing_info.get('sort_end', 0) - timing_info.get('sort_start', 0)) * 1000:.1f}ms")
|
|
1609
|
-
await ctx.debug(f"Formatting output: {(timing_info.get('format_end', 0) - timing_info.get('format_start', 0)) * 1000:.1f}ms")
|
|
1610
|
-
|
|
1611
|
-
# Log per-collection timings
|
|
1612
|
-
await ctx.debug(f"\n=== PER-COLLECTION TIMINGS ===")
|
|
1613
|
-
for ct in collection_timings:
|
|
1614
|
-
duration = (ct.get('end', 0) - ct.get('start', 0)) * 1000
|
|
1615
|
-
status = "ERROR" if 'error' in ct else "OK"
|
|
1616
|
-
await ctx.debug(f"{ct['name']}: {duration:.1f}ms ({status})")
|
|
1617
|
-
|
|
1618
|
-
return result_text
|
|
1619
|
-
|
|
1620
|
-
except Exception as e:
|
|
1621
|
-
await ctx.error(f"Search failed: {str(e)}")
|
|
1622
|
-
return f"Failed to search conversations: {str(e)}"
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
@mcp.tool()
|
|
1626
|
-
async def store_reflection(
|
|
1627
|
-
ctx: Context,
|
|
1628
|
-
content: str = Field(description="The insight or reflection to store"),
|
|
1629
|
-
tags: List[str] = Field(default=[], description="Tags to categorize this reflection")
|
|
1630
|
-
) -> str:
|
|
1631
|
-
"""Store an important insight or reflection for future reference."""
|
|
1632
|
-
|
|
1633
|
-
try:
|
|
1634
|
-
# Create reflections collection name
|
|
1635
|
-
collection_name = f"reflections{get_collection_suffix()}"
|
|
1636
|
-
|
|
1637
|
-
# Get current project context
|
|
1638
|
-
cwd = os.environ.get('MCP_CLIENT_CWD', os.getcwd())
|
|
1639
|
-
project_path = Path(cwd)
|
|
1640
|
-
|
|
1641
|
-
# Extract project name from path
|
|
1642
|
-
project_name = None
|
|
1643
|
-
path_parts = project_path.parts
|
|
1644
|
-
if 'projects' in path_parts:
|
|
1645
|
-
idx = path_parts.index('projects')
|
|
1646
|
-
if idx + 1 < len(path_parts):
|
|
1647
|
-
# Get all parts after 'projects' to form the project name
|
|
1648
|
-
# This handles cases like projects/Connectiva-App/connectiva-ai
|
|
1649
|
-
project_parts = path_parts[idx + 1:]
|
|
1650
|
-
project_name = '/'.join(project_parts)
|
|
1651
|
-
|
|
1652
|
-
# If no project detected, use the last directory name
|
|
1653
|
-
if not project_name:
|
|
1654
|
-
project_name = project_path.name
|
|
1655
|
-
|
|
1656
|
-
# Ensure collection exists
|
|
1657
|
-
try:
|
|
1658
|
-
collection_info = await qdrant_client.get_collection(collection_name)
|
|
1659
|
-
except:
|
|
1660
|
-
# Create collection if it doesn't exist
|
|
1661
|
-
await qdrant_client.create_collection(
|
|
1662
|
-
collection_name=collection_name,
|
|
1663
|
-
vectors_config=VectorParams(
|
|
1664
|
-
size=get_embedding_dimension(),
|
|
1665
|
-
distance=Distance.COSINE
|
|
1666
|
-
)
|
|
1667
|
-
)
|
|
1668
|
-
await ctx.debug(f"Created reflections collection: {collection_name}")
|
|
1669
|
-
|
|
1670
|
-
# Generate embedding for the reflection
|
|
1671
|
-
embedding = await generate_embedding(content)
|
|
1672
|
-
|
|
1673
|
-
# Create point with metadata including project context
|
|
1674
|
-
point_id = datetime.now().timestamp()
|
|
1675
|
-
point = PointStruct(
|
|
1676
|
-
id=int(point_id),
|
|
1677
|
-
vector=embedding,
|
|
1678
|
-
payload={
|
|
1679
|
-
"text": content,
|
|
1680
|
-
"tags": tags,
|
|
1681
|
-
"timestamp": datetime.now().isoformat(),
|
|
1682
|
-
"type": "reflection",
|
|
1683
|
-
"role": "user_reflection",
|
|
1684
|
-
"project": project_name, # Add project context
|
|
1685
|
-
"project_path": str(project_path) # Add full path for reference
|
|
1686
|
-
}
|
|
1687
|
-
)
|
|
1688
|
-
|
|
1689
|
-
# Store in Qdrant
|
|
1690
|
-
await qdrant_client.upsert(
|
|
1691
|
-
collection_name=collection_name,
|
|
1692
|
-
points=[point]
|
|
1693
|
-
)
|
|
1694
|
-
|
|
1695
|
-
tags_str = ', '.join(tags) if tags else 'none'
|
|
1696
|
-
return f"Reflection stored successfully with tags: {tags_str}"
|
|
1697
|
-
|
|
1698
|
-
except Exception as e:
|
|
1699
|
-
await ctx.error(f"Store failed: {str(e)}")
|
|
1700
|
-
return f"Failed to store reflection: {str(e)}"
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
@mcp.tool()
|
|
1704
|
-
async def quick_search(
|
|
1705
|
-
ctx: Context,
|
|
1706
|
-
query: str = Field(description="The search query to find semantically similar conversations"),
|
|
1707
|
-
min_score: float = Field(default=0.3, description="Minimum similarity score (0-1)"),
|
|
1708
|
-
project: Optional[str] = Field(default=None, description="Search specific project only. If not provided, searches current project based on working directory. Use 'all' to search across all projects.")
|
|
1709
|
-
) -> str:
|
|
1710
|
-
"""Quick search that returns only the count and top result for fast overview."""
|
|
1711
|
-
# MCP architectural limitation: MCP tools cannot call other MCP tools
|
|
1712
|
-
return """<error>
|
|
1713
|
-
MCP Architectural Limitation: This tool cannot directly call other MCP tools.
|
|
1714
652
|
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
min_score: float = Field(default=0.3, description="Minimum similarity score (0-1)"),
|
|
1754
|
-
project: Optional[str] = Field(default=None, description="Search specific project only")
|
|
1755
|
-
) -> str:
|
|
1756
|
-
"""Get additional search results after an initial search (pagination support)."""
|
|
1757
|
-
# MCP architectural limitation: MCP tools cannot call other MCP tools
|
|
1758
|
-
return """<error>
|
|
1759
|
-
MCP Architectural Limitation: This tool cannot directly call other MCP tools.
|
|
1760
|
-
|
|
1761
|
-
To get more search results, please:
|
|
1762
|
-
1. Call reflect_on_past directly with a higher limit parameter
|
|
1763
|
-
2. Or use the reflection-specialist agent to handle pagination
|
|
1764
|
-
|
|
1765
|
-
This limitation exists because MCP tools can only be orchestrated by the client (Claude),
|
|
1766
|
-
not by other tools within the MCP server.
|
|
1767
|
-
</error>"""
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
@mcp.tool()
|
|
1771
|
-
async def search_by_file(
|
|
1772
|
-
ctx: Context,
|
|
1773
|
-
file_path: str = Field(description="The file path to search for in conversations"),
|
|
1774
|
-
limit: int = Field(default=10, description="Maximum number of results to return"),
|
|
1775
|
-
project: Optional[str] = Field(default=None, description="Search specific project only. Use 'all' to search across all projects.")
|
|
1776
|
-
) -> str:
|
|
1777
|
-
"""Search for conversations that analyzed a specific file."""
|
|
1778
|
-
global qdrant_client
|
|
1779
|
-
|
|
1780
|
-
# Normalize file path
|
|
1781
|
-
normalized_path = file_path.replace("\\", "/").replace("/Users/", "~/")
|
|
1782
|
-
|
|
1783
|
-
# Determine which collections to search
|
|
1784
|
-
# If no project specified, search all collections
|
|
1785
|
-
collections = await get_all_collections() if not project else []
|
|
1786
|
-
|
|
1787
|
-
if project and project != 'all':
|
|
1788
|
-
# Filter collections for specific project - normalize first!
|
|
1789
|
-
normalized_project = normalize_project_name(project)
|
|
1790
|
-
project_hash = hashlib.md5(normalized_project.encode()).hexdigest()[:8]
|
|
1791
|
-
collection_prefix = f"conv_{project_hash}_"
|
|
1792
|
-
collections = [c for c in await get_all_collections() if c.startswith(collection_prefix)]
|
|
1793
|
-
elif project == 'all':
|
|
1794
|
-
collections = await get_all_collections()
|
|
1795
|
-
|
|
1796
|
-
if not collections:
|
|
1797
|
-
return "<search_by_file>\n<error>No collections found to search</error>\n</search_by_file>"
|
|
1798
|
-
|
|
1799
|
-
# Prepare results
|
|
1800
|
-
all_results = []
|
|
1801
|
-
|
|
1802
|
-
for collection_name in collections:
|
|
1803
|
-
try:
|
|
1804
|
-
# Use scroll to get all points and filter manually
|
|
1805
|
-
# Qdrant's array filtering can be tricky, so we'll filter in code
|
|
1806
|
-
scroll_result = await qdrant_client.scroll(
|
|
1807
|
-
collection_name=collection_name,
|
|
1808
|
-
limit=1000, # Get a batch
|
|
1809
|
-
with_payload=True
|
|
1810
|
-
)
|
|
1811
|
-
|
|
1812
|
-
# Filter results that contain the file
|
|
1813
|
-
for point in scroll_result[0]:
|
|
1814
|
-
payload = point.payload
|
|
1815
|
-
files_analyzed = payload.get('files_analyzed', [])
|
|
1816
|
-
files_edited = payload.get('files_edited', [])
|
|
1817
|
-
|
|
1818
|
-
# Check for exact match or if any file ends with the normalized path
|
|
1819
|
-
file_match = False
|
|
1820
|
-
for file in files_analyzed + files_edited:
|
|
1821
|
-
if file == normalized_path or file.endswith('/' + normalized_path) or file.endswith('\\' + normalized_path):
|
|
1822
|
-
file_match = True
|
|
1823
|
-
break
|
|
1824
|
-
|
|
1825
|
-
if file_match:
|
|
1826
|
-
all_results.append({
|
|
1827
|
-
'score': 1.0, # File match is always 1.0
|
|
1828
|
-
'payload': payload,
|
|
1829
|
-
'collection': collection_name
|
|
1830
|
-
})
|
|
1831
|
-
|
|
1832
|
-
except Exception as e:
|
|
1833
|
-
continue
|
|
1834
|
-
|
|
1835
|
-
# Sort by timestamp (newest first)
|
|
1836
|
-
all_results.sort(key=lambda x: x['payload'].get('timestamp', ''), reverse=True)
|
|
1837
|
-
|
|
1838
|
-
# Format results
|
|
1839
|
-
if not all_results:
|
|
1840
|
-
return f"""<search_by_file>
|
|
1841
|
-
<query>{file_path}</query>
|
|
1842
|
-
<normalized_path>{normalized_path}</normalized_path>
|
|
1843
|
-
<message>No conversations found that analyzed this file</message>
|
|
1844
|
-
</search_by_file>"""
|
|
1845
|
-
|
|
1846
|
-
results_text = []
|
|
1847
|
-
for i, result in enumerate(all_results[:limit]):
|
|
1848
|
-
payload = result['payload']
|
|
1849
|
-
timestamp = payload.get('timestamp', 'Unknown')
|
|
1850
|
-
conversation_id = payload.get('conversation_id', 'Unknown')
|
|
1851
|
-
project = payload.get('project', 'Unknown')
|
|
1852
|
-
text_preview = payload.get('text', '')[:200] + '...' if len(payload.get('text', '')) > 200 else payload.get('text', '')
|
|
1853
|
-
|
|
1854
|
-
# Check if file was edited or just read
|
|
1855
|
-
action = "edited" if normalized_path in payload.get('files_edited', []) else "analyzed"
|
|
1856
|
-
|
|
1857
|
-
# Get related tools used
|
|
1858
|
-
tool_summary = payload.get('tool_summary', {})
|
|
1859
|
-
tools_used = ', '.join(f"{tool}({count})" for tool, count in tool_summary.items())
|
|
1860
|
-
|
|
1861
|
-
results_text.append(f"""<result rank="{i+1}">
|
|
1862
|
-
<conversation_id>{conversation_id}</conversation_id>
|
|
1863
|
-
<project>{project}</project>
|
|
1864
|
-
<timestamp>{timestamp}</timestamp>
|
|
1865
|
-
<action>{action}</action>
|
|
1866
|
-
<tools_used>{tools_used}</tools_used>
|
|
1867
|
-
<preview>{text_preview}</preview>
|
|
1868
|
-
</result>""")
|
|
1869
|
-
|
|
1870
|
-
return f"""<search_by_file>
|
|
1871
|
-
<query>{file_path}</query>
|
|
1872
|
-
<normalized_path>{normalized_path}</normalized_path>
|
|
1873
|
-
<count>{len(all_results)}</count>
|
|
1874
|
-
<results>
|
|
1875
|
-
{''.join(results_text)}
|
|
1876
|
-
</results>
|
|
1877
|
-
</search_by_file>"""
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
@mcp.tool()
|
|
1881
|
-
async def search_by_concept(
|
|
1882
|
-
ctx: Context,
|
|
1883
|
-
concept: str = Field(description="The concept to search for (e.g., 'security', 'docker', 'testing')"),
|
|
1884
|
-
include_files: bool = Field(default=True, description="Include file information in results"),
|
|
1885
|
-
limit: int = Field(default=10, description="Maximum number of results to return"),
|
|
1886
|
-
project: Optional[str] = Field(default=None, description="Search specific project only. Use 'all' to search across all projects.")
|
|
1887
|
-
) -> str:
|
|
1888
|
-
"""Search for conversations about a specific development concept."""
|
|
1889
|
-
global qdrant_client
|
|
1890
|
-
|
|
1891
|
-
# Generate embedding for the concept
|
|
1892
|
-
embedding = await generate_embedding(concept)
|
|
1893
|
-
|
|
1894
|
-
# Determine which collections to search
|
|
1895
|
-
# If no project specified, search all collections
|
|
1896
|
-
collections = await get_all_collections() if not project else []
|
|
1897
|
-
|
|
1898
|
-
if project and project != 'all':
|
|
1899
|
-
# Filter collections for specific project
|
|
1900
|
-
normalized_project = normalize_project_name(project)
|
|
1901
|
-
project_hash = hashlib.md5(normalized_project.encode()).hexdigest()[:8]
|
|
1902
|
-
collection_prefix = f"conv_{project_hash}_"
|
|
1903
|
-
collections = [c for c in await get_all_collections() if c.startswith(collection_prefix)]
|
|
1904
|
-
elif project == 'all':
|
|
1905
|
-
collections = await get_all_collections()
|
|
1906
|
-
|
|
1907
|
-
if not collections:
|
|
1908
|
-
return "<search_by_concept>\n<error>No collections found to search</error>\n</search_by_concept>"
|
|
1909
|
-
|
|
1910
|
-
# First, check metadata health
|
|
1911
|
-
metadata_found = False
|
|
1912
|
-
total_points_checked = 0
|
|
1913
|
-
|
|
1914
|
-
for collection_name in collections[:3]: # Sample first 3 collections
|
|
1915
|
-
try:
|
|
1916
|
-
sample_points, _ = await qdrant_client.scroll(
|
|
1917
|
-
collection_name=collection_name,
|
|
1918
|
-
limit=10,
|
|
1919
|
-
with_payload=True
|
|
1920
|
-
)
|
|
1921
|
-
total_points_checked += len(sample_points)
|
|
1922
|
-
for point in sample_points:
|
|
1923
|
-
if 'concepts' in point.payload and point.payload['concepts']:
|
|
1924
|
-
metadata_found = True
|
|
1925
|
-
break
|
|
1926
|
-
if metadata_found:
|
|
1927
|
-
break
|
|
1928
|
-
except:
|
|
1929
|
-
continue
|
|
1930
|
-
|
|
1931
|
-
# Search all collections
|
|
1932
|
-
all_results = []
|
|
1933
|
-
|
|
1934
|
-
# If metadata exists, try metadata-based search first
|
|
1935
|
-
if metadata_found:
|
|
1936
|
-
for collection_name in collections:
|
|
1937
|
-
try:
|
|
1938
|
-
# Hybrid search: semantic + concept filter
|
|
1939
|
-
results = await qdrant_client.search(
|
|
1940
|
-
collection_name=collection_name,
|
|
1941
|
-
query_vector=embedding,
|
|
1942
|
-
query_filter=models.Filter(
|
|
1943
|
-
should=[
|
|
1944
|
-
models.FieldCondition(
|
|
1945
|
-
key="concepts",
|
|
1946
|
-
match=models.MatchAny(any=[concept.lower()])
|
|
1947
|
-
)
|
|
1948
|
-
]
|
|
1949
|
-
),
|
|
1950
|
-
limit=limit * 2, # Get more results for better filtering
|
|
1951
|
-
with_payload=True
|
|
1952
|
-
)
|
|
1953
|
-
|
|
1954
|
-
for point in results:
|
|
1955
|
-
payload = point.payload
|
|
1956
|
-
# Boost score if concept is in the concepts list
|
|
1957
|
-
score_boost = 0.2 if concept.lower() in payload.get('concepts', []) else 0.0
|
|
1958
|
-
all_results.append({
|
|
1959
|
-
'score': float(point.score) + score_boost,
|
|
1960
|
-
'payload': payload,
|
|
1961
|
-
'collection': collection_name,
|
|
1962
|
-
'search_type': 'metadata'
|
|
1963
|
-
})
|
|
1964
|
-
|
|
1965
|
-
except Exception as e:
|
|
1966
|
-
# Log unexpected errors but continue with other collections
|
|
1967
|
-
logger.debug(f"Error searching collection {collection_name}: {e}")
|
|
1968
|
-
continue
|
|
1969
|
-
|
|
1970
|
-
# If no results from metadata search OR no metadata exists, fall back to semantic search
|
|
1971
|
-
if not all_results:
|
|
1972
|
-
await ctx.debug(f"Falling back to semantic search for concept: {concept}")
|
|
1973
|
-
|
|
1974
|
-
for collection_name in collections:
|
|
1975
|
-
try:
|
|
1976
|
-
# Pure semantic search without filters
|
|
1977
|
-
results = await qdrant_client.search(
|
|
1978
|
-
collection_name=collection_name,
|
|
1979
|
-
query_vector=embedding,
|
|
1980
|
-
limit=limit,
|
|
1981
|
-
score_threshold=0.5, # Lower threshold for broader results
|
|
1982
|
-
with_payload=True
|
|
1983
|
-
)
|
|
1984
|
-
|
|
1985
|
-
for point in results:
|
|
1986
|
-
all_results.append({
|
|
1987
|
-
'score': float(point.score),
|
|
1988
|
-
'payload': point.payload,
|
|
1989
|
-
'collection': collection_name,
|
|
1990
|
-
'search_type': 'semantic'
|
|
1991
|
-
})
|
|
1992
|
-
|
|
1993
|
-
except Exception as e:
|
|
1994
|
-
# Log unexpected errors but continue with other collections
|
|
1995
|
-
logger.debug(f"Error searching collection {collection_name}: {e}")
|
|
1996
|
-
continue
|
|
1997
|
-
|
|
1998
|
-
# Sort by score and limit
|
|
1999
|
-
all_results.sort(key=lambda x: x['score'], reverse=True)
|
|
2000
|
-
all_results = all_results[:limit]
|
|
2001
|
-
|
|
2002
|
-
# Format results
|
|
2003
|
-
if not all_results:
|
|
2004
|
-
metadata_status = "with metadata" if metadata_found else "NO METADATA FOUND"
|
|
2005
|
-
return f"""<search_by_concept>
|
|
2006
|
-
<concept>{concept}</concept>
|
|
2007
|
-
<metadata_health>{metadata_status} (checked {total_points_checked} points)</metadata_health>
|
|
2008
|
-
<message>No conversations found about this concept. {'Try running: python scripts/delta-metadata-update.py' if not metadata_found else 'Try different search terms.'}</message>
|
|
2009
|
-
</search_by_concept>"""
|
|
2010
|
-
|
|
2011
|
-
results_text = []
|
|
2012
|
-
for i, result in enumerate(all_results):
|
|
2013
|
-
payload = result['payload']
|
|
2014
|
-
score = result['score']
|
|
2015
|
-
timestamp = payload.get('timestamp', 'Unknown')
|
|
2016
|
-
conversation_id = payload.get('conversation_id', 'Unknown')
|
|
2017
|
-
project = payload.get('project', 'Unknown')
|
|
2018
|
-
concepts = payload.get('concepts', [])
|
|
2019
|
-
|
|
2020
|
-
# Get text preview
|
|
2021
|
-
text_preview = payload.get('text', '')[:200] + '...' if len(payload.get('text', '')) > 200 else payload.get('text', '')
|
|
2022
|
-
|
|
2023
|
-
# File information
|
|
2024
|
-
files_info = ""
|
|
2025
|
-
if include_files:
|
|
2026
|
-
files_analyzed = payload.get('files_analyzed', [])[:5]
|
|
2027
|
-
if files_analyzed:
|
|
2028
|
-
files_info = f"\n<files_analyzed>{', '.join(files_analyzed)}</files_analyzed>"
|
|
2029
|
-
|
|
2030
|
-
# Related concepts
|
|
2031
|
-
related_concepts = [c for c in concepts if c != concept.lower()][:5]
|
|
2032
|
-
|
|
2033
|
-
results_text.append(f"""<result rank="{i+1}">
|
|
2034
|
-
<score>{score:.3f}</score>
|
|
2035
|
-
<conversation_id>{conversation_id}</conversation_id>
|
|
2036
|
-
<project>{project}</project>
|
|
2037
|
-
<timestamp>{timestamp}</timestamp>
|
|
2038
|
-
<concepts>{', '.join(concepts)}</concepts>
|
|
2039
|
-
<related_concepts>{', '.join(related_concepts)}</related_concepts>{files_info}
|
|
2040
|
-
<preview>{text_preview}</preview>
|
|
2041
|
-
</result>""")
|
|
2042
|
-
|
|
2043
|
-
# Determine if this was a fallback search
|
|
2044
|
-
used_fallback = any(r.get('search_type') == 'semantic' for r in all_results)
|
|
2045
|
-
metadata_status = "with metadata" if metadata_found else "NO METADATA FOUND"
|
|
2046
|
-
|
|
2047
|
-
return f"""<search_by_concept>
|
|
2048
|
-
<concept>{concept}</concept>
|
|
2049
|
-
<metadata_health>{metadata_status} (checked {total_points_checked} points)</metadata_health>
|
|
2050
|
-
<search_type>{'fallback_semantic' if used_fallback else 'metadata_based'}</search_type>
|
|
2051
|
-
<count>{len(all_results)}</count>
|
|
2052
|
-
<results>
|
|
2053
|
-
{''.join(results_text)}
|
|
2054
|
-
</results>
|
|
2055
|
-
</search_by_concept>"""
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
# Debug output
|
|
2059
|
-
print(f"[DEBUG] FastMCP server created with name: {mcp.name}")
|
|
2060
|
-
|
|
2061
|
-
@mcp.tool()
|
|
2062
|
-
async def get_full_conversation(
|
|
2063
|
-
ctx: Context,
|
|
2064
|
-
conversation_id: str = Field(description="The conversation ID from search results (cid)"),
|
|
2065
|
-
project: Optional[str] = Field(default=None, description="Optional project name to help locate the file")
|
|
2066
|
-
) -> str:
|
|
2067
|
-
"""Get the full JSONL conversation file path for a conversation ID.
|
|
2068
|
-
This allows agents to read complete conversations instead of truncated excerpts."""
|
|
2069
|
-
|
|
2070
|
-
# Base path for Claude conversations
|
|
2071
|
-
base_path = Path.home() / '.claude/projects'
|
|
2072
|
-
|
|
2073
|
-
# Build list of directories to search
|
|
2074
|
-
search_dirs = []
|
|
2075
|
-
|
|
2076
|
-
if project:
|
|
2077
|
-
# Try various project directory name formats
|
|
2078
|
-
sanitized_project = project.replace('/', '-')
|
|
2079
|
-
search_dirs.extend([
|
|
2080
|
-
base_path / project,
|
|
2081
|
-
base_path / sanitized_project,
|
|
2082
|
-
base_path / f"-Users-*-projects-{project}",
|
|
2083
|
-
base_path / f"-Users-*-projects-{sanitized_project}"
|
|
2084
|
-
])
|
|
2085
|
-
else:
|
|
2086
|
-
# Search all project directories
|
|
2087
|
-
search_dirs = list(base_path.glob("*"))
|
|
2088
|
-
|
|
2089
|
-
# Search for the JSONL file
|
|
2090
|
-
jsonl_path = None
|
|
2091
|
-
for search_dir in search_dirs:
|
|
2092
|
-
if not search_dir.is_dir():
|
|
2093
|
-
continue
|
|
2094
|
-
|
|
2095
|
-
potential_path = search_dir / f"{conversation_id}.jsonl"
|
|
2096
|
-
if potential_path.exists():
|
|
2097
|
-
jsonl_path = potential_path
|
|
2098
|
-
break
|
|
2099
|
-
|
|
2100
|
-
if not jsonl_path:
|
|
2101
|
-
# Try searching all directories as fallback
|
|
2102
|
-
for proj_dir in base_path.glob("*"):
|
|
2103
|
-
if proj_dir.is_dir():
|
|
2104
|
-
potential_path = proj_dir / f"{conversation_id}.jsonl"
|
|
2105
|
-
if potential_path.exists():
|
|
2106
|
-
jsonl_path = potential_path
|
|
2107
|
-
break
|
|
2108
|
-
|
|
2109
|
-
if not jsonl_path:
|
|
2110
|
-
return f"""<full_conversation>
|
|
2111
|
-
<conversation_id>{conversation_id}</conversation_id>
|
|
2112
|
-
<status>not_found</status>
|
|
2113
|
-
<message>Conversation file not found. Searched {len(search_dirs)} directories.</message>
|
|
2114
|
-
<hint>Try using the project parameter or check if the conversation ID is correct.</hint>
|
|
2115
|
-
</full_conversation>"""
|
|
2116
|
-
|
|
2117
|
-
# Get file stats
|
|
2118
|
-
file_stats = jsonl_path.stat()
|
|
2119
|
-
|
|
2120
|
-
# Count messages
|
|
2121
|
-
try:
|
|
2122
|
-
with open(jsonl_path, 'r', encoding='utf-8') as f:
|
|
2123
|
-
message_count = sum(1 for _ in f)
|
|
2124
|
-
except:
|
|
2125
|
-
message_count = 0
|
|
2126
|
-
|
|
2127
|
-
return f"""<full_conversation>
|
|
2128
|
-
<conversation_id>{conversation_id}</conversation_id>
|
|
2129
|
-
<status>found</status>
|
|
2130
|
-
<file_path>{jsonl_path}</file_path>
|
|
2131
|
-
<file_size>{file_stats.st_size}</file_size>
|
|
2132
|
-
<message_count>{message_count}</message_count>
|
|
2133
|
-
<project>{jsonl_path.parent.name}</project>
|
|
2134
|
-
<instructions>
|
|
2135
|
-
You can now use the Read tool to read the full conversation from:
|
|
2136
|
-
{jsonl_path}
|
|
2137
|
-
|
|
2138
|
-
Each line in the JSONL file is a separate message with complete content.
|
|
2139
|
-
This gives you access to:
|
|
2140
|
-
- Complete code blocks (not truncated)
|
|
2141
|
-
- Full problem descriptions and solutions
|
|
2142
|
-
- Entire debugging sessions
|
|
2143
|
-
- Complete architectural decisions and discussions
|
|
2144
|
-
</instructions>
|
|
2145
|
-
</full_conversation>"""
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
@mcp.tool()
|
|
2149
|
-
async def get_next_results(
|
|
2150
|
-
ctx: Context,
|
|
2151
|
-
query: str = Field(description="The original search query"),
|
|
2152
|
-
offset: int = Field(default=3, description="Number of results to skip (for pagination)"),
|
|
2153
|
-
limit: int = Field(default=3, description="Number of additional results to return"),
|
|
2154
|
-
min_score: float = Field(default=0.3, description="Minimum similarity score (0-1)"),
|
|
2155
|
-
project: Optional[str] = Field(default=None, description="Search specific project only")
|
|
2156
|
-
) -> str:
|
|
2157
|
-
"""Get additional search results after an initial search (pagination support)."""
|
|
2158
|
-
global qdrant_client, embedding_manager
|
|
2159
|
-
|
|
2160
|
-
try:
|
|
2161
|
-
# Generate embedding for the query
|
|
2162
|
-
embedding = await generate_embedding(query)
|
|
2163
|
-
|
|
2164
|
-
# Determine which collections to search
|
|
2165
|
-
if project == "all" or not project:
|
|
2166
|
-
# Search all collections if project is "all" or not specified
|
|
2167
|
-
collections = await get_all_collections()
|
|
2168
|
-
else:
|
|
2169
|
-
# Search specific project - normalize first!
|
|
2170
|
-
all_collections = await get_all_collections()
|
|
2171
|
-
normalized_project = normalize_project_name(project)
|
|
2172
|
-
project_hash = hashlib.md5(normalized_project.encode()).hexdigest()[:8]
|
|
2173
|
-
collections = [
|
|
2174
|
-
c for c in all_collections
|
|
2175
|
-
if c.startswith(f"conv_{project_hash}_")
|
|
2176
|
-
]
|
|
2177
|
-
if not collections:
|
|
2178
|
-
# Fall back to searching all collections
|
|
2179
|
-
collections = all_collections
|
|
2180
|
-
|
|
2181
|
-
if not collections:
|
|
2182
|
-
return """<next_results>
|
|
2183
|
-
<error>No collections available to search</error>
|
|
2184
|
-
</next_results>"""
|
|
2185
|
-
|
|
2186
|
-
# Collect all results from all collections
|
|
2187
|
-
all_results = []
|
|
2188
|
-
for collection_name in collections:
|
|
2189
|
-
try:
|
|
2190
|
-
# Check if collection exists
|
|
2191
|
-
collection_info = await qdrant_client.get_collection(collection_name)
|
|
2192
|
-
if not collection_info:
|
|
2193
|
-
continue
|
|
2194
|
-
|
|
2195
|
-
# Search with reasonable limit to account for offset
|
|
2196
|
-
max_search_limit = 100 # Define a reasonable cap
|
|
2197
|
-
search_limit = min(offset + limit + 10, max_search_limit)
|
|
2198
|
-
results = await qdrant_client.search(
|
|
2199
|
-
collection_name=collection_name,
|
|
2200
|
-
query_vector=embedding,
|
|
2201
|
-
limit=search_limit,
|
|
2202
|
-
score_threshold=min_score
|
|
2203
|
-
)
|
|
2204
|
-
|
|
2205
|
-
for point in results:
|
|
2206
|
-
payload = point.payload
|
|
2207
|
-
score = float(point.score)
|
|
2208
|
-
|
|
2209
|
-
# Apply time-based decay if enabled
|
|
2210
|
-
use_decay_bool = ENABLE_MEMORY_DECAY # Use global default
|
|
2211
|
-
if use_decay_bool and 'timestamp' in payload:
|
|
2212
|
-
try:
|
|
2213
|
-
timestamp = datetime.fromisoformat(payload['timestamp'].replace('Z', '+00:00'))
|
|
2214
|
-
age_days = (datetime.now(timezone.utc) - timestamp).total_seconds() / (24 * 60 * 60)
|
|
2215
|
-
# Use consistent half-life formula: decay = exp(-ln(2) * age / half_life)
|
|
2216
|
-
ln2 = math.log(2)
|
|
2217
|
-
decay_factor = math.exp(-ln2 * age_days / DECAY_SCALE_DAYS)
|
|
2218
|
-
# Apply multiplicative formula: score * ((1 - weight) + weight * decay)
|
|
2219
|
-
score = score * ((1 - DECAY_WEIGHT) + DECAY_WEIGHT * decay_factor)
|
|
2220
|
-
except (ValueError, TypeError) as e:
|
|
2221
|
-
# Log but continue - timestamp format issue shouldn't break search
|
|
2222
|
-
logger.debug(f"Failed to apply decay for timestamp {payload.get('timestamp')}: {e}")
|
|
2223
|
-
|
|
2224
|
-
all_results.append({
|
|
2225
|
-
'score': score,
|
|
2226
|
-
'payload': payload,
|
|
2227
|
-
'collection': collection_name
|
|
2228
|
-
})
|
|
2229
|
-
|
|
2230
|
-
except Exception as e:
|
|
2231
|
-
# Log unexpected errors but continue with other collections
|
|
2232
|
-
logger.debug(f"Error searching collection {collection_name}: {e}")
|
|
2233
|
-
continue
|
|
2234
|
-
|
|
2235
|
-
# Sort by score
|
|
2236
|
-
all_results.sort(key=lambda x: x['score'], reverse=True)
|
|
2237
|
-
|
|
2238
|
-
# Apply pagination
|
|
2239
|
-
paginated_results = all_results[offset:offset + limit]
|
|
2240
|
-
|
|
2241
|
-
if not paginated_results:
|
|
2242
|
-
return f"""<next_results>
|
|
2243
|
-
<query>{query}</query>
|
|
2244
|
-
<offset>{offset}</offset>
|
|
2245
|
-
<status>no_more_results</status>
|
|
2246
|
-
<message>No additional results found beyond offset {offset}</message>
|
|
2247
|
-
</next_results>"""
|
|
2248
|
-
|
|
2249
|
-
# Format results
|
|
2250
|
-
results_text = []
|
|
2251
|
-
for i, result in enumerate(paginated_results, start=offset + 1):
|
|
2252
|
-
payload = result['payload']
|
|
2253
|
-
score = result['score']
|
|
2254
|
-
timestamp = payload.get('timestamp', 'Unknown')
|
|
2255
|
-
conversation_id = payload.get('conversation_id', 'Unknown')
|
|
2256
|
-
project = payload.get('project', 'Unknown')
|
|
2257
|
-
|
|
2258
|
-
# Get text preview (store text once to avoid multiple calls)
|
|
2259
|
-
text = payload.get('text', '')
|
|
2260
|
-
text_preview = text[:300] + '...' if len(text) > 300 else text
|
|
2261
|
-
|
|
2262
|
-
results_text.append(f"""
|
|
2263
|
-
<result index="{i}">
|
|
2264
|
-
<score>{score:.3f}</score>
|
|
2265
|
-
<timestamp>{timestamp}</timestamp>
|
|
2266
|
-
<project>{project}</project>
|
|
2267
|
-
<conversation_id>{conversation_id}</conversation_id>
|
|
2268
|
-
<preview>{text_preview}</preview>
|
|
2269
|
-
</result>""")
|
|
2270
|
-
|
|
2271
|
-
# Check if there are more results available
|
|
2272
|
-
has_more = len(all_results) > (offset + limit)
|
|
2273
|
-
next_offset = offset + limit if has_more else None
|
|
2274
|
-
|
|
2275
|
-
return f"""<next_results>
|
|
2276
|
-
<query>{query}</query>
|
|
2277
|
-
<offset>{offset}</offset>
|
|
2278
|
-
<limit>{limit}</limit>
|
|
2279
|
-
<count>{len(paginated_results)}</count>
|
|
2280
|
-
<total_available>{len(all_results)}</total_available>
|
|
2281
|
-
<has_more>{has_more}</has_more>
|
|
2282
|
-
{f'<next_offset>{next_offset}</next_offset>' if next_offset else ''}
|
|
2283
|
-
<results>{''.join(results_text)}
|
|
2284
|
-
</results>
|
|
2285
|
-
</next_results>"""
|
|
2286
|
-
|
|
2287
|
-
except Exception as e:
|
|
2288
|
-
await ctx.error(f"Pagination failed: {str(e)}")
|
|
2289
|
-
return f"""<next_results>
|
|
2290
|
-
<error>Failed to get next results: {str(e)}</error>
|
|
2291
|
-
</next_results>"""
|
|
653
|
+
# Register temporal tools after all functions are defined
|
|
654
|
+
register_temporal_tools(
|
|
655
|
+
mcp,
|
|
656
|
+
qdrant_client,
|
|
657
|
+
QDRANT_URL,
|
|
658
|
+
get_all_collections,
|
|
659
|
+
generate_embedding,
|
|
660
|
+
initialize_embeddings,
|
|
661
|
+
normalize_project_name
|
|
662
|
+
)
|
|
663
|
+
print(f"[INFO] Temporal tools registered", file=sys.stderr)
|
|
664
|
+
|
|
665
|
+
# Register search tools
|
|
666
|
+
def get_embedding_manager():
|
|
667
|
+
"""Factory function to get the current embedding manager."""
|
|
668
|
+
from .embedding_manager import get_embedding_manager as get_em
|
|
669
|
+
return get_em()
|
|
670
|
+
|
|
671
|
+
# Initialize ProjectResolver for collection name mapping
|
|
672
|
+
# ProjectResolver needs a sync client, not async
|
|
673
|
+
from qdrant_client import QdrantClient as SyncQdrantClient
|
|
674
|
+
sync_qdrant_client = SyncQdrantClient(url=QDRANT_URL)
|
|
675
|
+
project_resolver = ProjectResolver(sync_qdrant_client)
|
|
676
|
+
|
|
677
|
+
register_search_tools(
|
|
678
|
+
mcp,
|
|
679
|
+
qdrant_client,
|
|
680
|
+
QDRANT_URL,
|
|
681
|
+
get_embedding_manager,
|
|
682
|
+
normalize_project_name,
|
|
683
|
+
ENABLE_MEMORY_DECAY,
|
|
684
|
+
DECAY_WEIGHT,
|
|
685
|
+
DECAY_SCALE_DAYS,
|
|
686
|
+
USE_NATIVE_DECAY,
|
|
687
|
+
NATIVE_DECAY_AVAILABLE,
|
|
688
|
+
decay_manager,
|
|
689
|
+
project_resolver # Pass the resolver
|
|
690
|
+
)
|
|
2292
691
|
|
|
692
|
+
# Register reflection tools
|
|
693
|
+
register_reflection_tools(
|
|
694
|
+
mcp,
|
|
695
|
+
qdrant_client,
|
|
696
|
+
QDRANT_URL,
|
|
697
|
+
get_embedding_manager,
|
|
698
|
+
normalize_project_name
|
|
699
|
+
)
|
|
2293
700
|
|
|
2294
701
|
# Run the server
|
|
2295
702
|
if __name__ == "__main__":
|