claude-self-reflect 3.2.3 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/.claude/agents/claude-self-reflect-test.md +595 -528
  2. package/.claude/agents/documentation-writer.md +1 -1
  3. package/.claude/agents/qdrant-specialist.md +2 -2
  4. package/.claude/agents/reflection-specialist.md +61 -5
  5. package/.claude/agents/search-optimizer.md +9 -7
  6. package/README.md +16 -9
  7. package/mcp-server/pyproject.toml +1 -1
  8. package/mcp-server/run-mcp.sh +49 -5
  9. package/mcp-server/src/app_context.py +64 -0
  10. package/mcp-server/src/config.py +57 -0
  11. package/mcp-server/src/connection_pool.py +286 -0
  12. package/mcp-server/src/decay_manager.py +106 -0
  13. package/mcp-server/src/embedding_manager.py +64 -40
  14. package/mcp-server/src/embeddings_old.py +141 -0
  15. package/mcp-server/src/models.py +64 -0
  16. package/mcp-server/src/parallel_search.py +371 -0
  17. package/mcp-server/src/project_resolver.py +33 -46
  18. package/mcp-server/src/reflection_tools.py +206 -0
  19. package/mcp-server/src/rich_formatting.py +196 -0
  20. package/mcp-server/src/search_tools.py +826 -0
  21. package/mcp-server/src/server.py +140 -1715
  22. package/mcp-server/src/temporal_design.py +132 -0
  23. package/mcp-server/src/temporal_tools.py +597 -0
  24. package/mcp-server/src/temporal_utils.py +384 -0
  25. package/mcp-server/src/utils.py +150 -67
  26. package/package.json +11 -1
  27. package/scripts/add-timestamp-indexes.py +134 -0
  28. package/scripts/check-collections.py +29 -0
  29. package/scripts/debug-august-parsing.py +76 -0
  30. package/scripts/debug-import-single.py +91 -0
  31. package/scripts/debug-project-resolver.py +82 -0
  32. package/scripts/debug-temporal-tools.py +135 -0
  33. package/scripts/delta-metadata-update.py +547 -0
  34. package/scripts/import-conversations-unified.py +65 -6
  35. package/scripts/importer/utils/project_normalizer.py +22 -9
  36. package/scripts/precompact-hook.sh +33 -0
  37. package/scripts/streaming-watcher.py +1443 -0
  38. package/scripts/utils.py +39 -0
  39. package/shared/__init__.py +5 -0
  40. package/shared/normalization.py +54 -0
@@ -4,21 +4,36 @@ import os
4
4
  import asyncio
5
5
  from pathlib import Path
6
6
  from typing import Any, Optional, List, Dict, Union
7
- from datetime import datetime, timezone
7
+ from datetime import datetime, timezone, timedelta
8
8
  import json
9
- import numpy as np
10
9
  import hashlib
11
10
  import time
12
11
  import logging
12
+ import math
13
13
  from xml.sax.saxutils import escape
14
14
 
15
15
  from fastmcp import FastMCP, Context
16
- from .utils import normalize_project_name
16
+
17
+ # Import from shared module for consistent normalization
18
+ import sys
19
+ sys.path.insert(0, str(Path(__file__).parent.parent.parent))
20
+ try:
21
+ from shared.normalization import normalize_project_name
22
+ except ImportError:
23
+ # Fall back to local utils if shared module not found
24
+ from .utils import normalize_project_name
25
+ import logging
26
+ logging.warning("Using legacy utils.normalize_project_name - shared module not found")
27
+
17
28
  from .project_resolver import ProjectResolver
29
+ from .temporal_utils import SessionDetector, TemporalParser, WorkSession, group_by_time_period
30
+ from .temporal_tools import register_temporal_tools
31
+ from .search_tools import register_search_tools
32
+ from .reflection_tools import register_reflection_tools
18
33
  from pydantic import BaseModel, Field
19
34
  from qdrant_client import AsyncQdrantClient, models
20
35
  from qdrant_client.models import (
21
- PointStruct, VectorParams, Distance
36
+ PointStruct, VectorParams, Distance, OrderBy
22
37
  )
23
38
 
24
39
  # Try to import newer Qdrant API for native decay
@@ -73,11 +88,49 @@ EMBEDDING_MODEL = os.getenv('EMBEDDING_MODEL', 'sentence-transformers/all-MiniLM
73
88
  # Import the robust embedding manager
74
89
  from .embedding_manager import get_embedding_manager
75
90
 
91
+ # Import new performance modules with fallback
92
+ try:
93
+ from .connection_pool import QdrantConnectionPool, CircuitBreaker
94
+ CONNECTION_POOL_AVAILABLE = True
95
+ except ImportError:
96
+ CONNECTION_POOL_AVAILABLE = False
97
+ logging.warning("Connection pool module not available")
98
+
99
+ try:
100
+ from .parallel_search import parallel_search_collections
101
+ PARALLEL_SEARCH_AVAILABLE = True
102
+ except ImportError:
103
+ PARALLEL_SEARCH_AVAILABLE = False
104
+ logging.warning("Parallel search module not available")
105
+
106
+ # Set default configuration values
107
+ MAX_RESULTS_PER_COLLECTION = 10
108
+ MAX_TOTAL_RESULTS = 1000
109
+ MAX_MEMORY_MB = 500
110
+ POOL_SIZE = 10
111
+ POOL_MAX_OVERFLOW = 5
112
+ POOL_TIMEOUT = 30.0
113
+ RETRY_ATTEMPTS = 3
114
+ RETRY_DELAY = 1.0
115
+ MAX_CONCURRENT_SEARCHES = 10
116
+ ENABLE_PARALLEL_SEARCH = True
117
+
118
+ try:
119
+ from .decay_manager import DecayManager
120
+ DECAY_MANAGER_AVAILABLE = True
121
+ except ImportError:
122
+ DECAY_MANAGER_AVAILABLE = False
123
+ logging.warning("Decay manager module not available")
124
+
76
125
  # Lazy initialization - models will be loaded on first use
77
126
  embedding_manager = None
78
127
  voyage_client = None # Keep for backward compatibility
79
128
  local_embedding_model = None # Keep for backward compatibility
80
129
 
130
+ # Initialize connection pool
131
+ qdrant_pool = None
132
+ circuit_breaker = None
133
+
81
134
  def initialize_embeddings():
82
135
  """Initialize embedding models with robust fallback."""
83
136
  global embedding_manager, voyage_client, local_embedding_model
@@ -89,7 +142,7 @@ def initialize_embeddings():
89
142
  if embedding_manager.model_type == 'voyage':
90
143
  voyage_client = embedding_manager.voyage_client
91
144
  elif embedding_manager.model_type == 'local':
92
- local_embedding_model = embedding_manager.model
145
+ local_embedding_model = embedding_manager.local_model
93
146
 
94
147
  return True
95
148
  except Exception as e:
@@ -97,9 +150,8 @@ def initialize_embeddings():
97
150
  return False
98
151
 
99
152
  # Debug environment loading and startup
100
- import sys
101
- import datetime as dt
102
- startup_time = dt.datetime.now().isoformat()
153
+ # Debug environment loading and startup
154
+ startup_time = datetime.now(timezone.utc).isoformat()
103
155
  print(f"[STARTUP] MCP Server starting at {startup_time}", file=sys.stderr)
104
156
  print(f"[STARTUP] Python: {sys.version}", file=sys.stderr)
105
157
  print(f"[STARTUP] Working directory: {os.getcwd()}", file=sys.stderr)
@@ -140,8 +192,32 @@ mcp = FastMCP(
140
192
  instructions="Search past conversations and store reflections with time-based memory decay"
141
193
  )
142
194
 
143
- # Create Qdrant client
144
- qdrant_client = AsyncQdrantClient(url=QDRANT_URL)
195
+ # Initialize Qdrant client with connection pooling if available
196
+ if CONNECTION_POOL_AVAILABLE and ENABLE_PARALLEL_SEARCH:
197
+ qdrant_pool = QdrantConnectionPool(
198
+ url=QDRANT_URL,
199
+ pool_size=POOL_SIZE,
200
+ max_overflow=POOL_MAX_OVERFLOW,
201
+ timeout=POOL_TIMEOUT,
202
+ retry_attempts=RETRY_ATTEMPTS,
203
+ retry_delay=RETRY_DELAY
204
+ )
205
+ # Create a wrapper for backward compatibility
206
+ qdrant_client = AsyncQdrantClient(url=QDRANT_URL)
207
+ circuit_breaker = CircuitBreaker(failure_threshold=5, recovery_timeout=60.0)
208
+ print(f"[INFO] Connection pool initialized with size {POOL_SIZE}", file=sys.stderr)
209
+ else:
210
+ # Fallback to single client
211
+ qdrant_client = AsyncQdrantClient(url=QDRANT_URL)
212
+ qdrant_pool = None
213
+ circuit_breaker = None
214
+ print(f"[INFO] Using single Qdrant client (no pooling)", file=sys.stderr)
215
+
216
+ # Initialize decay manager if available
217
+ decay_manager = None
218
+ if DECAY_MANAGER_AVAILABLE:
219
+ decay_manager = DecayManager()
220
+ print(f"[INFO] Decay manager initialized", file=sys.stderr)
145
221
 
146
222
  # Add MCP Resources for system status
147
223
  @mcp.resource("status://import-stats")
@@ -480,7 +556,14 @@ def get_embedding_dimension() -> int:
480
556
 
481
557
  def get_collection_suffix() -> str:
482
558
  """Get the collection suffix based on embedding provider."""
483
- if PREFER_LOCAL_EMBEDDINGS or not voyage_client:
559
+ # Use embedding_manager's model type if available
560
+ if embedding_manager and hasattr(embedding_manager, 'model_type'):
561
+ if embedding_manager.model_type == 'voyage':
562
+ return "_voyage"
563
+ else:
564
+ return "_local"
565
+ # Fallback to environment variable
566
+ elif PREFER_LOCAL_EMBEDDINGS:
484
567
  return "_local"
485
568
  else:
486
569
  return "_voyage"
@@ -566,1712 +649,54 @@ def aggregate_pattern_intelligence(results: List[SearchResult]) -> Dict[str, Any
566
649
  return intelligence
567
650
 
568
651
  # Register tools
569
- @mcp.tool()
570
- async def reflect_on_past(
571
- ctx: Context,
572
- query: str = Field(description="The search query to find semantically similar conversations"),
573
- limit: int = Field(default=5, description="Maximum number of results to return"),
574
- min_score: float = Field(default=0.7, description="Minimum similarity score (0-1)"),
575
- use_decay: Union[int, str] = Field(default=-1, description="Apply time-based decay: 1=enable, 0=disable, -1=use environment default (accepts int or str)"),
576
- project: Optional[str] = Field(default=None, description="Search specific project only. If not provided, searches current project based on working directory. Use 'all' to search across all projects."),
577
- include_raw: bool = Field(default=False, description="Include raw Qdrant payload data for debugging (increases response size)"),
578
- response_format: str = Field(default="xml", description="Response format: 'xml' or 'markdown'"),
579
- brief: bool = Field(default=False, description="Brief mode: returns minimal information for faster response"),
580
- mode: str = Field(default="full", description="Search mode: 'full' (all results with details), 'quick' (count + top result only), 'summary' (aggregated insights without individual results)")
581
- ) -> str:
582
- """Search for relevant past conversations using semantic search with optional time decay."""
583
-
584
- logger.info(f"=== SEARCH START === Query: '{query}', Project: '{project}', Limit: {limit}")
585
-
586
- # Validate mode parameter
587
- if mode not in ['full', 'quick', 'summary']:
588
- return f"<error>Invalid mode '{mode}'. Must be 'full', 'quick', or 'summary'</error>"
589
-
590
- # Start timing
591
- start_time = time.time()
592
- timing_info = {}
593
-
594
- # Normalize use_decay to integer
595
- timing_info['param_parsing_start'] = time.time()
596
- if isinstance(use_decay, str):
597
- try:
598
- use_decay = int(use_decay)
599
- except ValueError:
600
- raise ValueError("use_decay must be '1', '0', or '-1'")
601
- timing_info['param_parsing_end'] = time.time()
602
-
603
- # Parse decay parameter using integer approach
604
- should_use_decay = (
605
- True if use_decay == 1
606
- else False if use_decay == 0
607
- else ENABLE_MEMORY_DECAY # -1 or any other value
608
- )
609
-
610
- # Determine project scope
611
- target_project = project
612
-
613
- # Always get the working directory for logging purposes
614
- cwd = os.environ.get('MCP_CLIENT_CWD', os.getcwd())
615
- await ctx.debug(f"CWD: {cwd}, Project param: {project}")
616
-
617
- if project is None:
618
- # Use MCP_CLIENT_CWD environment variable set by run-mcp.sh
619
- # This contains the actual working directory where Claude Code is running
620
-
621
- # Extract project name from path (e.g., /Users/.../projects/project-name)
622
- path_parts = Path(cwd).parts
623
- if 'projects' in path_parts:
624
- idx = path_parts.index('projects')
625
- if idx + 1 < len(path_parts):
626
- target_project = path_parts[idx + 1]
627
- elif '.claude' in path_parts:
628
- # If we're in a .claude directory, go up to find project
629
- for i, part in enumerate(path_parts):
630
- if part == '.claude' and i > 0:
631
- target_project = path_parts[i - 1]
632
- break
633
-
634
- # If still no project detected, use the last directory name
635
- if target_project is None:
636
- target_project = Path(cwd).name
637
-
638
- await ctx.debug(f"Auto-detected project from path: {target_project}")
639
-
640
- # For project matching, we need to handle the dash-encoded format
641
- # Convert folder name to the format used in stored data
642
- if target_project != 'all':
643
- # The stored format uses full path with dashes, so we need to construct it
644
- # For now, let's try to match based on the end of the project name
645
- pass # We'll handle this differently in the filtering logic
646
-
647
- await ctx.debug(f"Searching for: {query}")
648
- await ctx.debug(f"Client working directory: {cwd}")
649
- await ctx.debug(f"Project scope: {target_project if target_project != 'all' else 'all projects'}")
650
- await ctx.debug(f"Decay enabled: {should_use_decay}")
651
- await ctx.debug(f"Native decay mode: {USE_NATIVE_DECAY}")
652
- await ctx.debug(f"ENABLE_MEMORY_DECAY env: {ENABLE_MEMORY_DECAY}")
653
- await ctx.debug(f"DECAY_WEIGHT: {DECAY_WEIGHT}, DECAY_SCALE_DAYS: {DECAY_SCALE_DAYS}")
654
-
655
- try:
656
- # We'll generate embeddings on-demand per collection type
657
- timing_info['embedding_prep_start'] = time.time()
658
- query_embeddings = {} # Cache embeddings by type
659
- timing_info['embedding_prep_end'] = time.time()
660
-
661
- # Get all collections
662
- timing_info['get_collections_start'] = time.time()
663
- all_collections = await get_all_collections()
664
- timing_info['get_collections_end'] = time.time()
665
-
666
- if not all_collections:
667
- return "No conversation collections found. Please import conversations first."
668
-
669
- # Filter collections by project if not searching all
670
- project_collections = [] # Define at this scope for later use
671
- if target_project != 'all':
672
- # Use ProjectResolver to find collections for this project
673
- resolver = ProjectResolver(qdrant_client)
674
- project_collections = resolver.find_collections_for_project(target_project)
675
-
676
- if not project_collections:
677
- # Fall back to old method for backward compatibility
678
- normalized_name = normalize_project_name(target_project)
679
- project_hash = hashlib.md5(normalized_name.encode()).hexdigest()[:8]
680
- project_collections = [
681
- c for c in all_collections
682
- if c.startswith(f"conv_{project_hash}_")
683
- ]
684
-
685
- # Always include reflections collections when searching a specific project
686
- reflections_collections = [c for c in all_collections if c.startswith('reflections')]
687
-
688
- if not project_collections:
689
- # Fall back to searching all collections but filtering by project metadata
690
- await ctx.debug(f"No collections found for project {target_project}, will filter by metadata")
691
- collections_to_search = all_collections
692
- else:
693
- await ctx.debug(f"Found {len(project_collections)} collections for project {target_project}")
694
- # Include both project collections and reflections
695
- collections_to_search = project_collections + reflections_collections
696
- # Remove duplicates
697
- collections_to_search = list(set(collections_to_search))
698
- else:
699
- collections_to_search = all_collections
700
-
701
- await ctx.debug(f"Searching across {len(collections_to_search)} collections")
702
- await ctx.debug(f"Using {'local' if PREFER_LOCAL_EMBEDDINGS or not voyage_client else 'Voyage AI'} embeddings")
703
-
704
- all_results = []
705
-
706
- # Search each collection
707
- timing_info['search_all_start'] = time.time()
708
- collection_timings = []
709
-
710
- # Report initial progress
711
- await ctx.report_progress(progress=0, total=len(collections_to_search))
712
-
713
- for idx, collection_name in enumerate(collections_to_search):
714
- collection_timing = {'name': collection_name, 'start': time.time()}
715
-
716
- # Report progress before searching each collection
717
- await ctx.report_progress(
718
- progress=idx,
719
- total=len(collections_to_search),
720
- message=f"Searching {collection_name}"
721
- )
722
-
723
- try:
724
- # Determine embedding type for this collection
725
- embedding_type_for_collection = 'voyage' if collection_name.endswith('_voyage') else 'local'
726
-
727
- # Generate or retrieve cached embedding for this type
728
- if embedding_type_for_collection not in query_embeddings:
729
- try:
730
- query_embeddings[embedding_type_for_collection] = await generate_embedding(query, force_type=embedding_type_for_collection)
731
- except Exception as e:
732
- await ctx.debug(f"Failed to generate {embedding_type_for_collection} embedding: {e}")
733
- continue
734
-
735
- query_embedding = query_embeddings[embedding_type_for_collection]
736
-
737
- if should_use_decay and USE_NATIVE_DECAY and NATIVE_DECAY_AVAILABLE:
738
- # Use native Qdrant decay with newer API
739
- await ctx.debug(f"Using NATIVE Qdrant decay (new API) for {collection_name}")
740
-
741
- # Build the query with native Qdrant decay formula using newer API
742
- query_obj = Query(
743
- nearest=query_embedding,
744
- formula=Formula(
745
- sum=[
746
- # Original similarity score
747
- Expression(variable="score"),
748
- # Decay boost term
749
- Expression(
750
- mult=MultExpression(
751
- mult=[
752
- # Decay weight
753
- Expression(constant=DECAY_WEIGHT),
754
- # Exponential decay function
755
- Expression(
756
- exp_decay=DecayParamsExpression(
757
- # Use timestamp field for decay
758
- x=Expression(datetime_key="timestamp"),
759
- # Decay from current time (server-side)
760
- target=Expression(datetime="now"),
761
- # Scale in milliseconds
762
- scale=DECAY_SCALE_DAYS * 24 * 60 * 60 * 1000,
763
- # Standard exponential decay midpoint
764
- midpoint=0.5
765
- )
766
- )
767
- ]
768
- )
769
- )
770
- ]
771
- )
772
- )
773
-
774
- # Execute query with native decay (new API)
775
- results = await qdrant_client.query_points(
776
- collection_name=collection_name,
777
- query=query_obj,
778
- limit=limit,
779
- score_threshold=min_score,
780
- with_payload=True
781
- )
782
- elif should_use_decay and USE_NATIVE_DECAY and not NATIVE_DECAY_AVAILABLE:
783
- # Use native Qdrant decay with older API
784
- await ctx.debug(f"Using NATIVE Qdrant decay (legacy API) for {collection_name}")
785
-
786
- # Build the query with native Qdrant decay formula using older API
787
- query_obj = FormulaQuery(
788
- nearest=query_embedding,
789
- formula=SumExpression(
790
- sum=[
791
- # Original similarity score
792
- 'score', # Variable expression can be a string
793
- # Decay boost term
794
- {
795
- 'mult': [
796
- # Decay weight (constant as float)
797
- DECAY_WEIGHT,
798
- # Exponential decay function
799
- {
800
- 'exp_decay': DecayParamsExpression(
801
- # Use timestamp field for decay
802
- x=DatetimeKeyExpression(datetime_key='timestamp'),
803
- # Decay from current time (server-side)
804
- target=DatetimeExpression(datetime='now'),
805
- # Scale in milliseconds
806
- scale=DECAY_SCALE_DAYS * 24 * 60 * 60 * 1000,
807
- # Standard exponential decay midpoint
808
- midpoint=0.5
809
- )
810
- }
811
- ]
812
- }
813
- ]
814
- )
815
- )
816
-
817
- # Execute query with native decay
818
- results = await qdrant_client.query_points(
819
- collection_name=collection_name,
820
- query=query_obj,
821
- limit=limit,
822
- score_threshold=min_score,
823
- with_payload=True
824
- )
825
-
826
- # Process results from native decay search
827
- for point in results.points:
828
- # Clean timestamp for proper parsing
829
- raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
830
- clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
831
-
832
- # Check project filter if we're searching all collections but want specific project
833
- point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
834
-
835
- # Special handling for reflections - they're global by default but can have project context
836
- is_reflection_collection = collection_name.startswith('reflections')
837
-
838
- # Handle project matching - check if the target project name appears at the end of the stored project path
839
- if target_project != 'all' and not project_collections and not is_reflection_collection:
840
- # The stored project name is like "-Users-username-projects-ShopifyMCPMockShop"
841
- # We want to match just "ShopifyMCPMockShop"
842
- # Also handle underscore/dash variations (procsolve-website vs procsolve_website)
843
- normalized_target = target_project.replace('-', '_')
844
- normalized_stored = point_project.replace('-', '_')
845
- if not (normalized_stored.endswith(f"_{normalized_target}") or
846
- normalized_stored == normalized_target or
847
- point_project.endswith(f"-{target_project}") or
848
- point_project == target_project):
849
- continue # Skip results from other projects
850
-
851
- # For reflections with project context, optionally filter by project
852
- if is_reflection_collection and target_project != 'all' and 'project' in point.payload:
853
- # Only filter if the reflection has project metadata
854
- reflection_project = point.payload.get('project', '')
855
- if reflection_project:
856
- # Normalize both for comparison (handle underscore/dash variations)
857
- normalized_target = target_project.replace('-', '_')
858
- normalized_reflection = reflection_project.replace('-', '_')
859
- if not (
860
- reflection_project == target_project or
861
- normalized_reflection == normalized_target or
862
- reflection_project.endswith(f"/{target_project}") or
863
- reflection_project.endswith(f"-{target_project}") or
864
- normalized_reflection.endswith(f"_{normalized_target}") or
865
- normalized_reflection.endswith(f"/{normalized_target}")
866
- ):
867
- continue # Skip reflections from other projects
868
-
869
- # Log pattern data
870
- patterns = point.payload.get('code_patterns')
871
- logger.info(f"DEBUG: Creating SearchResult for point {point.id} from {collection_name}: has_patterns={bool(patterns)}, pattern_keys={list(patterns.keys()) if patterns else None}")
872
-
873
- all_results.append(SearchResult(
874
- id=str(point.id),
875
- score=point.score, # Score already includes decay
876
- timestamp=clean_timestamp,
877
- role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
878
- excerpt=(point.payload.get('text', '')[:350] + '...' if len(point.payload.get('text', '')) > 350 else point.payload.get('text', '')),
879
- project_name=point_project,
880
- conversation_id=point.payload.get('conversation_id'),
881
- base_conversation_id=point.payload.get('base_conversation_id'),
882
- collection_name=collection_name,
883
- raw_payload=point.payload, # Always include payload for metadata extraction
884
- # Pattern intelligence metadata
885
- code_patterns=point.payload.get('code_patterns'),
886
- files_analyzed=point.payload.get('files_analyzed'),
887
- tools_used=list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
888
- concepts=point.payload.get('concepts')
889
- ))
890
-
891
- elif should_use_decay:
892
- # Use client-side decay (existing implementation)
893
- await ctx.debug(f"Using CLIENT-SIDE decay for {collection_name}")
894
-
895
- # Search without score threshold to get all candidates
896
- results = await qdrant_client.search(
897
- collection_name=collection_name,
898
- query_vector=query_embedding,
899
- limit=limit * 3, # Get more candidates for decay filtering
900
- with_payload=True
901
- )
902
-
903
- # Apply decay scoring manually
904
- now = datetime.now(timezone.utc)
905
- scale_ms = DECAY_SCALE_DAYS * 24 * 60 * 60 * 1000
906
-
907
- decay_results = []
908
- for point in results:
909
- try:
910
- # Get timestamp from payload
911
- timestamp_str = point.payload.get('timestamp')
912
- if timestamp_str:
913
- timestamp = datetime.fromisoformat(timestamp_str.replace('Z', '+00:00'))
914
- # Ensure timestamp is timezone-aware
915
- if timestamp.tzinfo is None:
916
- timestamp = timestamp.replace(tzinfo=timezone.utc)
917
- age_ms = (now - timestamp).total_seconds() * 1000
918
-
919
- # Calculate decay factor
920
- decay_factor = np.exp(-age_ms / scale_ms)
921
-
922
- # Apply decay formula
923
- adjusted_score = point.score + (DECAY_WEIGHT * decay_factor)
924
-
925
- # Debug: show the calculation
926
- age_days = age_ms / (24 * 60 * 60 * 1000)
927
- await ctx.debug(f"Point: age={age_days:.1f} days, original_score={point.score:.3f}, decay_factor={decay_factor:.3f}, adjusted_score={adjusted_score:.3f}")
928
- else:
929
- adjusted_score = point.score
930
-
931
- # Only include if above min_score after decay
932
- if adjusted_score >= min_score:
933
- decay_results.append((adjusted_score, point))
934
-
935
- except Exception as e:
936
- await ctx.debug(f"Error applying decay to point: {e}")
937
- decay_results.append((point.score, point))
938
-
939
- # Sort by adjusted score and take top results
940
- decay_results.sort(key=lambda x: x[0], reverse=True)
941
-
942
- # Convert to SearchResult format
943
- for adjusted_score, point in decay_results[:limit]:
944
- # Clean timestamp for proper parsing
945
- raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
946
- clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
947
-
948
- # Check project filter if we're searching all collections but want specific project
949
- point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
950
-
951
- # Special handling for reflections - they're global by default but can have project context
952
- is_reflection_collection = collection_name.startswith('reflections')
953
-
954
- # Handle project matching - check if the target project name appears at the end of the stored project path
955
- if target_project != 'all' and not project_collections and not is_reflection_collection:
956
- # The stored project name is like "-Users-username-projects-ShopifyMCPMockShop"
957
- # We want to match just "ShopifyMCPMockShop"
958
- # Also handle underscore/dash variations (procsolve-website vs procsolve_website)
959
- normalized_target = target_project.replace('-', '_')
960
- normalized_stored = point_project.replace('-', '_')
961
- if not (normalized_stored.endswith(f"_{normalized_target}") or
962
- normalized_stored == normalized_target or
963
- point_project.endswith(f"-{target_project}") or
964
- point_project == target_project):
965
- continue # Skip results from other projects
966
-
967
- # For reflections with project context, optionally filter by project
968
- if is_reflection_collection and target_project != 'all' and 'project' in point.payload:
969
- # Only filter if the reflection has project metadata
970
- reflection_project = point.payload.get('project', '')
971
- if reflection_project:
972
- # Normalize both for comparison (handle underscore/dash variations)
973
- normalized_target = target_project.replace('-', '_')
974
- normalized_reflection = reflection_project.replace('-', '_')
975
- if not (
976
- reflection_project == target_project or
977
- normalized_reflection == normalized_target or
978
- reflection_project.endswith(f"/{target_project}") or
979
- reflection_project.endswith(f"-{target_project}") or
980
- normalized_reflection.endswith(f"_{normalized_target}") or
981
- normalized_reflection.endswith(f"/{normalized_target}")
982
- ):
983
- continue # Skip reflections from other projects
984
-
985
- all_results.append(SearchResult(
986
- id=str(point.id),
987
- score=adjusted_score, # Use adjusted score
988
- timestamp=clean_timestamp,
989
- role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
990
- excerpt=(point.payload.get('text', '')[:350] + '...' if len(point.payload.get('text', '')) > 350 else point.payload.get('text', '')),
991
- project_name=point_project,
992
- conversation_id=point.payload.get('conversation_id'),
993
- base_conversation_id=point.payload.get('base_conversation_id'),
994
- collection_name=collection_name,
995
- raw_payload=point.payload, # Always include payload for metadata extraction
996
- # Pattern intelligence metadata
997
- code_patterns=point.payload.get('code_patterns'),
998
- files_analyzed=point.payload.get('files_analyzed'),
999
- tools_used=list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
1000
- concepts=point.payload.get('concepts')
1001
- ))
1002
- else:
1003
- # Standard search without decay
1004
- results = await qdrant_client.search(
1005
- collection_name=collection_name,
1006
- query_vector=query_embedding,
1007
- limit=limit * 2, # Get more results to account for filtering
1008
- score_threshold=min_score * 0.9, # Slightly lower threshold to catch v1 chunks
1009
- with_payload=True
1010
- )
1011
-
1012
- for point in results:
1013
- # Clean timestamp for proper parsing
1014
- raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
1015
- clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
1016
-
1017
- # Check project filter if we're searching all collections but want specific project
1018
- point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
1019
-
1020
- # Special handling for reflections - they're global by default but can have project context
1021
- is_reflection_collection = collection_name.startswith('reflections')
1022
-
1023
- # Handle project matching - check if the target project name appears at the end of the stored project path
1024
- if target_project != 'all' and not project_collections and not is_reflection_collection:
1025
- # The stored project name is like "-Users-username-projects-ShopifyMCPMockShop"
1026
- # We want to match just "ShopifyMCPMockShop"
1027
- # Also handle underscore/dash variations (procsolve-website vs procsolve_website)
1028
- normalized_target = target_project.replace('-', '_')
1029
- normalized_stored = point_project.replace('-', '_')
1030
- if not (normalized_stored.endswith(f"_{normalized_target}") or
1031
- normalized_stored == normalized_target or
1032
- point_project.endswith(f"-{target_project}") or
1033
- point_project == target_project):
1034
- continue # Skip results from other projects
1035
-
1036
- # For reflections with project context, optionally filter by project
1037
- if is_reflection_collection and target_project != 'all' and 'project' in point.payload:
1038
- # Only filter if the reflection has project metadata
1039
- reflection_project = point.payload.get('project', '')
1040
- if reflection_project:
1041
- # Normalize both for comparison (handle underscore/dash variations)
1042
- normalized_target = target_project.replace('-', '_')
1043
- normalized_reflection = reflection_project.replace('-', '_')
1044
- if not (
1045
- reflection_project == target_project or
1046
- normalized_reflection == normalized_target or
1047
- reflection_project.endswith(f"/{target_project}") or
1048
- reflection_project.endswith(f"-{target_project}") or
1049
- normalized_reflection.endswith(f"_{normalized_target}") or
1050
- normalized_reflection.endswith(f"/{normalized_target}")
1051
- ):
1052
- continue # Skip reflections from other projects
1053
-
1054
- # BOOST V2 CHUNKS: Apply score boost for v2 chunks (better quality)
1055
- original_score = point.score
1056
- final_score = original_score
1057
- chunking_version = point.payload.get('chunking_version', 'v1')
1058
-
1059
- if chunking_version == 'v2':
1060
- # Boost v2 chunks by 20% (configurable)
1061
- boost_factor = 1.2 # From migration config
1062
- final_score = min(1.0, original_score * boost_factor)
1063
- await ctx.debug(f"Boosted v2 chunk: {original_score:.3f} -> {final_score:.3f}")
1064
-
1065
- # Apply minimum score threshold after boosting
1066
- if final_score < min_score:
1067
- continue
1068
-
1069
- search_result = SearchResult(
1070
- id=str(point.id),
1071
- score=final_score,
1072
- timestamp=clean_timestamp,
1073
- role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
1074
- excerpt=(point.payload.get('text', '')[:350] + '...' if len(point.payload.get('text', '')) > 350 else point.payload.get('text', '')),
1075
- project_name=point_project,
1076
- conversation_id=point.payload.get('conversation_id'),
1077
- base_conversation_id=point.payload.get('base_conversation_id'),
1078
- collection_name=collection_name,
1079
- raw_payload=point.payload, # Always include payload for metadata extraction
1080
- # Pattern intelligence metadata
1081
- code_patterns=point.payload.get('code_patterns'),
1082
- files_analyzed=point.payload.get('files_analyzed'),
1083
- tools_used=list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
1084
- concepts=point.payload.get('concepts')
1085
- )
1086
-
1087
- all_results.append(search_result)
1088
-
1089
- except Exception as e:
1090
- await ctx.debug(f"Error searching {collection_name}: {str(e)}")
1091
- collection_timing['error'] = str(e)
1092
-
1093
- collection_timing['end'] = time.time()
1094
- collection_timings.append(collection_timing)
1095
-
1096
- timing_info['search_all_end'] = time.time()
1097
-
1098
- # Report completion of search phase
1099
- await ctx.report_progress(
1100
- progress=len(collections_to_search),
1101
- total=len(collections_to_search),
1102
- message="Search complete, processing results"
1103
- )
1104
-
1105
- # Apply base_conversation_id boosting before sorting
1106
- timing_info['boost_start'] = time.time()
1107
-
1108
- # Group results by base_conversation_id to identify related chunks
1109
- base_conversation_groups = {}
1110
- for result in all_results:
1111
- base_id = result.base_conversation_id
1112
- if base_id:
1113
- if base_id not in base_conversation_groups:
1114
- base_conversation_groups[base_id] = []
1115
- base_conversation_groups[base_id].append(result)
1116
-
1117
- # Apply boost to results from base conversations with multiple high-scoring chunks
1118
- base_conversation_boost = 0.1 # Boost factor for base conversation matching
1119
- for base_id, group_results in base_conversation_groups.items():
1120
- if len(group_results) > 1: # Multiple chunks from same base conversation
1121
- avg_score = sum(r.score for r in group_results) / len(group_results)
1122
- if avg_score > 0.8: # Only boost high-quality base conversations
1123
- for result in group_results:
1124
- result.score += base_conversation_boost
1125
- await ctx.debug(f"Boosted result from base_conversation_id {base_id}: {result.score:.3f}")
1126
-
1127
- timing_info['boost_end'] = time.time()
1128
-
1129
- # Sort by score and limit
1130
- timing_info['sort_start'] = time.time()
1131
- all_results.sort(key=lambda x: x.score, reverse=True)
1132
-
1133
- # Apply mode-specific limits
1134
- if mode == "quick":
1135
- # For quick mode, only keep the top result
1136
- all_results = all_results[:1]
1137
- elif mode == "summary":
1138
- # For summary mode, we'll process all results but not return individual ones
1139
- pass # Keep all for aggregation
1140
- else:
1141
- # For full mode, apply the normal limit
1142
- all_results = all_results[:limit]
1143
-
1144
- timing_info['sort_end'] = time.time()
1145
-
1146
- logger.info(f"Total results: {len(all_results)}, Mode: {mode}, Returning: {len(all_results[:limit])}")
1147
- for r in all_results[:3]: # Log first 3
1148
- logger.debug(f"Result: id={r.id}, has_patterns={bool(r.code_patterns)}, pattern_keys={list(r.code_patterns.keys()) if r.code_patterns else None}")
1149
-
1150
- if not all_results:
1151
- return f"No conversations found matching '{query}'. Try different keywords or check if conversations have been imported."
1152
-
1153
- # Aggregate pattern intelligence across results
1154
- pattern_intelligence = aggregate_pattern_intelligence(all_results)
1155
-
1156
- # Update indexing status before returning results
1157
- await update_indexing_status()
1158
-
1159
- # Format results based on response_format and mode
1160
- timing_info['format_start'] = time.time()
1161
-
1162
- # Handle mode-specific responses
1163
- if mode == "quick":
1164
- # Quick mode: return just count and top result
1165
- total_count = len(all_results) # Before we limited to 1
1166
- if response_format == "xml":
1167
- result_text = f"<quick_search>\n"
1168
- result_text += f" <count>{total_count}</count>\n"
1169
- if all_results:
1170
- top_result = all_results[0]
1171
- result_text += f" <top_result>\n"
1172
- result_text += f" <score>{top_result.score:.3f}</score>\n"
1173
- result_text += f" <excerpt>{escape(top_result.excerpt[:200])}</excerpt>\n"
1174
- result_text += f" <project>{escape(top_result.project_name)}</project>\n"
1175
- result_text += f" <conversation_id>{escape(top_result.conversation_id or '')}</conversation_id>\n"
1176
- result_text += f" </top_result>\n"
1177
- result_text += f"</quick_search>"
1178
- return result_text
1179
- else:
1180
- # Markdown format for quick mode
1181
- if all_results:
1182
- return f"**Found {total_count} matches**\n\nTop result (score: {all_results[0].score:.3f}):\n{all_results[0].excerpt[:200]}"
1183
- else:
1184
- return f"No matches found for '{query}'"
1185
-
1186
- elif mode == "summary":
1187
- # Summary mode: return aggregated insights without individual results
1188
- if not all_results:
1189
- return f"No conversations found to summarize for '{query}'"
1190
-
1191
- # Aggregate data
1192
- total_count = len(all_results)
1193
- avg_score = sum(r.score for r in all_results) / total_count
1194
-
1195
- # Extract common concepts and tools
1196
- all_concepts = []
1197
- all_tools = []
1198
- all_files = []
1199
- projects = set()
1200
-
1201
- for result in all_results:
1202
- if result.concepts:
1203
- all_concepts.extend(result.concepts)
1204
- if result.tools_used:
1205
- all_tools.extend(result.tools_used)
1206
- if result.files_analyzed:
1207
- all_files.extend(result.files_analyzed)
1208
- projects.add(result.project_name)
1209
-
1210
- # Count frequencies
1211
- from collections import Counter
1212
- concept_counts = Counter(all_concepts).most_common(5)
1213
- tool_counts = Counter(all_tools).most_common(5)
1214
-
1215
- if response_format == "xml":
1216
- result_text = f"<search_summary>\n"
1217
- result_text += f" <query>{escape(query)}</query>\n"
1218
- result_text += f" <total_matches>{total_count}</total_matches>\n"
1219
- result_text += f" <average_score>{avg_score:.3f}</average_score>\n"
1220
- result_text += f" <projects_involved>{len(projects)}</projects_involved>\n"
1221
- if concept_counts:
1222
- result_text += f" <common_concepts>\n"
1223
- for concept, count in concept_counts:
1224
- result_text += f" <concept count=\"{count}\">{escape(concept)}</concept>\n"
1225
- result_text += f" </common_concepts>\n"
1226
- if tool_counts:
1227
- result_text += f" <common_tools>\n"
1228
- for tool, count in tool_counts:
1229
- result_text += f" <tool count=\"{count}\">{escape(tool)}</tool>\n"
1230
- result_text += f" </common_tools>\n"
1231
- result_text += f"</search_summary>"
1232
- return result_text
1233
- else:
1234
- # Markdown format for summary
1235
- result_text = f"## Summary for: {query}\n\n"
1236
- result_text += f"- **Total matches**: {total_count}\n"
1237
- result_text += f"- **Average relevance**: {avg_score:.3f}\n"
1238
- result_text += f"- **Projects involved**: {len(projects)}\n\n"
1239
- if concept_counts:
1240
- result_text += "**Common concepts**:\n"
1241
- for concept, count in concept_counts:
1242
- result_text += f"- {concept} ({count} occurrences)\n"
1243
- if tool_counts:
1244
- result_text += "\n**Common tools**:\n"
1245
- for tool, count in tool_counts:
1246
- result_text += f"- {tool} ({count} uses)\n"
1247
- return result_text
1248
-
1249
- # Continue with normal formatting for full mode
1250
- if response_format == "xml":
1251
- # Add upfront summary for immediate visibility (before collapsible XML)
1252
- upfront_summary = ""
1253
-
1254
- # Show result summary
1255
- if all_results:
1256
- score_info = "high" if all_results[0].score >= 0.85 else "good" if all_results[0].score >= 0.75 else "partial"
1257
- upfront_summary += f"🎯 RESULTS: {len(all_results)} matches ({score_info} relevance, top score: {all_results[0].score:.3f})\n"
1258
-
1259
- # Show performance with indexing status inline
1260
- total_time = time.time() - start_time
1261
- indexing_info = f" | 📊 {indexing_status['indexed_conversations']}/{indexing_status['total_conversations']} indexed" if indexing_status["percentage"] < 100.0 else ""
1262
- upfront_summary += f"⚡ PERFORMANCE: {int(total_time * 1000)}ms ({len(collections_to_search)} collections searched{indexing_info})\n"
1263
- else:
1264
- upfront_summary += f"❌ NO RESULTS: No conversations found matching '{query}'\n"
1265
-
1266
- # XML format (compact tags for performance)
1267
- result_text = upfront_summary + "\n<search>\n"
1268
-
1269
- # Add indexing status if not fully baselined - put key stats in opening tag for immediate visibility
1270
- if indexing_status["percentage"] < 95.0:
1271
- result_text += f' <info status="indexing" progress="{indexing_status["percentage"]:.1f}%" backlog="{indexing_status["backlog_count"]}">\n'
1272
- result_text += f' <message>📊 Indexing: {indexing_status["indexed_conversations"]}/{indexing_status["total_conversations"]} conversations ({indexing_status["percentage"]:.1f}% complete, {indexing_status["backlog_count"]} pending)</message>\n'
1273
- result_text += f" </info>\n"
1274
-
1275
- # Add high-level result summary
1276
- if all_results:
1277
- # Count today's results
1278
- now = datetime.now(timezone.utc)
1279
- today_count = 0
1280
- yesterday_count = 0
1281
- week_count = 0
1282
-
1283
- for result in all_results:
1284
- timestamp_clean = result.timestamp.replace('Z', '+00:00') if result.timestamp.endswith('Z') else result.timestamp
1285
- timestamp_dt = datetime.fromisoformat(timestamp_clean)
1286
- if timestamp_dt.tzinfo is None:
1287
- timestamp_dt = timestamp_dt.replace(tzinfo=timezone.utc)
1288
-
1289
- days_ago = (now - timestamp_dt).days
1290
- if days_ago == 0:
1291
- today_count += 1
1292
- elif days_ago == 1:
1293
- yesterday_count += 1
1294
- if days_ago <= 7:
1295
- week_count += 1
1296
-
1297
- # Compact summary with key info in opening tag
1298
- time_info = ""
1299
- if today_count > 0:
1300
- time_info = f"{today_count} today"
1301
- elif yesterday_count > 0:
1302
- time_info = f"{yesterday_count} yesterday"
1303
- elif week_count > 0:
1304
- time_info = f"{week_count} this week"
1305
- else:
1306
- time_info = "older results"
1307
-
1308
- score_info = "high" if all_results[0].score >= 0.85 else "good" if all_results[0].score >= 0.75 else "partial"
1309
-
1310
- result_text += f' <summary count="{len(all_results)}" relevance="{score_info}" recency="{time_info}" top-score="{all_results[0].score:.3f}">\n'
1311
-
1312
- # Short preview of top result
1313
- top_excerpt = all_results[0].excerpt[:100].strip()
1314
- if '...' not in top_excerpt:
1315
- top_excerpt += "..."
1316
- result_text += f' <preview>{top_excerpt}</preview>\n'
1317
- result_text += f" </summary>\n"
1318
- else:
1319
- result_text += f" <result-summary>\n"
1320
- result_text += f" <headline>No matches found</headline>\n"
1321
- result_text += f" <relevance>No conversations matched your query</relevance>\n"
1322
- result_text += f" </result-summary>\n"
1323
-
1324
- result_text += f" <meta>\n"
1325
- result_text += f" <q>{query}</q>\n"
1326
- result_text += f" <scope>{target_project if target_project != 'all' else 'all'}</scope>\n"
1327
- result_text += f" <count>{len(all_results)}</count>\n"
1328
- if all_results:
1329
- result_text += f" <range>{all_results[-1].score:.3f}-{all_results[0].score:.3f}</range>\n"
1330
- result_text += f" <embed>{'local' if PREFER_LOCAL_EMBEDDINGS or not voyage_client else 'voyage'}</embed>\n"
1331
-
1332
- # Add timing metadata
1333
- total_time = time.time() - start_time
1334
- result_text += f" <perf>\n"
1335
- result_text += f" <ttl>{int(total_time * 1000)}</ttl>\n"
1336
- result_text += f" <emb>{int((timing_info.get('embedding_end', 0) - timing_info.get('embedding_start', 0)) * 1000)}</emb>\n"
1337
- result_text += f" <srch>{int((timing_info.get('search_all_end', 0) - timing_info.get('search_all_start', 0)) * 1000)}</srch>\n"
1338
- result_text += f" <cols>{len(collections_to_search)}</cols>\n"
1339
- result_text += f" </perf>\n"
1340
- result_text += f" </meta>\n"
1341
-
1342
- result_text += " <results>\n"
1343
- for i, result in enumerate(all_results):
1344
- result_text += f' <r rank="{i+1}">\n'
1345
- result_text += f" <s>{result.score:.3f}</s>\n"
1346
- result_text += f" <p>{result.project_name}</p>\n"
1347
-
1348
- # Calculate relative time
1349
- timestamp_clean = result.timestamp.replace('Z', '+00:00') if result.timestamp.endswith('Z') else result.timestamp
1350
- timestamp_dt = datetime.fromisoformat(timestamp_clean)
1351
- # Ensure both datetimes are timezone-aware
1352
- if timestamp_dt.tzinfo is None:
1353
- timestamp_dt = timestamp_dt.replace(tzinfo=timezone.utc)
1354
- now = datetime.now(timezone.utc)
1355
- days_ago = (now - timestamp_dt).days
1356
- if days_ago == 0:
1357
- time_str = "today"
1358
- elif days_ago == 1:
1359
- time_str = "yesterday"
1360
- else:
1361
- time_str = f"{days_ago}d"
1362
- result_text += f" <t>{time_str}</t>\n"
1363
-
1364
- if not brief:
1365
- # Extract title from first line of excerpt
1366
- excerpt_lines = result.excerpt.split('\n')
1367
- title = excerpt_lines[0][:80] + "..." if len(excerpt_lines[0]) > 80 else excerpt_lines[0]
1368
- result_text += f" <title>{title}</title>\n"
1369
-
1370
- # Key finding - summarize the main point
1371
- key_finding = result.excerpt[:100] + "..." if len(result.excerpt) > 100 else result.excerpt
1372
- result_text += f" <key-finding>{key_finding.strip()}</key-finding>\n"
1373
-
1374
- # Always include excerpt, but shorter in brief mode
1375
- if brief:
1376
- brief_excerpt = result.excerpt[:100] + "..." if len(result.excerpt) > 100 else result.excerpt
1377
- result_text += f" <excerpt>{brief_excerpt.strip()}</excerpt>\n"
1378
- else:
1379
- result_text += f" <excerpt><![CDATA[{result.excerpt}]]></excerpt>\n"
1380
-
1381
- if result.conversation_id:
1382
- result_text += f" <cid>{result.conversation_id}</cid>\n"
1383
-
1384
- # Include raw data if requested
1385
- if include_raw and result.raw_payload:
1386
- result_text += " <raw>\n"
1387
- result_text += f" <txt><![CDATA[{result.raw_payload.get('text', '')}]]></txt>\n"
1388
- result_text += f" <id>{result.id}</id>\n"
1389
- result_text += f" <dist>{1 - result.score:.3f}</dist>\n"
1390
- result_text += " <meta>\n"
1391
- for key, value in result.raw_payload.items():
1392
- if key != 'text':
1393
- result_text += f" <{key}>{value}</{key}>\n"
1394
- result_text += " </meta>\n"
1395
- result_text += " </raw>\n"
1396
-
1397
- # Add patterns if they exist - with detailed logging
1398
- if result.code_patterns and isinstance(result.code_patterns, dict):
1399
- logger.info(f"DEBUG: Point {result.id} has code_patterns dict with keys: {list(result.code_patterns.keys())}")
1400
- patterns_to_show = []
1401
- for category, patterns in result.code_patterns.items():
1402
- if patterns and isinstance(patterns, list) and len(patterns) > 0:
1403
- # Take up to 5 patterns from each category
1404
- patterns_to_show.append((category, patterns[:5]))
1405
- logger.info(f"DEBUG: Added category '{category}' with {len(patterns)} patterns")
1406
-
1407
- if patterns_to_show:
1408
- logger.info(f"DEBUG: Adding patterns XML for point {result.id}")
1409
- result_text += " <patterns>\n"
1410
- for category, patterns in patterns_to_show:
1411
- # Escape both category name and pattern content for XML safety
1412
- safe_patterns = ', '.join(escape(str(p)) for p in patterns)
1413
- result_text += f" <cat name=\"{escape(category)}\">{safe_patterns}</cat>\n"
1414
- result_text += " </patterns>\n"
1415
- else:
1416
- logger.info(f"DEBUG: Point {result.id} has code_patterns but no valid patterns to show")
1417
- else:
1418
- logger.info(f"DEBUG: Point {result.id} has no patterns. code_patterns={result.code_patterns}, type={type(result.code_patterns)}")
1419
-
1420
- if result.files_analyzed and len(result.files_analyzed) > 0:
1421
- result_text += f" <files>{', '.join(result.files_analyzed[:5])}</files>\n"
1422
- if result.concepts and len(result.concepts) > 0:
1423
- result_text += f" <concepts>{', '.join(result.concepts[:5])}</concepts>\n"
1424
-
1425
- # Include structured metadata for agent consumption
1426
- # This provides clean, parsed fields that agents can easily use
1427
- if hasattr(result, 'raw_payload') and result.raw_payload:
1428
- import json
1429
- payload = result.raw_payload
1430
-
1431
- # Files section - structured for easy agent parsing
1432
- files_analyzed = payload.get('files_analyzed', [])
1433
- files_edited = payload.get('files_edited', [])
1434
- if files_analyzed or files_edited:
1435
- result_text += " <files>\n"
1436
- if files_analyzed:
1437
- result_text += f" <analyzed count=\"{len(files_analyzed)}\">"
1438
- result_text += ", ".join(files_analyzed[:5]) # First 5 files
1439
- if len(files_analyzed) > 5:
1440
- result_text += f" ... and {len(files_analyzed)-5} more"
1441
- result_text += "</analyzed>\n"
1442
- if files_edited:
1443
- result_text += f" <edited count=\"{len(files_edited)}\">"
1444
- result_text += ", ".join(files_edited[:5]) # First 5 files
1445
- if len(files_edited) > 5:
1446
- result_text += f" ... and {len(files_edited)-5} more"
1447
- result_text += "</edited>\n"
1448
- result_text += " </files>\n"
1449
-
1450
- # Concepts section - clean list for agents
1451
- concepts = payload.get('concepts', [])
1452
- if concepts:
1453
- result_text += f" <concepts>{', '.join(concepts)}</concepts>\n"
1454
-
1455
- # Tools section - summarized with counts
1456
- tools_used = payload.get('tools_used', [])
1457
- if tools_used:
1458
- # Count tool usage
1459
- tool_counts = {}
1460
- for tool in tools_used:
1461
- tool_counts[tool] = tool_counts.get(tool, 0) + 1
1462
- # Sort by frequency
1463
- sorted_tools = sorted(tool_counts.items(), key=lambda x: x[1], reverse=True)
1464
- tool_summary = ", ".join(f"{tool}({count})" for tool, count in sorted_tools[:5])
1465
- if len(sorted_tools) > 5:
1466
- tool_summary += f" ... and {len(sorted_tools)-5} more"
1467
- result_text += f" <tools>{tool_summary}</tools>\n"
1468
-
1469
- # Code patterns section - structured by category
1470
- code_patterns = payload.get('code_patterns', {})
1471
- if code_patterns:
1472
- result_text += " <code_patterns>\n"
1473
- for category, patterns in code_patterns.items():
1474
- if patterns:
1475
- pattern_list = patterns if isinstance(patterns, list) else [patterns]
1476
- # Clean up pattern names
1477
- clean_patterns = []
1478
- for p in pattern_list[:5]:
1479
- # Remove common prefixes like $FUNC, $VAR
1480
- clean_p = str(p).replace('$FUNC', '').replace('$VAR', '').strip()
1481
- if clean_p:
1482
- clean_patterns.append(clean_p)
1483
- if clean_patterns:
1484
- result_text += f" <{category}>{', '.join(clean_patterns)}</{category}>\n"
1485
- result_text += " </code_patterns>\n"
1486
-
1487
- # Pattern inheritance info - shows propagation details
1488
- pattern_inheritance = payload.get('pattern_inheritance', {})
1489
- if pattern_inheritance:
1490
- source_chunk = pattern_inheritance.get('source_chunk', '')
1491
- confidence = pattern_inheritance.get('confidence', 0)
1492
- distance = pattern_inheritance.get('distance', 0)
1493
- if source_chunk:
1494
- result_text += f" <pattern_source chunk=\"{source_chunk}\" confidence=\"{confidence:.2f}\" distance=\"{distance}\"/>\n"
1495
-
1496
- # Message stats for context
1497
- msg_count = payload.get('message_count')
1498
- total_length = payload.get('total_length')
1499
- if msg_count or total_length:
1500
- stats_attrs = []
1501
- if msg_count:
1502
- stats_attrs.append(f'messages="{msg_count}"')
1503
- if total_length:
1504
- stats_attrs.append(f'length="{total_length}"')
1505
- result_text += f" <stats {' '.join(stats_attrs)}/>\n"
1506
-
1507
- # Raw metadata dump for backwards compatibility
1508
- # Kept minimal - only truly unique fields
1509
- remaining_metadata = {}
1510
- excluded_keys = {'text', 'conversation_id', 'timestamp', 'role', 'project', 'chunk_index',
1511
- 'files_analyzed', 'files_edited', 'concepts', 'tools_used',
1512
- 'code_patterns', 'pattern_inheritance', 'message_count', 'total_length',
1513
- 'chunking_version', 'chunk_method', 'chunk_overlap', 'migration_type'}
1514
- for key, value in payload.items():
1515
- if key not in excluded_keys and value is not None:
1516
- if isinstance(value, set):
1517
- value = list(value)
1518
- remaining_metadata[key] = value
1519
-
1520
- if remaining_metadata:
1521
- try:
1522
- # Only include if there's actually extra data
1523
- result_text += f" <metadata_extra><![CDATA[{json.dumps(remaining_metadata, default=str)}]]></metadata_extra>\n"
1524
- except:
1525
- pass
1526
-
1527
- result_text += " </r>\n"
1528
- result_text += " </results>\n"
1529
-
1530
- # Add aggregated pattern intelligence section
1531
- if pattern_intelligence and pattern_intelligence.get('total_unique_patterns', 0) > 0:
1532
- result_text += " <pattern_intelligence>\n"
1533
-
1534
- # Summary statistics
1535
- result_text += f" <summary>\n"
1536
- result_text += f" <unique_patterns>{pattern_intelligence['total_unique_patterns']}</unique_patterns>\n"
1537
- result_text += f" <pattern_diversity>{pattern_intelligence['pattern_diversity_score']:.2f}</pattern_diversity>\n"
1538
- result_text += f" </summary>\n"
1539
-
1540
- # Most common patterns
1541
- if pattern_intelligence.get('most_common_patterns'):
1542
- result_text += " <common_patterns>\n"
1543
- for pattern, count in pattern_intelligence['most_common_patterns'][:5]:
1544
- result_text += f" <pattern count=\"{count}\">{pattern}</pattern>\n"
1545
- result_text += " </common_patterns>\n"
1546
-
1547
- # Pattern categories
1548
- if pattern_intelligence.get('category_coverage'):
1549
- result_text += " <categories>\n"
1550
- for category, count in pattern_intelligence['category_coverage'].items():
1551
- result_text += f" <cat name=\"{category}\" count=\"{count}\"/>\n"
1552
- result_text += " </categories>\n"
1553
-
1554
- # Pattern combinations insight
1555
- if pattern_intelligence.get('pattern_combinations'):
1556
- combos = pattern_intelligence['pattern_combinations']
1557
- if combos.get('async_with_error_handling'):
1558
- result_text += " <insight>Async patterns combined with error handling detected</insight>\n"
1559
- if combos.get('react_with_state'):
1560
- result_text += " <insight>React hooks with state management patterns detected</insight>\n"
1561
-
1562
- # Files referenced across results
1563
- if pattern_intelligence.get('files_referenced') and len(pattern_intelligence['files_referenced']) > 0:
1564
- result_text += f" <files_across_results>{', '.join(pattern_intelligence['files_referenced'][:10])}</files_across_results>\n"
1565
-
1566
- # Concepts discussed
1567
- if pattern_intelligence.get('concepts_discussed') and len(pattern_intelligence['concepts_discussed']) > 0:
1568
- result_text += f" <concepts_discussed>{', '.join(pattern_intelligence['concepts_discussed'][:10])}</concepts_discussed>\n"
1569
-
1570
- result_text += " </pattern_intelligence>\n"
1571
-
1572
- result_text += "</search>"
1573
-
1574
- else:
1575
- # Markdown format (original)
1576
- result_text = f"Found {len(all_results)} relevant conversation(s) for '{query}':\n\n"
1577
- for i, result in enumerate(all_results):
1578
- result_text += f"**Result {i+1}** (Score: {result.score:.3f})\n"
1579
- # Handle timezone suffix 'Z' properly
1580
- timestamp_clean = result.timestamp.replace('Z', '+00:00') if result.timestamp.endswith('Z') else result.timestamp
1581
- result_text += f"Time: {datetime.fromisoformat(timestamp_clean).strftime('%Y-%m-%d %H:%M:%S')}\n"
1582
- result_text += f"Project: {result.project_name}\n"
1583
- result_text += f"Role: {result.role}\n"
1584
- result_text += f"Excerpt: {result.excerpt}\n"
1585
- result_text += "---\n\n"
1586
-
1587
- timing_info['format_end'] = time.time()
1588
-
1589
- # Log detailed timing breakdown
1590
- await ctx.debug(f"\n=== TIMING BREAKDOWN ===")
1591
- await ctx.debug(f"Total time: {(time.time() - start_time) * 1000:.1f}ms")
1592
- await ctx.debug(f"Embedding generation: {(timing_info.get('embedding_end', 0) - timing_info.get('embedding_start', 0)) * 1000:.1f}ms")
1593
- await ctx.debug(f"Get collections: {(timing_info.get('get_collections_end', 0) - timing_info.get('get_collections_start', 0)) * 1000:.1f}ms")
1594
- await ctx.debug(f"Search all collections: {(timing_info.get('search_all_end', 0) - timing_info.get('search_all_start', 0)) * 1000:.1f}ms")
1595
- await ctx.debug(f"Sorting results: {(timing_info.get('sort_end', 0) - timing_info.get('sort_start', 0)) * 1000:.1f}ms")
1596
- await ctx.debug(f"Formatting output: {(timing_info.get('format_end', 0) - timing_info.get('format_start', 0)) * 1000:.1f}ms")
1597
-
1598
- # Log per-collection timings
1599
- await ctx.debug(f"\n=== PER-COLLECTION TIMINGS ===")
1600
- for ct in collection_timings:
1601
- duration = (ct.get('end', 0) - ct.get('start', 0)) * 1000
1602
- status = "ERROR" if 'error' in ct else "OK"
1603
- await ctx.debug(f"{ct['name']}: {duration:.1f}ms ({status})")
1604
-
1605
- return result_text
1606
-
1607
- except Exception as e:
1608
- await ctx.error(f"Search failed: {str(e)}")
1609
- return f"Failed to search conversations: {str(e)}"
1610
-
1611
-
1612
- @mcp.tool()
1613
- async def store_reflection(
1614
- ctx: Context,
1615
- content: str = Field(description="The insight or reflection to store"),
1616
- tags: List[str] = Field(default=[], description="Tags to categorize this reflection")
1617
- ) -> str:
1618
- """Store an important insight or reflection for future reference."""
1619
-
1620
- try:
1621
- # Create reflections collection name
1622
- collection_name = f"reflections{get_collection_suffix()}"
1623
-
1624
- # Get current project context
1625
- cwd = os.environ.get('MCP_CLIENT_CWD', os.getcwd())
1626
- project_path = Path(cwd)
1627
-
1628
- # Extract project name from path
1629
- project_name = None
1630
- path_parts = project_path.parts
1631
- if 'projects' in path_parts:
1632
- idx = path_parts.index('projects')
1633
- if idx + 1 < len(path_parts):
1634
- # Get all parts after 'projects' to form the project name
1635
- # This handles cases like projects/Connectiva-App/connectiva-ai
1636
- project_parts = path_parts[idx + 1:]
1637
- project_name = '/'.join(project_parts)
1638
-
1639
- # If no project detected, use the last directory name
1640
- if not project_name:
1641
- project_name = project_path.name
1642
-
1643
- # Ensure collection exists
1644
- try:
1645
- collection_info = await qdrant_client.get_collection(collection_name)
1646
- except:
1647
- # Create collection if it doesn't exist
1648
- await qdrant_client.create_collection(
1649
- collection_name=collection_name,
1650
- vectors_config=VectorParams(
1651
- size=get_embedding_dimension(),
1652
- distance=Distance.COSINE
1653
- )
1654
- )
1655
- await ctx.debug(f"Created reflections collection: {collection_name}")
1656
-
1657
- # Generate embedding for the reflection
1658
- embedding = await generate_embedding(content)
1659
-
1660
- # Create point with metadata including project context
1661
- point_id = datetime.now().timestamp()
1662
- point = PointStruct(
1663
- id=int(point_id),
1664
- vector=embedding,
1665
- payload={
1666
- "text": content,
1667
- "tags": tags,
1668
- "timestamp": datetime.now().isoformat(),
1669
- "type": "reflection",
1670
- "role": "user_reflection",
1671
- "project": project_name, # Add project context
1672
- "project_path": str(project_path) # Add full path for reference
1673
- }
1674
- )
1675
-
1676
- # Store in Qdrant
1677
- await qdrant_client.upsert(
1678
- collection_name=collection_name,
1679
- points=[point]
1680
- )
1681
-
1682
- tags_str = ', '.join(tags) if tags else 'none'
1683
- return f"Reflection stored successfully with tags: {tags_str}"
1684
-
1685
- except Exception as e:
1686
- await ctx.error(f"Store failed: {str(e)}")
1687
- return f"Failed to store reflection: {str(e)}"
1688
-
1689
-
1690
- @mcp.tool()
1691
- async def quick_search(
1692
- ctx: Context,
1693
- query: str = Field(description="The search query to find semantically similar conversations"),
1694
- min_score: float = Field(default=0.7, description="Minimum similarity score (0-1)"),
1695
- project: Optional[str] = Field(default=None, description="Search specific project only. If not provided, searches current project based on working directory. Use 'all' to search across all projects.")
1696
- ) -> str:
1697
- """Quick search that returns only the count and top result for fast overview."""
1698
- # MCP architectural limitation: MCP tools cannot call other MCP tools
1699
- return """<error>
1700
- MCP Architectural Limitation: This tool cannot directly call other MCP tools.
1701
-
1702
- To perform a quick search, please:
1703
- 1. Call reflect_on_past directly with limit=1 and brief=True
1704
- 2. Or use the reflection-specialist agent for quick searches
1705
-
1706
- This limitation exists because MCP tools can only be orchestrated by the client (Claude),
1707
- not by other tools within the MCP server.
1708
- </error>"""
1709
-
1710
-
1711
- @mcp.tool()
1712
- async def search_summary(
1713
- ctx: Context,
1714
- query: str = Field(description="The search query to find semantically similar conversations"),
1715
- project: Optional[str] = Field(default=None, description="Search specific project only. If not provided, searches current project based on working directory. Use 'all' to search across all projects.")
1716
- ) -> str:
1717
- """Get aggregated insights from search results without individual result details."""
1718
- # MCP architectural limitation: MCP tools cannot call other MCP tools
1719
- # This is a fundamental constraint of the MCP protocol
1720
- return """<error>
1721
- MCP Architectural Limitation: This tool cannot directly call other MCP tools.
1722
-
1723
- To get a search summary, please use the reflection-specialist agent instead:
1724
- 1. Call the reflection-specialist agent
1725
- 2. Ask it to provide a summary of search results for your query
1726
-
1727
- Alternative: Call reflect_on_past directly and analyze the results yourself.
1728
-
1729
- This limitation exists because MCP tools can only be orchestrated by the client (Claude),
1730
- not by other tools within the MCP server.
1731
- </error>"""
1732
-
1733
-
1734
- @mcp.tool()
1735
- async def get_more_results(
1736
- ctx: Context,
1737
- query: str = Field(description="The original search query"),
1738
- offset: int = Field(default=3, description="Number of results to skip (for pagination)"),
1739
- limit: int = Field(default=3, description="Number of additional results to return"),
1740
- min_score: float = Field(default=0.7, description="Minimum similarity score (0-1)"),
1741
- project: Optional[str] = Field(default=None, description="Search specific project only")
1742
- ) -> str:
1743
- """Get additional search results after an initial search (pagination support)."""
1744
- # MCP architectural limitation: MCP tools cannot call other MCP tools
1745
- return """<error>
1746
- MCP Architectural Limitation: This tool cannot directly call other MCP tools.
1747
-
1748
- To get more search results, please:
1749
- 1. Call reflect_on_past directly with a higher limit parameter
1750
- 2. Or use the reflection-specialist agent to handle pagination
1751
-
1752
- This limitation exists because MCP tools can only be orchestrated by the client (Claude),
1753
- not by other tools within the MCP server.
1754
- </error>"""
1755
-
1756
-
1757
- @mcp.tool()
1758
- async def search_by_file(
1759
- ctx: Context,
1760
- file_path: str = Field(description="The file path to search for in conversations"),
1761
- limit: int = Field(default=10, description="Maximum number of results to return"),
1762
- project: Optional[str] = Field(default=None, description="Search specific project only. Use 'all' to search across all projects.")
1763
- ) -> str:
1764
- """Search for conversations that analyzed a specific file."""
1765
- global qdrant_client
1766
-
1767
- # Normalize file path
1768
- normalized_path = file_path.replace("\\", "/").replace("/Users/", "~/")
1769
-
1770
- # Determine which collections to search
1771
- # If no project specified, search all collections
1772
- collections = await get_all_collections() if not project else []
1773
-
1774
- if project and project != 'all':
1775
- # Filter collections for specific project
1776
- project_hash = hashlib.md5(project.encode()).hexdigest()[:8]
1777
- collection_prefix = f"conv_{project_hash}_"
1778
- collections = [c for c in await get_all_collections() if c.startswith(collection_prefix)]
1779
- elif project == 'all':
1780
- collections = await get_all_collections()
1781
-
1782
- if not collections:
1783
- return "<search_by_file>\n<error>No collections found to search</error>\n</search_by_file>"
1784
-
1785
- # Prepare results
1786
- all_results = []
1787
-
1788
- for collection_name in collections:
1789
- try:
1790
- # Use scroll to get all points and filter manually
1791
- # Qdrant's array filtering can be tricky, so we'll filter in code
1792
- scroll_result = await qdrant_client.scroll(
1793
- collection_name=collection_name,
1794
- limit=1000, # Get a batch
1795
- with_payload=True
1796
- )
1797
-
1798
- # Filter results that contain the file
1799
- for point in scroll_result[0]:
1800
- payload = point.payload
1801
- files_analyzed = payload.get('files_analyzed', [])
1802
- files_edited = payload.get('files_edited', [])
1803
-
1804
- # Check for exact match or if any file ends with the normalized path
1805
- file_match = False
1806
- for file in files_analyzed + files_edited:
1807
- if file == normalized_path or file.endswith('/' + normalized_path) or file.endswith('\\' + normalized_path):
1808
- file_match = True
1809
- break
1810
-
1811
- if file_match:
1812
- all_results.append({
1813
- 'score': 1.0, # File match is always 1.0
1814
- 'payload': payload,
1815
- 'collection': collection_name
1816
- })
1817
-
1818
- except Exception as e:
1819
- continue
1820
-
1821
- # Sort by timestamp (newest first)
1822
- all_results.sort(key=lambda x: x['payload'].get('timestamp', ''), reverse=True)
1823
-
1824
- # Format results
1825
- if not all_results:
1826
- return f"""<search_by_file>
1827
- <query>{file_path}</query>
1828
- <normalized_path>{normalized_path}</normalized_path>
1829
- <message>No conversations found that analyzed this file</message>
1830
- </search_by_file>"""
1831
-
1832
- results_text = []
1833
- for i, result in enumerate(all_results[:limit]):
1834
- payload = result['payload']
1835
- timestamp = payload.get('timestamp', 'Unknown')
1836
- conversation_id = payload.get('conversation_id', 'Unknown')
1837
- project = payload.get('project', 'Unknown')
1838
- text_preview = payload.get('text', '')[:200] + '...' if len(payload.get('text', '')) > 200 else payload.get('text', '')
1839
-
1840
- # Check if file was edited or just read
1841
- action = "edited" if normalized_path in payload.get('files_edited', []) else "analyzed"
1842
-
1843
- # Get related tools used
1844
- tool_summary = payload.get('tool_summary', {})
1845
- tools_used = ', '.join(f"{tool}({count})" for tool, count in tool_summary.items())
1846
-
1847
- results_text.append(f"""<result rank="{i+1}">
1848
- <conversation_id>{conversation_id}</conversation_id>
1849
- <project>{project}</project>
1850
- <timestamp>{timestamp}</timestamp>
1851
- <action>{action}</action>
1852
- <tools_used>{tools_used}</tools_used>
1853
- <preview>{text_preview}</preview>
1854
- </result>""")
1855
-
1856
- return f"""<search_by_file>
1857
- <query>{file_path}</query>
1858
- <normalized_path>{normalized_path}</normalized_path>
1859
- <count>{len(all_results)}</count>
1860
- <results>
1861
- {''.join(results_text)}
1862
- </results>
1863
- </search_by_file>"""
1864
-
1865
-
1866
- @mcp.tool()
1867
- async def search_by_concept(
1868
- ctx: Context,
1869
- concept: str = Field(description="The concept to search for (e.g., 'security', 'docker', 'testing')"),
1870
- include_files: bool = Field(default=True, description="Include file information in results"),
1871
- limit: int = Field(default=10, description="Maximum number of results to return"),
1872
- project: Optional[str] = Field(default=None, description="Search specific project only. Use 'all' to search across all projects.")
1873
- ) -> str:
1874
- """Search for conversations about a specific development concept."""
1875
- global qdrant_client
1876
-
1877
- # Generate embedding for the concept
1878
- embedding = await generate_embedding(concept)
1879
-
1880
- # Determine which collections to search
1881
- # If no project specified, search all collections
1882
- collections = await get_all_collections() if not project else []
1883
-
1884
- if project and project != 'all':
1885
- # Filter collections for specific project
1886
- normalized_project = normalize_project_name(project)
1887
- project_hash = hashlib.md5(normalized_project.encode()).hexdigest()[:8]
1888
- collection_prefix = f"conv_{project_hash}_"
1889
- collections = [c for c in await get_all_collections() if c.startswith(collection_prefix)]
1890
- elif project == 'all':
1891
- collections = await get_all_collections()
1892
-
1893
- if not collections:
1894
- return "<search_by_concept>\n<error>No collections found to search</error>\n</search_by_concept>"
1895
-
1896
- # First, check metadata health
1897
- metadata_found = False
1898
- total_points_checked = 0
1899
-
1900
- for collection_name in collections[:3]: # Sample first 3 collections
1901
- try:
1902
- sample_points, _ = await qdrant_client.scroll(
1903
- collection_name=collection_name,
1904
- limit=10,
1905
- with_payload=True
1906
- )
1907
- total_points_checked += len(sample_points)
1908
- for point in sample_points:
1909
- if 'concepts' in point.payload and point.payload['concepts']:
1910
- metadata_found = True
1911
- break
1912
- if metadata_found:
1913
- break
1914
- except:
1915
- continue
1916
-
1917
- # Search all collections
1918
- all_results = []
1919
-
1920
- # If metadata exists, try metadata-based search first
1921
- if metadata_found:
1922
- for collection_name in collections:
1923
- try:
1924
- # Hybrid search: semantic + concept filter
1925
- results = await qdrant_client.search(
1926
- collection_name=collection_name,
1927
- query_vector=embedding,
1928
- query_filter=models.Filter(
1929
- should=[
1930
- models.FieldCondition(
1931
- key="concepts",
1932
- match=models.MatchAny(any=[concept.lower()])
1933
- )
1934
- ]
1935
- ),
1936
- limit=limit * 2, # Get more results for better filtering
1937
- with_payload=True
1938
- )
1939
-
1940
- for point in results:
1941
- payload = point.payload
1942
- # Boost score if concept is in the concepts list
1943
- score_boost = 0.2 if concept.lower() in payload.get('concepts', []) else 0.0
1944
- all_results.append({
1945
- 'score': float(point.score) + score_boost,
1946
- 'payload': payload,
1947
- 'collection': collection_name,
1948
- 'search_type': 'metadata'
1949
- })
1950
-
1951
- except Exception as e:
1952
- # Log unexpected errors but continue with other collections
1953
- logger.debug(f"Error searching collection {collection_name}: {e}")
1954
- continue
1955
-
1956
- # If no results from metadata search OR no metadata exists, fall back to semantic search
1957
- if not all_results:
1958
- await ctx.debug(f"Falling back to semantic search for concept: {concept}")
1959
-
1960
- for collection_name in collections:
1961
- try:
1962
- # Pure semantic search without filters
1963
- results = await qdrant_client.search(
1964
- collection_name=collection_name,
1965
- query_vector=embedding,
1966
- limit=limit,
1967
- score_threshold=0.5, # Lower threshold for broader results
1968
- with_payload=True
1969
- )
1970
-
1971
- for point in results:
1972
- all_results.append({
1973
- 'score': float(point.score),
1974
- 'payload': point.payload,
1975
- 'collection': collection_name,
1976
- 'search_type': 'semantic'
1977
- })
1978
-
1979
- except Exception as e:
1980
- # Log unexpected errors but continue with other collections
1981
- logger.debug(f"Error searching collection {collection_name}: {e}")
1982
- continue
1983
-
1984
- # Sort by score and limit
1985
- all_results.sort(key=lambda x: x['score'], reverse=True)
1986
- all_results = all_results[:limit]
1987
-
1988
- # Format results
1989
- if not all_results:
1990
- metadata_status = "with metadata" if metadata_found else "NO METADATA FOUND"
1991
- return f"""<search_by_concept>
1992
- <concept>{concept}</concept>
1993
- <metadata_health>{metadata_status} (checked {total_points_checked} points)</metadata_health>
1994
- <message>No conversations found about this concept. {'Try running: python scripts/delta-metadata-update.py' if not metadata_found else 'Try different search terms.'}</message>
1995
- </search_by_concept>"""
1996
-
1997
- results_text = []
1998
- for i, result in enumerate(all_results):
1999
- payload = result['payload']
2000
- score = result['score']
2001
- timestamp = payload.get('timestamp', 'Unknown')
2002
- conversation_id = payload.get('conversation_id', 'Unknown')
2003
- project = payload.get('project', 'Unknown')
2004
- concepts = payload.get('concepts', [])
2005
-
2006
- # Get text preview
2007
- text_preview = payload.get('text', '')[:200] + '...' if len(payload.get('text', '')) > 200 else payload.get('text', '')
2008
-
2009
- # File information
2010
- files_info = ""
2011
- if include_files:
2012
- files_analyzed = payload.get('files_analyzed', [])[:5]
2013
- if files_analyzed:
2014
- files_info = f"\n<files_analyzed>{', '.join(files_analyzed)}</files_analyzed>"
2015
-
2016
- # Related concepts
2017
- related_concepts = [c for c in concepts if c != concept.lower()][:5]
2018
-
2019
- results_text.append(f"""<result rank="{i+1}">
2020
- <score>{score:.3f}</score>
2021
- <conversation_id>{conversation_id}</conversation_id>
2022
- <project>{project}</project>
2023
- <timestamp>{timestamp}</timestamp>
2024
- <concepts>{', '.join(concepts)}</concepts>
2025
- <related_concepts>{', '.join(related_concepts)}</related_concepts>{files_info}
2026
- <preview>{text_preview}</preview>
2027
- </result>""")
2028
-
2029
- # Determine if this was a fallback search
2030
- used_fallback = any(r.get('search_type') == 'semantic' for r in all_results)
2031
- metadata_status = "with metadata" if metadata_found else "NO METADATA FOUND"
2032
-
2033
- return f"""<search_by_concept>
2034
- <concept>{concept}</concept>
2035
- <metadata_health>{metadata_status} (checked {total_points_checked} points)</metadata_health>
2036
- <search_type>{'fallback_semantic' if used_fallback else 'metadata_based'}</search_type>
2037
- <count>{len(all_results)}</count>
2038
- <results>
2039
- {''.join(results_text)}
2040
- </results>
2041
- </search_by_concept>"""
2042
-
2043
-
2044
- # Debug output
2045
- print(f"[DEBUG] FastMCP server created with name: {mcp.name}")
2046
-
2047
- @mcp.tool()
2048
- async def get_full_conversation(
2049
- ctx: Context,
2050
- conversation_id: str = Field(description="The conversation ID from search results (cid)"),
2051
- project: Optional[str] = Field(default=None, description="Optional project name to help locate the file")
2052
- ) -> str:
2053
- """Get the full JSONL conversation file path for a conversation ID.
2054
- This allows agents to read complete conversations instead of truncated excerpts."""
2055
-
2056
- # Base path for Claude conversations
2057
- base_path = Path.home() / '.claude/projects'
2058
-
2059
- # Build list of directories to search
2060
- search_dirs = []
2061
-
2062
- if project:
2063
- # Try various project directory name formats
2064
- sanitized_project = project.replace('/', '-')
2065
- search_dirs.extend([
2066
- base_path / project,
2067
- base_path / sanitized_project,
2068
- base_path / f"-Users-*-projects-{project}",
2069
- base_path / f"-Users-*-projects-{sanitized_project}"
2070
- ])
2071
- else:
2072
- # Search all project directories
2073
- search_dirs = list(base_path.glob("*"))
2074
-
2075
- # Search for the JSONL file
2076
- jsonl_path = None
2077
- for search_dir in search_dirs:
2078
- if not search_dir.is_dir():
2079
- continue
2080
-
2081
- potential_path = search_dir / f"{conversation_id}.jsonl"
2082
- if potential_path.exists():
2083
- jsonl_path = potential_path
2084
- break
2085
-
2086
- if not jsonl_path:
2087
- # Try searching all directories as fallback
2088
- for proj_dir in base_path.glob("*"):
2089
- if proj_dir.is_dir():
2090
- potential_path = proj_dir / f"{conversation_id}.jsonl"
2091
- if potential_path.exists():
2092
- jsonl_path = potential_path
2093
- break
2094
-
2095
- if not jsonl_path:
2096
- return f"""<full_conversation>
2097
- <conversation_id>{conversation_id}</conversation_id>
2098
- <status>not_found</status>
2099
- <message>Conversation file not found. Searched {len(search_dirs)} directories.</message>
2100
- <hint>Try using the project parameter or check if the conversation ID is correct.</hint>
2101
- </full_conversation>"""
2102
-
2103
- # Get file stats
2104
- file_stats = jsonl_path.stat()
2105
-
2106
- # Count messages
2107
- try:
2108
- with open(jsonl_path, 'r', encoding='utf-8') as f:
2109
- message_count = sum(1 for _ in f)
2110
- except:
2111
- message_count = 0
2112
-
2113
- return f"""<full_conversation>
2114
- <conversation_id>{conversation_id}</conversation_id>
2115
- <status>found</status>
2116
- <file_path>{jsonl_path}</file_path>
2117
- <file_size>{file_stats.st_size}</file_size>
2118
- <message_count>{message_count}</message_count>
2119
- <project>{jsonl_path.parent.name}</project>
2120
- <instructions>
2121
- You can now use the Read tool to read the full conversation from:
2122
- {jsonl_path}
2123
-
2124
- Each line in the JSONL file is a separate message with complete content.
2125
- This gives you access to:
2126
- - Complete code blocks (not truncated)
2127
- - Full problem descriptions and solutions
2128
- - Entire debugging sessions
2129
- - Complete architectural decisions and discussions
2130
- </instructions>
2131
- </full_conversation>"""
2132
652
 
653
+ # Register temporal tools after all functions are defined
654
+ register_temporal_tools(
655
+ mcp,
656
+ qdrant_client,
657
+ QDRANT_URL,
658
+ get_all_collections,
659
+ generate_embedding,
660
+ initialize_embeddings,
661
+ normalize_project_name
662
+ )
663
+ print(f"[INFO] Temporal tools registered", file=sys.stderr)
664
+
665
+ # Register search tools
666
+ def get_embedding_manager():
667
+ """Factory function to get the current embedding manager."""
668
+ from .embedding_manager import get_embedding_manager as get_em
669
+ return get_em()
670
+
671
+ # Initialize ProjectResolver for collection name mapping
672
+ # ProjectResolver needs a sync client, not async
673
+ from qdrant_client import QdrantClient as SyncQdrantClient
674
+ sync_qdrant_client = SyncQdrantClient(url=QDRANT_URL)
675
+ project_resolver = ProjectResolver(sync_qdrant_client)
676
+
677
+ register_search_tools(
678
+ mcp,
679
+ qdrant_client,
680
+ QDRANT_URL,
681
+ get_embedding_manager,
682
+ normalize_project_name,
683
+ ENABLE_MEMORY_DECAY,
684
+ DECAY_WEIGHT,
685
+ DECAY_SCALE_DAYS,
686
+ USE_NATIVE_DECAY,
687
+ NATIVE_DECAY_AVAILABLE,
688
+ decay_manager,
689
+ project_resolver # Pass the resolver
690
+ )
2133
691
 
2134
- @mcp.tool()
2135
- async def get_next_results(
2136
- ctx: Context,
2137
- query: str = Field(description="The original search query"),
2138
- offset: int = Field(default=3, description="Number of results to skip (for pagination)"),
2139
- limit: int = Field(default=3, description="Number of additional results to return"),
2140
- min_score: float = Field(default=0.7, description="Minimum similarity score (0-1)"),
2141
- project: Optional[str] = Field(default=None, description="Search specific project only")
2142
- ) -> str:
2143
- """Get additional search results after an initial search (pagination support)."""
2144
- global qdrant_client, embedding_manager
2145
-
2146
- try:
2147
- # Generate embedding for the query
2148
- embedding = await generate_embedding(query)
2149
-
2150
- # Determine which collections to search
2151
- if project == "all" or not project:
2152
- # Search all collections if project is "all" or not specified
2153
- collections = await get_all_collections()
2154
- else:
2155
- # Search specific project
2156
- all_collections = await get_all_collections()
2157
- project_hash = hashlib.md5(project.encode()).hexdigest()[:8]
2158
- collections = [
2159
- c for c in all_collections
2160
- if c.startswith(f"conv_{project_hash}_")
2161
- ]
2162
- if not collections:
2163
- # Fall back to searching all collections
2164
- collections = all_collections
2165
-
2166
- if not collections:
2167
- return """<next_results>
2168
- <error>No collections available to search</error>
2169
- </next_results>"""
2170
-
2171
- # Collect all results from all collections
2172
- all_results = []
2173
- for collection_name in collections:
2174
- try:
2175
- # Check if collection exists
2176
- collection_info = await qdrant_client.get_collection(collection_name)
2177
- if not collection_info:
2178
- continue
2179
-
2180
- # Search with reasonable limit to account for offset
2181
- max_search_limit = 100 # Define a reasonable cap
2182
- search_limit = min(offset + limit + 10, max_search_limit)
2183
- results = await qdrant_client.search(
2184
- collection_name=collection_name,
2185
- query_vector=embedding,
2186
- limit=search_limit,
2187
- score_threshold=min_score
2188
- )
2189
-
2190
- for point in results:
2191
- payload = point.payload
2192
- score = float(point.score)
2193
-
2194
- # Apply time-based decay if enabled
2195
- use_decay_bool = ENABLE_MEMORY_DECAY # Use global default
2196
- if use_decay_bool and 'timestamp' in payload:
2197
- try:
2198
- timestamp = datetime.fromisoformat(payload['timestamp'].replace('Z', '+00:00'))
2199
- age_days = (datetime.now(timezone.utc) - timestamp).days
2200
- decay_factor = DECAY_WEIGHT + (1 - DECAY_WEIGHT) * math.exp(-age_days / DECAY_SCALE_DAYS)
2201
- score = score * decay_factor
2202
- except (ValueError, TypeError) as e:
2203
- # Log but continue - timestamp format issue shouldn't break search
2204
- logger.debug(f"Failed to apply decay for timestamp {payload.get('timestamp')}: {e}")
2205
-
2206
- all_results.append({
2207
- 'score': score,
2208
- 'payload': payload,
2209
- 'collection': collection_name
2210
- })
2211
-
2212
- except Exception as e:
2213
- # Log unexpected errors but continue with other collections
2214
- logger.debug(f"Error searching collection {collection_name}: {e}")
2215
- continue
2216
-
2217
- # Sort by score
2218
- all_results.sort(key=lambda x: x['score'], reverse=True)
2219
-
2220
- # Apply pagination
2221
- paginated_results = all_results[offset:offset + limit]
2222
-
2223
- if not paginated_results:
2224
- return f"""<next_results>
2225
- <query>{query}</query>
2226
- <offset>{offset}</offset>
2227
- <status>no_more_results</status>
2228
- <message>No additional results found beyond offset {offset}</message>
2229
- </next_results>"""
2230
-
2231
- # Format results
2232
- results_text = []
2233
- for i, result in enumerate(paginated_results, start=offset + 1):
2234
- payload = result['payload']
2235
- score = result['score']
2236
- timestamp = payload.get('timestamp', 'Unknown')
2237
- conversation_id = payload.get('conversation_id', 'Unknown')
2238
- project = payload.get('project', 'Unknown')
2239
-
2240
- # Get text preview (store text once to avoid multiple calls)
2241
- text = payload.get('text', '')
2242
- text_preview = text[:300] + '...' if len(text) > 300 else text
2243
-
2244
- results_text.append(f"""
2245
- <result index="{i}">
2246
- <score>{score:.3f}</score>
2247
- <timestamp>{timestamp}</timestamp>
2248
- <project>{project}</project>
2249
- <conversation_id>{conversation_id}</conversation_id>
2250
- <preview>{text_preview}</preview>
2251
- </result>""")
2252
-
2253
- # Check if there are more results available
2254
- has_more = len(all_results) > (offset + limit)
2255
- next_offset = offset + limit if has_more else None
2256
-
2257
- return f"""<next_results>
2258
- <query>{query}</query>
2259
- <offset>{offset}</offset>
2260
- <limit>{limit}</limit>
2261
- <count>{len(paginated_results)}</count>
2262
- <total_available>{len(all_results)}</total_available>
2263
- <has_more>{has_more}</has_more>
2264
- {f'<next_offset>{next_offset}</next_offset>' if next_offset else ''}
2265
- <results>{''.join(results_text)}
2266
- </results>
2267
- </next_results>"""
2268
-
2269
- except Exception as e:
2270
- await ctx.error(f"Pagination failed: {str(e)}")
2271
- return f"""<next_results>
2272
- <error>Failed to get next results: {str(e)}</error>
2273
- </next_results>"""
2274
-
692
+ # Register reflection tools
693
+ register_reflection_tools(
694
+ mcp,
695
+ qdrant_client,
696
+ QDRANT_URL,
697
+ get_embedding_manager,
698
+ normalize_project_name
699
+ )
2275
700
 
2276
701
  # Run the server
2277
702
  if __name__ == "__main__":