claude-self-reflect 3.2.4 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/.claude/agents/claude-self-reflect-test.md +595 -528
  2. package/.claude/agents/reflection-specialist.md +59 -3
  3. package/README.md +14 -5
  4. package/mcp-server/run-mcp.sh +49 -5
  5. package/mcp-server/src/app_context.py +64 -0
  6. package/mcp-server/src/config.py +57 -0
  7. package/mcp-server/src/connection_pool.py +286 -0
  8. package/mcp-server/src/decay_manager.py +106 -0
  9. package/mcp-server/src/embedding_manager.py +64 -40
  10. package/mcp-server/src/embeddings_old.py +141 -0
  11. package/mcp-server/src/models.py +64 -0
  12. package/mcp-server/src/parallel_search.py +371 -0
  13. package/mcp-server/src/project_resolver.py +5 -0
  14. package/mcp-server/src/reflection_tools.py +206 -0
  15. package/mcp-server/src/rich_formatting.py +196 -0
  16. package/mcp-server/src/search_tools.py +826 -0
  17. package/mcp-server/src/server.py +127 -1720
  18. package/mcp-server/src/temporal_design.py +132 -0
  19. package/mcp-server/src/temporal_tools.py +597 -0
  20. package/mcp-server/src/temporal_utils.py +384 -0
  21. package/mcp-server/src/utils.py +150 -67
  22. package/package.json +10 -1
  23. package/scripts/add-timestamp-indexes.py +134 -0
  24. package/scripts/check-collections.py +29 -0
  25. package/scripts/debug-august-parsing.py +76 -0
  26. package/scripts/debug-import-single.py +91 -0
  27. package/scripts/debug-project-resolver.py +82 -0
  28. package/scripts/debug-temporal-tools.py +135 -0
  29. package/scripts/delta-metadata-update.py +547 -0
  30. package/scripts/import-conversations-unified.py +53 -2
  31. package/scripts/precompact-hook.sh +33 -0
  32. package/scripts/streaming-watcher.py +1443 -0
  33. package/scripts/utils.py +39 -0
@@ -4,9 +4,8 @@ import os
4
4
  import asyncio
5
5
  from pathlib import Path
6
6
  from typing import Any, Optional, List, Dict, Union
7
- from datetime import datetime, timezone
7
+ from datetime import datetime, timezone, timedelta
8
8
  import json
9
- import numpy as np
10
9
  import hashlib
11
10
  import time
12
11
  import logging
@@ -27,10 +26,14 @@ except ImportError:
27
26
  logging.warning("Using legacy utils.normalize_project_name - shared module not found")
28
27
 
29
28
  from .project_resolver import ProjectResolver
29
+ from .temporal_utils import SessionDetector, TemporalParser, WorkSession, group_by_time_period
30
+ from .temporal_tools import register_temporal_tools
31
+ from .search_tools import register_search_tools
32
+ from .reflection_tools import register_reflection_tools
30
33
  from pydantic import BaseModel, Field
31
34
  from qdrant_client import AsyncQdrantClient, models
32
35
  from qdrant_client.models import (
33
- PointStruct, VectorParams, Distance
36
+ PointStruct, VectorParams, Distance, OrderBy
34
37
  )
35
38
 
36
39
  # Try to import newer Qdrant API for native decay
@@ -85,11 +88,49 @@ EMBEDDING_MODEL = os.getenv('EMBEDDING_MODEL', 'sentence-transformers/all-MiniLM
85
88
  # Import the robust embedding manager
86
89
  from .embedding_manager import get_embedding_manager
87
90
 
91
+ # Import new performance modules with fallback
92
+ try:
93
+ from .connection_pool import QdrantConnectionPool, CircuitBreaker
94
+ CONNECTION_POOL_AVAILABLE = True
95
+ except ImportError:
96
+ CONNECTION_POOL_AVAILABLE = False
97
+ logging.warning("Connection pool module not available")
98
+
99
+ try:
100
+ from .parallel_search import parallel_search_collections
101
+ PARALLEL_SEARCH_AVAILABLE = True
102
+ except ImportError:
103
+ PARALLEL_SEARCH_AVAILABLE = False
104
+ logging.warning("Parallel search module not available")
105
+
106
+ # Set default configuration values
107
+ MAX_RESULTS_PER_COLLECTION = 10
108
+ MAX_TOTAL_RESULTS = 1000
109
+ MAX_MEMORY_MB = 500
110
+ POOL_SIZE = 10
111
+ POOL_MAX_OVERFLOW = 5
112
+ POOL_TIMEOUT = 30.0
113
+ RETRY_ATTEMPTS = 3
114
+ RETRY_DELAY = 1.0
115
+ MAX_CONCURRENT_SEARCHES = 10
116
+ ENABLE_PARALLEL_SEARCH = True
117
+
118
+ try:
119
+ from .decay_manager import DecayManager
120
+ DECAY_MANAGER_AVAILABLE = True
121
+ except ImportError:
122
+ DECAY_MANAGER_AVAILABLE = False
123
+ logging.warning("Decay manager module not available")
124
+
88
125
  # Lazy initialization - models will be loaded on first use
89
126
  embedding_manager = None
90
127
  voyage_client = None # Keep for backward compatibility
91
128
  local_embedding_model = None # Keep for backward compatibility
92
129
 
130
+ # Initialize connection pool
131
+ qdrant_pool = None
132
+ circuit_breaker = None
133
+
93
134
  def initialize_embeddings():
94
135
  """Initialize embedding models with robust fallback."""
95
136
  global embedding_manager, voyage_client, local_embedding_model
@@ -101,7 +142,7 @@ def initialize_embeddings():
101
142
  if embedding_manager.model_type == 'voyage':
102
143
  voyage_client = embedding_manager.voyage_client
103
144
  elif embedding_manager.model_type == 'local':
104
- local_embedding_model = embedding_manager.model
145
+ local_embedding_model = embedding_manager.local_model
105
146
 
106
147
  return True
107
148
  except Exception as e:
@@ -109,9 +150,8 @@ def initialize_embeddings():
109
150
  return False
110
151
 
111
152
  # Debug environment loading and startup
112
- import sys
113
- import datetime as dt
114
- startup_time = dt.datetime.now().isoformat()
153
+ # Debug environment loading and startup
154
+ startup_time = datetime.now(timezone.utc).isoformat()
115
155
  print(f"[STARTUP] MCP Server starting at {startup_time}", file=sys.stderr)
116
156
  print(f"[STARTUP] Python: {sys.version}", file=sys.stderr)
117
157
  print(f"[STARTUP] Working directory: {os.getcwd()}", file=sys.stderr)
@@ -152,8 +192,32 @@ mcp = FastMCP(
152
192
  instructions="Search past conversations and store reflections with time-based memory decay"
153
193
  )
154
194
 
155
- # Create Qdrant client
156
- qdrant_client = AsyncQdrantClient(url=QDRANT_URL)
195
+ # Initialize Qdrant client with connection pooling if available
196
+ if CONNECTION_POOL_AVAILABLE and ENABLE_PARALLEL_SEARCH:
197
+ qdrant_pool = QdrantConnectionPool(
198
+ url=QDRANT_URL,
199
+ pool_size=POOL_SIZE,
200
+ max_overflow=POOL_MAX_OVERFLOW,
201
+ timeout=POOL_TIMEOUT,
202
+ retry_attempts=RETRY_ATTEMPTS,
203
+ retry_delay=RETRY_DELAY
204
+ )
205
+ # Create a wrapper for backward compatibility
206
+ qdrant_client = AsyncQdrantClient(url=QDRANT_URL)
207
+ circuit_breaker = CircuitBreaker(failure_threshold=5, recovery_timeout=60.0)
208
+ print(f"[INFO] Connection pool initialized with size {POOL_SIZE}", file=sys.stderr)
209
+ else:
210
+ # Fallback to single client
211
+ qdrant_client = AsyncQdrantClient(url=QDRANT_URL)
212
+ qdrant_pool = None
213
+ circuit_breaker = None
214
+ print(f"[INFO] Using single Qdrant client (no pooling)", file=sys.stderr)
215
+
216
+ # Initialize decay manager if available
217
+ decay_manager = None
218
+ if DECAY_MANAGER_AVAILABLE:
219
+ decay_manager = DecayManager()
220
+ print(f"[INFO] Decay manager initialized", file=sys.stderr)
157
221
 
158
222
  # Add MCP Resources for system status
159
223
  @mcp.resource("status://import-stats")
@@ -492,7 +556,14 @@ def get_embedding_dimension() -> int:
492
556
 
493
557
  def get_collection_suffix() -> str:
494
558
  """Get the collection suffix based on embedding provider."""
495
- if PREFER_LOCAL_EMBEDDINGS or not voyage_client:
559
+ # Use embedding_manager's model type if available
560
+ if embedding_manager and hasattr(embedding_manager, 'model_type'):
561
+ if embedding_manager.model_type == 'voyage':
562
+ return "_voyage"
563
+ else:
564
+ return "_local"
565
+ # Fallback to environment variable
566
+ elif PREFER_LOCAL_EMBEDDINGS:
496
567
  return "_local"
497
568
  else:
498
569
  return "_voyage"
@@ -578,1718 +649,54 @@ def aggregate_pattern_intelligence(results: List[SearchResult]) -> Dict[str, Any
578
649
  return intelligence
579
650
 
580
651
  # Register tools
581
- @mcp.tool()
582
- async def reflect_on_past(
583
- ctx: Context,
584
- query: str = Field(description="The search query to find semantically similar conversations"),
585
- limit: int = Field(default=5, description="Maximum number of results to return"),
586
- min_score: float = Field(default=0.3, description="Minimum similarity score (0-1)"),
587
- use_decay: Union[int, str] = Field(default=-1, description="Apply time-based decay: 1=enable, 0=disable, -1=use environment default (accepts int or str)"),
588
- project: Optional[str] = Field(default=None, description="Search specific project only. If not provided, searches current project based on working directory. Use 'all' to search across all projects."),
589
- include_raw: bool = Field(default=False, description="Include raw Qdrant payload data for debugging (increases response size)"),
590
- response_format: str = Field(default="xml", description="Response format: 'xml' or 'markdown'"),
591
- brief: bool = Field(default=False, description="Brief mode: returns minimal information for faster response"),
592
- mode: str = Field(default="full", description="Search mode: 'full' (all results with details), 'quick' (count + top result only), 'summary' (aggregated insights without individual results)")
593
- ) -> str:
594
- """Search for relevant past conversations using semantic search with optional time decay."""
595
-
596
- logger.info(f"=== SEARCH START === Query: '{query}', Project: '{project}', Limit: {limit}")
597
-
598
- # Validate mode parameter
599
- if mode not in ['full', 'quick', 'summary']:
600
- return f"<error>Invalid mode '{mode}'. Must be 'full', 'quick', or 'summary'</error>"
601
-
602
- # Start timing
603
- start_time = time.time()
604
- timing_info = {}
605
-
606
- # Normalize use_decay to integer
607
- timing_info['param_parsing_start'] = time.time()
608
- if isinstance(use_decay, str):
609
- try:
610
- use_decay = int(use_decay)
611
- except ValueError:
612
- raise ValueError("use_decay must be '1', '0', or '-1'")
613
- timing_info['param_parsing_end'] = time.time()
614
-
615
- # Parse decay parameter using integer approach
616
- should_use_decay = (
617
- True if use_decay == 1
618
- else False if use_decay == 0
619
- else ENABLE_MEMORY_DECAY # -1 or any other value
620
- )
621
-
622
- # Determine project scope
623
- target_project = project
624
-
625
- # Always get the working directory for logging purposes
626
- cwd = os.environ.get('MCP_CLIENT_CWD', os.getcwd())
627
- await ctx.debug(f"CWD: {cwd}, Project param: {project}")
628
-
629
- if project is None:
630
- # Use MCP_CLIENT_CWD environment variable set by run-mcp.sh
631
- # This contains the actual working directory where Claude Code is running
632
-
633
- # Extract project name from path (e.g., /Users/.../projects/project-name)
634
- path_parts = Path(cwd).parts
635
- if 'projects' in path_parts:
636
- idx = path_parts.index('projects')
637
- if idx + 1 < len(path_parts):
638
- target_project = path_parts[idx + 1]
639
- elif '.claude' in path_parts:
640
- # If we're in a .claude directory, go up to find project
641
- for i, part in enumerate(path_parts):
642
- if part == '.claude' and i > 0:
643
- target_project = path_parts[i - 1]
644
- break
645
-
646
- # If still no project detected, use the last directory name
647
- if target_project is None:
648
- target_project = Path(cwd).name
649
-
650
- await ctx.debug(f"Auto-detected project from path: {target_project}")
651
-
652
- # For project matching, we need to handle the dash-encoded format
653
- # Convert folder name to the format used in stored data
654
- if target_project != 'all':
655
- # The stored format uses full path with dashes, so we need to construct it
656
- # For now, let's try to match based on the end of the project name
657
- pass # We'll handle this differently in the filtering logic
658
-
659
- await ctx.debug(f"Searching for: {query}")
660
- await ctx.debug(f"Client working directory: {cwd}")
661
- await ctx.debug(f"Project scope: {target_project if target_project != 'all' else 'all projects'}")
662
- await ctx.debug(f"Decay enabled: {should_use_decay}")
663
- await ctx.debug(f"Native decay mode: {USE_NATIVE_DECAY}")
664
- await ctx.debug(f"ENABLE_MEMORY_DECAY env: {ENABLE_MEMORY_DECAY}")
665
- await ctx.debug(f"DECAY_WEIGHT: {DECAY_WEIGHT}, DECAY_SCALE_DAYS: {DECAY_SCALE_DAYS}")
666
-
667
- try:
668
- # We'll generate embeddings on-demand per collection type
669
- timing_info['embedding_prep_start'] = time.time()
670
- query_embeddings = {} # Cache embeddings by type
671
- timing_info['embedding_prep_end'] = time.time()
672
-
673
- # Get all collections
674
- timing_info['get_collections_start'] = time.time()
675
- all_collections = await get_all_collections()
676
- timing_info['get_collections_end'] = time.time()
677
-
678
- if not all_collections:
679
- return "No conversation collections found. Please import conversations first."
680
-
681
- # Filter collections by project if not searching all
682
- project_collections = [] # Define at this scope for later use
683
- if target_project != 'all':
684
- # Use ProjectResolver with sync client (resolver expects sync operations)
685
- from qdrant_client import QdrantClient as SyncQdrantClient
686
- sync_client = SyncQdrantClient(url=QDRANT_URL)
687
- resolver = ProjectResolver(sync_client)
688
- project_collections = resolver.find_collections_for_project(target_project)
689
-
690
- if not project_collections:
691
- # Fall back to old method for backward compatibility
692
- normalized_name = normalize_project_name(target_project)
693
- project_hash = hashlib.md5(normalized_name.encode()).hexdigest()[:8]
694
- project_collections = [
695
- c for c in all_collections
696
- if c.startswith(f"conv_{project_hash}_")
697
- ]
698
-
699
- # Always include reflections collections when searching a specific project
700
- reflections_collections = [c for c in all_collections if c.startswith('reflections')]
701
-
702
- if not project_collections:
703
- # Fall back to searching all collections but filtering by project metadata
704
- await ctx.debug(f"No collections found for project {target_project}, will filter by metadata")
705
- collections_to_search = all_collections
706
- else:
707
- await ctx.debug(f"Found {len(project_collections)} collections for project {target_project}")
708
- # Include both project collections and reflections
709
- collections_to_search = project_collections + reflections_collections
710
- # Remove duplicates
711
- collections_to_search = list(set(collections_to_search))
712
- else:
713
- collections_to_search = all_collections
714
-
715
- await ctx.debug(f"Searching across {len(collections_to_search)} collections")
716
- await ctx.debug(f"Using {'local' if PREFER_LOCAL_EMBEDDINGS or not voyage_client else 'Voyage AI'} embeddings")
717
-
718
- all_results = []
719
-
720
- # Search each collection
721
- timing_info['search_all_start'] = time.time()
722
- collection_timings = []
723
-
724
- # Report initial progress
725
- await ctx.report_progress(progress=0, total=len(collections_to_search))
726
-
727
- for idx, collection_name in enumerate(collections_to_search):
728
- collection_timing = {'name': collection_name, 'start': time.time()}
729
-
730
- # Report progress before searching each collection
731
- await ctx.report_progress(
732
- progress=idx,
733
- total=len(collections_to_search),
734
- message=f"Searching {collection_name}"
735
- )
736
-
737
- try:
738
- # Determine embedding type for this collection
739
- embedding_type_for_collection = 'voyage' if collection_name.endswith('_voyage') else 'local'
740
-
741
- # Generate or retrieve cached embedding for this type
742
- if embedding_type_for_collection not in query_embeddings:
743
- try:
744
- query_embeddings[embedding_type_for_collection] = await generate_embedding(query, force_type=embedding_type_for_collection)
745
- except Exception as e:
746
- await ctx.debug(f"Failed to generate {embedding_type_for_collection} embedding: {e}")
747
- continue
748
-
749
- query_embedding = query_embeddings[embedding_type_for_collection]
750
-
751
- if should_use_decay and USE_NATIVE_DECAY and NATIVE_DECAY_AVAILABLE:
752
- # Use native Qdrant decay with newer API
753
- await ctx.debug(f"Using NATIVE Qdrant decay (new API) for {collection_name}")
754
-
755
- # Build the query with native Qdrant decay formula using newer API
756
- # Convert half-life to seconds (Qdrant uses seconds for datetime)
757
- half_life_seconds = DECAY_SCALE_DAYS * 24 * 60 * 60
758
-
759
- # Build query using proper Python models as per Qdrant docs
760
- from qdrant_client import models
761
-
762
- query_obj = models.FormulaQuery(
763
- formula=models.SumExpression(
764
- sum=[
765
- "$score", # Original similarity score
766
- models.MultExpression(
767
- mult=[
768
- DECAY_WEIGHT, # Weight multiplier
769
- models.ExpDecayExpression(
770
- exp_decay=models.DecayParamsExpression(
771
- x=models.DatetimeKeyExpression(
772
- datetime_key="timestamp" # Payload field with datetime
773
- ),
774
- target=models.DatetimeExpression(
775
- datetime="now" # Current time on server
776
- ),
777
- scale=half_life_seconds, # Scale in seconds
778
- midpoint=0.5 # Half-life semantics
779
- )
780
- )
781
- ]
782
- )
783
- ]
784
- )
785
- )
786
-
787
- # Execute query with native decay (new API)
788
- results = await qdrant_client.query_points(
789
- collection_name=collection_name,
790
- query=query_obj,
791
- limit=limit,
792
- with_payload=True
793
- # No score_threshold - let Qdrant's decay formula handle relevance
794
- )
795
- elif should_use_decay and USE_NATIVE_DECAY and not NATIVE_DECAY_AVAILABLE:
796
- # Use native Qdrant decay with older API
797
- await ctx.debug(f"Using NATIVE Qdrant decay (legacy API) for {collection_name}")
798
-
799
- # Convert half-life to seconds (Qdrant uses seconds for datetime)
800
- half_life_seconds = DECAY_SCALE_DAYS * 24 * 60 * 60
801
-
802
- # Build the query with native Qdrant decay formula using older API
803
- # Use the same models but with FormulaQuery
804
- query_obj = FormulaQuery(
805
- nearest=query_embedding,
806
- formula=SumExpression(
807
- sum=[
808
- "$score", # Original similarity score
809
- {
810
- "mult": [
811
- DECAY_WEIGHT, # Weight multiplier
812
- {
813
- "exp_decay": DecayParamsExpression(
814
- x=DatetimeKeyExpression(datetime_key="timestamp"),
815
- target=DatetimeExpression(datetime="now"),
816
- scale=half_life_seconds, # Scale in seconds
817
- midpoint=0.5 # Half-life semantics
818
- )
819
- }
820
- ]
821
- }
822
- ]
823
- )
824
- )
825
-
826
- # Execute query with native decay
827
- results = await qdrant_client.query_points(
828
- collection_name=collection_name,
829
- query=query_obj,
830
- limit=limit,
831
- with_payload=True
832
- # No score_threshold - let Qdrant's decay formula handle relevance
833
- )
834
-
835
- # Process results from native decay search
836
- for point in results.points:
837
- # Clean timestamp for proper parsing
838
- raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
839
- clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
840
-
841
- # Check project filter if we're searching all collections but want specific project
842
- point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
843
-
844
- # Special handling for reflections - they're global by default but can have project context
845
- is_reflection_collection = collection_name.startswith('reflections')
846
-
847
- # Handle project matching - check if the target project name appears at the end of the stored project path
848
- if target_project != 'all' and not project_collections and not is_reflection_collection:
849
- # The stored project name is like "-Users-username-projects-ShopifyMCPMockShop"
850
- # We want to match just "ShopifyMCPMockShop"
851
- # Also handle underscore/dash variations (procsolve-website vs procsolve_website)
852
- normalized_target = target_project.replace('-', '_')
853
- normalized_stored = point_project.replace('-', '_')
854
- if not (normalized_stored.endswith(f"_{normalized_target}") or
855
- normalized_stored == normalized_target or
856
- point_project.endswith(f"-{target_project}") or
857
- point_project == target_project):
858
- continue # Skip results from other projects
859
-
860
- # For reflections with project context, optionally filter by project
861
- if is_reflection_collection and target_project != 'all' and 'project' in point.payload:
862
- # Only filter if the reflection has project metadata
863
- reflection_project = point.payload.get('project', '')
864
- if reflection_project:
865
- # Normalize both for comparison (handle underscore/dash variations)
866
- normalized_target = target_project.replace('-', '_')
867
- normalized_reflection = reflection_project.replace('-', '_')
868
- if not (
869
- reflection_project == target_project or
870
- normalized_reflection == normalized_target or
871
- reflection_project.endswith(f"/{target_project}") or
872
- reflection_project.endswith(f"-{target_project}") or
873
- normalized_reflection.endswith(f"_{normalized_target}") or
874
- normalized_reflection.endswith(f"/{normalized_target}")
875
- ):
876
- continue # Skip reflections from other projects
877
-
878
- # Log pattern data
879
- patterns = point.payload.get('code_patterns')
880
- logger.info(f"DEBUG: Creating SearchResult for point {point.id} from {collection_name}: has_patterns={bool(patterns)}, pattern_keys={list(patterns.keys()) if patterns else None}")
881
-
882
- all_results.append(SearchResult(
883
- id=str(point.id),
884
- score=point.score, # Score already includes decay
885
- timestamp=clean_timestamp,
886
- role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
887
- excerpt=(point.payload.get('text', '')[:350] + '...' if len(point.payload.get('text', '')) > 350 else point.payload.get('text', '')),
888
- project_name=point_project,
889
- conversation_id=point.payload.get('conversation_id'),
890
- base_conversation_id=point.payload.get('base_conversation_id'),
891
- collection_name=collection_name,
892
- raw_payload=point.payload, # Always include payload for metadata extraction
893
- # Pattern intelligence metadata
894
- code_patterns=point.payload.get('code_patterns'),
895
- files_analyzed=point.payload.get('files_analyzed'),
896
- tools_used=list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
897
- concepts=point.payload.get('concepts')
898
- ))
899
-
900
- elif should_use_decay:
901
- # Use client-side decay (existing implementation)
902
- await ctx.debug(f"Using CLIENT-SIDE decay for {collection_name}")
903
-
904
- # Search without score threshold to get all candidates
905
- results = await qdrant_client.search(
906
- collection_name=collection_name,
907
- query_vector=query_embedding,
908
- limit=limit * 3, # Get more candidates for decay filtering
909
- with_payload=True
910
- )
911
-
912
- # Apply decay scoring manually
913
- now = datetime.now(timezone.utc)
914
- scale_ms = DECAY_SCALE_DAYS * 24 * 60 * 60 * 1000
915
-
916
- decay_results = []
917
- for point in results:
918
- try:
919
- # Get timestamp from payload
920
- timestamp_str = point.payload.get('timestamp')
921
- if timestamp_str:
922
- timestamp = datetime.fromisoformat(timestamp_str.replace('Z', '+00:00'))
923
- # Ensure timestamp is timezone-aware
924
- if timestamp.tzinfo is None:
925
- timestamp = timestamp.replace(tzinfo=timezone.utc)
926
- age_ms = (now - timestamp).total_seconds() * 1000
927
-
928
- # Calculate decay factor using proper half-life formula
929
- # For half-life H: decay = exp(-ln(2) * age / H)
930
- ln2 = math.log(2)
931
- decay_factor = math.exp(-ln2 * age_ms / scale_ms)
932
-
933
- # Apply multiplicative decay formula to keep scores bounded [0, 1]
934
- # adjusted = score * ((1 - weight) + weight * decay)
935
- adjusted_score = point.score * ((1 - DECAY_WEIGHT) + DECAY_WEIGHT * decay_factor)
936
-
937
- # Debug: show the calculation
938
- age_days = age_ms / (24 * 60 * 60 * 1000)
939
- await ctx.debug(f"Point: age={age_days:.1f} days, original_score={point.score:.3f}, decay_factor={decay_factor:.3f}, adjusted_score={adjusted_score:.3f}")
940
- else:
941
- adjusted_score = point.score
942
-
943
- # Only include if above min_score after decay
944
- if adjusted_score >= min_score:
945
- decay_results.append((adjusted_score, point))
946
-
947
- except Exception as e:
948
- await ctx.debug(f"Error applying decay to point: {e}")
949
- decay_results.append((point.score, point))
950
-
951
- # Sort by adjusted score and take top results
952
- decay_results.sort(key=lambda x: x[0], reverse=True)
953
-
954
- # Convert to SearchResult format
955
- for adjusted_score, point in decay_results[:limit]:
956
- # Clean timestamp for proper parsing
957
- raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
958
- clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
959
-
960
- # Check project filter if we're searching all collections but want specific project
961
- point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
962
-
963
- # Special handling for reflections - they're global by default but can have project context
964
- is_reflection_collection = collection_name.startswith('reflections')
965
-
966
- # Handle project matching - check if the target project name appears at the end of the stored project path
967
- if target_project != 'all' and not project_collections and not is_reflection_collection:
968
- # The stored project name is like "-Users-username-projects-ShopifyMCPMockShop"
969
- # We want to match just "ShopifyMCPMockShop"
970
- # Also handle underscore/dash variations (procsolve-website vs procsolve_website)
971
- normalized_target = target_project.replace('-', '_')
972
- normalized_stored = point_project.replace('-', '_')
973
- if not (normalized_stored.endswith(f"_{normalized_target}") or
974
- normalized_stored == normalized_target or
975
- point_project.endswith(f"-{target_project}") or
976
- point_project == target_project):
977
- continue # Skip results from other projects
978
-
979
- # For reflections with project context, optionally filter by project
980
- if is_reflection_collection and target_project != 'all' and 'project' in point.payload:
981
- # Only filter if the reflection has project metadata
982
- reflection_project = point.payload.get('project', '')
983
- if reflection_project:
984
- # Normalize both for comparison (handle underscore/dash variations)
985
- normalized_target = target_project.replace('-', '_')
986
- normalized_reflection = reflection_project.replace('-', '_')
987
- if not (
988
- reflection_project == target_project or
989
- normalized_reflection == normalized_target or
990
- reflection_project.endswith(f"/{target_project}") or
991
- reflection_project.endswith(f"-{target_project}") or
992
- normalized_reflection.endswith(f"_{normalized_target}") or
993
- normalized_reflection.endswith(f"/{normalized_target}")
994
- ):
995
- continue # Skip reflections from other projects
996
-
997
- all_results.append(SearchResult(
998
- id=str(point.id),
999
- score=adjusted_score, # Use adjusted score
1000
- timestamp=clean_timestamp,
1001
- role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
1002
- excerpt=(point.payload.get('text', '')[:350] + '...' if len(point.payload.get('text', '')) > 350 else point.payload.get('text', '')),
1003
- project_name=point_project,
1004
- conversation_id=point.payload.get('conversation_id'),
1005
- base_conversation_id=point.payload.get('base_conversation_id'),
1006
- collection_name=collection_name,
1007
- raw_payload=point.payload, # Always include payload for metadata extraction
1008
- # Pattern intelligence metadata
1009
- code_patterns=point.payload.get('code_patterns'),
1010
- files_analyzed=point.payload.get('files_analyzed'),
1011
- tools_used=list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
1012
- concepts=point.payload.get('concepts')
1013
- ))
1014
- else:
1015
- # Standard search without decay
1016
- # Let Qdrant handle scoring natively
1017
- results = await qdrant_client.search(
1018
- collection_name=collection_name,
1019
- query_vector=query_embedding,
1020
- limit=limit * 2, # Get more results to account for filtering
1021
- with_payload=True
1022
- # No score_threshold - let Qdrant decide what's relevant
1023
- )
1024
-
1025
- for point in results:
1026
- # Clean timestamp for proper parsing
1027
- raw_timestamp = point.payload.get('timestamp', datetime.now().isoformat())
1028
- clean_timestamp = raw_timestamp.replace('Z', '+00:00') if raw_timestamp.endswith('Z') else raw_timestamp
1029
-
1030
- # Check project filter if we're searching all collections but want specific project
1031
- point_project = point.payload.get('project', collection_name.replace('conv_', '').replace('_voyage', '').replace('_local', ''))
1032
-
1033
- # Special handling for reflections - they're global by default but can have project context
1034
- is_reflection_collection = collection_name.startswith('reflections')
1035
-
1036
- # Handle project matching - check if the target project name appears at the end of the stored project path
1037
- if target_project != 'all' and not project_collections and not is_reflection_collection:
1038
- # The stored project name is like "-Users-username-projects-ShopifyMCPMockShop"
1039
- # We want to match just "ShopifyMCPMockShop"
1040
- # Also handle underscore/dash variations (procsolve-website vs procsolve_website)
1041
- normalized_target = target_project.replace('-', '_')
1042
- normalized_stored = point_project.replace('-', '_')
1043
- if not (normalized_stored.endswith(f"_{normalized_target}") or
1044
- normalized_stored == normalized_target or
1045
- point_project.endswith(f"-{target_project}") or
1046
- point_project == target_project):
1047
- continue # Skip results from other projects
1048
-
1049
- # For reflections with project context, optionally filter by project
1050
- if is_reflection_collection and target_project != 'all' and 'project' in point.payload:
1051
- # Only filter if the reflection has project metadata
1052
- reflection_project = point.payload.get('project', '')
1053
- if reflection_project:
1054
- # Normalize both for comparison (handle underscore/dash variations)
1055
- normalized_target = target_project.replace('-', '_')
1056
- normalized_reflection = reflection_project.replace('-', '_')
1057
- if not (
1058
- reflection_project == target_project or
1059
- normalized_reflection == normalized_target or
1060
- reflection_project.endswith(f"/{target_project}") or
1061
- reflection_project.endswith(f"-{target_project}") or
1062
- normalized_reflection.endswith(f"_{normalized_target}") or
1063
- normalized_reflection.endswith(f"/{normalized_target}")
1064
- ):
1065
- continue # Skip reflections from other projects
1066
-
1067
- # BOOST V2 CHUNKS: Apply score boost for v2 chunks (better quality)
1068
- original_score = point.score
1069
- final_score = original_score
1070
- chunking_version = point.payload.get('chunking_version', 'v1')
1071
-
1072
- if chunking_version == 'v2':
1073
- # Boost v2 chunks by 20% (configurable)
1074
- boost_factor = 1.2 # From migration config
1075
- final_score = min(1.0, original_score * boost_factor)
1076
- await ctx.debug(f"Boosted v2 chunk: {original_score:.3f} -> {final_score:.3f}")
1077
-
1078
- # Apply minimum score threshold after boosting
1079
- if final_score < min_score:
1080
- continue
1081
-
1082
- search_result = SearchResult(
1083
- id=str(point.id),
1084
- score=final_score,
1085
- timestamp=clean_timestamp,
1086
- role=point.payload.get('start_role', point.payload.get('role', 'unknown')),
1087
- excerpt=(point.payload.get('text', '')[:350] + '...' if len(point.payload.get('text', '')) > 350 else point.payload.get('text', '')),
1088
- project_name=point_project,
1089
- conversation_id=point.payload.get('conversation_id'),
1090
- base_conversation_id=point.payload.get('base_conversation_id'),
1091
- collection_name=collection_name,
1092
- raw_payload=point.payload, # Always include payload for metadata extraction
1093
- # Pattern intelligence metadata
1094
- code_patterns=point.payload.get('code_patterns'),
1095
- files_analyzed=point.payload.get('files_analyzed'),
1096
- tools_used=list(point.payload.get('tools_used', [])) if isinstance(point.payload.get('tools_used'), set) else point.payload.get('tools_used'),
1097
- concepts=point.payload.get('concepts')
1098
- )
1099
-
1100
- all_results.append(search_result)
1101
-
1102
- except Exception as e:
1103
- await ctx.debug(f"Error searching {collection_name}: {str(e)}")
1104
- collection_timing['error'] = str(e)
1105
-
1106
- collection_timing['end'] = time.time()
1107
- collection_timings.append(collection_timing)
1108
-
1109
- timing_info['search_all_end'] = time.time()
1110
-
1111
- # Report completion of search phase
1112
- await ctx.report_progress(
1113
- progress=len(collections_to_search),
1114
- total=len(collections_to_search),
1115
- message="Search complete, processing results"
1116
- )
1117
-
1118
- # Apply base_conversation_id boosting before sorting
1119
- timing_info['boost_start'] = time.time()
1120
-
1121
- # Group results by base_conversation_id to identify related chunks
1122
- base_conversation_groups = {}
1123
- for result in all_results:
1124
- base_id = result.base_conversation_id
1125
- if base_id:
1126
- if base_id not in base_conversation_groups:
1127
- base_conversation_groups[base_id] = []
1128
- base_conversation_groups[base_id].append(result)
1129
-
1130
- # Apply boost to results from base conversations with multiple high-scoring chunks
1131
- base_conversation_boost = 0.1 # Boost factor for base conversation matching
1132
- for base_id, group_results in base_conversation_groups.items():
1133
- if len(group_results) > 1: # Multiple chunks from same base conversation
1134
- avg_score = sum(r.score for r in group_results) / len(group_results)
1135
- if avg_score > 0.8: # Only boost high-quality base conversations
1136
- for result in group_results:
1137
- result.score += base_conversation_boost
1138
- await ctx.debug(f"Boosted result from base_conversation_id {base_id}: {result.score:.3f}")
1139
-
1140
- timing_info['boost_end'] = time.time()
1141
-
1142
- # Sort by score and limit
1143
- timing_info['sort_start'] = time.time()
1144
- all_results.sort(key=lambda x: x.score, reverse=True)
1145
-
1146
- # Apply mode-specific limits
1147
- if mode == "quick":
1148
- # For quick mode, only keep the top result
1149
- all_results = all_results[:1]
1150
- elif mode == "summary":
1151
- # For summary mode, we'll process all results but not return individual ones
1152
- pass # Keep all for aggregation
1153
- else:
1154
- # For full mode, apply the normal limit
1155
- all_results = all_results[:limit]
1156
-
1157
- timing_info['sort_end'] = time.time()
1158
-
1159
- logger.info(f"Total results: {len(all_results)}, Mode: {mode}, Returning: {len(all_results[:limit])}")
1160
- for r in all_results[:3]: # Log first 3
1161
- logger.debug(f"Result: id={r.id}, has_patterns={bool(r.code_patterns)}, pattern_keys={list(r.code_patterns.keys()) if r.code_patterns else None}")
1162
-
1163
- if not all_results:
1164
- return f"No conversations found matching '{query}'. Try different keywords or check if conversations have been imported."
1165
-
1166
- # Aggregate pattern intelligence across results
1167
- pattern_intelligence = aggregate_pattern_intelligence(all_results)
1168
-
1169
- # Update indexing status before returning results
1170
- await update_indexing_status()
1171
-
1172
- # Format results based on response_format and mode
1173
- timing_info['format_start'] = time.time()
1174
-
1175
- # Handle mode-specific responses
1176
- if mode == "quick":
1177
- # Quick mode: return just count and top result
1178
- total_count = len(all_results) # Before we limited to 1
1179
- if response_format == "xml":
1180
- result_text = f"<quick_search>\n"
1181
- result_text += f" <count>{total_count}</count>\n"
1182
- if all_results:
1183
- top_result = all_results[0]
1184
- result_text += f" <top_result>\n"
1185
- result_text += f" <score>{top_result.score:.3f}</score>\n"
1186
- result_text += f" <excerpt>{escape(top_result.excerpt[:200])}</excerpt>\n"
1187
- result_text += f" <project>{escape(top_result.project_name)}</project>\n"
1188
- result_text += f" <conversation_id>{escape(top_result.conversation_id or '')}</conversation_id>\n"
1189
- result_text += f" </top_result>\n"
1190
- result_text += f"</quick_search>"
1191
- return result_text
1192
- else:
1193
- # Markdown format for quick mode
1194
- if all_results:
1195
- return f"**Found {total_count} matches**\n\nTop result (score: {all_results[0].score:.3f}):\n{all_results[0].excerpt[:200]}"
1196
- else:
1197
- return f"No matches found for '{query}'"
1198
-
1199
- elif mode == "summary":
1200
- # Summary mode: return aggregated insights without individual results
1201
- if not all_results:
1202
- return f"No conversations found to summarize for '{query}'"
1203
-
1204
- # Aggregate data
1205
- total_count = len(all_results)
1206
- avg_score = sum(r.score for r in all_results) / total_count
1207
-
1208
- # Extract common concepts and tools
1209
- all_concepts = []
1210
- all_tools = []
1211
- all_files = []
1212
- projects = set()
1213
-
1214
- for result in all_results:
1215
- if result.concepts:
1216
- all_concepts.extend(result.concepts)
1217
- if result.tools_used:
1218
- all_tools.extend(result.tools_used)
1219
- if result.files_analyzed:
1220
- all_files.extend(result.files_analyzed)
1221
- projects.add(result.project_name)
1222
-
1223
- # Count frequencies
1224
- from collections import Counter
1225
- concept_counts = Counter(all_concepts).most_common(5)
1226
- tool_counts = Counter(all_tools).most_common(5)
1227
-
1228
- if response_format == "xml":
1229
- result_text = f"<search_summary>\n"
1230
- result_text += f" <query>{escape(query)}</query>\n"
1231
- result_text += f" <total_matches>{total_count}</total_matches>\n"
1232
- result_text += f" <average_score>{avg_score:.3f}</average_score>\n"
1233
- result_text += f" <projects_involved>{len(projects)}</projects_involved>\n"
1234
- if concept_counts:
1235
- result_text += f" <common_concepts>\n"
1236
- for concept, count in concept_counts:
1237
- result_text += f" <concept count=\"{count}\">{escape(concept)}</concept>\n"
1238
- result_text += f" </common_concepts>\n"
1239
- if tool_counts:
1240
- result_text += f" <common_tools>\n"
1241
- for tool, count in tool_counts:
1242
- result_text += f" <tool count=\"{count}\">{escape(tool)}</tool>\n"
1243
- result_text += f" </common_tools>\n"
1244
- result_text += f"</search_summary>"
1245
- return result_text
1246
- else:
1247
- # Markdown format for summary
1248
- result_text = f"## Summary for: {query}\n\n"
1249
- result_text += f"- **Total matches**: {total_count}\n"
1250
- result_text += f"- **Average relevance**: {avg_score:.3f}\n"
1251
- result_text += f"- **Projects involved**: {len(projects)}\n\n"
1252
- if concept_counts:
1253
- result_text += "**Common concepts**:\n"
1254
- for concept, count in concept_counts:
1255
- result_text += f"- {concept} ({count} occurrences)\n"
1256
- if tool_counts:
1257
- result_text += "\n**Common tools**:\n"
1258
- for tool, count in tool_counts:
1259
- result_text += f"- {tool} ({count} uses)\n"
1260
- return result_text
1261
-
1262
- # Continue with normal formatting for full mode
1263
- if response_format == "xml":
1264
- # Add upfront summary for immediate visibility (before collapsible XML)
1265
- upfront_summary = ""
1266
-
1267
- # Show result summary
1268
- if all_results:
1269
- score_info = "high" if all_results[0].score >= 0.85 else "good" if all_results[0].score >= 0.75 else "partial"
1270
- upfront_summary += f"🎯 RESULTS: {len(all_results)} matches ({score_info} relevance, top score: {all_results[0].score:.3f})\n"
1271
-
1272
- # Show performance with indexing status inline
1273
- total_time = time.time() - start_time
1274
- indexing_info = f" | 📊 {indexing_status['indexed_conversations']}/{indexing_status['total_conversations']} indexed" if indexing_status["percentage"] < 100.0 else ""
1275
- upfront_summary += f"⚡ PERFORMANCE: {int(total_time * 1000)}ms ({len(collections_to_search)} collections searched{indexing_info})\n"
1276
- else:
1277
- upfront_summary += f"❌ NO RESULTS: No conversations found matching '{query}'\n"
1278
-
1279
- # XML format (compact tags for performance)
1280
- result_text = upfront_summary + "\n<search>\n"
1281
-
1282
- # Add indexing status if not fully baselined - put key stats in opening tag for immediate visibility
1283
- if indexing_status["percentage"] < 95.0:
1284
- result_text += f' <info status="indexing" progress="{indexing_status["percentage"]:.1f}%" backlog="{indexing_status["backlog_count"]}">\n'
1285
- result_text += f' <message>📊 Indexing: {indexing_status["indexed_conversations"]}/{indexing_status["total_conversations"]} conversations ({indexing_status["percentage"]:.1f}% complete, {indexing_status["backlog_count"]} pending)</message>\n'
1286
- result_text += f" </info>\n"
1287
-
1288
- # Add high-level result summary
1289
- if all_results:
1290
- # Count today's results
1291
- now = datetime.now(timezone.utc)
1292
- today_count = 0
1293
- yesterday_count = 0
1294
- week_count = 0
1295
-
1296
- for result in all_results:
1297
- timestamp_clean = result.timestamp.replace('Z', '+00:00') if result.timestamp.endswith('Z') else result.timestamp
1298
- timestamp_dt = datetime.fromisoformat(timestamp_clean)
1299
- if timestamp_dt.tzinfo is None:
1300
- timestamp_dt = timestamp_dt.replace(tzinfo=timezone.utc)
1301
-
1302
- days_ago = (now - timestamp_dt).days
1303
- if days_ago == 0:
1304
- today_count += 1
1305
- elif days_ago == 1:
1306
- yesterday_count += 1
1307
- if days_ago <= 7:
1308
- week_count += 1
1309
-
1310
- # Compact summary with key info in opening tag
1311
- time_info = ""
1312
- if today_count > 0:
1313
- time_info = f"{today_count} today"
1314
- elif yesterday_count > 0:
1315
- time_info = f"{yesterday_count} yesterday"
1316
- elif week_count > 0:
1317
- time_info = f"{week_count} this week"
1318
- else:
1319
- time_info = "older results"
1320
-
1321
- score_info = "high" if all_results[0].score >= 0.85 else "good" if all_results[0].score >= 0.75 else "partial"
1322
-
1323
- result_text += f' <summary count="{len(all_results)}" relevance="{score_info}" recency="{time_info}" top-score="{all_results[0].score:.3f}">\n'
1324
-
1325
- # Short preview of top result
1326
- top_excerpt = all_results[0].excerpt[:100].strip()
1327
- if '...' not in top_excerpt:
1328
- top_excerpt += "..."
1329
- result_text += f' <preview>{top_excerpt}</preview>\n'
1330
- result_text += f" </summary>\n"
1331
- else:
1332
- result_text += f" <result-summary>\n"
1333
- result_text += f" <headline>No matches found</headline>\n"
1334
- result_text += f" <relevance>No conversations matched your query</relevance>\n"
1335
- result_text += f" </result-summary>\n"
1336
-
1337
- result_text += f" <meta>\n"
1338
- result_text += f" <q>{query}</q>\n"
1339
- result_text += f" <scope>{target_project if target_project != 'all' else 'all'}</scope>\n"
1340
- result_text += f" <count>{len(all_results)}</count>\n"
1341
- if all_results:
1342
- result_text += f" <range>{all_results[-1].score:.3f}-{all_results[0].score:.3f}</range>\n"
1343
- result_text += f" <embed>{'local' if PREFER_LOCAL_EMBEDDINGS or not voyage_client else 'voyage'}</embed>\n"
1344
-
1345
- # Add timing metadata
1346
- total_time = time.time() - start_time
1347
- result_text += f" <perf>\n"
1348
- result_text += f" <ttl>{int(total_time * 1000)}</ttl>\n"
1349
- result_text += f" <emb>{int((timing_info.get('embedding_end', 0) - timing_info.get('embedding_start', 0)) * 1000)}</emb>\n"
1350
- result_text += f" <srch>{int((timing_info.get('search_all_end', 0) - timing_info.get('search_all_start', 0)) * 1000)}</srch>\n"
1351
- result_text += f" <cols>{len(collections_to_search)}</cols>\n"
1352
- result_text += f" </perf>\n"
1353
- result_text += f" </meta>\n"
1354
-
1355
- result_text += " <results>\n"
1356
- for i, result in enumerate(all_results):
1357
- result_text += f' <r rank="{i+1}">\n'
1358
- result_text += f" <s>{result.score:.3f}</s>\n"
1359
- result_text += f" <p>{result.project_name}</p>\n"
1360
-
1361
- # Calculate relative time
1362
- timestamp_clean = result.timestamp.replace('Z', '+00:00') if result.timestamp.endswith('Z') else result.timestamp
1363
- timestamp_dt = datetime.fromisoformat(timestamp_clean)
1364
- # Ensure both datetimes are timezone-aware
1365
- if timestamp_dt.tzinfo is None:
1366
- timestamp_dt = timestamp_dt.replace(tzinfo=timezone.utc)
1367
- now = datetime.now(timezone.utc)
1368
- days_ago = (now - timestamp_dt).days
1369
- if days_ago == 0:
1370
- time_str = "today"
1371
- elif days_ago == 1:
1372
- time_str = "yesterday"
1373
- else:
1374
- time_str = f"{days_ago}d"
1375
- result_text += f" <t>{time_str}</t>\n"
1376
-
1377
- if not brief:
1378
- # Extract title from first line of excerpt
1379
- excerpt_lines = result.excerpt.split('\n')
1380
- title = excerpt_lines[0][:80] + "..." if len(excerpt_lines[0]) > 80 else excerpt_lines[0]
1381
- result_text += f" <title>{title}</title>\n"
1382
-
1383
- # Key finding - summarize the main point
1384
- key_finding = result.excerpt[:100] + "..." if len(result.excerpt) > 100 else result.excerpt
1385
- result_text += f" <key-finding>{key_finding.strip()}</key-finding>\n"
1386
-
1387
- # Always include excerpt, but shorter in brief mode
1388
- if brief:
1389
- brief_excerpt = result.excerpt[:100] + "..." if len(result.excerpt) > 100 else result.excerpt
1390
- result_text += f" <excerpt>{brief_excerpt.strip()}</excerpt>\n"
1391
- else:
1392
- result_text += f" <excerpt><![CDATA[{result.excerpt}]]></excerpt>\n"
1393
-
1394
- if result.conversation_id:
1395
- result_text += f" <cid>{result.conversation_id}</cid>\n"
1396
-
1397
- # Include raw data if requested
1398
- if include_raw and result.raw_payload:
1399
- result_text += " <raw>\n"
1400
- result_text += f" <txt><![CDATA[{result.raw_payload.get('text', '')}]]></txt>\n"
1401
- result_text += f" <id>{result.id}</id>\n"
1402
- result_text += f" <dist>{1 - result.score:.3f}</dist>\n"
1403
- result_text += " <meta>\n"
1404
- for key, value in result.raw_payload.items():
1405
- if key != 'text':
1406
- result_text += f" <{key}>{value}</{key}>\n"
1407
- result_text += " </meta>\n"
1408
- result_text += " </raw>\n"
1409
-
1410
- # Add patterns if they exist - with detailed logging
1411
- if result.code_patterns and isinstance(result.code_patterns, dict):
1412
- logger.info(f"DEBUG: Point {result.id} has code_patterns dict with keys: {list(result.code_patterns.keys())}")
1413
- patterns_to_show = []
1414
- for category, patterns in result.code_patterns.items():
1415
- if patterns and isinstance(patterns, list) and len(patterns) > 0:
1416
- # Take up to 5 patterns from each category
1417
- patterns_to_show.append((category, patterns[:5]))
1418
- logger.info(f"DEBUG: Added category '{category}' with {len(patterns)} patterns")
1419
-
1420
- if patterns_to_show:
1421
- logger.info(f"DEBUG: Adding patterns XML for point {result.id}")
1422
- result_text += " <patterns>\n"
1423
- for category, patterns in patterns_to_show:
1424
- # Escape both category name and pattern content for XML safety
1425
- safe_patterns = ', '.join(escape(str(p)) for p in patterns)
1426
- result_text += f" <cat name=\"{escape(category)}\">{safe_patterns}</cat>\n"
1427
- result_text += " </patterns>\n"
1428
- else:
1429
- logger.info(f"DEBUG: Point {result.id} has code_patterns but no valid patterns to show")
1430
- else:
1431
- logger.info(f"DEBUG: Point {result.id} has no patterns. code_patterns={result.code_patterns}, type={type(result.code_patterns)}")
1432
-
1433
- if result.files_analyzed and len(result.files_analyzed) > 0:
1434
- result_text += f" <files>{', '.join(result.files_analyzed[:5])}</files>\n"
1435
- if result.concepts and len(result.concepts) > 0:
1436
- result_text += f" <concepts>{', '.join(result.concepts[:5])}</concepts>\n"
1437
-
1438
- # Include structured metadata for agent consumption
1439
- # This provides clean, parsed fields that agents can easily use
1440
- if hasattr(result, 'raw_payload') and result.raw_payload:
1441
- import json
1442
- payload = result.raw_payload
1443
-
1444
- # Files section - structured for easy agent parsing
1445
- files_analyzed = payload.get('files_analyzed', [])
1446
- files_edited = payload.get('files_edited', [])
1447
- if files_analyzed or files_edited:
1448
- result_text += " <files>\n"
1449
- if files_analyzed:
1450
- result_text += f" <analyzed count=\"{len(files_analyzed)}\">"
1451
- result_text += ", ".join(files_analyzed[:5]) # First 5 files
1452
- if len(files_analyzed) > 5:
1453
- result_text += f" ... and {len(files_analyzed)-5} more"
1454
- result_text += "</analyzed>\n"
1455
- if files_edited:
1456
- result_text += f" <edited count=\"{len(files_edited)}\">"
1457
- result_text += ", ".join(files_edited[:5]) # First 5 files
1458
- if len(files_edited) > 5:
1459
- result_text += f" ... and {len(files_edited)-5} more"
1460
- result_text += "</edited>\n"
1461
- result_text += " </files>\n"
1462
-
1463
- # Concepts section - clean list for agents
1464
- concepts = payload.get('concepts', [])
1465
- if concepts:
1466
- result_text += f" <concepts>{', '.join(concepts)}</concepts>\n"
1467
-
1468
- # Tools section - summarized with counts
1469
- tools_used = payload.get('tools_used', [])
1470
- if tools_used:
1471
- # Count tool usage
1472
- tool_counts = {}
1473
- for tool in tools_used:
1474
- tool_counts[tool] = tool_counts.get(tool, 0) + 1
1475
- # Sort by frequency
1476
- sorted_tools = sorted(tool_counts.items(), key=lambda x: x[1], reverse=True)
1477
- tool_summary = ", ".join(f"{tool}({count})" for tool, count in sorted_tools[:5])
1478
- if len(sorted_tools) > 5:
1479
- tool_summary += f" ... and {len(sorted_tools)-5} more"
1480
- result_text += f" <tools>{tool_summary}</tools>\n"
1481
-
1482
- # Code patterns section - structured by category
1483
- code_patterns = payload.get('code_patterns', {})
1484
- if code_patterns:
1485
- result_text += " <code_patterns>\n"
1486
- for category, patterns in code_patterns.items():
1487
- if patterns:
1488
- pattern_list = patterns if isinstance(patterns, list) else [patterns]
1489
- # Clean up pattern names
1490
- clean_patterns = []
1491
- for p in pattern_list[:5]:
1492
- # Remove common prefixes like $FUNC, $VAR
1493
- clean_p = str(p).replace('$FUNC', '').replace('$VAR', '').strip()
1494
- if clean_p:
1495
- clean_patterns.append(clean_p)
1496
- if clean_patterns:
1497
- result_text += f" <{category}>{', '.join(clean_patterns)}</{category}>\n"
1498
- result_text += " </code_patterns>\n"
1499
-
1500
- # Pattern inheritance info - shows propagation details
1501
- pattern_inheritance = payload.get('pattern_inheritance', {})
1502
- if pattern_inheritance:
1503
- source_chunk = pattern_inheritance.get('source_chunk', '')
1504
- confidence = pattern_inheritance.get('confidence', 0)
1505
- distance = pattern_inheritance.get('distance', 0)
1506
- if source_chunk:
1507
- result_text += f" <pattern_source chunk=\"{source_chunk}\" confidence=\"{confidence:.2f}\" distance=\"{distance}\"/>\n"
1508
-
1509
- # Message stats for context
1510
- msg_count = payload.get('message_count')
1511
- total_length = payload.get('total_length')
1512
- if msg_count or total_length:
1513
- stats_attrs = []
1514
- if msg_count:
1515
- stats_attrs.append(f'messages="{msg_count}"')
1516
- if total_length:
1517
- stats_attrs.append(f'length="{total_length}"')
1518
- result_text += f" <stats {' '.join(stats_attrs)}/>\n"
1519
-
1520
- # Raw metadata dump for backwards compatibility
1521
- # Kept minimal - only truly unique fields
1522
- remaining_metadata = {}
1523
- excluded_keys = {'text', 'conversation_id', 'timestamp', 'role', 'project', 'chunk_index',
1524
- 'files_analyzed', 'files_edited', 'concepts', 'tools_used',
1525
- 'code_patterns', 'pattern_inheritance', 'message_count', 'total_length',
1526
- 'chunking_version', 'chunk_method', 'chunk_overlap', 'migration_type'}
1527
- for key, value in payload.items():
1528
- if key not in excluded_keys and value is not None:
1529
- if isinstance(value, set):
1530
- value = list(value)
1531
- remaining_metadata[key] = value
1532
-
1533
- if remaining_metadata:
1534
- try:
1535
- # Only include if there's actually extra data
1536
- result_text += f" <metadata_extra><![CDATA[{json.dumps(remaining_metadata, default=str)}]]></metadata_extra>\n"
1537
- except:
1538
- pass
1539
-
1540
- result_text += " </r>\n"
1541
- result_text += " </results>\n"
1542
-
1543
- # Add aggregated pattern intelligence section
1544
- if pattern_intelligence and pattern_intelligence.get('total_unique_patterns', 0) > 0:
1545
- result_text += " <pattern_intelligence>\n"
1546
-
1547
- # Summary statistics
1548
- result_text += f" <summary>\n"
1549
- result_text += f" <unique_patterns>{pattern_intelligence['total_unique_patterns']}</unique_patterns>\n"
1550
- result_text += f" <pattern_diversity>{pattern_intelligence['pattern_diversity_score']:.2f}</pattern_diversity>\n"
1551
- result_text += f" </summary>\n"
1552
-
1553
- # Most common patterns
1554
- if pattern_intelligence.get('most_common_patterns'):
1555
- result_text += " <common_patterns>\n"
1556
- for pattern, count in pattern_intelligence['most_common_patterns'][:5]:
1557
- result_text += f" <pattern count=\"{count}\">{pattern}</pattern>\n"
1558
- result_text += " </common_patterns>\n"
1559
-
1560
- # Pattern categories
1561
- if pattern_intelligence.get('category_coverage'):
1562
- result_text += " <categories>\n"
1563
- for category, count in pattern_intelligence['category_coverage'].items():
1564
- result_text += f" <cat name=\"{category}\" count=\"{count}\"/>\n"
1565
- result_text += " </categories>\n"
1566
-
1567
- # Pattern combinations insight
1568
- if pattern_intelligence.get('pattern_combinations'):
1569
- combos = pattern_intelligence['pattern_combinations']
1570
- if combos.get('async_with_error_handling'):
1571
- result_text += " <insight>Async patterns combined with error handling detected</insight>\n"
1572
- if combos.get('react_with_state'):
1573
- result_text += " <insight>React hooks with state management patterns detected</insight>\n"
1574
-
1575
- # Files referenced across results
1576
- if pattern_intelligence.get('files_referenced') and len(pattern_intelligence['files_referenced']) > 0:
1577
- result_text += f" <files_across_results>{', '.join(pattern_intelligence['files_referenced'][:10])}</files_across_results>\n"
1578
-
1579
- # Concepts discussed
1580
- if pattern_intelligence.get('concepts_discussed') and len(pattern_intelligence['concepts_discussed']) > 0:
1581
- result_text += f" <concepts_discussed>{', '.join(pattern_intelligence['concepts_discussed'][:10])}</concepts_discussed>\n"
1582
-
1583
- result_text += " </pattern_intelligence>\n"
1584
-
1585
- result_text += "</search>"
1586
-
1587
- else:
1588
- # Markdown format (original)
1589
- result_text = f"Found {len(all_results)} relevant conversation(s) for '{query}':\n\n"
1590
- for i, result in enumerate(all_results):
1591
- result_text += f"**Result {i+1}** (Score: {result.score:.3f})\n"
1592
- # Handle timezone suffix 'Z' properly
1593
- timestamp_clean = result.timestamp.replace('Z', '+00:00') if result.timestamp.endswith('Z') else result.timestamp
1594
- result_text += f"Time: {datetime.fromisoformat(timestamp_clean).strftime('%Y-%m-%d %H:%M:%S')}\n"
1595
- result_text += f"Project: {result.project_name}\n"
1596
- result_text += f"Role: {result.role}\n"
1597
- result_text += f"Excerpt: {result.excerpt}\n"
1598
- result_text += "---\n\n"
1599
-
1600
- timing_info['format_end'] = time.time()
1601
-
1602
- # Log detailed timing breakdown
1603
- await ctx.debug(f"\n=== TIMING BREAKDOWN ===")
1604
- await ctx.debug(f"Total time: {(time.time() - start_time) * 1000:.1f}ms")
1605
- await ctx.debug(f"Embedding generation: {(timing_info.get('embedding_end', 0) - timing_info.get('embedding_start', 0)) * 1000:.1f}ms")
1606
- await ctx.debug(f"Get collections: {(timing_info.get('get_collections_end', 0) - timing_info.get('get_collections_start', 0)) * 1000:.1f}ms")
1607
- await ctx.debug(f"Search all collections: {(timing_info.get('search_all_end', 0) - timing_info.get('search_all_start', 0)) * 1000:.1f}ms")
1608
- await ctx.debug(f"Sorting results: {(timing_info.get('sort_end', 0) - timing_info.get('sort_start', 0)) * 1000:.1f}ms")
1609
- await ctx.debug(f"Formatting output: {(timing_info.get('format_end', 0) - timing_info.get('format_start', 0)) * 1000:.1f}ms")
1610
-
1611
- # Log per-collection timings
1612
- await ctx.debug(f"\n=== PER-COLLECTION TIMINGS ===")
1613
- for ct in collection_timings:
1614
- duration = (ct.get('end', 0) - ct.get('start', 0)) * 1000
1615
- status = "ERROR" if 'error' in ct else "OK"
1616
- await ctx.debug(f"{ct['name']}: {duration:.1f}ms ({status})")
1617
-
1618
- return result_text
1619
-
1620
- except Exception as e:
1621
- await ctx.error(f"Search failed: {str(e)}")
1622
- return f"Failed to search conversations: {str(e)}"
1623
-
1624
-
1625
- @mcp.tool()
1626
- async def store_reflection(
1627
- ctx: Context,
1628
- content: str = Field(description="The insight or reflection to store"),
1629
- tags: List[str] = Field(default=[], description="Tags to categorize this reflection")
1630
- ) -> str:
1631
- """Store an important insight or reflection for future reference."""
1632
-
1633
- try:
1634
- # Create reflections collection name
1635
- collection_name = f"reflections{get_collection_suffix()}"
1636
-
1637
- # Get current project context
1638
- cwd = os.environ.get('MCP_CLIENT_CWD', os.getcwd())
1639
- project_path = Path(cwd)
1640
-
1641
- # Extract project name from path
1642
- project_name = None
1643
- path_parts = project_path.parts
1644
- if 'projects' in path_parts:
1645
- idx = path_parts.index('projects')
1646
- if idx + 1 < len(path_parts):
1647
- # Get all parts after 'projects' to form the project name
1648
- # This handles cases like projects/Connectiva-App/connectiva-ai
1649
- project_parts = path_parts[idx + 1:]
1650
- project_name = '/'.join(project_parts)
1651
-
1652
- # If no project detected, use the last directory name
1653
- if not project_name:
1654
- project_name = project_path.name
1655
-
1656
- # Ensure collection exists
1657
- try:
1658
- collection_info = await qdrant_client.get_collection(collection_name)
1659
- except:
1660
- # Create collection if it doesn't exist
1661
- await qdrant_client.create_collection(
1662
- collection_name=collection_name,
1663
- vectors_config=VectorParams(
1664
- size=get_embedding_dimension(),
1665
- distance=Distance.COSINE
1666
- )
1667
- )
1668
- await ctx.debug(f"Created reflections collection: {collection_name}")
1669
-
1670
- # Generate embedding for the reflection
1671
- embedding = await generate_embedding(content)
1672
-
1673
- # Create point with metadata including project context
1674
- point_id = datetime.now().timestamp()
1675
- point = PointStruct(
1676
- id=int(point_id),
1677
- vector=embedding,
1678
- payload={
1679
- "text": content,
1680
- "tags": tags,
1681
- "timestamp": datetime.now().isoformat(),
1682
- "type": "reflection",
1683
- "role": "user_reflection",
1684
- "project": project_name, # Add project context
1685
- "project_path": str(project_path) # Add full path for reference
1686
- }
1687
- )
1688
-
1689
- # Store in Qdrant
1690
- await qdrant_client.upsert(
1691
- collection_name=collection_name,
1692
- points=[point]
1693
- )
1694
-
1695
- tags_str = ', '.join(tags) if tags else 'none'
1696
- return f"Reflection stored successfully with tags: {tags_str}"
1697
-
1698
- except Exception as e:
1699
- await ctx.error(f"Store failed: {str(e)}")
1700
- return f"Failed to store reflection: {str(e)}"
1701
-
1702
-
1703
- @mcp.tool()
1704
- async def quick_search(
1705
- ctx: Context,
1706
- query: str = Field(description="The search query to find semantically similar conversations"),
1707
- min_score: float = Field(default=0.3, description="Minimum similarity score (0-1)"),
1708
- project: Optional[str] = Field(default=None, description="Search specific project only. If not provided, searches current project based on working directory. Use 'all' to search across all projects.")
1709
- ) -> str:
1710
- """Quick search that returns only the count and top result for fast overview."""
1711
- # MCP architectural limitation: MCP tools cannot call other MCP tools
1712
- return """<error>
1713
- MCP Architectural Limitation: This tool cannot directly call other MCP tools.
1714
652
 
1715
- To perform a quick search, please:
1716
- 1. Call reflect_on_past directly with limit=1 and brief=True
1717
- 2. Or use the reflection-specialist agent for quick searches
1718
-
1719
- This limitation exists because MCP tools can only be orchestrated by the client (Claude),
1720
- not by other tools within the MCP server.
1721
- </error>"""
1722
-
1723
-
1724
- @mcp.tool()
1725
- async def search_summary(
1726
- ctx: Context,
1727
- query: str = Field(description="The search query to find semantically similar conversations"),
1728
- project: Optional[str] = Field(default=None, description="Search specific project only. If not provided, searches current project based on working directory. Use 'all' to search across all projects.")
1729
- ) -> str:
1730
- """Get aggregated insights from search results without individual result details."""
1731
- # MCP architectural limitation: MCP tools cannot call other MCP tools
1732
- # This is a fundamental constraint of the MCP protocol
1733
- return """<error>
1734
- MCP Architectural Limitation: This tool cannot directly call other MCP tools.
1735
-
1736
- To get a search summary, please use the reflection-specialist agent instead:
1737
- 1. Call the reflection-specialist agent
1738
- 2. Ask it to provide a summary of search results for your query
1739
-
1740
- Alternative: Call reflect_on_past directly and analyze the results yourself.
1741
-
1742
- This limitation exists because MCP tools can only be orchestrated by the client (Claude),
1743
- not by other tools within the MCP server.
1744
- </error>"""
1745
-
1746
-
1747
- @mcp.tool()
1748
- async def get_more_results(
1749
- ctx: Context,
1750
- query: str = Field(description="The original search query"),
1751
- offset: int = Field(default=3, description="Number of results to skip (for pagination)"),
1752
- limit: int = Field(default=3, description="Number of additional results to return"),
1753
- min_score: float = Field(default=0.3, description="Minimum similarity score (0-1)"),
1754
- project: Optional[str] = Field(default=None, description="Search specific project only")
1755
- ) -> str:
1756
- """Get additional search results after an initial search (pagination support)."""
1757
- # MCP architectural limitation: MCP tools cannot call other MCP tools
1758
- return """<error>
1759
- MCP Architectural Limitation: This tool cannot directly call other MCP tools.
1760
-
1761
- To get more search results, please:
1762
- 1. Call reflect_on_past directly with a higher limit parameter
1763
- 2. Or use the reflection-specialist agent to handle pagination
1764
-
1765
- This limitation exists because MCP tools can only be orchestrated by the client (Claude),
1766
- not by other tools within the MCP server.
1767
- </error>"""
1768
-
1769
-
1770
- @mcp.tool()
1771
- async def search_by_file(
1772
- ctx: Context,
1773
- file_path: str = Field(description="The file path to search for in conversations"),
1774
- limit: int = Field(default=10, description="Maximum number of results to return"),
1775
- project: Optional[str] = Field(default=None, description="Search specific project only. Use 'all' to search across all projects.")
1776
- ) -> str:
1777
- """Search for conversations that analyzed a specific file."""
1778
- global qdrant_client
1779
-
1780
- # Normalize file path
1781
- normalized_path = file_path.replace("\\", "/").replace("/Users/", "~/")
1782
-
1783
- # Determine which collections to search
1784
- # If no project specified, search all collections
1785
- collections = await get_all_collections() if not project else []
1786
-
1787
- if project and project != 'all':
1788
- # Filter collections for specific project - normalize first!
1789
- normalized_project = normalize_project_name(project)
1790
- project_hash = hashlib.md5(normalized_project.encode()).hexdigest()[:8]
1791
- collection_prefix = f"conv_{project_hash}_"
1792
- collections = [c for c in await get_all_collections() if c.startswith(collection_prefix)]
1793
- elif project == 'all':
1794
- collections = await get_all_collections()
1795
-
1796
- if not collections:
1797
- return "<search_by_file>\n<error>No collections found to search</error>\n</search_by_file>"
1798
-
1799
- # Prepare results
1800
- all_results = []
1801
-
1802
- for collection_name in collections:
1803
- try:
1804
- # Use scroll to get all points and filter manually
1805
- # Qdrant's array filtering can be tricky, so we'll filter in code
1806
- scroll_result = await qdrant_client.scroll(
1807
- collection_name=collection_name,
1808
- limit=1000, # Get a batch
1809
- with_payload=True
1810
- )
1811
-
1812
- # Filter results that contain the file
1813
- for point in scroll_result[0]:
1814
- payload = point.payload
1815
- files_analyzed = payload.get('files_analyzed', [])
1816
- files_edited = payload.get('files_edited', [])
1817
-
1818
- # Check for exact match or if any file ends with the normalized path
1819
- file_match = False
1820
- for file in files_analyzed + files_edited:
1821
- if file == normalized_path or file.endswith('/' + normalized_path) or file.endswith('\\' + normalized_path):
1822
- file_match = True
1823
- break
1824
-
1825
- if file_match:
1826
- all_results.append({
1827
- 'score': 1.0, # File match is always 1.0
1828
- 'payload': payload,
1829
- 'collection': collection_name
1830
- })
1831
-
1832
- except Exception as e:
1833
- continue
1834
-
1835
- # Sort by timestamp (newest first)
1836
- all_results.sort(key=lambda x: x['payload'].get('timestamp', ''), reverse=True)
1837
-
1838
- # Format results
1839
- if not all_results:
1840
- return f"""<search_by_file>
1841
- <query>{file_path}</query>
1842
- <normalized_path>{normalized_path}</normalized_path>
1843
- <message>No conversations found that analyzed this file</message>
1844
- </search_by_file>"""
1845
-
1846
- results_text = []
1847
- for i, result in enumerate(all_results[:limit]):
1848
- payload = result['payload']
1849
- timestamp = payload.get('timestamp', 'Unknown')
1850
- conversation_id = payload.get('conversation_id', 'Unknown')
1851
- project = payload.get('project', 'Unknown')
1852
- text_preview = payload.get('text', '')[:200] + '...' if len(payload.get('text', '')) > 200 else payload.get('text', '')
1853
-
1854
- # Check if file was edited or just read
1855
- action = "edited" if normalized_path in payload.get('files_edited', []) else "analyzed"
1856
-
1857
- # Get related tools used
1858
- tool_summary = payload.get('tool_summary', {})
1859
- tools_used = ', '.join(f"{tool}({count})" for tool, count in tool_summary.items())
1860
-
1861
- results_text.append(f"""<result rank="{i+1}">
1862
- <conversation_id>{conversation_id}</conversation_id>
1863
- <project>{project}</project>
1864
- <timestamp>{timestamp}</timestamp>
1865
- <action>{action}</action>
1866
- <tools_used>{tools_used}</tools_used>
1867
- <preview>{text_preview}</preview>
1868
- </result>""")
1869
-
1870
- return f"""<search_by_file>
1871
- <query>{file_path}</query>
1872
- <normalized_path>{normalized_path}</normalized_path>
1873
- <count>{len(all_results)}</count>
1874
- <results>
1875
- {''.join(results_text)}
1876
- </results>
1877
- </search_by_file>"""
1878
-
1879
-
1880
- @mcp.tool()
1881
- async def search_by_concept(
1882
- ctx: Context,
1883
- concept: str = Field(description="The concept to search for (e.g., 'security', 'docker', 'testing')"),
1884
- include_files: bool = Field(default=True, description="Include file information in results"),
1885
- limit: int = Field(default=10, description="Maximum number of results to return"),
1886
- project: Optional[str] = Field(default=None, description="Search specific project only. Use 'all' to search across all projects.")
1887
- ) -> str:
1888
- """Search for conversations about a specific development concept."""
1889
- global qdrant_client
1890
-
1891
- # Generate embedding for the concept
1892
- embedding = await generate_embedding(concept)
1893
-
1894
- # Determine which collections to search
1895
- # If no project specified, search all collections
1896
- collections = await get_all_collections() if not project else []
1897
-
1898
- if project and project != 'all':
1899
- # Filter collections for specific project
1900
- normalized_project = normalize_project_name(project)
1901
- project_hash = hashlib.md5(normalized_project.encode()).hexdigest()[:8]
1902
- collection_prefix = f"conv_{project_hash}_"
1903
- collections = [c for c in await get_all_collections() if c.startswith(collection_prefix)]
1904
- elif project == 'all':
1905
- collections = await get_all_collections()
1906
-
1907
- if not collections:
1908
- return "<search_by_concept>\n<error>No collections found to search</error>\n</search_by_concept>"
1909
-
1910
- # First, check metadata health
1911
- metadata_found = False
1912
- total_points_checked = 0
1913
-
1914
- for collection_name in collections[:3]: # Sample first 3 collections
1915
- try:
1916
- sample_points, _ = await qdrant_client.scroll(
1917
- collection_name=collection_name,
1918
- limit=10,
1919
- with_payload=True
1920
- )
1921
- total_points_checked += len(sample_points)
1922
- for point in sample_points:
1923
- if 'concepts' in point.payload and point.payload['concepts']:
1924
- metadata_found = True
1925
- break
1926
- if metadata_found:
1927
- break
1928
- except:
1929
- continue
1930
-
1931
- # Search all collections
1932
- all_results = []
1933
-
1934
- # If metadata exists, try metadata-based search first
1935
- if metadata_found:
1936
- for collection_name in collections:
1937
- try:
1938
- # Hybrid search: semantic + concept filter
1939
- results = await qdrant_client.search(
1940
- collection_name=collection_name,
1941
- query_vector=embedding,
1942
- query_filter=models.Filter(
1943
- should=[
1944
- models.FieldCondition(
1945
- key="concepts",
1946
- match=models.MatchAny(any=[concept.lower()])
1947
- )
1948
- ]
1949
- ),
1950
- limit=limit * 2, # Get more results for better filtering
1951
- with_payload=True
1952
- )
1953
-
1954
- for point in results:
1955
- payload = point.payload
1956
- # Boost score if concept is in the concepts list
1957
- score_boost = 0.2 if concept.lower() in payload.get('concepts', []) else 0.0
1958
- all_results.append({
1959
- 'score': float(point.score) + score_boost,
1960
- 'payload': payload,
1961
- 'collection': collection_name,
1962
- 'search_type': 'metadata'
1963
- })
1964
-
1965
- except Exception as e:
1966
- # Log unexpected errors but continue with other collections
1967
- logger.debug(f"Error searching collection {collection_name}: {e}")
1968
- continue
1969
-
1970
- # If no results from metadata search OR no metadata exists, fall back to semantic search
1971
- if not all_results:
1972
- await ctx.debug(f"Falling back to semantic search for concept: {concept}")
1973
-
1974
- for collection_name in collections:
1975
- try:
1976
- # Pure semantic search without filters
1977
- results = await qdrant_client.search(
1978
- collection_name=collection_name,
1979
- query_vector=embedding,
1980
- limit=limit,
1981
- score_threshold=0.5, # Lower threshold for broader results
1982
- with_payload=True
1983
- )
1984
-
1985
- for point in results:
1986
- all_results.append({
1987
- 'score': float(point.score),
1988
- 'payload': point.payload,
1989
- 'collection': collection_name,
1990
- 'search_type': 'semantic'
1991
- })
1992
-
1993
- except Exception as e:
1994
- # Log unexpected errors but continue with other collections
1995
- logger.debug(f"Error searching collection {collection_name}: {e}")
1996
- continue
1997
-
1998
- # Sort by score and limit
1999
- all_results.sort(key=lambda x: x['score'], reverse=True)
2000
- all_results = all_results[:limit]
2001
-
2002
- # Format results
2003
- if not all_results:
2004
- metadata_status = "with metadata" if metadata_found else "NO METADATA FOUND"
2005
- return f"""<search_by_concept>
2006
- <concept>{concept}</concept>
2007
- <metadata_health>{metadata_status} (checked {total_points_checked} points)</metadata_health>
2008
- <message>No conversations found about this concept. {'Try running: python scripts/delta-metadata-update.py' if not metadata_found else 'Try different search terms.'}</message>
2009
- </search_by_concept>"""
2010
-
2011
- results_text = []
2012
- for i, result in enumerate(all_results):
2013
- payload = result['payload']
2014
- score = result['score']
2015
- timestamp = payload.get('timestamp', 'Unknown')
2016
- conversation_id = payload.get('conversation_id', 'Unknown')
2017
- project = payload.get('project', 'Unknown')
2018
- concepts = payload.get('concepts', [])
2019
-
2020
- # Get text preview
2021
- text_preview = payload.get('text', '')[:200] + '...' if len(payload.get('text', '')) > 200 else payload.get('text', '')
2022
-
2023
- # File information
2024
- files_info = ""
2025
- if include_files:
2026
- files_analyzed = payload.get('files_analyzed', [])[:5]
2027
- if files_analyzed:
2028
- files_info = f"\n<files_analyzed>{', '.join(files_analyzed)}</files_analyzed>"
2029
-
2030
- # Related concepts
2031
- related_concepts = [c for c in concepts if c != concept.lower()][:5]
2032
-
2033
- results_text.append(f"""<result rank="{i+1}">
2034
- <score>{score:.3f}</score>
2035
- <conversation_id>{conversation_id}</conversation_id>
2036
- <project>{project}</project>
2037
- <timestamp>{timestamp}</timestamp>
2038
- <concepts>{', '.join(concepts)}</concepts>
2039
- <related_concepts>{', '.join(related_concepts)}</related_concepts>{files_info}
2040
- <preview>{text_preview}</preview>
2041
- </result>""")
2042
-
2043
- # Determine if this was a fallback search
2044
- used_fallback = any(r.get('search_type') == 'semantic' for r in all_results)
2045
- metadata_status = "with metadata" if metadata_found else "NO METADATA FOUND"
2046
-
2047
- return f"""<search_by_concept>
2048
- <concept>{concept}</concept>
2049
- <metadata_health>{metadata_status} (checked {total_points_checked} points)</metadata_health>
2050
- <search_type>{'fallback_semantic' if used_fallback else 'metadata_based'}</search_type>
2051
- <count>{len(all_results)}</count>
2052
- <results>
2053
- {''.join(results_text)}
2054
- </results>
2055
- </search_by_concept>"""
2056
-
2057
-
2058
- # Debug output
2059
- print(f"[DEBUG] FastMCP server created with name: {mcp.name}")
2060
-
2061
- @mcp.tool()
2062
- async def get_full_conversation(
2063
- ctx: Context,
2064
- conversation_id: str = Field(description="The conversation ID from search results (cid)"),
2065
- project: Optional[str] = Field(default=None, description="Optional project name to help locate the file")
2066
- ) -> str:
2067
- """Get the full JSONL conversation file path for a conversation ID.
2068
- This allows agents to read complete conversations instead of truncated excerpts."""
2069
-
2070
- # Base path for Claude conversations
2071
- base_path = Path.home() / '.claude/projects'
2072
-
2073
- # Build list of directories to search
2074
- search_dirs = []
2075
-
2076
- if project:
2077
- # Try various project directory name formats
2078
- sanitized_project = project.replace('/', '-')
2079
- search_dirs.extend([
2080
- base_path / project,
2081
- base_path / sanitized_project,
2082
- base_path / f"-Users-*-projects-{project}",
2083
- base_path / f"-Users-*-projects-{sanitized_project}"
2084
- ])
2085
- else:
2086
- # Search all project directories
2087
- search_dirs = list(base_path.glob("*"))
2088
-
2089
- # Search for the JSONL file
2090
- jsonl_path = None
2091
- for search_dir in search_dirs:
2092
- if not search_dir.is_dir():
2093
- continue
2094
-
2095
- potential_path = search_dir / f"{conversation_id}.jsonl"
2096
- if potential_path.exists():
2097
- jsonl_path = potential_path
2098
- break
2099
-
2100
- if not jsonl_path:
2101
- # Try searching all directories as fallback
2102
- for proj_dir in base_path.glob("*"):
2103
- if proj_dir.is_dir():
2104
- potential_path = proj_dir / f"{conversation_id}.jsonl"
2105
- if potential_path.exists():
2106
- jsonl_path = potential_path
2107
- break
2108
-
2109
- if not jsonl_path:
2110
- return f"""<full_conversation>
2111
- <conversation_id>{conversation_id}</conversation_id>
2112
- <status>not_found</status>
2113
- <message>Conversation file not found. Searched {len(search_dirs)} directories.</message>
2114
- <hint>Try using the project parameter or check if the conversation ID is correct.</hint>
2115
- </full_conversation>"""
2116
-
2117
- # Get file stats
2118
- file_stats = jsonl_path.stat()
2119
-
2120
- # Count messages
2121
- try:
2122
- with open(jsonl_path, 'r', encoding='utf-8') as f:
2123
- message_count = sum(1 for _ in f)
2124
- except:
2125
- message_count = 0
2126
-
2127
- return f"""<full_conversation>
2128
- <conversation_id>{conversation_id}</conversation_id>
2129
- <status>found</status>
2130
- <file_path>{jsonl_path}</file_path>
2131
- <file_size>{file_stats.st_size}</file_size>
2132
- <message_count>{message_count}</message_count>
2133
- <project>{jsonl_path.parent.name}</project>
2134
- <instructions>
2135
- You can now use the Read tool to read the full conversation from:
2136
- {jsonl_path}
2137
-
2138
- Each line in the JSONL file is a separate message with complete content.
2139
- This gives you access to:
2140
- - Complete code blocks (not truncated)
2141
- - Full problem descriptions and solutions
2142
- - Entire debugging sessions
2143
- - Complete architectural decisions and discussions
2144
- </instructions>
2145
- </full_conversation>"""
2146
-
2147
-
2148
- @mcp.tool()
2149
- async def get_next_results(
2150
- ctx: Context,
2151
- query: str = Field(description="The original search query"),
2152
- offset: int = Field(default=3, description="Number of results to skip (for pagination)"),
2153
- limit: int = Field(default=3, description="Number of additional results to return"),
2154
- min_score: float = Field(default=0.3, description="Minimum similarity score (0-1)"),
2155
- project: Optional[str] = Field(default=None, description="Search specific project only")
2156
- ) -> str:
2157
- """Get additional search results after an initial search (pagination support)."""
2158
- global qdrant_client, embedding_manager
2159
-
2160
- try:
2161
- # Generate embedding for the query
2162
- embedding = await generate_embedding(query)
2163
-
2164
- # Determine which collections to search
2165
- if project == "all" or not project:
2166
- # Search all collections if project is "all" or not specified
2167
- collections = await get_all_collections()
2168
- else:
2169
- # Search specific project - normalize first!
2170
- all_collections = await get_all_collections()
2171
- normalized_project = normalize_project_name(project)
2172
- project_hash = hashlib.md5(normalized_project.encode()).hexdigest()[:8]
2173
- collections = [
2174
- c for c in all_collections
2175
- if c.startswith(f"conv_{project_hash}_")
2176
- ]
2177
- if not collections:
2178
- # Fall back to searching all collections
2179
- collections = all_collections
2180
-
2181
- if not collections:
2182
- return """<next_results>
2183
- <error>No collections available to search</error>
2184
- </next_results>"""
2185
-
2186
- # Collect all results from all collections
2187
- all_results = []
2188
- for collection_name in collections:
2189
- try:
2190
- # Check if collection exists
2191
- collection_info = await qdrant_client.get_collection(collection_name)
2192
- if not collection_info:
2193
- continue
2194
-
2195
- # Search with reasonable limit to account for offset
2196
- max_search_limit = 100 # Define a reasonable cap
2197
- search_limit = min(offset + limit + 10, max_search_limit)
2198
- results = await qdrant_client.search(
2199
- collection_name=collection_name,
2200
- query_vector=embedding,
2201
- limit=search_limit,
2202
- score_threshold=min_score
2203
- )
2204
-
2205
- for point in results:
2206
- payload = point.payload
2207
- score = float(point.score)
2208
-
2209
- # Apply time-based decay if enabled
2210
- use_decay_bool = ENABLE_MEMORY_DECAY # Use global default
2211
- if use_decay_bool and 'timestamp' in payload:
2212
- try:
2213
- timestamp = datetime.fromisoformat(payload['timestamp'].replace('Z', '+00:00'))
2214
- age_days = (datetime.now(timezone.utc) - timestamp).total_seconds() / (24 * 60 * 60)
2215
- # Use consistent half-life formula: decay = exp(-ln(2) * age / half_life)
2216
- ln2 = math.log(2)
2217
- decay_factor = math.exp(-ln2 * age_days / DECAY_SCALE_DAYS)
2218
- # Apply multiplicative formula: score * ((1 - weight) + weight * decay)
2219
- score = score * ((1 - DECAY_WEIGHT) + DECAY_WEIGHT * decay_factor)
2220
- except (ValueError, TypeError) as e:
2221
- # Log but continue - timestamp format issue shouldn't break search
2222
- logger.debug(f"Failed to apply decay for timestamp {payload.get('timestamp')}: {e}")
2223
-
2224
- all_results.append({
2225
- 'score': score,
2226
- 'payload': payload,
2227
- 'collection': collection_name
2228
- })
2229
-
2230
- except Exception as e:
2231
- # Log unexpected errors but continue with other collections
2232
- logger.debug(f"Error searching collection {collection_name}: {e}")
2233
- continue
2234
-
2235
- # Sort by score
2236
- all_results.sort(key=lambda x: x['score'], reverse=True)
2237
-
2238
- # Apply pagination
2239
- paginated_results = all_results[offset:offset + limit]
2240
-
2241
- if not paginated_results:
2242
- return f"""<next_results>
2243
- <query>{query}</query>
2244
- <offset>{offset}</offset>
2245
- <status>no_more_results</status>
2246
- <message>No additional results found beyond offset {offset}</message>
2247
- </next_results>"""
2248
-
2249
- # Format results
2250
- results_text = []
2251
- for i, result in enumerate(paginated_results, start=offset + 1):
2252
- payload = result['payload']
2253
- score = result['score']
2254
- timestamp = payload.get('timestamp', 'Unknown')
2255
- conversation_id = payload.get('conversation_id', 'Unknown')
2256
- project = payload.get('project', 'Unknown')
2257
-
2258
- # Get text preview (store text once to avoid multiple calls)
2259
- text = payload.get('text', '')
2260
- text_preview = text[:300] + '...' if len(text) > 300 else text
2261
-
2262
- results_text.append(f"""
2263
- <result index="{i}">
2264
- <score>{score:.3f}</score>
2265
- <timestamp>{timestamp}</timestamp>
2266
- <project>{project}</project>
2267
- <conversation_id>{conversation_id}</conversation_id>
2268
- <preview>{text_preview}</preview>
2269
- </result>""")
2270
-
2271
- # Check if there are more results available
2272
- has_more = len(all_results) > (offset + limit)
2273
- next_offset = offset + limit if has_more else None
2274
-
2275
- return f"""<next_results>
2276
- <query>{query}</query>
2277
- <offset>{offset}</offset>
2278
- <limit>{limit}</limit>
2279
- <count>{len(paginated_results)}</count>
2280
- <total_available>{len(all_results)}</total_available>
2281
- <has_more>{has_more}</has_more>
2282
- {f'<next_offset>{next_offset}</next_offset>' if next_offset else ''}
2283
- <results>{''.join(results_text)}
2284
- </results>
2285
- </next_results>"""
2286
-
2287
- except Exception as e:
2288
- await ctx.error(f"Pagination failed: {str(e)}")
2289
- return f"""<next_results>
2290
- <error>Failed to get next results: {str(e)}</error>
2291
- </next_results>"""
653
+ # Register temporal tools after all functions are defined
654
+ register_temporal_tools(
655
+ mcp,
656
+ qdrant_client,
657
+ QDRANT_URL,
658
+ get_all_collections,
659
+ generate_embedding,
660
+ initialize_embeddings,
661
+ normalize_project_name
662
+ )
663
+ print(f"[INFO] Temporal tools registered", file=sys.stderr)
664
+
665
+ # Register search tools
666
+ def get_embedding_manager():
667
+ """Factory function to get the current embedding manager."""
668
+ from .embedding_manager import get_embedding_manager as get_em
669
+ return get_em()
670
+
671
+ # Initialize ProjectResolver for collection name mapping
672
+ # ProjectResolver needs a sync client, not async
673
+ from qdrant_client import QdrantClient as SyncQdrantClient
674
+ sync_qdrant_client = SyncQdrantClient(url=QDRANT_URL)
675
+ project_resolver = ProjectResolver(sync_qdrant_client)
676
+
677
+ register_search_tools(
678
+ mcp,
679
+ qdrant_client,
680
+ QDRANT_URL,
681
+ get_embedding_manager,
682
+ normalize_project_name,
683
+ ENABLE_MEMORY_DECAY,
684
+ DECAY_WEIGHT,
685
+ DECAY_SCALE_DAYS,
686
+ USE_NATIVE_DECAY,
687
+ NATIVE_DECAY_AVAILABLE,
688
+ decay_manager,
689
+ project_resolver # Pass the resolver
690
+ )
2292
691
 
692
+ # Register reflection tools
693
+ register_reflection_tools(
694
+ mcp,
695
+ qdrant_client,
696
+ QDRANT_URL,
697
+ get_embedding_manager,
698
+ normalize_project_name
699
+ )
2293
700
 
2294
701
  # Run the server
2295
702
  if __name__ == "__main__":