@cmdoss/memwal-sdk 0.6.0

package/src/retrieval/AdvancedSearchService.ts

@@ -0,0 +1,630 @@
+/**
+ * AdvancedSearchService - Sophisticated Search & Filtering
+ * 
+ * Provides advanced search capabilities including:
+ * - Multi-dimensional filtering with facets
+ * - Semantic search with AI understanding  
+ * - Time-series analysis and temporal queries
+ * - Knowledge graph traversal
+ * - Search aggregations and analytics
+ * - Custom scoring algorithms
+ */
+
+import { UnifiedMemoryQuery, UnifiedMemoryResult } from './MemoryRetrievalService';
+import { EmbeddingService } from '../services/EmbeddingService';
+import { KnowledgeGraphManager } from '../graph/KnowledgeGraphManager';
+
+export interface SearchFilter {
+  type: 'category' | 'dateRange' | 'importance' | 'tags' | 'contentType' | 'similarity' | 'custom';
+  field: string;
+  operator: 'equals' | 'contains' | 'range' | 'greater' | 'less' | 'in' | 'regex';
+  value: any;
+  weight?: number;
+}
+
+export interface SearchAggregation {
+  name: string;
+  type: 'count' | 'sum' | 'avg' | 'max' | 'min' | 'histogram' | 'terms' | 'dateHistogram';
+  field: string;
+  size?: number;
+  interval?: string;
+  ranges?: Array<{ from?: number; to?: number; key?: string }>;
+}
+
+export interface SearchFacets {
+  categories: Array<{ value: string; count: number; selected?: boolean }>;
+  dateRanges: Array<{ label: string; from: Date; to: Date; count: number }>;
+  importanceRanges: Array<{ label: string; min: number; max: number; count: number }>;
+  tags: Array<{ value: string; count: number; coOccurrence?: string[] }>;
+  contentTypes: Array<{ value: string; count: number; extensions?: string[] }>;
+  similarityScores: Array<{ range: string; count: number; avgScore: number }>;
+}
+
+export interface SemanticSearchQuery {
+  query: string;
+  intent?: 'question' | 'statement' | 'command' | 'exploration';
+  context?: string;
+  entities?: string[];
+  semanticExpansion?: boolean;
+  conceptualDepth?: number;
+}
+
+export interface TemporalSearchQuery {
+  timeframe: 'last_hour' | 'today' | 'this_week' | 'this_month' | 'this_year' | 'custom';
+  customRange?: { start: Date; end: Date };
+  temporalPattern?: 'frequency' | 'seasonality' | 'trends' | 'anomalies';
+  groupBy?: 'hour' | 'day' | 'week' | 'month' | 'quarter' | 'year';
+}
+
+export interface GraphSearchQuery {
+  startNodes?: string[];
+  maxDepth: number;
+  relationshipTypes?: string[];
+  pathFinding?: 'shortest' | 'all' | 'weighted';
+  includeMetadata?: boolean;
+  traversalDirection?: 'outgoing' | 'incoming' | 'both';
+}
+
+export interface HybridSearchQuery {
+  vectorWeight: number;
+  semanticWeight: number;
+  keywordWeight: number;
+  graphWeight: number;
+  temporalWeight: number;
+  customScoring?: (result: UnifiedMemoryResult) => number;
+}
+
+export interface SearchResult {
+  results: UnifiedMemoryResult[];
+  facets: SearchFacets;
+  aggregations: Record<string, any>;
+  suggestions: {
+    queryExpansions: string[];
+    similarQueries: string[];
+    recommendedFilters: SearchFilter[];
+    explorationPaths: string[];
+  };
+  performance: {
+    totalTime: number;
+    searchTime: number;
+    aggregationTime: number;
+    facetTime: number;
+  };
+}
+
+/**
+ * Advanced Search Service with sophisticated filtering and analysis
+ */
+export class AdvancedSearchService {
+  private embeddingService: EmbeddingService;
+  private graphManager: KnowledgeGraphManager;
+
+  constructor(config?: {
+    embeddingService?: EmbeddingService;
+    graphManager?: KnowledgeGraphManager;
+  }) {
+    this.embeddingService = config?.embeddingService ?? new EmbeddingService();
+    this.graphManager = config?.graphManager ?? new KnowledgeGraphManager();
+  }
+
+  // ==================== ADVANCED SEARCH METHODS ====================
+
+  /**
+   * Multi-faceted search with dynamic filtering
+   */
+  async facetedSearch(
+    baseQuery: UnifiedMemoryQuery,
+    filters: SearchFilter[],
+    aggregations: SearchAggregation[],
+    options?: {
+      includeFacets?: boolean;
+      facetSize?: number;
+      timeout?: number;
+    }
+  ): Promise<SearchResult> {
+    const startTime = Date.now();
+
+    try {
+      // Apply filters to base query
+      const filteredQuery = this.applyFiltersToQuery(baseQuery, filters);
+      
+      // Execute base search (would call MemoryRetrievalService)
+      const baseResults = await this.executeBaseSearch(filteredQuery);
+      
+      // Generate facets if requested
+      let facets: SearchFacets = {
+        categories: [],
+        dateRanges: [],
+        importanceRanges: [],
+        tags: [],
+        contentTypes: [],
+        similarityScores: []
+      };
+      
+      let facetTime = 0;
+      if (options?.includeFacets !== false) {
+        const facetStart = Date.now();
+        facets = await this.generateFacets(baseResults, baseQuery);
+        facetTime = Date.now() - facetStart;
+      }
+
+      // Execute aggregations
+      const aggregationStart = Date.now();
+      const aggregationResults = await this.executeAggregations(baseResults, aggregations);
+      const aggregationTime = Date.now() - aggregationStart;
+
+      // Generate suggestions
+      const suggestions = await this.generateSearchSuggestions(baseQuery, baseResults);
+
+      return {
+        results: baseResults,
+        facets,
+        aggregations: aggregationResults,
+        suggestions,
+        performance: {
+          totalTime: Date.now() - startTime,
+          searchTime: Date.now() - startTime - facetTime - aggregationTime,
+          aggregationTime,
+          facetTime
+        }
+      };
+
+    } catch (error) {
+      throw new Error(`Faceted search failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
+    }
+  }
+
+  /**
+   * Semantic search with AI-powered understanding
+   */
+  async semanticSearch(
+    semanticQuery: SemanticSearchQuery,
+    baseQuery: UnifiedMemoryQuery
+  ): Promise<UnifiedMemoryResult[]> {
+    // Analyze query semantics
+    const semanticAnalysis = await this.analyzeSemanticQuery(semanticQuery);
+    
+    // Expand query if requested
+    let expandedQueries = [semanticQuery.query];
+    if (semanticQuery.semanticExpansion) {
+      expandedQueries = await this.expandQuerySemantics(semanticQuery.query);
+    }
+
+    // Search with expanded understanding
+    const allResults: UnifiedMemoryResult[] = [];
+    for (const query of expandedQueries) {
+      const queryResults = await this.executeSemanticQuery(query, semanticAnalysis, baseQuery);
+      allResults.push(...queryResults);
+    }
+
+    // Deduplicate and re-rank
+    const uniqueResults = this.deduplicateResults(allResults);
+    return this.rankSemanticResults(uniqueResults, semanticAnalysis);
+  }
+
+  /**
+   * Temporal search with time-series analysis
+   */
+  async temporalSearch(
+    temporalQuery: TemporalSearchQuery,
+    baseQuery: UnifiedMemoryQuery
+  ): Promise<{
+    results: UnifiedMemoryResult[];
+    timeline: Array<{
+      period: string;
+      count: number;
+      memories: string[];
+      trends: {
+        direction: 'up' | 'down' | 'stable';
+        strength: number;
+        anomalies: boolean;
+      };
+    }>;
+    patterns: {
+      peakHours: number[];
+      peakDays: string[];
+      seasonality: any;
+      correlations: Array<{ event: string; correlation: number }>;
+    };
+  }> {
+    // Convert temporal query to date range
+    const dateRange = this.convertTemporalQuery(temporalQuery);
+    
+    // Execute search with temporal constraints
+    const temporalBaseQuery = {
+      ...baseQuery,
+      dateRange
+    };
+    
+    const results = await this.executeBaseSearch(temporalBaseQuery);
+    
+    // Analyze temporal patterns
+    const timeline = this.generateTimeline(results, temporalQuery.groupBy || 'day');
+    const patterns = await this.analyzeTempralPatterns(results, temporalQuery);
+
+    return {
+      results,
+      timeline,
+      patterns
+    };
+  }
+
+  /**
+   * Knowledge graph traversal search
+   */
+  async graphSearch(
+    graphQuery: GraphSearchQuery,
+    baseQuery: UnifiedMemoryQuery
+  ): Promise<{
+    results: UnifiedMemoryResult[];
+    paths: Array<{
+      startNode: string;
+      endNode: string;
+      path: string[];
+      relationships: Array<{ from: string; to: string; type: string; weight: number }>;
+      score: number;
+    }>;
+    subgraphs: Array<{
+      nodes: string[];
+      relationships: Array<{ from: string; to: string; type: string }>;
+      centrality: Record<string, number>;
+    }>;
+  }> {
+    // Execute graph traversal
+    const traversalResults = await this.executeGraphTraversal(graphQuery, baseQuery);
+    
+    // Find related memories
+    const results = await this.findMemoriesFromGraphNodes(traversalResults.nodes, baseQuery);
+    
+    // Analyze graph structure
+    const paths = await this.findSignificantPaths(traversalResults, graphQuery);
+    const subgraphs = await this.identifySubgraphs(traversalResults, baseQuery.userId);
+
+    return {
+      results,
+      paths,
+      subgraphs
+    };
+  }
+
+  /**
+   * Hybrid search combining multiple algorithms
+   */
+  async hybridSearch(
+    hybridQuery: HybridSearchQuery,
+    baseQuery: UnifiedMemoryQuery,
+    specificQueries: {
+      semantic?: SemanticSearchQuery;
+      temporal?: TemporalSearchQuery;
+      graph?: GraphSearchQuery;
+    }
+  ): Promise<UnifiedMemoryResult[]> {
+    const searchResults: Array<{ results: UnifiedMemoryResult[]; weight: number }> = [];
+
+    // Vector search (base)
+    if (hybridQuery.vectorWeight > 0) {
+      const vectorResults = await this.executeBaseSearch({
+        ...baseQuery,
+        searchType: 'vector'
+      });
+      searchResults.push({ results: vectorResults, weight: hybridQuery.vectorWeight });
+    }
+
+    // Semantic search
+    if (hybridQuery.semanticWeight > 0 && specificQueries.semantic) {
+      const semanticResults = await this.semanticSearch(specificQueries.semantic, baseQuery);
+      searchResults.push({ results: semanticResults, weight: hybridQuery.semanticWeight });
+    }
+
+    // Keyword search
+    if (hybridQuery.keywordWeight > 0) {
+      const keywordResults = await this.executeBaseSearch({
+        ...baseQuery,
+        searchType: 'keyword'
+      });
+      searchResults.push({ results: keywordResults, weight: hybridQuery.keywordWeight });
+    }
+
+    // Graph search
+    if (hybridQuery.graphWeight > 0 && specificQueries.graph) {
+      const graphResults = await this.graphSearch(specificQueries.graph, baseQuery);
+      searchResults.push({ results: graphResults.results, weight: hybridQuery.graphWeight });
+    }
+
+    // Temporal search
+    if (hybridQuery.temporalWeight > 0 && specificQueries.temporal) {
+      const temporalResults = await this.temporalSearch(specificQueries.temporal, baseQuery);
+      searchResults.push({ results: temporalResults.results, weight: hybridQuery.temporalWeight });
+    }
+
+    // Combine results using weighted scoring
+    return this.combineHybridResults(searchResults, hybridQuery.customScoring);
+  }
+
+  // ==================== HELPER METHODS ====================
+
+  private applyFiltersToQuery(query: UnifiedMemoryQuery, filters: SearchFilter[]): UnifiedMemoryQuery {
+    const newQuery = { ...query };
+    
+    filters.forEach(filter => {
+      switch (filter.type) {
+        case 'category':
+          newQuery.categories = newQuery.categories || [];
+          if (filter.operator === 'equals') {
+            newQuery.categories.push(filter.value);
+          }
+          break;
+        case 'dateRange':
+          if (filter.operator === 'range') {
+            newQuery.dateRange = filter.value;
+          }
+          break;
+        case 'importance':
+          if (filter.operator === 'range') {
+            newQuery.importanceRange = filter.value;
+          }
+          break;
+        case 'tags':
+          newQuery.tags = newQuery.tags || [];
+          if (filter.operator === 'in') {
+            newQuery.tags.push(...filter.value);
+          }
+          break;
+      }
+    });
+
+    return newQuery;
+  }
+
+  private async executeBaseSearch(query: UnifiedMemoryQuery): Promise<UnifiedMemoryResult[]> {
+    // This would call the MemoryRetrievalService
+    // For now, return placeholder
+    return [];
+  }
+
+  private async generateFacets(
+    results: UnifiedMemoryResult[], 
+    query: UnifiedMemoryQuery
+  ): Promise<SearchFacets> {
+    // Generate category facets
+    const categoryMap = new Map<string, number>();
+    results.forEach(result => {
+      categoryMap.set(result.category, (categoryMap.get(result.category) || 0) + 1);
+    });
+
+    const categories = Array.from(categoryMap.entries()).map(([value, count]) => ({
+      value,
+      count
+    }));
+
+    // Generate other facets...
+    const dateRanges = this.generateDateRangeFacets(results);
+    const importanceRanges = this.generateImportanceRangeFacets(results);
+    const tags = this.generateTagFacets(results);
+    const contentTypes = this.generateContentTypeFacets(results);
+    const similarityScores = this.generateSimilarityFacets(results);
+
+    return {
+      categories,
+      dateRanges,
+      importanceRanges,
+      tags,
+      contentTypes,
+      similarityScores
+    };
+  }
+
+  private async executeAggregations(
+    results: UnifiedMemoryResult[],
+    aggregations: SearchAggregation[]
+  ): Promise<Record<string, any>> {
+    const aggregationResults: Record<string, any> = {};
+
+    for (const agg of aggregations) {
+      switch (agg.type) {
+        case 'count':
+          aggregationResults[agg.name] = results.length;
+          break;
+        case 'terms':
+          aggregationResults[agg.name] = this.executeTermsAggregation(results, agg);
+          break;
+        case 'dateHistogram':
+          aggregationResults[agg.name] = this.executeDateHistogramAggregation(results, agg);
+          break;
+        case 'histogram':
+          aggregationResults[agg.name] = this.executeHistogramAggregation(results, agg);
+          break;
+      }
+    }
+
+    return aggregationResults;
+  }
+
+  private async generateSearchSuggestions(
+    query: UnifiedMemoryQuery,
+    results: UnifiedMemoryResult[]
+  ): Promise<SearchResult['suggestions']> {
+    return {
+      queryExpansions: await this.generateQueryExpansions(query.query || ''),
+      similarQueries: await this.findSimilarQueries(query),
+      recommendedFilters: this.recommendFilters(results, query),
+      explorationPaths: await this.generateExplorationPaths(results, query)
+    };
+  }
+
+  // Placeholder implementations for complex methods
+  private async analyzeSemanticQuery(query: SemanticSearchQuery): Promise<any> {
+    // AI-powered semantic analysis
+    return {};
+  }
+
+  private async expandQuerySemantics(query: string): Promise<string[]> {
+    // Semantic query expansion
+    return [query];
+  }
+
+  private async executeSemanticQuery(
+    query: string, 
+    analysis: any, 
+    baseQuery: UnifiedMemoryQuery
+  ): Promise<UnifiedMemoryResult[]> {
+    return [];
+  }
+
+  private deduplicateResults(results: UnifiedMemoryResult[]): UnifiedMemoryResult[] {
+    const seen = new Set<string>();
+    return results.filter(result => {
+      if (seen.has(result.id)) return false;
+      seen.add(result.id);
+      return true;
+    });
+  }
+
+  private rankSemanticResults(
+    results: UnifiedMemoryResult[], 
+    analysis: any
+  ): UnifiedMemoryResult[] {
+    return results.sort((a, b) => b.relevanceScore - a.relevanceScore);
+  }
+
+  private convertTemporalQuery(query: TemporalSearchQuery): { start?: Date; end?: Date } {
+    const now = new Date();
+    
+    switch (query.timeframe) {
+      case 'today':
+        return {
+          start: new Date(now.getFullYear(), now.getMonth(), now.getDate()),
+          end: now
+        };
+      case 'this_week':
+        const weekStart = new Date(now);
+        weekStart.setDate(now.getDate() - now.getDay());
+        return { start: weekStart, end: now };
+      case 'custom':
+        return query.customRange || {};
+      default:
+        return {};
+    }
+  }
+
+  private generateTimeline(results: UnifiedMemoryResult[], groupBy: string): any[] {
+    // Generate timeline buckets
+    return [];
+  }
+
+  private async analyzeTempralPatterns(
+    results: UnifiedMemoryResult[], 
+    query: TemporalSearchQuery
+  ): Promise<any> {
+    // Analyze temporal patterns
+    return {
+      peakHours: [],
+      peakDays: [],
+      seasonality: {},
+      correlations: []
+    };
+  }
+
+  private async executeGraphTraversal(
+    query: GraphSearchQuery, 
+    baseQuery: UnifiedMemoryQuery
+  ): Promise<{ nodes: string[]; relationships: any[] }> {
+    return { nodes: [], relationships: [] };
+  }
+
+  private async findMemoriesFromGraphNodes(
+    nodes: string[], 
+    baseQuery: UnifiedMemoryQuery
+  ): Promise<UnifiedMemoryResult[]> {
+    return [];
+  }
+
+  private async findSignificantPaths(traversalResults: any, query: GraphSearchQuery): Promise<any[]> {
+    return [];
+  }
+
+  private async identifySubgraphs(traversalResults: any, userId: string): Promise<any[]> {
+    return [];
+  }
+
+  private combineHybridResults(
+    searchResults: Array<{ results: UnifiedMemoryResult[]; weight: number }>,
+    customScoring?: (result: UnifiedMemoryResult) => number
+  ): UnifiedMemoryResult[] {
+    const resultMap = new Map<string, UnifiedMemoryResult>();
+    const scoreMap = new Map<string, number>();
+
+    // Combine all results with weighted scores
+    searchResults.forEach(({ results, weight }) => {
+      results.forEach(result => {
+        const existingScore = scoreMap.get(result.id) || 0;
+        const newScore = customScoring ? customScoring(result) : result.relevanceScore;
+        scoreMap.set(result.id, existingScore + (newScore * weight));
+        
+        if (!resultMap.has(result.id)) {
+          resultMap.set(result.id, result);
+        }
+      });
+    });
+
+    // Update scores and sort
+    const finalResults = Array.from(resultMap.values());
+    finalResults.forEach(result => {
+      result.relevanceScore = scoreMap.get(result.id) || result.relevanceScore;
+    });
+
+    return finalResults.sort((a, b) => b.relevanceScore - a.relevanceScore);
+  }
+
+  // Facet generation helpers
+  private generateDateRangeFacets(results: UnifiedMemoryResult[]): SearchFacets['dateRanges'] {
+    return [];
+  }
+
+  private generateImportanceRangeFacets(results: UnifiedMemoryResult[]): SearchFacets['importanceRanges'] {
+    return [];
+  }
+
+  private generateTagFacets(results: UnifiedMemoryResult[]): SearchFacets['tags'] {
+    return [];
+  }
+
+  private generateContentTypeFacets(results: UnifiedMemoryResult[]): SearchFacets['contentTypes'] {
+    return [];
+  }
+
+  private generateSimilarityFacets(results: UnifiedMemoryResult[]): SearchFacets['similarityScores'] {
+    return [];
+  }
+
+  private executeTermsAggregation(results: UnifiedMemoryResult[], agg: SearchAggregation): any {
+    return {};
+  }
+
+  private executeDateHistogramAggregation(results: UnifiedMemoryResult[], agg: SearchAggregation): any {
+    return {};
+  }
+
+  private executeHistogramAggregation(results: UnifiedMemoryResult[], agg: SearchAggregation): any {
+    return {};
+  }
+
+  private async generateQueryExpansions(query: string): Promise<string[]> {
+    return [];
+  }
+
+  private async findSimilarQueries(query: UnifiedMemoryQuery): Promise<string[]> {
+    return [];
+  }
+
+  private recommendFilters(results: UnifiedMemoryResult[], query: UnifiedMemoryQuery): SearchFilter[] {
+    return [];
+  }
+
+  private async generateExplorationPaths(
+    results: UnifiedMemoryResult[], 
+    query: UnifiedMemoryQuery
+  ): Promise<string[]> {
+    return [];
+  }
+}