hokipoki 0.1.0

data/lib/hokipoki/intelligence/smart_retrieval_engine.rb

@@ -0,0 +1,401 @@
+# frozen_string_literal: true
+
+module Hokipoki
+  module Intelligence
+    # Smart Retrieval Engine with Real-time User Feedback
+    # Enhanced version of the original with comprehensive operation visibility
+    class SmartRetrievalEngine
+      include Singleton
+
+      def initialize
+        @logger = Rails.logger
+        @feedback = Feedback::DisplayManager.instance
+      end
+
+      # PATTERN-003: Intent-based analysis with surgical precision + real vector similarity
+      def retrieve_targeted_facts(query, token_budget: 1500, intent: 'auto')
+        start_time = Time.current
+
+        @feedback.performing_vector_retrieval(query, token_budget) do
+          @logger.info "🎯 SMART RETRIEVAL: Processing query with vector similarity"
+        end
+
+        begin
+          # Step 1: Analyze intent if auto
+          if intent == 'auto'
+            @feedback.debug_info('intent_analysis', 'detecting query intent')
+            intent = analyze_query_intent(query)
+            @feedback.debug_info('intent_detected', intent.to_s)
+          end
+
+          # Step 2: Multi-stage retrieval with vector similarity
+          @feedback.pulling_from_hive_mind('vectors')
+          facts = perform_multi_stage_retrieval(query, token_budget, intent)
+
+          # Step 3: Calculate processing metrics
+          processing_time = ((Time.current - start_time) * 1000).round(2)
+
+          if facts.present?
+            fact_count = facts.is_a?(Array) ? facts.length : 1
+            total_tokens = estimate_total_tokens(facts)
+            tokens_saved = calculate_tokens_saved(total_tokens, query)
+
+            @feedback.vector_results_found(fact_count, processing_time)
+            @feedback.vector_context_built(fact_count, total_tokens)
+            @feedback.total_tokens_saved(tokens_saved, get_session_total_saved(tokens_saved))
+            @feedback.token_usage(total_tokens, token_budget)
+
+            return facts
+          else
+            @feedback.vector_results_found(0, processing_time)
+            @feedback.operation_warning('vector_retrieval', 'falling back to pattern guidance')
+          end
+
+        rescue => e
+          @feedback.operation_error('vector_retrieval', e.message)
+          @logger.warn "Vector retrieval failed: #{e.message}, falling back to patterns"
+        end
+
+        # Fallback to pattern-based retrieval
+        @feedback.debug_info('fallback', 'using pattern-based retrieval')
+        fallback_result = get_pattern_fallback(query, intent)
+
+        if fallback_result
+          @feedback.vector_results_found(1, ((Time.current - start_time) * 1000).round(2))
+          return fallback_result
+        else
+          @feedback.vector_results_found(0, ((Time.current - start_time) * 1000).round(2))
+          return "Context: No specific guidance available for query: #{query}"
+        end
+      end
+
+      # Multi-stage retrieval with comprehensive feedback
+      def perform_multi_stage_retrieval(query, token_budget, intent)
+        @feedback.debug_info('retrieval_stages', 'starting multi-stage retrieval')
+
+        # Stage 1: Vector similarity search
+        @feedback.debug_info('stage_1', 'vector similarity search')
+        vector_candidates = perform_vector_similarity_search(query, intent)
+
+        # Stage 2: Rank by relevance
+        @feedback.debug_info('stage_2', 'ranking by relevance')
+        ranked_candidates = rank_by_relevance(vector_candidates, query, intent)
+
+        # Stage 3: Filter by quality
+        @feedback.debug_info('stage_3', 'filtering by quality')
+        filtered_candidates = filter_by_quality(ranked_candidates)
+
+        # Stage 4: Build context within token budget
+        @feedback.debug_info('stage_4', 'building final context')
+        final_context = build_contextual_response(filtered_candidates, query, token_budget)
+
+        final_context
+      end
+
+      # Vector similarity search with detailed feedback
+      def perform_vector_similarity_search(query, intent)
+        @feedback.debug_info('vector_search', "searching for intent: #{intent}")
+
+        candidates = []
+
+        # Search 1: Exact semantic match
+        @feedback.debug_info('search_exact', 'performing exact semantic match')
+        exact_matches = search_documents_by_content(query, similarity_threshold: 0.9, limit: 10)
+        candidates.concat(tag_results(exact_matches, :exact_match))
+        @feedback.debug_info('exact_results', "found #{exact_matches.length} exact matches")
+
+        # Search 2: Conceptual similarity
+        @feedback.debug_info('search_conceptual', 'performing conceptual similarity search')
+        conceptual_matches = search_documents_by_content(query, similarity_threshold: 0.7, limit: 15)
+        candidates.concat(tag_results(conceptual_matches, :conceptual_match))
+        @feedback.debug_info('conceptual_results', "found #{conceptual_matches.length} conceptual matches")
+
+        # Search 3: Intent-based search
+        @feedback.debug_info('search_intent', "performing intent-based search for #{intent}")
+        intent_matches = search_by_intent(query, intent)
+        candidates.concat(tag_results(intent_matches, :intent_match))
+        @feedback.debug_info('intent_results', "found #{intent_matches.length} intent matches")
+
+        # Remove duplicates
+        unique_candidates = deduplicate_candidates(candidates)
+        @feedback.debug_info('deduplication', "#{candidates.length} → #{unique_candidates.length} after deduplication")
+
+        unique_candidates
+      end
+
+      # Search documents with feedback
+      def search_documents_by_content(query, similarity_threshold: 0.7, limit: 10)
+        @feedback.debug_info('db_query', "searching documents (threshold: #{similarity_threshold})")
+
+        # Check if Document model is available
+        unless defined?(Document)
+          @feedback.operation_warning('db_search', 'Document model not available')
+          return []
+        end
+
+        begin
+          # Search documents with content matching
+          results = Document.where("content ILIKE ?", "%#{sanitize_query(query)}%")
+                           .limit(limit * 2) # Get extra for filtering
+                           .includes(:keywords)
+
+          @feedback.debug_info('db_results', "found #{results.count} database matches")
+
+          # Calculate similarity scores
+          scored_results = results.map do |doc|
+            similarity_score = calculate_content_similarity(query, doc.content)
+
+            {
+              document: doc,
+              content: doc.content,
+              similarity: similarity_score,
+              metadata: doc.metadata || {},
+              keywords: extract_keywords(doc),
+              relevance: assess_content_relevance(doc, query)
+            }
+          end
+
+          # Filter by threshold and sort
+          filtered_results = scored_results.select { |r| r[:similarity] >= similarity_threshold }
+          final_results = filtered_results.sort_by { |r| -(r[:similarity] * 0.7 + r[:relevance] * 0.3) }
+                                        .first(limit)
+
+          @feedback.debug_info('filtering', "#{scored_results.length} → #{final_results.length} after filtering")
+
+          final_results
+
+        rescue => e
+          @feedback.operation_error('db_search', e.message)
+          []
+        end
+      end
+
+      # Intent-based search with feedback
+      def search_by_intent(query, intent)
+        @feedback.debug_info('intent_search', "searching for #{intent} intent")
+
+        # Map intent to search strategies
+        search_strategy = get_search_strategy_for_intent(intent)
+        @feedback.debug_info('search_strategy', "using #{search_strategy[:fact_types].join(', ')} fact types")
+
+        return [] unless defined?(Document)
+
+        begin
+          intent_docs = Document.where("metadata->>'fact_type' IN (?)", search_strategy[:fact_types])
+                               .where("content ILIKE ?", "%#{sanitize_query(query)}%")
+                               .limit(10)
+
+          @feedback.debug_info('intent_results', "found #{intent_docs.count} intent-specific documents")
+
+          intent_docs.map do |doc|
+            base_relevance = calculate_content_relevance(doc, query)
+            boosted_relevance = base_relevance * search_strategy[:boost]
+
+            {
+              document: doc,
+              content: doc.content,
+              similarity: boosted_relevance,
+              metadata: doc.metadata || {},
+              keywords: extract_keywords(doc),
+              intent: intent,
+              fact_type: doc.metadata&.dig('fact_type')
+            }
+          end
+
+        rescue => e
+          @feedback.operation_error('intent_search', e.message)
+          []
+        end
+      end
+
+      # Build final context with token tracking
+      def build_contextual_response(candidates, query, token_budget)
+        return "" if candidates.empty?
+
+        @feedback.debug_info('context_building', "processing #{candidates.length} candidates")
+
+        # Sort by overall quality score
+        sorted_candidates = candidates.sort_by do |candidate|
+          -(candidate[:similarity] * 0.6 + candidate[:relevance] * 0.4)
+        end
+
+        # Build context within token budget
+        context_parts = []
+        used_tokens = 0
+
+        sorted_candidates.each_with_index do |candidate, index|
+          content = candidate[:content]
+          content_tokens = estimate_tokens(content)
+
+          if used_tokens + content_tokens <= token_budget
+            context_parts << content
+            used_tokens += content_tokens
+            @feedback.debug_info('context_add', "added part #{index + 1} (#{content_tokens} tokens)")
+          else
+            @feedback.debug_info('context_skip', "skipped part #{index + 1} (would exceed budget)")
+            break
+          end
+        end
+
+        if context_parts.any?
+          final_context = context_parts.join(" | ")
+          @feedback.vector_context_built(context_parts.length, used_tokens)
+          final_context
+        else
+          @feedback.operation_warning('context_building', 'no content fit within token budget')
+          ""
+        end
+      end
+
+      private
+
+      def analyze_query_intent(query)
+        return :implementation if query.match?(/how to|implement|create|build|fix|debug|error/i)
+        return :definition if query.match?(/what is|define|explain|meaning|concept/i)
+        return :frontend if query.match?(/css|style|html|frontend|design/i)
+        return :commands if query.match?(/command|run|execute|bash|terminal/i)
+        return :reference if query.match?(/example|show me|sample|demo/i)
+        :general
+      end
+
+      def get_search_strategy_for_intent(intent)
+        strategies = {
+          implementation: { fact_types: ['code_example', 'tutorial', 'implementation'], boost: 1.2 },
+          debugging: { fact_types: ['error_solution', 'debugging', 'troubleshooting'], boost: 1.1 },
+          definition: { fact_types: ['explanation', 'tutorial', 'definition'], boost: 1.0 },
+          reference: { fact_types: ['api_reference', 'documentation', 'specification'], boost: 1.3 },
+          frontend: { fact_types: ['css_example', 'html_example', 'frontend'], boost: 1.1 },
+          commands: { fact_types: ['command', 'terminal', 'script'], boost: 1.2 }
+        }
+
+        strategies[intent] || { fact_types: ['general'], boost: 1.0 }
+      end
+
+      def rank_by_relevance(candidates, query, intent)
+        candidates.map do |candidate|
+          # Calculate combined relevance score
+          similarity_score = candidate[:similarity] || 0.5
+          relevance_score = candidate[:relevance] || 0.5
+          intent_bonus = candidate[:intent] == intent ? 0.1 : 0.0
+
+          candidate[:combined_score] = similarity_score * 0.5 + relevance_score * 0.4 + intent_bonus
+          candidate
+        end.sort_by { |c| -c[:combined_score] }
+      end
+
+      def filter_by_quality(candidates)
+        quality_threshold = 0.3
+        candidates.select { |candidate| (candidate[:combined_score] || 0) >= quality_threshold }
+      end
+
+      def tag_results(results, source_type)
+        results.map do |result|
+          result[:source_type] = source_type
+          result
+        end
+      end
+
+      def deduplicate_candidates(candidates)
+        seen_docs = Set.new
+        candidates.reject do |candidate|
+          doc_id = candidate.dig(:document, :id) || candidate[:content]&.hash
+          if seen_docs.include?(doc_id)
+            true
+          else
+            seen_docs.add(doc_id)
+            false
+          end
+        end
+      end
+
+      def calculate_content_similarity(query, content)
+        # Simple word overlap similarity
+        query_words = query.downcase.split(/\W+/).reject(&:blank?)
+        content_words = content.downcase.split(/\W+/).reject(&:blank?)
+
+        return 0.0 if query_words.empty? || content_words.empty?
+
+        intersection = (query_words & content_words).size
+        union = (query_words | content_words).size
+
+        intersection.to_f / union
+      end
+
+      def calculate_content_relevance(document, query)
+        # Enhanced relevance calculation
+        query_terms = query.downcase.split(/\W+/).reject(&:blank?)
+        content_lower = document.content.downcase
+
+        # Count matches and assess density
+        matches = query_terms.count { |term| content_lower.include?(term) }
+        match_density = matches.to_f / [query_terms.length, 1].max
+
+        # Boost for document metadata
+        metadata_boost = document.metadata.present? ? 0.1 : 0.0
+
+        [match_density + metadata_boost, 1.0].min
+      end
+
+      def assess_content_relevance(document, query)
+        calculate_content_relevance(document, query)
+      end
+
+      def extract_keywords(document)
+        if document.respond_to?(:keywords) && document.keywords.respond_to?(:pluck)
+          document.keywords.pluck(:name)
+        else
+          []
+        end
+      end
+
+      def sanitize_query(query)
+        query.to_s.gsub(/['";\\]/, '').strip
+      end
+
+      def estimate_tokens(text)
+        (text.to_s.length * 0.25).ceil
+      end
+
+      def estimate_total_tokens(facts)
+        case facts
+        when Array
+          facts.sum { |fact| estimate_tokens(fact) }
+        when String
+          estimate_tokens(facts)
+        else
+          estimate_tokens(facts.to_s)
+        end
+      end
+
+      def calculate_tokens_saved(context_tokens, query)
+        # Estimate tokens saved by providing specific context vs generic response
+        baseline_tokens = query.length * 0.8 # Rough estimate of generic response
+        savings = [baseline_tokens - context_tokens, 0].max.round
+        savings
+      end
+
+      def get_session_total_saved(current_saved)
+        # Track session total (would be persistent in real implementation)
+        @session_tokens_saved ||= 0
+        @session_tokens_saved += current_saved
+        @session_tokens_saved
+      end
+
+      def get_pattern_fallback(query, intent)
+        # Basic pattern-based fallback
+        case intent
+        when :implementation
+          "Context: Implementation guidance for #{query} | Use best practices and error handling"
+        when :debugging
+          "Context: Debug #{query} | Check logs, verify configuration, test incrementally"
+        when :definition
+          "Context: #{query} definition | Core concepts and usage patterns"
+        when :reference
+          "Context: #{query} reference | API documentation and examples"
+        else
+          "Context: General guidance for #{query}"
+        end
+      end
+    end
+  end
+end