hokipoki 0.1.0 → 0.1.2

data/lib/hokipoki/intelligence/smart_retrieval_engine.rb

@@ -1,401 +0,0 @@
-# frozen_string_literal: true
-
-module Hokipoki
-  module Intelligence
-    # Smart Retrieval Engine with Real-time User Feedback
-    # Enhanced version of the original with comprehensive operation visibility
-    class SmartRetrievalEngine
-      include Singleton
-
-      def initialize
-        @logger = Rails.logger
-        @feedback = Feedback::DisplayManager.instance
-      end
-
-      # PATTERN-003: Intent-based analysis with surgical precision + real vector similarity
-      def retrieve_targeted_facts(query, token_budget: 1500, intent: 'auto')
-        start_time = Time.current
-
-        @feedback.performing_vector_retrieval(query, token_budget) do
-          @logger.info "🎯 SMART RETRIEVAL: Processing query with vector similarity"
-        end
-
-        begin
-          # Step 1: Analyze intent if auto
-          if intent == 'auto'
-            @feedback.debug_info('intent_analysis', 'detecting query intent')
-            intent = analyze_query_intent(query)
-            @feedback.debug_info('intent_detected', intent.to_s)
-          end
-
-          # Step 2: Multi-stage retrieval with vector similarity
-          @feedback.pulling_from_hive_mind('vectors')
-          facts = perform_multi_stage_retrieval(query, token_budget, intent)
-
-          # Step 3: Calculate processing metrics
-          processing_time = ((Time.current - start_time) * 1000).round(2)
-
-          if facts.present?
-            fact_count = facts.is_a?(Array) ? facts.length : 1
-            total_tokens = estimate_total_tokens(facts)
-            tokens_saved = calculate_tokens_saved(total_tokens, query)
-
-            @feedback.vector_results_found(fact_count, processing_time)
-            @feedback.vector_context_built(fact_count, total_tokens)
-            @feedback.total_tokens_saved(tokens_saved, get_session_total_saved(tokens_saved))
-            @feedback.token_usage(total_tokens, token_budget)
-
-            return facts
-          else
-            @feedback.vector_results_found(0, processing_time)
-            @feedback.operation_warning('vector_retrieval', 'falling back to pattern guidance')
-          end
-
-        rescue => e
-          @feedback.operation_error('vector_retrieval', e.message)
-          @logger.warn "Vector retrieval failed: #{e.message}, falling back to patterns"
-        end
-
-        # Fallback to pattern-based retrieval
-        @feedback.debug_info('fallback', 'using pattern-based retrieval')
-        fallback_result = get_pattern_fallback(query, intent)
-
-        if fallback_result
-          @feedback.vector_results_found(1, ((Time.current - start_time) * 1000).round(2))
-          return fallback_result
-        else
-          @feedback.vector_results_found(0, ((Time.current - start_time) * 1000).round(2))
-          return "Context: No specific guidance available for query: #{query}"
-        end
-      end
-
-      # Multi-stage retrieval with comprehensive feedback
-      def perform_multi_stage_retrieval(query, token_budget, intent)
-        @feedback.debug_info('retrieval_stages', 'starting multi-stage retrieval')
-
-        # Stage 1: Vector similarity search
-        @feedback.debug_info('stage_1', 'vector similarity search')
-        vector_candidates = perform_vector_similarity_search(query, intent)
-
-        # Stage 2: Rank by relevance
-        @feedback.debug_info('stage_2', 'ranking by relevance')
-        ranked_candidates = rank_by_relevance(vector_candidates, query, intent)
-
-        # Stage 3: Filter by quality
-        @feedback.debug_info('stage_3', 'filtering by quality')
-        filtered_candidates = filter_by_quality(ranked_candidates)
-
-        # Stage 4: Build context within token budget
-        @feedback.debug_info('stage_4', 'building final context')
-        final_context = build_contextual_response(filtered_candidates, query, token_budget)
-
-        final_context
-      end
-
-      # Vector similarity search with detailed feedback
-      def perform_vector_similarity_search(query, intent)
-        @feedback.debug_info('vector_search', "searching for intent: #{intent}")
-
-        candidates = []
-
-        # Search 1: Exact semantic match
-        @feedback.debug_info('search_exact', 'performing exact semantic match')
-        exact_matches = search_documents_by_content(query, similarity_threshold: 0.9, limit: 10)
-        candidates.concat(tag_results(exact_matches, :exact_match))
-        @feedback.debug_info('exact_results', "found #{exact_matches.length} exact matches")
-
-        # Search 2: Conceptual similarity
-        @feedback.debug_info('search_conceptual', 'performing conceptual similarity search')
-        conceptual_matches = search_documents_by_content(query, similarity_threshold: 0.7, limit: 15)
-        candidates.concat(tag_results(conceptual_matches, :conceptual_match))
-        @feedback.debug_info('conceptual_results', "found #{conceptual_matches.length} conceptual matches")
-
-        # Search 3: Intent-based search
-        @feedback.debug_info('search_intent', "performing intent-based search for #{intent}")
-        intent_matches = search_by_intent(query, intent)
-        candidates.concat(tag_results(intent_matches, :intent_match))
-        @feedback.debug_info('intent_results', "found #{intent_matches.length} intent matches")
-
-        # Remove duplicates
-        unique_candidates = deduplicate_candidates(candidates)
-        @feedback.debug_info('deduplication', "#{candidates.length} → #{unique_candidates.length} after deduplication")
-
-        unique_candidates
-      end
-
-      # Search documents with feedback
-      def search_documents_by_content(query, similarity_threshold: 0.7, limit: 10)
-        @feedback.debug_info('db_query', "searching documents (threshold: #{similarity_threshold})")
-
-        # Check if Document model is available
-        unless defined?(Document)
-          @feedback.operation_warning('db_search', 'Document model not available')
-          return []
-        end
-
-        begin
-          # Search documents with content matching
-          results = Document.where("content ILIKE ?", "%#{sanitize_query(query)}%")
-                           .limit(limit * 2) # Get extra for filtering
-                           .includes(:keywords)
-
-          @feedback.debug_info('db_results', "found #{results.count} database matches")
-
-          # Calculate similarity scores
-          scored_results = results.map do |doc|
-            similarity_score = calculate_content_similarity(query, doc.content)
-
-            {
-              document: doc,
-              content: doc.content,
-              similarity: similarity_score,
-              metadata: doc.metadata || {},
-              keywords: extract_keywords(doc),
-              relevance: assess_content_relevance(doc, query)
-            }
-          end
-
-          # Filter by threshold and sort
-          filtered_results = scored_results.select { |r| r[:similarity] >= similarity_threshold }
-          final_results = filtered_results.sort_by { |r| -(r[:similarity] * 0.7 + r[:relevance] * 0.3) }
-                                        .first(limit)
-
-          @feedback.debug_info('filtering', "#{scored_results.length} → #{final_results.length} after filtering")
-
-          final_results
-
-        rescue => e
-          @feedback.operation_error('db_search', e.message)
-          []
-        end
-      end
-
-      # Intent-based search with feedback
-      def search_by_intent(query, intent)
-        @feedback.debug_info('intent_search', "searching for #{intent} intent")
-
-        # Map intent to search strategies
-        search_strategy = get_search_strategy_for_intent(intent)
-        @feedback.debug_info('search_strategy', "using #{search_strategy[:fact_types].join(', ')} fact types")
-
-        return [] unless defined?(Document)
-
-        begin
-          intent_docs = Document.where("metadata->>'fact_type' IN (?)", search_strategy[:fact_types])
-                               .where("content ILIKE ?", "%#{sanitize_query(query)}%")
-                               .limit(10)
-
-          @feedback.debug_info('intent_results', "found #{intent_docs.count} intent-specific documents")
-
-          intent_docs.map do |doc|
-            base_relevance = calculate_content_relevance(doc, query)
-            boosted_relevance = base_relevance * search_strategy[:boost]
-
-            {
-              document: doc,
-              content: doc.content,
-              similarity: boosted_relevance,
-              metadata: doc.metadata || {},
-              keywords: extract_keywords(doc),
-              intent: intent,
-              fact_type: doc.metadata&.dig('fact_type')
-            }
-          end
-
-        rescue => e
-          @feedback.operation_error('intent_search', e.message)
-          []
-        end
-      end
-
-      # Build final context with token tracking
-      def build_contextual_response(candidates, query, token_budget)
-        return "" if candidates.empty?
-
-        @feedback.debug_info('context_building', "processing #{candidates.length} candidates")
-
-        # Sort by overall quality score
-        sorted_candidates = candidates.sort_by do |candidate|
-          -(candidate[:similarity] * 0.6 + candidate[:relevance] * 0.4)
-        end
-
-        # Build context within token budget
-        context_parts = []
-        used_tokens = 0
-
-        sorted_candidates.each_with_index do |candidate, index|
-          content = candidate[:content]
-          content_tokens = estimate_tokens(content)
-
-          if used_tokens + content_tokens <= token_budget
-            context_parts << content
-            used_tokens += content_tokens
-            @feedback.debug_info('context_add', "added part #{index + 1} (#{content_tokens} tokens)")
-          else
-            @feedback.debug_info('context_skip', "skipped part #{index + 1} (would exceed budget)")
-            break
-          end
-        end
-
-        if context_parts.any?
-          final_context = context_parts.join(" | ")
-          @feedback.vector_context_built(context_parts.length, used_tokens)
-          final_context
-        else
-          @feedback.operation_warning('context_building', 'no content fit within token budget')
-          ""
-        end
-      end
-
-      private
-
-      def analyze_query_intent(query)
-        return :implementation if query.match?(/how to|implement|create|build|fix|debug|error/i)
-        return :definition if query.match?(/what is|define|explain|meaning|concept/i)
-        return :frontend if query.match?(/css|style|html|frontend|design/i)
-        return :commands if query.match?(/command|run|execute|bash|terminal/i)
-        return :reference if query.match?(/example|show me|sample|demo/i)
-        :general
-      end
-
-      def get_search_strategy_for_intent(intent)
-        strategies = {
-          implementation: { fact_types: ['code_example', 'tutorial', 'implementation'], boost: 1.2 },
-          debugging: { fact_types: ['error_solution', 'debugging', 'troubleshooting'], boost: 1.1 },
-          definition: { fact_types: ['explanation', 'tutorial', 'definition'], boost: 1.0 },
-          reference: { fact_types: ['api_reference', 'documentation', 'specification'], boost: 1.3 },
-          frontend: { fact_types: ['css_example', 'html_example', 'frontend'], boost: 1.1 },
-          commands: { fact_types: ['command', 'terminal', 'script'], boost: 1.2 }
-        }
-
-        strategies[intent] || { fact_types: ['general'], boost: 1.0 }
-      end
-
-      def rank_by_relevance(candidates, query, intent)
-        candidates.map do |candidate|
-          # Calculate combined relevance score
-          similarity_score = candidate[:similarity] || 0.5
-          relevance_score = candidate[:relevance] || 0.5
-          intent_bonus = candidate[:intent] == intent ? 0.1 : 0.0
-
-          candidate[:combined_score] = similarity_score * 0.5 + relevance_score * 0.4 + intent_bonus
-          candidate
-        end.sort_by { |c| -c[:combined_score] }
-      end
-
-      def filter_by_quality(candidates)
-        quality_threshold = 0.3
-        candidates.select { |candidate| (candidate[:combined_score] || 0) >= quality_threshold }
-      end
-
-      def tag_results(results, source_type)
-        results.map do |result|
-          result[:source_type] = source_type
-          result
-        end
-      end
-
-      def deduplicate_candidates(candidates)
-        seen_docs = Set.new
-        candidates.reject do |candidate|
-          doc_id = candidate.dig(:document, :id) || candidate[:content]&.hash
-          if seen_docs.include?(doc_id)
-            true
-          else
-            seen_docs.add(doc_id)
-            false
-          end
-        end
-      end
-
-      def calculate_content_similarity(query, content)
-        # Simple word overlap similarity
-        query_words = query.downcase.split(/\W+/).reject(&:blank?)
-        content_words = content.downcase.split(/\W+/).reject(&:blank?)
-
-        return 0.0 if query_words.empty? || content_words.empty?
-
-        intersection = (query_words & content_words).size
-        union = (query_words | content_words).size
-
-        intersection.to_f / union
-      end
-
-      def calculate_content_relevance(document, query)
-        # Enhanced relevance calculation
-        query_terms = query.downcase.split(/\W+/).reject(&:blank?)
-        content_lower = document.content.downcase
-
-        # Count matches and assess density
-        matches = query_terms.count { |term| content_lower.include?(term) }
-        match_density = matches.to_f / [query_terms.length, 1].max
-
-        # Boost for document metadata
-        metadata_boost = document.metadata.present? ? 0.1 : 0.0
-
-        [match_density + metadata_boost, 1.0].min
-      end
-
-      def assess_content_relevance(document, query)
-        calculate_content_relevance(document, query)
-      end
-
-      def extract_keywords(document)
-        if document.respond_to?(:keywords) && document.keywords.respond_to?(:pluck)
-          document.keywords.pluck(:name)
-        else
-          []
-        end
-      end
-
-      def sanitize_query(query)
-        query.to_s.gsub(/['";\\]/, '').strip
-      end
-
-      def estimate_tokens(text)
-        (text.to_s.length * 0.25).ceil
-      end
-
-      def estimate_total_tokens(facts)
-        case facts
-        when Array
-          facts.sum { |fact| estimate_tokens(fact) }
-        when String
-          estimate_tokens(facts)
-        else
-          estimate_tokens(facts.to_s)
-        end
-      end
-
-      def calculate_tokens_saved(context_tokens, query)
-        # Estimate tokens saved by providing specific context vs generic response
-        baseline_tokens = query.length * 0.8 # Rough estimate of generic response
-        savings = [baseline_tokens - context_tokens, 0].max.round
-        savings
-      end
-
-      def get_session_total_saved(current_saved)
-        # Track session total (would be persistent in real implementation)
-        @session_tokens_saved ||= 0
-        @session_tokens_saved += current_saved
-        @session_tokens_saved
-      end
-
-      def get_pattern_fallback(query, intent)
-        # Basic pattern-based fallback
-        case intent
-        when :implementation
-          "Context: Implementation guidance for #{query} | Use best practices and error handling"
-        when :debugging
-          "Context: Debug #{query} | Check logs, verify configuration, test incrementally"
-        when :definition
-          "Context: #{query} definition | Core concepts and usage patterns"
-        when :reference
-          "Context: #{query} reference | API documentation and examples"
-        else
-          "Context: General guidance for #{query}"
-        end
-      end
-    end
-  end
-end