ragdoll-rails 0.1.8 → 0.1.11

data/app/controllers/ragdoll/search_controller.rb

@@ -0,0 +1,368 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class SearchController < ApplicationController
+    skip_before_action :verify_authenticity_token, only: [:search]
+    
+    def index
+      # Load popular queries for sidebar
+      @popular_queries = ::Ragdoll::Search.group(:query).count.sort_by { |query, count| -count }.first(10).to_h
+      
+      # Check if we're reconstructing a previous search
+      if params[:search_id].present?
+        begin
+          previous_search = ::Ragdoll::Search.find(params[:search_id])
+          @reconstructed_search = previous_search
+          
+          # Extract stored form parameters
+          search_options = previous_search.search_options.is_a?(Hash) ? previous_search.search_options : 
+                          (previous_search.search_options.present? ? JSON.parse(previous_search.search_options) : {})
+          search_filters = previous_search.search_filters.is_a?(Hash) ? previous_search.search_filters : 
+                          (previous_search.search_filters.present? ? JSON.parse(previous_search.search_filters) : {})
+          
+          form_params = search_options.dig('form_params') || {}
+          
+          # Reconstruct query and filters from stored search
+          @query = previous_search.query
+          @filters = {
+            document_type: form_params['document_type'] || search_filters['document_type'],
+            status: form_params['status'] || search_filters['status'],
+            limit: form_params['limit'] || search_filters['limit'] || 10,
+            threshold: form_params['threshold'] || search_filters['threshold'] || 0.001
+          }
+          
+          # Reconstruct boolean search options
+          @use_similarity_search = form_params['use_similarity_search'] || search_options['use_similarity'] || 'true'
+          @use_fulltext_search = form_params['use_fulltext_search'] || search_options['use_fulltext'] || 'true'
+          
+          ::Rails.logger.debug "🔍 Reconstructed search from ID #{params[:search_id]}: #{@query}"
+          
+        rescue ActiveRecord::RecordNotFound
+          ::Rails.logger.warn "🔍 Search ID #{params[:search_id]} not found"
+          # Fall back to default behavior
+        rescue => e
+          ::Rails.logger.error "🔍 Error reconstructing search: #{e.message}"
+          # Fall back to default behavior
+        end
+      end
+      
+      # Default values if not reconstructing a search
+      unless @reconstructed_search
+        @filters = {
+          document_type: params[:document_type],
+          status: params[:status],
+          limit: params[:limit]&.to_i || 10,
+          threshold: params[:threshold]&.to_f || (::Rails.env.development? ? 0.001 : 0.7)
+        }
+        @query = params[:query]
+        @use_similarity_search = params[:use_similarity_search] || 'true'
+        @use_fulltext_search = params[:use_fulltext_search] || 'true'
+      end
+      
+      @search_performed = false
+    end
+    
+    def search
+      ::Rails.logger.debug "🔍 Search called with params: #{params.inspect}"
+      ::Rails.logger.debug "🔍 Use similarity search: #{params[:use_similarity_search]}"
+      ::Rails.logger.debug "🔍 Use fulltext search: #{params[:use_fulltext_search]}"
+      @query = params[:query]
+      @filters = {
+        document_type: params[:document_type],
+        status: params[:status],
+        limit: params[:limit]&.to_i || 10,
+        threshold: params[:threshold]&.to_f || (::Rails.env.development? ? 0.001 : 0.7)  # Much lower threshold for development
+      }
+      ::Rails.logger.debug "🔍 Query: #{@query.inspect}, Filters: #{@filters.inspect}"
+      
+      # Initialize data needed for the view sidebar - load popular queries
+      @popular_queries = ::Ragdoll::Search.group(:query).count.sort_by { |query, count| -count }.first(10).to_h
+      
+      if @query.present?
+        begin
+          # Check which search types are enabled (default to both if neither param is set)
+          use_similarity = params[:use_similarity_search] != 'false'
+          use_fulltext = params[:use_fulltext_search] != 'false'
+          
+          @detailed_results = []
+          @below_threshold_results = []
+          @similarity_search_attempted = false
+          @similarity_threshold_used = @filters[:threshold]
+          
+          # Perform similarity search if enabled
+          if use_similarity
+            begin
+              search_params = {
+                query: @query,
+                limit: @filters[:limit],
+                threshold: @filters[:threshold]
+              }
+              
+              # Add document type filter if specified
+              if @filters[:document_type].present?
+                search_params[:document_type] = @filters[:document_type]
+              end
+              
+              # Add status filter if specified
+              if @filters[:status].present?
+                search_params[:status] = @filters[:status]
+              end
+              
+              search_response = ::Ragdoll.search(**search_params.merge(track_search: false))
+              
+              # The search returns a hash with :results and :statistics
+              @results = search_response.is_a?(Hash) ? search_response[:results] || [] : []
+              @similarity_stats = search_response.is_a?(Hash) ? search_response[:statistics] || {} : {}
+              
+              # Add similarity search results
+              @results.each do |result|
+                if result[:embedding_id] && result[:document_id]
+                  embedding = ::Ragdoll::Embedding.find(result[:embedding_id])
+                  document = ::Ragdoll::Document.find(result[:document_id])
+                  @detailed_results << {
+                    embedding: embedding,
+                    document: document,
+                    similarity: result[:similarity],
+                    content: result[:content],
+                    usage_count: embedding.usage_count,
+                    last_used: embedding.returned_at,
+                    search_type: 'similarity'
+                  }
+                end
+              end
+              
+              # Mark that similarity search was attempted
+              @similarity_search_attempted = true
+              
+              # Always gather statistics about all possible matches when similarity search returns limited results
+              similarity_results_count = @detailed_results.select { |r| r[:search_type] == 'similarity' }.count
+              ::Rails.logger.debug "🔍 Similarity results found: #{similarity_results_count}"
+              
+              # Gather statistics if we have no results OR if the threshold is relatively high (> 0.1)
+              # This ensures we provide helpful feedback even when the search succeeds with a lower threshold
+              should_gather_stats = similarity_results_count == 0 || @filters[:threshold] > 0.1
+              ::Rails.logger.debug "🔍 Should gather stats: #{should_gather_stats} (results: #{similarity_results_count}, threshold: #{@filters[:threshold]})"
+              
+              if should_gather_stats
+                ::Rails.logger.debug "🔍 Gathering below-threshold statistics..."
+                begin
+                  # Search again with minimal threshold to get all potential matches
+                  stats_params = search_params.merge(threshold: 0.0, limit: 100)
+                  stats_response = ::Ragdoll.search(**stats_params)
+                  
+                  ::Rails.logger.debug "🔍 Stats response: #{stats_response.inspect}"
+                  
+                  if stats_response.is_a?(Hash) && stats_response[:results]
+                    all_similarities = []
+                    stats_response[:results].each do |result|
+                      if result[:similarity]
+                        all_similarities << result[:similarity]
+                        # Store below-threshold results
+                        if result[:similarity] < @filters[:threshold] && result[:similarity] > 0
+                          @below_threshold_results << {
+                            document_id: result[:document_id],
+                            similarity: result[:similarity],
+                            content: result[:content]
+                          }
+                        end
+                      end
+                    end
+                    
+                    ::Rails.logger.debug "🔍 All similarities collected: #{all_similarities.inspect}"
+                    ::Rails.logger.debug "🔍 Threshold: #{@filters[:threshold]}"
+                    
+                    # Calculate statistics for display
+                    if all_similarities.any?
+                      below_threshold_count = all_similarities.count { |s| s < @filters[:threshold] && s > 0 }
+                      @below_threshold_stats = {
+                        count: below_threshold_count,
+                        highest: all_similarities.max,
+                        lowest: all_similarities.select { |s| s > 0 }.min,
+                        average: all_similarities.sum / all_similarities.size.to_f,
+                        suggested_threshold: all_similarities.select { |s| s > 0 }.min.round(3)
+                      }
+                      ::Rails.logger.debug "🔍 Below threshold stats: #{@below_threshold_stats.inspect}"
+                    else
+                      ::Rails.logger.debug "🔍 No similarities found in stats response"
+                    end
+                  else
+                    ::Rails.logger.debug "🔍 Stats response was not in expected format or had no results"
+                  end
+                rescue => stats_error
+                  ::Rails.logger.error "Stats gathering error: #{stats_error.message}"
+                end
+              end
+              
+            rescue => e
+              ::Rails.logger.error "Similarity search error: #{e.message}"
+              # Continue with fulltext search even if similarity search fails
+            end
+          end
+          
+          # Perform full-text search if enabled
+          if use_fulltext
+            fulltext_params = {
+              limit: @filters[:limit], 
+              threshold: @filters[:threshold]
+            }
+            
+            # Add document type filter if specified
+            if @filters[:document_type].present?
+              fulltext_params[:document_type] = @filters[:document_type]
+            end
+            
+            # Add status filter if specified
+            if @filters[:status].present?
+              fulltext_params[:status] = @filters[:status]
+            end
+            
+            fulltext_results = ::Ragdoll::Document.search_content(@query, **fulltext_params)
+            
+            # Collect fulltext similarities for statistics
+            fulltext_similarities = []
+            fulltext_results.each do |document|
+              # Avoid duplicates if document was already found in similarity search
+              unless @detailed_results.any? { |r| r[:document].id == document.id }
+                # Use the fulltext_similarity score from the enhanced search
+                fulltext_similarity = document.respond_to?(:fulltext_similarity) ? document.fulltext_similarity.to_f : 0.0
+                fulltext_similarities << fulltext_similarity if fulltext_similarity > 0
+                
+                @detailed_results << {
+                  document: document,
+                  content: document.metadata&.dig('summary') || document.title || "No summary available",
+                  search_type: 'fulltext',
+                  similarity: fulltext_similarity
+                }
+              end
+            end
+            
+            # Gather fulltext statistics if we have few results OR if threshold is high (> 0.1)
+            # This ensures consistent feedback regardless of which search types are enabled
+            fulltext_results_count = @detailed_results.select { |r| r[:search_type] == 'fulltext' }.count
+            should_gather_fulltext_stats = fulltext_results_count == 0 || @filters[:threshold] > 0.1
+            
+            if should_gather_fulltext_stats && !@below_threshold_stats
+              ::Rails.logger.debug "🔍 Gathering fulltext below-threshold statistics..."
+              begin
+                # Search again with lower threshold to get all potential matches
+                stats_params = fulltext_params.merge(threshold: 0.0, limit: 100)
+                all_fulltext_results = ::Ragdoll::Document.search_content(@query, **stats_params)
+                
+                all_fulltext_similarities = []
+                all_fulltext_results.each do |document|
+                  similarity = document.respond_to?(:fulltext_similarity) ? document.fulltext_similarity.to_f : 0.0
+                  if similarity > 0
+                    all_fulltext_similarities << similarity
+                    # Store below-threshold results
+                    if similarity < @filters[:threshold]
+                      @below_threshold_results << {
+                        document_id: document.id,
+                        similarity: similarity,
+                        content: document.metadata&.dig('summary') || document.title || "No summary available"
+                      }
+                    end
+                  end
+                end
+                
+                ::Rails.logger.debug "🔍 Fulltext similarities collected: #{all_fulltext_similarities.inspect}"
+                ::Rails.logger.debug "🔍 Threshold: #{@filters[:threshold]}"
+                
+                # Calculate statistics for display
+                if all_fulltext_similarities.any?
+                  below_threshold_count = all_fulltext_similarities.count { |s| s < @filters[:threshold] && s > 0 }
+                  @below_threshold_stats = {
+                    count: below_threshold_count,
+                    highest: all_fulltext_similarities.max,
+                    lowest: all_fulltext_similarities.select { |s| s > 0 }.min,
+                    average: all_fulltext_similarities.sum / all_fulltext_similarities.size.to_f,
+                    suggested_threshold: all_fulltext_similarities.select { |s| s > 0 }.min.round(3)
+                  }
+                  ::Rails.logger.debug "🔍 Fulltext below threshold stats: #{@below_threshold_stats.inspect}"
+                else
+                  ::Rails.logger.debug "🔍 No fulltext similarities found in stats response"
+                end
+              rescue => stats_error
+                ::Rails.logger.error "Fulltext stats gathering error: #{stats_error.message}"
+              end
+            end
+          end
+          
+          # Sort results by similarity score if available, otherwise by relevance
+          @detailed_results.sort_by! { |r| r[:similarity] ? -r[:similarity] : 0 }
+          
+          # Save search for analytics
+          search_type = case
+                       when use_similarity && use_fulltext then 'hybrid'
+                       when use_similarity then 'similarity'
+                       when use_fulltext then 'fulltext'
+                       else 'unknown'
+                       end
+          
+          similarity_results = @detailed_results.select { |r| r[:search_type] == 'similarity' }
+          similarities = similarity_results.map { |r| r[:similarity] }.compact
+          
+          # Save search for analytics without query embedding (which is optional)
+          begin
+            ::Ragdoll::Search.create!(
+              query: @query,
+              search_type: search_type,
+              results_count: @detailed_results.count,
+              max_similarity_score: similarities.any? ? similarities.max : nil,
+              min_similarity_score: similarities.any? ? similarities.min : nil,
+              avg_similarity_score: similarities.any? ? (similarities.sum / similarities.size.to_f) : nil,
+              search_filters: @filters.to_json,
+              search_options: {
+                threshold_used: @filters[:threshold],
+                similarity_results: similarity_results.count,
+                fulltext_results: @detailed_results.select { |r| r[:search_type] == 'fulltext' }.count,
+                use_similarity: use_similarity,
+                use_fulltext: use_fulltext,
+                # Store original form parameters for reconstruction
+                form_params: {
+                  use_similarity_search: params[:use_similarity_search],
+                  use_fulltext_search: params[:use_fulltext_search],
+                  limit: @filters[:limit],
+                  threshold: @filters[:threshold],
+                  document_type: @filters[:document_type],
+                  status: @filters[:status]
+                }
+              }.to_json
+            )
+            ::Rails.logger.debug "🔍 Search saved successfully"
+          rescue => e
+            ::Rails.logger.error "🔍 Failed to save search: #{e.message}"
+            # Continue without failing the search
+          end
+          
+          ::Rails.logger.debug "🔍 Search completed successfully. Results count: #{@detailed_results.count}"
+          ::Rails.logger.debug "🔍 Similarity search attempted: #{@similarity_search_attempted}"
+          ::Rails.logger.debug "🔍 Below threshold stats: #{@below_threshold_stats.inspect}"
+          ::Rails.logger.debug "🔍 Threshold used: #{@similarity_threshold_used}"
+          @search_performed = true
+          
+        rescue => e
+          ::Rails.logger.error "🔍 Search error: #{e.message}"
+          ::Rails.logger.error e.backtrace.join("\n")
+          @error = e.message
+          @search_performed = false
+        end
+      else
+        @search_performed = false
+      end
+      
+      respond_to do |format|
+        format.html { render :index }
+        format.json { 
+          json_response = { results: @detailed_results, error: @error }
+          if @similarity_search_attempted && @similarity_stats
+            json_response[:similarity_statistics] = {
+              threshold_used: @similarity_threshold_used,
+              stats: @similarity_stats
+            }
+          end
+          render json: json_response
+        }
+      end
+    end
+  end
+end

data/app/jobs/application_job.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class ApplicationJob < ActiveJob::Base
+  # Automatically retry jobs that encountered a deadlock
+  # retry_on ActiveRecord::Deadlocked
+
+  # Most jobs are safe to ignore if the underlying records are no longer available
+  # discard_on ActiveJob::DeserializationError
+end

data/app/jobs/ragdoll/bulk_document_processing_job.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class BulkDocumentProcessingJob < ApplicationJob
+    queue_as :default
+    
+    private
+    
+    def broadcast_status_update(session_id, data)
+      ActionCable.server.broadcast("bulk_upload_status_#{session_id}", data)
+    rescue => e
+      logger.error "Failed to broadcast status update: #{e.message}"
+    end
+    
+    def safe_log_operation(operation, details = {})
+      return unless defined?(RagdollLogging)
+      RagdollLogging.log_operation(operation, details)
+    rescue => e
+      logger.debug "Failed to log operation #{operation}: #{e.message}"
+    end
+    
+    def safe_log_error(operation, error, details = {})
+      return unless defined?(RagdollLogging)
+      RagdollLogging.log_error(operation, error, details)
+    rescue => e
+      logger.debug "Failed to log error #{operation}: #{e.message}"
+    end
+    
+    def safe_log_performance(operation, duration, details = {})
+      return unless defined?(RagdollLogging)
+      RagdollLogging.log_performance(operation, duration, details)
+    rescue => e
+      logger.debug "Failed to log performance #{operation}: #{e.message}"
+    end
+    
+    def perform(session_id, file_paths_data, force_duplicate = false)
+      start_time = Time.current
+      
+      # Initialize variables early to avoid nil errors in rescue block
+      total_files = file_paths_data&.size || 0
+      processed_count = 0
+      failed_files = []
+      
+      safe_log_operation("bulk_processing_start", {
+        session_id: session_id,
+        file_count: total_files,
+        force_duplicate: force_duplicate,
+        job_id: job_id
+      })
+      
+      logger.info "🚀 Starting bulk document processing job for session #{session_id}"
+      logger.info "📁 Processing #{total_files} files"
+      
+      # Early return if no files to process
+      if file_paths_data.nil? || file_paths_data.empty?
+        logger.warn "⚠️ No files provided for processing in session #{session_id}"
+        broadcast_status_update(session_id, {
+          type: 'upload_error',
+          error: 'No files provided for processing',
+          status: 'failed'
+        })
+        return
+      end
+      
+      # Broadcast upload start
+      broadcast_status_update(session_id, {
+        type: 'upload_start',
+        total_files: total_files,
+        status: 'processing',
+        started_at: Time.current.iso8601
+      })
+      
+      batch_size = 10  # Process 10 files at a time for async jobs
+      
+      file_paths_data.each_slice(batch_size).with_index do |file_batch, batch_index|
+        logger.info "📦 Processing batch #{batch_index + 1} of #{(total_files.to_f / batch_size).ceil}"
+        
+        file_batch.each do |file_data|
+          file_start_time = Time.current
+          
+          begin
+            temp_path = file_data[:temp_path]
+            original_filename = file_data[:original_filename]
+            
+            safe_log_operation("file_processing_start", {
+              session_id: session_id,
+              filename: original_filename,
+              temp_path: temp_path,
+              file_exists: File.exist?(temp_path),
+              file_size: File.exist?(temp_path) ? File.size(temp_path) : 0
+            })
+            
+            unless File.exist?(temp_path)
+              error_msg = "Temporary file not found: #{temp_path}"
+              safe_log_error("file_processing", StandardError.new(error_msg), {
+                session_id: session_id,
+                filename: original_filename,
+                temp_path: temp_path
+              })
+              next
+            end
+            
+            logger.info "🔄 Processing file: #{original_filename}"
+            
+            # Broadcast file start
+            progress_percentage = ((processed_count.to_f / total_files) * 100).round(1)
+            broadcast_status_update(session_id, {
+              type: 'file_start',
+              filename: original_filename,
+              processed: processed_count,
+              total: total_files,
+              percentage: progress_percentage,
+              status: 'processing',
+              batch_index: batch_index + 1,
+              total_batches: (total_files.to_f / batch_size).ceil
+            })
+            
+            # Process the document
+            ragdoll_start_time = Time.current
+            result = ::Ragdoll.add_document(path: temp_path, force: force_duplicate)
+            ragdoll_duration = Time.current - ragdoll_start_time
+            
+            safe_log_performance("ragdoll_add_document", ragdoll_duration, {
+              session_id: session_id,
+              filename: original_filename,
+              result_success: result && result[:success],
+              force_duplicate: force_duplicate
+            })
+            
+            if result && result[:success]
+              processed_count += 1
+              file_duration = Time.current - file_start_time
+              
+              safe_log_operation("file_processing_success", {
+                session_id: session_id,
+                filename: original_filename,
+                document_id: result[:document_id],
+                processing_duration: file_duration.round(3),
+                processed_count: processed_count,
+                total_files: total_files
+              })
+              
+              logger.info "✅ Successfully processed: #{original_filename}"
+              
+              # Broadcast success
+              broadcast_status_update(session_id, {
+                type: 'file_complete',
+                filename: original_filename,
+                processed: processed_count,
+                total: total_files,
+                percentage: ((processed_count.to_f / total_files) * 100).round(1),
+                status: 'completed',
+                document_id: result[:document_id],
+                processing_time: file_duration.round(3)
+              })
+            else
+              failed_files << original_filename
+              error_message = result ? result[:error] : 'Unknown error'
+              file_duration = Time.current - file_start_time
+              
+              safe_log_error("file_processing", StandardError.new(error_message), {
+                session_id: session_id,
+                filename: original_filename,
+                processing_duration: file_duration.round(3),
+                ragdoll_result: result,
+                temp_path: temp_path,
+                file_size: File.size(temp_path)
+              })
+              
+              logger.error "❌ Failed to process: #{original_filename} - #{error_message}"
+              
+              # Broadcast error
+              broadcast_status_update(session_id, {
+                type: 'file_error',
+                filename: original_filename,
+                processed: processed_count,
+                total: total_files,
+                percentage: ((processed_count.to_f / total_files) * 100).round(1),
+                status: 'failed',
+                error: error_message,
+                processing_time: file_duration.round(3)
+              })
+            end
+            
+            # Clean up temp file
+            File.delete(temp_path) if File.exist?(temp_path)
+            
+          rescue => e
+            failed_files << (file_data[:original_filename] || 'unknown file')
+            file_duration = Time.current - file_start_time
+            
+            safe_log_error("file_processing_exception", e, {
+              session_id: session_id,
+              filename: file_data[:original_filename],
+              temp_path: file_data[:temp_path],
+              processing_duration: file_duration.round(3),
+              file_data: file_data,
+              processed_count: processed_count,
+              total_files: total_files
+            })
+            
+            logger.error "💥 Exception processing file #{file_data[:original_filename]}: #{e.message}"
+            logger.error e.backtrace.join("\n")
+            
+            # Broadcast error
+            ActionCable.server.broadcast("ragdoll_file_processing_#{session_id}", {
+              type: 'file_error',
+              filename: file_data[:original_filename],
+              processed: processed_count,
+              total: total_files,
+              percentage: ((processed_count.to_f / total_files) * 100).round(1),
+              status: 'failed',
+              error: e.message
+            })
+          end
+        end
+        
+        # Force garbage collection after each batch
+        GC.start
+        
+        # Small delay between batches to prevent overwhelming the system
+        sleep(0.1)
+      end
+      
+      # Broadcast final completion
+      total_duration = Time.current - start_time
+      final_percentage = 100.0
+      broadcast_status_update(session_id, {
+        type: 'upload_complete',
+        processed: processed_count,
+        total: total_files,
+        failed: failed_files.size,
+        failed_files: failed_files,
+        percentage: final_percentage,
+        status: 'completed',
+        total_duration: total_duration.round(3),
+        completed_at: Time.current.iso8601
+      })
+      
+      safe_log_operation("bulk_processing_complete", {
+        session_id: session_id,
+        total_files: total_files,
+        processed_count: processed_count,
+        failed_count: failed_files.size,
+        failed_files: failed_files,
+        total_duration: total_duration.round(3),
+        avg_file_duration: total_files > 0 ? (total_duration / total_files).round(3) : 0
+      })
+      
+      logger.info "🎉 Bulk processing completed for session #{session_id}"
+      logger.info "📊 Results: #{processed_count}/#{total_files} successful, #{failed_files.size} failed"
+      
+    rescue => e
+      total_duration = Time.current - start_time
+      
+      safe_log_error("bulk_processing_job_failure", e, {
+        session_id: session_id,
+        total_files: total_files,
+        processed_count: processed_count,
+        failed_count: failed_files.size,
+        total_duration: total_duration.round(3),
+        job_id: job_id
+      })
+      
+      logger.error "💀 Bulk processing job failed for session #{session_id}: #{e.message}"
+      logger.error e.backtrace.join("\n")
+      
+      # Broadcast job failure
+      broadcast_status_update(session_id, {
+        type: 'upload_error',
+        error: e.message,
+        status: 'failed',
+        processed: processed_count,
+        total: total_files,
+        failed_at: Time.current.iso8601,
+        total_duration: total_duration.round(3)
+      })
+    end
+  end
+end