ragdoll-rails 0.1.9 → 0.1.11

data/app/controllers/ragdoll/configuration_controller.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class ConfigurationController < ApplicationController
+    def index
+      @configuration = ::Ragdoll.configuration
+      @available_providers = %w[openai anthropic google azure ollama huggingface]
+      @available_models = {
+        openai: ['text-embedding-3-small', 'text-embedding-3-large', 'text-embedding-ada-002'],
+        anthropic: ['claude-3-haiku-20240307', 'claude-3-sonnet-20240229', 'claude-3-opus-20240229'],
+        google: ['gemini-pro', 'gemini-1.5-flash', 'gemini-1.5-pro'],
+        azure: ['text-embedding-3-small', 'text-embedding-3-large'],
+        ollama: ['llama2', 'mistral', 'codellama'],
+        huggingface: ['sentence-transformers/all-MiniLM-L6-v2', 'sentence-transformers/all-mpnet-base-v2']
+      }
+      
+      @current_stats = {
+        total_documents: ::Ragdoll::Document.count,
+        total_embeddings: ::Ragdoll::Embedding.count,
+        embedding_dimensions: ::Ragdoll::Embedding.first&.embedding_dimensions || 0,
+        average_chunk_size: ::Ragdoll::Embedding.average('LENGTH(content)')&.round || 0
+      }
+    end
+    
+    def update
+      config_params = params.require(:configuration).permit(
+        :llm_provider,
+        :embedding_provider,
+        :embedding_model,
+        :chunk_size,
+        :chunk_overlap,
+        :max_search_results,
+        :search_similarity_threshold,
+        :enable_search_analytics,
+        :enable_document_summarization,
+        :enable_usage_tracking,
+        :usage_ranking_enabled,
+        :openai_api_key,
+        :anthropic_api_key,
+        :google_api_key,
+        :azure_api_key,
+        :ollama_url,
+        :huggingface_api_key
+      )
+      
+      begin
+        # Update configuration
+        config = ::Ragdoll.configuration
+        
+        config_params.each do |key, value|
+          # Convert string values to appropriate types
+          case key
+          when 'chunk_size', 'chunk_overlap', 'max_search_results'
+            config.send("#{key}=", value.to_i)
+          when 'search_similarity_threshold'
+            config.send("#{key}=", value.to_f)
+          when 'enable_search_analytics', 'enable_document_summarization', 'enable_usage_tracking', 'usage_ranking_enabled'
+            config.send("#{key}=", value == '1' || value == 'true')
+          else
+            config.send("#{key}=", value) if value.present?
+          end
+        end
+        
+        flash[:notice] = 'Configuration updated successfully.'
+        
+        # Test the configuration
+        begin
+          client = ::Ragdoll::Client.new
+          test_result = client.stats
+          flash[:notice] += " Configuration test successful."
+        rescue => e
+          flash[:warning] = "Configuration saved but test failed: #{e.message}"
+        end
+        
+      rescue => e
+        flash[:alert] = "Error updating configuration: #{e.message}"
+      end
+      
+      redirect_to ragdoll.configuration_path
+    end
+  end
+end

data/app/controllers/ragdoll/dashboard_controller.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class DashboardController < ApplicationController
+    def index
+      @stats = {
+        total_documents: ::Ragdoll::Document.count,
+        processed_documents: ::Ragdoll::Document.where(status: 'processed').count,
+        failed_documents: ::Ragdoll::Document.where(status: 'failed').count,
+        pending_documents: ::Ragdoll::Document.where(status: 'pending').count,
+        total_embeddings: ::Ragdoll::Embedding.count,
+        total_searches: ::Ragdoll::Search.count,
+        recent_searches: ::Ragdoll::Search.order(created_at: :desc).limit(5)
+      }
+      
+      @document_types = ::Ragdoll::Document.group(:document_type).count
+      @recent_documents = ::Ragdoll::Document.order(created_at: :desc).limit(10)
+      
+      # Usage analytics - join through embeddable (Content) to get to documents
+      @top_searched_documents = ::Ragdoll::Embedding
+        .joins("JOIN ragdoll_contents ON ragdoll_contents.id = ragdoll_embeddings.embeddable_id")
+        .joins("JOIN ragdoll_documents ON ragdoll_documents.id = ragdoll_contents.document_id")
+        .group('ragdoll_documents.title')
+        .order(Arel.sql('SUM(ragdoll_embeddings.usage_count) DESC'))
+        .limit(5)
+        .sum(:usage_count)
+    end
+    
+    def analytics
+      today = Date.current
+      week_start = today.beginning_of_week
+      month_start = today.beginning_of_month
+      
+      # Calculate search statistics
+      all_searches = ::Ragdoll::Search.all
+      searches_today = all_searches.where(created_at: today.beginning_of_day..today.end_of_day)
+      searches_this_week = all_searches.where(created_at: week_start.beginning_of_day..today.end_of_day)
+      searches_this_month = all_searches.where(created_at: month_start.beginning_of_day..today.end_of_day)
+      
+      # Comprehensive search analytics combining both pages
+      @search_analytics = {
+        total_searches: all_searches.count,
+        unique_queries: all_searches.distinct.count(:query),
+        searches_today: searches_today.count,
+        searches_this_week: searches_this_week.count,
+        searches_this_month: searches_this_month.count,
+        average_results: all_searches.average(:results_count)&.round(1) || 0,
+        average_similarity: all_searches.where.not(avg_similarity_score: nil).average(:avg_similarity_score)&.round(3) || 0,
+        avg_execution_time: all_searches.average(:execution_time_ms)&.round(1) || 0,
+        search_types: all_searches.group(:search_type).count
+      }
+      
+      # Top queries (most frequent)
+      @top_queries = all_searches
+        .group(:query)
+        .count
+        .sort_by { |query, count| -count }
+        .first(10)
+        .to_h
+      
+      # Search trends by day for the last 7 days
+      @search_trends = (6.days.ago.to_date..today).map do |date|
+        count = all_searches.where(created_at: date.beginning_of_day..date.end_of_day).count
+        [date.strftime('%m/%d'), count]
+      end.to_h
+      
+      # Most searched documents (using embedding usage as proxy)
+      @top_documents = ::Ragdoll::Embedding
+        .joins("JOIN ragdoll_contents ON ragdoll_contents.id = ragdoll_embeddings.embeddable_id")
+        .joins("JOIN ragdoll_documents ON ragdoll_documents.id = ragdoll_contents.document_id")
+        .group('ragdoll_documents.title')
+        .order(Arel.sql('SUM(ragdoll_embeddings.usage_count) DESC'))
+        .limit(10)
+        .sum(:usage_count)
+      
+      # Similarity score distribution
+      similarity_scores = all_searches.where.not(avg_similarity_score: nil).pluck(:avg_similarity_score)
+      @similarity_distribution = {
+        "0.9-1.0" => similarity_scores.count { |s| s >= 0.9 },
+        "0.8-0.9" => similarity_scores.count { |s| s >= 0.8 && s < 0.9 },
+        "0.7-0.8" => similarity_scores.count { |s| s >= 0.7 && s < 0.8 },
+        "0.6-0.7" => similarity_scores.count { |s| s >= 0.6 && s < 0.7 },
+        "0.5-0.6" => similarity_scores.count { |s| s >= 0.5 && s < 0.6 },
+        "< 0.5" => similarity_scores.count { |s| s < 0.5 }
+      }
+      
+      # System statistics
+      @system_stats = {
+        total_documents: ::Ragdoll::Document.count,
+        processed_documents: ::Ragdoll::Document.where(status: 'processed').count,
+        failed_documents: ::Ragdoll::Document.where(status: 'failed').count,
+        pending_documents: ::Ragdoll::Document.where(status: 'pending').count,
+        total_embeddings: ::Ragdoll::Embedding.count,
+        total_embedding_usage: ::Ragdoll::Embedding.sum(:usage_count)
+      }
+    end
+  end
+end

data/app/controllers/ragdoll/documents_controller.rb

@@ -0,0 +1,460 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class DocumentsController < ApplicationController
+    before_action :set_document, only: [:show, :edit, :update, :destroy, :preview, :reprocess, :download]
+    skip_before_action :verify_authenticity_token, only: [:upload_async, :bulk_upload, :create]
+    
+    def index
+      @documents = ::Ragdoll::Document.all
+      @documents = @documents.where(status: params[:status]) if params[:status].present?
+      @documents = @documents.where(document_type: params[:document_type]) if params[:document_type].present?
+      @documents = @documents.where('title ILIKE ?', "%#{params[:search]}%") if params[:search].present?
+      @documents = @documents.order(created_at: :desc)
+      
+      @document_types = ::Ragdoll::Document.distinct.pluck(:document_type).compact
+      @statuses = ::Ragdoll::Document.distinct.pluck(:status).compact
+    end
+    
+    def show
+      @embeddings = @document.all_embeddings
+      # Load recent searches for sidebar
+      @recent_searches = ::Ragdoll::Search.order(created_at: :desc).limit(10)
+    end
+    
+    def new
+      @document = ::Ragdoll::Document.new
+    end
+    
+    def create
+      if params[:ragdoll_document] && params[:ragdoll_document][:files].present?
+        uploaded_files = params[:ragdoll_document][:files]
+        force_duplicate = params[:ragdoll_document][:force_duplicate] == '1'
+        @results = []
+        
+        # Ensure uploaded_files is always an array
+        uploaded_files = [uploaded_files] unless uploaded_files.is_a?(Array)
+        
+        uploaded_files.each do |file|
+          begin
+            # Skip if file is not a valid upload object
+            next unless file.respond_to?(:original_filename)
+            
+            # Save uploaded file temporarily
+            temp_path = ::Rails.root.join('tmp', 'uploads', file.original_filename)
+            FileUtils.mkdir_p(File.dirname(temp_path))
+            File.binwrite(temp_path, file.read)
+            
+            # Use Ragdoll to add document with force option
+            result = ::Ragdoll.add_document(path: temp_path.to_s, force: force_duplicate)
+            
+            # Get the actual document object if successful
+            if result[:success] && result[:document_id]
+              document = ::Ragdoll::Document.find(result[:document_id])
+              duplicate_detected = result[:duplicate] || (result[:message] && result[:message].include?('already exists'))
+              @results << { 
+                file: file.original_filename, 
+                success: true, 
+                document: document, 
+                message: result[:message],
+                duplicate: duplicate_detected,
+                forced: force_duplicate
+              }
+            else
+              @results << { file: file.original_filename, success: false, error: result[:error] || "Unknown error" }
+            end
+            
+            # Clean up temp file
+            File.delete(temp_path) if File.exist?(temp_path)
+          rescue => e
+            filename = file.respond_to?(:original_filename) ? file.original_filename : file.to_s
+            @results << { file: filename, success: false, error: e.message }
+          end
+        end
+        
+        render :upload_results
+      elsif params[:ragdoll_document] && params[:ragdoll_document][:text_content].present?
+        begin
+          force_duplicate = params[:ragdoll_document][:force_duplicate] == '1'
+          
+          # For text content, we need to save it as a file first since Ragdoll.add_document expects a file
+          temp_path = ::Rails.root.join('tmp', 'uploads', "#{SecureRandom.hex(8)}.txt")
+          FileUtils.mkdir_p(File.dirname(temp_path))
+          File.write(temp_path, params[:ragdoll_document][:text_content])
+          
+          result = ::Ragdoll.add_document(path: temp_path.to_s, force: force_duplicate)
+          
+          # Clean up temp file
+          File.delete(temp_path) if File.exist?(temp_path)
+          
+          if result[:success] && result[:document_id]
+            document = ::Ragdoll::Document.find(result[:document_id])
+            duplicate_detected = result[:duplicate] || (result[:message] && result[:message].include?('already exists'))
+            
+            if duplicate_detected && !force_duplicate
+              redirect_to ragdoll.document_path(document), notice: 'Document already exists - returned existing document.'
+            elsif duplicate_detected && force_duplicate
+              redirect_to ragdoll.document_path(document), notice: 'Document was successfully created (duplicate forced).'
+            else
+              redirect_to ragdoll.document_path(document), notice: 'Document was successfully created.'
+            end
+          else
+            @document = ::Ragdoll::Document.new
+            @document.errors.add(:base, result[:error] || "Unknown error occurred")
+            render :new
+          end
+        rescue => e
+          @document = ::Ragdoll::Document.new
+          @document.errors.add(:base, e.message)
+          render :new
+        end
+      else
+        @document = ::Ragdoll::Document.new
+        @document.errors.add(:base, "Please provide either files or text content")
+        render :new
+      end
+    end
+    
+    def edit
+    end
+    
+    def update
+      if @document.update(document_params)
+        redirect_to ragdoll.document_path(@document), notice: 'Document was successfully updated.'
+      else
+        render :edit
+      end
+    end
+    
+    def destroy
+      @document.destroy
+      redirect_to ragdoll.documents_url, notice: 'Document was successfully deleted.'
+    end
+    
+    def preview
+      respond_to do |format|
+        format.html { render layout: false }
+        format.json { render json: { content: @document.content, metadata: @document.metadata } }
+      end
+    end
+    
+    def reprocess
+      begin
+        # Delete existing embeddings
+        @document.all_embeddings.destroy_all
+        
+        # Reprocess document
+        @document.update(status: 'pending')
+        
+        # Process embeddings in background
+        ::Ragdoll::GenerateEmbeddingsJob.perform_later(@document.id)
+        
+        redirect_to ragdoll.document_path(@document), notice: 'Document reprocessing initiated.'
+      rescue => e
+        redirect_to ragdoll.document_path(@document), alert: "Error reprocessing document: #{e.message}"
+      end
+    end
+    
+    def download
+      if @document.location.present? && File.exist?(@document.location)
+        send_file @document.location, filename: @document.title
+      else
+        redirect_to ragdoll.document_path(@document), alert: 'File not found.'
+      end
+    end
+    
+    def bulk_upload
+      logger.info "🔍 BULK UPLOAD METHOD CALLED"
+      logger.info "📊 Request params: #{params.inspect}"
+      logger.info "📊 Session ID: #{session.id}"
+      
+      if params[:directory_files].present?
+        # Use temp_session_id from frontend, session ID, or request ID as fallback
+        session_id = if params[:temp_session_id].present?
+                       params[:temp_session_id]
+                     elsif session.id.present?
+                       session.id.to_s
+                     else
+                       request.request_id
+                     end
+        
+        logger.info "📊 Using session_id: #{session_id}"
+        
+        uploaded_files = params[:directory_files]
+        force_duplicate = params[:force_duplicate] == '1'
+        
+        # Ensure uploaded_files is always an array
+        uploaded_files = [uploaded_files] unless uploaded_files.is_a?(Array)
+        
+        logger.info "📁 Processing #{uploaded_files.size} files"
+        
+        # Prepare file data for background job
+        file_paths_data = []
+        uploaded_files.each_with_index do |file, index|
+          next unless file.respond_to?(:original_filename)
+          
+          begin
+            # Generate unique temp filename
+            file_id = "#{session_id}_#{index}_#{Time.current.to_i}"
+            temp_path = ::Rails.root.join('tmp', 'uploads', "#{file_id}_#{file.original_filename}")
+            FileUtils.mkdir_p(File.dirname(temp_path))
+            File.binwrite(temp_path, file.read)
+            
+            file_paths_data << {
+              temp_path: temp_path.to_s,
+              original_filename: file.original_filename
+            }
+            
+            logger.info "📄 Queued file: #{file.original_filename}"
+          rescue => e
+            logger.error "❌ Failed to prepare file #{file.original_filename}: #{e.message}"
+          end
+        end
+        
+        if file_paths_data.any?
+          # Queue background job for processing
+          ::Ragdoll::BulkDocumentProcessingJob.perform_later(session_id, file_paths_data, force_duplicate)
+          
+          logger.info "🚀 Queued bulk processing job for #{file_paths_data.size} files"
+          
+          # Check if this is an AJAX request
+          if request.xhr? || request.format.json?
+            render json: {
+              success: true,
+              session_id: session_id,
+              file_count: file_paths_data.size,
+              message: "Upload started! Processing #{file_paths_data.size} files in the background."
+            }
+          else
+            flash[:notice] = "Upload started! Processing #{file_paths_data.size} files in the background. You can monitor progress below."
+            redirect_to ragdoll.documents_path
+          end
+        else
+          logger.warn "⚠️ No valid files found for processing"
+          
+          if request.xhr? || request.format.json?
+            render json: {
+              success: false,
+              error: "No valid files found for processing."
+            }, status: :unprocessable_entity
+          else
+            flash[:alert] = "No valid files found for processing."
+            redirect_to ragdoll.documents_path
+          end
+        end
+      else
+        logger.warn "⚠️ No files provided in bulk upload"
+        
+        if request.xhr? || request.format.json?
+          render json: {
+            success: false,
+            error: "Please select files to upload."
+          }, status: :bad_request
+        else
+          flash[:alert] = "Please select files to upload."
+          redirect_to ragdoll.documents_path
+        end
+      end
+    rescue => e
+      logger.error "💥 Fatal error in bulk_upload: #{e.message}"
+      logger.error e.backtrace.join("\n")
+      
+      if request.xhr? || request.format.json?
+        render json: {
+          success: false,
+          error: "Upload failed: #{e.message}"
+        }, status: :internal_server_error
+      else
+        flash[:alert] = "Upload failed: #{e.message}"
+        redirect_to ragdoll.documents_path
+      end
+    end
+    
+    def bulk_delete
+      if params[:document_ids].present?
+        documents = ::Ragdoll::Document.where(id: params[:document_ids])
+        count = documents.count
+        documents.destroy_all
+        flash[:notice] = "Successfully deleted #{count} documents."
+      else
+        flash[:alert] = "No documents selected for deletion."
+      end
+      
+      redirect_to ragdoll.documents_path
+    end
+    
+    def bulk_reprocess
+      if params[:document_ids].present?
+        documents = ::Ragdoll::Document.where(id: params[:document_ids])
+        documents.each do |document|
+          document.all_embeddings.destroy_all
+          document.update(status: 'pending')
+          ::Ragdoll::GenerateEmbeddingsJob.perform_later(document.id)
+        end
+        flash[:notice] = "Reprocessing initiated for #{documents.count} documents."
+      else
+        flash[:alert] = "No documents selected for reprocessing."
+      end
+      
+      redirect_to ragdoll.documents_path
+    end
+
+    def upload_async
+      logger.info "upload_async called with params: #{params.inspect}"
+      logger.info "Session ID: #{session.id}"
+      logger.info "Request ID: #{request.request_id}"
+      logger.info "Temp Session ID: #{params[:temp_session_id]}"
+      
+      if params[:ragdoll_document] && params[:ragdoll_document][:files].present?
+        # Priority: temp_session_id from frontend, then session ID, then request ID as fallback
+        session_id = if params[:temp_session_id].present?
+                       params[:temp_session_id]
+                     elsif session.id.present?
+                       session.id.to_s
+                     else
+                       request.request_id
+                     end
+        logger.info "Using session_id: #{session_id} (source: #{params[:temp_session_id].present? ? 'temp_session_id' : session.id.present? ? 'session' : 'request'})"
+        uploaded_files = params[:ragdoll_document][:files]
+        
+        logger.info "Files received: #{uploaded_files.inspect}"
+        
+        # Ensure uploaded_files is always an array
+        uploaded_files = [uploaded_files] unless uploaded_files.is_a?(Array)
+        
+        # Log file analysis for debugging count discrepancies
+        total_files = uploaded_files.count
+        valid_files = uploaded_files.select { |f| f && f.respond_to?(:original_filename) && f.original_filename.present? }
+        filtered_count = total_files - valid_files.count
+        
+        logger.info "📊 File upload analysis:"
+        logger.info "   Total files in upload array: #{total_files}"
+        logger.info "   Valid files with original_filename: #{valid_files.count}"
+        logger.info "   Files filtered out: #{filtered_count}"
+        
+        if filtered_count > 0
+          uploaded_files.each_with_index do |file, index|
+            unless file && file.respond_to?(:original_filename) && file.original_filename.present?
+              logger.warn "   Filtered file #{index + 1}: #{file.class} - #{file.inspect}"
+            end
+          end
+        end
+        
+        processed_count = 0
+        results = []
+        
+        # Process files in batches to avoid "too many open files" error
+        batch_size = 50  # Process 50 files at a time
+        uploaded_files.each_slice(batch_size).with_index do |file_batch, batch_index|
+          file_batch.each_with_index do |file, batch_file_index|
+            index = batch_index * batch_size + batch_file_index
+          next unless file.respond_to?(:original_filename)
+          
+          logger.info "Processing file #{index + 1}: #{file.original_filename}"
+          
+          begin
+            # Generate unique file ID
+            file_id = "#{session_id}_#{index}_#{Time.current.to_i}"
+            
+            # Save uploaded file temporarily
+            temp_path = ::Rails.root.join('tmp', 'uploads', "#{file_id}_#{file.original_filename}")
+            FileUtils.mkdir_p(File.dirname(temp_path))
+            File.binwrite(temp_path, file.read)
+            
+            logger.info "File saved to: #{temp_path}"
+            
+            # Try to queue background job first, fallback to direct processing
+            begin
+              if defined?(::Ragdoll::ProcessFileJob)
+                ::Ragdoll::ProcessFileJob.perform_later(file_id, session_id, file.original_filename, temp_path.to_s)
+                logger.info "Job queued for file: #{file_id}"
+                results << { file: file.original_filename, status: 'queued' }
+              else
+                raise "ProcessFileJob not available"
+              end
+            rescue => job_error
+              logger.warn "Background job failed, processing directly: #{job_error.message}"
+              
+              # Process directly if job system is not available  
+              force_duplicate = params[:ragdoll_document][:force_duplicate] == '1'
+              result = ::Ragdoll.add_document(path: temp_path.to_s, force: force_duplicate)
+              
+              if result[:success] && result[:document_id]
+                document = ::Ragdoll::Document.find(result[:document_id])
+                results << { 
+                  file: file.original_filename, 
+                  status: 'completed_sync',
+                  document_id: document.id
+                }
+                logger.info "File processed synchronously: #{file.original_filename}"
+              else
+                results << { 
+                  file: file.original_filename, 
+                  status: 'failed',
+                  error: result[:error] || 'Unknown error'
+                }
+              end
+              
+              # Clean up temp file for sync processing
+              File.delete(temp_path) if File.exist?(temp_path)
+            end
+            
+            processed_count += 1
+          rescue => file_error
+            logger.error "Error processing file #{file.original_filename}: #{file_error.message}"
+            results << { 
+              file: file.original_filename, 
+              status: 'failed',
+              error: file_error.message
+            }
+          end
+          end
+          
+          # Force garbage collection after each batch to free file descriptors
+          GC.start
+        end
+        
+        logger.info "Returning success response for #{processed_count} files"
+        render json: { 
+          success: true, 
+          session_id: session_id,
+          results: results,
+          message: "#{processed_count} file(s) processed" 
+        }
+      else
+        logger.error "No files provided in upload_async"
+        render json: { success: false, error: "No files provided" }, status: :bad_request
+      end
+    rescue => e
+      logger.error "Error in upload_async: #{e.message}"
+      logger.error e.backtrace.join("\n")
+      render json: { success: false, error: e.message }, status: :internal_server_error
+    end
+    
+    def status
+      @processing_stats = {
+        pending: ::Ragdoll::Document.where(status: 'pending').count,
+        processing: ::Ragdoll::Document.where(status: 'processing').count,
+        processed: ::Ragdoll::Document.where(status: 'processed').count,
+        failed: ::Ragdoll::Document.where(status: 'failed').count
+      }
+      
+      @recent_activity = ::Ragdoll::Document.order(updated_at: :desc).limit(20)
+      
+      respond_to do |format|
+        format.html
+        format.json { render json: @processing_stats }
+      end
+    end
+    
+    private
+    
+    def set_document
+      @document = ::Ragdoll::Document.find(params[:id])
+    end
+    
+    def document_params
+      params.require(:ragdoll_document).permit(:title, :content, :metadata, :status, :text_content, :force_duplicate, files: [])
+    end
+  end
+end