ragdoll-rails 0.1.8 → 0.1.11

data/app/controllers/ragdoll/api/v1/system_controller.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Api
+    module V1
+      class SystemController < BaseController
+        def stats
+          begin
+            client = ::Ragdoll::Client.new
+            ragdoll_stats = client.stats
+            
+            system_stats = {
+              ragdoll_version: ::Ragdoll::VERSION,
+              rails_version: Rails.version,
+              ruby_version: RUBY_VERSION,
+              
+              database_stats: {
+                documents: ::Ragdoll::Document.count,
+                embeddings: ::Ragdoll::Embedding.count,
+                searches: ::Ragdoll::Search.count,
+                database_size: calculate_database_size
+              },
+              
+              configuration: {
+                llm_provider: ::Ragdoll.configuration.llm_provider,
+                embedding_provider: ::Ragdoll.configuration.embedding_provider,
+                embedding_model: ::Ragdoll.configuration.embedding_model,
+                chunk_size: ::Ragdoll.configuration.chunk_size,
+                chunk_overlap: ::Ragdoll.configuration.chunk_overlap,
+                max_search_results: ::Ragdoll.configuration.max_search_results,
+                search_similarity_threshold: ::Ragdoll.configuration.search_similarity_threshold,
+                enable_search_analytics: ::Ragdoll.configuration.enable_search_analytics,
+                enable_document_summarization: ::Ragdoll.configuration.enable_document_summarization,
+                enable_usage_tracking: ::Ragdoll.configuration.enable_usage_tracking,
+                usage_ranking_enabled: ::Ragdoll.configuration.usage_ranking_enabled
+              },
+              
+              performance_metrics: {
+                average_search_time: calculate_average_search_time,
+                embedding_dimensions: ::Ragdoll::Embedding.first&.embedding_dimensions || 0,
+                average_document_size: ::Ragdoll::Document.average('LENGTH(summary)')&.round || 0,
+                average_chunks_per_document: ::Ragdoll::Document.count > 0 ? (::Ragdoll::Embedding.count.to_f / ::Ragdoll::Document.count).round || 0 : 0
+              },
+              
+              health_check: {
+                database_connection: database_healthy?,
+                embedding_service: embedding_service_healthy?
+              }
+            }.merge(ragdoll_stats)
+            
+            render json: system_stats
+          rescue => e
+            render_error("Error retrieving system stats: #{e.message}")
+          end
+        end
+        
+        private
+        
+        def calculate_database_size
+          # Simple approximation - in production you might want more sophisticated calculation
+          result = ActiveRecord::Base.connection.execute(
+            "SELECT pg_size_pretty(pg_database_size(current_database()))"
+          )
+          result.first['pg_size_pretty']
+        rescue
+          "Unknown"
+        end
+        
+        def calculate_average_search_time
+          # Calculate from actual search execution times if available
+          avg_time = ::Ragdoll::Search.where.not(execution_time_ms: nil).average(:execution_time_ms)
+          if avg_time
+            "#{avg_time.round}ms"
+          else
+            "< 100ms"
+          end
+        end
+        
+        def database_healthy?
+          ActiveRecord::Base.connection.active?
+        rescue
+          false
+        end
+        
+        def embedding_service_healthy?
+          begin
+            client = ::Ragdoll::Client.new
+            # Try a simple test to see if the embedding service is available
+            true
+          rescue
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/app/controllers/ragdoll/application_controller.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class ApplicationController < ActionController::Base
+    protect_from_forgery with: :exception
+    before_action :set_current_user
+    
+    layout 'ragdoll/application'
+    
+    private
+    
+    def set_current_user
+      # This can be overridden in the host application
+      # to set the current user for the Ragdoll engine
+    end
+  end
+end

data/app/controllers/ragdoll/configuration_controller.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class ConfigurationController < ApplicationController
+    def index
+      @configuration = ::Ragdoll.configuration
+      @available_providers = %w[openai anthropic google azure ollama huggingface]
+      @available_models = {
+        openai: ['text-embedding-3-small', 'text-embedding-3-large', 'text-embedding-ada-002'],
+        anthropic: ['claude-3-haiku-20240307', 'claude-3-sonnet-20240229', 'claude-3-opus-20240229'],
+        google: ['gemini-pro', 'gemini-1.5-flash', 'gemini-1.5-pro'],
+        azure: ['text-embedding-3-small', 'text-embedding-3-large'],
+        ollama: ['llama2', 'mistral', 'codellama'],
+        huggingface: ['sentence-transformers/all-MiniLM-L6-v2', 'sentence-transformers/all-mpnet-base-v2']
+      }
+      
+      @current_stats = {
+        total_documents: ::Ragdoll::Document.count,
+        total_embeddings: ::Ragdoll::Embedding.count,
+        embedding_dimensions: ::Ragdoll::Embedding.first&.embedding_dimensions || 0,
+        average_chunk_size: ::Ragdoll::Embedding.average('LENGTH(content)')&.round || 0
+      }
+    end
+    
+    def update
+      config_params = params.require(:configuration).permit(
+        :llm_provider,
+        :embedding_provider,
+        :embedding_model,
+        :chunk_size,
+        :chunk_overlap,
+        :max_search_results,
+        :search_similarity_threshold,
+        :enable_search_analytics,
+        :enable_document_summarization,
+        :enable_usage_tracking,
+        :usage_ranking_enabled,
+        :openai_api_key,
+        :anthropic_api_key,
+        :google_api_key,
+        :azure_api_key,
+        :ollama_url,
+        :huggingface_api_key
+      )
+      
+      begin
+        # Update configuration
+        config = ::Ragdoll.configuration
+        
+        config_params.each do |key, value|
+          # Convert string values to appropriate types
+          case key
+          when 'chunk_size', 'chunk_overlap', 'max_search_results'
+            config.send("#{key}=", value.to_i)
+          when 'search_similarity_threshold'
+            config.send("#{key}=", value.to_f)
+          when 'enable_search_analytics', 'enable_document_summarization', 'enable_usage_tracking', 'usage_ranking_enabled'
+            config.send("#{key}=", value == '1' || value == 'true')
+          else
+            config.send("#{key}=", value) if value.present?
+          end
+        end
+        
+        flash[:notice] = 'Configuration updated successfully.'
+        
+        # Test the configuration
+        begin
+          client = ::Ragdoll::Client.new
+          test_result = client.stats
+          flash[:notice] += " Configuration test successful."
+        rescue => e
+          flash[:warning] = "Configuration saved but test failed: #{e.message}"
+        end
+        
+      rescue => e
+        flash[:alert] = "Error updating configuration: #{e.message}"
+      end
+      
+      redirect_to ragdoll.configuration_path
+    end
+  end
+end

data/app/controllers/ragdoll/dashboard_controller.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class DashboardController < ApplicationController
+    def index
+      @stats = {
+        total_documents: ::Ragdoll::Document.count,
+        processed_documents: ::Ragdoll::Document.where(status: 'processed').count,
+        failed_documents: ::Ragdoll::Document.where(status: 'failed').count,
+        pending_documents: ::Ragdoll::Document.where(status: 'pending').count,
+        total_embeddings: ::Ragdoll::Embedding.count,
+        total_searches: ::Ragdoll::Search.count,
+        recent_searches: ::Ragdoll::Search.order(created_at: :desc).limit(5)
+      }
+      
+      @document_types = ::Ragdoll::Document.group(:document_type).count
+      @recent_documents = ::Ragdoll::Document.order(created_at: :desc).limit(10)
+      
+      # Usage analytics - join through embeddable (Content) to get to documents
+      @top_searched_documents = ::Ragdoll::Embedding
+        .joins("JOIN ragdoll_contents ON ragdoll_contents.id = ragdoll_embeddings.embeddable_id")
+        .joins("JOIN ragdoll_documents ON ragdoll_documents.id = ragdoll_contents.document_id")
+        .group('ragdoll_documents.title')
+        .order(Arel.sql('SUM(ragdoll_embeddings.usage_count) DESC'))
+        .limit(5)
+        .sum(:usage_count)
+    end
+    
+    def analytics
+      today = Date.current
+      week_start = today.beginning_of_week
+      month_start = today.beginning_of_month
+      
+      # Calculate search statistics
+      all_searches = ::Ragdoll::Search.all
+      searches_today = all_searches.where(created_at: today.beginning_of_day..today.end_of_day)
+      searches_this_week = all_searches.where(created_at: week_start.beginning_of_day..today.end_of_day)
+      searches_this_month = all_searches.where(created_at: month_start.beginning_of_day..today.end_of_day)
+      
+      # Comprehensive search analytics combining both pages
+      @search_analytics = {
+        total_searches: all_searches.count,
+        unique_queries: all_searches.distinct.count(:query),
+        searches_today: searches_today.count,
+        searches_this_week: searches_this_week.count,
+        searches_this_month: searches_this_month.count,
+        average_results: all_searches.average(:results_count)&.round(1) || 0,
+        average_similarity: all_searches.where.not(avg_similarity_score: nil).average(:avg_similarity_score)&.round(3) || 0,
+        avg_execution_time: all_searches.average(:execution_time_ms)&.round(1) || 0,
+        search_types: all_searches.group(:search_type).count
+      }
+      
+      # Top queries (most frequent)
+      @top_queries = all_searches
+        .group(:query)
+        .count
+        .sort_by { |query, count| -count }
+        .first(10)
+        .to_h
+      
+      # Search trends by day for the last 7 days
+      @search_trends = (6.days.ago.to_date..today).map do |date|
+        count = all_searches.where(created_at: date.beginning_of_day..date.end_of_day).count
+        [date.strftime('%m/%d'), count]
+      end.to_h
+      
+      # Most searched documents (using embedding usage as proxy)
+      @top_documents = ::Ragdoll::Embedding
+        .joins("JOIN ragdoll_contents ON ragdoll_contents.id = ragdoll_embeddings.embeddable_id")
+        .joins("JOIN ragdoll_documents ON ragdoll_documents.id = ragdoll_contents.document_id")
+        .group('ragdoll_documents.title')
+        .order(Arel.sql('SUM(ragdoll_embeddings.usage_count) DESC'))
+        .limit(10)
+        .sum(:usage_count)
+      
+      # Similarity score distribution
+      similarity_scores = all_searches.where.not(avg_similarity_score: nil).pluck(:avg_similarity_score)
+      @similarity_distribution = {
+        "0.9-1.0" => similarity_scores.count { |s| s >= 0.9 },
+        "0.8-0.9" => similarity_scores.count { |s| s >= 0.8 && s < 0.9 },
+        "0.7-0.8" => similarity_scores.count { |s| s >= 0.7 && s < 0.8 },
+        "0.6-0.7" => similarity_scores.count { |s| s >= 0.6 && s < 0.7 },
+        "0.5-0.6" => similarity_scores.count { |s| s >= 0.5 && s < 0.6 },
+        "< 0.5" => similarity_scores.count { |s| s < 0.5 }
+      }
+      
+      # System statistics
+      @system_stats = {
+        total_documents: ::Ragdoll::Document.count,
+        processed_documents: ::Ragdoll::Document.where(status: 'processed').count,
+        failed_documents: ::Ragdoll::Document.where(status: 'failed').count,
+        pending_documents: ::Ragdoll::Document.where(status: 'pending').count,
+        total_embeddings: ::Ragdoll::Embedding.count,
+        total_embedding_usage: ::Ragdoll::Embedding.sum(:usage_count)
+      }
+    end
+  end
+end