ragdoll-rails 0.1.9 → 0.1.11

data/app/components/ragdoll/empty_state_component.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class EmptyStateComponent < ApplicationComponent
+    def initialize(title:, message:, icon: nil, action_path: nil, action_text: nil)
+      @title = title
+      @message = message
+      @icon = icon
+      @action_path = action_path
+      @action_text = action_text
+    end
+
+    private
+
+    attr_reader :title, :message, :icon, :action_path, :action_text
+  end
+end

data/app/components/ragdoll/flash_messages_component.html.erb

@@ -0,0 +1,3 @@
+<% flash_messages.each do |flash_msg| %>
+  <%= render Ragdoll::AlertComponent.new(message: flash_msg[:message], type: flash_msg[:type]) %>
+<% end %>

data/app/components/ragdoll/flash_messages_component.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class FlashMessagesComponent < ApplicationComponent
+    def initialize(flash:)
+      @flash = flash
+    end
+
+    private
+
+    attr_reader :flash
+
+    def flash_type_to_alert_type(type)
+      case type.to_s
+      when 'notice'
+        'success'
+      when 'alert'
+        'danger'
+      when 'error'
+        'danger'
+      when 'warning'
+        'warning'
+      else
+        'info'
+      end
+    end
+
+    def flash_messages
+      flash.map do |type, message|
+        {
+          type: flash_type_to_alert_type(type),
+          message: message
+        }
+      end
+    end
+  end
+end

data/app/components/ragdoll/navbar_component.html.erb

@@ -0,0 +1,24 @@
+<nav class="navbar navbar-expand-lg navbar-dark bg-primary">
+  <div class="container">
+    <%= link_to brand_path, class: "navbar-brand d-flex align-items-center" do %>
+      <i class="fas fa-robot me-2"></i>
+      <%= brand_text %>
+    <% end %>
+    
+    <button class="navbar-toggler" type="button" data-bs-toggle="collapse" data-bs-target="#navbarNav">
+      <span class="navbar-toggler-icon"></span>
+    </button>
+    
+    <div class="collapse navbar-collapse" id="navbarNav">
+      <ul class="navbar-nav me-auto">
+        <% nav_items.each do |item| %>
+          <li class="nav-item">
+            <%= link_to item[:path], class: nav_link_classes(item[:path]) do %>
+              <i class="<%= item[:icon] %>"></i> <%= item[:text] %>
+            <% end %>
+          </li>
+        <% end %>
+      </ul>
+    </div>
+  </div>
+</nav>

data/app/components/ragdoll/navbar_component.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class NavbarComponent < ApplicationComponent
+    def initialize(brand_text: 'Ragdoll Engine', brand_path: nil)
+      @brand_text = brand_text
+      @brand_path = brand_path || main_app.root_path
+    end
+
+    private
+
+    attr_reader :brand_text, :brand_path
+
+    def nav_items
+      [
+        { text: 'Dashboard', path: ragdoll.dashboard_index_path, icon: 'fas fa-tachometer-alt' },
+        { text: 'Documents', path: ragdoll.documents_path, icon: 'fas fa-file-alt' },
+        { text: 'Search', path: ragdoll.search_index_path, icon: 'fas fa-search' },
+        { text: 'Jobs', path: ragdoll.jobs_path, icon: 'fas fa-tasks' },
+        { text: 'Analytics', path: ragdoll.analytics_path, icon: 'fas fa-chart-line' },
+        { text: 'Configuration', path: ragdoll.configuration_path, icon: 'fas fa-cog' }
+      ]
+    end
+
+    def nav_link_classes(path)
+      base_classes = ['nav-link']
+      base_classes << 'active' if current_page?(path)
+      base_classes.join(' ')
+    end
+  end
+end

data/app/components/ragdoll/page_header_component.html.erb

@@ -0,0 +1,13 @@
+<div class="d-flex justify-content-between align-items-center mb-4">
+  <div>
+    <h1>
+      <% if icon %>
+        <i class="<%= icon %>"></i>
+      <% end %>
+      <%= title %>
+    </h1>
+    <% if subtitle %>
+      <p class="text-muted"><%= subtitle %></p>
+    <% end %>
+  </div>
+</div>

data/app/components/ragdoll/page_header_component.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class PageHeaderComponent < ApplicationComponent
+    def initialize(title:, icon: nil, subtitle: nil)
+      @title = title
+      @icon = icon
+      @subtitle = subtitle
+    end
+
+    private
+
+    attr_reader :title, :icon, :subtitle
+  end
+end

data/app/components/ragdoll/stats_card_component.html.erb

@@ -0,0 +1,11 @@
+<div class="card border-<%= color %>">
+  <div class="card-body text-center">
+    <h5 class="card-title text-<%= color %>">
+      <i class="<%= icon %>"></i> <%= title %>
+    </h5>
+    <h2 class="text-<%= color %>"><%= value %></h2>
+    <% if description %>
+      <p class="text-muted mb-0"><%= description %></p>
+    <% end %>
+  </div>
+</div>

data/app/components/ragdoll/stats_card_component.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class StatsCardComponent < ApplicationComponent
+    def initialize(title:, value:, icon:, color: 'primary', description: nil)
+      @title = title
+      @value = value
+      @icon = icon
+      @color = color
+      @description = description
+    end
+
+    private
+
+    attr_reader :title, :value, :icon, :color, :description
+  end
+end

data/app/components/ragdoll/status_badge_component.html.erb

@@ -0,0 +1,3 @@
+<span class="badge <%= badge_class %>">
+  <%= badge_text %>
+</span>

data/app/components/ragdoll/status_badge_component.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class StatusBadgeComponent < ApplicationComponent
+    def initialize(status:)
+      @status = status
+    end
+
+    private
+
+    attr_reader :status
+
+    def badge_class
+      case status.to_s.downcase
+      when 'processed', 'completed', 'success'
+        'bg-success'
+      when 'failed', 'error'
+        'bg-danger'
+      when 'processing', 'pending', 'queued'
+        'bg-warning'
+      else
+        'bg-secondary'
+      end
+    end
+
+    def badge_text
+      status.to_s.humanize
+    end
+  end
+end

data/app/controllers/ragdoll/api/v1/analytics_controller.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Api
+    module V1
+      class AnalyticsController < BaseController
+        def index
+          analytics_data = {
+            document_stats: {
+              total_documents: ::Ragdoll::Document.count,
+              processed_documents: ::Ragdoll::Document.where(status: 'processed').count,
+              failed_documents: ::Ragdoll::Document.where(status: 'failed').count,
+              pending_documents: ::Ragdoll::Document.where(status: 'pending').count,
+              total_embeddings: ::Ragdoll::Embedding.count
+            },
+            
+            search_stats: {
+              total_searches: ::Ragdoll::Search.count,
+              unique_queries: ::Ragdoll::Search.distinct.count(:query),
+              searches_today: ::Ragdoll::Search.where(created_at: Date.current.beginning_of_day..Date.current.end_of_day).count,
+              searches_this_week: ::Ragdoll::Search.where(created_at: Date.current.beginning_of_week..Date.current.end_of_day).count,
+              average_similarity: ::Ragdoll::Search.where.not(avg_similarity_score: nil).average(:avg_similarity_score)&.round(3) || 0
+            },
+            
+            popular_queries: ::Ragdoll::Search.group(:query).count.sort_by { |query, count| -count }.first(10).to_h,
+            
+            document_types: ::Ragdoll::Document.group(:document_type).count,
+            
+            top_documents: ::Ragdoll::Embedding
+              .joins("JOIN ragdoll_contents ON ragdoll_contents.id = ragdoll_embeddings.embeddable_id")
+              .joins("JOIN ragdoll_documents ON ragdoll_documents.id = ragdoll_contents.document_id")
+              .group('ragdoll_documents.title')
+              .order('SUM(ragdoll_embeddings.usage_count) DESC')
+              .limit(10)
+              .sum(:usage_count),
+            
+            search_trends: (6.days.ago.to_date..Date.current).map do |date|
+              count = ::Ragdoll::Search.where(created_at: date.beginning_of_day..date.end_of_day).count
+              [date.strftime('%m/%d'), count]
+            end.to_h,
+            
+            embedding_usage: ::Ragdoll::Embedding
+              .joins("JOIN ragdoll_contents ON ragdoll_contents.id = ragdoll_embeddings.embeddable_id")
+              .joins("JOIN ragdoll_documents ON ragdoll_documents.id = ragdoll_contents.document_id")
+              .group('ragdoll_documents.title')
+              .order('SUM(ragdoll_embeddings.usage_count) DESC')
+              .limit(10)
+              .sum(:usage_count),
+            
+            similarity_distribution: build_similarity_distribution
+          }
+          
+          render json: analytics_data
+        end
+        
+        private
+        
+        def build_similarity_distribution
+          similarity_scores = ::Ragdoll::Search.where.not(avg_similarity_score: nil).pluck(:avg_similarity_score)
+          {
+            "0.9-1.0" => similarity_scores.count { |s| s >= 0.9 },
+            "0.8-0.9" => similarity_scores.count { |s| s >= 0.8 && s < 0.9 },
+            "0.7-0.8" => similarity_scores.count { |s| s >= 0.7 && s < 0.8 },
+            "0.6-0.7" => similarity_scores.count { |s| s >= 0.6 && s < 0.7 },
+            "0.5-0.6" => similarity_scores.count { |s| s >= 0.5 && s < 0.6 },
+            "< 0.5" => similarity_scores.count { |s| s < 0.5 }
+          }
+        end
+      end
+    end
+  end
+end

data/app/controllers/ragdoll/api/v1/base_controller.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Api
+    module V1
+      class BaseController < ActionController::API
+        protect_from_forgery with: :null_session
+        before_action :set_default_response_format
+        
+        private
+        
+        def set_default_response_format
+          request.format = :json
+        end
+        
+        def render_error(message, status = :unprocessable_entity)
+          render json: { error: message }, status: status
+        end
+        
+        def render_success(data = {}, message = nil)
+          response = { success: true }
+          response[:message] = message if message
+          response[:data] = data unless data.empty?
+          render json: response
+        end
+      end
+    end
+  end
+end

data/app/controllers/ragdoll/api/v1/documents_controller.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Api
+    module V1
+      class DocumentsController < BaseController
+        before_action :set_document, only: [:show, :update, :destroy, :reprocess]
+        
+        def index
+          documents = ::Ragdoll::Document.all
+          documents = documents.where(status: params[:status]) if params[:status].present?
+          documents = documents.where(document_type: params[:document_type]) if params[:document_type].present?
+          documents = documents.order(created_at: :desc)
+          
+          render json: {
+            documents: documents.map(&method(:document_json)),
+            total: documents.count
+          }
+        end
+        
+        def show
+          render json: {
+            document: document_json(@document),
+            embeddings: @document.all_embeddings.map(&method(:embedding_json))
+          }
+        end
+        
+        def create
+          begin
+            if params[:file].present?
+              # Handle file upload
+              temp_path = Rails.root.join('tmp', 'uploads', params[:file].original_filename)
+              FileUtils.mkdir_p(File.dirname(temp_path))
+              File.binwrite(temp_path, params[:file].read)
+              
+              result = ::Ragdoll.add_document(path: temp_path.to_s)
+              
+              if result[:success] && result[:document_id]
+                document = ::Ragdoll::Document.find(result[:document_id])
+                File.delete(temp_path) if File.exist?(temp_path)
+                render json: { document: document_json(document) }, status: :created
+              else
+                File.delete(temp_path) if File.exist?(temp_path)
+                return render_error(result[:error] || "Failed to create document")
+              end
+            elsif params[:content].present?
+              # Handle text content
+              temp_path = Rails.root.join('tmp', 'uploads', "#{SecureRandom.hex(8)}.txt")
+              FileUtils.mkdir_p(File.dirname(temp_path))
+              File.write(temp_path, params[:content])
+              
+              result = ::Ragdoll.add_document(path: temp_path.to_s)
+              
+              if result[:success] && result[:document_id]
+                document = ::Ragdoll::Document.find(result[:document_id])
+                File.delete(temp_path) if File.exist?(temp_path)
+                render json: { document: document_json(document) }, status: :created
+              else
+                File.delete(temp_path) if File.exist?(temp_path)
+                return render_error(result[:error] || "Failed to create document")
+              end
+            else
+              return render_error("Either file or content must be provided")
+            end
+            
+          rescue => e
+            render_error(e.message)
+          end
+        end
+        
+        def update
+          begin
+            if @document.update(document_params)
+              render json: { document: document_json(@document) }
+            else
+              render_error(@document.errors.full_messages.join(', '))
+            end
+          rescue => e
+            render_error(e.message)
+          end
+        end
+        
+        def destroy
+          begin
+            @document.destroy
+            render_success({}, "Document deleted successfully")
+          rescue => e
+            render_error(e.message)
+          end
+        end
+        
+        def reprocess
+          begin
+            @document.all_embeddings.destroy_all
+            @document.update(status: 'pending')
+            ::Ragdoll::GenerateEmbeddingsJob.perform_later(@document.id)
+            
+            render_success({}, "Document reprocessing initiated")
+          rescue => e
+            render_error(e.message)
+          end
+        end
+        
+        private
+        
+        def set_document
+          @document = ::Ragdoll::Document.find(params[:id])
+        rescue ActiveRecord::RecordNotFound
+          render_error("Document not found", :not_found)
+        end
+        
+        def document_params
+          params.require(:document).permit(:title, :content, :metadata, :status)
+        end
+        
+        def document_json(document)
+          {
+            id: document.id,
+            title: document.title,
+            content: document.content,
+            document_type: document.document_type,
+            location: document.location,
+            metadata: document.metadata,
+            status: document.status,
+            character_count: document.total_character_count,
+            word_count: document.total_word_count,
+            embedding_count: document.all_embeddings.count,
+            created_at: document.created_at,
+            updated_at: document.updated_at
+          }
+        end
+        
+        def embedding_json(embedding)
+          {
+            id: embedding.id,
+            content: embedding.content,
+            chunk_index: embedding.chunk_index,
+            model_name: embedding.model_name,
+            vector_dimensions: embedding.embedding_dimensions,
+            usage_count: embedding.usage_count,
+            last_used_at: embedding.returned_at,
+            created_at: embedding.created_at
+          }
+        end
+      end
+    end
+  end
+end

data/app/controllers/ragdoll/api/v1/search_controller.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Api
+    module V1
+      class SearchController < BaseController
+        def search
+          query = params[:query]
+          
+          if query.blank?
+            return render_error("Query parameter is required")
+          end
+          
+          begin
+            search_options = {
+              limit: params[:limit]&.to_i || 10,
+              threshold: params[:threshold]&.to_f || 0.7,
+              use_usage_ranking: params[:use_usage_ranking] == 'true'
+            }
+            
+            # Add document type filter if specified
+            if params[:document_type].present?
+              search_options[:document_type] = params[:document_type]
+            end
+            
+            # Add status filter if specified
+            if params[:status].present?
+              search_options[:status] = params[:status]
+            end
+            
+            search_response = ::Ragdoll.search(search_options.merge(query: query))
+            results = search_response.is_a?(Hash) ? search_response[:results] || [] : []
+            
+            # Format results with additional metadata
+            formatted_results = results.map do |result|
+              if result[:embedding_id] && result[:document_id]
+                embedding = ::Ragdoll::Embedding.find(result[:embedding_id])
+                document = ::Ragdoll::Document.find(result[:document_id])
+                {
+                  embedding_id: result[:embedding_id],
+                  document_id: result[:document_id],
+                  document_title: document.title,
+                  content: result[:content],
+                  similarity: result[:similarity],
+                  usage_count: embedding.usage_count,
+                  last_used_at: embedding.returned_at,
+                  chunk_index: embedding.chunk_index,
+                  document_type: document.document_type
+                }
+              end
+            end.compact
+            
+            # Save search for analytics
+            begin
+              similarities = formatted_results.map { |r| r[:similarity] }.compact
+              ::Ragdoll::Search.create!(
+                query: query,
+                search_type: 'api_semantic',
+                results_count: formatted_results.count,
+                max_similarity_score: similarities.any? ? similarities.max : nil,
+                min_similarity_score: similarities.any? ? similarities.min : nil,
+                avg_similarity_score: similarities.any? ? (similarities.sum / similarities.size.to_f) : nil,
+                search_filters: search_options.to_json,
+                search_options: {
+                  api_request: true,
+                  threshold_used: search_options[:threshold]
+                }.to_json
+              )
+            rescue => e
+              Rails.logger.error "Failed to save API search: #{e.message}"
+            end
+            
+            render json: {
+              query: query,
+              results: formatted_results,
+              total_results: formatted_results.count,
+              search_options: search_options
+            }
+            
+          rescue => e
+            render_error(e.message)
+          end
+        end
+      end
+    end
+  end
+end

data/app/controllers/ragdoll/api/v1/system_controller.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Api
+    module V1
+      class SystemController < BaseController
+        def stats
+          begin
+            client = ::Ragdoll::Client.new
+            ragdoll_stats = client.stats
+            
+            system_stats = {
+              ragdoll_version: ::Ragdoll::VERSION,
+              rails_version: Rails.version,
+              ruby_version: RUBY_VERSION,
+              
+              database_stats: {
+                documents: ::Ragdoll::Document.count,
+                embeddings: ::Ragdoll::Embedding.count,
+                searches: ::Ragdoll::Search.count,
+                database_size: calculate_database_size
+              },
+              
+              configuration: {
+                llm_provider: ::Ragdoll.configuration.llm_provider,
+                embedding_provider: ::Ragdoll.configuration.embedding_provider,
+                embedding_model: ::Ragdoll.configuration.embedding_model,
+                chunk_size: ::Ragdoll.configuration.chunk_size,
+                chunk_overlap: ::Ragdoll.configuration.chunk_overlap,
+                max_search_results: ::Ragdoll.configuration.max_search_results,
+                search_similarity_threshold: ::Ragdoll.configuration.search_similarity_threshold,
+                enable_search_analytics: ::Ragdoll.configuration.enable_search_analytics,
+                enable_document_summarization: ::Ragdoll.configuration.enable_document_summarization,
+                enable_usage_tracking: ::Ragdoll.configuration.enable_usage_tracking,
+                usage_ranking_enabled: ::Ragdoll.configuration.usage_ranking_enabled
+              },
+              
+              performance_metrics: {
+                average_search_time: calculate_average_search_time,
+                embedding_dimensions: ::Ragdoll::Embedding.first&.embedding_dimensions || 0,
+                average_document_size: ::Ragdoll::Document.average('LENGTH(summary)')&.round || 0,
+                average_chunks_per_document: ::Ragdoll::Document.count > 0 ? (::Ragdoll::Embedding.count.to_f / ::Ragdoll::Document.count).round || 0 : 0
+              },
+              
+              health_check: {
+                database_connection: database_healthy?,
+                embedding_service: embedding_service_healthy?
+              }
+            }.merge(ragdoll_stats)
+            
+            render json: system_stats
+          rescue => e
+            render_error("Error retrieving system stats: #{e.message}")
+          end
+        end
+        
+        private
+        
+        def calculate_database_size
+          # Simple approximation - in production you might want more sophisticated calculation
+          result = ActiveRecord::Base.connection.execute(
+            "SELECT pg_size_pretty(pg_database_size(current_database()))"
+          )
+          result.first['pg_size_pretty']
+        rescue
+          "Unknown"
+        end
+        
+        def calculate_average_search_time
+          # Calculate from actual search execution times if available
+          avg_time = ::Ragdoll::Search.where.not(execution_time_ms: nil).average(:execution_time_ms)
+          if avg_time
+            "#{avg_time.round}ms"
+          else
+            "< 100ms"
+          end
+        end
+        
+        def database_healthy?
+          ActiveRecord::Base.connection.active?
+        rescue
+          false
+        end
+        
+        def embedding_service_healthy?
+          begin
+            client = ::Ragdoll::Client.new
+            # Try a simple test to see if the embedding service is available
+            true
+          rescue
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/app/controllers/ragdoll/application_controller.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  class ApplicationController < ActionController::Base
+    protect_from_forgery with: :exception
+    before_action :set_current_user
+    
+    layout 'ragdoll/application'
+    
+    private
+    
+    def set_current_user
+      # This can be overridden in the host application
+      # to set the current user for the Ragdoll engine
+    end
+  end
+end