ragdoll 0.1.1 → 0.1.3

data/app/models/ragdoll/embedding.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "neighbor"
+
+module Ragdoll
+  class Embedding < ActiveRecord::Base
+    self.table_name = "ragdoll_embeddings"
+
+    # Use pgvector for vector similarity search
+    has_neighbors :embedding_vector
+
+    belongs_to :embeddable, polymorphic: true
+
+    validates :embeddable_id,    presence: true
+    validates :embeddable_type,  presence: true
+    validates :chunk_index,      presence: true, uniqueness: { scope: %i[embeddable_id embeddable_type] }
+    validates :embedding_vector, presence: true
+    validates :content,          presence: true
+
+    scope :by_model, lambda { |model|
+      # Use STI table for all content types
+      where(
+        "embeddable_id IN (SELECT id FROM ragdoll_contents WHERE embedding_model = ?)",
+        model
+      )
+    }
+    scope :recent,             -> { order(created_at: :desc) }
+    scope :frequently_used,    -> { where("usage_count > 0").order(usage_count: :desc) }
+    scope :by_chunk_order,     -> { order(:chunk_index) }
+    scope :by_embeddable_type, ->(type) { where(embeddable_type: type) }
+    scope :text_embeddings,    -> { where(embeddable_type: "Ragdoll::TextContent") }
+    scope :image_embeddings,   -> { where(embeddable_type: "Ragdoll::ImageContent") }
+    scope :audio_embeddings,   -> { where(embeddable_type: "Ragdoll::AudioContent") }
+
+    # JSON columns are handled natively by PostgreSQL - no serialization needed
+
+    # Callback for vector column updates (no-op for pgvector)
+    before_save :update_vector_columns
+
+    def embedding_dimensions
+      embedding_vector&.length || 0
+    end
+
+    # Access embedding_model via polymorphic relationship
+    def embedding_model
+      embeddable&.embedding_model
+    end
+
+    def mark_as_used!
+      increment!(:usage_count)
+      update!(returned_at: Time.current)
+    end
+
+    # PostgreSQL pgvector similarity search using neighbor gem
+    def self.search_similar(query_embedding, limit: 20,
+                            threshold: 0.8, filters: {})
+      # Apply filters
+      scope = all
+      scope = scope.where(embeddable_id: filters[:embeddable_id]) if filters[:embeddable_id]
+      scope = scope.where(embeddable_type: filters[:embeddable_type]) if filters[:embeddable_type]
+      scope = scope.by_model(filters[:embedding_model]) if filters[:embedding_model]
+
+      # Document-level filters require joining through embeddable (STI Content) to documents
+      if filters[:document_type]
+        scope = scope.joins("JOIN ragdoll_contents ON ragdoll_contents.id = ragdoll_embeddings.embeddable_id")
+                     .joins("JOIN ragdoll_documents ON ragdoll_documents.id = ragdoll_contents.document_id")
+                     .where("ragdoll_documents.document_type = ?", filters[:document_type])
+      end
+
+      # Use pgvector for similarity search
+      search_with_pgvector(query_embedding, scope, limit, threshold)
+    end
+
+    # Fast search using pgvector with neighbor gem
+    def self.search_with_pgvector(query_embedding, scope, limit, threshold)
+      # Use pgvector for similarity search
+      neighbor_results = scope
+                         .includes(:embeddable)
+                         .nearest_neighbors(:embedding_vector, query_embedding, distance: "cosine")
+                         .limit(limit * 2) # Get more to filter by threshold
+
+      results = []
+      highest_similarity = 0.0
+
+      neighbor_results.each do |embedding|
+        # Calculate cosine similarity (neighbor returns distance, we want similarity)
+        similarity = 1.0 - embedding.neighbor_distance
+
+        highest_similarity = similarity if similarity > highest_similarity
+        next if similarity < threshold
+
+        usage_score = calculate_usage_score(embedding)
+        combined_score = similarity + usage_score
+
+        results << build_result_hash(embedding, query_embedding, similarity, highest_similarity,
+                                     usage_score, combined_score)
+      end
+
+      # Sort by combined score and limit
+      results = results.sort_by { |r| -r[:combined_score] }.take(limit)
+      mark_embeddings_as_used(results)
+      results
+    end
+
+    private
+
+    # Calculate usage score for ranking
+    def self.calculate_usage_score(embedding)
+      usage_score = 0.0
+      if embedding.returned_at && embedding.usage_count.positive?
+        frequency_weight = 0.7
+        recency_weight = 0.3
+
+        frequency_score = [Math.log(embedding.usage_count + 1) / Math.log(100), 1.0].min
+        days_since_use = (Time.current - embedding.returned_at) / 1.day
+        recency_score = Math.exp(-days_since_use / 30)
+
+        usage_score = frequency_weight * frequency_score + recency_weight * recency_score
+      end
+      usage_score
+    end
+
+    # Build standardized result hash
+    def self.build_result_hash(embedding, query_embedding, similarity, highest_similarity, usage_score,
+                               combined_score)
+      {
+        embedding_id: embedding.id.to_s,
+        embeddable_id: embedding.embeddable_id.to_s,
+        embeddable_type: embedding.embeddable_type,
+        document_id: embedding.embeddable&.document_id&.to_s || "Unknown",
+        document_title: embedding.embeddable&.document&.title || "Unknown",
+        document_location: embedding.embeddable&.document&.location || "Unknown",
+        content: embedding.content,
+        similarity: similarity,
+        highest_similarity: highest_similarity,
+        distance: 1.0 - similarity,
+        chunk_index: embedding.chunk_index,
+        embedding_dimensions: query_embedding.length,
+        embedding_model: embedding.embeddable&.embedding_model,
+        usage_count: embedding.usage_count || 0,
+        returned_at: embedding.returned_at,
+        usage_score: usage_score,
+        combined_score: combined_score
+      }
+    end
+
+    # Mark embeddings as used for analytics
+    def self.mark_embeddings_as_used(results)
+      return if results.empty?
+
+      embedding_ids = results.map { |r| r[:embedding_id] }
+      where(id: embedding_ids).update_all(
+        usage_count: arel_table[:usage_count] + 1,
+        returned_at: Time.current
+      )
+    end
+
+    # Callback to update vector columns when embedding_vector changes
+    def update_vector_columns
+      # No additional processing needed for pgvector
+    end
+
+    def self.cosine_similarity(vec1, vec2)
+      return 0.0 if vec1.nil? || vec2.nil? || vec1.length != vec2.length
+
+      dot_product = vec1.zip(vec2).sum { |a, b| a * b }
+      magnitude1 = Math.sqrt(vec1.sum { |a| a * a })
+      magnitude2 = Math.sqrt(vec2.sum { |a| a * a })
+
+      return 0.0 if magnitude1 == 0.0 || magnitude2 == 0.0
+
+      dot_product / (magnitude1 * magnitude2)
+    end
+  end
+end

data/app/models/ragdoll/image_content.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "active_record"
+require_relative "content"
+
+module Ragdoll
+  class ImageContent < Content
+    validate :image_data_or_description_present
+
+    scope :recent, -> { order(created_at: :desc) }
+    scope :with_images, -> { where.not(data: [nil, ""]) }
+    scope :with_descriptions, -> { where.not(content: [nil, ""]) }
+
+    # Image content accessors - content field stores description for embedding
+    def description
+      content
+    end
+
+    def description=(value)
+      self.content = value
+    end
+
+    # Image file data accessor
+    def image_data
+      data
+    end
+
+    def image_data=(value)
+      self.data = value
+    end
+
+    # Image-specific technical metadata (raw file properties)
+    # This metadata is about the actual image file data, not AI-generated insights
+    def alt_text
+      metadata.dig("alt_text")
+    end
+
+    def alt_text=(value)
+      self.metadata = metadata.merge("alt_text" => value)
+    end
+
+    def embedding_count
+      embeddings.count
+    end
+
+    # Image file technical properties (stored in content metadata - raw file data)
+    def image_attached?
+      data.present?
+    end
+
+    def image_size
+      metadata.dig("file_size") || 0
+    end
+
+    def image_size=(value)
+      self.metadata = metadata.merge("file_size" => value)
+    end
+
+    def image_content_type
+      metadata.dig("content_type")
+    end
+
+    def image_content_type=(value)
+      self.metadata = metadata.merge("content_type" => value)
+    end
+
+    def image_filename
+      metadata.dig("filename")
+    end
+
+    def image_filename=(value)
+      self.metadata = metadata.merge("filename" => value)
+    end
+
+    def image_dimensions
+      width = metadata.dig("width")
+      height = metadata.dig("height")
+      return nil unless width && height
+
+      { width: width, height: height }
+    end
+
+    def set_image_dimensions(width, height)
+      self.metadata = metadata.merge("width" => width, "height" => height)
+    end
+
+    # Image format and technical details
+    def color_space
+      metadata.dig("color_space")
+    end
+
+    def color_space=(value)
+      self.metadata = metadata.merge("color_space" => value)
+    end
+
+    def bit_depth
+      metadata.dig("bit_depth")
+    end
+
+    def bit_depth=(value)
+      self.metadata = metadata.merge("bit_depth" => value)
+    end
+
+    # Generate description from image file using LLM vision capabilities
+    def generate_description_from_image!(options = {})
+      return false unless image_attached? || file_path_available?
+
+      begin
+        image_path = get_image_path
+        return false unless image_path
+
+        # Use the image description service
+        description_service = Ragdoll::ImageDescriptionService.new
+
+        generated_description = description_service.generate_description(image_path, options)
+
+        if generated_description.present?
+          self.description = generated_description
+          save!
+          return true
+        end
+
+        false
+      rescue StandardError => e
+        puts "Failed to generate image description: #{e.message}"
+        false
+      end
+    end
+
+    # Override content for embedding to combine description and alt_text
+    def content_for_embedding
+      content_parts = []
+      content_parts << alt_text if alt_text.present?
+      content_parts << description if description.present?
+      content_parts.join(" ")
+    end
+
+    def generate_embeddings!
+      return unless should_generate_embeddings?
+
+      embedding_content = content_for_embedding
+      return if embedding_content.blank?
+
+      # Generate embeddings using the base class method
+      super
+    end
+
+    # Override should_generate_embeddings to check for content
+    def should_generate_embeddings?
+      content_for_embedding.present? && embeddings.empty?
+    end
+
+    def self.stats
+      {
+        total_image_contents: count,
+        by_model: group(:embedding_model).count,
+        total_embeddings: joins(:embeddings).count,
+        with_images: with_images.count,
+        with_descriptions: with_descriptions.count,
+        average_image_size: joins(:image_attachment).average("active_storage_blobs.byte_size")
+      }
+    end
+
+    private
+
+    def file_path_available?
+      document&.location&.present? && File.exist?(document.location)
+    end
+
+    def get_image_path
+      if file_path_available?
+        # Use document location if it's an image file
+        document.location if image_file?(document.location)
+      elsif image_attached?
+        # Try to get path from stored data (if it's a file path)
+        data if data&.start_with?("/")
+      end
+    end
+
+    def image_file?(file_path)
+      return false unless file_path
+
+      image_extensions = %w[.jpg .jpeg .png .gif .bmp .webp .svg .ico .tiff .tif]
+      ext = File.extname(file_path).downcase
+      image_extensions.include?(ext)
+    end
+
+    def image_data_or_description_present
+      return if image_attached? || description.present? || alt_text.present?
+
+      errors.add(:base, "Must have either image data or description/alt_text")
+    end
+  end
+end

data/app/models/ragdoll/text_content.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "active_record"
+require_relative "content"
+
+module Ragdoll
+  class TextContent < Content
+    validates :content, presence: true
+
+    scope :recent, -> { order(created_at: :desc) }
+
+    # Text-specific processing configuration stored in content metadata
+    # This metadata is about the raw content processing, not AI-generated insights
+    def chunk_size
+      metadata.dig('chunk_size') || 1000
+    end
+
+    def chunk_size=(value)
+      self.metadata = metadata.merge('chunk_size' => value)
+    end
+
+    def overlap
+      metadata.dig('overlap') || 200
+    end
+
+    def overlap=(value)
+      self.metadata = metadata.merge('overlap' => value)
+    end
+
+    # Content-specific technical metadata (file processing info)
+    def encoding
+      metadata.dig('encoding')
+    end
+
+    def encoding=(value)
+      self.metadata = metadata.merge('encoding' => value)
+    end
+
+    def line_count
+      metadata.dig('line_count')
+    end
+
+    def line_count=(value)
+      self.metadata = metadata.merge('line_count' => value)
+    end
+
+    def word_count
+      content&.split&.length || 0
+    end
+
+    def character_count
+      content&.length || 0
+    end
+
+    def embedding_count
+      embeddings.count
+    end
+
+    # Text-specific processing methods
+    def chunks
+      return [] if content.blank?
+
+      chunks = []
+      start_pos = 0
+
+      while start_pos < content.length
+        end_pos = [start_pos + chunk_size, content.length].min
+
+        # Try to break at word boundary if not at end
+        if end_pos < content.length
+          last_space = content.rindex(" ", end_pos)
+          end_pos = last_space if last_space && last_space > start_pos
+        end
+
+        chunk_content = content[start_pos...end_pos].strip
+        if chunk_content.present?
+          chunks << {
+            content: chunk_content,
+            start_position: start_pos,
+            end_position: end_pos,
+            chunk_index: chunks.length
+          }
+        end
+
+        break if end_pos >= content.length
+
+        start_pos = [end_pos - overlap, start_pos + 1].max
+      end
+
+      chunks
+    end
+
+    def generate_embeddings!
+      return if content.blank?
+
+      # Clear existing embeddings
+      embeddings.destroy_all
+
+      # Use TextChunker to split content into manageable chunks
+      chunks = Ragdoll::TextChunker.chunk(content)
+
+      # Generate embeddings for each chunk
+      embedding_service = Ragdoll::EmbeddingService.new
+
+      chunks.each_with_index do |chunk_text, index|
+        begin
+          vector = embedding_service.generate_embedding(chunk_text)
+
+          embeddings.create!(
+            content: chunk_text,
+            embedding_vector: vector,
+            chunk_index: index
+          )
+        rescue StandardError => e
+          puts "Failed to generate embedding for chunk #{index}: #{e.message}"
+        end
+      end
+
+      update!(metadata: (metadata || {}).merge("embeddings_generated_at" => Time.current))
+    end
+
+    # Override content for embedding to use the text content
+    def content_for_embedding
+      content
+    end
+
+    def self.stats
+      {
+        total_text_contents:  count,
+        by_model:             group(:embedding_model).count,
+        total_embeddings:     joins(:embeddings).count,
+        average_word_count:   average("LENGTH(content) - LENGTH(REPLACE(content, ' ', '')) + 1"),
+        average_chunk_size:   average(:chunk_size)
+      }
+    end
+  end
+end

data/app/services/ragdoll/configuration_service.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  # Service class for centralized configuration logic
+  # Provides a clean interface for accessing configuration with validation
+  class ConfigurationService
+    def initialize(config = nil)
+      @config = config || Ragdoll.config
+    end
+
+    # Expose config as a public method as well for backward compatibility
+    def config
+      @config
+    end
+
+    # Resolve model for a task with inheritance support
+    def resolve_model(task_type, content_type = :text)
+      case task_type
+      when :embedding
+        @config.embedding_model(content_type)
+      when :summary, :keywords
+        # Check for task-specific model, fall back to default
+        task_model = @config.models.text_generation[task_type]
+        task_model || @config.models.text_generation[:default]
+      else
+        @config.models.text_generation[:default]
+      end
+    end
+
+    # Get provider credentials with fallback to default provider
+    def provider_credentials(provider = nil)
+      provider ||= @config.llm_providers[:default_provider]
+      credentials = @config.llm_providers[provider]
+      
+      if credentials.nil?
+        raise Ragdoll::ConfigurationError, "Provider '#{provider}' not configured"
+      end
+      
+      credentials
+    end
+
+    # Get chunking configuration for content type
+    def chunking_config(content_type = :text)
+      @config.processing[content_type]&.dig(:chunking) || 
+      @config.processing[:default][:chunking]
+    end
+
+    # Get search configuration
+    def search_config
+      @config.processing[:search]
+    end
+
+    # Get prompt template with validation
+    def prompt_template(template_name = :rag_enhancement)
+      template = @config.prompt_templates[template_name]
+      
+      if template.nil?
+        raise Ragdoll::ConfigurationError, "Prompt template '#{template_name}' not found"
+      end
+      
+      template
+    end
+
+    # Validate configuration completeness
+    def validate!
+      errors = []
+
+      # Check required database configuration
+      errors << "Database password not configured" if @config.database[:password].nil?
+      
+      # Check default LLM provider configuration
+      default_provider = @config.llm_providers[:default_provider]
+      if default_provider.nil?
+        errors << "Default LLM provider not specified"
+      else
+        provider_config = @config.llm_providers[default_provider]
+        if provider_config.nil?
+          errors << "Default provider '#{default_provider}' not configured"
+        elsif provider_config[:api_key].nil?
+          errors << "API key for default provider '#{default_provider}' not configured"
+        end
+      end
+
+      # Check embedding configuration
+      if @config.models.embedding[:text].nil?
+        errors << "Text embedding model not configured"
+      end
+
+      # Ensure log directory can be created
+      log_dir = File.dirname(@config.logging[:filepath])
+      unless Dir.exist?(log_dir) || File.writable?(File.dirname(log_dir))
+        errors << "Cannot create log directory '#{log_dir}'"
+      end
+
+      unless errors.empty?
+        raise Ragdoll::ConfigurationError, "Configuration validation failed:\n  - #{errors.join("\n  - ")}"
+      end
+
+      true
+    end
+
+    # Check if configuration is valid without raising
+    def valid?
+      validate!
+      true
+    rescue Ragdoll::ConfigurationError
+      false
+    end
+
+    # Expose config for access
+    attr_reader :config
+  end
+end