ragdoll-rails 0.0.1 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ac0f85d4a125cc90ca0a5187ae327eeb1c88de8380c3a5929ec79c52d7db54a
-  data.tar.gz: 7eb055ea9faecbdbdb5a51cce6256ed5350de814447de02d20d09e6a7dd3a86f
+  metadata.gz: c65d1b42c610da8ca427a88eae05c3e7636d2b3521a2c0ecccf09efc55af335d
+  data.tar.gz: d1ca9240447923aeeb896c57da732d92b874fa3ebf2645306d74f5ee435e642d
 SHA512:
-  metadata.gz: 9e52d30d90f0641b02f7b32fab536f3465666c38ab06d9008ae90a67c0fec9e8037614049adb63479607dad320d15e6a51d32e469eca7a80b4439029c5aa031a
-  data.tar.gz: a5397b0ae44775dd35bddf5a4551bfc456f6c856958695ee9fae0da0117b4ccfef89c612ca0e9182eb310866b61c0b42d5e7302b2160186c6144297a075239d6
+  metadata.gz: e656136397c64bbb15923dea45d071dc7ea8a71f3d8334ea7ed69ee1e08d2c067b0358af02e996f822f8fdf57908bfbc47de75aadebdbbe811e5273b242b0594
+  data.tar.gz: 1ca6e7f05d39f3135d2623cb3d2c12a6d7822553ed462f991631e4b3d05be6b0e77e667913b72219f08ce06145937b42f9366fa803cb92133adce6221c621cee

data/lib/ragdoll/rails/configuration.rb

@@ -21,13 +21,5 @@ module Ragdoll
       end
     end
 
-    def self.configuration
-      @configuration ||= Configuration.new
-    end
-
-    def self.configure
-      yield(configuration)
-      configuration.configure_core
-    end
   end
 end

data/lib/ragdoll/rails/version.rb

@@ -4,6 +4,6 @@
 
 module Ragdoll
   module Rails
-    VERSION = "0.0.1"
+    VERSION = "0.1.8"
   end
 end

data/lib/ragdoll/rails.rb

@@ -2,7 +2,12 @@
 
 require_relative 'rails/version'
 require_relative 'rails/configuration'
-require_relative 'rails/engine'
+begin
+  require_relative 'rails/engine'
+rescue LoadError, NameError => e
+  # Skip engine loading if Rails is not available (e.g., in tests)
+  puts "Warning: Could not load Rails engine: #{e.message}" if ENV['RAGDOLL_VERBOSE']
+end
 
 module Ragdoll
   module Rails

data/lib/ragdoll-rails.rb

@@ -6,6 +6,7 @@ require_relative "ragdoll/rails"
 # The ragdoll gem provides the core business logic functionality
 begin
   require 'ragdoll'
-rescue LoadError
-  raise LoadError, "The ragdoll gem is required for ragdoll-rails to function. Please add 'gem \"ragdoll\"' to your Gemfile."
+rescue LoadError => e
+  puts "Warning: Could not load ragdoll gem: #{e.message}"
+  puts "Please ensure 'gem \"ragdoll\"' is in your Gemfile"
 end

metadata

@@ -1,14 +1,42 @@
 --- !ruby/object:Gem::Specification
 name: ragdoll-rails
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.8
 platform: ruby
 authors:
 - Dewayne VanHoozer
 bindir: bin
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
-dependencies: []
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ragdoll
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.0
 description: Rails engine providing ActiveRecord integration, background jobs, and
   UI components for the Ragdoll RAG (Retrieval-Augmented Generation) system
 email:
@@ -19,17 +47,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - Rakefile
-- app/models/ragdoll/document.rb
-- app/models/ragdoll/embedding.rb
-- app/models/ragdoll/search.rb
 - config/initializers/ragdoll.rb
 - config/routes.rb
-- db/migrate/20250218123456_create_documents.rb
-- db/migrate/20250219123456_create_ragdoll_embeddings.rb
-- db/migrate/20250220123456_update_embeddings_vector_column.rb
-- db/migrate/20250223123457_add_metadata_and_foreign_key_to_ragdoll_tables.rb
-- db/migrate/20250225123456_add_summary_to_ragdoll_documents.rb
-- db/migrate/20250226123456_add_usage_tracking_to_ragdoll_embeddings.rb
 - lib/generators/ragdoll/init/init_generator.rb
 - lib/generators/ragdoll/init/templates/INSTALL
 - lib/generators/ragdoll/init/templates/ragdoll_config.rb

data/app/models/ragdoll/document.rb

@@ -1,120 +0,0 @@
-# This file defines the Rails-specific Document model for the Ragdoll Rails engine.
-# This model is separate from Ragdoll::Core::Models::Document to avoid conflicts.
-
-# frozen_string_literal: true
-
-module Ragdoll
-  module Rails
-    class Document < ApplicationRecord
-      self.table_name = 'ragdoll_documents'
-    
-    # Associations
-    has_many :ragdoll_embeddings, class_name: 'Ragdoll::Rails::Embedding', foreign_key: 'document_id', dependent: :destroy
-    has_one_attached :file if respond_to?(:has_one_attached)
-    
-    # Validations
-    validates :location, presence: true, uniqueness: true
-    validates :status, inclusion: { in: %w[pending processing completed failed] }
-    validates :chunk_size, numericality: { greater_than: 0 }, allow_nil: true
-    validates :chunk_overlap, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true
-    
-    # Scopes
-    scope :completed, -> { where(status: 'completed') }
-    scope :failed, -> { where(status: 'failed') }
-    scope :processing, -> { where(status: 'processing') }
-    scope :pending, -> { where(status: 'pending') }
-    scope :by_type, ->(type) { where(document_type: type) }
-    scope :with_summaries, -> { where.not(summary: nil) }
-    scope :needs_summary, -> { where(summary: nil).completed }
-    
-    # Search configuration
-    searchkick text_middle: [:title, :summary, :content, :metadata_name, :metadata_summary] if defined?(Searchkick)
-
-    def search_data
-      return {} unless defined?(Searchkick)
-      
-      {
-        title: title,
-        summary: summary,
-        content: content,
-        metadata_name: metadata&.dig('name'),
-        metadata_summary: metadata&.dig('summary'),
-        document_type: document_type,
-        status: status
-      }
-    end
-    
-    # Summary-related methods
-    def has_summary?
-      summary.present?
-    end
-    
-    def summary_stale?
-      return false unless has_summary?
-      return true unless summary_generated_at
-      
-      # Consider summary stale if document was updated after summary generation
-      updated_at > summary_generated_at
-    end
-    
-    def needs_summary?
-      return false unless content.present?
-      # Business logic should be handled by ragdoll gem
-      # TODO: Delegate to Ragdoll.needs_summary?(content, summary, summary_generated_at)
-      
-      !has_summary? || summary_stale?
-    end
-    
-    def summary_word_count
-      return 0 unless summary.present?
-      summary.split.length
-    end
-    
-    def regenerate_summary!
-      # Business logic for summary generation should be handled by the ragdoll gem
-      # This is a placeholder that delegates to the core ragdoll functionality
-      return false unless content.present?
-      
-      # TODO: Delegate to Ragdoll gem's summarization functionality
-      # summarization_result = Ragdoll.generate_summary(content, options)
-      # Update the model with the result
-      
-      Rails.logger.warn "Summary regeneration not implemented - should delegate to ragdoll gem"
-      false
-    end
-    
-    # Processing status helpers
-    def completed?
-      status == 'completed'
-    end
-    
-    def failed?
-      status == 'failed'
-    end
-    
-    def processing?
-      status == 'processing'
-    end
-    
-    def pending?
-      status == 'pending'
-    end
-    
-    # Content helpers
-    def word_count
-      return 0 unless content.present?
-      content.split.length
-    end
-    
-    def character_count
-      return 0 unless content.present?
-      content.length
-    end
-    
-    def processing_duration
-      return nil unless processing_started_at && processing_finished_at
-      processing_finished_at - processing_started_at
-    end
-    end
-  end
-end

data/app/models/ragdoll/embedding.rb

@@ -1,31 +0,0 @@
-# This file defines the Rails-specific Embedding model for the Ragdoll Rails engine.
-# This model is separate from Ragdoll::Core::Models::Embedding to avoid conflicts.
-
-# frozen_string_literal: true
-
-module Ragdoll
-  module Rails
-    class Embedding < ApplicationRecord
-      searchkick text_middle: [:metadata_content, :metadata_propositions] if defined?(Searchkick)
-
-    belongs_to :document, class_name: 'Ragdoll::Rails::Document'
-
-    # Override dangerous attribute to allow access to model_name column
-    def self.dangerous_attribute_method?(name)
-      name.to_s == 'model_name' ? false : super
-    end
-
-    def search_data
-      return {} unless defined?(Searchkick)
-      
-      {
-        metadata_content: metadata['content'],
-        metadata_propositions: metadata['propositions']
-      }
-    end
-
-    # Assuming the vector column is named 'vector'
-    neighbor :vector, method: :euclidean if respond_to?(:neighbor)
-    end
-  end
-end

data/app/models/ragdoll/search.rb

@@ -1,201 +0,0 @@
-# frozen_string_literal: true
-
-module Ragdoll
-  module Rails
-    class Search < ApplicationRecord
-      self.table_name = 'ragdoll_searches'
-
-    # Override dangerous attribute to allow access to model_name column
-    def self.dangerous_attribute_method?(name)
-      name.to_s == 'model_name' ? false : super
-    end
-
-    # Validations
-    validates :query, presence: true, length: { minimum: 1, maximum: 10000 }
-    validates :search_type, presence: true, inclusion: { in: %w[semantic keyword hybrid] }
-    validates :result_count, presence: true, numericality: { greater_than_or_equal_to: 0 }
-    validates :search_time, numericality: { greater_than: 0 }, allow_nil: true
-    validates :model_name, presence: true, length: { maximum: 255 }
-
-    # Scopes
-    scope :recent, -> { order(created_at: :desc) }
-    scope :by_type, ->(type) { where(search_type: type) }
-    scope :successful, -> { where('result_count > 0') }
-    scope :failed, -> { where(result_count: 0) }
-    scope :by_model, ->(model) { where(model_name: model) }
-    scope :within_days, ->(days) { where(created_at: days.days.ago..) }
-    scope :slow_searches, ->(threshold = 2.0) { where('search_time > ?', threshold) }
-
-    # Callbacks
-    before_validation :set_defaults
-    before_save :normalize_query
-    after_create :update_analytics, if: -> { Ragdoll.configuration.enable_search_analytics }
-
-    # Class methods
-    def self.analytics(days: 30)
-      searches = within_days(days)
-      
-      {
-        total_searches: searches.count,
-        unique_queries: searches.distinct.count(:query),
-        average_results: searches.average(:result_count)&.round(2) || 0,
-        average_search_time: searches.where.not(search_time: nil).average(:search_time)&.round(3) || 0,
-        success_rate: calculate_success_rate(searches),
-        most_common_queries: most_common_queries(searches),
-        search_types: searches.group(:search_type).count,
-        models_used: searches.group(:model_name).count,
-        performance_stats: performance_statistics(searches)
-      }
-    end
-
-    def self.most_common_queries(searches = all, limit: 10)
-      searches
-        .group(:query)
-        .count
-        .sort_by { |_, count| -count }
-        .first(limit)
-        .map { |query, count| { query: query, count: count } }
-    end
-
-    def self.calculate_success_rate(searches = all)
-      total = searches.count
-      return 0 if total == 0
-      
-      successful_count = searches.successful.count
-      (successful_count.to_f / total * 100).round(2)
-    end
-
-    def self.performance_statistics(searches = all)
-      searches_with_time = searches.where.not(search_time: nil)
-      return {} if searches_with_time.empty?
-
-      times = searches_with_time.pluck(:search_time).sort
-      
-      {
-        fastest: times.first,
-        slowest: times.last,
-        median: calculate_median(times),
-        percentile_95: calculate_percentile(times, 95),
-        slow_search_count: searches.slow_searches.count
-      }
-    end
-
-    def self.calculate_median(sorted_array)
-      length = sorted_array.length
-      return 0 if length == 0
-      
-      if length.odd?
-        sorted_array[length / 2]
-      else
-        (sorted_array[length / 2 - 1] + sorted_array[length / 2]) / 2.0
-      end
-    end
-
-    def self.calculate_percentile(sorted_array, percentile)
-      return 0 if sorted_array.empty?
-      
-      index = (percentile / 100.0 * (sorted_array.length - 1)).round
-      sorted_array[index]
-    end
-
-    # Instance methods
-    def successful?
-      result_count > 0
-    end
-
-    def failed?
-      result_count == 0
-    end
-
-    def slow?(threshold = 2.0)
-      search_time && search_time > threshold
-    end
-
-    def embedding_vector
-      return nil unless query_embedding
-      
-      if query_embedding.is_a?(String)
-        # Handle string representation of vector
-        JSON.parse(query_embedding)
-      else
-        query_embedding
-      end
-    rescue JSON::ParserError
-      nil
-    end
-
-    def result_ids
-      return [] unless results.is_a?(Hash)
-      
-      results['result_ids'] || results[:result_ids] || []
-    end
-
-    def filter_summary
-      return 'None' if filters.blank?
-      
-      filter_parts = []
-      filters.each do |key, value|
-        filter_parts << "#{key}: #{value}"
-      end
-      
-      filter_parts.join(', ')
-    end
-
-    def performance_category
-      return 'unknown' unless search_time
-      
-      case search_time
-      when 0..0.5
-        'fast'
-      when 0.5..1.0
-        'normal'
-      when 1.0..2.0
-        'slow'
-      else
-        'very_slow'
-      end
-    end
-
-    def to_analytics_hash
-      {
-        id: id,
-        query: query,
-        search_type: search_type,
-        result_count: result_count,
-        search_time: search_time,
-        performance_category: performance_category,
-        successful: successful?,
-        model_name: model_name,
-        filters: filter_summary,
-        created_at: created_at
-      }
-    end
-
-    private
-
-    def set_defaults
-      self.search_type ||= 'semantic'
-      self.result_count ||= 0
-      self.filters ||= {}
-      self.results ||= {}
-      self.model_name ||= Ragdoll.configuration.embedding_model
-    end
-
-    def normalize_query
-      return unless query
-      
-      # Remove excessive whitespace
-      self.query = query.strip.gsub(/\s+/, ' ')
-      
-      # Truncate if too long
-      self.query = query.truncate(10000) if query.length > 10000
-    end
-
-    def update_analytics
-      # This could be extended to update real-time analytics
-      # For now, it's just a placeholder for future enhancements
-      Rails.logger.debug "Search recorded: #{query} (#{result_count} results in #{search_time}s)"
-    end
-    end
-  end
-end

data/db/migrate/20250218123456_create_documents.rb

@@ -1,46 +0,0 @@
-# Creates the core documents table for the Ragdoll RAG (Retrieval-Augmented Generation) system.
-# This table stores document metadata, content, and processing status information.
-class CreateRagdollDocuments < ActiveRecord::Migration[8.0]
-  def change
-    # Enable PostgreSQL extensions required for advanced text processing and vector operations
-    enable_extension 'pg_trgm' unless extension_enabled?('pg_trgm')        # Trigram matching for fuzzy text search
-    enable_extension 'fuzzystrmatch' unless extension_enabled?('fuzzystrmatch') # Fuzzy string matching algorithms
-    enable_extension 'vector' unless extension_enabled?('vector')          # Vector operations for embedding similarity
-
-    create_table :ragdoll_documents, comment: 'Core documents table storing files and content for RAG processing. Each row represents a document that can be chunked into embeddings for semantic search and AI retrieval.' do |t|
-      # File system reference - where the original document is stored
-      t.string :location, null: false, comment: 'File system path or URL to the original document. Required field that uniquely identifies the source location of the document for re-processing or reference.'
-      
-      # Document content and derived data
-      t.text :content, comment: 'Full extracted text content of the document. This is the raw text that will be chunked and embedded for semantic search. May be large for documents like PDFs or web pages.'
-      t.text :summary, comment: 'AI-generated summary of the document content. Created during processing to provide quick overviews and improve search relevance. Generated by summarization models.'
-      
-      # Document classification and metadata
-      t.string :document_type, comment: 'Classification of the document type (e.g., "pdf", "text", "markdown", "html"). Used for applying type-specific processing rules and display formatting.'
-      t.string :title, comment: 'Human-readable title of the document. May be extracted from filename, document metadata, or content analysis. Used for display and identification purposes.'
-      t.string :source_type, comment: 'Origin source of the document (e.g., "file", "url", "api", "upload"). Helps track how the document entered the system for auditing and re-processing.'
-      
-      # Chunking configuration - controls how document is split for embedding
-      t.integer :chunk_size, comment: 'Number of characters per chunk when splitting document for embedding. Larger chunks capture more context but may exceed model limits. Typical values: 500-2000 characters.'
-      t.integer :chunk_overlap, comment: 'Number of characters to overlap between consecutive chunks. Prevents context loss at chunk boundaries. Typically 10-20% of chunk_size to maintain coherence.'
-      
-      # Flexible metadata storage for document-specific information
-      t.jsonb :metadata, default: {}, comment: 'Flexible JSON storage for document-specific metadata such as author, creation date, file size, extraction settings, or custom tags. Indexed with GIN for efficient querying.'
-      
-      # Processing lifecycle tracking
-      t.datetime :processing_started_at, comment: 'Timestamp when document processing (chunking and embedding generation) began. Used for monitoring processing duration and identifying stuck jobs.'
-      t.datetime :processing_finished_at, comment: 'Timestamp when document processing completed successfully or failed. Used with started_at to calculate processing time and identify performance issues.'
-      t.string :status, default: 'pending', comment: 'Current processing status of the document. Values: "pending" (not yet processed), "processing" (currently being processed), "processed" (successfully completed), "failed" (processing error occurred). Used for job queue management and user feedback.'
-
-      # Standard Rails timestamps for audit trails
-      t.timestamps comment: 'Standard Rails created_at and updated_at timestamps for tracking when document records are created and modified in the database.'
-    end
-
-    # Indexes for performance optimization
-    add_index :ragdoll_documents, :location, unique: true, comment: 'Unique constraint on location prevents duplicate documents and enables fast lookups by file path or URL'
-    add_index :ragdoll_documents, :document_type, comment: 'Index for filtering documents by type, commonly used in admin interfaces and type-specific processing queries'
-    add_index :ragdoll_documents, :status, comment: 'Index for filtering by processing status, critical for job queue management and monitoring dashboard queries'
-    add_index :ragdoll_documents, :metadata, using: :gin, comment: 'GIN index on JSONB metadata field enables efficient queries on nested JSON properties and full-text search within metadata'
-    add_index :ragdoll_documents, :processing_started_at, comment: 'Index for sorting and filtering by processing start time, used for monitoring processing queues and performance analysis'
-  end
-end

data/db/migrate/20250219123456_create_ragdoll_embeddings.rb

@@ -1,41 +0,0 @@
-# Creates the embeddings table for storing document chunks and their vector representations.
-# This table is the core of the RAG system's semantic search capabilities.
-class CreateRagdollEmbeddings < ActiveRecord::Migration[8.0]
-  def change
-    create_table :ragdoll_embeddings, comment: 'Stores document chunks and their vector embeddings for semantic search. Each row represents a chunk of text from a document along with its AI-generated vector representation for similarity matching.' do |t|
-      # Parent document reference - establishes the chunk-to-document relationship
-      t.references :document, null: false, foreign_key: { to_table: :ragdoll_documents }, comment: 'Foreign key reference to the parent document in ragdoll_documents table. Required field that links each chunk back to its source document for context and retrieval.'
-      
-      # The actual text content of this chunk
-      t.text :content, null: false, comment: 'The actual text content of this chunk extracted from the parent document. Required field containing the text that will be used for semantic matching and returned in search results. Typically 500-2000 characters.'
-      
-      # Vector embedding for semantic search (pgvector format for optimal similarity calculations)
-      t.vector :embedding, limit: 1536, comment: 'High-dimensional vector representation of the content generated by AI embedding models. Stored in pgvector format for efficient cosine similarity calculations. Dimension typically 1536 for OpenAI models.'
-      
-      # Model identification for compatibility and reprocessing
-      t.string :model_name, comment: 'Name or identifier of the AI model used to generate this embedding (e.g., "text-embedding-3-small", "sentence-transformers/all-MiniLM-L6-v2"). Critical for ensuring embedding compatibility during search operations.'
-      
-      # Token usage tracking for cost analysis and optimization
-      t.integer :token_count, comment: 'Number of tokens consumed by the AI model when generating this embedding. Used for cost tracking, billing analysis, and optimizing chunk sizes to minimize API costs while maintaining quality.'
-      
-      # Chunk ordering within the parent document
-      t.integer :chunk_index, comment: 'Sequential position of this chunk within the parent document (0-based). Used for maintaining document order, reconstructing original text flow, and providing contextual information in search results.'
-      
-      # Flexible storage for chunk-specific metadata
-      t.jsonb :metadata, default: {}, comment: 'Flexible JSON storage for chunk-specific metadata such as section headers, page numbers, processing parameters, or semantic tags. Indexed with GIN for efficient querying of nested properties.'
-      
-      # Classification of embedding content type
-      t.string :embedding_type, default: 'text', comment: 'Type of content that was embedded (e.g., "text", "code", "table", "heading"). Allows for type-specific search strategies and filtering. Default "text" covers most document content.'
-
-      # Standard Rails timestamps for audit trails
-      t.timestamps comment: 'Standard Rails created_at and updated_at timestamps for tracking when embedding records are created and modified. Critical for debugging and performance analysis.'
-    end
-
-    # Performance indexes for efficient querying
-    add_index :ragdoll_embeddings, :document_id, comment: 'Index for efficiently retrieving all chunks belonging to a specific document. Essential for document reprocessing and chunk management operations.'
-    add_index :ragdoll_embeddings, :chunk_index, comment: 'Index for sorting chunks by their position within documents. Used for maintaining document order and providing sequential context in search results.'
-    add_index :ragdoll_embeddings, :embedding_type, comment: 'Index for filtering embeddings by content type. Enables type-specific search strategies (e.g., searching only code chunks or headings) and analytics on content distribution.'
-    add_index :ragdoll_embeddings, :metadata, using: :gin, comment: 'GIN index on JSONB metadata field enables efficient queries on nested JSON properties and supports complex filtering on chunk-specific attributes.'
-    add_index :ragdoll_embeddings, :embedding, using: :hnsw, opclass: :vector_cosine_ops, comment: 'Hierarchical Navigable Small World (HNSW) index optimized for cosine similarity searches on vector embeddings. Critical for fast semantic search performance at scale.'
-  end
-end

data/db/migrate/20250220123456_update_embeddings_vector_column.rb

@@ -1,41 +0,0 @@
-# Updates the embeddings table to support variable vector dimensions and optimize queries.
-# Originally intended to convert to pgvector format but maintains text compatibility for broader database support.
-class UpdateEmbeddingsVectorColumn < ActiveRecord::Migration[8.0]
-  def up
-    # Remove the limit constraint to allow variable length vectors
-    # Different AI models produce embeddings with different dimensions (e.g., 1536 for OpenAI, 384 for some sentence transformers)
-    change_column :ragdoll_embeddings, :embedding, :vector, limit: nil, comment: 'High-dimensional vector representation with variable dimensions removed to support multiple embedding models. Allows for mixing different model outputs while maintaining pgvector compatibility.'
-    
-    # Add column to track embedding vector dimensions for validation and compatibility
-    add_column :ragdoll_embeddings, :embedding_dimensions, :integer, comment: 'Number of dimensions in the embedding vector (e.g., 1536 for OpenAI text-embedding-3-small). Critical for ensuring embedding compatibility during similarity searches and preventing dimension mismatches.'
-    
-    # Update existing records to set their dimensions based on actual vector data
-    # This ensures data integrity for existing embeddings
-    execute <<~SQL
-      UPDATE ragdoll_embeddings 
-      SET embedding_dimensions = array_length(embedding::real[], 1) 
-      WHERE embedding IS NOT NULL
-    SQL
-    
-    # Add index on embedding_dimensions for efficient filtering by vector size
-    # Used when searching embeddings to ensure only compatible vectors are compared
-    add_index :ragdoll_embeddings, :embedding_dimensions, comment: 'Index for filtering embeddings by vector dimension size. Essential for ensuring only compatible embeddings are compared during similarity searches and avoiding dimension mismatch errors.'
-    
-    # Add composite index on model_name and embedding_dimensions for optimized similarity searches
-    # This combination is frequently queried together when finding similar embeddings
-    add_index :ragdoll_embeddings, [:model_name, :embedding_dimensions], 
-              name: 'index_ragdoll_embeddings_on_model_and_dimensions',
-              comment: 'Composite index on model name and vector dimensions. Optimizes the common query pattern of finding embeddings from the same model with matching dimensions for similarity calculations.'
-  end
-
-  def down
-    # Remove the new columns and indexes in reverse order
-    remove_index :ragdoll_embeddings, :embedding_dimensions
-    remove_index :ragdoll_embeddings, name: 'index_ragdoll_embeddings_on_model_and_dimensions'
-    remove_column :ragdoll_embeddings, :embedding_dimensions
-    
-    # Note: The original plan was to restore vector type, but this would fail with mixed dimensions
-    # Restore the original limit (this will fail if there are vectors with different dimensions)
-    change_column :ragdoll_embeddings, :embedding, :vector, limit: 1536
-  end
-end

data/db/migrate/20250223123457_add_metadata_and_foreign_key_to_ragdoll_tables.rb

@@ -1,37 +0,0 @@
-# Creates the searches table for tracking user search queries and analytics.
-# This table enables search analytics, caching, and performance monitoring for the RAG system.
-class CreateRagdollSearches < ActiveRecord::Migration[8.0]
-  def change
-    create_table :ragdoll_searches, comment: 'Tracks user search queries and results for analytics and performance monitoring. Each row represents a search performed by a user, storing both the query and metadata about results.' do |t|
-      # User's original search query
-      t.text :query, null: false, comment: 'The original search query text entered by the user. Required field that captures the exact search terms for analytics, popular queries tracking, and search result caching.'
-      
-      # Embedding of the search query for similarity calculations
-      t.vector :query_embedding, limit: 1536, comment: 'Vector embedding of the search query. Generated using the same model as document embeddings to enable semantic similarity calculations. Stored in pgvector format for efficient similarity operations.'
-      
-      # Search classification and method
-      t.string :search_type, default: 'semantic', comment: 'Type of search performed (e.g., "semantic", "keyword", "hybrid"). Allows for different search strategies and helps analyze which search methods are most effective for users.'
-      
-      # Search parameters and constraints
-      t.jsonb :filters, default: {}, comment: 'JSON object containing search filters applied (e.g., document_type, date_range, status). Used for analyzing how users refine searches and for recreating search contexts.'
-      
-      # Search results metadata  
-      t.jsonb :results, default: {}, comment: 'JSON object containing search result metadata such as result IDs, similarity scores, and ranking information. Enables search result caching and detailed analytics on result quality.'
-      t.integer :result_count, default: 0, comment: 'Number of results returned by this search. Used for analytics on search effectiveness and identifying queries that return too few or too many results.'
-      
-      # Performance tracking
-      t.float :search_time, comment: 'Time in seconds taken to execute this search query. Critical for performance monitoring, identifying slow queries, and optimizing search algorithms.'
-      
-      # Model identification for compatibility
-      t.string :model_name, comment: 'Name of the AI model used to generate the query embedding (e.g., "text-embedding-3-small"). Ensures search compatibility and enables analytics by model performance.'
-
-      # Standard Rails timestamps
-      t.timestamps comment: 'Standard Rails created_at and updated_at timestamps. The created_at timestamp is particularly important for search analytics and identifying search patterns over time.'
-    end
-
-    # Indexes for analytics and performance
-    add_index :ragdoll_searches, :search_type, comment: 'Index for filtering searches by type, used in analytics dashboards to compare effectiveness of different search strategies.'
-    add_index :ragdoll_searches, :query_embedding, using: :hnsw, opclass: :vector_cosine_ops, comment: 'HNSW index on query embeddings for finding similar queries. Enables query recommendation, duplicate detection, and search clustering analysis.'
-    add_index :ragdoll_searches, :created_at, comment: 'Index for time-based queries and analytics. Essential for generating search trend reports, popular queries over time, and performance monitoring.'
-  end
-end

data/db/migrate/20250225123456_add_summary_to_ragdoll_documents.rb

@@ -1,17 +0,0 @@
-# Adds summary metadata tracking to documents table for AI-generated summaries.
-# Enhances the existing summary field with provenance and timing information.
-class AddSummaryMetadataToRagdollDocuments < ActiveRecord::Migration[8.0]
-  def change
-    # Summary field already exists from initial migration, just add metadata fields for tracking
-    
-    # Timestamp tracking for summary generation
-    add_column :ragdoll_documents, :summary_generated_at, :timestamp, comment: 'Timestamp when the AI-generated summary was created. Used for cache invalidation, determining summary freshness, and analytics on summary generation performance.'
-    
-    # Model identification for summary provenance
-    add_column :ragdoll_documents, :summary_model, :string, comment: 'Name/identifier of the AI model used to generate the summary (e.g., "gpt-3.5-turbo", "claude-3-haiku"). Critical for tracking summary quality, cost analysis, and ensuring consistency in summary style.'
-    
-    # Indexes for summary metadata queries and analytics
-    add_index :ragdoll_documents, :summary_generated_at, comment: 'Index for filtering and sorting documents by summary generation time. Used for cache invalidation queries and identifying documents with stale summaries that need regeneration.'
-    add_index :ragdoll_documents, :summary_model, comment: 'Index for filtering documents by the AI model used for summarization. Enables analytics on summary quality by model and batch re-summarization with newer models.'
-  end
-end

data/db/migrate/20250226123456_add_usage_tracking_to_ragdoll_embeddings.rb

@@ -1,28 +0,0 @@
-# Adds usage tracking functionality to embeddings for intelligent ranking and caching.
-# Enables the system to learn which embeddings are most valuable and prioritize them in search results.
-class AddUsageTrackingToRagdollEmbeddings < ActiveRecord::Migration[8.0]
-  def change
-    # Usage frequency tracking
-    add_column :ragdoll_embeddings, :usage_count, :integer, default: 0, null: false, comment: 'Number of times this embedding has been returned in search results. Incremented each time the embedding appears in search results. Used for frequency-based ranking to surface more relevant content.'
-    
-    # Recency tracking for temporal relevance
-    add_column :ragdoll_embeddings, :returned_at, :timestamp, comment: 'Timestamp of the most recent time this embedding was returned in search results. Used for recency-based ranking algorithms and cache warming strategies. NULL indicates never used.'
-    
-    # Performance indexes for usage-based ranking queries
-    add_index :ragdoll_embeddings, :usage_count, name: 'index_ragdoll_embeddings_on_usage_count', comment: 'Index for sorting embeddings by usage frequency. Critical for popularity-based ranking algorithms and identifying most/least used content.'
-    add_index :ragdoll_embeddings, :returned_at, name: 'index_ragdoll_embeddings_on_returned_at', comment: 'Index for sorting embeddings by recency of use. Enables temporal ranking algorithms and cache warming strategies based on recent usage patterns.'
-    
-    # Composite index for advanced ranking algorithms that combine frequency and recency
-    add_index :ragdoll_embeddings, [:usage_count, :returned_at], 
-              name: 'index_ragdoll_embeddings_on_usage_and_recency',
-              comment: 'Composite index for complex ranking algorithms that combine usage frequency with recency. Optimizes queries that balance popular content with recently accessed content for intelligent search result ranking.'
-    
-    # Data migration to ensure existing records have proper default values
-    # Usage count defaults to 0, returned_at remains null until first usage
-    reversible do |dir|
-      dir.up do
-        execute "UPDATE ragdoll_embeddings SET usage_count = 0 WHERE usage_count IS NULL"
-      end
-    end
-  end
-end