ragdoll-rails 0.0.1

data/app/models/ragdoll/search.rb

@@ -0,0 +1,201 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Rails
+    class Search < ApplicationRecord
+      self.table_name = 'ragdoll_searches'
+
+    # Override dangerous attribute to allow access to model_name column
+    def self.dangerous_attribute_method?(name)
+      name.to_s == 'model_name' ? false : super
+    end
+
+    # Validations
+    validates :query, presence: true, length: { minimum: 1, maximum: 10000 }
+    validates :search_type, presence: true, inclusion: { in: %w[semantic keyword hybrid] }
+    validates :result_count, presence: true, numericality: { greater_than_or_equal_to: 0 }
+    validates :search_time, numericality: { greater_than: 0 }, allow_nil: true
+    validates :model_name, presence: true, length: { maximum: 255 }
+
+    # Scopes
+    scope :recent, -> { order(created_at: :desc) }
+    scope :by_type, ->(type) { where(search_type: type) }
+    scope :successful, -> { where('result_count > 0') }
+    scope :failed, -> { where(result_count: 0) }
+    scope :by_model, ->(model) { where(model_name: model) }
+    scope :within_days, ->(days) { where(created_at: days.days.ago..) }
+    scope :slow_searches, ->(threshold = 2.0) { where('search_time > ?', threshold) }
+
+    # Callbacks
+    before_validation :set_defaults
+    before_save :normalize_query
+    after_create :update_analytics, if: -> { Ragdoll.configuration.enable_search_analytics }
+
+    # Class methods
+    def self.analytics(days: 30)
+      searches = within_days(days)
+      
+      {
+        total_searches: searches.count,
+        unique_queries: searches.distinct.count(:query),
+        average_results: searches.average(:result_count)&.round(2) || 0,
+        average_search_time: searches.where.not(search_time: nil).average(:search_time)&.round(3) || 0,
+        success_rate: calculate_success_rate(searches),
+        most_common_queries: most_common_queries(searches),
+        search_types: searches.group(:search_type).count,
+        models_used: searches.group(:model_name).count,
+        performance_stats: performance_statistics(searches)
+      }
+    end
+
+    def self.most_common_queries(searches = all, limit: 10)
+      searches
+        .group(:query)
+        .count
+        .sort_by { |_, count| -count }
+        .first(limit)
+        .map { |query, count| { query: query, count: count } }
+    end
+
+    def self.calculate_success_rate(searches = all)
+      total = searches.count
+      return 0 if total == 0
+      
+      successful_count = searches.successful.count
+      (successful_count.to_f / total * 100).round(2)
+    end
+
+    def self.performance_statistics(searches = all)
+      searches_with_time = searches.where.not(search_time: nil)
+      return {} if searches_with_time.empty?
+
+      times = searches_with_time.pluck(:search_time).sort
+      
+      {
+        fastest: times.first,
+        slowest: times.last,
+        median: calculate_median(times),
+        percentile_95: calculate_percentile(times, 95),
+        slow_search_count: searches.slow_searches.count
+      }
+    end
+
+    def self.calculate_median(sorted_array)
+      length = sorted_array.length
+      return 0 if length == 0
+      
+      if length.odd?
+        sorted_array[length / 2]
+      else
+        (sorted_array[length / 2 - 1] + sorted_array[length / 2]) / 2.0
+      end
+    end
+
+    def self.calculate_percentile(sorted_array, percentile)
+      return 0 if sorted_array.empty?
+      
+      index = (percentile / 100.0 * (sorted_array.length - 1)).round
+      sorted_array[index]
+    end
+
+    # Instance methods
+    def successful?
+      result_count > 0
+    end
+
+    def failed?
+      result_count == 0
+    end
+
+    def slow?(threshold = 2.0)
+      search_time && search_time > threshold
+    end
+
+    def embedding_vector
+      return nil unless query_embedding
+      
+      if query_embedding.is_a?(String)
+        # Handle string representation of vector
+        JSON.parse(query_embedding)
+      else
+        query_embedding
+      end
+    rescue JSON::ParserError
+      nil
+    end
+
+    def result_ids
+      return [] unless results.is_a?(Hash)
+      
+      results['result_ids'] || results[:result_ids] || []
+    end
+
+    def filter_summary
+      return 'None' if filters.blank?
+      
+      filter_parts = []
+      filters.each do |key, value|
+        filter_parts << "#{key}: #{value}"
+      end
+      
+      filter_parts.join(', ')
+    end
+
+    def performance_category
+      return 'unknown' unless search_time
+      
+      case search_time
+      when 0..0.5
+        'fast'
+      when 0.5..1.0
+        'normal'
+      when 1.0..2.0
+        'slow'
+      else
+        'very_slow'
+      end
+    end
+
+    def to_analytics_hash
+      {
+        id: id,
+        query: query,
+        search_type: search_type,
+        result_count: result_count,
+        search_time: search_time,
+        performance_category: performance_category,
+        successful: successful?,
+        model_name: model_name,
+        filters: filter_summary,
+        created_at: created_at
+      }
+    end
+
+    private
+
+    def set_defaults
+      self.search_type ||= 'semantic'
+      self.result_count ||= 0
+      self.filters ||= {}
+      self.results ||= {}
+      self.model_name ||= Ragdoll.configuration.embedding_model
+    end
+
+    def normalize_query
+      return unless query
+      
+      # Remove excessive whitespace
+      self.query = query.strip.gsub(/\s+/, ' ')
+      
+      # Truncate if too long
+      self.query = query.truncate(10000) if query.length > 10000
+    end
+
+    def update_analytics
+      # This could be extended to update real-time analytics
+      # For now, it's just a placeholder for future enhancements
+      Rails.logger.debug "Search recorded: #{query} (#{result_count} results in #{search_time}s)"
+    end
+    end
+  end
+end

data/config/initializers/ragdoll.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Initializer for Ragdoll engine
+Ragdoll.configure do |config|
+  # Set configuration options here
+end

data/config/routes.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Ragdoll::Engine.routes.draw do
+  # Define your engine routes here
+end

data/db/migrate/20250218123456_create_documents.rb

@@ -0,0 +1,46 @@
+# Creates the core documents table for the Ragdoll RAG (Retrieval-Augmented Generation) system.
+# This table stores document metadata, content, and processing status information.
+class CreateRagdollDocuments < ActiveRecord::Migration[8.0]
+  def change
+    # Enable PostgreSQL extensions required for advanced text processing and vector operations
+    enable_extension 'pg_trgm' unless extension_enabled?('pg_trgm')        # Trigram matching for fuzzy text search
+    enable_extension 'fuzzystrmatch' unless extension_enabled?('fuzzystrmatch') # Fuzzy string matching algorithms
+    enable_extension 'vector' unless extension_enabled?('vector')          # Vector operations for embedding similarity
+
+    create_table :ragdoll_documents, comment: 'Core documents table storing files and content for RAG processing. Each row represents a document that can be chunked into embeddings for semantic search and AI retrieval.' do |t|
+      # File system reference - where the original document is stored
+      t.string :location, null: false, comment: 'File system path or URL to the original document. Required field that uniquely identifies the source location of the document for re-processing or reference.'
+      
+      # Document content and derived data
+      t.text :content, comment: 'Full extracted text content of the document. This is the raw text that will be chunked and embedded for semantic search. May be large for documents like PDFs or web pages.'
+      t.text :summary, comment: 'AI-generated summary of the document content. Created during processing to provide quick overviews and improve search relevance. Generated by summarization models.'
+      
+      # Document classification and metadata
+      t.string :document_type, comment: 'Classification of the document type (e.g., "pdf", "text", "markdown", "html"). Used for applying type-specific processing rules and display formatting.'
+      t.string :title, comment: 'Human-readable title of the document. May be extracted from filename, document metadata, or content analysis. Used for display and identification purposes.'
+      t.string :source_type, comment: 'Origin source of the document (e.g., "file", "url", "api", "upload"). Helps track how the document entered the system for auditing and re-processing.'
+      
+      # Chunking configuration - controls how document is split for embedding
+      t.integer :chunk_size, comment: 'Number of characters per chunk when splitting document for embedding. Larger chunks capture more context but may exceed model limits. Typical values: 500-2000 characters.'
+      t.integer :chunk_overlap, comment: 'Number of characters to overlap between consecutive chunks. Prevents context loss at chunk boundaries. Typically 10-20% of chunk_size to maintain coherence.'
+      
+      # Flexible metadata storage for document-specific information
+      t.jsonb :metadata, default: {}, comment: 'Flexible JSON storage for document-specific metadata such as author, creation date, file size, extraction settings, or custom tags. Indexed with GIN for efficient querying.'
+      
+      # Processing lifecycle tracking
+      t.datetime :processing_started_at, comment: 'Timestamp when document processing (chunking and embedding generation) began. Used for monitoring processing duration and identifying stuck jobs.'
+      t.datetime :processing_finished_at, comment: 'Timestamp when document processing completed successfully or failed. Used with started_at to calculate processing time and identify performance issues.'
+      t.string :status, default: 'pending', comment: 'Current processing status of the document. Values: "pending" (not yet processed), "processing" (currently being processed), "processed" (successfully completed), "failed" (processing error occurred). Used for job queue management and user feedback.'
+
+      # Standard Rails timestamps for audit trails
+      t.timestamps comment: 'Standard Rails created_at and updated_at timestamps for tracking when document records are created and modified in the database.'
+    end
+
+    # Indexes for performance optimization
+    add_index :ragdoll_documents, :location, unique: true, comment: 'Unique constraint on location prevents duplicate documents and enables fast lookups by file path or URL'
+    add_index :ragdoll_documents, :document_type, comment: 'Index for filtering documents by type, commonly used in admin interfaces and type-specific processing queries'
+    add_index :ragdoll_documents, :status, comment: 'Index for filtering by processing status, critical for job queue management and monitoring dashboard queries'
+    add_index :ragdoll_documents, :metadata, using: :gin, comment: 'GIN index on JSONB metadata field enables efficient queries on nested JSON properties and full-text search within metadata'
+    add_index :ragdoll_documents, :processing_started_at, comment: 'Index for sorting and filtering by processing start time, used for monitoring processing queues and performance analysis'
+  end
+end

data/db/migrate/20250219123456_create_ragdoll_embeddings.rb

@@ -0,0 +1,41 @@
+# Creates the embeddings table for storing document chunks and their vector representations.
+# This table is the core of the RAG system's semantic search capabilities.
+class CreateRagdollEmbeddings < ActiveRecord::Migration[8.0]
+  def change
+    create_table :ragdoll_embeddings, comment: 'Stores document chunks and their vector embeddings for semantic search. Each row represents a chunk of text from a document along with its AI-generated vector representation for similarity matching.' do |t|
+      # Parent document reference - establishes the chunk-to-document relationship
+      t.references :document, null: false, foreign_key: { to_table: :ragdoll_documents }, comment: 'Foreign key reference to the parent document in ragdoll_documents table. Required field that links each chunk back to its source document for context and retrieval.'
+      
+      # The actual text content of this chunk
+      t.text :content, null: false, comment: 'The actual text content of this chunk extracted from the parent document. Required field containing the text that will be used for semantic matching and returned in search results. Typically 500-2000 characters.'
+      
+      # Vector embedding for semantic search (pgvector format for optimal similarity calculations)
+      t.vector :embedding, limit: 1536, comment: 'High-dimensional vector representation of the content generated by AI embedding models. Stored in pgvector format for efficient cosine similarity calculations. Dimension typically 1536 for OpenAI models.'
+      
+      # Model identification for compatibility and reprocessing
+      t.string :model_name, comment: 'Name or identifier of the AI model used to generate this embedding (e.g., "text-embedding-3-small", "sentence-transformers/all-MiniLM-L6-v2"). Critical for ensuring embedding compatibility during search operations.'
+      
+      # Token usage tracking for cost analysis and optimization
+      t.integer :token_count, comment: 'Number of tokens consumed by the AI model when generating this embedding. Used for cost tracking, billing analysis, and optimizing chunk sizes to minimize API costs while maintaining quality.'
+      
+      # Chunk ordering within the parent document
+      t.integer :chunk_index, comment: 'Sequential position of this chunk within the parent document (0-based). Used for maintaining document order, reconstructing original text flow, and providing contextual information in search results.'
+      
+      # Flexible storage for chunk-specific metadata
+      t.jsonb :metadata, default: {}, comment: 'Flexible JSON storage for chunk-specific metadata such as section headers, page numbers, processing parameters, or semantic tags. Indexed with GIN for efficient querying of nested properties.'
+      
+      # Classification of embedding content type
+      t.string :embedding_type, default: 'text', comment: 'Type of content that was embedded (e.g., "text", "code", "table", "heading"). Allows for type-specific search strategies and filtering. Default "text" covers most document content.'
+
+      # Standard Rails timestamps for audit trails
+      t.timestamps comment: 'Standard Rails created_at and updated_at timestamps for tracking when embedding records are created and modified. Critical for debugging and performance analysis.'
+    end
+
+    # Performance indexes for efficient querying
+    add_index :ragdoll_embeddings, :document_id, comment: 'Index for efficiently retrieving all chunks belonging to a specific document. Essential for document reprocessing and chunk management operations.'
+    add_index :ragdoll_embeddings, :chunk_index, comment: 'Index for sorting chunks by their position within documents. Used for maintaining document order and providing sequential context in search results.'
+    add_index :ragdoll_embeddings, :embedding_type, comment: 'Index for filtering embeddings by content type. Enables type-specific search strategies (e.g., searching only code chunks or headings) and analytics on content distribution.'
+    add_index :ragdoll_embeddings, :metadata, using: :gin, comment: 'GIN index on JSONB metadata field enables efficient queries on nested JSON properties and supports complex filtering on chunk-specific attributes.'
+    add_index :ragdoll_embeddings, :embedding, using: :hnsw, opclass: :vector_cosine_ops, comment: 'Hierarchical Navigable Small World (HNSW) index optimized for cosine similarity searches on vector embeddings. Critical for fast semantic search performance at scale.'
+  end
+end

data/db/migrate/20250220123456_update_embeddings_vector_column.rb

@@ -0,0 +1,41 @@
+# Updates the embeddings table to support variable vector dimensions and optimize queries.
+# Originally intended to convert to pgvector format but maintains text compatibility for broader database support.
+class UpdateEmbeddingsVectorColumn < ActiveRecord::Migration[8.0]
+  def up
+    # Remove the limit constraint to allow variable length vectors
+    # Different AI models produce embeddings with different dimensions (e.g., 1536 for OpenAI, 384 for some sentence transformers)
+    change_column :ragdoll_embeddings, :embedding, :vector, limit: nil, comment: 'High-dimensional vector representation with variable dimensions removed to support multiple embedding models. Allows for mixing different model outputs while maintaining pgvector compatibility.'
+    
+    # Add column to track embedding vector dimensions for validation and compatibility
+    add_column :ragdoll_embeddings, :embedding_dimensions, :integer, comment: 'Number of dimensions in the embedding vector (e.g., 1536 for OpenAI text-embedding-3-small). Critical for ensuring embedding compatibility during similarity searches and preventing dimension mismatches.'
+    
+    # Update existing records to set their dimensions based on actual vector data
+    # This ensures data integrity for existing embeddings
+    execute <<~SQL
+      UPDATE ragdoll_embeddings 
+      SET embedding_dimensions = array_length(embedding::real[], 1) 
+      WHERE embedding IS NOT NULL
+    SQL
+    
+    # Add index on embedding_dimensions for efficient filtering by vector size
+    # Used when searching embeddings to ensure only compatible vectors are compared
+    add_index :ragdoll_embeddings, :embedding_dimensions, comment: 'Index for filtering embeddings by vector dimension size. Essential for ensuring only compatible embeddings are compared during similarity searches and avoiding dimension mismatch errors.'
+    
+    # Add composite index on model_name and embedding_dimensions for optimized similarity searches
+    # This combination is frequently queried together when finding similar embeddings
+    add_index :ragdoll_embeddings, [:model_name, :embedding_dimensions], 
+              name: 'index_ragdoll_embeddings_on_model_and_dimensions',
+              comment: 'Composite index on model name and vector dimensions. Optimizes the common query pattern of finding embeddings from the same model with matching dimensions for similarity calculations.'
+  end
+
+  def down
+    # Remove the new columns and indexes in reverse order
+    remove_index :ragdoll_embeddings, :embedding_dimensions
+    remove_index :ragdoll_embeddings, name: 'index_ragdoll_embeddings_on_model_and_dimensions'
+    remove_column :ragdoll_embeddings, :embedding_dimensions
+    
+    # Note: The original plan was to restore vector type, but this would fail with mixed dimensions
+    # Restore the original limit (this will fail if there are vectors with different dimensions)
+    change_column :ragdoll_embeddings, :embedding, :vector, limit: 1536
+  end
+end

data/db/migrate/20250223123457_add_metadata_and_foreign_key_to_ragdoll_tables.rb

@@ -0,0 +1,37 @@
+# Creates the searches table for tracking user search queries and analytics.
+# This table enables search analytics, caching, and performance monitoring for the RAG system.
+class CreateRagdollSearches < ActiveRecord::Migration[8.0]
+  def change
+    create_table :ragdoll_searches, comment: 'Tracks user search queries and results for analytics and performance monitoring. Each row represents a search performed by a user, storing both the query and metadata about results.' do |t|
+      # User's original search query
+      t.text :query, null: false, comment: 'The original search query text entered by the user. Required field that captures the exact search terms for analytics, popular queries tracking, and search result caching.'
+      
+      # Embedding of the search query for similarity calculations
+      t.vector :query_embedding, limit: 1536, comment: 'Vector embedding of the search query. Generated using the same model as document embeddings to enable semantic similarity calculations. Stored in pgvector format for efficient similarity operations.'
+      
+      # Search classification and method
+      t.string :search_type, default: 'semantic', comment: 'Type of search performed (e.g., "semantic", "keyword", "hybrid"). Allows for different search strategies and helps analyze which search methods are most effective for users.'
+      
+      # Search parameters and constraints
+      t.jsonb :filters, default: {}, comment: 'JSON object containing search filters applied (e.g., document_type, date_range, status). Used for analyzing how users refine searches and for recreating search contexts.'
+      
+      # Search results metadata  
+      t.jsonb :results, default: {}, comment: 'JSON object containing search result metadata such as result IDs, similarity scores, and ranking information. Enables search result caching and detailed analytics on result quality.'
+      t.integer :result_count, default: 0, comment: 'Number of results returned by this search. Used for analytics on search effectiveness and identifying queries that return too few or too many results.'
+      
+      # Performance tracking
+      t.float :search_time, comment: 'Time in seconds taken to execute this search query. Critical for performance monitoring, identifying slow queries, and optimizing search algorithms.'
+      
+      # Model identification for compatibility
+      t.string :model_name, comment: 'Name of the AI model used to generate the query embedding (e.g., "text-embedding-3-small"). Ensures search compatibility and enables analytics by model performance.'
+
+      # Standard Rails timestamps
+      t.timestamps comment: 'Standard Rails created_at and updated_at timestamps. The created_at timestamp is particularly important for search analytics and identifying search patterns over time.'
+    end
+
+    # Indexes for analytics and performance
+    add_index :ragdoll_searches, :search_type, comment: 'Index for filtering searches by type, used in analytics dashboards to compare effectiveness of different search strategies.'
+    add_index :ragdoll_searches, :query_embedding, using: :hnsw, opclass: :vector_cosine_ops, comment: 'HNSW index on query embeddings for finding similar queries. Enables query recommendation, duplicate detection, and search clustering analysis.'
+    add_index :ragdoll_searches, :created_at, comment: 'Index for time-based queries and analytics. Essential for generating search trend reports, popular queries over time, and performance monitoring.'
+  end
+end

data/db/migrate/20250225123456_add_summary_to_ragdoll_documents.rb

@@ -0,0 +1,17 @@
+# Adds summary metadata tracking to documents table for AI-generated summaries.
+# Enhances the existing summary field with provenance and timing information.
+class AddSummaryMetadataToRagdollDocuments < ActiveRecord::Migration[8.0]
+  def change
+    # Summary field already exists from initial migration, just add metadata fields for tracking
+    
+    # Timestamp tracking for summary generation
+    add_column :ragdoll_documents, :summary_generated_at, :timestamp, comment: 'Timestamp when the AI-generated summary was created. Used for cache invalidation, determining summary freshness, and analytics on summary generation performance.'
+    
+    # Model identification for summary provenance
+    add_column :ragdoll_documents, :summary_model, :string, comment: 'Name/identifier of the AI model used to generate the summary (e.g., "gpt-3.5-turbo", "claude-3-haiku"). Critical for tracking summary quality, cost analysis, and ensuring consistency in summary style.'
+    
+    # Indexes for summary metadata queries and analytics
+    add_index :ragdoll_documents, :summary_generated_at, comment: 'Index for filtering and sorting documents by summary generation time. Used for cache invalidation queries and identifying documents with stale summaries that need regeneration.'
+    add_index :ragdoll_documents, :summary_model, comment: 'Index for filtering documents by the AI model used for summarization. Enables analytics on summary quality by model and batch re-summarization with newer models.'
+  end
+end

data/db/migrate/20250226123456_add_usage_tracking_to_ragdoll_embeddings.rb

@@ -0,0 +1,28 @@
+# Adds usage tracking functionality to embeddings for intelligent ranking and caching.
+# Enables the system to learn which embeddings are most valuable and prioritize them in search results.
+class AddUsageTrackingToRagdollEmbeddings < ActiveRecord::Migration[8.0]
+  def change
+    # Usage frequency tracking
+    add_column :ragdoll_embeddings, :usage_count, :integer, default: 0, null: false, comment: 'Number of times this embedding has been returned in search results. Incremented each time the embedding appears in search results. Used for frequency-based ranking to surface more relevant content.'
+    
+    # Recency tracking for temporal relevance
+    add_column :ragdoll_embeddings, :returned_at, :timestamp, comment: 'Timestamp of the most recent time this embedding was returned in search results. Used for recency-based ranking algorithms and cache warming strategies. NULL indicates never used.'
+    
+    # Performance indexes for usage-based ranking queries
+    add_index :ragdoll_embeddings, :usage_count, name: 'index_ragdoll_embeddings_on_usage_count', comment: 'Index for sorting embeddings by usage frequency. Critical for popularity-based ranking algorithms and identifying most/least used content.'
+    add_index :ragdoll_embeddings, :returned_at, name: 'index_ragdoll_embeddings_on_returned_at', comment: 'Index for sorting embeddings by recency of use. Enables temporal ranking algorithms and cache warming strategies based on recent usage patterns.'
+    
+    # Composite index for advanced ranking algorithms that combine frequency and recency
+    add_index :ragdoll_embeddings, [:usage_count, :returned_at], 
+              name: 'index_ragdoll_embeddings_on_usage_and_recency',
+              comment: 'Composite index for complex ranking algorithms that combine usage frequency with recency. Optimizes queries that balance popular content with recently accessed content for intelligent search result ranking.'
+    
+    # Data migration to ensure existing records have proper default values
+    # Usage count defaults to 0, returned_at remains null until first usage
+    reversible do |dir|
+      dir.up do
+        execute "UPDATE ragdoll_embeddings SET usage_count = 0 WHERE usage_count IS NULL"
+      end
+    end
+  end
+end

data/lib/generators/ragdoll/init/init_generator.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'rails/generators'
+
+module Ragdoll
+  module Generators
+    class InitGenerator < Rails::Generators::Base
+      desc "Create Ragdoll configuration initializer"
+      source_root File.expand_path("templates", __dir__)
+
+      def create_initializer_file
+        template "ragdoll_config.rb", "config/initializers/ragdoll_config.rb"
+      end
+
+      def show_readme
+        readme "INSTALL" if behavior == :invoke
+      end
+
+      private
+
+      def application_name
+        Rails.application.class.name.split("::").first.underscore
+      end
+    end
+  end
+end

data/lib/generators/ragdoll/init/templates/INSTALL

@@ -0,0 +1,56 @@
+===============================================================================
+
+    Ragdoll Configuration Created Successfully!
+
+===============================================================================
+
+The Ragdoll initializer has been created at:
+  config/initializers/ragdoll_config.rb
+
+Next steps:
+
+1. Configure your LLM provider API keys in environment variables:
+   
+   For OpenAI:
+     export OPENAI_API_KEY="your-api-key-here"
+   
+   For Anthropic:
+     export ANTHROPIC_API_KEY="your-api-key-here"
+   
+   For Google:
+     export GOOGLE_API_KEY="your-api-key-here"
+     export GOOGLE_PROJECT_ID="your-project-id"
+
+2. Install and run database migrations:
+   
+   rails ragdoll:install:migrations
+   rails db:migrate
+
+3. Ensure you have PostgreSQL with pgvector extension installed:
+   
+   # In PostgreSQL console:
+   CREATE EXTENSION IF NOT EXISTS vector;
+
+4. Configure Sidekiq for background job processing (optional):
+   
+   # Add to your Gemfile if not already present:
+   gem 'sidekiq'
+   
+   # Start Sidekiq:
+   bundle exec sidekiq
+
+5. Start using Ragdoll in your Rails application:
+   
+   # Add documents
+   Ragdoll.add_document('/path/to/document.pdf')
+   
+   # Enhance prompts with context
+   enhanced = Ragdoll.enhance_prompt('How do I configure the database?')
+   
+   # Use with your LLM
+   response = RubyLLM.ask(enhanced[:enhanced_prompt])
+
+For more information, visit:
+  https://github.com/MadBomber/ragdoll-rails
+
+===============================================================================

data/lib/generators/ragdoll/init/templates/ragdoll_config.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+# Ragdoll RAG (Retrieval-Augmented Generation) Configuration
+# This initializer configures the Ragdoll Rails engine for your application.
+
+Ragdoll.configure do |config|
+  # LLM Provider Configuration
+  # Supported providers: :openai, :anthropic, :google, :azure, :ollama, :huggingface
+  config.llm_provider = :openai
+  
+  # Optional: Use a different provider for embeddings (defaults to llm_provider)
+  # config.embedding_provider = :openai
+
+  # Provider-specific API configurations
+  # Add your API keys and configuration here
+  config.llm_config = {
+    openai: { 
+      api_key: ENV['OPENAI_API_KEY']
+      # organization: ENV['OPENAI_ORGANIZATION'],  # optional
+      # project: ENV['OPENAI_PROJECT']              # optional
+    },
+    anthropic: { 
+      api_key: ENV['ANTHROPIC_API_KEY'] 
+    },
+    google: { 
+      api_key: ENV['GOOGLE_API_KEY'],
+      project_id: ENV['GOOGLE_PROJECT_ID']
+    },
+    azure: {
+      api_key: ENV['AZURE_API_KEY'],
+      endpoint: ENV['AZURE_ENDPOINT'],
+      api_version: ENV['AZURE_API_VERSION']
+    },
+    ollama: {
+      endpoint: ENV['OLLAMA_ENDPOINT'] || 'http://localhost:11434'
+    },
+    huggingface: {
+      api_key: ENV['HUGGINGFACE_API_KEY']
+    }
+  }
+
+  # Embedding Model Configuration
+  # Examples: 'text-embedding-3-small', 'text-embedding-3-large', 'text-embedding-ada-002'
+  config.embedding_model = 'text-embedding-3-small'
+  
+  # Default model for chat/completion
+  config.default_model = 'gpt-4o-mini'
+
+  # Text Processing Configuration
+  config.chunk_size = 1000
+  config.chunk_overlap = 200
+
+  # Search Configuration
+  config.search_similarity_threshold = 0.7
+  config.max_search_results = 10
+
+  # Vector Configuration
+  # Maximum dimensions supported (supports variable-length vectors)
+  config.max_embedding_dimensions = 3072
+
+  # Background Jobs Configuration
+  # Set to false if you don't want to use background jobs for document processing
+  config.use_background_jobs = true
+
+  # Analytics Configuration
+  config.enable_search_analytics = true
+  config.cache_embeddings = true
+
+  # Custom Prompt Template (optional)
+  # Use {{context}} and {{prompt}} placeholders
+  # config.prompt_template = <<~TEMPLATE
+  #   Based on the following context, please answer the question.
+  #   
+  #   Context:
+  #   {{context}}
+  #   
+  #   Question: {{prompt}}
+  #   
+  #   Answer:
+  # TEMPLATE
+end
+
+# Optional: Configure Rails-specific settings
+Ragdoll::Rails.configure do |config|
+  # Enable/disable background job processing
+  config.use_background_jobs = true
+  
+  # Background job queue name
+  config.queue_name = :ragdoll
+  
+  # Maximum file size for uploads (in bytes)
+  config.max_file_size = 10.megabytes
+  
+  # Allowed file types for document upload
+  config.allowed_file_types = %w[pdf docx txt md html htm json xml csv]
+end

data/lib/ragdoll/rails/configuration.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Rails
+    class Configuration
+      # Rails-specific configuration options
+      attr_accessor :use_background_jobs, :job_queue, :job_adapter, :queue_name, :max_file_size, :allowed_file_types
+      
+      def initialize
+        @use_background_jobs = true
+        @job_queue = :default
+        @job_adapter = :sidekiq
+        @queue_name = :ragdoll
+        @max_file_size = 10 * 1024 * 1024 # 10MB
+        @allowed_file_types = %w[pdf docx txt md html htm json xml csv]
+      end
+
+      def configure_core
+        # Delegate to core ragdoll gem configuration
+        # This would configure the core ragdoll gem based on Rails-specific settings
+      end
+    end
+
+    def self.configuration
+      @configuration ||= Configuration.new
+    end
+
+    def self.configure
+      yield(configuration)
+      configuration.configure_core
+    end
+  end
+end