ragdoll 0.1.1 → 0.1.8

data/lib/ragdoll/core/client.rb

@@ -6,14 +6,22 @@ module Ragdoll
   module Core
     class Client
       def initialize
+        # Setup configuration services
+        @config_service = Ragdoll::ConfigurationService.new
+        @model_resolver = Ragdoll::ModelResolver.new(@config_service)
+
         # Setup logging
         setup_logging
 
         # Setup database connection
-        Database.setup(Ragdoll.config.database_config)
-
-        @embedding_service = EmbeddingService.new
-        @search_engine = SearchEngine.new(@embedding_service)
+        Database.setup(@config_service.config.database)
+
+        @embedding_service = Ragdoll::EmbeddingService.new(
+          client: nil,
+          config_service: @config_service,
+          model_resolver: @model_resolver
+        )
+        @search_engine = Ragdoll::SearchEngine.new(@embedding_service, config_service: @config_service)
       end
 
       # Primary method for RAG applications
@@ -88,7 +96,7 @@ module Ragdoll
         query_embedding = @embedding_service.generate_embedding(query)
 
         # Perform hybrid search
-        results = Models::Document.hybrid_search(query, query_embedding: query_embedding, **options)
+        results = Ragdoll::Document.hybrid_search(query, query_embedding: query_embedding, **options)
 
         {
           query: query,
@@ -111,30 +119,28 @@ module Ragdoll
       # Document management
       def add_document(path:)
         # Parse the document
-        parsed = DocumentProcessor.parse(path)
+        parsed = Ragdoll::DocumentProcessor.parse(path)
 
         # Extract title from metadata or use filename
         title = parsed[:metadata][:title] ||
                 File.basename(path, File.extname(path))
 
         # Add document to database
-        doc_id = DocumentManagement.add_document(path, parsed[:content], {
+        doc_id = Ragdoll::DocumentManagement.add_document(path, parsed[:content], {
                                                    title: title,
                                                    document_type: parsed[:document_type],
                                                    **parsed[:metadata]
                                                  })
 
-
         # Queue background jobs for processing if content is available
         embeddings_queued = false
         if parsed[:content].present?
-          Ragdoll::Core::Jobs::GenerateEmbeddings.perform_later(doc_id)
-          Ragdoll::Core::Jobs::GenerateSummary.perform_later(doc_id)
-          Ragdoll::Core::Jobs::ExtractKeywords.perform_later(doc_id)
+          Ragdoll::GenerateEmbeddingsJob.perform_later(doc_id)
+          Ragdoll::GenerateSummaryJob.perform_later(doc_id)
+          Ragdoll::ExtractKeywordsJob.perform_later(doc_id)
           embeddings_queued = true
         end
 
-
         # Return success information
         {
           success: true,
@@ -155,16 +161,16 @@ module Ragdoll
 
       def add_text(content:, title:, **options)
         # Add document to database
-        doc_id = DocumentManagement.add_document(title, content, {
+        doc_id = Ragdoll::DocumentManagement.add_document(title, content, {
                                                    title: title,
                                                    document_type: "text",
                                                    **options
                                                  })
 
         # Queue background job for embeddings
-        Ragdoll::Core::Jobs::GenerateEmbeddings.perform_later(doc_id,
-                                                              chunk_size: options[:chunk_size],
-                                                              chunk_overlap: options[:chunk_overlap])
+        Ragdoll::GenerateEmbeddingsJob.perform_later(doc_id,
+                                                     chunk_size: options[:chunk_size],
+                                                     chunk_overlap: options[:chunk_overlap])
 
         doc_id
       end
@@ -188,7 +194,7 @@ module Ragdoll
       end
 
       def get_document(id:)
-        document_hash = DocumentManagement.get_document(id)
+        document_hash = Ragdoll::DocumentManagement.get_document(id)
         return nil unless document_hash
 
         # DocumentManagement.get_document already returns a hash with all needed info
@@ -196,7 +202,7 @@ module Ragdoll
       end
 
       def document_status(id:)
-        document = Models::Document.find(id)
+        document = Ragdoll::Document.find(id)
         embeddings_count = document.all_embeddings.count
 
         {
@@ -228,25 +234,25 @@ module Ragdoll
       end
 
       def update_document(id:, **updates)
-        DocumentManagement.update_document(id, **updates)
+        Ragdoll::DocumentManagement.update_document(id, **updates)
       end
 
       def delete_document(id:)
-        DocumentManagement.delete_document(id)
+        Ragdoll::DocumentManagement.delete_document(id)
       end
 
       def list_documents(**options)
-        DocumentManagement.list_documents(options)
+        Ragdoll::DocumentManagement.list_documents(options)
       end
 
       # Analytics and stats
       def stats
-        DocumentManagement.get_document_stats
+        Ragdoll::DocumentManagement.get_document_stats
       end
 
       def search_analytics(days: 30)
         # This could be implemented with additional database queries
-        Models::Embedding.where("returned_at > ?", days.days.ago)
+        Ragdoll::Embedding.where("returned_at > ?", days.days.ago)
                          .group("DATE(returned_at)")
                          .count
       end
@@ -265,14 +271,13 @@ module Ragdoll
         require "active_job"
 
         # Create log directory if it doesn't exist
-        # FIXME: log_file is not in current config structure
-        log_file = Ragdoll.config.logging_config[:filepath] || File.join(Dir.home, ".ragdoll", "ragdoll.log")
+        log_file = @config_service.config.logging[:filepath]
         log_dir = File.dirname(log_file)
         FileUtils.mkdir_p(log_dir) unless Dir.exist?(log_dir)
 
         # Set up logger with appropriate level
         logger = Logger.new(log_file)
-        logger.level = case Ragdoll.config.logging_config[:level]
+        logger.level = case @config_service.config.logging[:level]
                        when :debug then Logger::DEBUG
                        when :info then Logger::INFO
                        when :warn then Logger::WARN
@@ -290,26 +295,12 @@ module Ragdoll
       end
 
       def build_enhanced_prompt(original_prompt, context)
-        # FIXME: prompt_template is not in current config structure
-        template = default_prompt_template
+        template = @config_service.config.prompt_template(:rag_enhancement)
 
         template
           .gsub("{{context}}", context)
           .gsub("{{prompt}}", original_prompt)
       end
-
-      def default_prompt_template
-        <<~TEMPLATE
-          You are an AI assistant. Use the following context to help answer the user's question. If the context doesn't contain relevant information, say so.
-
-          Context:
-          {{context}}
-
-          Question: {{prompt}}
-
-          Answer:
-        TEMPLATE
-      end
     end
   end
 end

data/lib/ragdoll/core/configuration.rb

@@ -3,6 +3,7 @@
 require "yaml"
 require "fileutils"
 require "ostruct"
+require_relative "model"
 
 module Ragdoll
   module Core
@@ -12,134 +13,162 @@ module Ragdoll
       class ConfigurationLoadUnknownError < StandardError; end
 
       DEFAULT = {
-        directory: File.join(Dir.home, ".ragdoll"),
-        filepath: File.join(Dir.home, ".ragdoll", "config.yml"),
+        # Base directory for all Ragdoll files - single source of truth
+        base_directory: File.join(Dir.home, ".config", "ragdoll"),
+
+        # Configuration file path derived from base directory
+        config_filepath: File.join(Dir.home, ".config", "ragdoll", "config.yml"),
+
+        # Model configurations organized by purpose with inheritance support
         models: {
-          default: "openai/gpt-4o",
-          summary: "openai/gpt-4o",
-          keywords: "openai/gpt-4o",
-          embedding: {
-            text: "text-embedding-3-small",
-            image: "image-embedding-3-small", # FIXME
-            audio: "audio-embedding-3-small",  # FIXME
+          text_generation: {
+            default: -> { Model.new(ENV.fetch("RAGDOLL_DEFAULT_TEXT_MODEL", "openai/gpt-4o")) },
+            summary: -> { Model.new(ENV.fetch("RAGDOLL_SUMMARY_MODEL", "openai/gpt-4o")) },
+            keywords: -> { Model.new(ENV.fetch("RAGDOLL_KEYWORDS_MODEL", "openai/gpt-4o")) }
           },
+          embedding: {
+            provider: :openai,
+            text: -> { Model.new(ENV.fetch("RAGDOLL_TEXT_EMBEDDING_MODEL", "openai/text-embedding-3-small")) },
+            image: -> { Model.new(ENV.fetch("RAGDOLL_IMAGE_EMBEDDING_MODEL", "openai/clip-vit-base-patch32")) },
+            audio: -> { Model.new(ENV.fetch("RAGDOLL_AUDIO_EMBEDDING_MODEL", "openai/whisper-1")) },
+            max_dimensions: 3072,
+            cache_embeddings: true
+          }
         },
-        chunking: {
+
+        # Processing configuration by content type
+        processing: {
           text: {
-            max_tokens: 1000,
-            overlap: 200,
-          },
-          image: {
-            max_tokens: 4096,
-            overlap: 128,
-          },
-          audio: {
-            max_tokens: 4096,
-            overlap: 128,
+            chunking: {
+              max_tokens: 1000,
+              overlap: 200
+            }
           },
           default: {
-            max_tokens: 4096,
-            overlap: 128,
+            chunking: {
+              max_tokens: 4096,
+              overlap: 128
+            }
           },
+          search: {
+            similarity_threshold: 0.7,
+            max_results: 10,
+            analytics: {
+              enable: true,
+              usage_tracking_enabled: true,
+              ranking_enabled: true,
+              recency_weight: 0.3,
+              frequency_weight: 0.7,
+              similarity_weight: 1.0
+            }
+          }
         },
-        ruby_llm_config: {
+
+        # LLM provider configurations (renamed from ruby_llm_config)
+        llm_providers: {
+          default_provider: :openai,
           openai: {
-            api_key: -> { ENV["OPENAI_API_KEY"] },
-            organization: -> { ENV["OPENAI_ORGANIZATION"] },
-            project: -> { ENV["OPENAI_PROJECT"] },
+            api_key: -> { ENV.fetch("OPENAI_API_KEY", nil) },
+            organization: -> { ENV.fetch("OPENAI_ORGANIZATION", nil) },
+            project: -> { ENV.fetch("OPENAI_PROJECT", nil) }
           },
           anthropic: {
-            api_key: -> { ENV["ANTHROPIC_API_KEY"] },
+            api_key: -> { ENV.fetch("ANTHROPIC_API_KEY", nil) }
           },
           google: {
-            api_key: -> { ENV["GOOGLE_API_KEY"] },
-            project_id: -> { ENV["GOOGLE_PROJECT_ID"] },
+            api_key: -> { ENV.fetch("GOOGLE_API_KEY", nil) },
+            project_id: -> { ENV.fetch("GOOGLE_PROJECT_ID", nil) }
           },
           azure: {
-            api_key: -> { ENV["AZURE_OPENAI_API_KEY"] },
-            endpoint: -> { ENV["AZURE_OPENAI_ENDPOINT"] },
-            api_version: -> { ENV["AZURE_OPENAI_API_VERSION"] || "2024-02-01" },
+            api_key: -> { ENV.fetch("AZURE_OPENAI_API_KEY", nil) },
+            endpoint: -> { ENV.fetch("AZURE_OPENAI_ENDPOINT", nil) },
+            api_version: -> { ENV.fetch("AZURE_OPENAI_API_VERSION", "2024-02-01") }
           },
           ollama: {
-            endpoint: -> { ENV["OLLAMA_ENDPOINT"] || "http://localhost:11434/v1" },
+            endpoint: -> { ENV.fetch("OLLAMA_ENDPOINT", "http://localhost:11434") }
           },
           huggingface: {
-            api_key: -> { ENV["HUGGINGFACE_API_KEY"] },
+            api_key: -> { ENV.fetch("HUGGINGFACE_API_KEY", nil) }
           },
           openrouter: {
-            api_key: -> { ENV["OPENROUTER_API_KEY"] },
-          },
-        },
-        embedding_config: {
-          provider: :openai,
-          cache_embeddings: true,
-          max_embedding_dimensions: 3072, # Support up to text-embedding-3-large
+            api_key: -> { ENV.fetch("OPENROUTER_API_KEY", nil) }
+          }
         },
-        summarization_config: {
+
+        # Summarization configuration
+        summarization: {
           enable: true,
           max_length: 300,
-          min_content_length: 300,
+          min_content_length: 300
         },
-        database_config: {
+
+        # Database configuration with standardized ENV variable name
+        database: {
           adapter: "postgresql",
           database: "ragdoll_development",
           username: "ragdoll",
-          password: -> { ENV["DATABASE_PASSWORD"] },
+          password: -> { ENV.fetch("RAGDOLL_DATABASE_PASSWORD", nil) },
           host: "localhost",
           port: 5432,
           auto_migrate: true,
-          logger: nil, # Set to Logger.new(STDOUT) for debugging
+          logger: nil
         },
-        logging_config: {
-          log_level: :warn,
-          log_directory: File.join(Dir.home, ".ragdoll"),
-          log_filepath: File.join(Dir.home, ".ragdoll", "ragdoll.log"),
-        },
-        search: {
-          similarity_threshold: 0.7,
-          max_results: 10,
-          enable_analytics: true,
-          enable_usage_tracking: true,
-          usage_ranking_enabled: true,
-          usage_recency_weight: 0.3,
-          usage_frequency_weight: 0.7,
-          usage_similarity_weight: 1.0,
+
+        # Logging configuration with corrected key names and path derivation
+        logging: {
+          level: :warn, # Fixed: was log_level, now matches usage
+          directory: File.join(Dir.home, ".config", "ragdoll", "logs"),
+          filepath: File.join(Dir.home, ".config", "ragdoll", "logs", "ragdoll.log")
         },
-      }
+
+        # Prompt templates for customizable text generation
+        prompt_templates: {
+          rag_enhancement: <<~TEMPLATE.strip
+            You are an AI assistant. Use the following context to help answer the user's question.
+            If the context doesn't contain relevant information, say so.
+
+            Context:
+            {{context}}
+
+            Question: {{prompt}}
+
+            Answer:
+          TEMPLATE
+        }
+
+      }.freeze
 
       def initialize(config = {})
         merged_config = deep_merge(self.class::DEFAULT, config)
-        resolved_config = resolve_procs(merged_config)
+        resolved_config = resolve_procs(merged_config, [])
         @config = OpenStruct.new(resolved_config)
       end
 
       def self.load(path: nil)
-        path ||= DEFAULT[:filepath]
+        path ||= DEFAULT[:config_filepath]
 
-        unless File.exist?(path)
-          raise ConfigurationFileNotFoundError, "Configuration file not found: #{path}"
-        end
+        raise ConfigurationFileNotFoundError, "Configuration file not found: #{path}" unless File.exist?(path)
 
         new(YAML.safe_load_file(path) || {})
       rescue Errno::ENOENT
         raise ConfigurationFileNotFoundError, "Configuration file not found: #{path}"
-      rescue => e
+      rescue StandardError => e
         raise ConfigurationLoadUnknownError, "Failed to load configuration from #{path}: #{e.message}"
       end
 
       def save(path: nil)
         if path.nil?
-          path = @config.filepath
+          path = @config.config_filepath
         else
-          save_filepath = @config.filepath
-          @config.filepath = path
+          save_filepath = @config.config_filepath
+          @config.config_filepath = path
         end
 
         FileUtils.mkdir_p(File.dirname(path))
 
         File.write(path, @config.to_yaml)
-      rescue => e
-        @config.filepath = save_filepath unless save_filepath.nil?
+      rescue StandardError => e
+        @config.config_filepath = save_filepath unless save_filepath.nil?
         raise ConfigurationSaveError, "Failed to save configuration to #{path}: #{e.message}"
       end
 
@@ -160,6 +189,35 @@ module Ragdoll
         end
       end
 
+      # Resolve model with inheritance support
+      # Returns the model string for a given task, with inheritance from default
+      def resolve_model(task_type)
+        case task_type
+        when :embedding
+          @config.models[:embedding]
+        when :text, :summary, :keywords, :default
+          @config.models[:text_generation][task_type] || @config.models[:text_generation][:default]
+        else
+          @config.models[:text_generation][:default]
+        end
+      end
+
+      # Get provider credentials for a given provider
+      def provider_credentials(provider = nil)
+        provider ||= @config.llm_providers[:default_provider]
+        @config.llm_providers[provider] || {}
+      end
+
+      # Resolve embedding model for content type
+      def embedding_model(content_type = :text)
+        @config.models[:embedding][content_type] || @config.models[:embedding][:text]
+      end
+
+      # Get prompt template
+      def prompt_template(template_name = :rag_enhancement)
+        @config.prompt_templates[template_name]
+      end
+
       # Enable method delegation to the internal OpenStruct
       def method_missing(method_name, *args, &block)
         @config.send(method_name, *args, &block)
@@ -171,103 +229,29 @@ module Ragdoll
 
       private
 
-      def resolve_procs(obj)
+      def resolve_procs(obj, path = [])
         case obj
         when Hash
-          obj.transform_values { |v| resolve_procs(v) }
+          obj.each_with_object({}) { |(k, v), result| result[k] = resolve_procs(v, path + [k]) }
         when Proc
           obj.call
+        when String
+          # Convert strings to Model instances in the models configuration section
+          if path.length >= 2 && path[0] == :models
+            Model.new(obj)
+          else
+            obj
+          end
         else
           obj
         end
       end
 
       def deep_merge(hash1, hash2)
-        hash1.merge(hash2) do |key, oldval, newval|
+        hash1.merge(hash2) do |_key, oldval, newval|
           oldval.is_a?(Hash) && newval.is_a?(Hash) ? deep_merge(oldval, newval) : newval
         end
       end
     end
   end
 end
-
-__END__
-
-{
-directory: "/Users/dewayne/.ragdoll",
- filepath: "/Users/dewayne/.ragdoll/config.yml",
- embedding_config:
-  {default:
-    {model: "openai/gpt-4o-mini", summary_model: "openai/gpt-4o-mini", keywords_model: "openai/gpt-4o-mini", max_dimensions: 3072},
-   text: {model: "openai/text-embedding-3-small", max_tokens: 1000, overlap: 200},
-   image: {model: "laion/CLIP-ViT-H-14", max_tokens: 4096, overlap: 128},
-   audio: {model: "openl3", transcription_model: "openai/whisper-large-v2", max_tokens: 4096, overlap: 128}},
- chunking: {text: {max_tokens: 1000, overlap: 200}, default: {max_tokens: 4096, overlap: 128}},
- ruby_llm_config:
-  {openai: {api_key: "***", organization: nil, project: nil},
-   anthropic:
-    {api_key: "***"},
-   google: {api_key: "***", project_id: nil},
-   azure: {api_key: nil, endpoint: nil, api_version: "2024-02-01"},
-   ollama: {endpoint: "http://localhost:11434/v1"},
-   huggingface: {api_key: nil},
-   openrouter: {api_key: nil}},
- summarization_config: {enable: true, model: nil, max_length: 300, min_content_length: 300},
- database_config:
-  {adapter: "postgresql",
-   database: "ragdoll_development",
-   username: "ragdoll",
-   password: "ragdoll",
-   host: "localhost",
-   port: 5432,
-   pool: 20,
-   timeout: 5000,
-   auto_migrate: true,
-   logger: nil},
- logging_config: {level: :warn, directory: "/Users/dewayne/.ragdoll", filepath: "/Users/dewayne/.ragdoll/ragdoll.log"},
- search:
-  {similarity_threshold: 0.7,
-   max_results: 10,
-   enable_analytics: true,
-   enable_usage_tracking: true,
-   usage_ranking_enabled: true,
-   usage_recency_weight: 0.3,
-   usage_frequency_weight: 0.7,
-   usage_similarity_weight: 1.0},
- llm_provider: :openai,
- openai_api_key: "***",
- llm_config:
-  {openai: {api_key: "***", organization: nil, project: nil},
-   anthropic:
-    {api_key: "***"},
-   google: {api_key: "***", project_id: nil},
-   azure: {api_key: nil, endpoint: nil, api_version: "2024-02-01"},
-   ollama: {endpoint: "http://localhost:11434"},
-   huggingface: {api_key: nil},
-   openrouter: {api_key: nil}},
- embedding_provider: :openai,
- embedding_model: "text-embedding-3-small",
- max_embedding_dimensions: 3072,
- cache_embeddings: true,
- default_model: "gpt-4o-mini",
- summary_provider_model: "openai/gpt-4o-mini",
- keywords_provider_model: "openai/gpt-4o-mini",
- embeddings_provider_model: "openai/text-embedding-3-small",
- summary_model: nil,
- chunk_size: 1000,
- chunk_overlap: 200,
- enable_document_summarization: true,
- summary_max_length: 300,
- summary_min_content_length: 300,
- prompt_template: nil,
- search_similarity_threshold: 0.7,
- max_search_results: 10,
- enable_search_analytics: true,
- enable_usage_tracking: true,
- usage_ranking_enabled: true,
- usage_recency_weight: 0.3,
- usage_frequency_weight: 0.7,
- usage_similarity_weight: 1.0,
- log_level: :warn,
- log_file: "/Users/dewayne/.ragdoll/ragdoll.log"
-}

data/lib/ragdoll/core/database.rb

@@ -125,7 +125,7 @@ module Ragdoll
           adapter: "postgresql",
           database: "ragdoll_development",
           username: "ragdoll",
-          password: ENV["RAGDOLL_DATABASE_PASSWORD"],
+          password: ENV.fetch("RAGDOLL_DATABASE_PASSWORD", nil),
           host: "localhost",
           port: 5432,
           auto_migrate: true,

data/lib/ragdoll/core/model.rb

@@ -0,0 +1,45 @@
+# lib/ragdoll/core/model.rb
+# frozen_string_literal: true
+
+module Ragdoll
+  module Core
+    # Model represents a provider and model name.
+    # It is initialized with a string in the format "provider/model".
+    # The provider is optional.
+    # Can be initialized with nil or empty string.
+    Model = Data.define(:name) do
+      # @return [Symbol, nil] the provider part of the name, or nil if not present.
+      def provider
+        return nil if name.nil? || name.empty?
+
+        parts = name.split("/", 2)
+        return nil if parts.length < 2 || parts.first.empty?
+
+        parts.first.to_sym
+      end
+
+      # @return [String, nil] the model part of the name, or nil if name is nil/empty.
+      def model
+        return nil if name.nil? || name.empty?
+
+        parts = name.split("/", 2)
+        parts.length < 2 ? name : parts.last
+      end
+
+      # @return [String] the original name string, or empty string if name is nil.
+      def to_s
+        name.nil? ? "" : name
+      end
+
+      # @return [Hash] a hash representation of the model.
+      def to_h
+        { provider: provider, model: model }
+      end
+
+      # YAML serialization - save as string name
+      def encode_with(coder)
+        coder.scalar = name
+      end
+    end
+  end
+end

data/lib/ragdoll/core/version.rb

@@ -3,6 +3,6 @@
 
 module Ragdoll
   module Core
-    VERSION = "0.1.1"
+    VERSION = "0.1.8"
   end
 end