ragdoll 0.1.0 → 0.1.3

data/lib/ragdoll/core/database.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "active_record"
+require "logger"
+
+module Ragdoll
+  module Core
+    class Database
+      def self.setup(config = {})
+        database_config = default_config.merge(config)
+
+        # Set up ActiveRecord connection
+        ActiveRecord::Base.establish_connection(database_config)
+
+        # Set up logging if specified
+        ActiveRecord::Base.logger = database_config[:logger] if database_config[:logger]
+
+        # Auto-migrate if specified
+        return unless database_config[:auto_migrate]
+
+        migrate!
+      end
+
+      def self.migrate!
+        # Get the path to the gem root directory
+        # Current file is lib/ragdoll/core/database.rb, so go up 3 levels to get to gem root
+        gem_root = File.expand_path(File.join(File.dirname(__FILE__), "..", "..", ".."))
+        migration_paths = [
+          File.join(gem_root, "db", "migrate")
+        ]
+
+        ActiveRecord::Migration.verbose = true
+
+        # Ensure schema_migrations table exists first
+        unless ActiveRecord::Base.connection.table_exists?("schema_migrations")
+          ActiveRecord::Base.connection.create_table("schema_migrations", id: false) do |t|
+            t.string :version, null: false
+          end
+          ActiveRecord::Base.connection.add_index("schema_migrations", :version, unique: true)
+        end
+
+        # Debug migration path (silenced for clean test output)
+        # puts "Migration path: #{migration_paths.first}" if ActiveRecord::Migration.verbose
+        migration_files = Dir[File.join(migration_paths.first, "*.rb")].sort
+        # puts "Found #{migration_files.length} migration files" if ActiveRecord::Migration.verbose
+
+        # Load and run each migration manually since ActiveRecord migration context seems broken
+        migration_files.each do |migration_file|
+          # Extract version from filename
+          version = File.basename(migration_file, ".rb").split("_").first
+
+          # Skip if already migrated
+          next if ActiveRecord::Base.connection.select_values(
+            "SELECT version FROM schema_migrations WHERE version = '#{version}'"
+          ).any?
+
+          # Load the migration file to define the class
+          require migration_file
+
+          # Get the migration class - convert snake_case to CamelCase
+          filename_parts = File.basename(migration_file, ".rb").split("_")[1..]
+          migration_class_name = filename_parts.map(&:capitalize).join
+
+          begin
+            migration_class = Object.const_get(migration_class_name)
+          rescue NameError
+            puts "Warning: Could not find migration class #{migration_class_name} in #{migration_file}"
+            next
+          end
+
+          # Run the migration quietly
+          old_verbose = ActiveRecord::Migration.verbose
+          ActiveRecord::Migration.verbose = false
+          migration_class.migrate(:up)
+          ActiveRecord::Migration.verbose = old_verbose
+
+          # Record the migration
+          ActiveRecord::Base.connection.insert(
+            "INSERT INTO schema_migrations (version) VALUES ('#{version}')"
+          )
+
+          # Silenced migration progress - uncomment for debugging
+          # puts "Migrated #{migration_class_name}" if ActiveRecord::Migration.verbose
+        end
+      end
+
+      def self.reset!
+        ActiveRecord::Migration.verbose = false
+
+        # Drop all tables in correct order (respecting foreign key constraints)
+        # Order: dependent tables first, then parent tables
+        tables_to_drop = %w[
+          ragdoll_embeddings
+          ragdoll_text_contents
+          ragdoll_image_contents
+          ragdoll_audio_contents
+          ragdoll_documents
+          schema_migrations
+        ]
+
+        tables_to_drop.each do |table|
+          if ActiveRecord::Base.connection.table_exists?(table)
+            # For PostgreSQL, we can use CASCADE to drop dependent objects
+            if ActiveRecord::Base.connection.adapter_name.downcase.include?("postgresql")
+              ActiveRecord::Base.connection.execute("DROP TABLE IF EXISTS #{table} CASCADE")
+            else
+              ActiveRecord::Base.connection.drop_table(table)
+            end
+          end
+        end
+
+        migrate!
+      end
+
+      def self.connected?
+        ActiveRecord::Base.connected?
+      end
+
+      def self.disconnect!
+        ActiveRecord::Base.clear_all_connections!
+      end
+
+      def self.default_config
+        {
+          adapter: "postgresql",
+          database: "ragdoll_development",
+          username: "ragdoll",
+          password: ENV.fetch("RAGDOLL_DATABASE_PASSWORD", nil),
+          host: "localhost",
+          port: 5432,
+          auto_migrate: true,
+          logger: Logger.new($stdout, level: Logger::WARN)
+        }
+      end
+
+      def self.migration_paths
+        [File.join(File.dirname(__FILE__), "..", "..", "..", "..", "db", "migrate")]
+      end
+    end
+  end
+end

data/lib/ragdoll/core/errors.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Ragdoll
+  module Core
+    class Error < StandardError; end
+    class EmbeddingError < Error; end
+    class SearchError < Error; end
+    class DocumentError < Error; end
+    class ConfigurationError < Error; end
+  end
+end

data/lib/ragdoll/core/model.rb

@@ -0,0 +1,45 @@
+# lib/ragdoll/core/model.rb
+# frozen_string_literal: true
+
+module Ragdoll
+  module Core
+    # Model represents a provider and model name.
+    # It is initialized with a string in the format "provider/model".
+    # The provider is optional.
+    # Can be initialized with nil or empty string.
+    Model = Data.define(:name) do
+      # @return [Symbol, nil] the provider part of the name, or nil if not present.
+      def provider
+        return nil if name.nil? || name.empty?
+
+        parts = name.split("/", 2)
+        return nil if parts.length < 2 || parts.first.empty?
+
+        parts.first.to_sym
+      end
+
+      # @return [String, nil] the model part of the name, or nil if name is nil/empty.
+      def model
+        return nil if name.nil? || name.empty?
+
+        parts = name.split("/", 2)
+        parts.length < 2 ? name : parts.last
+      end
+
+      # @return [String] the original name string, or empty string if name is nil.
+      def to_s
+        name.nil? ? "" : name
+      end
+
+      # @return [Hash] a hash representation of the model.
+      def to_h
+        { provider: provider, model: model }
+      end
+
+      # YAML serialization - save as string name
+      def encode_with(coder)
+        coder.scalar = name
+      end
+    end
+  end
+end

data/lib/ragdoll/core/shrine_config.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "shrine"
+require "shrine/storage/file_system"
+
+# Configure Shrine with filesystem storage
+Shrine.storages = {
+  cache: Shrine::Storage::FileSystem.new("tmp/uploads", prefix: "cache"),
+  store: Shrine::Storage::FileSystem.new("uploads")
+}
+
+Shrine.plugin :activerecord
+Shrine.plugin :cached_attachment_data
+Shrine.plugin :restore_cached_data
+Shrine.plugin :rack_file
+Shrine.plugin :validation_helpers
+Shrine.plugin :determine_mime_type
+
+# File uploader for documents
+class FileUploader < Shrine
+  plugin :validation_helpers
+  plugin :determine_mime_type
+
+  Attacher.validate do
+    validate_max_size 50.megabytes
+    validate_mime_type %w[
+      application/pdf
+      application/vnd.openxmlformats-officedocument.wordprocessingml.document
+      text/plain
+      text/html
+      text/markdown
+      application/json
+    ]
+  end
+end
+
+# Image uploader for image content
+class ImageUploader < Shrine
+  plugin :validation_helpers
+  plugin :determine_mime_type
+
+  Attacher.validate do
+    validate_max_size 10.megabytes
+    validate_mime_type %w[
+      image/jpeg
+      image/png
+      image/gif
+      image/webp
+      image/bmp
+      image/tiff
+    ]
+  end
+end
+
+# Audio uploader for audio content
+class AudioUploader < Shrine
+  plugin :validation_helpers
+  plugin :determine_mime_type
+
+  Attacher.validate do
+    validate_max_size 100.megabytes
+    validate_mime_type %w[
+      audio/mpeg
+      audio/wav
+      audio/mp4
+      audio/webm
+      audio/ogg
+      audio/flac
+    ]
+  end
+end

data/lib/ragdoll/core/version.rb

@@ -0,0 +1,8 @@
+# ragdoll/core/version.rb
+# frozen_string_literal: true
+
+module Ragdoll
+  module Core
+    VERSION = "0.1.3"
+  end
+end

data/lib/ragdoll/core.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "delegate"
+require "debug_me"
+include DebugMe
+$DEBUG_ME = true
+
+# require_relative "../extensions/openstruct_merge"  # File doesn't exist
+
+# Add app/models, app/jobs, app/services, and app/lib to the load path
+$LOAD_PATH.unshift(File.expand_path("../../app/models", __dir__))
+$LOAD_PATH.unshift(File.expand_path("../../app/jobs", __dir__))
+$LOAD_PATH.unshift(File.expand_path("../../app/services", __dir__))
+$LOAD_PATH.unshift(File.expand_path("../../app/lib", __dir__))
+
+require_relative "core/version"
+require_relative "core/errors"
+require_relative "core/model"
+require_relative "core/configuration"
+# Require services from app/services/ragdoll
+require "ragdoll/configuration_service"
+require "ragdoll/model_resolver"
+require_relative "core/database"
+require_relative "core/shrine_config"
+
+# Require models from app/models/ragdoll
+require "ragdoll/document"
+require "ragdoll/embedding"
+require "ragdoll/content"
+require "ragdoll/text_content"
+require "ragdoll/audio_content"
+require "ragdoll/image_content"
+require "ragdoll/document_processor"
+require "ragdoll/document_management"
+require "ragdoll/text_chunker"
+require "ragdoll/embedding_service"
+require "ragdoll/text_generation_service"
+require "ragdoll/search_engine"
+require "ragdoll/image_description_service"
+require "ragdoll/metadata_generator"
+# Require from app/lib/ragdoll
+require "ragdoll/metadata_schemas"
+# Require jobs from app/jobs/ragdoll
+require "ragdoll/generate_embeddings_job"
+require "ragdoll/generate_summary_job"
+require "ragdoll/extract_keywords_job"
+require "ragdoll/extract_text_job"
+require_relative "core/client"
+
+module Ragdoll
+  def self.config
+    @config ||= Core::Configuration.new
+  end
+
+  module Core
+    extend SingleForwardable
+
+    def self.config
+      @config ||= Configuration.new
+    end
+
+    def self.configuration
+      config
+    end
+
+    def self.configure
+      yield(config)
+    end
+
+    # Reset configuration (useful for testing)
+    def self.reset_configuration!
+      @config = Configuration.new
+      @default_client = nil
+    end
+
+    # Factory method for creating clients
+    def self.client(_config = nil)
+      Client.new
+    end
+
+    # Delegate high-level API methods to default client
+    def_delegators :default_client, :add_document, :search, :enhance_prompt,
+                   :get_document, :document_status, :list_documents, :delete_document,
+                   :update_document, :get_context, :search_similar_content,
+                   :add_directory, :stats, :healthy?, :hybrid_search
+
+    def self.default_client
+      @default_client ||= Client.new
+    end
+  end
+end

data/lib/ragdoll-core.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "ragdoll/core"

data/lib/ragdoll.rb

@@ -1,12 +1,249 @@
-# This file is the main entry point for the Ragdoll gem, requiring all necessary components.
-
 # frozen_string_literal: true
 
-# frozen_string_literal: true
+require "debug_me"
+include DebugMe
+$DEBUG_ME = true
 
-require "ragdoll/version"
-require "ragdoll/engine"
+require "delegate"
+require_relative "ragdoll/core"
 
 module Ragdoll
-  class Error < StandardError; end
+  class << self
+
+    #################
+    # Configuration #
+    #################
+
+    # Retrieve the current configuration.
+    # @example
+    #   config = Ragdoll.config
+    #   puts config.database_config[:adapter]
+    # @example
+    #   current_config = Ragdoll.configuration
+    #   puts current_config.models[:default]
+    # @return [Ragdoll::Core::Configuration] the current configuration instance.
+    def config
+      Core.config
+    end
+
+    # Configure the Ragdoll module.
+    # @yieldparam config [Ragdoll::Core::Configuration] the configuration instance to modify.
+    # @example
+    #   Ragdoll.configure do |config|
+    #     config.database_config[:adapter] = "postgres"
+    #   end
+    # @yield [Ragdoll::Core::Configuration] yields the configuration instance for modification.
+    def configure(*args, **kwargs, &block)
+      Ragdoll::Core.configure(*args, **kwargs, &block)
+    end
+
+    # Access the current configuration.
+    # @param args [Array] additional arguments for configuration.
+    # @param kwargs [Hash] keyword arguments for configuration.
+    # @return [Ragdoll::Core::Configuration] the current configuration instance.
+    def configuration(*args, **kwargs)
+      Ragdoll::Core.configuration(*args, **kwargs)
+    end
+
+    # @example
+    #   Ragdoll.reset_configuration!
+    #   puts Ragdoll.config.models[:default]
+    def reset_configuration!(*args, **kwargs)
+      Ragdoll::Core.reset_configuration!(*args, **kwargs)
+    end
+
+
+    #######################
+    # Document Management #
+    #######################
+
+    # Add a directory of documents to the system.
+    # @param path [String] the path to the directory containing documents.
+    # @example
+    #   Ragdoll.add_directory(path: "/path/to/documents", recursive: true)
+    # @param recursive [Boolean] whether to add documents from subdirectories.
+    def add_directory(*args, **kwargs)
+      Ragdoll::Core.add_directory(*args, **kwargs)
+    end
+
+    # Add a single document to the system.
+    # @example
+    #   Ragdoll.add_document(path: "/path/to/document.txt")
+    # @param path [String] the file path of the document to add.
+    def add_document(*args, **kwargs)
+      Ragdoll::Core.add_document(*args, **kwargs)
+    end
+    alias_method :add, :add_document
+
+    # Retrieve a document by its identifier.
+    # @param id [String] the identifier of the document to retrieve.
+    # @example
+    #   document = Ragdoll.get_document(id: "123")
+    #   puts document[:title] if document
+    # @return [Hash, nil] the document data or nil if not found.
+    def get_document(*args, **kwargs)
+      Ragdoll::Core.get_document(*args, **kwargs)
+    end
+    alias_method :get, :get_document
+
+    # List all documents in the system.
+    # @param options [Hash] options for listing documents, such as limit and offset.
+    # @example
+    #   documents = Ragdoll.list_documents(limit: 10)
+    #   documents.each { |doc| puts doc[:title] }
+    # @return [Array<Hash>] an array of document data.
+    def list_documents(*args, **kwargs)
+      Ragdoll::Core.list_documents(*args, **kwargs)
+    end
+    alias_method :list, :list_documents
+
+    # Delete a document by its identifier.
+    # @param id [String] the identifier of the document to delete.
+    # @example
+    #   success = Ragdoll.delete_document(id: "123")
+    #   puts "Deleted" if success
+    # @return [Boolean] true if the document was successfully deleted.
+    def delete_document(*args, **kwargs)
+      Ragdoll::Core.delete_document(*args, **kwargs)
+    end
+    alias_method :delete, :delete_document
+
+    # Get the status of a document.
+    # @param id [String] the identifier of the document to check status.
+    # @example
+    #   status = Ragdoll.document_status(id: "123")
+    #   puts status[:status]
+    # @return [Hash] the status information of the document.
+    def document_status(*args, **kwargs)
+      Ragdoll::Core.document_status(*args, **kwargs)
+    end
+    alias_method :status, :document_status
+
+    # Update a document's information.
+    # @param id [String] the identifier of the document to update.
+    # @param updates [Hash] the fields to update in the document.
+    # @example
+    #   updated_doc = Ragdoll.update_document(id: "123", title: "New Title")
+    #   puts updated_doc[:title]
+    # @return [Hash] the updated document data.
+    def update_document(*args, **kwargs)
+      Ragdoll::Core.update_document(*args, **kwargs)
+    end
+    alias_method :update, :update_document
+
+    # Retrieve all documents.
+    # @example
+    #   all_docs = Ragdoll.documents
+    #   all_docs.each { |doc| puts doc.title }
+    # @return [ActiveRecord::Relation] a relation of all documents.
+    def documents
+      Ragdoll::Document.all
+    end
+    alias_method :docs, :documents
+
+    #############
+    # Retrieval #
+    #############
+
+    # FIXME: This high-level API method should be able to take a query that is
+    #        a string or a file.  If its a file, then the downstream Process will
+    #        be responsible for reading the file and passing the contents to the
+    #        search method based upon whether the content is text, image or audio.
+
+    # Perform a search for documents based on a query.
+    # @param query [String] the search query string.
+    # @param options [Hash] additional search options, such as filters and limits.
+    # @example
+    #   response = Ragdoll.search(query: "example search")
+    #   response[:results].each { |result| puts result[:document_title] }
+    # @return [Hash] the search results.
+    def search(*args, **kwargs)
+      Ragdoll::Core.search(*args, **kwargs)
+    end
+
+    # Enhance a prompt with additional context.
+    # @param prompt [String] the original prompt to enhance.
+    # @param context_limit [Integer] the number of context chunks to include.
+    # @param options [Hash] additional options for enhancing the prompt.
+    # @example
+    #   enhanced = Ragdoll.enhance_prompt(prompt: "What is AI?", context_limit: 3)
+    #   puts enhanced[:enhanced_prompt]
+    # @return [Hash] the enhanced prompt data.
+    def enhance_prompt(*args, **kwargs)
+      Ragdoll::Core.enhance_prompt(*args, **kwargs)
+    end
+
+    # Retrieve context for a given query.
+    # @param query [String] the query to retrieve context for.
+    # @param limit [Integer] the number of context chunks to retrieve.
+    # @param options [Hash] additional options for context retrieval.
+    # @example
+    #   context = Ragdoll.get_context(query: "AI", limit: 5)
+    #   puts context[:combined_context]
+    # @return [Hash] the context data.
+    def get_context(*args, **kwargs)
+      Ragdoll::Core.get_context(*args, **kwargs)
+    end
+
+    # Search for content similar to a given query.
+    # @param query [String] the query to find similar content for.
+    # @param options [Hash] additional options for the search, such as filters and limits.
+    # @example
+    #   similar_content = Ragdoll.search_similar_content(query: "AI")
+    #   similar_content.each { |content| puts content[:document_title] }
+    # @return [Array<Hash>] an array of similar content data.
+    def search_similar_content(*args, **kwargs)
+      Ragdoll::Core.search_similar_content(*args, **kwargs)
+    end
+
+
+    ###############
+    # Misc. Stuff #
+    ###############
+
+    # Retrieve statistics about the system.
+    # @example
+    #   stats = Ragdoll.stats
+    #   puts stats[:total_documents]
+    # @return [Hash] the system statistics.
+    def stats(*args, **kwargs)
+      Ragdoll::Core.stats(*args, **kwargs)
+    end
+
+    # Check if the system is healthy.
+    # @example
+    #   puts "System is healthy" if Ragdoll.healthy?
+    # @return [Boolean] true if the system is healthy.
+    def healthy?(*args, **kwargs)
+      Ragdoll::Core.healthy?(*args, **kwargs)
+    end
+
+    # Retrieve the client instance.
+    # @example
+    #   client = Ragdoll.client
+    #   puts client.inspect
+    # @return [Ragdoll::Core::Client] the client instance.
+    def client(*args, **kwargs)
+      Ragdoll::Core.client(*args, **kwargs)
+    end
+
+    # Retrieve the version information of the Ragdoll modules.
+    # @example
+    #   versions = Ragdoll.version
+    #   versions.each { |version| puts version }
+    # @return [Array<String>] an array of version strings for each module.
+    def version
+      versions = []
+
+      ObjectSpace.each_object(Module) do |mod|
+        if mod.name =~ /^Ragdoll::\w+$/
+          if defined?(mod::VERSION) && mod::VERSION.is_a?(String)
+            versions << "#{mod.name}: #{mod::VERSION}"
+          end
+        end
+      end
+
+      versions
+    end
+  end
 end