ragdoll 0.1.3 → 0.1.9

data/db/migrate/008_create_ragdoll_search_results.rb

@@ -0,0 +1,49 @@
+class CreateRagdollSearchResults < ActiveRecord::Migration[7.0]
+  def change
+    # Junction table for tracking which embeddings were returned for each search
+    create_table :ragdoll_search_results,
+      comment: "Junction table linking searches to their returned embeddings" do |t|
+
+      t.references :search, null: false, foreign_key: { to_table: :ragdoll_searches },
+        comment: "Reference to the search query"
+
+      t.references :embedding, null: false, foreign_key: { to_table: :ragdoll_embeddings },
+        comment: "Reference to the returned embedding"
+
+      t.float :similarity_score, null: false,
+        comment: "Similarity score for this result"
+
+      t.integer :result_rank, null: false,
+        comment: "Ranking position of this result (1-based)"
+
+      t.boolean :clicked, default: false,
+        comment: "Whether user interacted with this result"
+
+      t.datetime :clicked_at,
+        comment: "Timestamp when result was clicked/selected"
+
+      t.timestamps null: false,
+        comment: "Standard creation and update timestamps"
+
+      ###########
+      # Indexes #
+      ###########
+
+      t.index [:search_id, :result_rank],
+        name: "idx_search_results_search_rank",
+        comment: "Index for retrieving results in ranked order"
+
+      t.index [:embedding_id, :similarity_score],
+        name: "idx_search_results_embedding_score", 
+        comment: "Index for analyzing embedding performance"
+
+      t.index :similarity_score,
+        name: "idx_search_results_similarity",
+        comment: "Index for similarity score analysis"
+
+      t.index [:clicked, :clicked_at],
+        name: "idx_search_results_clicks",
+        comment: "Index for click-through analysis"
+    end
+  end
+end

data/lib/ragdoll/core/client.rb

@@ -49,7 +49,15 @@ module Ragdoll
 
       # Get relevant context without prompt enhancement
       def get_context(query:, limit: 10, **options)
-        results = search_similar_content(query: query, limit: limit, **options)
+        search_response = search_similar_content(query: query, limit: limit, **options)
+        
+        # Handle both old format (array) and new format (hash with results/statistics)
+        if search_response.is_a?(Hash) && search_response.key?(:results)
+          results = search_response[:results]
+        else
+          # Fallback for old format
+          results = search_response || []
+        end
 
         context_chunks = results.map do |result|
           {
@@ -76,13 +84,31 @@ module Ragdoll
 
       # Semantic search++ should incorporate hybrid search
       def search(query:, **options)
-        results = search_similar_content(query: query, **options)
-
-        {
-          query: query,
-          results: results,
-          total_results: results.length
-        }
+        # Pass through tracking options to the search engine
+        search_response = search_similar_content(query: query, **options)
+        
+        # Handle both old format (array) and new format (hash with results/statistics)
+        if search_response.is_a?(Hash) && search_response.key?(:results)
+          results = search_response[:results]
+          statistics = search_response[:statistics]
+          execution_time_ms = search_response[:execution_time_ms]
+          
+          {
+            query: query,
+            results: results,
+            total_results: results.length,
+            statistics: statistics,
+            execution_time_ms: execution_time_ms
+          }
+        else
+          # Fallback for old format
+          results = search_response || []
+          {
+            query: query,
+            results: results,
+            total_results: results.length
+          }
+        end
       end
 
       # Search similar content (core functionality)
@@ -92,11 +118,52 @@ module Ragdoll
 
       # Hybrid search combining semantic and full-text search
       def hybrid_search(query:, **options)
+        start_time = Time.current
+        
+        # Extract tracking options
+        session_id = options[:session_id]
+        user_id = options[:user_id]
+        track_search = options.fetch(:track_search, true)
+        
         # Generate embedding for the query
         query_embedding = @embedding_service.generate_embedding(query)
 
         # Perform hybrid search
         results = Ragdoll::Document.hybrid_search(query, query_embedding: query_embedding, **options)
+        
+        execution_time = ((Time.current - start_time) * 1000).round
+        
+        # Record search if tracking enabled
+        if track_search && query && !query.empty?
+          begin
+            # Format results for search recording - hybrid search returns different format
+            search_results = results.map do |result|
+              {
+                embedding_id: result[:embedding_id] || result[:id],
+                similarity: result[:similarity] || result[:score] || 0.0
+              }
+            end
+            
+            # Extract filters from options
+            filters = options.slice(:document_type, :status).compact
+            search_options = options.slice(:limit, :semantic_weight, :text_weight).compact
+            
+            Ragdoll::Search.record_search(
+              query: query,
+              query_embedding: query_embedding,
+              results: search_results,
+              search_type: "hybrid",
+              filters: filters,
+              options: search_options,
+              execution_time_ms: execution_time,
+              session_id: session_id,
+              user_id: user_id
+            )
+          rescue => e
+            # Log error but don't fail the search
+            puts "Warning: Hybrid search tracking failed: #{e.message}" if ENV["RAGDOLL_DEBUG"]
+          end
+        end
 
         {
           query: query,

data/lib/ragdoll/core/model.rb

@@ -8,6 +8,14 @@ module Ragdoll
     # The provider is optional.
     # Can be initialized with nil or empty string.
     Model = Data.define(:name) do
+      def initialize(name:)
+        # Handle case where a Model object is passed instead of a string
+        if name.is_a?(Model)
+          super(name: name.name)
+        else
+          super(name: name)
+        end
+      end
       # @return [Symbol, nil] the provider part of the name, or nil if not present.
       def provider
         return nil if name.nil? || name.empty?
@@ -31,6 +39,11 @@ module Ragdoll
         name.nil? ? "" : name
       end
 
+      # @return [Boolean] true if the name is nil or empty.
+      def empty?
+        name.nil? || name.empty?
+      end
+
       # @return [Hash] a hash representation of the model.
       def to_h
         { provider: provider, model: model }

data/lib/ragdoll/core/version.rb

@@ -3,6 +3,6 @@
 
 module Ragdoll
   module Core
-    VERSION = "0.1.3"
+    VERSION = "0.1.9"
   end
 end

data/lib/ragdoll/core.rb

@@ -30,6 +30,8 @@ require "ragdoll/content"
 require "ragdoll/text_content"
 require "ragdoll/audio_content"
 require "ragdoll/image_content"
+require "ragdoll/search"
+require "ragdoll/search_result"
 require "ragdoll/document_processor"
 require "ragdoll/document_management"
 require "ragdoll/text_chunker"

data/lib/ragdoll.rb

@@ -196,6 +196,23 @@ module Ragdoll
       Ragdoll::Core.search_similar_content(*args, **kwargs)
     end
 
+    # Perform hybrid search combining semantic and full-text search.
+    # @param query [String] the search query string.
+    # @param semantic_weight [Float] weight for semantic search results (0.0 - 1.0).
+    # @param text_weight [Float] weight for full-text search results (0.0 - 1.0).
+    # @param options [Hash] additional search options, such as filters and limits.
+    # @example
+    #   results = Ragdoll.hybrid_search(
+    #     query: "machine learning",
+    #     semantic_weight: 0.7,
+    #     text_weight: 0.3
+    #   )
+    #   results.each { |result| puts result[:document_title] }
+    # @return [Array<Hash>] an array of search results combining semantic and text search.
+    def hybrid_search(*args, **kwargs)
+      Ragdoll::Core.hybrid_search(*args, **kwargs)
+    end
+
 
     ###############
     # Misc. Stuff #

data/lib/tasks/db.rake

@@ -8,9 +8,9 @@ namespace :db do
     require_relative "../ragdoll-core"
 
     config = Ragdoll::Core.configuration
-    puts "Creating database with config: #{config.database_config.inspect}"
+    puts "Creating database with config: #{config.database.inspect}"
 
-    case config.database_config[:adapter]
+    case config.database[:adapter]
     when "postgresql"
       puts "PostgreSQL database setup - running as superuser to create database and role..."
       
@@ -20,8 +20,8 @@ namespace :db do
         database: 'postgres', # Connect to postgres database initially
         username: ENV.fetch('POSTGRES_SUPERUSER', 'postgres'),
         password: ENV['POSTGRES_SUPERUSER_PASSWORD'],
-        host: config.database_config[:host] || 'localhost',
-        port: config.database_config[:port] || 5432
+        host: config.database[:host] || 'localhost',
+        port: config.database[:port] || 5432
       )
       
       # Run individual SQL commands to avoid transaction block issues
@@ -62,8 +62,8 @@ namespace :db do
         database: 'ragdoll_development',
         username: ENV.fetch('POSTGRES_SUPERUSER', 'postgres'),
         password: ENV['POSTGRES_SUPERUSER_PASSWORD'],
-        host: config.database_config[:host] || 'localhost',
-        port: config.database_config[:port] || 5432
+        host: config.database[:host] || 'localhost',
+        port: config.database[:port] || 5432
       )
       
       ActiveRecord::Base.connection.execute <<-SQL
@@ -94,11 +94,11 @@ namespace :db do
     require_relative "../ragdoll-core"
 
     config = Ragdoll::Core.configuration
-    puts "Dropping database with config: #{config.database_config.inspect}"
+    puts "Dropping database with config: #{config.database.inspect}"
 
-    case config.database_config[:adapter]
+    case config.database[:adapter]
     when "postgresql", "mysql2"
-      puts "For #{config.database_config[:adapter]}, please drop the database manually on your server"
+      puts "For #{config.database[:adapter]}, please drop the database manually on your server"
     end
 
     puts "Database drop completed"
@@ -216,9 +216,9 @@ namespace :db do
 
     config = Ragdoll::Core.configuration
 
-    case config.database_config[:adapter]
+    case config.database[:adapter]
     when "postgresql"
-      db_config = config.database_config
+      db_config = config.database
       psql_cmd = "psql"
       psql_cmd += " -h #{db_config[:host]}" if db_config[:host]
       psql_cmd += " -p #{db_config[:port]}" if db_config[:port]
@@ -227,7 +227,7 @@ namespace :db do
       puts "Opening PostgreSQL console..."
       system(psql_cmd)
     when "mysql2"
-      db_config = config.database_config
+      db_config = config.database
       mysql_cmd = "mysql"
       mysql_cmd += " -h #{db_config[:host]}" if db_config[:host]
       mysql_cmd += " -P #{db_config[:port]}" if db_config[:port]
@@ -237,7 +237,7 @@ namespace :db do
       puts "Opening MySQL console..."
       system(mysql_cmd)
     else
-      puts "Console not supported for adapter: #{config.database_config[:adapter]}"
+      puts "Console not supported for adapter: #{config.database[:adapter]}"
     end
   end