searchkick 2.3.2 → 4.4.1

data/lib/searchkick/logging.rb

@@ -129,10 +129,10 @@ module Searchkick
   end
 
   module SearchkickWithInstrumentation
-    def multi_search(searches, **options)
+    def multi_search(searches)
       event = {
         name: "Multi Search",
-        body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join
+        body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join,
       }
       ActiveSupport::Notifications.instrument("multi_search.searchkick", event) do
         super
@@ -162,12 +162,17 @@ module Searchkick
 
       payload = event.payload
       name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
-      type = payload[:query][:type]
+
       index = payload[:query][:index].is_a?(Array) ? payload[:query][:index].join(",") : payload[:query][:index]
+      type = payload[:query][:type]
+      request_params = payload[:query].except(:index, :type, :body)
 
-      # no easy way to tell which host the client will use
-      host = Searchkick.client.transport.hosts.first
-      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -d '#{payload[:query][:body].to_json}'"
+      params = []
+      request_params.each do |k, v|
+        params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
+      end
+
+      debug "  #{color(name, YELLOW, true)}  #{index}#{type ? "/#{type.join(',')}" : ''}/_search#{params.any? ? '?' + params.join('&') : nil} #{payload[:query][:body].to_json}"
     end
 
     def request(event)
@@ -187,9 +192,7 @@ module Searchkick
       payload = event.payload
       name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
 
-      # no easy way to tell which host the client will use
-      host = Searchkick.client.transport.hosts.first
-      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -d '#{payload[:body]}'"
+      debug "  #{color(name, YELLOW, true)}  _msearch #{payload[:body]}"
     end
   end
 
@@ -232,10 +235,11 @@ module Searchkick
     end
   end
 end
-Searchkick::Query.send(:prepend, Searchkick::QueryWithInstrumentation)
-Searchkick::Index.send(:prepend, Searchkick::IndexWithInstrumentation)
-Searchkick::Indexer.send(:prepend, Searchkick::IndexerWithInstrumentation)
-Searchkick.singleton_class.send(:prepend, Searchkick::SearchkickWithInstrumentation)
+
+Searchkick::Query.prepend(Searchkick::QueryWithInstrumentation)
+Searchkick::Index.prepend(Searchkick::IndexWithInstrumentation)
+Searchkick::Indexer.prepend(Searchkick::IndexerWithInstrumentation)
+Searchkick.singleton_class.prepend(Searchkick::SearchkickWithInstrumentation)
 Searchkick::LogSubscriber.attach_to :searchkick
 ActiveSupport.on_load(:action_controller) do
   include Searchkick::ControllerRuntime

data/lib/searchkick/model.rb

@@ -1,10 +1,12 @@
 module Searchkick
   module Model
     def searchkick(**options)
-      unknown_keywords = options.keys - [:_all, :batch_size, :callbacks, :conversions, :default_fields,
-        :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :language,
-        :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :settings, :similarity,
-        :special_characters, :stem_conversions, :suggest, :synonyms, :text_end,
+      options = Searchkick.model_options.merge(options)
+
+      unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
+        :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
+        :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
+        :special_characters, :stem, :stem_conversions, :stem_exclusion, :stemmer_override, :suggest, :synonyms, :text_end,
         :text_middle, :text_start, :word, :wordnet, :word_end, :word_middle, :word_start]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
 
@@ -12,63 +14,54 @@ module Searchkick
 
       Searchkick.models << self
 
+      options[:_type] ||= -> { searchkick_index.klass_document_type(self, true) }
+      options[:class_name] = model_name.name
+
+      callbacks = options.key?(:callbacks) ? options[:callbacks] : :inline
+      unless [:inline, true, false, :async, :queue].include?(callbacks)
+        raise ArgumentError, "Invalid value for callbacks"
+      end
+
+      index_name =
+        if options[:index_name]
+          options[:index_name]
+        elsif options[:index_prefix].respond_to?(:call)
+          -> { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
+        else
+          [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
+        end
+
       class_eval do
         cattr_reader :searchkick_options, :searchkick_klass
 
-        callbacks = options.key?(:callbacks) ? options[:callbacks] : true
-
         class_variable_set :@@searchkick_options, options.dup
         class_variable_set :@@searchkick_klass, self
-        class_variable_set :@@searchkick_callbacks, callbacks
-        class_variable_set :@@searchkick_index, options[:index_name] ||
-          (options[:index_prefix].respond_to?(:call) && proc { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }) ||
-          [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
+        class_variable_set :@@searchkick_index, index_name
+        class_variable_set :@@searchkick_index_cache, {}
 
         class << self
           def searchkick_search(term = "*", **options, &block)
-            Searchkick.search(term, {model: self}.merge(options), &block)
+            # TODO throw error in next major version
+            Searchkick.warn("calling search on a relation is deprecated") if Searchkick.relation?(self)
+
+            Searchkick.search(term, model: self, **options, &block)
           end
           alias_method Searchkick.search_method_name, :searchkick_search if Searchkick.search_method_name
 
-          def searchkick_index
-            index = class_variable_get :@@searchkick_index
-            index = index.call if index.respond_to? :call
-            Searchkick::Index.new(index, searchkick_options)
+          def searchkick_index(name: nil)
+            index = name || class_variable_get(:@@searchkick_index)
+            index = index.call if index.respond_to?(:call)
+            index_cache = class_variable_get(:@@searchkick_index_cache)
+            index_cache[index] ||= Searchkick::Index.new(index, searchkick_options)
           end
           alias_method :search_index, :searchkick_index unless method_defined?(:search_index)
 
-          def enable_search_callbacks
-            class_variable_set :@@searchkick_callbacks, true
-          end
-
-          def disable_search_callbacks
-            class_variable_set :@@searchkick_callbacks, false
-          end
-
-          def search_callbacks?
-            class_variable_get(:@@searchkick_callbacks) && Searchkick.callbacks?
-          end
-
-          def searchkick_reindex(method_name = nil, full: false, **options)
-            scoped = (respond_to?(:current_scope) && respond_to?(:default_scoped) && current_scope && current_scope.to_sql != default_scoped.to_sql) ||
+          def searchkick_reindex(method_name = nil, **options)
+            # TODO relation = Searchkick.relation?(self)
+            relation = (respond_to?(:current_scope) && respond_to?(:default_scoped) && current_scope && current_scope.to_sql != default_scoped.to_sql) ||
               (respond_to?(:queryable) && queryable != unscoped.with_default_scope)
 
-            refresh = options.fetch(:refresh, !scoped)
-
-            if method_name
-              # update
-              searchkick_index.import_scope(searchkick_klass, method_name: method_name)
-              searchkick_index.refresh if refresh
-              true
-            elsif scoped && !full
-              # reindex association
-              searchkick_index.import_scope(searchkick_klass)
-              searchkick_index.refresh if refresh
-              true
-            else
-              # full reindex
-              searchkick_index.reindex_scope(searchkick_klass, options)
-            end
+            searchkick_index.reindex(searchkick_klass, method_name, scoped: relation, **options)
           end
           alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex)
 
@@ -77,68 +70,30 @@ module Searchkick
           end
         end
 
-        callback_name = callbacks == :async ? :reindex_async : :reindex
+        # always add callbacks, even when callbacks is false
+        # so Model.callbacks block can be used
         if respond_to?(:after_commit)
-          after_commit callback_name, if: proc { self.class.search_callbacks? }
+          after_commit :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
         elsif respond_to?(:after_save)
-          after_save callback_name, if: proc { self.class.search_callbacks? }
-          after_destroy callback_name, if: proc { self.class.search_callbacks? }
+          after_save :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
+          after_destroy :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
         end
 
-        def reindex(method_name = nil, refresh: false, async: false, mode: nil)
-          klass_options = self.class.searchkick_index.options
-
-          if mode.nil?
-            mode =
-              if async
-                :async
-              elsif Searchkick.callbacks_value
-                Searchkick.callbacks_value
-              elsif klass_options.key?(:callbacks) && klass_options[:callbacks] != :async
-                # TODO remove 2nd condition in next major version
-                klass_options[:callbacks]
-              end
-          end
-
-          case mode
-          when :queue
-            if method_name
-              raise Searchkick::Error, "Partial reindex not supported with queue option"
-            else
-              self.class.searchkick_index.reindex_queue.push(id.to_s)
-            end
-          when :async
-            if method_name
-              # TODO support Mongoid and NoBrainer and non-id primary keys
-              Searchkick::BulkReindexJob.perform_later(
-                class_name: self.class.name,
-                record_ids: [id.to_s],
-                method_name: method_name ? method_name.to_s : nil
-              )
-            else
-              self.class.searchkick_index.reindex_record_async(self)
-            end
-          else
-            if method_name
-              self.class.searchkick_index.update_record(self, method_name)
-            else
-              self.class.searchkick_index.reindex_record(self)
-            end
-            self.class.searchkick_index.refresh if refresh
-          end
+        def reindex(method_name = nil, **options)
+          RecordIndexer.new(self).reindex(method_name, **options)
         end unless method_defined?(:reindex)
 
-        # TODO remove this method in next major version
-        def reindex_async
-          reindex(async: true)
-        end unless method_defined?(:reindex_async)
-
+        # TODO switch to keyword arguments
         def similar(options = {})
-          self.class.searchkick_index.similar_record(self, options)
+          self.class.searchkick_index.similar_record(self, **options)
         end unless method_defined?(:similar)
 
         def search_data
-          respond_to?(:to_hash) ? to_hash : serializable_hash
+          data = respond_to?(:to_hash) ? to_hash : serializable_hash
+          data.delete("id")
+          data.delete("_id")
+          data.delete("_type")
+          data
         end unless method_defined?(:search_data)
 
         def should_index?

data/lib/searchkick/multi_search.rb

@@ -2,25 +2,24 @@ module Searchkick
   class MultiSearch
     attr_reader :queries
 
-    def initialize(queries, retry_misspellings: false)
+    def initialize(queries)
       @queries = queries
-      @retry_misspellings = retry_misspellings
     end
 
     def perform
       if queries.any?
-        perform_search(queries, retry_misspellings: @retry_misspellings)
+        perform_search(queries)
       end
     end
 
     private
 
-    def perform_search(queries, retry_misspellings: true)
-      responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
+    def perform_search(search_queries, perform_retry: true)
+      responses = client.msearch(body: search_queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
 
       retry_queries = []
-      queries.each_with_index do |query, i|
-        if retry_misspellings && query.retry_misspellings?(responses[i])
+      search_queries.each_with_index do |query, i|
+        if perform_retry && query.retry_misspellings?(responses[i])
           query.send(:prepare) # okay, since we don't want to expose this method outside Searchkick
           retry_queries << query
         else
@@ -28,11 +27,11 @@ module Searchkick
         end
       end
 
-      if retry_misspellings && retry_queries.any?
-        perform_search(retry_queries, retry_misspellings: false)
+      if retry_queries.any?
+        perform_search(retry_queries, perform_retry: false)
       end
 
-      queries
+      search_queries
     end
 
     def client

data/lib/searchkick/process_batch_job.rb

@@ -1,8 +1,12 @@
 module Searchkick
   class ProcessBatchJob < ActiveJob::Base
-    queue_as :searchkick
+    queue_as { Searchkick.queue_name }
+
+    def perform(class_name:, record_ids:, index_name: nil)
+      # separate routing from id
+      routing = Hash[record_ids.map { |r| r.split(/(?<!\|)\|(?!\|)/, 2).map { |v| v.gsub("||", "|") } }]
+      record_ids = routing.keys
 
-    def perform(class_name:, record_ids:)
       klass = class_name.constantize
       scope = Searchkick.load_records(klass, record_ids)
       scope = scope.search_import if scope.respond_to?(:search_import)
@@ -10,10 +14,19 @@ module Searchkick
 
       # determine which records to delete
       delete_ids = record_ids - records.map { |r| r.id.to_s }
-      delete_records = delete_ids.map { |id| m = klass.new; m.id = id; m }
+      delete_records = delete_ids.map do |id|
+        m = klass.new
+        m.id = id
+        if routing[id]
+          m.define_singleton_method(:search_routing) do
+            routing[id]
+          end
+        end
+        m
+      end
 
       # bulk reindex
-      index = klass.searchkick_index
+      index = klass.searchkick_index(name: index_name)
       Searchkick.callbacks(:bulk) do
         index.bulk_index(records) if records.any?
         index.bulk_delete(delete_records) if delete_records.any?

data/lib/searchkick/process_queue_job.rb

@@ -1,22 +1,30 @@
 module Searchkick
   class ProcessQueueJob < ActiveJob::Base
-    queue_as :searchkick
+    queue_as { Searchkick.queue_name }
 
-    def perform(class_name:)
+    def perform(class_name:, index_name: nil, inline: false)
       model = class_name.constantize
+      limit = model.searchkick_options[:batch_size] || 1000
 
-      limit = model.searchkick_index.options[:batch_size] || 1000
-      record_ids = model.searchkick_index.reindex_queue.reserve(limit: limit)
-      if record_ids.any?
-        Searchkick::ProcessBatchJob.perform_later(
-          class_name: model.name,
-          record_ids: record_ids
-        )
-        # TODO when moving to reliable queuing, mark as complete
+      loop do
+        record_ids = model.searchkick_index(name: index_name).reindex_queue.reserve(limit: limit)
+        if record_ids.any?
+          batch_options = {
+            class_name: class_name,
+            record_ids: record_ids,
+            index_name: index_name
+          }
 
-        if record_ids.size == limit
-          Searchkick::ProcessQueueJob.perform_later(class_name: class_name)
+          if inline
+            # use new.perform to avoid excessive logging
+            Searchkick::ProcessBatchJob.new.perform(**batch_options)
+          else
+            Searchkick::ProcessBatchJob.perform_later(**batch_options)
+          end
+
+          # TODO when moving to reliable queuing, mark as complete
         end
+        break unless record_ids.size == limit
       end
     end
   end