searchkick 4.6.3 → 5.5.2

data/lib/searchkick/index_options.rb

@@ -7,21 +7,19 @@ module Searchkick
     end
 
     def index_options
-      custom_mapping = options[:mappings] || {}
-      if below70? && custom_mapping.keys.map(&:to_sym).include?(:properties)
-        # add type
-        custom_mapping = {index_type => custom_mapping}
-      end
+      # mortal symbols are garbage collected in Ruby 2.2+
+      custom_settings = (options[:settings] || {}).deep_symbolize_keys
+      custom_mappings = (options[:mappings] || {}).deep_symbolize_keys
 
       if options[:mappings] && !options[:merge_mappings]
-        settings = options[:settings] || {}
-        mappings = custom_mapping
+        settings = custom_settings
+        mappings = custom_mappings
       else
-        settings = generate_settings
-        mappings = generate_mappings.symbolize_keys.deep_merge(custom_mapping.symbolize_keys)
+        settings = generate_settings.deep_symbolize_keys.deep_merge(custom_settings)
+        mappings = generate_mappings.deep_symbolize_keys.deep_merge(custom_mappings)
       end
 
-      set_deep_paging(settings) if options[:deep_paging]
+      set_deep_paging(settings) if options[:deep_paging] || options[:max_result_window]
 
       {
         settings: settings,
@@ -162,16 +160,28 @@ module Searchkick
         settings[:number_of_replicas] = 0
       end
 
-      # TODO remove in Searchkick 5 (classic no longer supported)
       if options[:similarity]
         settings[:similarity] = {default: {type: options[:similarity]}}
       end
 
-      unless below62?
-        settings[:index] = {
-          max_ngram_diff: 49,
-          max_shingle_diff: 4
-        }
+      settings[:index] = {
+        max_ngram_diff: 49,
+        max_shingle_diff: 4
+      }
+
+      if options[:knn]
+        unless Searchkick.knn_support?
+          if Searchkick.opensearch?
+            raise Error, "knn requires OpenSearch 2.4+"
+          else
+            raise Error, "knn requires Elasticsearch 8.6+"
+          end
+        end
+
+        if Searchkick.opensearch? && options[:knn].any? { |_, v| !v[:distance].nil? }
+          # only enable if doing approximate search
+          settings[:index][:knn] = true
+        end
       end
 
       if options[:case_sensitive]
@@ -180,13 +190,8 @@ module Searchkick
         end
       end
 
-      # TODO do this last in Searchkick 5
-      settings = settings.symbolize_keys.deep_merge((options[:settings] || {}).symbolize_keys)
-
       add_synonyms(settings)
       add_search_synonyms(settings)
-      # TODO remove in Searchkick 5
-      add_wordnet(settings) if options[:wordnet]
 
       if options[:special_characters] == false
         settings[:analysis][:analyzer].each_value do |analyzer_settings|
@@ -223,19 +228,7 @@ module Searchkick
             type: "smartcn"
           }
         )
-      when "japanese"
-        settings[:analysis][:analyzer].merge!(
-          default_analyzer => {
-            type: "kuromoji"
-          },
-          searchkick_search: {
-            type: "kuromoji"
-          },
-          searchkick_search2: {
-            type: "kuromoji"
-          }
-        )
-      when "japanese2"
+      when "japanese", "japanese2"
         analyzer = {
           type: "custom",
           tokenizer: "kuromoji_tokenizer",
@@ -379,16 +372,15 @@ module Searchkick
         }
       end
 
-      mapping_options = Hash[
+      mapping_options =
         [:suggest, :word, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight, :searchable, :filterable]
-          .map { |type| [type, (options[type] || []).map(&:to_s)] }
-      ]
+          .to_h { |type| [type, (options[type] || []).map(&:to_s)] }
 
       word = options[:word] != false && (!options[:match] || options[:match] == :word)
 
       mapping_options[:searchable].delete("_all")
 
-      analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer}
+      analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer.to_s}
 
       mapping_options.values.flatten.uniq.each do |field|
         fields = {}
@@ -429,6 +421,75 @@ module Searchkick
         mapping[field] = shape_options.merge(type: "geo_shape")
       end
 
+      (options[:knn] || []).each do |field, knn_options|
+        distance = knn_options[:distance]
+
+        if Searchkick.opensearch?
+          if distance.nil?
+            # avoid server crash if method not specified
+            raise ArgumentError, "Must specify a distance for OpenSearch"
+          end
+
+          vector_options = {
+            type: "knn_vector",
+            dimension: knn_options[:dimensions]
+          }
+
+          if !distance.nil?
+            space_type =
+              case distance
+              when "cosine"
+                "cosinesimil"
+              when "euclidean"
+                "l2"
+              when "inner_product"
+                "innerproduct"
+              else
+                raise ArgumentError, "Unknown distance: #{distance}"
+              end
+
+            vector_options[:method] = {
+              name: "hnsw",
+              space_type: space_type,
+              engine: "lucene",
+              parameters: knn_options.slice(:m, :ef_construction)
+            }
+          end
+
+          mapping[field.to_s] = vector_options
+        else
+          vector_options = {
+            type: "dense_vector",
+            dims: knn_options[:dimensions],
+            index: !distance.nil?
+          }
+
+          if !distance.nil?
+            vector_options[:similarity] =
+              case distance
+              when "cosine"
+                "cosine"
+              when "euclidean"
+                "l2_norm"
+              when "inner_product"
+                "max_inner_product"
+              else
+                raise ArgumentError, "Unknown distance: #{distance}"
+              end
+
+            vector_index_options = knn_options.slice(:m, :ef_construction)
+            if vector_index_options.any?
+              # TODO no quantization by default in Searchkick 6
+              # int8_hnsw was made the default in Elasticsearch 8.14.0
+              type = Searchkick.server_below?("8.14.0") ? "hnsw" : "int8_hnsw"
+              vector_options[:index_options] = {type: type}.merge(vector_index_options)
+            end
+          end
+
+          mapping[field.to_s] = vector_options
+        end
+      end
+
       if options[:inheritance]
         mapping[:type] = keyword_mapping
       end
@@ -481,10 +542,6 @@ module Searchkick
         ]
       }
 
-      if below70?
-        mappings = {index_type => mappings}
-      end
-
       mappings
     end
 
@@ -533,14 +590,14 @@ module Searchkick
         end
         settings[:analysis][:filter][:searchkick_synonym_graph] = synonym_graph
 
-        if options[:language] == "japanese2"
+        if ["japanese", "japanese2"].include?(options[:language])
           [:searchkick_search, :searchkick_search2].each do |analyzer|
             settings[:analysis][:analyzer][analyzer][:filter].insert(4, "searchkick_synonym_graph")
           end
         else
           [:searchkick_search2, :searchkick_word_search].each do |analyzer|
             unless settings[:analysis][:analyzer][analyzer].key?(:filter)
-              raise Searchkick::Error, "Search synonyms are not supported yet for language"
+              raise Error, "Search synonyms are not supported yet for language"
             end
 
             settings[:analysis][:analyzer][analyzer][:filter].insert(2, "searchkick_synonym_graph")
@@ -549,25 +606,10 @@ module Searchkick
       end
     end
 
-    def add_wordnet(settings)
-      settings[:analysis][:filter][:searchkick_wordnet] = {
-        type: "synonym",
-        format: "wordnet",
-        synonyms_path: Searchkick.wordnet_path
-      }
-
-      settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_wordnet")
-      settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_wordnet"
-
-      %w(word_start word_middle word_end).each do |type|
-        settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_wordnet")
-      end
-    end
-
     def set_deep_paging(settings)
       if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"]
         settings[:index] ||= {}
-        settings[:index][:max_result_window] = 1_000_000_000
+        settings[:index][:max_result_window] = options[:max_result_window] || 1_000_000_000
       end
     end
 
@@ -587,14 +629,6 @@ module Searchkick
       :searchkick_index
     end
 
-    def below62?
-      Searchkick.server_below?("6.2.0")
-    end
-
-    def below70?
-      Searchkick.server_below?("7.0.0")
-    end
-
     def below73?
       Searchkick.server_below?("7.3.0")
     end

data/lib/searchkick/indexer.rb

@@ -1,3 +1,5 @@
+# thread-local (technically fiber-local) indexer
+# used to aggregate bulk callbacks across models
 module Searchkick
   class Indexer
     attr_reader :queued_items
@@ -14,15 +16,20 @@ module Searchkick
     def perform
       items = @queued_items
       @queued_items = []
-      if items.any?
-        response = Searchkick.client.bulk(body: items)
-        if response["errors"]
-          first_with_error = response["items"].map do |item|
-            (item["index"] || item["delete"] || item["update"])
-          end.find { |item| item["error"] }
-          raise Searchkick::ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
-        end
+
+      return if items.empty?
+
+      response = Searchkick.client.bulk(body: items)
+      if response["errors"]
+        # note: delete does not set error when item not found
+        first_with_error = response["items"].map do |item|
+          (item["index"] || item["delete"] || item["update"])
+        end.find { |item| item["error"] }
+        raise ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
       end
+
+      # maybe return response in future
+      nil
     end
   end
 end

data/lib/searchkick/log_subscriber.rb

@@ -0,0 +1,57 @@
+# based on https://gist.github.com/mnutt/566725
+module Searchkick
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def self.runtime=(value)
+      Thread.current[:searchkick_runtime] = value
+    end
+
+    def self.runtime
+      Thread.current[:searchkick_runtime] ||= 0
+    end
+
+    def self.reset_runtime
+      rt = runtime
+      self.runtime = 0
+      rt
+    end
+
+    def search(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      index = payload[:query][:index].is_a?(Array) ? payload[:query][:index].join(",") : payload[:query][:index]
+      type = payload[:query][:type]
+      request_params = payload[:query].except(:index, :type, :body)
+
+      params = []
+      request_params.each do |k, v|
+        params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
+      end
+
+      debug "  #{color(name, YELLOW, bold: true)}  #{index}#{type ? "/#{type.join(',')}" : ''}/_search#{params.any? ? '?' + params.join('&') : nil} #{payload[:query][:body].to_json}"
+    end
+
+    def request(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      debug "  #{color(name, YELLOW, bold: true)}  #{payload.except(:name).to_json}"
+    end
+
+    def multi_search(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      debug "  #{color(name, YELLOW, bold: true)}  _msearch #{payload[:body]}"
+    end
+  end
+end

data/lib/searchkick/middleware.rb

@@ -1,10 +1,17 @@
-require "faraday/middleware"
+require "faraday"
 
 module Searchkick
   class Middleware < Faraday::Middleware
     def call(env)
-      if env[:method] == :get && env[:url].path.to_s.end_with?("/_search")
+      path = env[:url].path.to_s
+      if path.end_with?("/_search")
         env[:request][:timeout] = Searchkick.search_timeout
+      elsif path.end_with?("/_msearch")
+        # assume no concurrent searches for timeout for now
+        searches = env[:request_body].count("\n") / 2
+        # do not allow timeout to exceed Searchkick.timeout
+        timeout = [Searchkick.search_timeout * searches, Searchkick.timeout].min
+        env[:request][:timeout] = timeout
       end
       @app.call(env)
     end

data/lib/searchkick/model.rb

@@ -4,10 +4,10 @@ module Searchkick
       options = Searchkick.model_options.merge(options)
 
       unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
-        :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
-        :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
+        :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :knn, :language,
+        :locations, :mappings, :match, :max_result_window, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
         :special_characters, :stem, :stemmer, :stem_conversions, :stem_exclusion, :stemmer_override, :suggest, :synonyms, :text_end,
-        :text_middle, :text_start, :word, :wordnet, :word_end, :word_middle, :word_start]
+        :text_middle, :text_start, :unscope, :word, :word_end, :word_middle, :word_start]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
 
       raise "Only call searchkick once per model" if respond_to?(:searchkick_index)
@@ -22,52 +22,78 @@ module Searchkick
         raise ArgumentError, "Invalid value for callbacks"
       end
 
-      index_name =
-        if options[:index_name]
-          options[:index_name]
-        elsif options[:index_prefix].respond_to?(:call)
-          -> { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
-        else
-          [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
-        end
+      base = self
+
+      mod = Module.new
+      include(mod)
+      mod.module_eval do
+        def reindex(method_name = nil, mode: nil, refresh: false)
+          self.class.searchkick_index.reindex([self], method_name: method_name, mode: mode, refresh: refresh, single: true)
+        end unless base.method_defined?(:reindex)
+
+        def similar(**options)
+          self.class.searchkick_index.similar_record(self, **options)
+        end unless base.method_defined?(:similar)
+
+        def search_data
+          data = respond_to?(:to_hash) ? to_hash : serializable_hash
+          data.delete("id")
+          data.delete("_id")
+          data.delete("_type")
+          data
+        end unless base.method_defined?(:search_data)
+
+        def should_index?
+          true
+        end unless base.method_defined?(:should_index?)
+      end
 
       class_eval do
-        cattr_reader :searchkick_options, :searchkick_klass
+        cattr_reader :searchkick_options, :searchkick_klass, instance_reader: false
 
         class_variable_set :@@searchkick_options, options.dup
         class_variable_set :@@searchkick_klass, self
-        class_variable_set :@@searchkick_index, index_name
-        class_variable_set :@@searchkick_index_cache, {}
+        class_variable_set :@@searchkick_index_cache, Searchkick::IndexCache.new
 
         class << self
           def searchkick_search(term = "*", **options, &block)
-            # TODO throw error in next major version
-            Searchkick.warn("calling search on a relation is deprecated") if Searchkick.relation?(self)
+            if Searchkick.relation?(self)
+              raise Searchkick::Error, "search must be called on model, not relation"
+            end
 
             Searchkick.search(term, model: self, **options, &block)
           end
           alias_method Searchkick.search_method_name, :searchkick_search if Searchkick.search_method_name
 
           def searchkick_index(name: nil)
-            index = name || class_variable_get(:@@searchkick_index)
-            index = index.call if index.respond_to?(:call)
+            index_name = name || searchkick_klass.searchkick_index_name
+            index_name = index_name.call if index_name.respond_to?(:call)
             index_cache = class_variable_get(:@@searchkick_index_cache)
-            index_cache[index] ||= Searchkick::Index.new(index, searchkick_options)
+            index_cache.fetch(index_name) { Searchkick::Index.new(index_name, searchkick_options) }
           end
           alias_method :search_index, :searchkick_index unless method_defined?(:search_index)
 
           def searchkick_reindex(method_name = nil, **options)
-            # TODO relation = Searchkick.relation?(self)
-            relation = (respond_to?(:current_scope) && respond_to?(:default_scoped) && current_scope && current_scope.to_sql != default_scoped.to_sql) ||
-              (respond_to?(:queryable) && queryable != unscoped.with_default_scope)
-
-            searchkick_index.reindex(searchkick_klass, method_name, scoped: relation, **options)
+            searchkick_index.reindex(self, method_name: method_name, **options)
           end
           alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex)
 
           def searchkick_index_options
             searchkick_index.index_options
           end
+
+          def searchkick_index_name
+            @searchkick_index_name ||= begin
+              options = class_variable_get(:@@searchkick_options)
+              if options[:index_name]
+                options[:index_name]
+              elsif options[:index_prefix].respond_to?(:call)
+                -> { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
+              else
+                [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
+              end
+            end
+          end
         end
 
         # always add callbacks, even when callbacks is false
@@ -78,33 +104,6 @@ module Searchkick
           after_save :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
           after_destroy :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
         end
-
-        def reindex(method_name = nil, **options)
-          RecordIndexer.new(self).reindex(method_name, **options)
-        end unless method_defined?(:reindex)
-
-        # TODO switch to keyword arguments
-        def similar(options = {})
-          self.class.searchkick_index.similar_record(self, **options)
-        end unless method_defined?(:similar)
-
-        def search_data
-          data = respond_to?(:to_hash) ? to_hash : serializable_hash
-          data.delete("id")
-          data.delete("_id")
-          data.delete("_type")
-          data
-        end unless method_defined?(:search_data)
-
-        def should_index?
-          true
-        end unless method_defined?(:should_index?)
-
-        if defined?(Cequel) && self < Cequel::Record && !method_defined?(:destroyed?)
-          def destroyed?
-            transient?
-          end
-        end
       end
     end
   end

data/lib/searchkick/process_batch_job.rb

@@ -3,34 +3,18 @@ module Searchkick
     queue_as { Searchkick.queue_name }
 
     def perform(class_name:, record_ids:, index_name: nil)
-      # separate routing from id
-      routing = Hash[record_ids.map { |r| r.split(/(?<!\|)\|(?!\|)/, 2).map { |v| v.gsub("||", "|") } }]
-      record_ids = routing.keys
+      model = Searchkick.load_model(class_name)
+      index = model.searchkick_index(name: index_name)
 
-      klass = class_name.constantize
-      scope = Searchkick.load_records(klass, record_ids)
-      scope = scope.search_import if scope.respond_to?(:search_import)
-      records = scope.select(&:should_index?)
-
-      # determine which records to delete
-      delete_ids = record_ids - records.map { |r| r.id.to_s }
-      delete_records = delete_ids.map do |id|
-        m = klass.new
-        m.id = id
-        if routing[id]
-          m.define_singleton_method(:search_routing) do
-            routing[id]
-          end
+      items =
+        record_ids.map do |r|
+          parts = r.split(/(?<!\|)\|(?!\|)/, 2)
+            .map { |v| v.gsub("||", "|") }
+          {id: parts[0], routing: parts[1]}
         end
-        m
-      end
 
-      # bulk reindex
-      index = klass.searchkick_index(name: index_name)
-      Searchkick.callbacks(:bulk) do
-        index.bulk_index(records) if records.any?
-        index.bulk_delete(delete_records) if delete_records.any?
-      end
+      relation = Searchkick.scope(model)
+      RecordIndexer.new(index).reindex_items(relation, items, method_name: nil)
     end
   end
 end

data/lib/searchkick/process_queue_job.rb

@@ -3,11 +3,12 @@ module Searchkick
     queue_as { Searchkick.queue_name }
 
     def perform(class_name:, index_name: nil, inline: false)
-      model = class_name.constantize
+      model = Searchkick.load_model(class_name)
+      index = model.searchkick_index(name: index_name)
       limit = model.searchkick_options[:batch_size] || 1000
 
       loop do
-        record_ids = model.searchkick_index(name: index_name).reindex_queue.reserve(limit: limit)
+        record_ids = index.reindex_queue.reserve(limit: limit)
         if record_ids.any?
           batch_options = {
             class_name: class_name,