searchkick 4.6.3 → 5.0.0

data/lib/searchkick/index.rb

@@ -1,5 +1,3 @@
-require "searchkick/index_options"
-
 module Searchkick
   class Index
     attr_reader :name, :options
@@ -40,12 +38,15 @@ module Searchkick
       client.indices.exists_alias name: name
     end
 
+    # call to_h for consistent results between elasticsearch gem 7 and 8
+    # could do for all API calls, but just do for ones where return value is focus for now
     def mapping
-      client.indices.get_mapping index: name
+      client.indices.get_mapping(index: name).to_h
     end
 
+    # call to_h for consistent results between elasticsearch gem 7 and 8
     def settings
-      client.indices.get_settings index: name
+      client.indices.get_settings(index: name).to_h
     end
 
     def refresh_interval
@@ -97,7 +98,7 @@ module Searchkick
       record_data = RecordData.new(self, record).record_data
 
       # remove underscore
-      get_options = Hash[record_data.map { |k, v| [k.to_s.sub(/\A_/, "").to_sym, v] }]
+      get_options = record_data.to_h { |k, v| [k.to_s.sub(/\A_/, "").to_sym, v] }
 
       client.get(get_options)["_source"]
     end
@@ -127,32 +128,47 @@ module Searchkick
       indices
     end
 
-    # record based
-    # use helpers for notifications
-
     def store(record)
-      bulk_indexer.bulk_index([record])
+      notify(record, "Store") do
+        queue_index([record])
+      end
     end
 
     def remove(record)
-      bulk_indexer.bulk_delete([record])
+      notify(record, "Remove") do
+        queue_delete([record])
+      end
     end
 
     def update_record(record, method_name)
-      bulk_indexer.bulk_update([record], method_name)
+      notify(record, "Update") do
+        queue_update([record], method_name)
+      end
     end
 
     def bulk_delete(records)
-      bulk_indexer.bulk_delete(records)
+      return if records.empty?
+
+      notify_bulk(records, "Delete") do
+        queue_delete(records)
+      end
     end
 
     def bulk_index(records)
-      bulk_indexer.bulk_index(records)
+      return if records.empty?
+
+      notify_bulk(records, "Import") do
+        queue_index(records)
+      end
     end
     alias_method :import, :bulk_index
 
     def bulk_update(records, method_name)
-      bulk_indexer.bulk_update(records, method_name)
+      return if records.empty?
+
+      notify_bulk(records, "Update") do
+        queue_update(records, method_name)
+      end
     end
 
     def search_id(record)
@@ -163,20 +179,12 @@ module Searchkick
       RecordData.new(self, record).document_type
     end
 
-    # TODO use like: [{_index: ..., _id: ...}] in Searchkick 5
     def similar_record(record, **options)
-      like_text = retrieve(record).to_hash
-        .keep_if { |k, _| !options[:fields] || options[:fields].map(&:to_s).include?(k) }
-        .values.compact.join(" ")
-
-      options[:where] ||= {}
-      options[:where][:_id] ||= {}
-      options[:where][:_id][:not] = Array(options[:where][:_id][:not]) + [record.id.to_s]
       options[:per_page] ||= 10
-      options[:similar] = true
+      options[:similar] = [RecordData.new(self, record).record_data]
+      options[:models] ||= [record.class] unless options.key?(:model)
 
-      # TODO use index class instead of record class
-      Searchkick.search(like_text, model: record.class, **options)
+      Searchkick.search("*", **options)
     end
 
     def reload_synonyms
@@ -186,8 +194,9 @@ module Searchkick
         raise Error, "Requires Elasticsearch 7.3+" if Searchkick.server_below?("7.3.0")
         begin
           client.transport.perform_request("GET", "#{CGI.escape(name)}/_reload_search_analyzers")
-        rescue Elasticsearch::Transport::Transport::Errors::MethodNotAllowed
-          raise Error, "Requires non-OSS version of Elasticsearch"
+        rescue => e
+          raise Error, "Requires non-OSS version of Elasticsearch" if Searchkick.not_allowed_error?(e)
+          raise e
         end
       end
     end
@@ -200,29 +209,35 @@ module Searchkick
 
     # reindex
 
-    def reindex(relation, method_name, scoped:, full: false, scope: nil, **options)
+    # note: this is designed to be used internally
+    # so it does not check object matches index class
+    def reindex(object, method_name: nil, full: false, **options)
+      if object.is_a?(Array)
+        # note: purposefully skip full
+        return reindex_records(object, method_name: method_name, **options)
+      end
+
+      if !object.respond_to?(:searchkick_klass)
+        raise Error, "Cannot reindex object"
+      end
+
+      scoped = Searchkick.relation?(object)
+      # call searchkick_klass for inheritance
+      relation = scoped ? object.all : Searchkick.scope(object.searchkick_klass).all
+
       refresh = options.fetch(:refresh, !scoped)
       options.delete(:refresh)
 
-      if method_name
-        # TODO throw ArgumentError
-        Searchkick.warn("unsupported keywords: #{options.keys.map(&:inspect).join(", ")}") if options.any?
+      if method_name || (scoped && !full)
+        mode = options.delete(:mode) || :inline
+        raise ArgumentError, "unsupported keywords: #{options.keys.map(&:inspect).join(", ")}" if options.any?
 
-        # update
-        import_scope(relation, method_name: method_name, scope: scope)
-        self.refresh if refresh
-        true
-      elsif scoped && !full
-        # TODO throw ArgumentError
-        Searchkick.warn("unsupported keywords: #{options.keys.map(&:inspect).join(", ")}") if options.any?
-
-        # reindex association
-        import_scope(relation, scope: scope)
+        # import only
+        import_scope(relation, method_name: method_name, mode: mode)
         self.refresh if refresh
         true
       else
-        # full reindex
-        reindex_scope(relation, scope: scope, **options)
+        full_reindex(relation, **options)
       end
     end
 
@@ -234,15 +249,14 @@ module Searchkick
     end
 
     def import_scope(relation, **options)
-      bulk_indexer.import_scope(relation, **options)
+      relation_indexer.reindex(relation, **options)
     end
 
     def batches_left
-      bulk_indexer.batches_left
+      relation_indexer.batches_left
     end
 
-    # other
-
+    # private
     def klass_document_type(klass, ignore_type = false)
       @klass_document_type[[klass, ignore_type]] ||= begin
         if !ignore_type && klass.searchkick_klass.searchkick_options[:_type]
@@ -255,7 +269,7 @@ module Searchkick
       end
     end
 
-    # should not be public
+    # private
     def conversions_fields
       @conversions_fields ||= begin
         conversions = Array(options[:conversions])
@@ -263,10 +277,12 @@ module Searchkick
       end
     end
 
+    # private
     def suggest_fields
       @suggest_fields ||= Array(options[:suggest]).map(&:to_s)
     end
 
+    # private
     def locations_fields
       @locations_fields ||= begin
         locations = Array(options[:locations])
@@ -285,8 +301,20 @@ module Searchkick
       Searchkick.client
     end
 
-    def bulk_indexer
-      @bulk_indexer ||= BulkIndexer.new(self)
+    def queue_index(records)
+      Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).index_data })
+    end
+
+    def queue_delete(records)
+      Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(self, r).delete_data })
+    end
+
+    def queue_update(records, method_name)
+      Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).update_data(method_name) })
+    end
+
+    def relation_indexer
+      @relation_indexer ||= RelationIndexer.new(self)
     end
 
     def index_settings
@@ -297,9 +325,19 @@ module Searchkick
       index.import_scope(relation, **import_options)
     end
 
+    def reindex_records(object, mode: nil, refresh: false, **options)
+      mode ||= Searchkick.callbacks_value || @options[:callbacks] || true
+      mode = :inline if mode == :bulk
+
+      result = RecordIndexer.new(self).reindex(object, mode: mode, full: false, **options)
+      self.refresh if refresh
+      result
+    end
+
     # https://gist.github.com/jarosan/3124884
     # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
-    def reindex_scope(relation, import: true, resume: false, retain: false, async: false, refresh_interval: nil, scope: nil)
+    # TODO deprecate async in favor of mode: :async, wait: true/false
+    def full_reindex(relation, import: true, resume: false, retain: false, async: false, refresh_interval: nil, scope: nil)
       if resume
         index_name = all_indices.sort.last
         raise Searchkick::Error, "No index to resume" unless index_name
@@ -313,9 +351,9 @@ module Searchkick
       end
 
       import_options = {
-        resume: resume,
-        async: async,
+        mode: (async ? :async : :inline),
         full: true,
+        resume: resume,
         scope: scope
       }
 
@@ -367,7 +405,7 @@ module Searchkick
       end
     rescue => e
       if Searchkick.transport_error?(e) && e.message.include?("No handler for type [text]")
-        raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 6 or greater"
+        raise UnsupportedVersionError
       end
 
       raise e
@@ -382,5 +420,34 @@ module Searchkick
         raise Searchkick::Error, "Safety check failed - only run one Model.reindex per model at a time"
       end
     end
+
+    def notify(record, name)
+      if Searchkick.callbacks_value == :bulk
+        yield
+      else
+        name = "#{record.class.searchkick_klass.name} #{name}" if record && record.class.searchkick_klass
+        event = {
+          name: name,
+          id: search_id(record)
+        }
+        ActiveSupport::Notifications.instrument("request.searchkick", event) do
+          yield
+        end
+      end
+    end
+
+    def notify_bulk(records, name)
+      if Searchkick.callbacks_value == :bulk
+        yield
+      else
+        event = {
+          name: "#{records.first.class.searchkick_klass.name} #{name}",
+          count: records.size
+        }
+        ActiveSupport::Notifications.instrument("request.searchkick", event) do
+          yield
+        end
+      end
+    end
   end
 end

data/lib/searchkick/index_cache.rb

@@ -0,0 +1,30 @@
+module Searchkick
+  class IndexCache
+    def initialize(max_size: 20)
+      @data = {}
+      @mutex = Mutex.new
+      @max_size = max_size
+    end
+
+    # probably a better pattern for this
+    # but keep it simple
+    def fetch(name)
+      # thread-safe in MRI without mutex
+      # due to how context switching works
+      @mutex.synchronize do
+        if @data.key?(name)
+          @data[name]
+        else
+          @data.clear if @data.size >= @max_size
+          @data[name] = yield
+        end
+      end
+    end
+
+    def clear
+      @mutex.synchronize do
+        @data.clear
+      end
+    end
+  end
+end

data/lib/searchkick/index_options.rb

@@ -7,18 +7,16 @@ module Searchkick
     end
 
     def index_options
-      custom_mapping = options[:mappings] || {}
-      if below70? && custom_mapping.keys.map(&:to_sym).include?(:properties)
-        # add type
-        custom_mapping = {index_type => custom_mapping}
-      end
+      # mortal symbols are garbage collected in Ruby 2.2+
+      custom_settings = (options[:settings] || {}).deep_symbolize_keys
+      custom_mappings = (options[:mappings] || {}).deep_symbolize_keys
 
       if options[:mappings] && !options[:merge_mappings]
-        settings = options[:settings] || {}
-        mappings = custom_mapping
+        settings = custom_settings
+        mappings = custom_mappings
       else
-        settings = generate_settings
-        mappings = generate_mappings.symbolize_keys.deep_merge(custom_mapping.symbolize_keys)
+        settings = generate_settings.deep_symbolize_keys.deep_merge(custom_settings)
+        mappings = generate_mappings.deep_symbolize_keys.deep_merge(custom_mappings)
       end
 
       set_deep_paging(settings) if options[:deep_paging]
@@ -162,17 +160,14 @@ module Searchkick
         settings[:number_of_replicas] = 0
       end
 
-      # TODO remove in Searchkick 5 (classic no longer supported)
       if options[:similarity]
         settings[:similarity] = {default: {type: options[:similarity]}}
       end
 
-      unless below62?
-        settings[:index] = {
-          max_ngram_diff: 49,
-          max_shingle_diff: 4
-        }
-      end
+      settings[:index] = {
+        max_ngram_diff: 49,
+        max_shingle_diff: 4
+      }
 
       if options[:case_sensitive]
         settings[:analysis][:analyzer].each do |_, analyzer|
@@ -180,13 +175,8 @@ module Searchkick
         end
       end
 
-      # TODO do this last in Searchkick 5
-      settings = settings.symbolize_keys.deep_merge((options[:settings] || {}).symbolize_keys)
-
       add_synonyms(settings)
       add_search_synonyms(settings)
-      # TODO remove in Searchkick 5
-      add_wordnet(settings) if options[:wordnet]
 
       if options[:special_characters] == false
         settings[:analysis][:analyzer].each_value do |analyzer_settings|
@@ -223,19 +213,7 @@ module Searchkick
             type: "smartcn"
           }
         )
-      when "japanese"
-        settings[:analysis][:analyzer].merge!(
-          default_analyzer => {
-            type: "kuromoji"
-          },
-          searchkick_search: {
-            type: "kuromoji"
-          },
-          searchkick_search2: {
-            type: "kuromoji"
-          }
-        )
-      when "japanese2"
+      when "japanese", "japanese2"
         analyzer = {
           type: "custom",
           tokenizer: "kuromoji_tokenizer",
@@ -379,16 +357,15 @@ module Searchkick
         }
       end
 
-      mapping_options = Hash[
+      mapping_options =
         [:suggest, :word, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight, :searchable, :filterable]
-          .map { |type| [type, (options[type] || []).map(&:to_s)] }
-      ]
+          .to_h { |type| [type, (options[type] || []).map(&:to_s)] }
 
       word = options[:word] != false && (!options[:match] || options[:match] == :word)
 
       mapping_options[:searchable].delete("_all")
 
-      analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer}
+      analyzed_field_options = {type: default_type, index: true, analyzer: default_analyzer.to_s}
 
       mapping_options.values.flatten.uniq.each do |field|
         fields = {}
@@ -481,10 +458,6 @@ module Searchkick
         ]
       }
 
-      if below70?
-        mappings = {index_type => mappings}
-      end
-
       mappings
     end
 
@@ -533,7 +506,7 @@ module Searchkick
         end
         settings[:analysis][:filter][:searchkick_synonym_graph] = synonym_graph
 
-        if options[:language] == "japanese2"
+        if ["japanese", "japanese2"].include?(options[:language])
           [:searchkick_search, :searchkick_search2].each do |analyzer|
             settings[:analysis][:analyzer][analyzer][:filter].insert(4, "searchkick_synonym_graph")
           end
@@ -549,21 +522,6 @@ module Searchkick
       end
     end
 
-    def add_wordnet(settings)
-      settings[:analysis][:filter][:searchkick_wordnet] = {
-        type: "synonym",
-        format: "wordnet",
-        synonyms_path: Searchkick.wordnet_path
-      }
-
-      settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_wordnet")
-      settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_wordnet"
-
-      %w(word_start word_middle word_end).each do |type|
-        settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_wordnet")
-      end
-    end
-
     def set_deep_paging(settings)
       if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"]
         settings[:index] ||= {}
@@ -587,14 +545,6 @@ module Searchkick
       :searchkick_index
     end
 
-    def below62?
-      Searchkick.server_below?("6.2.0")
-    end
-
-    def below70?
-      Searchkick.server_below?("7.0.0")
-    end
-
     def below73?
       Searchkick.server_below?("7.3.0")
     end

data/lib/searchkick/indexer.rb

@@ -1,3 +1,5 @@
+# thread-local (technically fiber-local) indexer
+# used to aggregate bulk callbacks across models
 module Searchkick
   class Indexer
     attr_reader :queued_items
@@ -14,15 +16,20 @@ module Searchkick
     def perform
       items = @queued_items
       @queued_items = []
-      if items.any?
-        response = Searchkick.client.bulk(body: items)
-        if response["errors"]
-          first_with_error = response["items"].map do |item|
-            (item["index"] || item["delete"] || item["update"])
-          end.find { |item| item["error"] }
-          raise Searchkick::ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
-        end
+
+      return if items.empty?
+
+      response = Searchkick.client.bulk(body: items)
+      if response["errors"]
+        # note: delete does not set error when item not found
+        first_with_error = response["items"].map do |item|
+          (item["index"] || item["delete"] || item["update"])
+        end.find { |item| item["error"] }
+        raise ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
       end
+
+      # maybe return response in future
+      nil
     end
   end
 end

data/lib/searchkick/log_subscriber.rb

@@ -0,0 +1,57 @@
+# based on https://gist.github.com/mnutt/566725
+module Searchkick
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def self.runtime=(value)
+      Thread.current[:searchkick_runtime] = value
+    end
+
+    def self.runtime
+      Thread.current[:searchkick_runtime] ||= 0
+    end
+
+    def self.reset_runtime
+      rt = runtime
+      self.runtime = 0
+      rt
+    end
+
+    def search(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      index = payload[:query][:index].is_a?(Array) ? payload[:query][:index].join(",") : payload[:query][:index]
+      type = payload[:query][:type]
+      request_params = payload[:query].except(:index, :type, :body)
+
+      params = []
+      request_params.each do |k, v|
+        params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
+      end
+
+      debug "  #{color(name, YELLOW, true)}  #{index}#{type ? "/#{type.join(',')}" : ''}/_search#{params.any? ? '?' + params.join('&') : nil} #{payload[:query][:body].to_json}"
+    end
+
+    def request(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      debug "  #{color(name, YELLOW, true)}  #{payload.except(:name).to_json}"
+    end
+
+    def multi_search(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      debug "  #{color(name, YELLOW, true)}  _msearch #{payload[:body]}"
+    end
+  end
+end

data/lib/searchkick/middleware.rb

@@ -1,4 +1,4 @@
-require "faraday/middleware"
+require "faraday"
 
 module Searchkick
   class Middleware < Faraday::Middleware