searchkick 2.5.0 → 3.0.0

data/lib/searchkick/index_options.rb

@@ -4,35 +4,23 @@ module Searchkick
       options = @options
       language = options[:language]
       language = language.call if language.respond_to?(:call)
-      type = options[:_type] || :_default_
-      type = type.call if type.respond_to?(:call)
+      index_type = options[:_type]
+      index_type = index_type.call if index_type.respond_to?(:call)
 
       if options[:mappings] && !options[:merge_mappings]
         settings = options[:settings] || {}
         mappings = options[:mappings]
       else
-        below22 = Searchkick.server_below?("2.2.0")
-        below50 = Searchkick.server_below?("5.0.0-alpha1")
         below60 = Searchkick.server_below?("6.0.0-alpha1")
-        default_type = below50 ? "string" : "text"
+        default_type = "text"
         default_analyzer = :searchkick_index
-        keyword_mapping =
-          if below50
-            {
-              type: default_type,
-              index: "not_analyzed"
-            }
-          else
-            {
-              type: "keyword"
-            }
-          end
+        keyword_mapping = {type: "keyword"}
 
-        all = options.key?(:_all) ? options[:_all] : below60
-        index_true_value = below50 ? "analyzed" : true
-        index_false_value = below50 ? "no" : false
+        all = options.key?(:_all) ? options[:_all] : false
+        index_true_value = true
+        index_false_value = false
 
-        keyword_mapping[:ignore_above] = (options[:ignore_above] || 30000) unless below22
+        keyword_mapping[:ignore_above] = options[:ignore_above] || 30000
 
         settings = {
           analysis: {
@@ -40,7 +28,7 @@ module Searchkick
               searchkick_keyword: {
                 type: "custom",
                 tokenizer: "keyword",
-                filter: ["lowercase"] + (options[:stem_conversions] == false ? [] : ["searchkick_stemmer"])
+                filter: ["lowercase"] + (options[:stem_conversions] ? ["searchkick_stemmer"] : [])
               },
               default_analyzer => {
                 type: "custom",
@@ -139,7 +127,6 @@ module Searchkick
               },
               searchkick_stemmer: {
                 # use stemmer if language is lowercase, snowball otherwise
-                # TODO deprecate language option in favor of stemmer
                 type: language == language.to_s.downcase ? "stemmer" : "snowball",
                 language: language || "English"
               }
@@ -155,6 +142,22 @@ module Searchkick
           }
         }
 
+        if language == "chinese"
+          settings[:analysis][:analyzer].merge!(
+            default_analyzer => {
+              type: "ik_smart"
+            },
+            searchkick_search: {
+              type: "ik_smart"
+            },
+            searchkick_search2: {
+              type: "ik_max_word"
+            }
+          )
+
+          settings[:analysis][:filter].delete(:searchkick_stemmer)
+        end
+
         if Searchkick.env == "test"
           settings[:number_of_shards] = 1
           settings[:number_of_replicas] = 0
@@ -175,7 +178,7 @@ module Searchkick
           settings[:analysis][:filter][:searchkick_synonym] = {
             type: "synonym",
             # only remove a single space from synonyms so three-word synonyms will fail noisily instead of silently
-            synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.map { |s| s.sub(/\s+/, "") }.join(",") : s }.map(&:downcase)
+            synonyms: synonyms.select { |s| s.size > 1 }.map { |s| s.is_a?(Array) ? s.map { |s2| s2.sub(/\s+/, "") }.join(",") : s }.map(&:downcase)
           }
           # choosing a place for the synonym filter when stemming is not easy
           # https://groups.google.com/forum/#!topic/elasticsearch/p7qcQlgHdB8
@@ -210,7 +213,7 @@ module Searchkick
         end
 
         if options[:special_characters] == false
-          settings[:analysis][:analyzer].each do |_, analyzer_settings|
+          settings[:analysis][:analyzer].each_value do |analyzer_settings|
             analyzer_settings[:filter].reject! { |f| f == "asciifolding" }
           end
         end
@@ -320,7 +323,7 @@ module Searchkick
         multi_field = dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}"))
 
         mappings = {
-          type => {
+          index_type => {
             properties: mapping,
             _routing: routing,
             # https://gist.github.com/kimchy/2898285
@@ -338,7 +341,7 @@ module Searchkick
 
         if below60
           all_enabled = all && (!options[:searchable] || options[:searchable].to_a.map(&:to_s).include?("_all"))
-          mappings[type][:_all] = all_enabled ? analyzed_field_options : {enabled: false}
+          mappings[index_type][:_all] = all_enabled ? analyzed_field_options : {enabled: false}
         end
 
         mappings = mappings.deep_merge(options[:mappings] || {})

data/lib/searchkick/logging.rb

@@ -129,7 +129,7 @@ module Searchkick
   end
 
   module SearchkickWithInstrumentation
-    def multi_search(searches, **options)
+    def multi_search(searches)
       event = {
         name: "Multi Search",
         body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join
@@ -167,7 +167,7 @@ module Searchkick
 
       # no easy way to tell which host the client will use
       host = Searchkick.client.transport.hosts.first
-      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -d '#{payload[:query][:body].to_json}'"
+      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -H 'Content-Type: application/json' -d '#{payload[:query][:body].to_json}'"
     end
 
     def request(event)
@@ -189,7 +189,7 @@ module Searchkick
 
       # no easy way to tell which host the client will use
       host = Searchkick.client.transport.hosts.first
-      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -d '#{payload[:body]}'"
+      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -H 'Content-Type: application/json' -d '#{payload[:body]}'"
     end
   end
 
@@ -232,10 +232,11 @@ module Searchkick
     end
   end
 end
-Searchkick::Query.send(:prepend, Searchkick::QueryWithInstrumentation)
-Searchkick::Index.send(:prepend, Searchkick::IndexWithInstrumentation)
-Searchkick::Indexer.send(:prepend, Searchkick::IndexerWithInstrumentation)
-Searchkick.singleton_class.send(:prepend, Searchkick::SearchkickWithInstrumentation)
+
+Searchkick::Query.prepend(Searchkick::QueryWithInstrumentation)
+Searchkick::Index.prepend(Searchkick::IndexWithInstrumentation)
+Searchkick::Indexer.prepend(Searchkick::IndexerWithInstrumentation)
+Searchkick.singleton_class.prepend(Searchkick::SearchkickWithInstrumentation)
 Searchkick::LogSubscriber.attach_to :searchkick
 ActiveSupport.on_load(:action_controller) do
   include Searchkick::ControllerRuntime

data/lib/searchkick/model.rb

@@ -1,6 +1,8 @@
 module Searchkick
   module Model
     def searchkick(**options)
+      options = Searchkick.model_options.merge(options)
+
       unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :conversions, :default_fields,
         :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
         :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :settings, :similarity,
@@ -12,19 +14,29 @@ module Searchkick
 
       Searchkick.models << self
 
-      options[:_type] ||= -> { searchkick_index.klass_document_type(self, true) } if options[:inheritance]
+      options[:_type] ||= -> { searchkick_index.klass_document_type(self, true) }
+
+      callbacks = options.key?(:callbacks) ? options[:callbacks] : true
+      unless [true, false, :async, :queue].include?(callbacks)
+        raise ArgumentError, "Invalid value for callbacks"
+      end
+
+      index_name =
+        if options[:index_name]
+          options[:index_name]
+        elsif options[:index_prefix].respond_to?(:call)
+          -> { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
+        else
+          [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
+        end
 
       class_eval do
         cattr_reader :searchkick_options, :searchkick_klass
 
-        callbacks = options.key?(:callbacks) ? options[:callbacks] : true
-
         class_variable_set :@@searchkick_options, options.dup
         class_variable_set :@@searchkick_klass, self
-        class_variable_set :@@searchkick_callbacks, callbacks
-        class_variable_set :@@searchkick_index, options[:index_name] ||
-          (options[:index_prefix].respond_to?(:call) && proc { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }) ||
-          [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
+        class_variable_set :@@searchkick_index, index_name
+        class_variable_set :@@searchkick_index_cache, {}
 
         class << self
           def searchkick_search(term = "*", **options, &block)
@@ -33,44 +45,18 @@ module Searchkick
           alias_method Searchkick.search_method_name, :searchkick_search if Searchkick.search_method_name
 
           def searchkick_index
-            index = class_variable_get :@@searchkick_index
-            index = index.call if index.respond_to? :call
-            Searchkick::Index.new(index, searchkick_options)
+            index = class_variable_get(:@@searchkick_index)
+            index = index.call if index.respond_to?(:call)
+            index_cache = class_variable_get(:@@searchkick_index_cache)
+            index_cache[index] ||= Searchkick::Index.new(index, searchkick_options)
           end
           alias_method :search_index, :searchkick_index unless method_defined?(:search_index)
 
-          def enable_search_callbacks
-            class_variable_set :@@searchkick_callbacks, true
-          end
-
-          def disable_search_callbacks
-            class_variable_set :@@searchkick_callbacks, false
-          end
-
-          def search_callbacks?
-            class_variable_get(:@@searchkick_callbacks) && Searchkick.callbacks?
-          end
-
-          def searchkick_reindex(method_name = nil, full: false, **options)
+          def searchkick_reindex(method_name = nil, **options)
             scoped = (respond_to?(:current_scope) && respond_to?(:default_scoped) && current_scope && current_scope.to_sql != default_scoped.to_sql) ||
               (respond_to?(:queryable) && queryable != unscoped.with_default_scope)
 
-            refresh = options.fetch(:refresh, !scoped)
-
-            if method_name
-              # update
-              searchkick_index.import_scope(searchkick_klass, method_name: method_name)
-              searchkick_index.refresh if refresh
-              true
-            elsif scoped && !full
-              # reindex association
-              searchkick_index.import_scope(searchkick_klass)
-              searchkick_index.refresh if refresh
-              true
-            else
-              # full reindex
-              searchkick_index.reindex_scope(searchkick_klass, options)
-            end
+            searchkick_index.reindex(searchkick_klass, method_name, scoped: scoped, **options)
           end
           alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex)
 
@@ -79,68 +65,29 @@ module Searchkick
           end
         end
 
-        callback_name = callbacks == :async ? :reindex_async : :reindex
+        # always add callbacks, even when callbacks is false
+        # so Model.callbacks block can be used
         if respond_to?(:after_commit)
-          after_commit callback_name, if: proc { self.class.search_callbacks? }
+          after_commit :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
         elsif respond_to?(:after_save)
-          after_save callback_name, if: proc { self.class.search_callbacks? }
-          after_destroy callback_name, if: proc { self.class.search_callbacks? }
+          after_save :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
+          after_destroy :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
         end
 
-        def reindex(method_name = nil, refresh: false, async: false, mode: nil)
-          klass_options = self.class.searchkick_index.options
-
-          if mode.nil?
-            mode =
-              if async
-                :async
-              elsif Searchkick.callbacks_value
-                Searchkick.callbacks_value
-              elsif klass_options.key?(:callbacks) && klass_options[:callbacks] != :async
-                # TODO remove 2nd condition in next major version
-                klass_options[:callbacks]
-              end
-          end
-
-          case mode
-          when :queue
-            if method_name
-              raise Searchkick::Error, "Partial reindex not supported with queue option"
-            else
-              self.class.searchkick_index.reindex_queue.push(id.to_s)
-            end
-          when :async
-            if method_name
-              # TODO support Mongoid and NoBrainer and non-id primary keys
-              Searchkick::BulkReindexJob.perform_later(
-                class_name: self.class.name,
-                record_ids: [id.to_s],
-                method_name: method_name ? method_name.to_s : nil
-              )
-            else
-              self.class.searchkick_index.reindex_record_async(self)
-            end
-          else
-            if method_name
-              self.class.searchkick_index.update_record(self, method_name)
-            else
-              self.class.searchkick_index.reindex_record(self)
-            end
-            self.class.searchkick_index.refresh if refresh
-          end
+        def reindex(method_name = nil, **options)
+          RecordIndexer.new(self).reindex(method_name, **options)
         end unless method_defined?(:reindex)
 
-        # TODO remove this method in next major version
-        def reindex_async
-          reindex(async: true)
-        end unless method_defined?(:reindex_async)
-
         def similar(options = {})
           self.class.searchkick_index.similar_record(self, options)
         end unless method_defined?(:similar)
 
         def search_data
-          respond_to?(:to_hash) ? to_hash : serializable_hash
+          data = respond_to?(:to_hash) ? to_hash : serializable_hash
+          data.delete("id")
+          data.delete("_id")
+          data.delete("_type")
+          data
         end unless method_defined?(:search_data)
 
         def should_index?

data/lib/searchkick/multi_search.rb

@@ -2,25 +2,24 @@ module Searchkick
   class MultiSearch
     attr_reader :queries
 
-    def initialize(queries, retry_misspellings: false)
+    def initialize(queries)
       @queries = queries
-      @retry_misspellings = retry_misspellings
     end
 
     def perform
       if queries.any?
-        perform_search(queries, retry_misspellings: @retry_misspellings)
+        perform_search(queries)
       end
     end
 
     private
 
-    def perform_search(queries, retry_misspellings: true)
+    def perform_search(queries)
       responses = client.msearch(body: queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
 
       retry_queries = []
       queries.each_with_index do |query, i|
-        if retry_misspellings && query.retry_misspellings?(responses[i])
+        if query.retry_misspellings?(responses[i])
           query.send(:prepare) # okay, since we don't want to expose this method outside Searchkick
           retry_queries << query
         else
@@ -28,8 +27,8 @@ module Searchkick
         end
       end
 
-      if retry_misspellings && retry_queries.any?
-        perform_search(retry_queries, retry_misspellings: false)
+      if retry_queries.any?
+        perform_search(retry_queries)
       end
 
       queries

data/lib/searchkick/query.rb

@@ -135,7 +135,7 @@ module Searchkick
         if searchkick_index
           puts "Model Search Data"
           begin
-            pp klass.first(3).map { |r| {index: searchkick_index.record_data(r).merge(data: searchkick_index.send(:search_data, r))}}
+            pp(klass.first(3).map { |r| {index: searchkick_index.record_data(r).merge(data: searchkick_index.send(:search_data, r))}})
           rescue => e
             puts "#{e.class.name}: #{e.message}"
           end
@@ -179,14 +179,14 @@ module Searchkick
         e.message.include?("No query registered for [function_score]")
       )
 
-        raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 2 or greater"
+        raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
       elsif status_code == 400
         if (
           e.message.include?("bool query does not support [filter]") ||
           e.message.include?("[bool] filter does not support [filter]")
         )
 
-          raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 2 or greater"
+          raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
         elsif e.message.include?("[multi_match] analyzer [searchkick_search] not found")
           raise InvalidQueryError, "Bad mapping - run #{reindex_command}"
         else
@@ -212,15 +212,13 @@ module Searchkick
 
       # pagination
       page = [options[:page].to_i, 1].max
-      per_page = (options[:limit] || options[:per_page] || 1_000).to_i
+      per_page = (options[:limit] || options[:per_page] || 10_000).to_i
       padding = [options[:padding].to_i, 0].max
       offset = options[:offset] || (page - 1) * per_page + padding
 
       # model and eager loading
       load = options[:load].nil? ? true : options[:load]
 
-      conversions_fields = Array(options[:conversions] || searchkick_options[:conversions]).map(&:to_s)
-
       all = term == "*"
 
       @json = options[:body]
@@ -228,12 +226,15 @@ module Searchkick
         ignored_options = options.keys & [:aggs, :boost,
           :boost_by, :boost_by_distance, :boost_where, :conversions, :conversions_term, :exclude, :explain,
           :fields, :highlight, :indices_boost, :limit, :match, :misspellings, :offset, :operator, :order,
-          :padding, :page, :per_page, :select, :smart_aggs, :suggest, :where]
-        warn "The body option replaces the entire body, so the following options are ignored: #{ignored_options.join(", ")}" if ignored_options.any?
+          :padding, :page, :per_page, :profile, :select, :smart_aggs, :suggest, :where]
+        raise ArgumentError, "Options incompatible with body option: #{ignored_options.join(", ")}" if ignored_options.any?
         payload = @json
       else
+        must_not = []
+        should = []
+
         if options[:similar]
-          payload = {
+          query = {
             more_like_this: {
               like: term,
               min_doc_freq: 1,
@@ -241,11 +242,14 @@ module Searchkick
               analyzer: "searchkick_search2"
             }
           }
+          if fields.all? { |f| f.start_with?("*.") }
+            raise ArgumentError, "Must specify fields to search"
+          end
           if fields != ["_all"]
-            payload[:more_like_this][:fields] = fields
+            query[:more_like_this][:fields] = fields
           end
         elsif all
-          payload = {
+          query = {
             match_all: {}
           }
         else
@@ -327,10 +331,19 @@ module Searchkick
             end
 
             if misspellings != false && match_type == :match
-              qs.concat qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) }
+              qs.concat(qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) })
             end
 
-            q2 = qs.map { |q| {match_type => {field => q}} }
+            if field.start_with?("*.")
+              q2 = qs.map { |q| {multi_match: q.merge(fields: [field], type: match_type == :match_phrase ? "phrase" : "best_fields")} }
+              if below61?
+                q2.each do |q|
+                  q[:multi_match].delete(:fuzzy_transpositions)
+                end
+              end
+            else
+              q2 = qs.map { |q| {match_type => {field => q}} }
+            end
 
             # boost exact matches more
             if field =~ /\.word_(start|middle|end)\z/ && searchkick_options[:word] != false
@@ -348,28 +361,11 @@ module Searchkick
               queries_to_add.concat(q2)
             end
 
-            if options[:exclude]
-              must_not =
-                Array(options[:exclude]).map do |phrase|
-                  {
-                    match_phrase: {
-                      exclude_field => {
-                        query: phrase,
-                        analyzer: exclude_analyzer
-                      }
-                    }
-                  }
-                end
+            queries.concat(queries_to_add)
 
-              queries_to_add = [{
-                bool: {
-                  should: queries_to_add,
-                  must_not: must_not
-                }
-              }]
+            if options[:exclude]
+              must_not.concat(set_exclude(exclude_field, exclude_analyzer))
             end
-
-            queries.concat(queries_to_add)
           end
 
           payload = {
@@ -378,78 +374,15 @@ module Searchkick
             }
           }
 
-          if conversions_fields.present? && options[:conversions] != false
-            shoulds = []
-            conversions_fields.each do |conversions_field|
-              # wrap payload in a bool query
-              script_score = {field_value_factor: {field: "#{conversions_field}.count"}}
-
-              shoulds << {
-                nested: {
-                  path: conversions_field,
-                  score_mode: "sum",
-                  query: {
-                    function_score: {
-                      boost_mode: "replace",
-                      query: {
-                        match: {
-                          "#{conversions_field}.query" => options[:conversions_term] || term
-                        }
-                      }
-                    }.merge(script_score)
-                  }
-                }
-              }
-            end
-            payload = {
-              bool: {
-                must: payload,
-                should: shoulds
-              }
-            }
-          end
-        end
-
-        custom_filters = []
-        multiply_filters = []
-
-        set_boost_by(multiply_filters, custom_filters)
-        set_boost_where(custom_filters)
-        set_boost_by_distance(custom_filters) if options[:boost_by_distance]
+          should.concat(set_conversions)
 
-        if custom_filters.any?
-          payload = {
-            function_score: {
-              functions: custom_filters,
-              query: payload,
-              score_mode: "sum"
-            }
-          }
-        end
-
-        if multiply_filters.any?
-          payload = {
-            function_score: {
-              functions: multiply_filters,
-              query: payload,
-              score_mode: "multiply"
-            }
-          }
+          query = payload
         end
 
         payload = {
-          query: payload,
           size: per_page,
           from: offset
         }
-        payload[:explain] = options[:explain] if options[:explain]
-        payload[:profile] = options[:profile] if options[:profile]
-
-        # order
-        set_order(payload) if options[:order]
-
-        # indices_boost
-        set_boost_by_indices(payload)
 
         # type when inheritance
         where = (options[:where] || {}).dup
@@ -465,8 +398,26 @@ module Searchkick
         # aggregations
         set_aggregations(payload, filters, post_filters) if options[:aggs]
 
-        # filters
-        set_filters(payload, filters, post_filters)
+        # post filters
+        set_post_filters(payload, post_filters) if post_filters.any?
+
+        custom_filters = []
+        multiply_filters = []
+
+        set_boost_by(multiply_filters, custom_filters)
+        set_boost_where(custom_filters)
+        set_boost_by_distance(custom_filters) if options[:boost_by_distance]
+
+        payload[:query] = build_query(query, filters, should, must_not, custom_filters, multiply_filters)
+
+        payload[:explain] = options[:explain] if options[:explain]
+        payload[:profile] = options[:profile] if options[:profile]
+
+        # order
+        set_order(payload) if options[:order]
+
+        # indices_boost
+        set_boost_by_indices(payload)
 
         # suggestions
         set_suggestions(payload, options[:suggest]) if options[:suggest]
@@ -512,7 +463,7 @@ module Searchkick
     def set_fields
       boost_fields = {}
       fields = options[:fields] || searchkick_options[:default_fields] || searchkick_options[:searchable]
-      all = searchkick_options.key?(:_all) ? searchkick_options[:_all] : below60?
+      all = searchkick_options.key?(:_all) ? searchkick_options[:_all] : false
       default_match = options[:match] || searchkick_options[:match] || :word
       fields =
         if fields
@@ -529,12 +480,88 @@ module Searchkick
           ["_all.phrase"]
         elsif term == "*"
           []
-        else
+        elsif default_match == :exact
           raise ArgumentError, "Must specify fields to search"
+        else
+          [default_match == :word ? "*.analyzed" : "*.#{default_match}"]
         end
       [boost_fields, fields]
     end
 
+    def build_query(query, filters, should, must_not, custom_filters, multiply_filters)
+      if filters.any? || must_not.any? || should.any?
+        bool = {must: query}
+        bool[:filter] = filters if filters.any?      # where
+        bool[:must_not] = must_not if must_not.any?  # exclude
+        bool[:should] = should if should.any?        # conversions
+        query = {bool: bool}
+      end
+
+      if custom_filters.any?
+        query = {
+          function_score: {
+            functions: custom_filters,
+            query: query,
+            score_mode: "sum"
+          }
+        }
+      end
+
+      if multiply_filters.any?
+        query = {
+          function_score: {
+            functions: multiply_filters,
+            query: query,
+            score_mode: "multiply"
+          }
+        }
+      end
+
+      query
+    end
+
+    def set_conversions
+      conversions_fields = Array(options[:conversions] || searchkick_options[:conversions]).map(&:to_s)
+      if conversions_fields.present? && options[:conversions] != false
+        conversions_fields.map do |conversions_field|
+          {
+            nested: {
+              path: conversions_field,
+              score_mode: "sum",
+              query: {
+                function_score: {
+                  boost_mode: "replace",
+                  query: {
+                    match: {
+                      "#{conversions_field}.query" => options[:conversions_term] || term
+                    }
+                  },
+                  field_value_factor: {
+                    field: "#{conversions_field}.count"
+                  }
+                }
+              }
+            }
+          }
+        end
+      else
+        []
+      end
+    end
+
+    def set_exclude(field, analyzer)
+      Array(options[:exclude]).map do |phrase|
+        {
+          multi_match: {
+            fields: [field],
+            query: phrase,
+            analyzer: analyzer,
+            type: "phrase"
+          }
+        }
+      end
+    end
+
     def set_boost_by_distance(custom_filters)
       boost_by_distance = options[:boost_by_distance] || {}
 
@@ -564,11 +591,11 @@ module Searchkick
       if boost_by.is_a?(Array)
         boost_by = Hash[boost_by.map { |f| [f, {factor: 1}] }]
       elsif boost_by.is_a?(Hash)
-        multiply_by, boost_by = boost_by.partition { |_, v| v[:boost_mode] == "multiply" }.map { |i| Hash[i] }
+        multiply_by, boost_by = boost_by.partition { |_, v| v.delete(:boost_mode) == "multiply" }.map { |i| Hash[i] }
       end
       boost_by[options[:boost]] = {factor: 1} if options[:boost]
 
-      custom_filters.concat boost_filters(boost_by, log: true)
+      custom_filters.concat boost_filters(boost_by, modifier: "ln2p")
       multiply_filters.concat boost_filters(multiply_by || {})
     end
 
@@ -631,7 +658,8 @@ module Searchkick
 
     def set_highlights(payload, fields)
       payload[:highlight] = {
-        fields: Hash[fields.map { |f| [f, {}] }]
+        fields: Hash[fields.map { |f| [f, {}] }],
+        fragment_size: below60? ? 30000 : 0
       }
 
       if options[:highlight].is_a?(Hash)
@@ -683,7 +711,7 @@ module Searchkick
               ranges: agg_options[:date_ranges]
             }.merge(shared_agg_options)
           }
-        elsif histogram = agg_options[:date_histogram]
+        elsif (histogram = agg_options[:date_histogram])
           interval = histogram[:interval]
           payload[:aggs][field] = {
             date_histogram: {
@@ -691,7 +719,7 @@ module Searchkick
               interval: interval
             }
           }
-        elsif metric = @@metric_aggs.find { |k| agg_options.has_key?(k) }
+        elsif (metric = @@metric_aggs.find { |k| agg_options.has_key?(k) })
           payload[:aggs][field] = {
             metric => {
               field: agg_options[metric][:field] || field
@@ -735,30 +763,18 @@ module Searchkick
       end
     end
 
-    def set_filters(payload, filters, post_filters)
-      if post_filters.any?
-        payload[:post_filter] = {
-          bool: {
-            filter: post_filters
-          }
+    def set_post_filters(payload, post_filters)
+      payload[:post_filter] = {
+        bool: {
+          filter: post_filters
         }
-      end
-
-      if filters.any?
-        # more efficient query if no aggs
-        payload[:query] = {
-          bool: {
-            must: payload[:query],
-            filter: filters
-          }
-        }
-      end
+      }
     end
 
     # TODO id transformation for arrays
     def set_order(payload)
       order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
-      id_field = below50? ? :_id : :_uid
+      id_field = :_uid
       payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? id_field : k, v] }]
     end
 
@@ -889,50 +905,37 @@ module Searchkick
     end
 
     def custom_filter(field, value, factor)
-      if below50?
-        {
-          filter: {
-            bool: {
-              must: where_filters(field => value)
-            }
-          },
-          boost_factor: factor
+      {
+        filter: where_filters(field => value),
+        weight: factor
+      }
+    end
+
+    def boost_filter(field, factor: 1, modifier: nil, missing: nil)
+      script_score = {
+        field_value_factor: {
+          field: field,
+          factor: factor.to_f,
+          modifier: modifier
         }
+      }
+
+      if missing
+        script_score[:field_value_factor][:missing] = missing.to_f
       else
-        {
-          filter: where_filters(field => value),
-          weight: factor
+        script_score[:filter] = {
+          exists: {
+            field: field
+          }
         }
       end
+
+      script_score
     end
 
-    def boost_filters(boost_by, options = {})
+    def boost_filters(boost_by, modifier: nil)
       boost_by.map do |field, value|
-        log = value.key?(:log) ? value[:log] : options[:log]
-        value[:factor] ||= 1
-        script_score = {
-          field_value_factor: {
-            field: field,
-            factor: value[:factor].to_f,
-            modifier: value[:modifier] || (log ? "ln2p" : nil)
-          }
-        }
-
-        if value[:missing]
-          if below50?
-            raise ArgumentError, "The missing option for boost_by is not supported in Elasticsearch < 5"
-          else
-            script_score[:field_value_factor][:missing] = value[:missing].to_f
-          end
-        else
-          script_score[:filter] = {
-            exists: {
-              field: field
-            }
-          }
-        end
-
-        script_score
+        boost_filter(field, modifier: modifier, **value)
       end
     end
 
@@ -957,12 +960,12 @@ module Searchkick
       end
     end
 
-    def below50?
-      Searchkick.server_below?("5.0.0-alpha1")
-    end
-
     def below60?
       Searchkick.server_below?("6.0.0-alpha1")
     end
+
+    def below61?
+      Searchkick.server_below?("6.1.0-alpha1")
+    end
   end
 end