searchkick 2.3.2 → 4.4.1

data/lib/searchkick/query.rb

@@ -12,14 +12,14 @@ module Searchkick
       :took, :error, :model_name, :entry_name, :total_count, :total_entries,
       :current_page, :per_page, :limit_value, :padding, :total_pages, :num_pages,
       :offset_value, :offset, :previous_page, :prev_page, :next_page, :first_page?, :last_page?,
-      :out_of_range?, :hits, :response, :to_a, :first
+      :out_of_range?, :hits, :response, :to_a, :first, :scroll
 
     def initialize(klass, term = "*", **options)
-      unknown_keywords = options.keys - [:aggs, :body, :body_options, :boost,
-        :boost_by, :boost_by_distance, :boost_where, :conversions, :conversions_term, :debug, :emoji, :exclude, :execute, :explain,
+      unknown_keywords = options.keys - [:aggs, :block, :body, :body_options, :boost,
+        :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_term, :debug, :emoji, :exclude, :execute, :explain,
         :fields, :highlight, :includes, :index_name, :indices_boost, :limit, :load,
-        :match, :misspellings, :model_includes, :offset, :operator, :order, :padding, :page, :per_page, :profile,
-        :request_params, :routing, :select, :similar, :smart_aggs, :suggest, :track, :type, :where]
+        :match, :misspellings, :models, :model_includes, :offset, :operator, :order, :padding, :page, :per_page, :profile,
+        :request_params, :routing, :scope_results, :scroll, :select, :similar, :smart_aggs, :suggest, :total_entries, :track, :type, :where]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
 
       term = term.to_s
@@ -39,6 +39,7 @@ module Searchkick
       @misspellings = false
       @misspellings_below = nil
       @highlighted_fields = nil
+      @index_mapping = nil
 
       prepare
     end
@@ -56,9 +57,19 @@ module Searchkick
     end
 
     def params
+      if options[:models]
+        @index_mapping = {}
+        Array(options[:models]).each do |model|
+          # there can be multiple models per index name due to inheritance - see #1259
+          (@index_mapping[model.searchkick_index.name] ||= []) << model
+        end
+      end
+
       index =
         if options[:index_name]
           Array(options[:index_name]).map { |v| v.respond_to?(:searchkick_index) ? v.searchkick_index.name : v }.join(",")
+        elsif options[:models]
+          @index_mapping.keys.join(",")
         elsif searchkick_index
           searchkick_index.name
         else
@@ -71,6 +82,7 @@ module Searchkick
       }
       params[:type] = @type if @type
       params[:routing] = @routing if @routing
+      params[:scroll] = @scroll if @scroll
       params.merge!(options[:request_params]) if options[:request_params]
       params
     end
@@ -94,11 +106,16 @@ module Searchkick
       query = params
       type = query[:type]
       index = query[:index].is_a?(Array) ? query[:index].join(",") : query[:index]
+      request_params = query.except(:index, :type, :body)
 
       # no easy way to tell which host the client will use
       host = Searchkick.client.transport.hosts.first
       credentials = host[:user] || host[:password] ? "#{host[:user]}:#{host[:password]}@" : nil
-      "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -d '#{query[:body].to_json}'"
+      params = ["pretty"]
+      request_params.each do |k, v|
+        params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
+      end
+      "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?#{params.join('&')} -H 'Content-Type: application/json' -d '#{query[:body].to_json}'"
     end
 
     def handle_response(response)
@@ -111,11 +128,19 @@ module Searchkick
         model_includes: options[:model_includes],
         json: !@json.nil?,
         match_suffix: @match_suffix,
+        highlight: options[:highlight],
         highlighted_fields: @highlighted_fields || [],
-        misspellings: @misspellings
+        misspellings: @misspellings,
+        term: term,
+        scope_results: options[:scope_results],
+        total_entries: options[:total_entries],
+        index_mapping: @index_mapping,
+        suggest: options[:suggest],
+        scroll: options[:scroll]
       }
 
       if options[:debug]
+        # can remove when minimum Ruby version is 2.5
         require "pp"
 
         puts "Searchkick Version: #{Searchkick::VERSION}"
@@ -133,7 +158,7 @@ module Searchkick
         if searchkick_index
           puts "Model Search Data"
           begin
-            pp klass.first(3).map { |r| {index: searchkick_index.record_data(r).merge(data: searchkick_index.send(:search_data, r))}}
+            pp klass.limit(3).map { |r| RecordData.new(searchkick_index, r).index_data }
           rescue => e
             puts "#{e.class.name}: #{e.message}"
           end
@@ -161,7 +186,7 @@ module Searchkick
     end
 
     def retry_misspellings?(response)
-      @misspellings_below && response["hits"]["total"] < @misspellings_below
+      @misspellings_below && Searchkick::Results.new(searchkick_klass, response).total_count < @misspellings_below
     end
 
     private
@@ -177,15 +202,15 @@ module Searchkick
         e.message.include?("No query registered for [function_score]")
       )
 
-        raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 2 or greater"
+        raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
       elsif status_code == 400
         if (
           e.message.include?("bool query does not support [filter]") ||
           e.message.include?("[bool] filter does not support [filter]")
         )
 
-          raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 2 or greater"
-        elsif e.message.include?("[multi_match] analyzer [searchkick_search] not found")
+          raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
+        elsif e.message =~ /analyzer \[searchkick_.+\] not found/
           raise InvalidQueryError, "Bad mapping - run #{reindex_command}"
         else
           raise InvalidQueryError, e.message
@@ -210,40 +235,47 @@ module Searchkick
 
       # pagination
       page = [options[:page].to_i, 1].max
-      per_page = (options[:limit] || options[:per_page] || 1_000).to_i
+      # maybe use index.max_result_window in the future
+      default_limit = searchkick_options[:deep_paging] ? 1_000_000_000 : 10_000
+      per_page = (options[:limit] || options[:per_page] || default_limit).to_i
       padding = [options[:padding].to_i, 0].max
       offset = options[:offset] || (page - 1) * per_page + padding
+      scroll = options[:scroll]
 
       # model and eager loading
       load = options[:load].nil? ? true : options[:load]
 
-      conversions_fields = Array(options[:conversions] || searchkick_options[:conversions]).map(&:to_s)
-
       all = term == "*"
 
       @json = options[:body]
       if @json
         ignored_options = options.keys & [:aggs, :boost,
-          :boost_by, :boost_by_distance, :boost_where, :conversions, :conversions_term, :exclude, :explain,
-          :fields, :highlight, :indices_boost, :limit, :match, :misspellings, :offset, :operator, :order,
-          :padding, :page, :per_page, :select, :smart_aggs, :suggest, :where]
-        warn "The body option replaces the entire body, so the following options are ignored: #{ignored_options.join(", ")}" if ignored_options.any?
+          :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_term, :exclude, :explain,
+          :fields, :highlight, :indices_boost, :match, :misspellings, :operator, :order,
+          :profile, :select, :smart_aggs, :suggest, :where]
+        raise ArgumentError, "Options incompatible with body option: #{ignored_options.join(", ")}" if ignored_options.any?
         payload = @json
       else
+        must_not = []
+        should = []
+
         if options[:similar]
-          payload = {
+          query = {
             more_like_this: {
-              like_text: term,
+              like: term,
               min_doc_freq: 1,
               min_term_freq: 1,
               analyzer: "searchkick_search2"
             }
           }
+          if fields.all? { |f| f.start_with?("*.") }
+            raise ArgumentError, "Must specify fields to search"
+          end
           if fields != ["_all"]
-            payload[:more_like_this][:fields] = fields
+            query[:more_like_this][:fields] = fields
           end
-        elsif all
-          payload = {
+        elsif all && !options[:exclude]
+          query = {
             match_all: {}
           }
         else
@@ -272,6 +304,15 @@ module Searchkick
             prefix_length = (misspellings.is_a?(Hash) && misspellings[:prefix_length]) || 0
             default_max_expansions = @misspellings_below ? 20 : 3
             max_expansions = (misspellings.is_a?(Hash) && misspellings[:max_expansions]) || default_max_expansions
+            misspellings_fields = misspellings.is_a?(Hash) && misspellings.key?(:fields) && misspellings[:fields].map(&:to_s)
+
+            if misspellings_fields
+              missing_fields = misspellings_fields - fields.map { |f| base_field(f) }
+              if missing_fields.any?
+                raise ArgumentError, "All fields in per-field misspellings must also be specified in fields option"
+              end
+            end
+
             @misspellings = true
           else
             @misspellings = false
@@ -306,12 +347,16 @@ module Searchkick
             exclude_analyzer = nil
             exclude_field = field
 
+            field_misspellings = misspellings && (!misspellings_fields || misspellings_fields.include?(base_field(field)))
+
             if field == "_all" || field.end_with?(".analyzed")
-              shared_options[:cutoff_frequency] = 0.001 unless operator == "and" || misspellings == false
-              qs.concat [
-                shared_options.merge(analyzer: "searchkick_search"),
-                shared_options.merge(analyzer: "searchkick_search2")
-              ]
+              shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false || (!below73? && !track_total_hits?)
+              qs << shared_options.merge(analyzer: "searchkick_search")
+
+              # searchkick_search and searchkick_search2 are the same for ukrainian
+              unless %w(japanese korean polish ukrainian vietnamese).include?(searchkick_options[:language])
+                qs << shared_options.merge(analyzer: "searchkick_search2")
+              end
               exclude_analyzer = "searchkick_search2"
             elsif field.end_with?(".exact")
               f = field.split(".")[0..-2].join(".")
@@ -324,11 +369,20 @@ module Searchkick
               exclude_analyzer = analyzer
             end
 
-            if misspellings != false && match_type == :match
-              qs.concat qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) }
+            if field_misspellings != false && match_type == :match
+              qs.concat(qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) })
             end
 
-            q2 = qs.map { |q| {match_type => {field => q}} }
+            if field.start_with?("*.")
+              q2 = qs.map { |q| {multi_match: q.merge(fields: [field], type: match_type == :match_phrase ? "phrase" : "best_fields")} }
+              if below61?
+                q2.each do |q|
+                  q[:multi_match].delete(:fuzzy_transpositions)
+                end
+              end
+            else
+              q2 = qs.map { |q| {match_type => {field => q}} }
+            end
 
             # boost exact matches more
             if field =~ /\.word_(start|middle|end)\z/ && searchkick_options[:word] != false
@@ -346,100 +400,81 @@ module Searchkick
               queries_to_add.concat(q2)
             end
 
-            if options[:exclude]
-              must_not =
-                Array(options[:exclude]).map do |phrase|
-                  {
-                    match_phrase: {
-                      exclude_field => {
-                        query: phrase,
-                        analyzer: exclude_analyzer
-                      }
-                    }
-                  }
-                end
+            queries << queries_to_add
 
-              queries_to_add = [{
-                bool: {
-                  should: queries_to_add,
-                  must_not: must_not
-                }
-              }]
+            if options[:exclude]
+              must_not.concat(set_exclude(exclude_field, exclude_analyzer))
             end
-
-            queries.concat(queries_to_add)
           end
 
-          payload = {
-            dis_max: {
-              queries: queries
+          # all + exclude option
+          if all
+            query = {
+              match_all: {}
             }
-          }
-
-          if conversions_fields.present? && options[:conversions] != false
-            shoulds = []
-            conversions_fields.each do |conversions_field|
-              # wrap payload in a bool query
-              script_score = {field_value_factor: {field: "#{conversions_field}.count"}}
 
-              shoulds << {
-                nested: {
-                  path: conversions_field,
-                  score_mode: "sum",
-                  query: {
-                    function_score: {
-                      boost_mode: "replace",
-                      query: {
-                        match: {
-                          "#{conversions_field}.query" => options[:conversions_term] || term
-                        }
-                      }
-                    }.merge(script_score)
-                  }
-                }
-              }
-            end
+            should = []
+          else
+            # higher score for matching more fields
             payload = {
               bool: {
-                must: payload,
-                should: shoulds
+                should: queries.map { |qs| {dis_max: {queries: qs}} }
               }
             }
+
+            should.concat(set_conversions)
+          end
+
+          query = payload
+        end
+
+        payload = {}
+
+        # type when inheritance
+        where = (options[:where] || {}).dup
+        if searchkick_options[:inheritance] && (options[:type] || (klass != searchkick_klass && searchkick_index))
+          where[:type] = [options[:type] || klass].flatten.map { |v| searchkick_index.klass_document_type(v, true) }
+        end
+
+        models = Array(options[:models])
+        if models.any? { |m| m != m.searchkick_klass }
+          # aliases are not supported with _index in ES below 7.5
+          # see https://github.com/elastic/elasticsearch/pull/46640
+          if below75?
+            Searchkick.warn("Passing child models to models option throws off hits and pagination - use type option instead")
+          else
+            index_type_or =
+              models.map do |m|
+                v = {_index: m.searchkick_index.name}
+                v[:type] = m.searchkick_index.klass_document_type(m, true) if m != m.searchkick_klass
+                v
+              end
+
+            where[:or] = Array(where[:or]) + [index_type_or]
           end
         end
 
+        # start everything as efficient filters
+        # move to post_filters as aggs demand
+        filters = where_filters(where)
+        post_filters = []
+
+        # aggregations
+        set_aggregations(payload, filters, post_filters) if options[:aggs]
+
+        # post filters
+        set_post_filters(payload, post_filters) if post_filters.any?
+
         custom_filters = []
         multiply_filters = []
 
         set_boost_by(multiply_filters, custom_filters)
         set_boost_where(custom_filters)
         set_boost_by_distance(custom_filters) if options[:boost_by_distance]
+        set_boost_by_recency(custom_filters) if options[:boost_by_recency]
 
-        if custom_filters.any?
-          payload = {
-            function_score: {
-              functions: custom_filters,
-              query: payload,
-              score_mode: "sum"
-            }
-          }
-        end
-
-        if multiply_filters.any?
-          payload = {
-            function_score: {
-              functions: multiply_filters,
-              query: payload,
-              score_mode: "multiply"
-            }
-          }
-        end
+        payload[:query] = build_query(query, filters, should, must_not, custom_filters, multiply_filters)
 
-        payload = {
-          query: payload,
-          size: per_page,
-          from: offset
-        }
         payload[:explain] = options[:explain] if options[:explain]
         payload[:profile] = options[:profile] if options[:profile]
 
@@ -449,13 +484,6 @@ module Searchkick
         # indices_boost
         set_boost_by_indices(payload)
 
-        # filters
-        filters = where_filters(options[:where])
-        set_filters(payload, filters) if filters.any?
-
-        # aggregations
-        set_aggregations(payload) if options[:aggs]
-
         # suggestions
         set_suggestions(payload, options[:suggest]) if options[:suggest]
 
@@ -466,8 +494,7 @@ module Searchkick
         payload[:timeout] ||= "#{Searchkick.search_timeout + 1}s"
 
         # An empty array will cause only the _id and _type for each hit to be returned
-        # doc for :select - http://www.elasticsearch.org/guide/reference/api/search/fields/
-        # doc for :select_v2 - https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-source-filtering.html
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-source-filtering.html
         if options[:select]
           if options[:select] == []
             # intuitively [] makes sense to return no fields, but ES by default returns all fields
@@ -480,28 +507,49 @@ module Searchkick
         end
       end
 
+      # pagination
+      pagination_options = options[:page] || options[:limit] || options[:per_page] || options[:offset] || options[:padding]
+      if !options[:body] || pagination_options
+        payload[:size] = per_page
+        payload[:from] = offset if offset > 0
+      end
+
       # type
-      if options[:type] || (klass != searchkick_klass && searchkick_index)
+      if !searchkick_options[:inheritance] && (options[:type] || (klass != searchkick_klass && searchkick_index))
         @type = [options[:type] || klass].flatten.map { |v| searchkick_index.klass_document_type(v) }
       end
 
       # routing
       @routing = options[:routing] if options[:routing]
 
+      if track_total_hits?
+        payload[:track_total_hits] = true
+      end
+
       # merge more body options
       payload = payload.deep_merge(options[:body_options]) if options[:body_options]
 
+      # run block
+      options[:block].call(payload) if options[:block]
+
+      # scroll optimization when interating over all docs
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html
+      if options[:scroll] && payload[:query] == {match_all: {}}
+        payload[:sort] ||= ["_doc"]
+      end
+
       @body = payload
       @page = page
       @per_page = per_page
       @padding = padding
       @load = load
+      @scroll = scroll
     end
 
     def set_fields
       boost_fields = {}
       fields = options[:fields] || searchkick_options[:default_fields] || searchkick_options[:searchable]
-      all = searchkick_options.key?(:_all) ? searchkick_options[:_all] : below60?
+      all = searchkick_options.key?(:_all) ? searchkick_options[:_all] : false
       default_match = options[:match] || searchkick_options[:match] || :word
       fields =
         if fields
@@ -516,12 +564,89 @@ module Searchkick
           ["_all"]
         elsif all && default_match == :phrase
           ["_all.phrase"]
-        else
+        elsif term != "*" && default_match == :exact
           raise ArgumentError, "Must specify fields to search"
+        else
+          [default_match == :word ? "*.analyzed" : "*.#{default_match}"]
         end
       [boost_fields, fields]
     end
 
+    def build_query(query, filters, should, must_not, custom_filters, multiply_filters)
+      if filters.any? || must_not.any? || should.any?
+        bool = {}
+        bool[:must] = query if query
+        bool[:filter] = filters if filters.any?      # where
+        bool[:must_not] = must_not if must_not.any?  # exclude
+        bool[:should] = should if should.any?        # conversions
+        query = {bool: bool}
+      end
+
+      if custom_filters.any?
+        query = {
+          function_score: {
+            functions: custom_filters,
+            query: query,
+            score_mode: "sum"
+          }
+        }
+      end
+
+      if multiply_filters.any?
+        query = {
+          function_score: {
+            functions: multiply_filters,
+            query: query,
+            score_mode: "multiply"
+          }
+        }
+      end
+
+      query
+    end
+
+    def set_conversions
+      conversions_fields = Array(options[:conversions] || searchkick_options[:conversions]).map(&:to_s)
+      if conversions_fields.present? && options[:conversions] != false
+        conversions_fields.map do |conversions_field|
+          {
+            nested: {
+              path: conversions_field,
+              score_mode: "sum",
+              query: {
+                function_score: {
+                  boost_mode: "replace",
+                  query: {
+                    match: {
+                      "#{conversions_field}.query" => options[:conversions_term] || term
+                    }
+                  },
+                  field_value_factor: {
+                    field: "#{conversions_field}.count"
+                  }
+                }
+              }
+            }
+          }
+        end
+      else
+        []
+      end
+    end
+
+    def set_exclude(field, analyzer)
+      Array(options[:exclude]).map do |phrase|
+        {
+          multi_match: {
+            fields: [field],
+            query: phrase,
+            analyzer: analyzer,
+            type: "phrase"
+          }
+        }
+      end
+    end
+
     def set_boost_by_distance(custom_filters)
       boost_by_distance = options[:boost_by_distance] || {}
 
@@ -535,9 +660,11 @@ module Searchkick
         unless attributes[:origin]
           raise ArgumentError, "boost_by_distance requires :origin"
         end
-        function_params = attributes.select { |k, _| [:origin, :scale, :offset, :decay].include?(k) }
+
+        function_params = attributes.except(:factor, :function)
         function_params[:origin] = location_value(function_params[:origin])
         custom_filters << {
+          weight: attributes[:factor] || 1,
           attributes[:function] => {
             field => function_params
           }
@@ -545,16 +672,29 @@ module Searchkick
       end
     end
 
+    def set_boost_by_recency(custom_filters)
+      options[:boost_by_recency].each do |field, attributes|
+        attributes = {function: :gauss, origin: Time.now}.merge(attributes)
+
+        custom_filters << {
+          weight: attributes[:factor] || 1,
+          attributes[:function] => {
+            field => attributes.except(:factor, :function)
+          }
+        }
+      end
+    end
+
     def set_boost_by(multiply_filters, custom_filters)
       boost_by = options[:boost_by] || {}
       if boost_by.is_a?(Array)
         boost_by = Hash[boost_by.map { |f| [f, {factor: 1}] }]
       elsif boost_by.is_a?(Hash)
-        multiply_by, boost_by = boost_by.partition { |_, v| v[:boost_mode] == "multiply" }.map { |i| Hash[i] }
+        multiply_by, boost_by = boost_by.partition { |_, v| v.delete(:boost_mode) == "multiply" }.map { |i| Hash[i] }
       end
       boost_by[options[:boost]] = {factor: 1} if options[:boost]
 
-      custom_filters.concat boost_filters(boost_by, log: true)
+      custom_filters.concat boost_filters(boost_by, modifier: "ln2p")
       multiply_filters.concat boost_filters(multiply_by || {})
     end
 
@@ -577,11 +717,9 @@ module Searchkick
     def set_boost_by_indices(payload)
       return unless options[:indices_boost]
 
-      indices_boost = options[:indices_boost].each_with_object({}) do |(key, boost), memo|
+      indices_boost = options[:indices_boost].map do |key, boost|
         index = key.respond_to?(:searchkick_index) ? key.searchkick_index.name : key
-        # try to use index explicitly instead of alias: https://github.com/elasticsearch/elasticsearch/issues/4756
-        index_by_alias = Searchkick.client.indices.get_alias(index: index).keys.first
-        memo[index_by_alias || index] = boost
+        {index => boost}
       end
 
       payload[:indices_boost] = indices_boost
@@ -617,7 +755,8 @@ module Searchkick
 
     def set_highlights(payload, fields)
       payload[:highlight] = {
-        fields: Hash[fields.map { |f| [f, {}] }]
+        fields: Hash[fields.map { |f| [f, {}] }],
+        fragment_size: 0
       }
 
       if options[:highlight].is_a?(Hash)
@@ -646,15 +785,14 @@ module Searchkick
       @highlighted_fields = payload[:highlight][:fields].keys
     end
 
-    def set_aggregations(payload)
+    def set_aggregations(payload, filters, post_filters)
       aggs = options[:aggs]
       payload[:aggs] = {}
 
       aggs = Hash[aggs.map { |f| [f, {}] }] if aggs.is_a?(Array) # convert to more advanced syntax
-
       aggs.each do |field, agg_options|
         size = agg_options[:limit] ? agg_options[:limit] : 1_000
-        shared_agg_options = agg_options.slice(:order, :min_doc_count)
+        shared_agg_options = agg_options.except(:limit, :field, :ranges, :date_ranges, :where)
 
         if agg_options[:ranges]
           payload[:aggs][field] = {
@@ -670,20 +808,16 @@ module Searchkick
               ranges: agg_options[:date_ranges]
             }.merge(shared_agg_options)
           }
-        elsif histogram = agg_options[:date_histogram]
-          interval = histogram[:interval]
+        elsif (histogram = agg_options[:date_histogram])
           payload[:aggs][field] = {
-            date_histogram: {
-              field: histogram[:field],
-              interval: interval
-            }
-          }
-        elsif metric = @@metric_aggs.find { |k| agg_options.has_key?(k) }
+            date_histogram: histogram
+          }.merge(shared_agg_options)
+        elsif (metric = @@metric_aggs.find { |k| agg_options.has_key?(k) })
           payload[:aggs][field] = {
             metric => {
               field: agg_options[metric][:field] || field
             }
-          }
+          }.merge(shared_agg_options)
         else
           payload[:aggs][field] = {
             terms: {
@@ -696,6 +830,17 @@ module Searchkick
         where = {}
         where = (options[:where] || {}).reject { |k| k == field } unless options[:smart_aggs] == false
         agg_filters = where_filters(where.merge(agg_options[:where] || {}))
+
+        # only do one level comparison for simplicity
+        filters.select! do |filter|
+          if agg_filters.include?(filter)
+            true
+          else
+            post_filters << filter
+            false
+          end
+        end
+
         if agg_filters.any?
           payload[:aggs][field] = {
             filter: {
@@ -711,32 +856,27 @@ module Searchkick
       end
     end
 
-    def set_filters(payload, filters)
-      if options[:aggs]
-        payload[:post_filter] = {
-          bool: {
-            filter: filters
-          }
-        }
-      else
-        # more efficient query if no aggs
-        payload[:query] = {
-          bool: {
-            must: payload[:query],
-            filter: filters
-          }
+    def set_post_filters(payload, post_filters)
+      payload[:post_filter] = {
+        bool: {
+          filter: post_filters
         }
-      end
+      }
     end
 
     # TODO id transformation for arrays
     def set_order(payload)
       order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
-      id_field = below50? ? :_id : :_uid
+      id_field = :_id
       payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? id_field : k, v] }]
     end
 
     def where_filters(where)
+      # if where.respond_to?(:permitted?) && !where.permitted?
+      #   # TODO check in more places
+      #   Searchkick.warn("Passing unpermitted parameters will raise an exception in Searchkick 5")
+      # end
+
       filters = []
       (where || {}).each do |field, value|
         field = :_id if field.to_s == "id"
@@ -751,10 +891,17 @@ module Searchkick
           filters << {bool: {must_not: where_filters(value)}}
         elsif field == :_and
           filters << {bool: {must: value.map { |or_statement| {bool: {filter: where_filters(or_statement)}} }}}
+        # elsif field == :_script
+        #   filters << {script: {script: {source: value, lang: "painless"}}}
         else
           # expand ranges
           if value.is_a?(Range)
-            value = {gte: value.first, (value.exclude_end? ? :lt : :lte) => value.last}
+            # infinite? added in Ruby 2.4
+            if value.end.nil? || (value.end.respond_to?(:infinite?) && value.end.infinite?)
+              value = {gte: value.first}
+            else
+              value = {gte: value.first, (value.exclude_end? ? :lt : :lte) => value.last}
+            end
           end
 
           value = {in: value} if value.is_a?(Array)
@@ -762,7 +909,7 @@ module Searchkick
           if value.is_a?(Hash)
             value.each do |op, op_value|
               case op
-              when :within, :bottom_right
+              when :within, :bottom_right, :bottom_left
                 # do nothing
               when :near
                 filters << {
@@ -797,9 +944,35 @@ module Searchkick
                     }
                   }
                 }
+              when :top_right
+                filters << {
+                  geo_bounding_box: {
+                    field => {
+                      top_right: location_value(op_value),
+                      bottom_left: location_value(value[:bottom_left])
+                    }
+                  }
+                }
+              when :like
+                # based on Postgres
+                # https://www.postgresql.org/docs/current/functions-matching.html
+                # % matches zero or more characters
+                # _ matches one character
+                # \ is escape character
+                # escape Lucene reserved characters
+                # https://www.elastic.co/guide/en/elasticsearch/reference/current/regexp-syntax.html#regexp-optional-operators
+                reserved = %w(. ? + * | { } [ ] ( ) " \\)
+                regex = op_value.dup
+                reserved.each do |v|
+                  regex.gsub!(v, "\\" + v)
+                end
+                regex = regex.gsub(/(?<!\\)%/, ".*").gsub(/(?<!\\)_/, ".").gsub("\\%", "%").gsub("\\_", "_")
+                filters << {regexp: {field => {value: regex}}}
+              when :prefix
+                filters << {prefix: {field => {value: op_value}}}
               when :regexp # support for regexp queries without using a regexp ruby object
                 filters << {regexp: {field => {value: op_value}}}
-              when :not # not equal
+              when :not, :_not # not equal
                 filters << {bool: {must_not: term_filters(field, op_value)}}
               when :all
                 op_value.each do |val|
@@ -807,6 +980,8 @@ module Searchkick
                 end
               when :in
                 filters << term_filters(field, op_value)
+              when :exists
+                filters << {exists: {field: field}}
               else
                 range_query =
                   case op
@@ -847,57 +1022,78 @@ module Searchkick
       elsif value.nil?
         {bool: {must_not: {exists: {field: field}}}}
       elsif value.is_a?(Regexp)
-        {regexp: {field => {value: value.source}}}
+        if value.casefold?
+          Searchkick.warn("Case-insensitive flag does not work with Elasticsearch")
+        end
+
+        source = value.source
+        unless source.start_with?("\\A") && source.end_with?("\\z")
+          # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-regexp-query.html
+          Searchkick.warn("Regular expressions are always anchored in Elasticsearch")
+        end
+
+        # TODO handle other anchor characters, like ^, $, \Z
+        if source.start_with?("\\A")
+          source = source[2..-1]
+        else
+          # TODO uncomment in Searchkick 5
+          # source = ".*#{source}"
+        end
+
+        if source.end_with?("\\z")
+          source = source[0..-3]
+        else
+          # TODO uncomment in Searchkick 5
+          # source = "#{source}.*"
+        end
+
+        {regexp: {field => {value: source, flags: "NONE"}}}
       else
-        {term: {field => value}}
+        # TODO add this for other values
+        if value.as_json.is_a?(Enumerable)
+          # query will fail, but this is better
+          # same message as Active Record
+          # TODO make TypeError
+          # raise InvalidQueryError for backward compatibility
+          raise Searchkick::InvalidQueryError, "can't cast #{value.class.name}"
+        end
+
+        {term: {field => {value: value}}}
       end
     end
 
     def custom_filter(field, value, factor)
-      if below50?
-        {
-          filter: {
-            bool: {
-              must: where_filters(field => value)
-            }
-          },
-          boost_factor: factor
+      {
+        filter: where_filters(field => value),
+        weight: factor
+      }
+    end
+
+    def boost_filter(field, factor: 1, modifier: nil, missing: nil)
+      script_score = {
+        field_value_factor: {
+          field: field,
+          factor: factor.to_f,
+          modifier: modifier
         }
+      }
+
+      if missing
+        script_score[:field_value_factor][:missing] = missing.to_f
       else
-        {
-          filter: where_filters(field => value),
-          weight: factor
+        script_score[:filter] = {
+          exists: {
+            field: field
+          }
         }
       end
+
+      script_score
     end
 
-    def boost_filters(boost_by, options = {})
+    def boost_filters(boost_by, modifier: nil)
       boost_by.map do |field, value|
-        log = value.key?(:log) ? value[:log] : options[:log]
-        value[:factor] ||= 1
-        script_score = {
-          field_value_factor: {
-            field: field,
-            factor: value[:factor].to_f,
-            modifier: log ? "ln2p" : nil
-          }
-        }
-
-        if value[:missing]
-          if below50?
-            raise ArgumentError, "The missing option for boost_by is not supported in Elasticsearch < 5"
-          else
-            script_score[:field_value_factor][:missing] = value[:missing].to_f
-          end
-        else
-          script_score[:filter] = {
-            exists: {
-              field: field
-            }
-          }
-        end
-
-        script_score
+        boost_filter(field, modifier: modifier, **value)
       end
     end
 
@@ -922,12 +1118,32 @@ module Searchkick
       end
     end
 
-    def below50?
-      Searchkick.server_below?("5.0.0-alpha1")
+    def base_field(k)
+      k.sub(/\.(analyzed|word_start|word_middle|word_end|text_start|text_middle|text_end|exact)\z/, "")
+    end
+
+    def track_total_hits?
+      (searchkick_options[:deep_paging] && !below70?) || body_options[:track_total_hits]
+    end
+
+    def body_options
+      options[:body_options] || {}
+    end
+
+    def below61?
+      Searchkick.server_below?("6.1.0")
+    end
+
+    def below70?
+      Searchkick.server_below?("7.0.0")
+    end
+
+    def below73?
+      Searchkick.server_below?("7.3.0")
     end
 
-    def below60?
-      Searchkick.server_below?("6.0.0-alpha1")
+    def below75?
+      Searchkick.server_below?("7.5.0")
     end
   end
 end