searchkick 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 750ced99cb1b0929710eba206bde31907f0d9544
-  data.tar.gz: 454d078f8900948def90b1a88b40bde8aec561b5
+  metadata.gz: 9328dde4ac72c1fb914fdcbffe6a544e6f64b335
+  data.tar.gz: e59db37c34cc78f48139a8f6d604d2c4eb108d06
 SHA512:
-  metadata.gz: 5139a9a1f326fa97f58f09fc52d5988ff14661e8e4ec9a3240c6e4c10160c577d5b6231caab980101951a37cbdbbf5631e4df70c5060f5a21bd5beb9ae58dfec
-  data.tar.gz: 841981f0feaafcbeb39295282b018d4672bc41c115f31ce3d1b63964345197942082aac4eeff5b43190669ef81ad5a0fd3b0ac8afb5f271154058f0045fb3760
+  metadata.gz: 181ac8dc2d783231b7696094583312ca73e4b1104f7a91af76cde94f30ffcd6c4d89bd88ace568fcf1825776ae348c8bb2c1e7b50061407251d6805fcf45281a
+  data.tar.gz: 786f655864dbdc1f9d978779000b6ae762be9ea476f832515cba1b46fa923c9dee8174bc92dff247580fdabc10a4d8aa4aaacb2690c0eea9d3fb3881e96e8d55

data/.gitignore

@@ -18,3 +18,5 @@ tmp
 *.log
 .DS_Store
 .ruby-*
+.idea/
+*.sqlite3

data/.travis.yml

@@ -5,7 +5,7 @@ services:
   - mongodb
 before_install:
   - ./test/ci/before_install.sh
-script: bundle exec rake test
+script: RUBYOPT=W0 bundle exec rake test
 before_script:
   - psql -c 'create database searchkick_test;' -U postgres
 notifications:
@@ -22,10 +22,22 @@ gemfile:
   - test/gemfiles/mongoid3.gemfile
   - test/gemfiles/mongoid4.gemfile
   - test/gemfiles/mongoid5.gemfile
+env:
+  - ELASTICSEARCH_VERSION=2.3.0
 matrix:
   include:
-    - gemfile: test/gemfiles/nobrainer.gemfile
-      env: NOBRAINER=true
+    - gemfile: Gemfile
+      env: ELASTICSEARCH_VERSION=1.0.0
+    - gemfile: Gemfile
+      env: ELASTICSEARCH_VERSION=1.7.0
+    - gemfile: Gemfile
+      env: ELASTICSEARCH_VERSION=2.0.0
+    - gemfile: Gemfile
+      env: ELASTICSEARCH_VERSION=5.0.0-alpha2
+    # - gemfile: test/gemfiles/nobrainer.gemfile
+    #   env: NOBRAINER=true
   allow_failures:
-    - gemfile: test/gemfiles/nobrainer.gemfile
-      env: NOBRAINER=true
+    - gemfile: Gemfile
+      env: ELASTICSEARCH_VERSION=5.0.0-alpha2
+    # - gemfile: test/gemfiles/nobrainer.gemfile
+    #   env: NOBRAINER=true

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 1.3.0
+
+- Added support for Elasticsearch 5.0 alpha
+- Added support for phrase matches
+- Added support for procs for `index_prefix` option
+
 ## 1.2.1
 
 - Added `multi_search` method

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 Andrew Kane
+Copyright (c) 2013-2016 Andrew Kane
 
 MIT License
 

data/README.md

@@ -21,8 +21,6 @@ Plus:
 - “Did you mean” suggestions
 - works with ActiveRecord, Mongoid, and NoBrainer
 
-**Searchkick 1.0 was just released!** See [instructions for upgrading](#100)
-
 :speech_balloon: Get [handcrafted updates](http://chartkick.us7.list-manage.com/subscribe?u=952c861f99eb43084e0a49f98&id=6ea6541e8e&group[0][4]=true) for new features
 
 :tangerine: Battle-tested at [Instacart](https://www.instacart.com/opensource)
@@ -247,6 +245,12 @@ Available options are:
 User.search params[:q], fields: [{email: :exact}, :name]
 ```
 
+### Phrase Matches
+
+```ruby
+User.search "fresh honey", match: :phrase
+```
+
 ### Language
 
 Searchkick defaults to English for stemming.  To change this, use:
@@ -329,7 +333,7 @@ gem 'gemoji-parser'
 And use:
 
 ```ruby
-Product.search "[emoji go here]", emoji: true
+Product.search "🍨🍰", emoji: true
 ```
 
 ### Indexing
@@ -633,6 +637,12 @@ price_ranges = [{to: 20}, {from: 20, to: 50}, {from: 50}]
 Product.search "*", aggs: {price: {ranges: price_ranges}}
 ```
 
+Minimum document count
+
+```ruby
+Product.search "apples", aggs: {store_id: {min_doc_count: 2}}
+```
+
 #### Moving From Facets
 
 1. Replace `facets` with `aggs` in searches. **Note:** Stats facets are not supported at this time.
@@ -838,7 +848,7 @@ Searchkick supports [Elasticsearch’s routing feature](https://www.elastic.co/b
 class Business < ActiveRecord::Base
   searchkick routing: true
 
-  def searchkick_routing
+  def search_routing
     city_id
   end
 end
@@ -919,20 +929,20 @@ Searchkick uses `ENV["ELASTICSEARCH_URL"]` for the Elasticsearch server.  This d
 
 ### Heroku
 
-Choose an add-on: [SearchBox](https://elements.heroku.com/addons/searchbox), [Bonsai](https://elements.heroku.com/addons/bonsai), or [Found](https://elements.heroku.com/addons/foundelasticsearch).
+Choose an add-on: [SearchBox](https://elements.heroku.com/addons/searchbox), [Bonsai](https://elements.heroku.com/addons/bonsai), or [Elastic Cloud](https://elements.heroku.com/addons/foundelasticsearch).
 
 ```sh
 # SearchBox
-heroku addons:add searchbox:starter
-heroku config:add ELASTICSEARCH_URL=`heroku config:get SEARCHBOX_URL`
+heroku addons:create searchbox:starter
+heroku config:set ELASTICSEARCH_URL=`heroku config:get SEARCHBOX_URL`
 
 # Bonsai
-heroku addons:add bonsai
-heroku config:add ELASTICSEARCH_URL=`heroku config:get BONSAI_URL`
+heroku addons:create bonsai
+heroku config:set ELASTICSEARCH_URL=`heroku config:get BONSAI_URL`
 
 # Found
-heroku addons:add foundelasticsearch
-heroku config:add ELASTICSEARCH_URL=`heroku config:get FOUNDELASTICSEARCH_URL`
+heroku addons:create foundelasticsearch
+heroku config:set ELASTICSEARCH_URL=`heroku config:get FOUNDELASTICSEARCH_URL`
 ```
 
 Then deploy and reindex:
@@ -1104,7 +1114,7 @@ Searchkick.multi_search([fresh_products, frozen_products])
 
 Then use `fresh_products` and `frozen_products` as typical results.
 
-**Note:** Errors are not raised as with single requests. Use the `error` method on each query to check for errors. Also, the `below` option for misspellings is ignored.
+**Note:** Errors are not raised as with single requests. Use the `error` method on each query to check for errors. Also, if you use the `below` option for misspellings, misspellings will be disabled.
 
 ## Reference
 
@@ -1316,6 +1326,10 @@ product.reindex # don't forget this
 Product.searchkick_index.refresh # or this
 ```
 
+## Multi-Tenancy
+
+Check out [this great post](https://www.tiagoamaro.com.br/2014/12/11/multi-tenancy-with-searchkick/) on the [Apartment](https://github.com/influitive/apartment) gem. Follow a similar pattern if you use another gem.
+
 ## Migrating from Tire
 
 1. Change `search` methods to `tire.search` and add index name in existing search calls

data/lib/searchkick.rb

@@ -58,6 +58,10 @@ module Searchkick
     @server_version ||= client.info["version"]["number"]
   end
 
+  def self.server_below?(version)
+    Gem::Version.new(server_version) < Gem::Version.new(version)
+  end
+
   def self.enable_callbacks
     self.callbacks_value = nil
   end

data/lib/searchkick/index.rb

@@ -36,7 +36,7 @@ module Searchkick
         begin
           client.indices.get_alias(name: name).keys
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          []
+          {}
         end
       actions = old_indices.map { |old_name| {remove: {index: old_name, alias: name}} } + [{add: {index: new_name, alias: name}}]
       client.indices.update_aliases body: {actions: actions}
@@ -146,7 +146,7 @@ module Searchkick
         begin
           client.indices.get_aliases
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          []
+          {}
         end
       indices = all_indices.select { |k, v| (v.empty? || v["aliases"].empty?) && k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys
       indices.each do |index|
@@ -218,6 +218,22 @@ module Searchkick
         settings = options[:settings] || {}
         mappings = options[:mappings]
       else
+        below22 = Searchkick.server_below?("2.2.0")
+        below50 = Searchkick.server_below?("5.0.0-alpha1")
+        default_type = below50 ? "string" : "text"
+        default_analyzer = below50 ? :default_index : :default
+        keyword_mapping =
+          if below50
+            {
+              type: default_type,
+              index: "not_analyzed"
+            }
+          else
+            {
+              type: "keyword"
+            }
+          end
+
         settings = {
           analysis: {
             analyzer: {
@@ -226,7 +242,7 @@ module Searchkick
                 tokenizer: "keyword",
                 filter: ["lowercase"] + (options[:stem_conversions] == false ? [] : ["searchkick_stemmer"])
               },
-              default_index: {
+              default_analyzer => {
                 type: "custom",
                 # character filters -> tokenizer -> token filters
                 # https://www.elastic.co/guide/en/elasticsearch/guide/current/analysis-intro.html
@@ -380,8 +396,8 @@ module Searchkick
           # - Only apply the synonym expansion at index time
           # - Don't have the synonym filter applied search
           # - Use directional synonyms where appropriate. You want to make sure that you're not injecting terms that are too general.
-          settings[:analysis][:analyzer][:default_index][:filter].insert(4, "searchkick_synonym")
-          settings[:analysis][:analyzer][:default_index][:filter] << "searchkick_synonym"
+          settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_synonym")
+          settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_synonym"
 
           %w(word_start word_middle word_end).each do |type|
             settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_synonym")
@@ -395,8 +411,8 @@ module Searchkick
             synonyms_path: Searchkick.wordnet_path
           }
 
-          settings[:analysis][:analyzer][:default_index][:filter].insert(4, "searchkick_wordnet")
-          settings[:analysis][:analyzer][:default_index][:filter] << "searchkick_wordnet"
+          settings[:analysis][:analyzer][default_analyzer][:filter].insert(4, "searchkick_wordnet")
+          settings[:analysis][:analyzer][default_analyzer][:filter] << "searchkick_wordnet"
 
           %w(word_start word_middle word_end).each do |type|
             settings[:analysis][:analyzer]["searchkick_#{type}_index".to_sym][:filter].insert(2, "searchkick_wordnet")
@@ -416,7 +432,7 @@ module Searchkick
           mapping[conversions_field] = {
             type: "nested",
             properties: {
-              query: {type: "string", analyzer: "searchkick_keyword"},
+              query: {type: default_type, analyzer: "searchkick_keyword"},
               count: {type: "integer"}
             }
           }
@@ -430,32 +446,39 @@ module Searchkick
         word = options[:word] != false && (!options[:match] || options[:match] == :word)
 
         mapping_options.values.flatten.uniq.each do |field|
-          field_mapping = {
-            type: "multi_field",
-            fields: {}
-          }
+          fields = {}
 
-          unless mapping_options[:only_analyzed].include?(field)
-            field_mapping[:fields][field] = {type: "string", index: "not_analyzed"}
+          if mapping_options[:only_analyzed].include?(field)
+            fields[field] = {type: default_type, index: "no"}
+          else
+            fields[field] = keyword_mapping
           end
 
           if !options[:searchable] || mapping_options[:searchable].include?(field)
             if word
-              field_mapping[:fields]["analyzed"] = {type: "string", index: "analyzed"}
+              fields["analyzed"] = {type: default_type, index: "analyzed", analyzer: default_analyzer}
 
               if mapping_options[:highlight].include?(field)
-                field_mapping[:fields]["analyzed"][:term_vector] = "with_positions_offsets"
+                fields["analyzed"][:term_vector] = "with_positions_offsets"
               end
             end
 
-            mapping_options.except(:highlight, :searchable, :only_analyzed).each do |type, fields|
-              if options[:match] == type || fields.include?(field)
-                field_mapping[:fields][type] = {type: "string", index: "analyzed", analyzer: "searchkick_#{type}_index"}
+            mapping_options.except(:highlight, :searchable, :only_analyzed).each do |type, f|
+              if options[:match] == type || f.include?(field)
+                fields[type] = {type: default_type, index: "analyzed", analyzer: "searchkick_#{type}_index"}
               end
             end
           end
 
-          mapping[field] = field_mapping
+          mapping[field] =
+            if below50
+              {
+                type: "multi_field",
+                fields: fields
+              }
+            elsif fields[field]
+              fields[field].merge(fields: fields.except(field))
+           end
         end
 
         (options[:locations] || []).map(&:to_s).each do |field|
@@ -466,7 +489,7 @@ module Searchkick
 
         (options[:unsearchable] || []).map(&:to_s).each do |field|
           mapping[field] = {
-            type: "string",
+            type: default_type,
             index: "no"
           }
         end
@@ -484,21 +507,35 @@ module Searchkick
           # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
           # however, we can include the not_analyzed field in _all
           # and the _all index analyzer will take care of it
-          "{name}" => {type: "string", index: "not_analyzed", include_in_all: !options[:searchable]}
+          "{name}" => keyword_mapping.merge(include_in_all: !options[:searchable])
         }
 
+        dynamic_fields["{name}"][:ignore_above] = 256 unless below22
+
         unless options[:searchable]
           if options[:match] && options[:match] != :word
-            dynamic_fields[options[:match]] = {type: "string", index: "analyzed", analyzer: "searchkick_#{options[:match]}_index"}
+            dynamic_fields[options[:match]] = {type: default_type, index: "analyzed", analyzer: "searchkick_#{options[:match]}_index"}
           end
 
           if word
-            dynamic_fields["analyzed"] = {type: "string", index: "analyzed"}
+            dynamic_fields["analyzed"] = {type: default_type, index: "analyzed"}
           end
         end
 
+        # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
+        multi_field =
+          if below50
+            {
+              type: "multi_field",
+              fields: dynamic_fields
+            }
+          else
+            dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}"))
+          end
+
         mappings = {
           _default_: {
+            _all: {type: default_type, index: "analyzed", analyzer: default_analyzer},
             properties: mapping,
             _routing: routing,
             # https://gist.github.com/kimchy/2898285
@@ -507,11 +544,7 @@ module Searchkick
                 string_template: {
                   match: "*",
                   match_mapping_type: "string",
-                  mapping: {
-                    # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
-                    type: "multi_field",
-                    fields: dynamic_fields
-                  }
+                  mapping: multi_field
                 }
               }
             ]

data/lib/searchkick/model.rb

@@ -15,7 +15,9 @@ module Searchkick
         class_variable_set :@@searchkick_options, options.dup
         class_variable_set :@@searchkick_klass, self
         class_variable_set :@@searchkick_callbacks, callbacks
-        class_variable_set :@@searchkick_index, options[:index_name] || [options[:index_prefix], model_name.plural, Searchkick.env].compact.join("_")
+        class_variable_set :@@searchkick_index, options[:index_name] ||
+          (options[:index_prefix].respond_to?(:call) && proc { [options[:index_prefix].call, model_name.plural, Searchkick.env].compact.join("_") }) ||
+          [options[:index_prefix], model_name.plural, Searchkick.env].compact.join("_")
 
         class << self
           def searchkick_search(term = nil, options = {}, &block)

data/lib/searchkick/query.rb

@@ -149,28 +149,7 @@ module Searchkick
     end
 
     def prepare
-      boost_fields = {}
-      fields = options[:fields] || searchkick_options[:searchable]
-      fields =
-        if fields
-          if options[:autocomplete]
-            fields.map { |f| "#{f}.autocomplete" }
-          else
-            fields.map do |value|
-              k, v = value.is_a?(Hash) ? value.to_a.first : [value, options[:match] || searchkick_options[:match] || :word]
-              k2, boost = k.to_s.split("^", 2)
-              field = "#{k2}.#{v == :word ? 'analyzed' : v}"
-              boost_fields[field] = boost.to_f if boost
-              field
-            end
-          end
-        else
-          if options[:autocomplete]
-            (searchkick_options[:autocomplete] || []).map { |f| "#{f}.autocomplete" }
-          else
-            ["_all"]
-          end
-        end
+      boost_fields, fields = set_fields
 
       operator = options[:operator] || (options[:partial] ? "or" : "and")
 
@@ -187,7 +166,6 @@ module Searchkick
       personalize_field = searchkick_options[:personalize]
 
       all = term == "*"
-      facet_limits = {}
 
       options[:json] ||= options[:body]
       if options[:json]
@@ -256,10 +234,19 @@ module Searchkick
               factor = boost_fields[field] || 1
               shared_options = {
                 query: term,
-                operator: operator,
                 boost: 10 * factor
               }
 
+              match_type =
+                if field.end_with?(".phrase")
+                  field = field.sub(/\.phrase\z/, ".analyzed")
+                  :match_phrase
+                else
+                  :match
+                end
+
+              shared_options[:operator] = operator if match_type == :match || below50?
+
               if field == "_all" || field.end_with?(".analyzed")
                 shared_options[:cutoff_frequency] = 0.001 unless operator == "and" || misspellings == false
                 qs.concat [
@@ -274,11 +261,11 @@ module Searchkick
                 qs << shared_options.merge(analyzer: analyzer)
               end
 
-              if misspellings != false
+              if misspellings != false && (match_type == :match || below50?)
                 qs.concat qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) }
               end
 
-              queries.concat(qs.map { |q| {match: {field => q}} })
+              queries.concat(qs.map { |q| {match_type => {field => q}} })
             end
 
             payload = {
@@ -324,52 +311,9 @@ module Searchkick
         custom_filters = []
         multiply_filters = []
 
-        boost_by = options[:boost_by] || {}
-
-        if boost_by.is_a?(Array)
-          boost_by = Hash[boost_by.map { |f| [f, {factor: 1}] }]
-        elsif boost_by.is_a?(Hash)
-          multiply_by, boost_by = boost_by.partition { |_, v| v[:boost_mode] == "multiply" }.map { |i| Hash[i] }
-        end
-        boost_by[options[:boost]] = {factor: 1} if options[:boost]
-
-        custom_filters.concat boost_filters(boost_by, log: true)
-        multiply_filters.concat boost_filters(multiply_by || {})
-
-        boost_where = options[:boost_where] || {}
-        if options[:user_id] && personalize_field
-          boost_where[personalize_field] = options[:user_id]
-        end
-        if options[:personalize]
-          boost_where = boost_where.merge(options[:personalize])
-        end
-        boost_where.each do |field, value|
-          if value.is_a?(Array) && value.first.is_a?(Hash)
-            value.each do |value_factor|
-              custom_filters << custom_filter(field, value_factor[:value], value_factor[:factor])
-            end
-          elsif value.is_a?(Hash)
-            custom_filters << custom_filter(field, value[:value], value[:factor])
-          else
-            factor = 1000
-            custom_filters << custom_filter(field, value, factor)
-          end
-        end
-
-        boost_by_distance = options[:boost_by_distance]
-        if boost_by_distance
-          boost_by_distance = {function: :gauss, scale: "5mi"}.merge(boost_by_distance)
-          if !boost_by_distance[:field] || !boost_by_distance[:origin]
-            raise ArgumentError, "boost_by_distance requires :field and :origin"
-          end
-          function_params = boost_by_distance.select { |k, _| [:origin, :scale, :offset, :decay].include?(k) }
-          function_params[:origin] = location_value(function_params[:origin])
-          custom_filters << {
-            boost_by_distance[:function] => {
-              boost_by_distance[:field] => function_params
-            }
-          }
-        end
+        set_boost_by(multiply_filters, custom_filters)
+        set_boost_where(custom_filters, personalize_field)
+        set_boost_by_distance(custom_filters) if options[:boost_by_distance]
 
         if custom_filters.any?
           payload = {
@@ -399,187 +343,23 @@ module Searchkick
         payload[:explain] = options[:explain] if options[:explain]
 
         # order
-        if options[:order]
-          order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
-          # TODO id transformation for arrays
-          payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? :_id : k, v] }]
-        end
+        set_order(payload) if options[:order]
 
         # filters
         filters = where_filters(options[:where])
-        if filters.any?
-          if options[:facets] || options[:aggs]
-            payload[:filter] = {
-              and: filters
-            }
-          else
-            # more efficient query if no facets
-            payload[:query] = {
-              filtered: {
-                query: payload[:query],
-                filter: {
-                  and: filters
-                }
-              }
-            }
-          end
-        end
+        set_filters(payload, filters) if filters.any?
 
         # facets
-        if options[:facets]
-          facets = options[:facets] || {}
-          facets = Hash[facets.map { |f| [f, {}] }] if facets.is_a?(Array) # convert to more advanced syntax
-
-          payload[:facets] = {}
-          facets.each do |field, facet_options|
-            # ask for extra facets due to
-            # https://github.com/elasticsearch/elasticsearch/issues/1305
-            size = facet_options[:limit] ? facet_options[:limit] + 150 : 1_000
-
-            if facet_options[:ranges]
-              payload[:facets][field] = {
-                range: {
-                  field.to_sym => facet_options[:ranges]
-                }
-              }
-            elsif facet_options[:stats]
-              payload[:facets][field] = {
-                terms_stats: {
-                  key_field: field,
-                  value_script: below14? ? "doc.score" : "_score",
-                  size: size
-                }
-              }
-            else
-              payload[:facets][field] = {
-                terms: {
-                  field: facet_options[:field] || field,
-                  size: size
-                }
-              }
-            end
-
-            facet_limits[field] = facet_options[:limit] if facet_options[:limit]
-
-            # offset is not possible
-            # http://elasticsearch-users.115913.n3.nabble.com/Is-pagination-possible-in-termsStatsFacet-td3422943.html
-
-            facet_options.deep_merge!(where: options.fetch(:where, {}).reject { |k| k == field }) if options[:smart_facets] == true
-            facet_filters = where_filters(facet_options[:where])
-            if facet_filters.any?
-              payload[:facets][field][:facet_filter] = {
-                and: {
-                  filters: facet_filters
-                }
-              }
-            end
-          end
-        end
+        set_facets(payload) if options[:facets]
 
         # aggregations
-        if options[:aggs]
-          aggs = options[:aggs]
-          payload[:aggs] = {}
-
-          aggs = Hash[aggs.map { |f| [f, {}] }] if aggs.is_a?(Array) # convert to more advanced syntax
-
-          aggs.each do |field, agg_options|
-            size = agg_options[:limit] ? agg_options[:limit] : 1_000
-            shared_agg_options = agg_options.slice(:order)
-
-            if agg_options[:ranges]
-              payload[:aggs][field] = {
-                range: {
-                  field: agg_options[:field] || field,
-                  ranges: agg_options[:ranges]
-                }.merge(shared_agg_options)
-              }
-            elsif agg_options[:date_ranges]
-              payload[:aggs][field] = {
-                date_range: {
-                  field: agg_options[:field] || field,
-                  ranges: agg_options[:date_ranges]
-                }.merge(shared_agg_options)
-              }
-            else
-              payload[:aggs][field] = {
-                terms: {
-                  field: agg_options[:field] || field,
-                  size: size
-                }.merge(shared_agg_options)
-              }
-            end
-
-            where = {}
-            where = (options[:where] || {}).reject { |k| k == field } unless options[:smart_aggs] == false
-            agg_filters = where_filters(where.merge(agg_options[:where] || {}))
-            if agg_filters.any?
-              payload[:aggs][field] = {
-                filter: {
-                  bool: {
-                    must: agg_filters
-                  }
-                },
-                aggs: {
-                  field => payload[:aggs][field]
-                }
-              }
-            end
-          end
-        end
+        set_aggregations(payload) if options[:aggs]
 
         # suggestions
-        if options[:suggest]
-          suggest_fields = (searchkick_options[:suggest] || []).map(&:to_s)
-
-          # intersection
-          if options[:fields]
-            suggest_fields &= options[:fields].map { |v| (v.is_a?(Hash) ? v.keys.first : v).to_s.split("^", 2).first }
-          end
-
-          if suggest_fields.any?
-            payload[:suggest] = {text: term}
-            suggest_fields.each do |field|
-              payload[:suggest][field] = {
-                phrase: {
-                  field: "#{field}.suggest"
-                }
-              }
-            end
-          end
-        end
+        set_suggestions(payload) if options[:suggest]
 
         # highlight
-        if options[:highlight]
-          payload[:highlight] = {
-            fields: Hash[fields.map { |f| [f, {}] }]
-          }
-
-          if options[:highlight].is_a?(Hash)
-            if (tag = options[:highlight][:tag])
-              payload[:highlight][:pre_tags] = [tag]
-              payload[:highlight][:post_tags] = [tag.to_s.gsub(/\A</, "</")]
-            end
-
-            if (fragment_size = options[:highlight][:fragment_size])
-              payload[:highlight][:fragment_size] = fragment_size
-            end
-            if (encoder = options[:highlight][:encoder])
-              payload[:highlight][:encoder] = encoder
-            end
-
-            highlight_fields = options[:highlight][:fields]
-            if highlight_fields
-              payload[:highlight][:fields] = {}
-
-              highlight_fields.each do |name, opts|
-                payload[:highlight][:fields]["#{name}.#{@match_suffix}"] = opts || {}
-              end
-            end
-          end
-
-          @highlighted_fields = payload[:highlight][:fields].keys
-        end
+        set_highlights(payload, fields) if options[:highlight]
 
         # An empty array will cause only the _id and _type for each hit to be returned
         # doc for :select - http://www.elasticsearch.org/guide/reference/api/search/fields/
@@ -606,13 +386,286 @@ module Searchkick
       end
 
       @body = payload
-      @facet_limits = facet_limits
+      @facet_limits = @facet_limits || {}
       @page = page
       @per_page = per_page
       @padding = padding
       @load = load
     end
 
+    def set_fields
+      boost_fields = {}
+      fields = options[:fields] || searchkick_options[:searchable]
+      fields =
+        if fields
+          if options[:autocomplete]
+            fields.map { |f| "#{f}.autocomplete" }
+          else
+            fields.map do |value|
+              k, v = value.is_a?(Hash) ? value.to_a.first : [value, options[:match] || searchkick_options[:match] || :word]
+              k2, boost = k.to_s.split("^", 2)
+              field = "#{k2}.#{v == :word ? 'analyzed' : v}"
+              boost_fields[field] = boost.to_f if boost
+              field
+            end
+          end
+        else
+          if options[:autocomplete]
+            (searchkick_options[:autocomplete] || []).map { |f| "#{f}.autocomplete" }
+          else
+            ["_all"]
+          end
+        end
+      [boost_fields, fields]
+    end
+
+    def set_boost_by_distance(custom_filters)
+      boost_by_distance = options[:boost_by_distance] || {}
+      boost_by_distance = {function: :gauss, scale: "5mi"}.merge(boost_by_distance)
+      if !boost_by_distance[:field] || !boost_by_distance[:origin]
+        raise ArgumentError, "boost_by_distance requires :field and :origin"
+      end
+      function_params = boost_by_distance.select { |k, _| [:origin, :scale, :offset, :decay].include?(k) }
+      function_params[:origin] = location_value(function_params[:origin])
+      custom_filters << {
+        boost_by_distance[:function] => {
+          boost_by_distance[:field] => function_params
+        }
+      }
+    end
+
+    def set_boost_by(multiply_filters, custom_filters)
+      boost_by = options[:boost_by] || {}
+      if boost_by.is_a?(Array)
+        boost_by = Hash[boost_by.map { |f| [f, {factor: 1}] }]
+      elsif boost_by.is_a?(Hash)
+        multiply_by, boost_by = boost_by.partition { |_, v| v[:boost_mode] == "multiply" }.map { |i| Hash[i] }
+      end
+      boost_by[options[:boost]] = {factor: 1} if options[:boost]
+
+      custom_filters.concat boost_filters(boost_by, log: true)
+      multiply_filters.concat boost_filters(multiply_by || {})
+    end
+
+    def set_boost_where(custom_filters, personalize_field)
+      boost_where = options[:boost_where] || {}
+      if options[:user_id] && personalize_field
+        boost_where[personalize_field] = options[:user_id]
+      end
+      if options[:personalize]
+        boost_where = boost_where.merge(options[:personalize])
+      end
+      boost_where.each do |field, value|
+        if value.is_a?(Array) && value.first.is_a?(Hash)
+          value.each do |value_factor|
+            custom_filters << custom_filter(field, value_factor[:value], value_factor[:factor])
+          end
+        elsif value.is_a?(Hash)
+          custom_filters << custom_filter(field, value[:value], value[:factor])
+        else
+          factor = 1000
+          custom_filters << custom_filter(field, value, factor)
+        end
+      end
+    end
+
+    def set_suggestions(payload)
+      suggest_fields = (searchkick_options[:suggest] || []).map(&:to_s)
+
+      # intersection
+      if options[:fields]
+        suggest_fields &= options[:fields].map { |v| (v.is_a?(Hash) ? v.keys.first : v).to_s.split("^", 2).first }
+      end
+
+      if suggest_fields.any?
+        payload[:suggest] = {text: term}
+        suggest_fields.each do |field|
+          payload[:suggest][field] = {
+            phrase: {
+              field: "#{field}.suggest"
+            }
+          }
+        end
+      end
+    end
+
+    def set_highlights(payload, fields)
+      payload[:highlight] = {
+        fields: Hash[fields.map { |f| [f, {}] }]
+      }
+
+      if options[:highlight].is_a?(Hash)
+        if (tag = options[:highlight][:tag])
+          payload[:highlight][:pre_tags] = [tag]
+          payload[:highlight][:post_tags] = [tag.to_s.gsub(/\A</, "</")]
+        end
+
+        if (fragment_size = options[:highlight][:fragment_size])
+          payload[:highlight][:fragment_size] = fragment_size
+        end
+        if (encoder = options[:highlight][:encoder])
+          payload[:highlight][:encoder] = encoder
+        end
+
+        highlight_fields = options[:highlight][:fields]
+        if highlight_fields
+          payload[:highlight][:fields] = {}
+
+          highlight_fields.each do |name, opts|
+            payload[:highlight][:fields]["#{name}.#{@match_suffix}"] = opts || {}
+          end
+        end
+      end
+
+      @highlighted_fields = payload[:highlight][:fields].keys
+    end
+
+    def set_aggregations(payload)
+      aggs = options[:aggs]
+      payload[:aggs] = {}
+
+      aggs = Hash[aggs.map { |f| [f, {}] }] if aggs.is_a?(Array) # convert to more advanced syntax
+
+      aggs.each do |field, agg_options|
+        size = agg_options[:limit] ? agg_options[:limit] : 1_000
+        shared_agg_options = agg_options.slice(:order, :min_doc_count)
+
+        if agg_options[:ranges]
+          payload[:aggs][field] = {
+            range: {
+              field: agg_options[:field] || field,
+              ranges: agg_options[:ranges]
+            }.merge(shared_agg_options)
+          }
+        elsif agg_options[:date_ranges]
+          payload[:aggs][field] = {
+            date_range: {
+              field: agg_options[:field] || field,
+              ranges: agg_options[:date_ranges]
+            }.merge(shared_agg_options)
+          }
+        else
+          payload[:aggs][field] = {
+            terms: {
+              field: agg_options[:field] || field,
+              size: size
+            }.merge(shared_agg_options)
+          }
+        end
+
+        where = {}
+        where = (options[:where] || {}).reject { |k| k == field } unless options[:smart_aggs] == false
+        agg_filters = where_filters(where.merge(agg_options[:where] || {}))
+        if agg_filters.any?
+          payload[:aggs][field] = {
+            filter: {
+              bool: {
+                must: agg_filters
+              }
+            },
+            aggs: {
+              field => payload[:aggs][field]
+            }
+          }
+        end
+      end
+    end
+
+    def set_facets(payload)
+      facets = options[:facets] || {}
+      facets = Hash[facets.map { |f| [f, {}] }] if facets.is_a?(Array) # convert to more advanced syntax
+      facet_limits = {}
+      payload[:facets] = {}
+
+      facets.each do |field, facet_options|
+        # ask for extra facets due to
+        # https://github.com/elasticsearch/elasticsearch/issues/1305
+        size = facet_options[:limit] ? facet_options[:limit] + 150 : 1_000
+
+        if facet_options[:ranges]
+          payload[:facets][field] = {
+            range: {
+              field.to_sym => facet_options[:ranges]
+            }
+          }
+        elsif facet_options[:stats]
+          payload[:facets][field] = {
+            terms_stats: {
+              key_field: field,
+              value_script: below14? ? "doc.score" : "_score",
+              size: size
+            }
+          }
+        else
+          payload[:facets][field] = {
+            terms: {
+              field: facet_options[:field] || field,
+              size: size
+            }
+          }
+        end
+
+        facet_limits[field] = facet_options[:limit] if facet_options[:limit]
+
+        # offset is not possible
+        # http://elasticsearch-users.115913.n3.nabble.com/Is-pagination-possible-in-termsStatsFacet-td3422943.html
+
+        facet_options.deep_merge!(where: options.fetch(:where, {}).reject { |k| k == field }) if options[:smart_facets] == true
+        facet_filters = where_filters(facet_options[:where])
+        if facet_filters.any?
+          payload[:facets][field][:facet_filter] = {
+            and: {
+              filters: facet_filters
+            }
+          }
+        end
+      end
+
+      @facet_limits = facet_limits
+    end
+
+    def set_filters(payload, filters)
+      if options[:facets] || options[:aggs]
+        if below20?
+          payload[:filter] = {
+            and: filters
+          }
+        else
+          payload[:post_filter] = {
+            bool: {
+              filter: filters
+            }
+          }
+        end
+      else
+        # more efficient query if no facets
+        if below20?
+          payload[:query] = {
+            filtered: {
+              query: payload[:query],
+              filter: {
+                and: filters
+              }
+            }
+          }
+        else
+          payload[:query] = {
+            bool: {
+              must: payload[:query],
+              filter: filters
+            }
+          }
+        end
+      end
+    end
+
+    # TODO id transformation for arrays
+    def set_order(payload)
+      order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
+      id_field = below50? ? :_id : :_uid
+      payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? id_field : k, v] }]
+    end
+
     def where_filters(where)
       filters = []
       (where || {}).each do |field, value|
@@ -620,7 +673,11 @@ module Searchkick
 
         if field == :or
           value.each do |or_clause|
-            filters << {or: or_clause.map { |or_statement| {and: where_filters(or_statement)} }}
+            if below50?
+              filters << {or: or_clause.map { |or_statement| {and: where_filters(or_statement)} }}
+            else
+              filters << {bool: {should: or_clause.map { |or_statement| {bool: {filter: where_filters(or_statement)}} }}}
+            end
           end
         else
           # expand ranges
@@ -654,7 +711,11 @@ module Searchkick
               when :regexp # support for regexp queries without using a regexp ruby object
                 filters << {regexp: {field => {value: op_value}}}
               when :not # not equal
-                filters << {not: {filter: term_filters(field, op_value)}}
+                if below50?
+                  filters << {not: {filter: term_filters(field, op_value)}}
+                else
+                  filters << {bool: {must_not: term_filters(field, op_value)}}
+                end
               when :all
                 op_value.each do |value|
                   filters << term_filters(field, value)
@@ -694,12 +755,20 @@ module Searchkick
     def term_filters(field, value)
       if value.is_a?(Array) # in query
         if value.any?(&:nil?)
-          {or: [term_filters(field, nil), term_filters(field, value.compact)]}
+          if below50?
+            {or: [term_filters(field, nil), term_filters(field, value.compact)]}
+          else
+            {bool: {should: [term_filters(field, nil), term_filters(field, value.compact)]}}
+          end
         else
           {in: {field => value}}
         end
       elsif value.nil?
-        {missing: {"field" => field, existence: true, null_value: true}}
+        if below50?
+          {missing: {field: field, existence: true, null_value: true}}
+        else
+          {bool: {must_not: {exists: {field: field}}}}
+        end
       elsif value.is_a?(Regexp)
         {regexp: {field => {value: value.source}}}
       else
@@ -708,12 +777,19 @@ module Searchkick
     end
 
     def custom_filter(field, value, factor)
-      {
-        filter: {
-          and: where_filters(field => value)
-        },
-        boost_factor: factor
-      }
+      if below50?
+        {
+          filter: {
+            and: where_filters(field => value)
+          },
+          boost_factor: factor
+        }
+      else
+        {
+          filter: where_filters(field => value),
+          weight: factor
+        }
+      end
     end
 
     def boost_filters(boost_by, options = {})
@@ -747,19 +823,19 @@ module Searchkick
     end
 
     def below12?
-      below_version?("1.2.0")
+      Searchkick.server_below?("1.2.0")
     end
 
     def below14?
-      below_version?("1.4.0")
+      Searchkick.server_below?("1.4.0")
     end
 
     def below20?
-      below_version?("2.0.0")
+      Searchkick.server_below?("2.0.0")
     end
 
-    def below_version?(version)
-      Gem::Version.new(Searchkick.server_version) < Gem::Version.new(version)
+    def below50?
+      Searchkick.server_below?("5.0.0-alpha1")
     end
   end
 end