searchkick 0.9.1 → 1.0.0

data/lib/searchkick/query.rb

@@ -92,24 +92,37 @@ module Searchkick
               shared_options = {
                 query: term,
                 operator: operator,
-                boost: factor
+                boost: 10 * factor
               }
 
-              if field == "_all" || field.end_with?(".analyzed")
-                misspellings = options.key?(:misspellings) ? options[:misspellings] : options[:mispellings] # why not?
-                if misspellings != false
-                  edit_distance = (misspellings.is_a?(Hash) && (misspellings[:edit_distance] || misspellings[:distance])) || 1
-                  transpositions = (misspellings.is_a?(Hash) && misspellings[:transpositions] == true) ? {fuzzy_transpositions: true} : {}
-                  prefix_length = (misspellings.is_a?(Hash) && misspellings[:prefix_length]) || 0
-                  qs.concat [
-                    shared_options.merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: 3, analyzer: "searchkick_search").merge(transpositions),
-                    shared_options.merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: 3, analyzer: "searchkick_search2").merge(transpositions)
-                  ]
+              misspellings =
+                if options.key?(:misspellings)
+                  options[:misspellings]
+                elsif options.key?(:mispellings)
+                  options[:mispellings] # why not?
+                else
+                  true
                 end
+
+              if misspellings != false
+                edit_distance = (misspellings.is_a?(Hash) && (misspellings[:edit_distance] || misspellings[:distance])) || 1
+                transpositions =
+                  if misspellings.is_a?(Hash) && misspellings.key?(:transpositions)
+                    {fuzzy_transpositions: misspellings[:transpositions]}
+                  elsif below14?
+                    {}
+                  else
+                    {fuzzy_transpositions: true}
+                  end
+                prefix_length = (misspellings.is_a?(Hash) && misspellings[:prefix_length]) || 0
+                max_expansions = (misspellings.is_a?(Hash) && misspellings[:max_expansions]) || 3
+              end
+
+              if field == "_all" || field.end_with?(".analyzed")
                 shared_options[:cutoff_frequency] = 0.001 unless operator == "and" || misspellings == false
                 qs.concat [
-                  shared_options.merge(boost: 10 * factor, analyzer: "searchkick_search"),
-                  shared_options.merge(boost: 10 * factor, analyzer: "searchkick_search2")
+                  shared_options.merge(analyzer: "searchkick_search"),
+                  shared_options.merge(analyzer: "searchkick_search2")
                 ]
               elsif field.end_with?(".exact")
                 f = field.split(".")[0..-2].join(".")
@@ -119,6 +132,10 @@ module Searchkick
                 qs << shared_options.merge(analyzer: analyzer)
               end
 
+              if misspellings != false
+                qs.concat qs.map { |q| q.except(:cutoff_frequency).merge(fuzziness: edit_distance, prefix_length: prefix_length, max_expansions: max_expansions, boost: factor).merge(transpositions) }
+              end
+
               queries.concat(qs.map { |q| {match: {field => q}} })
             end
 
@@ -135,7 +152,7 @@ module Searchkick
               if below12?
                 {script_score: {script: "doc['count'].value"}}
               else
-                {field_value_factor: {field: "count"}}
+                {field_value_factor: {field: "#{conversions_field}.count"}}
               end
 
             payload = {
@@ -144,13 +161,13 @@ module Searchkick
                 should: {
                   nested: {
                     path: conversions_field,
-                    score_mode: "total",
+                    score_mode: "sum",
                     query: {
                       function_score: {
                         boost_mode: "replace",
                         query: {
                           match: {
-                            query: term
+                            "#{conversions_field}.query" => term
                           }
                         }
                       }.merge(script_score)
@@ -170,11 +187,9 @@ module Searchkick
         if boost_by.is_a?(Array)
           boost_by = Hash[boost_by.map { |f| [f, {factor: 1}] }]
         elsif boost_by.is_a?(Hash)
-          multiply_by, boost_by = boost_by.partition { |k,v| v[:boost_mode] == "multiply" }.map{ |i| Hash[i] }
-        end
-        if options[:boost]
-          boost_by[options[:boost]] = {factor: 1}
+          multiply_by, boost_by = boost_by.partition { |_, v| v[:boost_mode] == "multiply" }.map { |i| Hash[i] }
         end
+        boost_by[options[:boost]] = {factor: 1} if options[:boost]
 
         custom_filters.concat boost_filters(boost_by, log: true)
         multiply_filters.concat boost_filters(multiply_by || {})
@@ -189,12 +204,10 @@ module Searchkick
         boost_where.each do |field, value|
           if value.is_a?(Array) && value.first.is_a?(Hash)
             value.each do |value_factor|
-              value, factor = value_factor[:value], value_factor[:factor]
-              custom_filters << custom_filter(field, value, factor)
+              custom_filters << custom_filter(field, value_factor[:value], value_factor[:factor])
             end
           elsif value.is_a?(Hash)
-            value, factor = value[:value], value[:factor]
-            custom_filters << custom_filter(field, value, factor)
+            custom_filters << custom_filter(field, value[:value], value[:factor])
           else
             factor = 1000
             custom_filters << custom_filter(field, value, factor)
@@ -207,7 +220,7 @@ module Searchkick
           if !boost_by_distance[:field] || !boost_by_distance[:origin]
             raise ArgumentError, "boost_by_distance requires :field and :origin"
           end
-          function_params = boost_by_distance.select { |k, v| [:origin, :scale, :offset, :decay].include?(k) }
+          function_params = boost_by_distance.select { |k, _| [:origin, :scale, :offset, :decay].include?(k) }
           function_params[:origin] = function_params[:origin].reverse
           custom_filters << {
             boost_by_distance[:function] => {
@@ -253,7 +266,7 @@ module Searchkick
         # filters
         filters = where_filters(options[:where])
         if filters.any?
-          if options[:facets]
+          if options[:facets] || options[:aggs]
             payload[:filter] = {
               and: filters
             }
@@ -273,9 +286,7 @@ module Searchkick
         # facets
         if options[:facets]
           facets = options[:facets] || {}
-          if facets.is_a?(Array) # convert to more advanced syntax
-            facets = Hash[facets.map { |f| [f, {}] }]
-          end
+          facets = Hash[facets.map { |f| [f, {}] }] if facets.is_a?(Array) # convert to more advanced syntax
 
           payload[:facets] = {}
           facets.each do |field, facet_options|
@@ -311,7 +322,7 @@ module Searchkick
             # offset is not possible
             # http://elasticsearch-users.115913.n3.nabble.com/Is-pagination-possible-in-termsStatsFacet-td3422943.html
 
-            facet_options.deep_merge!(where: options[:where].reject { |k| k == field }) if options[:smart_facets] == true
+            facet_options.deep_merge!(where: options.fetch(:where, {}).reject { |k| k == field }) if options[:smart_facets] == true
             facet_filters = where_filters(facet_options[:where])
             if facet_filters.any?
               payload[:facets][field][:facet_filter] = {
@@ -323,6 +334,57 @@ module Searchkick
           end
         end
 
+        # aggregations
+        if options[:aggs]
+          aggs = options[:aggs]
+          payload[:aggs] = {}
+
+          aggs = aggs.map { |f| [f, {}] }.to_h if aggs.is_a?(Array) # convert to more advanced syntax
+
+          aggs.each do |field, agg_options|
+            size = agg_options[:limit] ? agg_options[:limit] : 100_000
+
+            if agg_options[:ranges]
+              payload[:aggs][field] = {
+                range: {
+                  field: agg_options[:field] || field,
+                  ranges: agg_options[:ranges]
+                }
+              }
+            elsif agg_options[:date_ranges]
+              payload[:aggs][field] = {
+                date_range: {
+                  field: agg_options[:field] || field,
+                  ranges: agg_options[:date_ranges]
+                }
+              }
+            else
+              payload[:aggs][field] = {
+                terms: {
+                  field: agg_options[:field] || field,
+                  size: size
+                }
+              }
+            end
+
+            where = {}
+            where = (options[:where] || {}).reject { |k| k == field } unless options[:smart_aggs] == false
+            agg_filters = where_filters(where.merge(agg_options[:where] || {}))
+            if agg_filters.any?
+              payload[:aggs][field] = {
+                filter: {
+                  bool: {
+                    must: agg_filters
+                  }
+                },
+                aggs: {
+                  field => payload[:aggs][field]
+                }
+              }
+            end
+          end
+        end
+
         # suggestions
         if options[:suggest]
           suggest_fields = (searchkick_options[:suggest] || []).map(&:to_s)
@@ -356,6 +418,10 @@ module Searchkick
               payload[:highlight][:post_tags] = [tag.to_s.gsub(/\A</, "</")]
             end
 
+            if (fragment_size = options[:highlight][:fragment_size])
+              payload[:highlight][:fragment_size] = fragment_size
+            end
+
             highlight_fields = options[:highlight][:fields]
             if highlight_fields
               payload[:highlight][:fields] = {}
@@ -380,9 +446,7 @@ module Searchkick
         end
 
         # routing
-        if options[:routing]
-          @routing = options[:routing]
-        end
+        @routing = options[:routing] if options[:routing]
       end
 
       @body = payload
@@ -478,9 +542,7 @@ module Searchkick
             value = {gte: value.first, (value.exclude_end? ? :lt : :lte) => value.last}
           end
 
-          if value.is_a?(Array)
-            value = {in: value}
-          end
+          value = {in: value} if value.is_a?(Array)
 
           if value.is_a?(Hash)
             value.each do |op, op_value|
@@ -506,9 +568,11 @@ module Searchkick
               when :regexp # support for regexp queries without using a regexp ruby object
                 filters << {regexp: {field => {value: op_value}}}
               when :not # not equal
-                filters << {not: term_filters(field, op_value)}
+                filters << {not: {filter: term_filters(field, op_value)}}
               when :all
-                filters << {terms: {field => op_value, execution: "and"}}
+                op_value.each do |value|
+                  filters << term_filters(field, value)
+                end
               when :in
                 filters << term_filters(field, op_value)
               else
@@ -560,7 +624,7 @@ module Searchkick
     def custom_filter(field, value, factor)
       {
         filter: {
-          and: where_filters({field => value})
+          and: where_filters(field => value)
         },
         boost_factor: factor
       }
@@ -596,6 +660,10 @@ module Searchkick
       below_version?("1.4.0")
     end
 
+    def below20?
+      below_version?("2.0.0")
+    end
+
     def below_version?(version)
       Gem::Version.new(Searchkick.server_version) < Gem::Version.new(version)
     end

data/lib/searchkick/reindex_job.rb

@@ -1,6 +1,5 @@
 module Searchkick
   class ReindexJob
-
     def initialize(klass, id)
       @klass = klass
       @id = id
@@ -23,6 +22,5 @@ module Searchkick
         index.store record
       end
     end
-
   end
 end

data/lib/searchkick/reindex_v2_job.rb

@@ -19,6 +19,5 @@ module Searchkick
         index.store record
       end
     end
-
   end
 end

data/lib/searchkick/results.rb

@@ -26,7 +26,7 @@ module Searchkick
           # results can have different types
           results = {}
 
-          hits.group_by { |hit, i| hit["_type"] }.each do |type, grouped_hits|
+          hits.group_by { |hit, _| hit["_type"] }.each do |type, grouped_hits|
             results[type] = results_query(type.camelize.constantize, grouped_hits).to_a.index_by { |r| r.id.to_s }
           end
 
@@ -75,6 +75,23 @@ module Searchkick
       response["facets"]
     end
 
+    def aggs
+      @aggs ||= begin
+        response["aggregations"].dup.each do |field, filtered_agg|
+          buckets = filtered_agg[field]
+          # move the buckets one level above into the field hash
+          if buckets
+            filtered_agg.delete(field)
+            filtered_agg.merge!(buckets)
+          end
+        end
+      end
+    end
+
+    def took
+      response["took"]
+    end
+
     def model_name
       klass.model_name
     end

data/lib/searchkick/tasks.rb

@@ -1,7 +1,6 @@
 require "rake"
 
 namespace :searchkick do
-
   desc "reindex model"
   task reindex: :environment do
     if ENV["CLASS"]
@@ -31,5 +30,4 @@ namespace :searchkick do
     end
 
   end
-
 end

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "0.9.1"
+  VERSION = "1.0.0"
 end

data/test/aggs_test.rb

@@ -0,0 +1,102 @@
+require_relative "test_helper"
+
+class AggsTest < Minitest::Test
+  def setup
+    super
+    store [
+      {name: "Product Show", latitude: 37.7833, longitude: 12.4167, store_id: 1, in_stock: true, color: "blue", price: 21, created_at: 2.days.ago},
+      {name: "Product Hide", latitude: 29.4167, longitude: -98.5000, store_id: 2, in_stock: false, color: "green", price: 25, created_at: 2.days.from_now},
+      {name: "Product B", latitude: 43.9333, longitude: -122.4667, store_id: 2, in_stock: false, color: "red", price: 5},
+      {name: "Foo", latitude: 43.9333, longitude: 12.4667, store_id: 3, in_stock: false, color: "yellow", price: 15}
+    ]
+  end
+
+  def test_basic
+    assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: [:store_id])
+  end
+
+  def test_where
+    assert_equal ({1 => 1}), store_agg(aggs: {store_id: {where: {in_stock: true}}})
+  end
+
+  def test_field
+    assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: {store_id: {}})
+    assert_equal ({1 => 1, 2 => 2}), store_agg(aggs: {store_id: {field: "store_id"}})
+    assert_equal ({1 => 1, 2 => 2}), store_agg({aggs: {store_id_new: {field: "store_id"}}}, "store_id_new")
+  end
+
+  def test_limit
+    agg = Product.search("Product", aggs: {store_id: {limit: 1}}).aggs["store_id"]
+    assert_equal 1, agg["buckets"].size
+    # assert_equal 3, agg["doc_count"]
+    assert_equal(1, agg["sum_other_doc_count"]) if Gem::Version.new(Searchkick.server_version) >= Gem::Version.new("1.4.0")
+  end
+
+  def test_ranges
+    price_ranges = [{to: 10}, {from: 10, to: 20}, {from: 20}]
+    agg = Product.search("Product", aggs: {price: {ranges: price_ranges}}).aggs["price"]
+
+    assert_equal 3, agg["buckets"].size
+    assert_equal 10.0, agg["buckets"][0]["to"]
+    assert_equal 20.0, agg["buckets"][2]["from"]
+    assert_equal 1, agg["buckets"][0]["doc_count"]
+    assert_equal 0, agg["buckets"][1]["doc_count"]
+    assert_equal 2, agg["buckets"][2]["doc_count"]
+  end
+
+  def test_date_ranges
+    ranges = [{to: 1.day.ago}, {from: 1.day.ago, to: 1.day.from_now}, {from: 1.day.from_now}]
+    agg = Product.search("Product", aggs: {created_at: {date_ranges: ranges}}).aggs["created_at"]
+
+    assert_equal 1, agg["buckets"][0]["doc_count"]
+    assert_equal 1, agg["buckets"][1]["doc_count"]
+    assert_equal 1, agg["buckets"][2]["doc_count"]
+  end
+
+  def test_query_where
+    assert_equal ({1 => 1}), store_agg(where: {in_stock: true}, aggs: [:store_id])
+  end
+
+  def test_two_wheres
+    assert_equal ({2 => 1}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false}}})
+  end
+
+  def test_where_override
+    assert_equal ({}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false, color: "blue"}}})
+    assert_equal ({2 => 1}), store_agg(where: {color: "blue"}, aggs: {store_id: {where: {in_stock: false, color: "red"}}})
+  end
+
+  def test_skip
+    assert_equal ({1 => 1, 2 => 2}), store_agg(where: {store_id: 2}, aggs: [:store_id])
+  end
+
+  def test_skip_complex
+    assert_equal ({1 => 1, 2 => 1}), store_agg(where: {store_id: 2, price: {gt: 5}}, aggs: [:store_id])
+  end
+
+  def test_multiple
+    assert_equal ({"store_id" => {1 => 1, 2 => 2}, "color" => {"blue" => 1, "green" => 1, "red" => 1}}), store_multiple_aggs(aggs: [:store_id, :color])
+  end
+
+  def test_smart_aggs_false
+    assert_equal ({2 => 2}), store_agg(where: {color: "red"}, aggs: {store_id: {where: {in_stock: false}}}, smart_aggs: false)
+    assert_equal ({2 => 2}), store_agg(where: {color: "blue"}, aggs: {store_id: {where: {in_stock: false}}}, smart_aggs: false)
+  end
+
+  protected
+
+  def buckets_as_hash(agg)
+    agg["buckets"].map { |v| [v["key"], v["doc_count"]] }.to_h
+  end
+
+  def store_agg(options, agg_key = "store_id")
+    buckets = Product.search("Product", options).aggs[agg_key]
+    buckets_as_hash(buckets)
+  end
+
+  def store_multiple_aggs(options)
+    Product.search("Product", options).aggs.map do |field, filtered_agg|
+      [field, buckets_as_hash(filtered_agg)]
+    end.to_h
+  end
+end

data/test/autocomplete_test.rb

@@ -1,7 +1,6 @@
 require_relative "test_helper"
 
-class TestAutocomplete < Minitest::Test
-
+class AutocompleteTest < Minitest::Test
   def test_autocomplete
     store_names ["Hummus"]
     assert_search "hum", ["Hummus"], autocomplete: true
@@ -63,5 +62,4 @@ class TestAutocomplete < Minitest::Test
     store_names ["hi@example.org"]
     assert_search "hi@example.org", ["hi@example.org"], fields: [{name: :exact}]
   end
-
 end