searchkick 1.3.1 → 1.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8305aa9120f165e3db35f1dc1a5662ac29f97b0c
-  data.tar.gz: 3dc530c3469c997ac3be7d579a56b3f3e7e63501
+  metadata.gz: e749d5351a6a6132cf79bb8294ad8e86c72968b0
+  data.tar.gz: d6ebce4be756ff0cbd9ac60653b95c9e0aaddbba
 SHA512:
-  metadata.gz: 0f72db4715189311702fbfdb14ab8a4d7757ba8de1774af53ccbea564ac187ed74ade723f4aa14d9a5221ec0c1e15cd98e6d7f5aa39312c734607baa02b47483
-  data.tar.gz: 3b9bdbbb2e189abb6226d69b45ffd20c42ac1b272b90b20beb8357de6b3b180818aa76879952585d3c8ddba5b18058650310c62dd397800bd95b1deafccad417
+  metadata.gz: a31724df713fd03b8ddabd57ffd16c1b346023a003a5bf6fcaa6be4578d19f4480fb3eee94e9863c1cd5179bca9ef17ee67a7503c18f9a3a7af64930ee65e6ac
+  data.tar.gz: 76bf046bc1a442cc8c209613512d5aad192fd6eb428dadf5d48fcacc9a6e4a87600701a6e0cdfb58bf4df542a9650e264d446a1a6bf496d6b3220866f3960a2e

data/.travis.yml

@@ -14,7 +14,7 @@ notifications:
     on_failure: change
 gemfile:
   - Gemfile
-  - test/gemfiles/activerecord50.gemfile
+  - test/gemfiles/activerecord42.gemfile
   - test/gemfiles/activerecord41.gemfile
   - test/gemfiles/activerecord40.gemfile
   - test/gemfiles/activerecord32.gemfile

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 1.3.2
+
+- Added `body_options` option
+- Added `date_histogram` aggregation
+- Added `indices_boost` option
+- Added support for multiple conversions
+
 ## 1.3.1
 
 - Fixed error with Ruby 2.0

data/Gemfile

@@ -4,5 +4,5 @@ source "https://rubygems.org"
 gemspec
 
 gem "sqlite3"
-gem "activerecord", "~> 4.2.0"
+gem "activerecord", "~> 5.0.0"
 gem "gemoji-parser"

data/README.md

@@ -472,7 +472,7 @@ Next, add conversions to the index.
 class Product < ActiveRecord::Base
   has_many :searches, class_name: "Searchjoy::Search"
 
-  searchkick conversions: "conversions" # name of field
+  searchkick conversions: ["conversions"] # name of field
 
   def search_data
     {
@@ -643,6 +643,12 @@ Minimum document count
 Product.search "apples", aggs: {store_id: {min_doc_count: 2}}
 ```
 
+Date histogram
+
+```ruby
+Product.search "pear", aggs: {products_per_year: {date_histogram: {field: :created_at, interval: :year}}}
+```
+
 #### Moving From Facets
 
 1. Replace `facets` with `aggs` in searches. **Note:** Stats facets are not supported at this time.
@@ -1068,6 +1074,7 @@ class Product < ActiveRecord::Base
   }
 end
 ```
+**Note:** If you use a custom mapping, you'll need to use [custom searching](#advanced-search) as well.
 
 To keep the mappings and settings generated by Searchkick, use:
 
@@ -1095,10 +1102,16 @@ products.response
 
 To modify the query generated by Searchkick, use:
 
+```ruby
+products = Product.search "milk", body_options: {min_score: 1}
+```
+
+or
+
 ```ruby
 products =
   Product.search "apples" do |body|
-    body[:query] = {match_all: {}}
+    body[:min_score] = 1
   end
 ```
 
@@ -1116,6 +1129,20 @@ Then use `fresh_products` and `frozen_products` as typical results.
 
 **Note:** Errors are not raised as with single requests. Use the `error` method on each query to check for errors. Also, if you use the `below` option for misspellings, misspellings will be disabled.
 
+### Multiple Indices
+
+Search across multiple indices with:
+
+```ruby
+Searchkick.search "milk", index_name: [Product, Category]
+```
+
+Boost specific indices with:
+
+```ruby
+indices_boost: {Category => 2, Product => 1}
+```
+
 ## Reference
 
 Reindex one record
@@ -1184,6 +1211,35 @@ class Product < ActiveRecord::Base
 end
 ```
 
+Multiple conversion fields
+
+```ruby
+class Product < ActiveRecord::Base
+  has_many :searches, class_name: "Searchjoy::Search"
+
+  # searchkick also supports multiple "conversions" fields
+  searchkick conversions: ["unique_user_conversions", "total_conversions"]
+
+  def search_data
+    {
+      name: name,
+      unique_user_conversions: searches.group(:query).uniq.count(:user_id)
+      # {"ice cream" => 234, "chocolate" => 67, "cream" => 2}
+      total_conversions: searches.group(:query).count
+      # {"ice cream" => 412, "chocolate" => 117, "cream" => 6}
+    }
+  end
+end
+```
+
+and during query time:
+
+```ruby
+Product.search("banana") # boost by both fields (default)
+Product.search("banana", conversions: "total_conversions") # only boost by total_conversions
+Product.search("banana", conversions: false) # no conversion boosting
+```
+
 Change timeout
 
 ```ruby
@@ -1274,12 +1330,6 @@ class Product < ActiveRecord::Base
 end
 ```
 
-Search multiple models
-
-```ruby
-Searchkick.search "milk", index_name: [Product, Category]
-```
-
 Reindex all models - Rails only
 
 ```sh

data/lib/searchkick.rb

@@ -107,8 +107,10 @@ module Searchkick
     if items.any?
       response = client.bulk(body: items)
       if response["errors"]
-        first_item = response["items"].first
-        raise Searchkick::ImportError, (first_item["index"] || first_item["delete"])["error"]
+        first_with_error = response["items"].map do |item|
+          (item["index"] || item["delete"])
+        end.find { |item| item["error"] }
+        raise Searchkick::ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
       end
     end
   end

data/lib/searchkick/index.rb

@@ -430,7 +430,7 @@ module Searchkick
         mapping = {}
 
         # conversions
-        if (conversions_field = options[:conversions])
+        Array(options[:conversions]).each do |conversions_field|
           mapping[conversions_field] = {
             type: "nested",
             properties: {
@@ -570,7 +570,7 @@ module Searchkick
       if klass.respond_to?(:document_type)
         klass.document_type
       else
-        klass.model_name.to_s.underscore
+        klass.model_name.singular
       end
     end
 
@@ -602,9 +602,10 @@ module Searchkick
       source = source.inject({}) { |memo, (k, v)| memo[k.to_s] = v; memo }.except("_id")
 
       # conversions
-      conversions_field = options[:conversions]
-      if conversions_field && source[conversions_field]
-        source[conversions_field] = source[conversions_field].map { |k, v| {query: k, count: v} }
+      Array(options[:conversions]).each do |conversions_field|
+        if source[conversions_field]
+          source[conversions_field] = source[conversions_field].map { |k, v| {query: k, count: v} }
+        end
       end
 
       # hack to prevent generator field doesn't exist error

data/lib/searchkick/query.rb

@@ -162,8 +162,8 @@ module Searchkick
       # model and eagar loading
       load = options[:load].nil? ? true : options[:load]
 
-      conversions_field = searchkick_options[:conversions]
-      personalize_field = searchkick_options[:personalize]
+      conversions_fields = Array(options[:conversions] || searchkick_options[:conversions])
+      personalize_field  = searchkick_options[:personalize]
 
       all = term == "*"
 
@@ -275,34 +275,38 @@ module Searchkick
             }
           end
 
-          if conversions_field && options[:conversions] != false
-            # wrap payload in a bool query
-            script_score =
-              if below12?
-                {script_score: {script: "doc['count'].value"}}
-              else
-                {field_value_factor: {field: "#{conversions_field}.count"}}
-              end
+          if conversions_fields.present? && options[:conversions] != false
+            shoulds = []
+            conversions_fields.each do |conversions_field|
+              # wrap payload in a bool query
+              script_score =
+                if below12?
+                  {script_score: {script: "doc['count'].value"}}
+                else
+                  {field_value_factor: {field: "#{conversions_field}.count"}}
+                end
 
+              shoulds <<  {
+                            nested: {
+                              path: conversions_field,
+                              score_mode: "sum",
+                              query: {
+                                function_score: {
+                                  boost_mode: "replace",
+                                  query: {
+                                    match: {
+                                      "#{conversions_field}.query" => term
+                                    }
+                                  }
+                                }.merge(script_score)
+                              }
+                            }
+                          }
+            end
             payload = {
               bool: {
                 must: payload,
-                should: {
-                  nested: {
-                    path: conversions_field,
-                    score_mode: "sum",
-                    query: {
-                      function_score: {
-                        boost_mode: "replace",
-                        query: {
-                          match: {
-                            "#{conversions_field}.query" => term
-                          }
-                        }
-                      }.merge(script_score)
-                    }
-                  }
-                }
+                should: shoulds
               }
             }
           end
@@ -345,6 +349,9 @@ module Searchkick
         # order
         set_order(payload) if options[:order]
 
+        # indices_boost
+        set_boost_by_indices(payload)
+
         # filters
         filters = where_filters(options[:where])
         set_filters(payload, filters) if filters.any?
@@ -385,6 +392,9 @@ module Searchkick
         @routing = options[:routing] if options[:routing]
       end
 
+      # merge more body options
+      payload = payload.deep_merge(options[:body_options]) if options[:body_options]
+
       @body = payload
       @facet_limits = @facet_limits || {}
       @page = page
@@ -469,6 +479,19 @@ module Searchkick
       end
     end
 
+    def set_boost_by_indices(payload)
+      return unless options[:indices_boost]
+
+      indices_boost = options[:indices_boost].each_with_object({}) do |(key, boost), memo|
+        index = key.respond_to?(:searchkick_index) ? key.searchkick_index.name : key
+        # try to use index explicitly instead of alias: https://github.com/elasticsearch/elasticsearch/issues/4756
+        index_by_alias = Searchkick.client.indices.get_alias(index: index).keys.first
+        memo[index_by_alias || index] = boost
+      end
+
+      payload[:indices_boost] = indices_boost
+    end
+
     def set_suggestions(payload)
       suggest_fields = (searchkick_options[:suggest] || []).map(&:to_s)
 
@@ -544,6 +567,14 @@ module Searchkick
               ranges: agg_options[:date_ranges]
             }.merge(shared_agg_options)
           }
+        elsif histogram = agg_options[:date_histogram]
+          interval = histogram[:interval]
+          payload[:aggs][field] = {
+            date_histogram: {
+              field: histogram[:field],
+              interval: interval
+            }
+          }
         else
           payload[:aggs][field] = {
             terms: {

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "1.3.1"
+  VERSION = "1.3.2"
 end

data/test/aggs_test.rb

@@ -96,6 +96,29 @@ class AggsTest < Minitest::Test
     assert_equal ({2 => 2}), store_agg(where: {color: "blue"}, aggs: {store_id: {where: {in_stock: false}}}, smart_aggs: false)
   end
 
+  def test_aggs_group_by_date
+    store [{name: "Old Product", created_at: 3.years.ago}]
+    aggs = Product.search(
+      "Product",
+      {
+        aggs: {
+          products_per_year: {
+            date_histogram: {
+              field: :created_at,
+              interval: :year
+            }
+          }
+        }
+      }
+    ).aggs
+
+    if elasticsearch_below20?
+      assert_equal 2, aggs["products_per_year"]["buckets"].size
+    else
+      assert_equal 4, aggs["products_per_year"]["buckets"].size
+    end
+  end
+
   protected
 
   def buckets_as_hash(agg)

data/test/boost_test.rb

@@ -10,6 +10,24 @@ class BoostTest < Minitest::Test
       {name: "Tomato C", conversions: {"tomato" => 3}}
     ]
     assert_order "tomato", ["Tomato C", "Tomato B", "Tomato A"]
+    assert_equal_scores "tomato", {conversions: false}
+  end
+
+  def test_multiple_conversions
+    skip if elasticsearch_below14?
+
+    store [
+      {name: "Speaker A", conversions_a: {"speaker" => 1}, conversions_b: {"speaker" => 6}},
+      {name: "Speaker B", conversions_a: {"speaker" => 2}, conversions_b: {"speaker" => 5}},
+      {name: "Speaker C", conversions_a: {"speaker" => 3}, conversions_b: {"speaker" => 4}},
+    ], Speaker
+
+    assert_equal_scores "speaker", {conversions: false}, Speaker
+    assert_equal_scores "speaker", {}, Speaker
+    assert_equal_scores "speaker", {conversions: ["conversions_a", "conversions_b"]}, Speaker
+    assert_equal_scores "speaker", {conversions: ["conversions_b", "conversions_a"]}, Speaker
+    assert_order "speaker", ["Speaker C", "Speaker B", "Speaker A"], {conversions: "conversions_a"}, Speaker
+    assert_order "speaker", ["Speaker A", "Speaker B", "Speaker C"], {conversions: "conversions_b"}, Speaker
   end
 
   def test_conversions_stemmed
@@ -141,4 +159,11 @@ class BoostTest < Minitest::Test
     ]
     assert_order "san", ["San Francisco", "San Antonio", "San Marino"], boost_by_distance: {field: :location, origin: {lat: 37, lon: -122}, scale: "1000mi"}
   end
+
+  def test_boost_by_indices
+    store_names ["Rex"], Animal
+    store_names ["Rexx"], Product
+
+    assert_order "Rex", ["Rexx", "Rex"], {index_name: [Animal, Product], indices_boost: {Animal => 1, Product => 200}}, Store
+  end
 end

data/test/errors_test.rb

@@ -0,0 +1,19 @@
+require_relative "test_helper"
+
+class ErrorsTest < Minitest::Test
+  def test_bulk_import_raises_error
+    valid_dog = Dog.new(name: "2016-01-02")
+    invalid_dog = Dog.new(name: "Ol' One-Leg")
+    index = Searchkick::Index.new "dogs", mappings: {
+      dog: {
+        properties: {
+          name: {type: "date"}
+        }
+      }
+    }
+    index.store valid_dog
+    error = assert_raises(Searchkick::ImportError) do
+      index.bulk_index [valid_dog, invalid_dog]
+    end
+  end
+end

data/test/gemfiles/{activerecord50.gemfile → activerecord42.gemfile}

@@ -4,4 +4,4 @@ source 'https://rubygems.org'
 gemspec path: "../../"
 
 gem "sqlite3"
-gem "activerecord", "~> 5.0.0.rc1"
+gem "activerecord", "~> 4.2.0"

data/test/gemfiles/nobrainer.gemfile

@@ -3,4 +3,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in searchkick.gemspec
 gemspec path: "../../"
 
-gem "nobrainer", "0.27.0"
+gem "nobrainer"

data/test/query_test.rb

@@ -10,4 +10,14 @@ class QueryTest < Minitest::Test
     assert_equal ["Apple", "Milk"], query.map(&:name).sort
     assert_equal ["Apple", "Milk"], query.execute.map(&:name).sort
   end
+
+  def test_with_effective_min_score
+    store_names ["Milk", "Milk2"]
+    assert_equal ["Milk"], Product.search("Milk", body_options: { min_score: 0.1 }).map(&:name)
+  end
+
+  def test_with_uneffective_min_score
+    store_names ["Milk", "Milk2"]
+    assert_equal ["Milk", "Milk2"], Product.search("Milk", body_options: { min_score: 0.0001 }).map(&:name)
+  end
 end

data/test/sql_test.rb

@@ -52,6 +52,13 @@ class SqlTest < Minitest::Test
     assert_search "product", ["Product"], where: {latitude: {gt: 79}}
   end
 
+  # body_options
+
+  def test_body_options_should_merge_into_body
+    query = Product.search({ query: { name: "milk"}, body_options: { min_score: 1.0 }}, execute: false)
+    assert_equal 1.0, query.body[:min_score]
+  end
+
   # load
 
   def test_load_default

data/test/test_helper.rb

@@ -93,6 +93,12 @@ if defined?(Mongoid)
     field :name
   end
 
+  class Speaker
+    include Mongoid::Document
+
+    field :name
+  end
+
   class Animal
     include Mongoid::Document
 
@@ -137,6 +143,13 @@ elsif defined?(NoBrainer)
     field :name, type: String
   end
 
+  class Speaker
+    include NoBrainer::Document
+
+    field :id,   type: Object
+    field :name, type: String
+  end
+
   class Animal
     include NoBrainer::Document
 
@@ -162,7 +175,7 @@ else
   # migrations
   ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
 
-  ActiveRecord::Base.raise_in_transactional_callbacks = true if ActiveRecord::Base.respond_to?(:raise_in_transactional_callbacks=)
+  ActiveRecord::Base.raise_in_transactional_callbacks = true if ActiveRecord::VERSION::STRING.start_with?("4.2.")
 
   if defined?(Apartment)
     class Rails
@@ -221,6 +234,10 @@ else
     t.string :name
   end
 
+  ActiveRecord::Migration.create_table :speakers do |t|
+    t.string :name
+  end
+
   ActiveRecord::Migration.create_table :animals do |t|
     t.string :name
     t.string :type
@@ -233,6 +250,9 @@ else
     has_many :products
   end
 
+  class Speaker < ActiveRecord::Base
+  end
+
   class Animal < ActiveRecord::Base
   end
 
@@ -310,6 +330,20 @@ class Store
   end
 end
 
+class Speaker
+  searchkick \
+    conversions: ["conversions_a", "conversions_b"]
+
+  attr_accessor :conversions_a, :conversions_b
+
+  def search_data
+    serializable_hash.except("id").merge(
+      conversions_a: conversions_a,
+      conversions_b: conversions_b,
+    )
+  end
+end
+
 class Animal
   searchkick \
     autocomplete: [:name],
@@ -325,12 +359,14 @@ Product.create!(name: "Set mapping")
 
 Store.reindex
 Animal.reindex
+Speaker.reindex
 
 class Minitest::Test
   def setup
     Product.destroy_all
     Store.destroy_all
     Animal.destroy_all
+    Speaker.destroy_all
   end
 
   protected
@@ -355,6 +391,10 @@ class Minitest::Test
     assert_equal expected, klass.search(term, options).map(&:name)
   end
 
+  def assert_equal_scores(term, options = {}, klass = Product)
+    assert_equal 1, klass.search(term, options).hits.map { |a| a['_score'] }.uniq.size
+  end
+
   def assert_first(term, expected, options = {}, klass = Product)
     assert_equal expected, klass.search(term, options).map(&:name).first
   end

metadata

@@ -1,97 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.3.2
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-10 00:00:00.000000000 Z
+date: 2016-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
   name: hashie
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Intelligent search made easy
@@ -101,8 +101,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -126,12 +126,13 @@ files:
 - test/callbacks_test.rb
 - test/ci/before_install.sh
 - test/dangerous_reindex_test.rb
+- test/errors_test.rb
 - test/facets_test.rb
 - test/gemfiles/activerecord31.gemfile
 - test/gemfiles/activerecord32.gemfile
 - test/gemfiles/activerecord40.gemfile
 - test/gemfiles/activerecord41.gemfile
-- test/gemfiles/activerecord50.gemfile
+- test/gemfiles/activerecord42.gemfile
 - test/gemfiles/apartment.gemfile
 - test/gemfiles/mongoid2.gemfile
 - test/gemfiles/mongoid3.gemfile
@@ -170,17 +171,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.6.1
 signing_key: 
 specification_version: 4
 summary: Searchkick learns what your users are looking for. As more people search,
@@ -193,12 +194,13 @@ test_files:
 - test/callbacks_test.rb
 - test/ci/before_install.sh
 - test/dangerous_reindex_test.rb
+- test/errors_test.rb
 - test/facets_test.rb
 - test/gemfiles/activerecord31.gemfile
 - test/gemfiles/activerecord32.gemfile
 - test/gemfiles/activerecord40.gemfile
 - test/gemfiles/activerecord41.gemfile
-- test/gemfiles/activerecord50.gemfile
+- test/gemfiles/activerecord42.gemfile
 - test/gemfiles/apartment.gemfile
 - test/gemfiles/mongoid2.gemfile
 - test/gemfiles/mongoid3.gemfile