searchkick 1.4.0 → 1.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f7ebf358ab6ac1ae3d08be867a20c9617a4ef0e9
-  data.tar.gz: 4935de64421b02b6a27505888f588c9e161f13f0
+  metadata.gz: 9994b20546ec9b93a6fe9b34a0e329868aa3371b
+  data.tar.gz: 77ab1a591fe5e599b3d640d271aafe3f0642b47d
 SHA512:
-  metadata.gz: dfd3083681354eb0b1aab35d6b1f2d383326d8b6a2106af167b3f9cc9420ac974fec80c4dec30881e7e6903fb785a6492c643a14e36429bdb93fac21f0dc2193
-  data.tar.gz: 136e074de01969cb08fc54c35ee3468477b9d4612583510d327559fceee0218950e459b1228c32b3b5a54c60e81f3653ad7e22c0ab626a5193a9a355253bcc88
+  metadata.gz: 8cf24561e0d7d871311b6ec5d50f115d654ce64c77422c1e51a07d13831b3ae81327ff1505d59d1b707d6b4926fad61e094178e703ee7a454cffafaf80b77464
+  data.tar.gz: 95a15798d483c444398a88c6940e88275eceb850231e869a272ab0bbc659089ca651780eba84ab5ac95f9a24aa1a8aed3e05da88474e6b093aedd265e69c7c7a

data/.travis.yml

@@ -1,7 +1,7 @@
+sudo: false
 language: ruby
 rvm: 2.2.5
 services:
-  - elasticsearch
   - mongodb
 before_install:
   - ./test/ci/before_install.sh
@@ -24,17 +24,22 @@ gemfile:
   - test/gemfiles/mongoid4.gemfile
   - test/gemfiles/mongoid5.gemfile
 env:
-  - ELASTICSEARCH_VERSION=2.4.1
+  - ELASTICSEARCH_VERSION=5.1.1
+jdk: oraclejdk8
 matrix:
   include:
     - gemfile: Gemfile
       env: ELASTICSEARCH_VERSION=1.0.0
+      jdk: oraclejdk7
     - gemfile: Gemfile
       env: ELASTICSEARCH_VERSION=1.7.0
+      jdk: oraclejdk7
     - gemfile: Gemfile
       env: ELASTICSEARCH_VERSION=2.0.0
+      jdk: oraclejdk7
     - gemfile: Gemfile
-      env: ELASTICSEARCH_VERSION=5.0.0
-  allow_failures:
+      env: ELASTICSEARCH_VERSION=2.4.1
+      jdk: oraclejdk7
     - gemfile: Gemfile
-      env: ELASTICSEARCH_VERSION=5.0.0
+      env: ELASTICSEARCH_VERSION=5.0.1
+      jdk: oraclejdk8

data/CHANGELOG.md

@@ -1,5 +1,12 @@
+## 1.4.1
+
+- Added `partial_reindex` method
+- Added `debug` option to `search` method
+- Added `profile` option
+
 ## 1.4.0
 
+- Official support for Elasticsearch 5
 - Boost exact matches for partial matching
 - Added `searchkick_debug` method
 - Added `geo_polygon` filter

data/README.md

@@ -44,8 +44,6 @@ Add this line to your application’s Gemfile:
 gem 'searchkick'
 ```
 
-For Elasticsearch 5.0, use the version `1.4` and above. For Elasticsearch 2.0, use the version `1.0` and above.
-
 Add searchkick to models you want to search.
 
 ```ruby
@@ -543,7 +541,7 @@ First, specify which fields use this feature. This is necessary since autocomple
 
 ```ruby
 class Book < ActiveRecord::Base
-  searchkick match: :word_start, searchable: [:title, :author]
+  searchkick word_start: [:title, :author]
 end
 ```
 
@@ -565,6 +563,7 @@ class BooksController < ApplicationController
   def autocomplete
     render json: Book.search(params[:query], {
       fields: ["title^5", "author"],
+      match: :word_start,
       limit: 10,
       load: false,
       misspellings: {below: 5}
@@ -579,11 +578,18 @@ Then add the search box and JavaScript code to a view.
 <input type="text" id="query" name="query" />
 
 <script src="jquery.js"></script>
-<script src="typeahead.js"></script>
+<script src="typeahead.bundle.js"></script>
 <script>
-  $("#query").typeahead({
-    name: "book",
-    remote: "/books/autocomplete?query=%QUERY"
+  var books = new Bloodhound({
+    datumTokenizer: Bloodhound.tokenizers.whitespace,
+    queryTokenizer: Bloodhound.tokenizers.whitespace,
+    remote: {
+      url: '/books/autocomplete?query=%QUERY',
+      wildcard: '%QUERY'
+    }
+  });
+  $('#query').typeahead(null, {
+    source: books
   });
 </script>
 ```
@@ -852,7 +858,7 @@ Bounded by a box
 City.search "san", where: {location: {top_left: {lat: 38, lon: -123}, bottom_right: {lat: 37, lon: -122}}}
 ```
 
-Bounded by a polygon [master]
+Bounded by a polygon
 
 ```ruby
 City.search "san", where: {location: {geo_polygon: {points: [{lat: 38, lon: -123}, {lat: 39, lon: -123}, {lat: 37, lon: 122}]}}}
@@ -1055,7 +1061,7 @@ require "typhoeus/adapters/faraday"
 Ethon.logger.level = Logger::WARN
 ```
 
-**Note:** Typhoeus is not available for Windows.
+If you run into issues on Windows, check out [this post](https://www.rastating.com/fixing-issues-in-typhoeus-and-httparty-on-windows/).
 
 ### Automatic Failover
 
@@ -1207,6 +1213,21 @@ Product.where("id > 100000").find_in_batches do |batch|
 end
 ```
 
+Reindex a subset of attributes
+
+```ruby
+class Product < ActiveRecord::Base
+  def search_prices
+    {
+      price: price,
+      sale_price: sale_price
+    }
+  end
+end
+
+Product.partial_reindex(:search_prices)
+```
+
 Remove old indices
 
 ```ruby
@@ -1517,6 +1538,8 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 - Write, clarify, or fix documentation
 - Suggest or add new features
 
+If you’re looking for ideas, [try here](https://github.com/ankane/searchkick/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22).
+
 To get started with development and testing:
 
 ```sh

data/lib/searchkick/index.rb

@@ -67,6 +67,10 @@ module Searchkick
     end
     alias_method :import, :bulk_index
 
+    def bulk_update(records, method_name)
+      Searchkick.queue_items(records.map { |r| {update: record_data(r).merge(data: {doc: search_data(r, method_name)})} })
+    end
+
     def record_data(r)
       data = {
         _index: name,
@@ -221,6 +225,7 @@ module Searchkick
 
     def import_scope(scope, options = {})
       batch_size = @options[:batch_size] || 1000
+      method_name = options[:method_name]
 
       # use scope for import
       scope = scope.search_import if scope.respond_to?(:search_import)
@@ -234,7 +239,7 @@ module Searchkick
         end
 
         scope.find_in_batches batch_size: batch_size do |batch|
-          import batch.select(&:should_index?)
+          import_or_update batch.select(&:should_index?), method_name
         end
       else
         # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
@@ -244,14 +249,18 @@ module Searchkick
         scope.all.each do |item|
           items << item if item.should_index?
           if items.length == batch_size
-            import items
+            import_or_update items, method_name
             items = []
           end
         end
-        import items
+        import_or_update items, method_name
       end
     end
 
+    def import_or_update(records, method_name)
+      method_name ? bulk_update(records, method_name) : import(records)
+    end
+
     # other
 
     def tokens(text, options = {})
@@ -285,13 +294,14 @@ module Searchkick
       id.is_a?(Numeric) ? id : id.to_s
     end
 
-    def search_data(record)
-      source = record.search_data
+    def search_data(record, method_name = nil)
+      partial_reindex = !method_name.nil?
+      source = record.send(method_name || :search_data)
       options = record.class.searchkick_options
 
       # stringify fields
       # remove _id since search_id is used instead
-      source = source.each_with_object({}) { |(k, v), memo| memo[k.to_s] = v; memo }.except("_id")
+      source = source.each_with_object({}) { |(k, v), memo| memo[k.to_s] = v; memo }.except("_id", "_type")
 
       # conversions
       Array(options[:conversions]).map(&:to_s).each do |conversions_field|
@@ -302,7 +312,7 @@ module Searchkick
 
       # hack to prevent generator field doesn't exist error
       (options[:suggest] || []).map(&:to_s).each do |field|
-        source[field] = nil unless source[field]
+        source[field] = nil if !source[field] && !partial_reindex
       end
 
       # locations

data/lib/searchkick/index_options.rb

@@ -25,7 +25,7 @@ module Searchkick
             }
           end
 
-        keyword_mapping[:ignore_above] = 256 unless below22
+        keyword_mapping[:ignore_above] = (options[:ignore_above] || 256) unless below22
 
         settings = {
           analysis: {
@@ -308,7 +308,7 @@ module Searchkick
           dynamic_fields["{name}"] = {type: default_type, index: "no"}
         end
 
-        dynamic_fields["{name}"][:ignore_above] = 256 unless below22
+        dynamic_fields["{name}"][:ignore_above] = (options[:ignore_above] || 256) unless below22
 
         unless options[:searchable]
           if options[:match] && options[:match] != :word

data/lib/searchkick/model.rb

@@ -54,6 +54,13 @@ module Searchkick
           end
           alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex)
 
+          def searchkick_partial_reindex(method_name)
+            searchkick_index.import_scope(searchkick_klass, method_name: method_name)
+            searchkick_index.refresh
+            true
+          end
+          alias_method :partial_reindex, :searchkick_partial_reindex unless method_defined?(:partial_reindex)
+
           def clean_indices
             searchkick_index.clean_indices
           end
@@ -71,30 +78,7 @@ module Searchkick
           end
 
           def searchkick_debug
-            require "pp"
-
-            puts "Model Searchkick Options"
-            pp searchkick_options
-            puts
-
-            puts "Model Sample Search Data"
-            begin
-              pp first(3).map { |r| {index: searchkick_index.record_data(r).merge(data: searchkick_index.send(:search_data, r))}}
-            rescue => e
-              puts "#{e.class.name}: #{e.message}"
-            end
-            puts
-
-            puts "Elasticsearch Mapping"
-            puts JSON.pretty_generate(searchkick_index.mapping)
-            puts
-
-            puts "Elasticsearch Settings"
-            puts JSON.pretty_generate(searchkick_index.settings)
-            puts
-
-            puts "Elasticsearch Sample Results"
-            puts JSON.pretty_generate(search("*", load: false, limit: 3).response)
+            warn "Use debug option with search method instead"
 
             nil # do not return anything, as this is strictly used for manual debugging
           end
@@ -117,6 +101,12 @@ module Searchkick
           self.class.searchkick_index.reindex_record_async(self)
         end unless method_defined?(:reindex_async)
 
+        def partial_reindex(method_name)
+          self.class.searchkick_index.bulk_update([self], method_name)
+          self.class.searchkick_index.refresh
+          true
+        end unless method_defined?(:partial_reindex)
+
         def similar(options = {})
           self.class.searchkick_index.similar_record(self, options)
         end unless method_defined?(:similar)

data/lib/searchkick/query.rb

@@ -112,11 +112,50 @@ module Searchkick
         padding: @padding,
         load: @load,
         includes: options[:include] || options[:includes],
-        json: !options[:json].nil?,
+        json: !@json.nil?,
         match_suffix: @match_suffix,
         highlighted_fields: @highlighted_fields || []
       }
 
+      if options[:debug]
+        require "pp"
+
+        puts "Searchkick Version: #{Searchkick::VERSION}"
+        puts "Elasticsearch Version: #{Searchkick.server_version}"
+        puts
+
+        puts "Model Searchkick Options"
+        pp searchkick_options
+        puts
+
+        puts "Search Options"
+        pp options
+        puts
+
+        puts "Model Search Data"
+        begin
+          pp klass.first(3).map { |r| {index: searchkick_index.record_data(r).merge(data: searchkick_index.send(:search_data, r))}}
+        rescue => e
+          puts "#{e.class.name}: #{e.message}"
+        end
+        puts
+
+        puts "Elasticsearch Mapping"
+        puts JSON.pretty_generate(searchkick_index.mapping)
+        puts
+
+        puts "Elasticsearch Settings"
+        puts JSON.pretty_generate(searchkick_index.settings)
+        puts
+
+        puts "Elasticsearch Query"
+        puts to_curl
+        puts
+
+        puts "Elasticsearch Results"
+        puts JSON.pretty_generate(response)
+      end
+
       # set execute for multi search
       @execute = Searchkick::Results.new(searchkick_klass, response, opts)
     end
@@ -173,9 +212,9 @@ module Searchkick
 
       all = term == "*"
 
-      options[:json] ||= options[:body]
-      if options[:json]
-        payload = options[:json]
+      @json = options[:json] || options[:body]
+      if @json
+        payload = @json
       else
         if options[:query]
           payload = options[:query]
@@ -365,6 +404,7 @@ module Searchkick
           from: offset
         }
         payload[:explain] = options[:explain] if options[:explain]
+        payload[:profile] = options[:profile] if options[:profile]
 
         # order
         set_order(payload) if options[:order]

data/lib/searchkick/results.rb

@@ -181,7 +181,11 @@ module Searchkick
       if options[:includes]
         records =
           if defined?(NoBrainer::Document) && records < NoBrainer::Document
-            records.preload(options[:includes])
+            if Gem.loaded_specs["nobrainer"].version >= Gem::Version.new("0.21")
+              records.eager_load(options[:includes])
+            else
+              records.preload(options[:includes])
+            end
           else
             records.includes(options[:includes])
           end
@@ -196,7 +200,7 @@ module Searchkick
       elsif records.respond_to?(:queryable)
         # Mongoid 3+
         records.queryable.for_ids(ids)
-      elsif records.respond_to?(:unscoped) && records.all.respond_to?(:preload)
+      elsif records.respond_to?(:unscoped) && [:preload, :eager_load].any? { |m| records.all.respond_to?(m) }
         # Nobrainer
         records.unscoped.where(:id.in => ids)
       else

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "1.4.0"
+  VERSION = "1.4.1"
 end

data/test/ci/before_install.sh

@@ -4,15 +4,14 @@ set -e
 
 gem install bundler
 
-# https://docs.travis-ci.com/user/database-setup/#ElasticSearch
-sudo apt-get purge elasticsearch
 if [[ $ELASTICSEARCH_VERSION == 1* ]]; then
-  curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-$ELASTICSEARCH_VERSION.deb
+  curl -L -O https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-$ELASTICSEARCH_VERSION.tar.gz
 elif [[ $ELASTICSEARCH_VERSION == 2* ]]; then
-  curl -O https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/$ELASTICSEARCH_VERSION/elasticsearch-$ELASTICSEARCH_VERSION.deb
+  curl -L -O https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/$ELASTICSEARCH_VERSION/elasticsearch-$ELASTICSEARCH_VERSION.tar.gz
 else
-  curl -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ELASTICSEARCH_VERSION.deb
+  curl -L -O https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-$ELASTICSEARCH_VERSION.tar.gz
 fi
-sudo dpkg -i --force-confnew elasticsearch-$ELASTICSEARCH_VERSION.deb
-sudo service elasticsearch restart
-sleep 10
+tar -xvf elasticsearch-$ELASTICSEARCH_VERSION.tar.gz
+cd elasticsearch-$ELASTICSEARCH_VERSION/bin
+./elasticsearch -d
+wget -O- --waitretry=1 --tries=30 --retry-connrefused -v http://127.0.0.1:9200/

data/test/gemfiles/nobrainer.gemfile

@@ -3,4 +3,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in searchkick.gemspec
 gemspec path: "../../"
 
-gem "nobrainer"
+gem "nobrainer", ">= 0.21.0"

data/test/match_test.rb

@@ -34,7 +34,9 @@ class MatchTest < Minitest::Test
   end
 
   def test_percent
-    store_names ["1% Milk", "2% Milk", "Whole Milk"]
+    # Note: "2% Milk" doesn't get matched in ES below 5.1.1
+    # This could be a bug since it has an edit distance of 1
+    store_names ["1% Milk", "Whole Milk"]
     assert_search "1%", ["1% Milk"]
   end
 
@@ -184,6 +186,11 @@ class MatchTest < Minitest::Test
     assert_search "ben and jerrys", ["Ben and Jerry's"]
   end
 
+  def test_apostrophe_search
+    store_names ["Ben and Jerrys"]
+    assert_search "ben and jerry's", ["Ben and Jerrys"]
+  end
+
   def test_ampersand_index
     store_names ["Ben & Jerry's"]
     assert_search "ben and jerrys", ["Ben & Jerry's"]

data/test/partial_reindex_test.rb

@@ -0,0 +1,58 @@
+require_relative "test_helper"
+
+class PartialReindexTest < Minitest::Test
+  def test_class_method
+    store [{name: "Hi", color: "Blue"}]
+
+    # normal search
+    assert_search "hi", ["Hi"], fields: [:name], load: false
+    assert_search "blue", ["Hi"], fields: [:color], load: false
+
+    # update
+    product = Product.first
+    product.name = "Bye"
+    product.color = "Red"
+    Searchkick.callbacks(false) do
+      product.save!
+    end
+    Product.searchkick_index.refresh
+
+    # index not updated
+    assert_search "hi", ["Hi"], fields: [:name], load: false
+    assert_search "blue", ["Hi"], fields: [:color], load: false
+
+    # partial reindex
+    Product.partial_reindex(:search_name)
+
+    # name updated, but not color
+    assert_search "bye", ["Bye"], fields: [:name], load: false
+    assert_search "blue", ["Bye"], fields: [:color], load: false
+  end
+
+  def test_instance_method
+    store [{name: "Hi", color: "Blue"}]
+
+    # normal search
+    assert_search "hi", ["Hi"], fields: [:name], load: false
+    assert_search "blue", ["Hi"], fields: [:color], load: false
+
+    # update
+    product = Product.first
+    product.name = "Bye"
+    product.color = "Red"
+    Searchkick.callbacks(false) do
+      product.save!
+    end
+    Product.searchkick_index.refresh
+
+    # index not updated
+    assert_search "hi", ["Hi"], fields: [:name], load: false
+    assert_search "blue", ["Hi"], fields: [:color], load: false
+
+    product.partial_reindex(:search_name)
+
+    # name updated, but not color
+    assert_search "bye", ["Bye"], fields: [:name], load: false
+    assert_search "blue", ["Bye"], fields: [:color], load: false
+  end
+end

data/test/sql_test.rb

@@ -55,7 +55,7 @@ class SqlTest < Minitest::Test
   # body_options
 
   def test_body_options_should_merge_into_body
-    query = Product.search({query: {name: "milk"}, body_options: {min_score: 1.0}}, execute: false)
+    query = Product.search("*", body_options: {min_score: 1.0}, execute: false)
     assert_equal 1.0, query.body[:min_score]
   end
 
@@ -81,6 +81,12 @@ class SqlTest < Minitest::Test
     assert_kind_of Hash, Product.search("product", load: false, include: [:store]).first
   end
 
+  def test_load_false_nested_object
+    aisle = {"id" => 1, "name" => "Frozen"}
+    store [{name: "Product A", aisle: aisle}]
+    assert_equal aisle, Product.search("product", load: false).first.aisle.to_hash
+  end
+
   # select
 
   def test_select
@@ -218,14 +224,16 @@ class SqlTest < Minitest::Test
     assert_nil result.user_ids
   end
 
-  # other tests
+  # nested
 
-  def test_nested_object
-    aisle = {"id" => 1, "name" => "Frozen"}
-    store [{name: "Product A", aisle: aisle}]
-    assert_equal aisle, Product.search("product", load: false).first.aisle.to_hash
+  def test_nested_search
+    skip
+    store [{name: "Product A", aisle: {"id" => 1, "name" => "Frozen"}}]
+    assert_search "frozen", ["Product A"], fields: ["aisle.name"], debug: true
   end
 
+  # other tests
+
   def test_include
     skip unless defined?(ActiveRecord)
     store_names ["Product A"]

data/test/test_helper.rb

@@ -309,6 +309,12 @@ class Product
   def should_index?
     name != "DO NOT INDEX"
   end
+
+  def search_name
+    {
+      name: name
+    }
+  end
 end
 
 class Store

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-27 00:00:00.000000000 Z
+date: 2016-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -150,6 +150,7 @@ files:
 - test/multi_tenancy_test.rb
 - test/order_test.rb
 - test/pagination_test.rb
+- test/partial_reindex_test.rb
 - test/query_test.rb
 - test/records_test.rb
 - test/reindex_job_test.rb
@@ -218,6 +219,7 @@ test_files:
 - test/multi_tenancy_test.rb
 - test/order_test.rb
 - test/pagination_test.rb
+- test/partial_reindex_test.rb
 - test/query_test.rb
 - test/records_test.rb
 - test/reindex_job_test.rb