searchkick 2.0.0 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06b1b68ac3cb57317a8fdd0c94e3988ffed37852
-  data.tar.gz: b5a51c3c2126b88749bd114c0355a9e593f2cdfa
+  metadata.gz: 17c7f39c1470d38bc637340ae0031997a0b0134c
+  data.tar.gz: 0167c2d1448536bf5b2b48120f56e992c22aed8e
 SHA512:
-  metadata.gz: b520ec977fec7ac5a65aa6a5aacd9fb4c6142a079385bd63dce20b28ccd59f7cf84993daa4e1f68f8befba383ceb2246ce2a78b38846ec43e7d4432bb9587b40
-  data.tar.gz: b0032add9b2f7ed647ecbc9f2db958f5f240a0725b3990ea24697290e8b61c5e2d6afdf8777742fafaa2fbb33407f0d62f6480239522099869a9bccaf4b9092e
+  metadata.gz: a6bf61245813c31fc922a29c5aa4998df13c9c1f51eba6c248ea1bfbba435f048de37a069d23cad6fc041618a235974895cb7466c08ee24cb14eb39296fc908d
+  data.tar.gz: b50909d603658acd1c9562470c2fd1de404cffd2f93a22c1ca926003afaaabcf33ea70a14ba69d96e3b94b50af241b91cbd267492c079416e2b9ae500c6cf32e

data/.travis.yml

@@ -26,7 +26,7 @@ matrix:
       env: ELASTICSEARCH_VERSION=2.0.0
       jdk: oraclejdk7
     - gemfile: Gemfile
-      env: ELASTICSEARCH_VERSION=2.4.1
+      env: ELASTICSEARCH_VERSION=2.4.3
       jdk: oraclejdk7
     - gemfile: Gemfile
       env: ELASTICSEARCH_VERSION=5.0.1

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.0.1
+
+- Added `search_hit` and `search_highlights` methods to models
+- Improved reindex performance
+
 ## 2.0.0
 
 - Added support for `reindex` on associations

data/README.md

@@ -254,7 +254,7 @@ Available options are:
 ### Exact Matches
 
 ```ruby
-User.search params[:q], fields: [{email: :exact}, :name]
+User.search query, fields: [{email: :exact}, :name]
 ```
 
 ### Phrase Matches
@@ -431,7 +431,7 @@ Product.reindex(resume: true)
 
 #### No need to reindex
 
-- App starts
+- app starts
 
 ### Stay Synced
 
@@ -784,8 +784,8 @@ bands = Band.search "cinema", fields: [:name], highlight: true
 View the highlighted fields with:
 
 ```ruby
-bands.with_details.each do |band, details|
-  puts details[:highlight][:name] # "Two Door <em>Cinema</em> Club"
+bands.each do |band|
+  band.search_highlights[:name] # "Two Door <em>Cinema</em> Club"
 end
 ```
 
@@ -1013,7 +1013,7 @@ heroku run rake searchkick:reindex CLASS=Product
 Include `elasticsearch 1.0.15` or greater in your Gemfile.
 
 ```ruby
-gem "elasticsearch", ">= 1.0.15"
+gem 'elasticsearch', '>= 1.0.15'
 ```
 
 Create an initializer `config/initializers/elasticsearch.rb` with:
@@ -1088,15 +1088,23 @@ See [Production Rails](https://github.com/ankane/production_rails) for other goo
 
 ## Performance
 
+### JSON Generation
+
+Significantly increase performance with faster JSON generation. Add [Oj](https://github.com/ohler55/oj) to your Gemfile.
+
+```ruby
+gem 'oj'
+```
+
 ### Persistent HTTP Connections
 
-For the best performance, add [Typhoeus](https://github.com/typhoeus/typhoeus) to your Gemfile.
+Significantly increase performance with persistent HTTP connections. Add [Typhoeus](https://github.com/typhoeus/typhoeus) to your Gemfile.
 
 ```ruby
 gem 'typhoeus'
 ```
 
-And create an initializer to reduce log noise with:
+To reduce log noise, create an initializer with:
 
 ```ruby
 Ethon.logger = Logger.new("/dev/null")
@@ -1126,7 +1134,7 @@ end
 
 ### Routing
 
-Searchkick supports [Elasticsearch’s routing feature](https://www.elastic.co/blog/customizing-your-document-routing).
+Searchkick supports [Elasticsearch’s routing feature](https://www.elastic.co/blog/customizing-your-document-routing), which can significantly speed up searches.
 
 ```ruby
 class Business < ActiveRecord::Base
@@ -1245,24 +1253,22 @@ User.search "san", fields: ["address.city"], where: {"address.zip_code" => 12345
 Reindex one record
 
 ```ruby
-product = Product.find 10
+product = Product.find(1)
 product.reindex
 # or to reindex in the background
 product.reindex_async
 ```
 
-Reindex more than one record without recreating the index
+Reindex multiple records
 
 ```ruby
-some_company.products.reindex
+Product.where(store_id: 1).reindex
 ```
 
-Reindex large set of records in batches
+Reindex associations
 
 ```ruby
-Product.where("id > 100000").find_in_batches do |batch|
-  Product.searchkick_index.import(batch)
-end
+store.products.reindex
 ```
 
 Reindex a subset of attributes (partial reindex)

data/Rakefile

@@ -6,3 +6,7 @@ Rake::TestTask.new do |t|
   t.libs << "test"
   t.pattern = "test/**/*_test.rb"
 end
+
+task :benchmark do
+  require_relative "benchmark/benchmark"
+end

data/benchmark/Gemfile

@@ -0,0 +1,19 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in searchkick.gemspec
+gemspec path: "../"
+
+gem "sqlite3"
+gem "activerecord", "~> 5.0.0"
+gem "activerecord-import"
+
+# performance
+gem "typhoeus"
+gem "oj"
+
+# profiling
+gem "ruby-prof"
+gem "allocation_stats"
+gem "get_process_mem"
+gem "memory_profiler"
+gem "allocation_tracer"

data/benchmark/benchmark.rb

@@ -0,0 +1,63 @@
+require "bundler/setup"
+Bundler.require(:default)
+require "active_record"
+require "benchmark"
+
+ActiveRecord::Base.default_timezone = :utc
+ActiveRecord::Base.time_zone_aware_attributes = true
+ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+
+ActiveRecord::Migration.create_table :products do |t|
+  t.string :name
+  t.string :color
+  t.integer :store_id
+end
+
+class Product < ActiveRecord::Base
+  searchkick batch_size: 100
+
+  def search_data
+    {
+      name: name,
+      color: color,
+      store_id: store_id
+    }
+  end
+end
+
+Product.import ["name", "color", "store_id"], 20000.times.map { |i| ["Product #{i}", ["red", "blue"].sample, rand(10)] }
+
+puts "Imported"
+
+result = nil
+report = nil
+stats = nil
+
+# p GetProcessMem.new.mb
+
+time =
+  Benchmark.realtime do
+    # result = RubyProf.profile do
+    # report = MemoryProfiler.report do
+    # stats = AllocationStats.trace do
+    Product.reindex
+    # end
+  end
+
+# p GetProcessMem.new.mb
+
+puts time.round(1)
+puts Product.searchkick_index.total_docs
+
+if result
+  printer = RubyProf::GraphPrinter.new(result)
+  printer.print(STDOUT, min_percent: 5)
+end
+
+if report
+  puts report.pretty_print
+end
+
+if stats
+  puts result.allocations(alias_paths: true).group_by(:sourcefile, :class).to_text
+end

data/lib/searchkick/index.rb

@@ -7,6 +7,7 @@ module Searchkick
     def initialize(name, options = {})
       @name = name
       @options = options
+      @klass_document_type = {} # cache
     end
 
     def create(body = {})
@@ -270,10 +271,12 @@ module Searchkick
     end
 
     def klass_document_type(klass)
-      if klass.respond_to?(:document_type)
-        klass.document_type
-      else
-        klass.model_name.to_s.underscore
+      @klass_document_type[klass] ||= begin
+        if klass.respond_to?(:document_type)
+          klass.document_type
+        else
+          klass.model_name.to_s.underscore
+        end
       end
     end
 
@@ -296,42 +299,48 @@ module Searchkick
       id.is_a?(Numeric) ? id : id.to_s
     end
 
+    EXCLUDED_ATTRIBUTES = ["_id", "_type"]
+
     def search_data(record, method_name = nil)
       partial_reindex = !method_name.nil?
-      source = record.send(method_name || :search_data)
       options = record.class.searchkick_options
 
-      # stringify fields
       # remove _id since search_id is used instead
-      source = source.each_with_object({}) { |(k, v), memo| memo[k.to_s] = v; memo }.except("_id", "_type")
+      source = record.send(method_name || :search_data).each_with_object({}) { |(k, v), memo| memo[k.to_s] = v; memo }.except(*EXCLUDED_ATTRIBUTES)
 
       # conversions
-      Array(options[:conversions]).map(&:to_s).each do |conversions_field|
-        if source[conversions_field]
-          source[conversions_field] = source[conversions_field].map { |k, v| {query: k, count: v} }
+      if options[:conversions]
+        Array(options[:conversions]).map(&:to_s).each do |conversions_field|
+          if source[conversions_field]
+            source[conversions_field] = source[conversions_field].map { |k, v| {query: k, count: v} }
+          end
         end
       end
 
       # hack to prevent generator field doesn't exist error
-      (options[:suggest] || []).map(&:to_s).each do |field|
-        source[field] = nil if !source[field] && !partial_reindex
+      if options[:suggest]
+        options[:suggest].map(&:to_s).each do |field|
+          source[field] = nil if !source[field] && !partial_reindex
+        end
       end
 
       # locations
-      (options[:locations] || []).map(&:to_s).each do |field|
-        if source[field]
-          if !source[field].is_a?(Hash) && (source[field].first.is_a?(Array) || source[field].first.is_a?(Hash))
-            # multiple locations
-            source[field] = source[field].map { |a| location_value(a) }
-          else
-            source[field] = location_value(source[field])
+      if options[:locations]
+        options[:locations].map(&:to_s).each do |field|
+          if source[field]
+            if !source[field].is_a?(Hash) && (source[field].first.is_a?(Array) || source[field].first.is_a?(Hash))
+              # multiple locations
+              source[field] = source[field].map { |a| location_value(a) }
+            else
+              source[field] = location_value(source[field])
+            end
           end
         end
       end
 
       cast_big_decimal(source)
 
-      source.as_json
+      source
     end
 
     def location_value(value)

data/lib/searchkick/index_options.rb

@@ -252,15 +252,7 @@ module Searchkick
             end
           end
 
-          mapping[field] =
-            if below50
-              {
-                type: "multi_field",
-                fields: fields
-              }
-            elsif fields[field]
-              fields[field].merge(fields: fields.except(field))
-            end
+          mapping[field] = fields[field].merge(fields: fields.except(field))
         end
 
         (options[:locations] || []).map(&:to_s).each do |field|
@@ -307,15 +299,7 @@ module Searchkick
         end
 
         # http://www.elasticsearch.org/guide/reference/mapping/multi-field-type/
-        multi_field =
-          if below50
-            {
-              type: "multi_field",
-              fields: dynamic_fields
-            }
-          else
-            dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}"))
-          end
+        multi_field = dynamic_fields["{name}"].merge(fields: dynamic_fields.except("{name}"))
 
         all_enabled = !options[:searchable] || options[:searchable].to_a.include?("_all")
 

data/lib/searchkick/results.rb

@@ -32,7 +32,22 @@ module Searchkick
 
           # sort
           hits.map do |hit|
-            results[hit["_type"]][hit["_id"].to_s]
+            result = results[hit["_type"]][hit["_id"].to_s]
+            if result
+              unless result.respond_to?(:search_hit)
+                result.define_singleton_method(:search_hit) do
+                  hit
+                end
+              end
+
+              if hit["highlight"] && !result.respond_to?(:search_highlights)
+                highlights = Hash[hit["highlight"].map { |k, v| [(options[:json] ? k : k.sub(/\.#{@options[:match_suffix]}\z/, "")).to_sym, v.first] }]
+                result.define_singleton_method(:search_highlights) do
+                  highlights
+                end
+              end
+            end
+            result
           end.compact
         else
           hits.map do |hit|
@@ -187,21 +202,21 @@ module Searchkick
           end
       end
 
-      if records.respond_to?(:primary_key) && records.primary_key
-        # ActiveRecord
-        records.where(records.primary_key => ids)
-      elsif records.respond_to?(:all) && records.all.respond_to?(:for_ids)
-        # Mongoid 2
-        records.all.for_ids(ids)
-      elsif records.respond_to?(:queryable)
-        # Mongoid 3+
-        records.queryable.for_ids(ids)
-      elsif records.respond_to?(:unscoped) && [:preload, :eager_load].any? { |m| records.all.respond_to?(m) }
-        # Nobrainer
-        records.unscoped.where(:id.in => ids)
-      else
-        raise "Not sure how to load records"
-      end
+      records =
+        if records.respond_to?(:primary_key)
+          # ActiveRecord
+          records.where(records.primary_key => ids) if records.primary_key
+        elsif records.respond_to?(:queryable)
+          # Mongoid 3+
+          records.queryable.for_ids(ids)
+        elsif records.respond_to?(:unscoped) && :id.respond_to?(:in)
+          # Nobrainer
+          records.unscoped.where(:id.in => ids)
+        end
+
+      raise Searchkick::Error, "Not sure how to load records" if !records
+
+      records
     end
 
     def base_field(k)

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "2.0.0"
+  VERSION = "2.0.1"
 end

data/searchkick.gemspec

@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
 
   spec.files         = `git ls-files`.split($/)
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features|benchmark)/})
   spec.require_paths = ["lib"]
 
   spec.add_dependency "activemodel", ">= 4.1"

data/test/aggs_test.rb

@@ -98,19 +98,22 @@ class AggsTest < Minitest::Test
 
   def test_aggs_group_by_date
     store [{name: "Old Product", created_at: 3.years.ago}]
-    aggs = Product.search(
-      "Product",
-              aggs: {
-                products_per_year: {
-                  date_histogram: {
-                    field: :created_at,
-                    interval: :year
-                  }
-                }
-              }
-    ).aggs
-
-    assert_equal 4, aggs["products_per_year"]["buckets"].size
+    products =
+      Product.search("Product", {
+        where: {
+          created_at: {lt: Time.now}
+        },
+        aggs: {
+          products_per_year: {
+            date_histogram: {
+              field: :created_at,
+              interval: :year
+            }
+          }
+        }
+      })
+
+    assert_equal 4, products.aggs["products_per_year"]["buckets"].size
   end
 
   protected

data/test/highlight_test.rb

@@ -3,42 +3,42 @@ require_relative "test_helper"
 class HighlightTest < Minitest::Test
   def test_basic
     store_names ["Two Door Cinema Club"]
-    assert_equal "Two Door <em>Cinema</em> Club", Product.search("cinema", fields: [:name], highlight: true).with_details.first[1][:highlight][:name]
+    assert_equal "Two Door <em>Cinema</em> Club", Product.search("cinema", fields: [:name], highlight: true).first.search_highlights[:name]
   end
 
   def test_tag
     store_names ["Two Door Cinema Club"]
-    assert_equal "Two Door <strong>Cinema</strong> Club", Product.search("cinema", fields: [:name], highlight: {tag: "<strong>"}).with_details.first[1][:highlight][:name]
+    assert_equal "Two Door <strong>Cinema</strong> Club", Product.search("cinema", fields: [:name], highlight: {tag: "<strong>"}).first.search_highlights[:name]
   end
 
   def test_multiple_fields
     store [{name: "Two Door Cinema Club", color: "Cinema Orange"}]
-    highlight = Product.search("cinema", fields: [:name, :color], highlight: true).with_details.first[1][:highlight]
-    assert_equal "Two Door <em>Cinema</em> Club", highlight[:name]
-    assert_equal "<em>Cinema</em> Orange", highlight[:color]
+    highlights = Product.search("cinema", fields: [:name, :color], highlight: true).first.search_highlights
+    assert_equal "Two Door <em>Cinema</em> Club", highlights[:name]
+    assert_equal "<em>Cinema</em> Orange", highlights[:color]
   end
 
   def test_fields
     store [{name: "Two Door Cinema Club", color: "Cinema Orange"}]
-    highlight = Product.search("cinema", fields: [:name, :color], highlight: {fields: [:name]}).with_details.first[1][:highlight]
-    assert_equal "Two Door <em>Cinema</em> Club", highlight[:name]
-    assert_nil highlight[:color]
+    highlights = Product.search("cinema", fields: [:name, :color], highlight: {fields: [:name]}).first.search_highlights
+    assert_equal "Two Door <em>Cinema</em> Club", highlights[:name]
+    assert_nil highlights[:color]
   end
 
   def test_field_options
     store_names ["Two Door Cinema Club are a Northern Irish indie rock band"]
     fragment_size = ENV["MATCH"] == "word_start" ? 26 : 20
-    assert_equal "Two Door <em>Cinema</em> Club are", Product.search("cinema", fields: [:name], highlight: {fields: {name: {fragment_size: fragment_size}}}).with_details.first[1][:highlight][:name]
+    assert_equal "Two Door <em>Cinema</em> Club are", Product.search("cinema", fields: [:name], highlight: {fields: {name: {fragment_size: fragment_size}}}).first.search_highlights[:name]
   end
 
   def test_multiple_words
     store_names ["Hello World Hello"]
-    assert_equal "<em>Hello</em> World <em>Hello</em>", Product.search("hello", fields: [:name], highlight: true).with_details.first[1][:highlight][:name]
+    assert_equal "<em>Hello</em> World <em>Hello</em>", Product.search("hello", fields: [:name], highlight: true).first.search_highlights[:name]
   end
 
   def test_encoder
     store_names ["<b>Hello</b>"]
-    assert_equal "&lt;b&gt;<em>Hello</em>&lt;&#x2F;b&gt;", Product.search("hello", fields: [:name], highlight: {encoder: "html"}, misspellings: false).with_details.first[1][:highlight][:name]
+    assert_equal "&lt;b&gt;<em>Hello</em>&lt;&#x2F;b&gt;", Product.search("hello", fields: [:name], highlight: {encoder: "html"}, misspellings: false).first.search_highlights[:name]
   end
 
   def test_body
@@ -58,6 +58,11 @@ class HighlightTest < Minitest::Test
         }
       }
     }
-    assert_equal "Two Door <strong>Cinema</strong> Club", Product.search(body: body).with_details.first[1][:highlight][:"name.analyzed"]
+    assert_equal "Two Door <strong>Cinema</strong> Club", Product.search(body: body).first.search_highlights[:"name.analyzed"]
+  end
+
+  def test_legacy
+    store_names ["Two Door Cinema Club"]
+    assert_equal "Two Door <em>Cinema</em> Club", Product.search("cinema", fields: [:name], highlight: true).with_details.first[1][:highlight][:name]
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-29 00:00:00.000000000 Z
+date: 2016-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -108,6 +108,8 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- benchmark/Gemfile
+- benchmark/benchmark.rb
 - lib/searchkick.rb
 - lib/searchkick/index.rb
 - lib/searchkick/index_options.rb
@@ -190,6 +192,8 @@ summary: Searchkick learns what your users are looking for. As more people searc
   it gets smarter and the results get better. It’s friendly for developers - and magical
   for your users.
 test_files:
+- benchmark/Gemfile
+- benchmark/benchmark.rb
 - test/aggs_test.rb
 - test/autocomplete_test.rb
 - test/boost_test.rb