searchkick 2.0.2 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 01c193778cc86142dca494a2aa0b0ac8c6474d0b
-  data.tar.gz: 7ff77b2f385a8004acaf1dabb4935f90d61c26f8
+  metadata.gz: 63b182684f14e8e5c0b30e0505a188d7bbc06f85
+  data.tar.gz: 9227d3877f3b6cf944a3413fe4478ec17393bc0c
 SHA512:
-  metadata.gz: cbeefdbb0c40dfb0c32789e7468d3b570c7014fe9b27a202fca7b310820fd3c223264e360456ea68ca5bb13378e3770f448b3237abb9f61385f22d72942b09b0
-  data.tar.gz: 199fb639bcc0d883e87848009e35d2ed0e62e491c5ada17d0eeab3844539513f27c09af46ebb17051f402973dabed86e46e8d6bd08e18b9b3dc43ce3ac2d74e0
+  metadata.gz: '08f71c79a49fa5c0c43d4d1d489b0040bfec789edd6c3e3737b48fd749a7dc1ee4d7b2337aa0e57db35a29a20c14ce13c5a0b653441151f64f2241b59f69a4f5'
+  data.tar.gz: 9f864b9b4d890a1a220364abd8e6d0376d06e90d853d34201630f3599bc580a5fee3cf41604b79cbed7bf25ee5524352056119cd9c4a8b0883017476639523da

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 2.0.3
+
+- Added `async` option to `reindex` [experimental]
+- Added `misspellings?` method to results
+
 ## 2.0.2
 
 - Added `retain` option to `reindex`

data/README.md

@@ -253,12 +253,16 @@ Available options are:
 
 ### Exact Matches
 
+To match a field exactly (case-insensitive), use:
+
 ```ruby
 User.search query, fields: [{email: :exact}, :name]
 ```
 
 ### Phrase Matches
 
+To only match the exact order, use:
+
 ```ruby
 User.search "fresh honey", match: :phrase
 ```
@@ -1152,6 +1156,37 @@ Reindex and search with:
 Business.search "ice cream", routing: params[:city_id]
 ```
 
+## Large Data Sets
+
+### Background Reindexing [experimental, ActiveRecord only]
+
+For large data sets, you can use background jobs to parallelize reindexing.
+
+```ruby
+Product.reindex(async: true)
+# {index_name: "products_production_20170111210018065"}
+```
+
+Once the jobs complete, promote the new index with:
+
+```ruby
+Product.searchkick_index.promote(index_name)
+```
+
+You can optionally track the status with Redis:
+
+```ruby
+Searchkick.redis = Redis.new
+```
+
+And use:
+
+```ruby
+Searchkick.reindex_status(index_name)
+```
+
+For more tips, check out [Keeping Elasticsearch in Sync](https://www.elastic.co/blog/found-keeping-elasticsearch-in-sync).
+
 ## Advanced
 
 Prefer to use the [Elasticsearch DSL](https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-queries.html) but still want awesome features like zero-downtime reindexing?
@@ -1453,10 +1488,6 @@ Product.search "api", misspellings: {prefix_length: 2} # api, apt, no ahi
 Product.search "ah", misspellings: {prefix_length: 2} # ah, no aha
 ```
 
-## Large Data Sets
-
-For large data sets, check out [Keeping Elasticsearch in Sync](https://www.elastic.co/blog/found-keeping-elasticsearch-in-sync). Searchkick will make this easy in the future.
-
 ## Testing
 
 This section could use some love.

data/benchmark/Gemfile

@@ -3,9 +3,13 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in searchkick.gemspec
 gemspec path: "../"
 
-gem "sqlite3"
+# gem "sqlite3"
+gem "pg"
 gem "activerecord", "~> 5.0.0"
 gem "activerecord-import"
+gem "activejob"
+gem "redis"
+gem "sidekiq"
 
 # performance
 gem "typhoeus"

data/benchmark/benchmark.rb

@@ -2,19 +2,27 @@ require "bundler/setup"
 Bundler.require(:default)
 require "active_record"
 require "benchmark"
+require "active_support/notifications"
+
+ActiveSupport::Notifications.subscribe "request.searchkick" do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  p event.duration
+end
+
+ActiveJob::Base.queue_adapter = :sidekiq
+
+Searchkick.redis = Redis.new
 
 ActiveRecord::Base.default_timezone = :utc
 ActiveRecord::Base.time_zone_aware_attributes = true
-ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+# ActiveRecord::Base.establish_connection adapter: "sqlite3", database: "/tmp/searchkick"
+ActiveRecord::Base.establish_connection "postgresql://localhost/searchkick_demo_development"
+# ActiveRecord::Base.logger = Logger.new(STDOUT)
 
-ActiveRecord::Migration.create_table :products do |t|
-  t.string :name
-  t.string :color
-  t.integer :store_id
-end
+ActiveJob::Base.logger = nil
 
 class Product < ActiveRecord::Base
-  searchkick batch_size: 100
+  searchkick batch_size: 1000
 
   def search_data
     {
@@ -25,7 +33,15 @@ class Product < ActiveRecord::Base
   end
 end
 
-Product.import ["name", "color", "store_id"], 20000.times.map { |i| ["Product #{i}", ["red", "blue"].sample, rand(10)] }
+total_docs = 100000
+
+# ActiveRecord::Migration.create_table :products, force: :cascade do |t|
+#   t.string :name
+#   t.string :color
+#   t.integer :store_id
+# end
+
+# Product.import ["name", "color", "store_id"], total_docs.times.map { |i| ["Product #{i}", ["red", "blue"].sample, rand(10)] }
 
 puts "Imported"
 
@@ -35,19 +51,37 @@ stats = nil
 
 # p GetProcessMem.new.mb
 
+Product.searchkick_index.delete rescue nil
+
 time =
   Benchmark.realtime do
     # result = RubyProf.profile do
     # report = MemoryProfiler.report do
     # stats = AllocationStats.trace do
-    Product.reindex
+    reindex = Product.reindex(async: true)
+    p reindex
     # end
+
+    60.times do |i|
+      if reindex.is_a?(Hash)
+        docs = Searchkick::Index.new(reindex[:index_name]).total_docs
+      else
+        docs = Product.searchkick_index.total_docs
+      end
+      puts "#{i}: #{docs}"
+      if docs == total_docs
+        break
+      end
+      p Searchkick.reindex_status(reindex[:index_name]) if reindex.is_a?(Hash)
+      sleep(1)
+      # Product.searchkick_index.refresh
+    end
   end
 
 # p GetProcessMem.new.mb
 
 puts time.round(1)
-puts Product.searchkick_index.total_docs
+
 
 if result
   printer = RubyProf::GraphPrinter.new(result)

data/lib/searchkick/bulk_reindex_job.rb

@@ -0,0 +1,12 @@
+module Searchkick
+  class BulkReindexJob < ActiveJob::Base
+    queue_as :searchkick
+
+    def perform(class_name:, record_ids: nil, index_name: nil, method_name: nil, batch_id: nil, min_id: nil, max_id: nil)
+      klass = class_name.constantize
+      index = index_name ? Searchkick::Index.new(index_name) : klass.searchkick_index
+      record_ids ||= min_id..max_id
+      index.import_scope(klass.where(klass.primary_key => record_ids), method_name: method_name, batch: true, batch_id: batch_id)
+    end
+  end
+end

data/lib/searchkick/index.rb

@@ -38,6 +38,10 @@ module Searchkick
       client.indices.get_settings index: name
     end
 
+    def update_settings(settings)
+      client.indices.put_settings index: name, body: settings
+    end
+
     def promote(new_name)
       old_indices =
         begin
@@ -187,7 +191,7 @@ module Searchkick
 
     # https://gist.github.com/jarosan/3124884
     # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
-    def reindex_scope(scope, import: true, resume: false, retain: false)
+    def reindex_scope(scope, import: true, resume: false, retain: false, async: false)
       if resume
         index_name = all_indices.sort.last
         raise Searchkick::Error, "No index to resume" unless index_name
@@ -201,30 +205,58 @@ module Searchkick
       # check if alias exists
       if alias_exists?
         # import before promotion
-        index.import_scope(scope, resume: resume) if import
+        index.import_scope(scope, resume: resume, async: async, full: true) if import
 
         # get existing indices to remove
-        promote(index.name)
-        clean_indices unless retain
+        unless async
+          promote(index.name)
+          clean_indices unless retain
+        end
       else
         delete if exists?
         promote(index.name)
 
         # import after promotion
-        index.import_scope(scope, resume: resume) if import
+        index.import_scope(scope, resume: resume, async: async, full: true) if import
       end
 
-      index.refresh
-
-      true
+      if async
+        {index_name: index.name}
+      else
+        index.refresh
+        true
+      end
     end
 
-    def import_scope(scope, resume: false, method_name: nil)
+    def import_scope(scope, resume: false, method_name: nil, async: false, batch: false, batch_id: nil, full: false)
       batch_size = @options[:batch_size] || 1000
 
       # use scope for import
       scope = scope.search_import if scope.respond_to?(:search_import)
-      if scope.respond_to?(:find_in_batches)
+
+      if batch
+        import_or_update scope.to_a, method_name, async
+        Searchkick.redis.srem(batches_key, batch_id) if batch_id && Searchkick.redis
+      elsif full && async
+        # TODO expire Redis key
+        primary_key = scope.primary_key
+        starting_id = scope.minimum(primary_key)
+        max_id = scope.maximum(primary_key)
+        batches_count = ((max_id - starting_id + 1) / batch_size.to_f).ceil
+
+        batches_count.times do |i|
+          batch_id = i + 1
+          min_id = starting_id + (i * batch_size)
+          Searchkick::BulkReindexJob.perform_later(
+            class_name: scope.model_name.name,
+            min_id: min_id,
+            max_id: min_id + batch_size - 1,
+            index_name: name,
+            batch_id: batch_id
+          )
+          Searchkick.redis.sadd(batches_key, batch_id) if Searchkick.redis
+        end
+      elsif scope.respond_to?(:find_in_batches)
         if resume
           # use total docs instead of max id since there's not a great way
           # to get the max _id without scripting since it's a string
@@ -233,8 +265,10 @@ module Searchkick
           scope = scope.where("id > ?", total_docs)
         end
 
+        scope = scope.select("id").except(:includes, :preload) if async
+
         scope.find_in_batches batch_size: batch_size do |batch|
-          import_or_update batch.select(&:should_index?), method_name
+          import_or_update batch, method_name, async
         end
       else
         # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
@@ -242,27 +276,18 @@ module Searchkick
         items = []
         # TODO add resume
         scope.all.each do |item|
-          items << item if item.should_index?
+          items << item
           if items.length == batch_size
-            import_or_update items, method_name
+            import_or_update items, method_name, async
             items = []
           end
         end
-        import_or_update items, method_name
+        import_or_update items, method_name, async
       end
     end
 
-    def import_or_update(records, method_name)
-      retries = 0
-      begin
-        method_name ? bulk_update(records, method_name) : import(records)
-      rescue Faraday::ClientError => e
-        if retries < 1
-          retries += 1
-          retry
-        end
-        raise e
-      end
+    def batches_left
+      Searchkick.redis.scard(batches_key) if Searchkick.redis
     end
 
     # other
@@ -373,5 +398,34 @@ module Searchkick
         obj
       end
     end
+
+    def import_or_update(records, method_name, async)
+      if records.any?
+        if async
+          Searchkick::BulkReindexJob.perform_later(
+            class_name: records.first.class.name,
+            record_ids: records.map(&:id),
+            index_name: name,
+            method_name: method_name ? method_name.to_s : nil
+          )
+        else
+          retries = 0
+          records = records.select(&:should_index?)
+          begin
+            method_name ? bulk_update(records, method_name) : import(records)
+          rescue Faraday::ClientError => e
+            if retries < 1
+              retries += 1
+              retry
+            end
+            raise e
+          end
+        end
+      end
+    end
+
+    def batches_key
+      "searchkick:reindex:#{name}:batches"
+    end
   end
 end

data/lib/searchkick/model.rb

@@ -58,15 +58,16 @@ module Searchkick
               # update
               searchkick_index.import_scope(searchkick_klass, method_name: method_name)
               searchkick_index.refresh if refresh
+              true
             elsif scoped && !full
               # reindex association
               searchkick_index.import_scope(searchkick_klass)
               searchkick_index.refresh if refresh
+              true
             else
               # full reindex
               searchkick_index.reindex_scope(searchkick_klass, options)
             end
-            true
           end
           alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex)
 
@@ -83,17 +84,27 @@ module Searchkick
           after_destroy callback_name, if: proc { self.class.search_callbacks? }
         end
 
-        def reindex(method_name = nil, refresh: false)
-          if method_name
-            self.class.searchkick_index.bulk_update([self], method_name)
+        def reindex(method_name = nil, refresh: false, async: false)
+          if async
+            if method_name
+              # TODO support Mongoid and NoBrainer and non-id primary keys
+              Searchkick::BulkReindexJob.perform_later(class_name: self.class.name, record_ids: [id.to_s], method_name: method_name ? method_name.to_s : nil)
+            else
+              self.class.searchkick_index.reindex_record_async(self)
+            end
           else
-            self.class.searchkick_index.reindex_record(self)
+            if method_name
+              self.class.searchkick_index.bulk_update([self], method_name)
+            else
+              self.class.searchkick_index.reindex_record(self)
+            end
+            self.class.searchkick_index.refresh if refresh
           end
-          self.class.searchkick_index.refresh if refresh
         end unless method_defined?(:reindex)
 
+        # TODO remove this method in next major version
         def reindex_async
-          self.class.searchkick_index.reindex_record_async(self)
+          reindex(async: true)
         end unless method_defined?(:reindex_async)
 
         def similar(options = {})

data/lib/searchkick/query.rb

@@ -34,6 +34,7 @@ module Searchkick
       # prevent Ruby warnings
       @type = nil
       @routing = nil
+      @misspellings = false
       @misspellings_below = nil
       @highlighted_fields = nil
 
@@ -107,7 +108,8 @@ module Searchkick
         includes: options[:includes],
         json: !@json.nil?,
         match_suffix: @match_suffix,
-        highlighted_fields: @highlighted_fields || []
+        highlighted_fields: @highlighted_fields || [],
+        misspellings: @misspellings
       }
 
       if options[:debug]
@@ -256,6 +258,9 @@ module Searchkick
             prefix_length = (misspellings.is_a?(Hash) && misspellings[:prefix_length]) || 0
             default_max_expansions = @misspellings_below ? 20 : 3
             max_expansions = (misspellings.is_a?(Hash) && misspellings[:max_expansions]) || default_max_expansions
+            @misspellings = true
+          else
+            @misspellings = false
           end
 
           fields.each do |field|

data/lib/searchkick/results.rb

@@ -184,6 +184,10 @@ module Searchkick
       @response["hits"]["hits"]
     end
 
+    def misspellings?
+      @options[:misspellings]
+    end
+
     private
 
     def results_query(records, hits)

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "2.0.2"
+  VERSION = "2.0.3"
 end

data/lib/searchkick.rb

@@ -19,7 +19,10 @@ begin
 rescue LoadError
   # do nothing
 end
-require "searchkick/reindex_v2_job" if defined?(ActiveJob)
+if defined?(ActiveJob)
+  require "searchkick/bulk_reindex_job"
+  require "searchkick/reindex_v2_job"
+end
 
 module Searchkick
   class Error < StandardError; end
@@ -30,7 +33,7 @@ module Searchkick
   class ImportError < Error; end
 
   class << self
-    attr_accessor :search_method_name, :wordnet_path, :timeout, :models, :client_options
+    attr_accessor :search_method_name, :wordnet_path, :timeout, :models, :client_options, :redis
     attr_writer :client, :env, :search_timeout
     attr_reader :aws_credentials
   end
@@ -129,6 +132,16 @@ module Searchkick
     @client = nil # reset client
   end
 
+  def self.reindex_status(index_name)
+    if redis
+      batches_left = Searchkick::Index.new(index_name).batches_left
+      {
+        completed: batches_left == 0,
+        batches_left: batches_left
+      }
+    end
+  end
+
   # private
   def self.indexer
     Thread.current[:searchkick_indexer] ||= Searchkick::Indexer.new

data/test/match_test.rb

@@ -203,6 +203,11 @@ class MatchTest < Minitest::Test
     assert_search "fresh honey", ["Fresh Honey"], match: :phrase
   end
 
+  def test_phrase_order
+    store_names ["Wheat Bread", "Whole Wheat Bread"]
+    assert_order "wheat bread", ["Wheat Bread", "Whole Wheat Bread"], match: :phrase
+  end
+
   def test_unsearchable
     store [
       {name: "Unsearchable", description: "Almond"}

data/test/misspellings_test.rb

@@ -39,8 +39,18 @@ class MisspellingsTest < Minitest::Test
     assert_search "abc", ["abc", "abd"], misspellings: {below: 2}
   end
 
+  def test_misspellings_below_unmet_result
+    store_names ["abc", "abd", "aee"]
+    assert Product.search("abc", misspellings: {below: 2}).misspellings?
+  end
+
   def test_misspellings_below_met
     store_names ["abc", "abd", "aee"]
     assert_search "abc", ["abc"], misspellings: {below: 1}
   end
+
+  def test_misspellings_below_met_result
+    store_names ["abc", "abd", "aee"]
+    assert !Product.search("abc", misspellings: {below: 1}).misspellings?
+  end
 end

data/test/reindex_test.rb

@@ -22,4 +22,21 @@ class ReindexTest < Minitest::Test
     store.products.reindex(refresh: true)
     assert_search "product", ["Product A", "Product B"]
   end
+
+  def test_async
+    skip unless defined?(ActiveJob) && defined?(ActiveRecord)
+
+    Searchkick.callbacks(false) do
+      store_names ["Product A"]
+    end
+    reindex = Product.reindex(async: true)
+    assert_search "product", []
+
+    index = Searchkick::Index.new(reindex[:index_name])
+    index.refresh
+    assert_equal 1, index.total_docs
+
+    Product.searchkick_index.promote(reindex[:index_name])
+    assert_search "product", ["Product A"]
+  end
 end

data/test/test_helper.rb

@@ -372,7 +372,8 @@ class Animal
   searchkick \
     text_start: [:name],
     suggest: [:name],
-    index_name: -> { "#{name.tableize}-#{Date.today.year}" }
+    index_name: -> { "#{name.tableize}-#{Date.today.year}" },
+    callbacks: defined?(ActiveJob) ? :async : true
     # wordnet: true
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 2.0.2
+  version: 2.0.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-09 00:00:00.000000000 Z
+date: 2017-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -111,6 +111,7 @@ files:
 - benchmark/Gemfile
 - benchmark/benchmark.rb
 - lib/searchkick.rb
+- lib/searchkick/bulk_reindex_job.rb
 - lib/searchkick/index.rb
 - lib/searchkick/index_options.rb
 - lib/searchkick/indexer.rb