searchkick 2.0.3 → 2.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 63b182684f14e8e5c0b30e0505a188d7bbc06f85
-  data.tar.gz: 9227d3877f3b6cf944a3413fe4478ec17393bc0c
+  metadata.gz: 275674a7d9ef11e2faba8426a3a85172d84cf16c
+  data.tar.gz: 3134c16ed33df803187e4b8b75dd3f8167745e62
 SHA512:
-  metadata.gz: '08f71c79a49fa5c0c43d4d1d489b0040bfec789edd6c3e3737b48fd749a7dc1ee4d7b2337aa0e57db35a29a20c14ce13c5a0b653441151f64f2241b59f69a4f5'
-  data.tar.gz: 9f864b9b4d890a1a220364abd8e6d0376d06e90d853d34201630f3599bc580a5fee3cf41604b79cbed7bf25ee5524352056119cd9c4a8b0883017476639523da
+  metadata.gz: bf4ab3f4fb5557051f9e5bbc25fd9aaf26f3c5403e473e86f6108f5418a53cc3ee391ec0cf204cc717ca0df7f96ba5e83deeb42bb725bc6025f6258c067d9c7d
+  data.tar.gz: dba920c1264cf007eb5a4759c83f3bf0269a8b36bf381b4e4f6c8b23089abfb1a90c373a3ae6853f4145bae751b1181f7bcd77fb06cd839cd45c8e350c0f22ea

data/.travis.yml

@@ -3,6 +3,7 @@ language: ruby
 rvm: 2.3.1
 services:
   - mongodb
+  - redis-server
 before_install:
   - ./test/ci/before_install.sh
 script: RUBYOPT=W0 bundle exec rake test

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 2.0.4
+
+- Added support for queuing updates [experimental]
+- Added `refresh_interval` option to `reindex`
+- Prefer `search_index` over `searchkick_index`
+
 ## 2.0.3
 
 - Added `async` option to `reindex` [experimental]

data/Gemfile

@@ -8,3 +8,4 @@ gem "activerecord", "~> 5.0.0"
 gem "gemoji-parser"
 gem "typhoeus"
 gem "activejob"
+gem "redis"

data/README.md

@@ -1185,6 +1185,34 @@ And use:
 Searchkick.reindex_status(index_name)
 ```
 
+### Queues [master, experimental]
+
+You can also queue updates and do them in bulk for better performance. First, set up Redis in an initializer.
+
+```ruby
+Searchkick.redis = Redis.new
+```
+
+And ask your models to queue updates.
+
+```ruby
+class Product < ActiveRecord::Base
+  searchkick callbacks: :queue
+end
+```
+
+Then, set up a background job to run.
+
+```ruby
+Searchkick::ProcessQueueJob.perform_later(class_name: "Product")
+```
+
+You can check the queue length with:
+
+```ruby
+Product.searchkick_index.reindex_queue.length
+```
+
 For more tips, check out [Keeping Elasticsearch in Sync](https://www.elastic.co/blog/found-keeping-elasticsearch-in-sync).
 
 ## Advanced

data/lib/searchkick.rb

@@ -5,6 +5,7 @@ require "searchkick/version"
 require "searchkick/index_options"
 require "searchkick/index"
 require "searchkick/indexer"
+require "searchkick/reindex_queue"
 require "searchkick/results"
 require "searchkick/query"
 require "searchkick/model"
@@ -21,6 +22,8 @@ rescue LoadError
 end
 if defined?(ActiveJob)
   require "searchkick/bulk_reindex_job"
+  require "searchkick/process_queue_job"
+  require "searchkick/process_batch_job"
   require "searchkick/reindex_v2_job"
 end
 
@@ -142,6 +145,25 @@ module Searchkick
     end
   end
 
+  # private
+  def self.load_records(records, ids)
+    records =
+      if records.respond_to?(:primary_key)
+        # ActiveRecord
+        records.where(records.primary_key => ids) if records.primary_key
+      elsif records.respond_to?(:queryable)
+        # Mongoid 3+
+        records.queryable.for_ids(ids)
+      elsif records.respond_to?(:unscoped) && :id.respond_to?(:in)
+        # Nobrainer
+        records.unscoped.where(:id.in => ids)
+      end
+
+    raise Searchkick::Error, "Not sure how to load records" if !records
+
+    records
+  end
+
   # private
   def self.indexer
     Thread.current[:searchkick_indexer] ||= Searchkick::Indexer.new

data/lib/searchkick/bulk_reindex_job.rb

@@ -6,7 +6,7 @@ module Searchkick
       klass = class_name.constantize
       index = index_name ? Searchkick::Index.new(index_name) : klass.searchkick_index
       record_ids ||= min_id..max_id
-      index.import_scope(klass.where(klass.primary_key => record_ids), method_name: method_name, batch: true, batch_id: batch_id)
+      index.import_scope(Searchkick.load_records(klass, record_ids), method_name: method_name, batch: true, batch_id: batch_id)
     end
   end
 end

data/lib/searchkick/index.rb

@@ -38,11 +38,22 @@ module Searchkick
       client.indices.get_settings index: name
     end
 
+    def refresh_interval
+      settings.values.first["settings"]["index"]["refresh_interval"]
+    end
+
     def update_settings(settings)
       client.indices.put_settings index: name, body: settings
     end
 
-    def promote(new_name)
+    def promote(new_name, update_refresh_interval: false)
+      if update_refresh_interval
+        new_index = Searchkick::Index.new(new_name)
+        settings = options[:settings] || {}
+        refresh_interval = (settings[:index] && settings[:index][:refresh_interval]) || "1s"
+        new_index.update_settings(index: {refresh_interval: refresh_interval})
+      end
+
       old_indices =
         begin
           client.indices.get_alias(name: name).keys
@@ -135,6 +146,12 @@ module Searchkick
       search_model(record.class, like_text, options)
     end
 
+    # queue
+
+    def reindex_queue
+      Searchkick::ReindexQueue.new(name)
+    end
+
     # search
 
     def search_model(searchkick_klass, term = "*", **options, &block)
@@ -191,7 +208,7 @@ module Searchkick
 
     # https://gist.github.com/jarosan/3124884
     # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
-    def reindex_scope(scope, import: true, resume: false, retain: false, async: false)
+    def reindex_scope(scope, import: true, resume: false, retain: false, async: false, refresh_interval: nil)
       if resume
         index_name = all_indices.sort.last
         raise Searchkick::Error, "No index to resume" unless index_name
@@ -199,7 +216,9 @@ module Searchkick
       else
         clean_indices unless retain
 
-        index = create_index(index_options: scope.searchkick_index_options)
+        index_options = scope.searchkick_index_options
+        index_options.deep_merge!(settings: {index: {refresh_interval: refresh_interval}}) if refresh_interval
+        index = create_index(index_options: index_options)
       end
 
       # check if alias exists
@@ -209,12 +228,12 @@ module Searchkick
 
         # get existing indices to remove
         unless async
-          promote(index.name)
+          promote(index.name, update_refresh_interval: !refresh_interval.nil?)
           clean_indices unless retain
         end
       else
         delete if exists?
-        promote(index.name)
+        promote(index.name, update_refresh_interval: !refresh_interval.nil?)
 
         # import after promotion
         index.import_scope(scope, resume: resume, async: async, full: true) if import
@@ -238,23 +257,27 @@ module Searchkick
         import_or_update scope.to_a, method_name, async
         Searchkick.redis.srem(batches_key, batch_id) if batch_id && Searchkick.redis
       elsif full && async
-        # TODO expire Redis key
-        primary_key = scope.primary_key
-        starting_id = scope.minimum(primary_key)
-        max_id = scope.maximum(primary_key)
-        batches_count = ((max_id - starting_id + 1) / batch_size.to_f).ceil
-
-        batches_count.times do |i|
-          batch_id = i + 1
-          min_id = starting_id + (i * batch_size)
-          Searchkick::BulkReindexJob.perform_later(
-            class_name: scope.model_name.name,
-            min_id: min_id,
-            max_id: min_id + batch_size - 1,
-            index_name: name,
-            batch_id: batch_id
-          )
-          Searchkick.redis.sadd(batches_key, batch_id) if Searchkick.redis
+        if scope.respond_to?(:primary_key)
+          # TODO expire Redis key
+          primary_key = scope.primary_key
+          starting_id = scope.minimum(primary_key) || 0
+          max_id = scope.maximum(primary_key) || 0
+          batches_count = ((max_id - starting_id + 1) / batch_size.to_f).ceil
+
+          batches_count.times do |i|
+            batch_id = i + 1
+            min_id = starting_id + (i * batch_size)
+            Searchkick::BulkReindexJob.perform_later(
+              class_name: scope.model_name.name,
+              min_id: min_id,
+              max_id: min_id + batch_size - 1,
+              index_name: name,
+              batch_id: batch_id
+            )
+            Searchkick.redis.sadd(batches_key, batch_id) if Searchkick.redis
+          end
+        else
+          raise Searchkick::Error, "async option only supported for ActiveRecord"
         end
       elsif scope.respond_to?(:find_in_batches)
         if resume

data/lib/searchkick/model.rb

@@ -35,6 +35,7 @@ module Searchkick
             index = index.call if index.respond_to? :call
             Searchkick::Index.new(index, searchkick_options)
           end
+          alias_method :search_index, :searchkick_index unless method_defined?(:search_index)
 
           def enable_search_callbacks
             class_variable_set :@@searchkick_callbacks, true
@@ -84,11 +85,36 @@ module Searchkick
           after_destroy callback_name, if: proc { self.class.search_callbacks? }
         end
 
-        def reindex(method_name = nil, refresh: false, async: false)
-          if async
+        def reindex(method_name = nil, refresh: false, async: false, mode: nil)
+          klass_options = self.class.searchkick_index.options
+
+          if mode.nil?
+            mode =
+              if async
+                :async
+              elsif Searchkick.callbacks_value
+                Searchkick.callbacks_value
+              elsif klass_options.key?(:callbacks) && klass_options[:callbacks] != :async
+                # TODO remove 2nd condition in next major version
+                klass_options[:callbacks]
+              end
+          end
+
+          case mode
+          when :queue
+            if method_name
+              raise Searchkick::Error, "Partial reindex not supported with queue option"
+            else
+              self.class.searchkick_index.reindex_queue.push(id.to_s)
+            end
+          when :async
             if method_name
               # TODO support Mongoid and NoBrainer and non-id primary keys
-              Searchkick::BulkReindexJob.perform_later(class_name: self.class.name, record_ids: [id.to_s], method_name: method_name ? method_name.to_s : nil)
+              Searchkick::BulkReindexJob.perform_later(
+                class_name: self.class.name,
+                record_ids: [id.to_s],
+                method_name: method_name ? method_name.to_s : nil
+              )
             else
               self.class.searchkick_index.reindex_record_async(self)
             end

data/lib/searchkick/process_batch_job.rb

@@ -0,0 +1,23 @@
+module Searchkick
+  class ProcessBatchJob < ActiveJob::Base
+    queue_as :searchkick
+
+    def perform(class_name:, record_ids:)
+      klass = class_name.constantize
+      scope = Searchkick.load_records(klass, record_ids)
+      scope = scope.search_import if scope.respond_to?(:search_import)
+      records = scope.select(&:should_index?)
+
+      # determine which records to delete
+      delete_ids = record_ids - records.map { |r| r.id.to_s }
+      delete_records = delete_ids.map { |id| m = klass.new; m.id = id; m }
+
+      # bulk reindex
+      index = klass.searchkick_index
+      Searchkick.callbacks(:bulk) do
+        index.bulk_index(records)
+        index.bulk_delete(delete_records)
+      end
+    end
+  end
+end

data/lib/searchkick/process_queue_job.rb

@@ -0,0 +1,23 @@
+module Searchkick
+  class ProcessQueueJob < ActiveJob::Base
+    queue_as :searchkick
+
+    def perform(class_name:)
+      model = class_name.constantize
+
+      limit = 1000
+      record_ids = Searchkick::ReindexQueue.new(model.searchkick_index.name).reserve(limit: limit)
+      if record_ids.any?
+        Searchkick::ProcessBatchJob.perform_later(
+          class_name: model.name,
+          record_ids: record_ids
+        )
+        # TODO when moving to reliable queuing, mark as complete
+
+        if record_ids.size == limit
+          Searchkick::ProcessQueueJob.perform_later(class_name: class_name)
+        end
+      end
+    end
+  end
+end

data/lib/searchkick/reindex_queue.rb

@@ -0,0 +1,40 @@
+module Searchkick
+  class ReindexQueue
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+    end
+
+    def push(record_id)
+      redis.lpush(redis_key, record_id)
+    end
+
+    # TODO use reliable queuing
+    def reserve(limit: 1000)
+      record_ids = Set.new
+      while record_ids.size < limit && record_id = redis.rpop(redis_key)
+        record_ids << record_id
+      end
+      record_ids.to_a
+    end
+
+    def clear
+      redis.del(redis_key)
+    end
+
+    def length
+      redis.llen(redis_key)
+    end
+
+    private
+
+    def redis
+      Searchkick.redis
+    end
+
+    def redis_key
+      "searchkick:reindex_queue:#{name}"
+    end
+  end
+end

data/lib/searchkick/results.rb

@@ -206,21 +206,7 @@ module Searchkick
           end
       end
 
-      records =
-        if records.respond_to?(:primary_key)
-          # ActiveRecord
-          records.where(records.primary_key => ids) if records.primary_key
-        elsif records.respond_to?(:queryable)
-          # Mongoid 3+
-          records.queryable.for_ids(ids)
-        elsif records.respond_to?(:unscoped) && :id.respond_to?(:in)
-          # Nobrainer
-          records.unscoped.where(:id.in => ids)
-        end
-
-      raise Searchkick::Error, "Not sure how to load records" if !records
-
-      records
+      Searchkick.load_records(records, ids)
     end
 
     def base_field(k)

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "2.0.3"
+  VERSION = "2.0.4"
 end

data/test/callbacks_test.rb

@@ -24,4 +24,36 @@ class CallbacksTest < Minitest::Test
     Product.searchkick_index.refresh
     assert_search "product", ["Product A", "Product B"]
   end
+
+  def test_queue
+    skip unless defined?(ActiveJob) && defined?(Redis)
+
+    reindex_queue = Product.searchkick_index.reindex_queue
+    reindex_queue.clear
+
+    Searchkick.callbacks(:queue) do
+      store_names ["Product A", "Product B"]
+    end
+    Product.searchkick_index.refresh
+    assert_search "product", [], load: false
+    assert_equal 2, reindex_queue.length
+
+    Searchkick::ProcessQueueJob.perform_later(class_name: "Product")
+    Product.searchkick_index.refresh
+    assert_search "product", ["Product A", "Product B"], load: false
+    assert_equal 0, reindex_queue.length
+
+    Searchkick.callbacks(:queue) do
+      Product.where(name: "Product B").destroy_all
+      Product.create!(name: "Product C")
+    end
+    Product.searchkick_index.refresh
+    assert_search "product", ["Product A", "Product B"], load: false
+    assert_equal 2, reindex_queue.length
+
+    Searchkick::ProcessQueueJob.perform_later(class_name: "Product")
+    Product.searchkick_index.refresh
+    assert_search "product", ["Product A", "Product C"], load: false
+    assert_equal 0, reindex_queue.length
+  end
 end

data/test/gemfiles/mongoid6.gemfile

@@ -5,3 +5,4 @@ gemspec path: "../../"
 
 gem "mongoid", "~> 6.0.0"
 gem "activejob"
+gem "redis"

data/test/gemfiles/nobrainer.gemfile

@@ -5,3 +5,4 @@ gemspec path: "../../"
 
 gem "nobrainer", ">= 0.21.0"
 gem "activejob"
+gem "redis"

data/test/reindex_test.rb

@@ -39,4 +39,17 @@ class ReindexTest < Minitest::Test
     Product.searchkick_index.promote(reindex[:index_name])
     assert_search "product", ["Product A"]
   end
+
+  def test_refresh_interval
+    reindex = Product.reindex(refresh_interval: "30s", async: true, import: false)
+    index = Searchkick::Index.new(reindex[:index_name])
+    assert_nil Product.search_index.refresh_interval
+    assert_equal "30s", index.refresh_interval
+
+    Product.search_index.promote(index.name, update_refresh_interval: true)
+    assert_equal "1s", index.refresh_interval
+    assert_equal "1s", Product.search_index.refresh_interval
+  ensure
+    Product.reindex
+  end
 end

data/test/test_helper.rb

@@ -14,6 +14,8 @@ File.delete("elasticsearch.log") if File.exist?("elasticsearch.log")
 Searchkick.client.transport.logger = Logger.new("elasticsearch.log")
 Searchkick.search_timeout = 5
 
+Searchkick.redis = Redis.new if defined?(Redis)
+
 puts "Running against Elasticsearch #{Searchkick.server_version}"
 
 I18n.config.enforce_available_locales = true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.4
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-13 00:00:00.000000000 Z
+date: 2017-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -118,7 +118,10 @@ files:
 - lib/searchkick/logging.rb
 - lib/searchkick/middleware.rb
 - lib/searchkick/model.rb
+- lib/searchkick/process_batch_job.rb
+- lib/searchkick/process_queue_job.rb
 - lib/searchkick/query.rb
+- lib/searchkick/reindex_queue.rb
 - lib/searchkick/reindex_v2_job.rb
 - lib/searchkick/results.rb
 - lib/searchkick/tasks.rb