searchkick 1.3.3 → 1.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 95d6fb192a6e28ebc711b99439f23ad5f27990ab
-  data.tar.gz: 7b749691174ca6b03e471c75e4afb304f09869ef
+  metadata.gz: e771f2aea318aada5efd68350c101abca94a77a8
+  data.tar.gz: 4c3d8566133e79b6456609a9d50faf84be229454
 SHA512:
-  metadata.gz: 37e07bcc09568bc7ca4806b5623eb53436fcdfcca773fd12398886240aefddb0666a5385d13146b7667fc582f9bedf6a9ce1193ecff4e7fe5c94484d9053fcf1
-  data.tar.gz: 9a6d7f8844242656226dca9f2d984ee7740c43e3ccd0724a4bacb3f03b5426860140b1f5eebd554e70a6f9db42f008c92267b7b2c0c86a6d0057949c5ac26205
+  metadata.gz: 67375bbcd44840170e0fc29eeb84308f4e6814b24c30e8c8e6d0ed177df04dc038f7d0f4084d9cba19a2f5116249bb8313a37b01707e4163a83aeccc89a812c1
+  data.tar.gz: e31a38b61ed0ddf786f7aaf8be45eb39a938122305a9886d60fc33eaac335b267c468e1fb01511bf5863ca390bea86b216e822461422656e956c423948a3361c

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 1.3.4
+
+- Added `resume` option to reindex
+- Added search timeout to payload
+
 ## 1.3.3
 
 - Fix for namespaced models (broken in 1.3.2)

data/README.md

@@ -372,6 +372,12 @@ class Product < ActiveRecord::Base
 end
 ```
 
+If a reindex is interrupted, you can resume it with:
+
+```ruby
+Product.reindex(resume: true)
+```
+
 ### To Reindex, or Not to Reindex
 
 #### Reindex

data/lib/searchkick/index.rb

@@ -140,34 +140,60 @@ module Searchkick
       index
     end
 
-    # remove old indices that start w/ index_name
-    def clean_indices
-      all_indices =
+    def all_indices(options = {})
+      indices =
         begin
           client.indices.get_aliases
         rescue Elasticsearch::Transport::Transport::Errors::NotFound
           {}
         end
-      indices = all_indices.select { |k, v| (v.empty? || v["aliases"].empty?) && k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys
+      indices = indices.select { |k, v| v.empty? || v["aliases"].empty? } if options[:unaliased]
+      indices.select { |k, v| k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys
+    end
+
+    # remove old indices that start w/ index_name
+    def clean_indices
+      indices = all_indices(unaliased: true)
       indices.each do |index|
         Searchkick::Index.new(index).delete
       end
       indices
     end
 
+    def total_docs
+      response =
+        client.search(
+          index: name,
+          body: {
+            fields: [],
+            query: {match_all: {}},
+            size: 0
+          }
+        )
+
+      response["hits"]["total"]
+    end
+
     # https://gist.github.com/jarosan/3124884
     # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
     def reindex_scope(scope, options = {})
       skip_import = options[:import] == false
+      resume = options[:resume]
 
-      clean_indices
+      if resume
+        index_name = all_indices.sort.last
+        raise Searchkick::Error, "No index to resume" unless index_name
+        index = Searchkick::Index.new(index_name)
+      else
+        clean_indices
 
-      index = create_index(index_options: scope.searchkick_index_options)
+        index = create_index(index_options: scope.searchkick_index_options)
+      end
 
       # check if alias exists
       if alias_exists?
         # import before swap
-        index.import_scope(scope) unless skip_import
+        index.import_scope(scope, resume: resume) unless skip_import
 
         # get existing indices to remove
         swap(index.name)
@@ -177,7 +203,7 @@ module Searchkick
         swap(index.name)
 
         # import after swap
-        index.import_scope(scope) unless skip_import
+        index.import_scope(scope, resume: resume) unless skip_import
       end
 
       index.refresh
@@ -185,12 +211,20 @@ module Searchkick
       true
     end
 
-    def import_scope(scope)
+    def import_scope(scope, options = {})
       batch_size = @options[:batch_size] || 1000
 
       # use scope for import
       scope = scope.search_import if scope.respond_to?(:search_import)
       if scope.respond_to?(:find_in_batches)
+        if options[:resume]
+          # use total docs instead of max id since there's not a great way
+          # to get the max _id without scripting since it's a string
+
+          # TODO use primary key and prefix with table name
+          scope = scope.where("id > ?", total_docs)
+        end
+
         scope.find_in_batches batch_size: batch_size do |batch|
           import batch.select(&:should_index?)
         end
@@ -198,6 +232,7 @@ module Searchkick
         # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
         # use cursor for Mongoid
         items = []
+        # TODO add resume
         scope.all.each do |item|
           items << item if item.should_index?
           if items.length == batch_size

data/lib/searchkick/query.rb

@@ -368,6 +368,9 @@ module Searchkick
         # highlight
         set_highlights(payload, fields) if options[:highlight]
 
+        # timeout shortly after client times out
+        payload[:timeout] ||= "#{Searchkick.search_timeout + 1}s"
+
         # An empty array will cause only the _id and _type for each hit to be returned
         # doc for :select - http://www.elasticsearch.org/guide/reference/api/search/fields/
         # doc for :select_v2 - https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-source-filtering.html

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "1.3.3"
+  VERSION = "1.3.4"
 end

data/test/query_test.rb

@@ -13,11 +13,19 @@ class QueryTest < Minitest::Test
 
   def test_with_effective_min_score
     store_names ["Milk", "Milk2"]
-    assert_equal ["Milk"], Product.search("Milk", body_options: { min_score: 0.1 }).map(&:name)
+    assert_search "milk", ["Milk"], body_options: {min_score: 0.1}
   end
 
   def test_with_uneffective_min_score
     store_names ["Milk", "Milk2"]
-    assert_equal ["Milk", "Milk2"], Product.search("Milk", body_options: { min_score: 0.0001 }).map(&:name)
+    assert_search "milk", ["Milk", "Milk2"], body_options: {min_score: 0.0001}
+  end
+
+  def test_default_timeout
+    assert_equal "6s", Product.search("*", execute: false).body[:timeout]
+  end
+
+  def test_timeout_override
+    assert_equal "1s", Product.search("*", body_options: {timeout: "1s"}, execute: false).body[:timeout]
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 1.3.3
+  version: 1.3.4
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-02 00:00:00.000000000 Z
+date: 2016-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel