searchkick 4.1.1 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 79cd2e4afe02598dbdc951ed222c98ab51c8f6b0580c7beb70d7f6f5e9223c8f
-  data.tar.gz: 804ae10724df4b1f72fc93f157e185cd6acbc734952c64e1c596d20f26360ce9
+  metadata.gz: '0529381ceedcb4630640d4ae37b02ac2e950c3088a757946b930e56e0a8d4591'
+  data.tar.gz: 539ee2b8da1632f5b86ff3da063e5b815fac2ba3edb5efc2b3d2d508cdb9c002
 SHA512:
-  metadata.gz: 732852787481b5aaf69edda3850c50846d5d07a55ba73d41dbee864988f04c5480c8c1535af6b82374982da4f95de5735f36433e6a37b46a41053ecc975fc9ef
-  data.tar.gz: 6d4408714af66540fc2868d2d9d4bd7b72d468150f5a29c9200d4eb349ee89f3155c98536742eb0015f905de644345e832af3a752613007d424cbb2c353051c2
+  metadata.gz: cfb0aa73064ee937db83d948086bf5f9f1a81bc597f871288786f18ddaff1f39edf202c3a877d4147c3e963c1a736130f4250e10238444924347dbf8854b15c0
+  data.tar.gz: d4733742e12185d5f6296361948a20a2cc11ad14cf0f2c254cbcb80b5f5c64ba6754eb46161b22e33d786363a94b7e79db8b55d567e77ee8dc46ed3a9d7ab246

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 4.2.0 (2019-12-18)
+
+- Added safety check for multiple `Model.reindex`
+- Added `deep_paging` option
+- Added request parameters to search notifications and curl representation
+- Removed curl from search notifications to prevent confusion
+
 ## 4.1.1 (2019-11-19)
 
 - Added `chinese2` and `korean2` languages

data/README.md

@@ -137,8 +137,6 @@ Limit / offset
 limit: 20, offset: 40
 ```
 
-**Note:** By default, Elasticsearch [limits pagination](#deep-pagination) to the first 10,000 results for performance
-
 Select
 
 ```ruby
@@ -182,6 +180,8 @@ Get the full response from Elasticsearch
 results.response
 ```
 
+**Note:** By default, Elasticsearch [limits paging](#deep-paging-master) to the first 10,000 results for performance. With Elasticsearch 7, this applies to the total count as well.
+
 ### Boosting
 
 Boost important fields
@@ -554,7 +554,7 @@ Searchkick.callbacks(false) do
 end
 ```
 
-#### Associations
+### Associations
 
 Data is **not** automatically synced when an association is updated. If this is desired, add a callback to reindex:
 
@@ -1518,20 +1518,20 @@ end
 products.clear_scroll
 ```
 
-## Deep Pagination
+## Deep Paging
 
-By default, Elasticsearch limits pagination to the first 10,000 results [for performance](https://www.elastic.co/guide/en/elasticsearch/guide/current/pagination.html). We don’t recommend changing this, but if you need to, you can use:
+By default, Elasticsearch limits paging to the first 10,000 results. [Here’s why](https://www.elastic.co/guide/en/elasticsearch/guide/current/pagination.html). We don’t recommend changing this, but if you really need all results, you can use:
 
 ```ruby
 class Product < ApplicationRecord
-  searchkick settings: {index: {max_result_window: 1000000000}}
+  searchkick deep_paging: true
 end
 ```
 
-And search with:
+If you just need an accurate total count with Elasticsearch 7, you can instead use:
 
 ```ruby
-Product.search("pears", limit: 1000000000, body_options: {track_total_hits: true})
+Product.search("pears", body_options: {track_total_hits: true})
 ```
 
 ## Nested Data

data/lib/searchkick/index.rb

@@ -47,7 +47,7 @@ module Searchkick
     end
 
     def refresh_interval
-      settings.values.first["settings"]["index"]["refresh_interval"]
+      index_settings["refresh_interval"]
     end
 
     def update_settings(settings)
@@ -249,6 +249,11 @@ module Searchkick
       end
     end
 
+    # private
+    def uuid
+      index_settings["uuid"]
+    end
+
     protected
 
     def client
@@ -259,6 +264,10 @@ module Searchkick
       @bulk_indexer ||= BulkIndexer.new(self)
     end
 
+    def index_settings
+      settings.values.first["settings"]["index"]
+    end
+
     def import_before_promotion(index, relation, **import_options)
       index.import_scope(relation, **import_options)
     end
@@ -285,6 +294,8 @@ module Searchkick
         scope: scope
       }
 
+      uuid = index.uuid
+
       # check if alias exists
       alias_exists = alias_exists?
       if alias_exists
@@ -292,6 +303,7 @@ module Searchkick
 
         # get existing indices to remove
         unless async
+          check_uuid(uuid, index.uuid)
           promote(index.name, update_refresh_interval: !refresh_interval.nil?)
           clean_indices unless retain
         end
@@ -316,6 +328,7 @@ module Searchkick
           # already promoted if alias didn't exist
           if alias_exists
             puts "Jobs complete. Promoting..."
+            check_uuid(uuid, index.uuid)
             promote(index.name, update_refresh_interval: !refresh_interval.nil?)
           end
           clean_indices unless retain
@@ -334,5 +347,15 @@ module Searchkick
 
       raise e
     end
+
+    # safety check
+    # still a chance for race condition since its called before promotion
+    # ideal is for user to disable automatic index creation
+    # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-creation
+    def check_uuid(old_uuid, new_uuid)
+      if old_uuid != new_uuid
+        raise Searchkick::Error, "Safety check failed - only run one Model.reindex per model at a time"
+      end
+    end
   end
 end

data/lib/searchkick/index_options.rb

@@ -456,6 +456,13 @@ module Searchkick
         mappings = mappings.symbolize_keys.deep_merge(custom_mapping.symbolize_keys)
       end
 
+      if options[:deep_paging]
+        if !settings.dig(:index, :max_result_window) && !settings[:"index.max_result_window"]
+          settings[:index] ||= {}
+          settings[:index][:max_result_window] = 1_000_000_000
+        end
+      end
+
       {
         settings: settings,
         mappings: mappings

data/lib/searchkick/logging.rb

@@ -132,7 +132,7 @@ module Searchkick
     def multi_search(searches)
       event = {
         name: "Multi Search",
-        body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join
+        body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join,
       }
       ActiveSupport::Notifications.instrument("multi_search.searchkick", event) do
         super
@@ -162,14 +162,17 @@ module Searchkick
 
       payload = event.payload
       name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
-      type = payload[:query][:type]
+
       index = payload[:query][:index].is_a?(Array) ? payload[:query][:index].join(",") : payload[:query][:index]
+      type = payload[:query][:type]
+      request_params = payload[:query].except(:index, :type, :body)
+
+      params = []
+      request_params.each do |k, v|
+        params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
+      end
 
-      # no easy way to tell which host the client will use
-      host = Searchkick.client.transport.hosts.first
-      params = ["pretty"]
-      params << "scroll=#{payload[:query][:scroll]}" if payload[:query][:scroll]
-      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?#{params.join('&')} -H 'Content-Type: application/json' -d '#{payload[:query][:body].to_json}'"
+      debug "  #{color(name, YELLOW, true)}  #{index}#{type ? "/#{type.join(',')}" : ''}/_search#{params.any? ? '?' + params.join('&') : nil} #{payload[:query][:body].to_json}"
     end
 
     def request(event)
@@ -189,9 +192,7 @@ module Searchkick
       payload = event.payload
       name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
 
-      # no easy way to tell which host the client will use
-      host = Searchkick.client.transport.hosts.first
-      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -H 'Content-Type: application/json' -d '#{payload[:body]}'"
+      debug "  #{color(name, YELLOW, true)}  _msearch #{payload[:body]}"
     end
   end
 

data/lib/searchkick/model.rb

@@ -3,7 +3,7 @@ module Searchkick
     def searchkick(**options)
       options = Searchkick.model_options.merge(options)
 
-      unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :default_fields,
+      unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
         :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
         :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :settings, :similarity,
         :special_characters, :stem, :stem_conversions, :suggest, :synonyms, :text_end,

data/lib/searchkick/query.rb

@@ -106,12 +106,15 @@ module Searchkick
       query = params
       type = query[:type]
       index = query[:index].is_a?(Array) ? query[:index].join(",") : query[:index]
+      request_params = query.except(:index, :type, :body)
 
       # no easy way to tell which host the client will use
       host = Searchkick.client.transport.hosts.first
       credentials = host[:user] || host[:password] ? "#{host[:user]}:#{host[:password]}@" : nil
       params = ["pretty"]
-      params << "scroll=#{options[:scroll]}" if options[:scroll]
+      request_params.each do |k, v|
+        params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
+      end
       "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?#{params.join('&')} -H 'Content-Type: application/json' -d '#{query[:body].to_json}'"
     end
 
@@ -232,7 +235,9 @@ module Searchkick
 
       # pagination
       page = [options[:page].to_i, 1].max
-      per_page = (options[:limit] || options[:per_page] || 10_000).to_i
+      # maybe use index.max_result_window in the future
+      default_limit = searchkick_options[:deep_paging] ? 1_000_000_000 : 10_000
+      per_page = (options[:limit] || options[:per_page] || default_limit).to_i
       padding = [options[:padding].to_i, 0].max
       offset = options[:offset] || (page - 1) * per_page + padding
       scroll = options[:scroll]
@@ -433,19 +438,20 @@ module Searchkick
 
         models = Array(options[:models])
         if models.any? { |m| m != m.searchkick_klass }
-          Searchkick.warn("Passing child models to models option throws off hits and pagination - use type option instead")
-
-          # TODO uncomment once aliases are supported with _index
-          # should be ES 7.5
+          # aliases are not supported with _index in ES below 7.5
           # see https://github.com/elastic/elasticsearch/pull/46640
-          # index_type_or =
-          #   models.map do |m|
-          #     v = {_index: m.searchkick_index.name}
-          #     v[:type] = m.searchkick_index.klass_document_type(m, true) if m != m.searchkick_klass
-          #     v
-          #   end
-
-          # where[:or] = Array(where[:or]) + [index_type_or]
+          if below75?
+            Searchkick.warn("Passing child models to models option throws off hits and pagination - use type option instead")
+          else
+            index_type_or =
+              models.map do |m|
+                v = {_index: m.searchkick_index.name}
+                v[:type] = m.searchkick_index.klass_document_type(m, true) if m != m.searchkick_klass
+                v
+              end
+
+            where[:or] = Array(where[:or]) + [index_type_or]
+          end
         end
 
         # start everything as efficient filters
@@ -516,6 +522,10 @@ module Searchkick
       # routing
       @routing = options[:routing] if options[:routing]
 
+      if searchkick_options[:deep_paging] && !below70?
+        payload[:track_total_hits] = true
+      end
+
       # merge more body options
       payload = payload.deep_merge(options[:body_options]) if options[:body_options]
 
@@ -1097,5 +1107,9 @@ module Searchkick
     def below70?
       Searchkick.server_below?("7.0.0")
     end
+
+    def below75?
+      Searchkick.server_below?("7.5.0")
+    end
   end
 end

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "4.1.1"
+  VERSION = "4.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 4.1.1
+  version: 4.2.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-20 00:00:00.000000000 Z
+date: 2019-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -145,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Intelligent search made easy with Rails and Elasticsearch