searchkick 4.0.0 → 4.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e1a7c30f19f80bab77d1c44dc8778877881d8178ab52c2c3e48cb0fe1f31ac4
-  data.tar.gz: 3091f32cea77c628e629e28dc865d700203f2265393ff44526e86b8ce899ae46
+  metadata.gz: bf57aedd7e85672dd9bc5f60f22719af2f5cb1797e1afa8f16c426ea291dd4db
+  data.tar.gz: 54542375eb2cb3c5b7917304cd7e68530a826641c8f1a556af551083964a8849
 SHA512:
-  metadata.gz: 8d25cc6e9ac21a3f62199255aa5ab9911f2ab3fd970d79417a29063eeb944497da84e1db5ba840cc75eba549bee524c2a9b5b68d624dcadf0d4aa9bfcbc6f4fd
-  data.tar.gz: 9fff1372bb24c8c8d2a59a1821d5ad299c9fcc1e2a73e53bbda2f3fec6b8aeddff0e00692451ec3ece0708762b9ccafe370b0ca7fc2d1e623a5bf12688a78ecd
+  metadata.gz: 5029c0fc16c139f4358786400aad6c36751c6335221e9985f66e5ac308152817db9cd41e709ff6f2b18e731842c85b1a17b9aaeb649b7895393e7001aad8426d
+  data.tar.gz: 63bd24be4b7b41c539c0acd0c0c8bc6502095707539d235f8ddc9fc49aa2634c2fb602e40ce77e771d151945118d2c4da73d26bcda8c4290d61eae4b6c44b468

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 4.0.1
+
+- Added support for scroll API
+- Made type optional for custom mapping for Elasticsearch 6
+- Fixed error when suggestions empty
+- Fixed `models` option with inheritance
+
 ## 4.0.0
 
 - Added support for Elasticsearch 7

data/README.md

@@ -1044,10 +1044,10 @@ Visit the Shield page and reset your password. You’ll need to add the username
 heroku config:get FOUNDELASTICSEARCH_URL
 ```
 
-And add `elastic:password@` right after `https://`:
+And add `elastic:password@` right after `https://` and add port `9243` at the end:
 
 ```sh
-heroku config:set ELASTICSEARCH_URL=https://elastic:password@12345.us-east-1.aws.found.io
+heroku config:set ELASTICSEARCH_URL=https://elastic:password@12345.us-east-1.aws.found.io:9243
 ```
 
 Then deploy and reindex:
@@ -1396,10 +1396,8 @@ Create a custom mapping:
 ```ruby
 class Product < ApplicationRecord
   searchkick mappings: {
-    product: {
-      properties: {
-        name: {type: "keyword"}
-      }
+    properties: {
+      name: {type: "keyword"}
     }
   }
 end
@@ -1479,6 +1477,21 @@ Boost specific models with:
 indices_boost: {Category => 2, Product => 1}
 ```
 
+## Scroll API
+
+To retrieve a very large number of results, use the [scroll API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html).
+
+```ruby
+products = Product.search "*", scroll: "1m"
+while products.any?
+  # do something ...
+
+  products = products.scroll
+end
+```
+
+You should call `scroll` on each new set of results, not the original result.
+
 ## Nested Data
 
 To query nested data, use dot notation.

data/lib/searchkick.rb

@@ -67,7 +67,7 @@ module Searchkick
   end
 
   def self.search_timeout
-    @search_timeout || timeout
+    (defined?(@search_timeout) && @search_timeout) || timeout
   end
 
   def self.server_version

data/lib/searchkick/index_options.rb

@@ -5,18 +5,29 @@ module Searchkick
       language = options[:language]
       language = language.call if language.respond_to?(:call)
 
+      below62 = Searchkick.server_below?("6.2.0")
+      below70 = Searchkick.server_below?("7.0.0")
+
+      if below70
+        index_type = options[:_type]
+        index_type = index_type.call if index_type.respond_to?(:call)
+      end
+
+      custom_mapping = options[:mapping] || {}
+      if below70 && custom_mapping.keys.map(&:to_sym).include?(:properties)
+        # add type
+        custom_mapping = {index_type => custom_mapping}
+      end
+
       if options[:mappings] && !options[:merge_mappings]
         settings = options[:settings] || {}
-        mappings = options[:mappings]
+        mappings = custom_mapping
       else
-        below62 = Searchkick.server_below?("6.2.0")
-        below70 = Searchkick.server_below?("7.0.0")
 
         default_type = "text"
         default_analyzer = :searchkick_index
         keyword_mapping = {type: "keyword"}
 
-        all = options.key?(:_all) ? options[:_all] : false
         index_true_value = true
         index_false_value = false
 
@@ -412,12 +423,10 @@ module Searchkick
         }
 
         if below70
-          index_type = options[:_type]
-          index_type = index_type.call if index_type.respond_to?(:call)
           mappings = {index_type => mappings}
         end
 
-        mappings = mappings.symbolize_keys.deep_merge((options[:mappings] || {}).symbolize_keys)
+        mappings = mappings.symbolize_keys.deep_merge(custom_mapping.symbolize_keys)
       end
 
       {

data/lib/searchkick/logging.rb

@@ -167,7 +167,9 @@ module Searchkick
 
       # no easy way to tell which host the client will use
       host = Searchkick.client.transport.hosts.first
-      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -H 'Content-Type: application/json' -d '#{payload[:query][:body].to_json}'"
+      params = ["pretty"]
+      params << "scroll=#{payload[:query][:scroll]}" if payload[:query][:scroll]
+      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?#{params.join('&')} -H 'Content-Type: application/json' -d '#{payload[:query][:body].to_json}'"
     end
 
     def request(event)

data/lib/searchkick/query.rb

@@ -12,14 +12,14 @@ module Searchkick
       :took, :error, :model_name, :entry_name, :total_count, :total_entries,
       :current_page, :per_page, :limit_value, :padding, :total_pages, :num_pages,
       :offset_value, :offset, :previous_page, :prev_page, :next_page, :first_page?, :last_page?,
-      :out_of_range?, :hits, :response, :to_a, :first
+      :out_of_range?, :hits, :response, :to_a, :first, :scroll
 
     def initialize(klass, term = "*", **options)
       unknown_keywords = options.keys - [:aggs, :block, :body, :body_options, :boost,
         :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_term, :debug, :emoji, :exclude, :execute, :explain,
         :fields, :highlight, :includes, :index_name, :indices_boost, :limit, :load,
         :match, :misspellings, :models, :model_includes, :offset, :operator, :order, :padding, :page, :per_page, :profile,
-        :request_params, :routing, :scope_results, :select, :similar, :smart_aggs, :suggest, :total_entries, :track, :type, :where]
+        :request_params, :routing, :scope_results, :scroll, :select, :similar, :smart_aggs, :suggest, :total_entries, :track, :type, :where]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
 
       term = term.to_s
@@ -60,7 +60,8 @@ module Searchkick
       if options[:models]
         @index_mapping = {}
         Array(options[:models]).each do |model|
-          @index_mapping[model.searchkick_index.name] = model
+          # there can be multiple models per index name due to inheritance - see #1259
+          (@index_mapping[model.searchkick_index.name] ||= []) << model
         end
       end
 
@@ -81,6 +82,7 @@ module Searchkick
       }
       params[:type] = @type if @type
       params[:routing] = @routing if @routing
+      params[:scroll] = @scroll if @scroll
       params.merge!(options[:request_params]) if options[:request_params]
       params
     end
@@ -108,7 +110,9 @@ module Searchkick
       # no easy way to tell which host the client will use
       host = Searchkick.client.transport.hosts.first
       credentials = host[:user] || host[:password] ? "#{host[:user]}:#{host[:password]}@" : nil
-      "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -H 'Content-Type: application/json' -d '#{query[:body].to_json}'"
+      params = ["pretty"]
+      params << "scroll=#{options[:scroll]}" if options[:scroll]
+      "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?#{params.join('&')} -H 'Content-Type: application/json' -d '#{query[:body].to_json}'"
     end
 
     def handle_response(response)
@@ -127,7 +131,9 @@ module Searchkick
         term: term,
         scope_results: options[:scope_results],
         total_entries: options[:total_entries],
-        index_mapping: @index_mapping
+        index_mapping: @index_mapping,
+        suggest: options[:suggest],
+        scroll: options[:scroll]
       }
 
       if options[:debug]
@@ -228,6 +234,7 @@ module Searchkick
       per_page = (options[:limit] || options[:per_page] || 10_000).to_i
       padding = [options[:padding].to_i, 0].max
       offset = options[:offset] || (page - 1) * per_page + padding
+      scroll = options[:scroll]
 
       # model and eager loading
       load = options[:load].nil? ? true : options[:load]
@@ -423,6 +430,21 @@ module Searchkick
           where[:type] = [options[:type] || klass].flatten.map { |v| searchkick_index.klass_document_type(v, true) }
         end
 
+        models = Array(options[:models])
+        if models.any? { |m| m != m.searchkick_klass }
+          warn "[searchkick] WARNING: Passing child models to models option throws off hits and pagination - use type option instead"
+
+          # uncomment once aliases are supported with _index
+          # index_type_or =
+          #   models.map do |m|
+          #     v = {_index: m.searchkick_index.name}
+          #     v[:type] = m.searchkick_index.klass_document_type(m, true) if m != m.searchkick_klass
+          #     v
+          #   end
+
+          # where[:or] = Array(where[:or]) + [index_type_or]
+        end
+
         # start everything as efficient filters
         # move to post_filters as aggs demand
         filters = where_filters(where)
@@ -480,7 +502,7 @@ module Searchkick
       pagination_options = options[:page] || options[:limit] || options[:per_page] || options[:offset] || options[:padding]
       if !options[:body] || pagination_options
         payload[:size] = per_page
-        payload[:from] = offset
+        payload[:from] = offset if offset > 0
       end
 
       # type
@@ -497,11 +519,18 @@ module Searchkick
       # run block
       options[:block].call(payload) if options[:block]
 
+      # scroll optimization when interating over all docs
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html
+      if options[:scroll] && payload[:query] == {match_all: {}}
+        payload[:sort] ||= ["_doc"]
+      end
+
       @body = payload
       @page = page
       @per_page = per_page
       @padding = padding
       @load = load
+      @scroll = scroll
     end
 
     def set_fields

data/lib/searchkick/results.rb

@@ -26,15 +26,19 @@ module Searchkick
           results = {}
 
           hits.group_by { |hit, _| hit["_index"] }.each do |index, grouped_hits|
-            klass =
+            klasses =
               if @klass
-                @klass
+                [@klass]
               else
                 index_alias = index.split("_")[0..-2].join("_")
-                (options[:index_mapping] || {})[index_alias]
+                Array((options[:index_mapping] || {})[index_alias])
               end
-            raise Searchkick::Error, "Unknown model for index: #{index}" unless klass
-            results[index] = results_query(klass, grouped_hits).to_a.index_by { |r| r.id.to_s }
+            raise Searchkick::Error, "Unknown model for index: #{index}" unless klasses.any?
+
+            results[index] = {}
+            klasses.each do |klass|
+              results[index].merge!(results_query(klass, grouped_hits).to_a.index_by { |r| r.id.to_s })
+            end
           end
 
           missing_ids = []
@@ -90,7 +94,7 @@ module Searchkick
     def suggestions
       if response["suggest"]
         response["suggest"].values.flat_map { |v| v.first["options"] }.sort_by { |o| -o["score"] }.map { |o| o["text"] }.uniq
-      elsif options[:term] == "*"
+      elsif options[:suggest] || options[:term] == "*" # TODO remove 2nd term
         []
       else
         raise "Pass `suggest: true` to the search method for suggestions"
@@ -217,6 +221,29 @@ module Searchkick
       @options[:misspellings]
     end
 
+    def scroll_id
+      @response["_scroll_id"]
+    end
+
+    def scroll
+      raise Searchkick::Error, "Pass `scroll` option to the search method for scrolling" unless scroll_id
+
+      params = {
+        scroll: options[:scroll],
+        scroll_id: scroll_id
+      }
+
+      begin
+        Searchkick::Results.new(@klass, Searchkick.client.scroll(params), @options)
+      rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
+        if e.class.to_s =~ /NotFound/ && e.message =~ /search_context_missing_exception/i
+          raise Searchkick::Error, "Scroll id has expired"
+        else
+          raise e
+        end
+      end
+    end
+
     private
 
     def results_query(records, hits)

data/lib/searchkick/version.rb

@@ -1,3 +1,3 @@
 module Searchkick
-  VERSION = "4.0.0"
+  VERSION = "4.0.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: searchkick
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.0.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-12 00:00:00.000000000 Z
+date: 2019-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel