searchkick 4.6.3 → 5.2.2

data/lib/searchkick/indexer.rb

@@ -1,3 +1,5 @@
+# thread-local (technically fiber-local) indexer
+# used to aggregate bulk callbacks across models
 module Searchkick
   class Indexer
     attr_reader :queued_items
@@ -14,15 +16,20 @@ module Searchkick
     def perform
       items = @queued_items
       @queued_items = []
-      if items.any?
-        response = Searchkick.client.bulk(body: items)
-        if response["errors"]
-          first_with_error = response["items"].map do |item|
-            (item["index"] || item["delete"] || item["update"])
-          end.find { |item| item["error"] }
-          raise Searchkick::ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
-        end
+
+      return if items.empty?
+
+      response = Searchkick.client.bulk(body: items)
+      if response["errors"]
+        # note: delete does not set error when item not found
+        first_with_error = response["items"].map do |item|
+          (item["index"] || item["delete"] || item["update"])
+        end.find { |item| item["error"] }
+        raise ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
       end
+
+      # maybe return response in future
+      nil
     end
   end
 end

data/lib/searchkick/log_subscriber.rb

@@ -0,0 +1,57 @@
+# based on https://gist.github.com/mnutt/566725
+module Searchkick
+  class LogSubscriber < ActiveSupport::LogSubscriber
+    def self.runtime=(value)
+      Thread.current[:searchkick_runtime] = value
+    end
+
+    def self.runtime
+      Thread.current[:searchkick_runtime] ||= 0
+    end
+
+    def self.reset_runtime
+      rt = runtime
+      self.runtime = 0
+      rt
+    end
+
+    def search(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      index = payload[:query][:index].is_a?(Array) ? payload[:query][:index].join(",") : payload[:query][:index]
+      type = payload[:query][:type]
+      request_params = payload[:query].except(:index, :type, :body)
+
+      params = []
+      request_params.each do |k, v|
+        params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
+      end
+
+      debug "  #{color(name, YELLOW, bold: true)}  #{index}#{type ? "/#{type.join(',')}" : ''}/_search#{params.any? ? '?' + params.join('&') : nil} #{payload[:query][:body].to_json}"
+    end
+
+    def request(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      debug "  #{color(name, YELLOW, bold: true)}  #{payload.except(:name).to_json}"
+    end
+
+    def multi_search(event)
+      self.class.runtime += event.duration
+      return unless logger.debug?
+
+      payload = event.payload
+      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
+
+      debug "  #{color(name, YELLOW, bold: true)}  _msearch #{payload[:body]}"
+    end
+  end
+end

data/lib/searchkick/middleware.rb

@@ -1,10 +1,17 @@
-require "faraday/middleware"
+require "faraday"
 
 module Searchkick
   class Middleware < Faraday::Middleware
     def call(env)
-      if env[:method] == :get && env[:url].path.to_s.end_with?("/_search")
+      path = env[:url].path.to_s
+      if path.end_with?("/_search")
         env[:request][:timeout] = Searchkick.search_timeout
+      elsif path.end_with?("/_msearch")
+        # assume no concurrent searches for timeout for now
+        searches = env[:request_body].count("\n") / 2
+        # do not allow timeout to exceed Searchkick.timeout
+        timeout = [Searchkick.search_timeout * searches, Searchkick.timeout].min
+        env[:request][:timeout] = timeout
       end
       @app.call(env)
     end

data/lib/searchkick/model.rb

@@ -5,9 +5,9 @@ module Searchkick
 
       unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
         :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
-        :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
+        :locations, :mappings, :match, :max_result_window, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
         :special_characters, :stem, :stemmer, :stem_conversions, :stem_exclusion, :stemmer_override, :suggest, :synonyms, :text_end,
-        :text_middle, :text_start, :word, :wordnet, :word_end, :word_middle, :word_start]
+        :text_middle, :text_start, :unscope, :word, :word_end, :word_middle, :word_start]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
 
       raise "Only call searchkick once per model" if respond_to?(:searchkick_index)
@@ -22,52 +22,78 @@ module Searchkick
         raise ArgumentError, "Invalid value for callbacks"
       end
 
-      index_name =
-        if options[:index_name]
-          options[:index_name]
-        elsif options[:index_prefix].respond_to?(:call)
-          -> { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
-        else
-          [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
-        end
+      base = self
+
+      mod = Module.new
+      include(mod)
+      mod.module_eval do
+        def reindex(method_name = nil, mode: nil, refresh: false)
+          self.class.searchkick_index.reindex([self], method_name: method_name, mode: mode, refresh: refresh, single: true)
+        end unless base.method_defined?(:reindex)
+
+        def similar(**options)
+          self.class.searchkick_index.similar_record(self, **options)
+        end unless base.method_defined?(:similar)
+
+        def search_data
+          data = respond_to?(:to_hash) ? to_hash : serializable_hash
+          data.delete("id")
+          data.delete("_id")
+          data.delete("_type")
+          data
+        end unless base.method_defined?(:search_data)
+
+        def should_index?
+          true
+        end unless base.method_defined?(:should_index?)
+      end
 
       class_eval do
-        cattr_reader :searchkick_options, :searchkick_klass
+        cattr_reader :searchkick_options, :searchkick_klass, instance_reader: false
 
         class_variable_set :@@searchkick_options, options.dup
         class_variable_set :@@searchkick_klass, self
-        class_variable_set :@@searchkick_index, index_name
-        class_variable_set :@@searchkick_index_cache, {}
+        class_variable_set :@@searchkick_index_cache, Searchkick::IndexCache.new
 
         class << self
           def searchkick_search(term = "*", **options, &block)
-            # TODO throw error in next major version
-            Searchkick.warn("calling search on a relation is deprecated") if Searchkick.relation?(self)
+            if Searchkick.relation?(self)
+              raise Searchkick::Error, "search must be called on model, not relation"
+            end
 
             Searchkick.search(term, model: self, **options, &block)
           end
           alias_method Searchkick.search_method_name, :searchkick_search if Searchkick.search_method_name
 
           def searchkick_index(name: nil)
-            index = name || class_variable_get(:@@searchkick_index)
-            index = index.call if index.respond_to?(:call)
+            index_name = name || searchkick_klass.searchkick_index_name
+            index_name = index_name.call if index_name.respond_to?(:call)
             index_cache = class_variable_get(:@@searchkick_index_cache)
-            index_cache[index] ||= Searchkick::Index.new(index, searchkick_options)
+            index_cache.fetch(index_name) { Searchkick::Index.new(index_name, searchkick_options) }
           end
           alias_method :search_index, :searchkick_index unless method_defined?(:search_index)
 
           def searchkick_reindex(method_name = nil, **options)
-            # TODO relation = Searchkick.relation?(self)
-            relation = (respond_to?(:current_scope) && respond_to?(:default_scoped) && current_scope && current_scope.to_sql != default_scoped.to_sql) ||
-              (respond_to?(:queryable) && queryable != unscoped.with_default_scope)
-
-            searchkick_index.reindex(searchkick_klass, method_name, scoped: relation, **options)
+            searchkick_index.reindex(self, method_name: method_name, **options)
           end
           alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex)
 
           def searchkick_index_options
             searchkick_index.index_options
           end
+
+          def searchkick_index_name
+            @searchkick_index_name ||= begin
+              options = class_variable_get(:@@searchkick_options)
+              if options[:index_name]
+                options[:index_name]
+              elsif options[:index_prefix].respond_to?(:call)
+                -> { [options[:index_prefix].call, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_") }
+              else
+                [options.key?(:index_prefix) ? options[:index_prefix] : Searchkick.index_prefix, model_name.plural, Searchkick.env, Searchkick.index_suffix].compact.join("_")
+              end
+            end
+          end
         end
 
         # always add callbacks, even when callbacks is false
@@ -78,33 +104,6 @@ module Searchkick
           after_save :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
           after_destroy :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
         end
-
-        def reindex(method_name = nil, **options)
-          RecordIndexer.new(self).reindex(method_name, **options)
-        end unless method_defined?(:reindex)
-
-        # TODO switch to keyword arguments
-        def similar(options = {})
-          self.class.searchkick_index.similar_record(self, **options)
-        end unless method_defined?(:similar)
-
-        def search_data
-          data = respond_to?(:to_hash) ? to_hash : serializable_hash
-          data.delete("id")
-          data.delete("_id")
-          data.delete("_type")
-          data
-        end unless method_defined?(:search_data)
-
-        def should_index?
-          true
-        end unless method_defined?(:should_index?)
-
-        if defined?(Cequel) && self < Cequel::Record && !method_defined?(:destroyed?)
-          def destroyed?
-            transient?
-          end
-        end
       end
     end
   end

data/lib/searchkick/process_batch_job.rb

@@ -3,34 +3,18 @@ module Searchkick
     queue_as { Searchkick.queue_name }
 
     def perform(class_name:, record_ids:, index_name: nil)
-      # separate routing from id
-      routing = Hash[record_ids.map { |r| r.split(/(?<!\|)\|(?!\|)/, 2).map { |v| v.gsub("||", "|") } }]
-      record_ids = routing.keys
+      model = Searchkick.load_model(class_name)
+      index = model.searchkick_index(name: index_name)
 
-      klass = class_name.constantize
-      scope = Searchkick.load_records(klass, record_ids)
-      scope = scope.search_import if scope.respond_to?(:search_import)
-      records = scope.select(&:should_index?)
-
-      # determine which records to delete
-      delete_ids = record_ids - records.map { |r| r.id.to_s }
-      delete_records = delete_ids.map do |id|
-        m = klass.new
-        m.id = id
-        if routing[id]
-          m.define_singleton_method(:search_routing) do
-            routing[id]
-          end
+      items =
+        record_ids.map do |r|
+          parts = r.split(/(?<!\|)\|(?!\|)/, 2)
+            .map { |v| v.gsub("||", "|") }
+          {id: parts[0], routing: parts[1]}
         end
-        m
-      end
 
-      # bulk reindex
-      index = klass.searchkick_index(name: index_name)
-      Searchkick.callbacks(:bulk) do
-        index.bulk_index(records) if records.any?
-        index.bulk_delete(delete_records) if delete_records.any?
-      end
+      relation = Searchkick.scope(model)
+      RecordIndexer.new(index).reindex_items(relation, items, method_name: nil)
     end
   end
 end

data/lib/searchkick/process_queue_job.rb

@@ -3,11 +3,12 @@ module Searchkick
     queue_as { Searchkick.queue_name }
 
     def perform(class_name:, index_name: nil, inline: false)
-      model = class_name.constantize
+      model = Searchkick.load_model(class_name)
+      index = model.searchkick_index(name: index_name)
       limit = model.searchkick_options[:batch_size] || 1000
 
       loop do
-        record_ids = model.searchkick_index(name: index_name).reindex_queue.reserve(limit: limit)
+        record_ids = index.reindex_queue.reserve(limit: limit)
         if record_ids.any?
           batch_options = {
             class_name: class_name,

data/lib/searchkick/query.rb

@@ -18,7 +18,7 @@ module Searchkick
 
     def initialize(klass, term = "*", **options)
       unknown_keywords = options.keys - [:aggs, :block, :body, :body_options, :boost,
-        :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_term, :debug, :emoji, :exclude, :execute, :explain,
+        :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_term, :debug, :emoji, :exclude, :explain,
         :fields, :highlight, :includes, :index_name, :indices_boost, :limit, :load,
         :match, :misspellings, :models, :model_includes, :offset, :operator, :order, :padding, :page, :per_page, :profile,
         :request_params, :routing, :scope_results, :scroll, :select, :similar, :smart_aggs, :suggest, :total_entries, :track, :type, :where]
@@ -148,9 +148,6 @@ module Searchkick
       }
 
       if options[:debug]
-        # can remove when minimum Ruby version is 2.5
-        require "pp"
-
         puts "Searchkick Version: #{Searchkick::VERSION}"
         puts "Elasticsearch Version: #{Searchkick.server_version}"
         puts
@@ -190,11 +187,11 @@ module Searchkick
       end
 
       # set execute for multi search
-      @execute = Searchkick::Results.new(searchkick_klass, response, opts)
+      @execute = Results.new(searchkick_klass, response, opts)
     end
 
     def retry_misspellings?(response)
-      @misspellings_below && Searchkick::Results.new(searchkick_klass, response).total_count < @misspellings_below
+      @misspellings_below && Results.new(searchkick_klass, response).total_count < @misspellings_below
     end
 
     private
@@ -202,7 +199,11 @@ module Searchkick
     def handle_error(e)
       status_code = e.message[1..3].to_i
       if status_code == 404
-        raise MissingIndexError, "Index missing - run #{reindex_command}"
+        if e.message.include?("No search context found for id")
+          raise MissingIndexError, "No search context found for id"
+        else
+          raise MissingIndexError, "Index missing - run #{reindex_command}"
+        end
       elsif status_code == 500 && (
         e.message.include?("IllegalArgumentException[minimumSimilarity >= 1]") ||
         e.message.include?("No query registered for [multi_match]") ||
@@ -210,14 +211,14 @@ module Searchkick
         e.message.include?("No query registered for [function_score]")
       )
 
-        raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
+        raise UnsupportedVersionError
       elsif status_code == 400
         if (
           e.message.include?("bool query does not support [filter]") ||
           e.message.include?("[bool] filter does not support [filter]")
         )
 
-          raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
+          raise UnsupportedVersionError
         elsif e.message =~ /analyzer \[searchkick_.+\] not found/
           raise InvalidQueryError, "Bad mapping - run #{reindex_command}"
         else
@@ -233,7 +234,14 @@ module Searchkick
     end
 
     def execute_search
-      Searchkick.client.search(params)
+      name = searchkick_klass ? "#{searchkick_klass.name} Search" : "Search"
+      event = {
+        name: name,
+        query: params
+      }
+      ActiveSupport::Notifications.instrument("search.searchkick", event) do
+        Searchkick.client.search(params)
+      end
     end
 
     def prepare
@@ -247,9 +255,15 @@ module Searchkick
       default_limit = searchkick_options[:deep_paging] ? 1_000_000_000 : 10_000
       per_page = (options[:limit] || options[:per_page] || default_limit).to_i
       padding = [options[:padding].to_i, 0].max
-      offset = options[:offset] || (page - 1) * per_page + padding
+      offset = (options[:offset] || (page - 1) * per_page + padding).to_i
       scroll = options[:scroll]
 
+      max_result_window = searchkick_options[:max_result_window]
+      if max_result_window
+        offset = max_result_window if offset > max_result_window
+        per_page = max_result_window - offset if offset + per_page > max_result_window
+      end
+
       # model and eager loading
       load = options[:load].nil? ? true : options[:load]
 
@@ -268,9 +282,10 @@ module Searchkick
         should = []
 
         if options[:similar]
+          like = options[:similar] == true ? term : options[:similar]
           query = {
             more_like_this: {
-              like: term,
+              like: like,
               min_doc_freq: 1,
               min_term_freq: 1,
               analyzer: "searchkick_search2"
@@ -383,11 +398,6 @@ module Searchkick
 
             if field.start_with?("*.")
               q2 = qs.map { |q| {multi_match: q.merge(fields: [field], type: match_type == :match_phrase ? "phrase" : "best_fields")} }
-              if below61?
-                q2.each do |q|
-                  q[:multi_match].delete(:fuzzy_transpositions)
-                end
-              end
             else
               q2 = qs.map { |q| {match_type => {field => q}} }
             end
@@ -439,7 +449,7 @@ module Searchkick
         payload = {}
 
         # type when inheritance
-        where = (options[:where] || {}).dup
+        where = ensure_permitted(options[:where] || {}).dup
         if searchkick_options[:inheritance] && (options[:type] || (klass != searchkick_klass && searchkick_index))
           where[:type] = [options[:type] || klass].flatten.map { |v| searchkick_index.klass_document_type(v, true) }
         end
@@ -499,7 +509,7 @@ module Searchkick
         set_highlights(payload, fields) if options[:highlight]
 
         # timeout shortly after client times out
-        payload[:timeout] ||= "#{Searchkick.search_timeout + 1}s"
+        payload[:timeout] ||= "#{((Searchkick.search_timeout + 1) * 1000).round}ms"
 
         # An empty array will cause only the _id and _type for each hit to be returned
         # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-source-filtering.html
@@ -696,9 +706,9 @@ module Searchkick
     def set_boost_by(multiply_filters, custom_filters)
       boost_by = options[:boost_by] || {}
       if boost_by.is_a?(Array)
-        boost_by = Hash[boost_by.map { |f| [f, {factor: 1}] }]
+        boost_by = boost_by.to_h { |f| [f, {factor: 1}] }
       elsif boost_by.is_a?(Hash)
-        multiply_by, boost_by = boost_by.partition { |_, v| v.delete(:boost_mode) == "multiply" }.map { |i| Hash[i] }
+        multiply_by, boost_by = boost_by.partition { |_, v| v.delete(:boost_mode) == "multiply" }.map(&:to_h)
       end
       boost_by[options[:boost]] = {factor: 1} if options[:boost]
 
@@ -763,7 +773,7 @@ module Searchkick
 
     def set_highlights(payload, fields)
       payload[:highlight] = {
-        fields: Hash[fields.map { |f| [f, {}] }],
+        fields: fields.to_h { |f| [f, {}] },
         fragment_size: 0
       }
 
@@ -797,7 +807,7 @@ module Searchkick
       aggs = options[:aggs]
       payload[:aggs] = {}
 
-      aggs = Hash[aggs.map { |f| [f, {}] }] if aggs.is_a?(Array) # convert to more advanced syntax
+      aggs = aggs.to_h { |f| [f, {}] } if aggs.is_a?(Array) # convert to more advanced syntax
       aggs.each do |field, agg_options|
         size = agg_options[:limit] ? agg_options[:limit] : 1_000
         shared_agg_options = agg_options.except(:limit, :field, :ranges, :date_ranges, :where)
@@ -836,8 +846,9 @@ module Searchkick
         end
 
         where = {}
-        where = (options[:where] || {}).reject { |k| k == field } unless options[:smart_aggs] == false
-        agg_filters = where_filters(where.merge(agg_options[:where] || {}))
+        where = ensure_permitted(options[:where] || {}).reject { |k| k == field } unless options[:smart_aggs] == false
+        agg_where = ensure_permitted(agg_options[:where] || {})
+        agg_filters = where_filters(where.merge(agg_where))
 
         # only do one level comparison for simplicity
         filters.select! do |filter|
@@ -873,19 +884,16 @@ module Searchkick
     end
 
     def set_order(payload)
-      order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
-      id_field = :_id
-      # TODO no longer map id to _id in Searchkick 5
-      # since sorting on _id is deprecated in Elasticsearch
-      payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? id_field : k, v] }]
+      payload[:sort] = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
     end
 
-    def where_filters(where)
-      # if where.respond_to?(:permitted?) && !where.permitted?
-      #   # TODO check in more places
-      #   Searchkick.warn("Passing unpermitted parameters will raise an exception in Searchkick 5")
-      # end
+    # provides *very* basic protection from unfiltered parameters
+    # this is not meant to be comprehensive and may be expanded in the future
+    def ensure_permitted(obj)
+      obj.to_h
+    end
 
+    def where_filters(where)
       filters = []
       (where || {}).each do |field, value|
         field = :_id if field.to_s == "id"
@@ -1007,7 +1015,7 @@ module Searchkick
                   when :lte
                     {to: op_value, include_upper: true}
                   else
-                    raise "Unknown where operator: #{op.inspect}"
+                    raise ArgumentError, "Unknown where operator: #{op.inspect}"
                   end
                 # issue 132
                 if (existing = filters.find { |f| f[:range] && f[:range][field] })
@@ -1036,29 +1044,25 @@ module Searchkick
         {bool: {must_not: {exists: {field: field}}}}
       elsif value.is_a?(Regexp)
         source = value.source
-        unless source.start_with?("\\A") && source.end_with?("\\z")
-          # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-regexp-query.html
-          Searchkick.warn("Regular expressions are always anchored in Elasticsearch")
-        end
+
+        # TODO handle other regexp options
 
         # TODO handle other anchor characters, like ^, $, \Z
         if source.start_with?("\\A")
           source = source[2..-1]
         else
-          # TODO uncomment in Searchkick 5
-          # source = ".*#{source}"
+          source = ".*#{source}"
         end
 
         if source.end_with?("\\z")
           source = source[0..-3]
         else
-          # TODO uncomment in Searchkick 5
-          # source = "#{source}.*"
+          source = "#{source}.*"
         end
 
         if below710?
           if value.casefold?
-            Searchkick.warn("Case-insensitive flag does not work with Elasticsearch < 7.10")
+            raise ArgumentError, "Case-insensitive flag does not work with Elasticsearch < 7.10"
           end
           {regexp: {field => {value: source, flags: "NONE"}}}
         else
@@ -1069,9 +1073,7 @@ module Searchkick
         if value.as_json.is_a?(Enumerable)
           # query will fail, but this is better
           # same message as Active Record
-          # TODO make TypeError
-          # raise InvalidQueryError for backward compatibility
-          raise Searchkick::InvalidQueryError, "can't cast #{value.class.name}"
+          raise TypeError, "can't cast #{value.class.name}"
         end
 
         {term: {field => {value: value}}}
@@ -1150,21 +1152,13 @@ module Searchkick
     end
 
     def track_total_hits?
-      (searchkick_options[:deep_paging] && !below70?) || body_options[:track_total_hits]
+      searchkick_options[:deep_paging] || body_options[:track_total_hits]
     end
 
     def body_options
       options[:body_options] || {}
     end
 
-    def below61?
-      Searchkick.server_below?("6.1.0")
-    end
-
-    def below70?
-      Searchkick.server_below?("7.0.0")
-    end
-
     def below73?
       Searchkick.server_below?("7.3.0")
     end

data/lib/searchkick/record_data.rb

@@ -25,6 +25,7 @@ module Searchkick
       {delete: record_data}
     end
 
+    # custom id can be useful for load: false
     def search_id
       id = record.respond_to?(:search_document_id) ? record.search_document_id : record.id
       id.is_a?(Numeric) ? id : id.to_s
@@ -39,7 +40,6 @@ module Searchkick
         _index: index.name,
         _id: search_id
       }
-      data[:_type] = document_type if Searchkick.server_below7?
       data[:routing] = record.search_routing if record.respond_to?(:search_routing)
       data
     end