searchkick 4.0.0 → 4.4.1

data/lib/searchkick/logging.rb

@@ -132,7 +132,7 @@ module Searchkick
     def multi_search(searches)
       event = {
         name: "Multi Search",
-        body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join
+        body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join,
       }
       ActiveSupport::Notifications.instrument("multi_search.searchkick", event) do
         super
@@ -162,12 +162,17 @@ module Searchkick
 
       payload = event.payload
       name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
-      type = payload[:query][:type]
+
       index = payload[:query][:index].is_a?(Array) ? payload[:query][:index].join(",") : payload[:query][:index]
+      type = payload[:query][:type]
+      request_params = payload[:query].except(:index, :type, :body)
+
+      params = []
+      request_params.each do |k, v|
+        params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
+      end
 
-      # no easy way to tell which host the client will use
-      host = Searchkick.client.transport.hosts.first
-      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -H 'Content-Type: application/json' -d '#{payload[:query][:body].to_json}'"
+      debug "  #{color(name, YELLOW, true)}  #{index}#{type ? "/#{type.join(',')}" : ''}/_search#{params.any? ? '?' + params.join('&') : nil} #{payload[:query][:body].to_json}"
     end
 
     def request(event)
@@ -187,9 +192,7 @@ module Searchkick
       payload = event.payload
       name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
 
-      # no easy way to tell which host the client will use
-      host = Searchkick.client.transport.hosts.first
-      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -H 'Content-Type: application/json' -d '#{payload[:body]}'"
+      debug "  #{color(name, YELLOW, true)}  _msearch #{payload[:body]}"
     end
   end
 

data/lib/searchkick/model.rb

@@ -3,10 +3,10 @@ module Searchkick
     def searchkick(**options)
       options = Searchkick.model_options.merge(options)
 
-      unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :default_fields,
+      unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
         :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :language,
-        :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :settings, :similarity,
-        :special_characters, :stem, :stem_conversions, :suggest, :synonyms, :text_end,
+        :locations, :mappings, :match, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
+        :special_characters, :stem, :stem_conversions, :stem_exclusion, :stemmer_override, :suggest, :synonyms, :text_end,
         :text_middle, :text_start, :word, :wordnet, :word_end, :word_middle, :word_start]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
 
@@ -15,6 +15,7 @@ module Searchkick
       Searchkick.models << self
 
       options[:_type] ||= -> { searchkick_index.klass_document_type(self, true) }
+      options[:class_name] = model_name.name
 
       callbacks = options.key?(:callbacks) ? options[:callbacks] : :inline
       unless [:inline, true, false, :async, :queue].include?(callbacks)
@@ -40,12 +41,15 @@ module Searchkick
 
         class << self
           def searchkick_search(term = "*", **options, &block)
-            Searchkick.search(term, {model: self}.merge(options), &block)
+            # TODO throw error in next major version
+            Searchkick.warn("calling search on a relation is deprecated") if Searchkick.relation?(self)
+
+            Searchkick.search(term, model: self, **options, &block)
           end
           alias_method Searchkick.search_method_name, :searchkick_search if Searchkick.search_method_name
 
-          def searchkick_index
-            index = class_variable_get(:@@searchkick_index)
+          def searchkick_index(name: nil)
+            index = name || class_variable_get(:@@searchkick_index)
             index = index.call if index.respond_to?(:call)
             index_cache = class_variable_get(:@@searchkick_index_cache)
             index_cache[index] ||= Searchkick::Index.new(index, searchkick_options)
@@ -53,10 +57,11 @@ module Searchkick
           alias_method :search_index, :searchkick_index unless method_defined?(:search_index)
 
           def searchkick_reindex(method_name = nil, **options)
-            scoped = (respond_to?(:current_scope) && respond_to?(:default_scoped) && current_scope && current_scope.to_sql != default_scoped.to_sql) ||
+            # TODO relation = Searchkick.relation?(self)
+            relation = (respond_to?(:current_scope) && respond_to?(:default_scoped) && current_scope && current_scope.to_sql != default_scoped.to_sql) ||
               (respond_to?(:queryable) && queryable != unscoped.with_default_scope)
 
-            searchkick_index.reindex(searchkick_klass, method_name, scoped: scoped, **options)
+            searchkick_index.reindex(searchkick_klass, method_name, scoped: relation, **options)
           end
           alias_method :reindex, :searchkick_reindex unless method_defined?(:reindex)
 
@@ -78,8 +83,9 @@ module Searchkick
           RecordIndexer.new(self).reindex(method_name, **options)
         end unless method_defined?(:reindex)
 
+        # TODO switch to keyword arguments
         def similar(options = {})
-          self.class.searchkick_index.similar_record(self, options)
+          self.class.searchkick_index.similar_record(self, **options)
         end unless method_defined?(:similar)
 
         def search_data

data/lib/searchkick/process_batch_job.rb

@@ -2,7 +2,7 @@ module Searchkick
   class ProcessBatchJob < ActiveJob::Base
     queue_as { Searchkick.queue_name }
 
-    def perform(class_name:, record_ids:)
+    def perform(class_name:, record_ids:, index_name: nil)
       # separate routing from id
       routing = Hash[record_ids.map { |r| r.split(/(?<!\|)\|(?!\|)/, 2).map { |v| v.gsub("||", "|") } }]
       record_ids = routing.keys
@@ -26,7 +26,7 @@ module Searchkick
       end
 
       # bulk reindex
-      index = klass.searchkick_index
+      index = klass.searchkick_index(name: index_name)
       Searchkick.callbacks(:bulk) do
         index.bulk_index(records) if records.any?
         index.bulk_delete(delete_records) if delete_records.any?

data/lib/searchkick/process_queue_job.rb

@@ -2,21 +2,29 @@ module Searchkick
   class ProcessQueueJob < ActiveJob::Base
     queue_as { Searchkick.queue_name }
 
-    def perform(class_name:)
+    def perform(class_name:, index_name: nil, inline: false)
       model = class_name.constantize
+      limit = model.searchkick_options[:batch_size] || 1000
 
-      limit = model.searchkick_index.options[:batch_size] || 1000
-      record_ids = model.searchkick_index.reindex_queue.reserve(limit: limit)
-      if record_ids.any?
-        Searchkick::ProcessBatchJob.perform_later(
-          class_name: model.name,
-          record_ids: record_ids
-        )
-        # TODO when moving to reliable queuing, mark as complete
+      loop do
+        record_ids = model.searchkick_index(name: index_name).reindex_queue.reserve(limit: limit)
+        if record_ids.any?
+          batch_options = {
+            class_name: class_name,
+            record_ids: record_ids,
+            index_name: index_name
+          }
 
-        if record_ids.size == limit
-          Searchkick::ProcessQueueJob.perform_later(class_name: class_name)
+          if inline
+            # use new.perform to avoid excessive logging
+            Searchkick::ProcessBatchJob.new.perform(**batch_options)
+          else
+            Searchkick::ProcessBatchJob.perform_later(**batch_options)
+          end
+
+          # TODO when moving to reliable queuing, mark as complete
         end
+        break unless record_ids.size == limit
       end
     end
   end

data/lib/searchkick/query.rb

@@ -12,14 +12,14 @@ module Searchkick
       :took, :error, :model_name, :entry_name, :total_count, :total_entries,
       :current_page, :per_page, :limit_value, :padding, :total_pages, :num_pages,
       :offset_value, :offset, :previous_page, :prev_page, :next_page, :first_page?, :last_page?,
-      :out_of_range?, :hits, :response, :to_a, :first
+      :out_of_range?, :hits, :response, :to_a, :first, :scroll
 
     def initialize(klass, term = "*", **options)
       unknown_keywords = options.keys - [:aggs, :block, :body, :body_options, :boost,
         :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_term, :debug, :emoji, :exclude, :execute, :explain,
         :fields, :highlight, :includes, :index_name, :indices_boost, :limit, :load,
         :match, :misspellings, :models, :model_includes, :offset, :operator, :order, :padding, :page, :per_page, :profile,
-        :request_params, :routing, :scope_results, :select, :similar, :smart_aggs, :suggest, :total_entries, :track, :type, :where]
+        :request_params, :routing, :scope_results, :scroll, :select, :similar, :smart_aggs, :suggest, :total_entries, :track, :type, :where]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
 
       term = term.to_s
@@ -60,7 +60,8 @@ module Searchkick
       if options[:models]
         @index_mapping = {}
         Array(options[:models]).each do |model|
-          @index_mapping[model.searchkick_index.name] = model
+          # there can be multiple models per index name due to inheritance - see #1259
+          (@index_mapping[model.searchkick_index.name] ||= []) << model
         end
       end
 
@@ -81,6 +82,7 @@ module Searchkick
       }
       params[:type] = @type if @type
       params[:routing] = @routing if @routing
+      params[:scroll] = @scroll if @scroll
       params.merge!(options[:request_params]) if options[:request_params]
       params
     end
@@ -104,11 +106,16 @@ module Searchkick
       query = params
       type = query[:type]
       index = query[:index].is_a?(Array) ? query[:index].join(",") : query[:index]
+      request_params = query.except(:index, :type, :body)
 
       # no easy way to tell which host the client will use
       host = Searchkick.client.transport.hosts.first
       credentials = host[:user] || host[:password] ? "#{host[:user]}:#{host[:password]}@" : nil
-      "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -H 'Content-Type: application/json' -d '#{query[:body].to_json}'"
+      params = ["pretty"]
+      request_params.each do |k, v|
+        params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
+      end
+      "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?#{params.join('&')} -H 'Content-Type: application/json' -d '#{query[:body].to_json}'"
     end
 
     def handle_response(response)
@@ -127,10 +134,13 @@ module Searchkick
         term: term,
         scope_results: options[:scope_results],
         total_entries: options[:total_entries],
-        index_mapping: @index_mapping
+        index_mapping: @index_mapping,
+        suggest: options[:suggest],
+        scroll: options[:scroll]
       }
 
       if options[:debug]
+        # can remove when minimum Ruby version is 2.5
         require "pp"
 
         puts "Searchkick Version: #{Searchkick::VERSION}"
@@ -225,9 +235,12 @@ module Searchkick
 
       # pagination
       page = [options[:page].to_i, 1].max
-      per_page = (options[:limit] || options[:per_page] || 10_000).to_i
+      # maybe use index.max_result_window in the future
+      default_limit = searchkick_options[:deep_paging] ? 1_000_000_000 : 10_000
+      per_page = (options[:limit] || options[:per_page] || default_limit).to_i
       padding = [options[:padding].to_i, 0].max
       offset = options[:offset] || (page - 1) * per_page + padding
+      scroll = options[:scroll]
 
       # model and eager loading
       load = options[:load].nil? ? true : options[:load]
@@ -337,7 +350,7 @@ module Searchkick
             field_misspellings = misspellings && (!misspellings_fields || misspellings_fields.include?(base_field(field)))
 
             if field == "_all" || field.end_with?(".analyzed")
-              shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false
+              shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false || (!below73? && !track_total_hits?)
               qs << shared_options.merge(analyzer: "searchkick_search")
 
               # searchkick_search and searchkick_search2 are the same for ukrainian
@@ -423,6 +436,24 @@ module Searchkick
           where[:type] = [options[:type] || klass].flatten.map { |v| searchkick_index.klass_document_type(v, true) }
         end
 
+        models = Array(options[:models])
+        if models.any? { |m| m != m.searchkick_klass }
+          # aliases are not supported with _index in ES below 7.5
+          # see https://github.com/elastic/elasticsearch/pull/46640
+          if below75?
+            Searchkick.warn("Passing child models to models option throws off hits and pagination - use type option instead")
+          else
+            index_type_or =
+              models.map do |m|
+                v = {_index: m.searchkick_index.name}
+                v[:type] = m.searchkick_index.klass_document_type(m, true) if m != m.searchkick_klass
+                v
+              end
+
+            where[:or] = Array(where[:or]) + [index_type_or]
+          end
+        end
+
         # start everything as efficient filters
         # move to post_filters as aggs demand
         filters = where_filters(where)
@@ -480,7 +511,7 @@ module Searchkick
       pagination_options = options[:page] || options[:limit] || options[:per_page] || options[:offset] || options[:padding]
       if !options[:body] || pagination_options
         payload[:size] = per_page
-        payload[:from] = offset
+        payload[:from] = offset if offset > 0
       end
 
       # type
@@ -491,17 +522,28 @@ module Searchkick
       # routing
       @routing = options[:routing] if options[:routing]
 
+      if track_total_hits?
+        payload[:track_total_hits] = true
+      end
+
       # merge more body options
       payload = payload.deep_merge(options[:body_options]) if options[:body_options]
 
       # run block
       options[:block].call(payload) if options[:block]
 
+      # scroll optimization when interating over all docs
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html
+      if options[:scroll] && payload[:query] == {match_all: {}}
+        payload[:sort] ||= ["_doc"]
+      end
+
       @body = payload
       @page = page
       @per_page = per_page
       @padding = padding
       @load = load
+      @scroll = scroll
     end
 
     def set_fields
@@ -532,7 +574,8 @@ module Searchkick
 
     def build_query(query, filters, should, must_not, custom_filters, multiply_filters)
       if filters.any? || must_not.any? || should.any?
-        bool = {must: query}
+        bool = {}
+        bool[:must] = query if query
         bool[:filter] = filters if filters.any?      # where
         bool[:must_not] = must_not if must_not.any?  # exclude
         bool[:should] = should if should.any?        # conversions
@@ -829,6 +872,11 @@ module Searchkick
     end
 
     def where_filters(where)
+      # if where.respond_to?(:permitted?) && !where.permitted?
+      #   # TODO check in more places
+      #   Searchkick.warn("Passing unpermitted parameters will raise an exception in Searchkick 5")
+      # end
+
       filters = []
       (where || {}).each do |field, value|
         field = :_id if field.to_s == "id"
@@ -843,6 +891,8 @@ module Searchkick
           filters << {bool: {must_not: where_filters(value)}}
         elsif field == :_and
           filters << {bool: {must: value.map { |or_statement| {bool: {filter: where_filters(or_statement)}} }}}
+        # elsif field == :_script
+        #   filters << {script: {script: {source: value, lang: "painless"}}}
         else
           # expand ranges
           if value.is_a?(Range)
@@ -903,8 +953,23 @@ module Searchkick
                     }
                   }
                 }
+              when :like
+                # based on Postgres
+                # https://www.postgresql.org/docs/current/functions-matching.html
+                # % matches zero or more characters
+                # _ matches one character
+                # \ is escape character
+                # escape Lucene reserved characters
+                # https://www.elastic.co/guide/en/elasticsearch/reference/current/regexp-syntax.html#regexp-optional-operators
+                reserved = %w(. ? + * | { } [ ] ( ) " \\)
+                regex = op_value.dup
+                reserved.each do |v|
+                  regex.gsub!(v, "\\" + v)
+                end
+                regex = regex.gsub(/(?<!\\)%/, ".*").gsub(/(?<!\\)_/, ".").gsub("\\%", "%").gsub("\\_", "_")
+                filters << {regexp: {field => {value: regex}}}
               when :prefix
-                filters << {prefix: {field => op_value}}
+                filters << {prefix: {field => {value: op_value}}}
               when :regexp # support for regexp queries without using a regexp ruby object
                 filters << {regexp: {field => {value: op_value}}}
               when :not, :_not # not equal
@@ -915,6 +980,8 @@ module Searchkick
                 end
               when :in
                 filters << term_filters(field, op_value)
+              when :exists
+                filters << {exists: {field: field}}
               else
                 range_query =
                   case op
@@ -955,9 +1022,43 @@ module Searchkick
       elsif value.nil?
         {bool: {must_not: {exists: {field: field}}}}
       elsif value.is_a?(Regexp)
-        {regexp: {field => {value: value.source, flags: "NONE"}}}
+        if value.casefold?
+          Searchkick.warn("Case-insensitive flag does not work with Elasticsearch")
+        end
+
+        source = value.source
+        unless source.start_with?("\\A") && source.end_with?("\\z")
+          # https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-regexp-query.html
+          Searchkick.warn("Regular expressions are always anchored in Elasticsearch")
+        end
+
+        # TODO handle other anchor characters, like ^, $, \Z
+        if source.start_with?("\\A")
+          source = source[2..-1]
+        else
+          # TODO uncomment in Searchkick 5
+          # source = ".*#{source}"
+        end
+
+        if source.end_with?("\\z")
+          source = source[0..-3]
+        else
+          # TODO uncomment in Searchkick 5
+          # source = "#{source}.*"
+        end
+
+        {regexp: {field => {value: source, flags: "NONE"}}}
       else
-        {term: {field => value}}
+        # TODO add this for other values
+        if value.as_json.is_a?(Enumerable)
+          # query will fail, but this is better
+          # same message as Active Record
+          # TODO make TypeError
+          # raise InvalidQueryError for backward compatibility
+          raise Searchkick::InvalidQueryError, "can't cast #{value.class.name}"
+        end
+
+        {term: {field => {value: value}}}
       end
     end
 
@@ -1021,6 +1122,14 @@ module Searchkick
       k.sub(/\.(analyzed|word_start|word_middle|word_end|text_start|text_middle|text_end|exact)\z/, "")
     end
 
+    def track_total_hits?
+      (searchkick_options[:deep_paging] && !below70?) || body_options[:track_total_hits]
+    end
+
+    def body_options
+      options[:body_options] || {}
+    end
+
     def below61?
       Searchkick.server_below?("6.1.0")
     end
@@ -1028,5 +1137,13 @@ module Searchkick
     def below70?
       Searchkick.server_below?("7.0.0")
     end
+
+    def below73?
+      Searchkick.server_below?("7.3.0")
+    end
+
+    def below75?
+      Searchkick.server_below?("7.5.0")
+    end
   end
 end