searchkick 4.0.0 → 5.0.0

data/lib/searchkick/query.rb

@@ -1,5 +1,6 @@
 module Searchkick
   class Query
+    include Enumerable
     extend Forwardable
 
     @@metric_aggs = [:avg, :cardinality, :max, :min, :sum]
@@ -12,20 +13,21 @@ module Searchkick
       :took, :error, :model_name, :entry_name, :total_count, :total_entries,
       :current_page, :per_page, :limit_value, :padding, :total_pages, :num_pages,
       :offset_value, :offset, :previous_page, :prev_page, :next_page, :first_page?, :last_page?,
-      :out_of_range?, :hits, :response, :to_a, :first
+      :out_of_range?, :hits, :response, :to_a, :first, :scroll, :highlights, :with_highlights,
+      :with_score, :misspellings?, :scroll_id, :clear_scroll, :missing_records, :with_hit
 
     def initialize(klass, term = "*", **options)
       unknown_keywords = options.keys - [:aggs, :block, :body, :body_options, :boost,
-        :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_term, :debug, :emoji, :exclude, :execute, :explain,
+        :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_term, :debug, :emoji, :exclude, :explain,
         :fields, :highlight, :includes, :index_name, :indices_boost, :limit, :load,
         :match, :misspellings, :models, :model_includes, :offset, :operator, :order, :padding, :page, :per_page, :profile,
-        :request_params, :routing, :scope_results, :select, :similar, :smart_aggs, :suggest, :total_entries, :track, :type, :where]
+        :request_params, :routing, :scope_results, :scroll, :select, :similar, :smart_aggs, :suggest, :total_entries, :track, :type, :where]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
 
       term = term.to_s
 
       if options[:emoji]
-        term = EmojiParser.parse_unicode(term) { |e| " #{e.name} " }.strip
+        term = EmojiParser.parse_unicode(term) { |e| " #{e.name.tr('_', ' ')} " }.strip
       end
 
       @klass = klass
@@ -60,7 +62,8 @@ module Searchkick
       if options[:models]
         @index_mapping = {}
         Array(options[:models]).each do |model|
-          @index_mapping[model.searchkick_index.name] = model
+          # there can be multiple models per index name due to inheritance - see #1259
+          (@index_mapping[model.searchkick_index.name] ||= []) << model
         end
       end
 
@@ -72,7 +75,8 @@ module Searchkick
         elsif searchkick_index
           searchkick_index.name
         else
-          "_all"
+          # fixes warning about accessing system indices
+          "*,-.*"
         end
 
       params = {
@@ -81,6 +85,7 @@ module Searchkick
       }
       params[:type] = @type if @type
       params[:routing] = @routing if @routing
+      params[:scroll] = @scroll if @scroll
       params.merge!(options[:request_params]) if options[:request_params]
       params
     end
@@ -104,11 +109,21 @@ module Searchkick
       query = params
       type = query[:type]
       index = query[:index].is_a?(Array) ? query[:index].join(",") : query[:index]
+      request_params = query.except(:index, :type, :body)
 
       # no easy way to tell which host the client will use
-      host = Searchkick.client.transport.hosts.first
+      host =
+        if Searchkick.client.transport.respond_to?(:transport)
+          Searchkick.client.transport.transport.hosts.first
+        else
+          Searchkick.client.transport.hosts.first
+        end
       credentials = host[:user] || host[:password] ? "#{host[:user]}:#{host[:password]}@" : nil
-      "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -H 'Content-Type: application/json' -d '#{query[:body].to_json}'"
+      params = ["pretty"]
+      request_params.each do |k, v|
+        params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
+      end
+      "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?#{params.join('&')} -H 'Content-Type: application/json' -d '#{query[:body].to_json}'"
     end
 
     def handle_response(response)
@@ -127,12 +142,12 @@ module Searchkick
         term: term,
         scope_results: options[:scope_results],
         total_entries: options[:total_entries],
-        index_mapping: @index_mapping
+        index_mapping: @index_mapping,
+        suggest: options[:suggest],
+        scroll: options[:scroll]
       }
 
       if options[:debug]
-        require "pp"
-
         puts "Searchkick Version: #{Searchkick::VERSION}"
         puts "Elasticsearch Version: #{Searchkick.server_version}"
         puts
@@ -192,14 +207,14 @@ module Searchkick
         e.message.include?("No query registered for [function_score]")
       )
 
-        raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
+        raise UnsupportedVersionError
       elsif status_code == 400
         if (
           e.message.include?("bool query does not support [filter]") ||
           e.message.include?("[bool] filter does not support [filter]")
         )
 
-          raise UnsupportedVersionError, "This version of Searchkick requires Elasticsearch 5 or greater"
+          raise UnsupportedVersionError
         elsif e.message =~ /analyzer \[searchkick_.+\] not found/
           raise InvalidQueryError, "Bad mapping - run #{reindex_command}"
         else
@@ -215,7 +230,14 @@ module Searchkick
     end
 
     def execute_search
-      Searchkick.client.search(params)
+      name = searchkick_klass ? "#{searchkick_klass.name} Search" : "Search"
+      event = {
+        name: name,
+        query: params
+      }
+      ActiveSupport::Notifications.instrument("search.searchkick", event) do
+        Searchkick.client.search(params)
+      end
     end
 
     def prepare
@@ -225,9 +247,12 @@ module Searchkick
 
       # pagination
       page = [options[:page].to_i, 1].max
-      per_page = (options[:limit] || options[:per_page] || 10_000).to_i
+      # maybe use index.max_result_window in the future
+      default_limit = searchkick_options[:deep_paging] ? 1_000_000_000 : 10_000
+      per_page = (options[:limit] || options[:per_page] || default_limit).to_i
       padding = [options[:padding].to_i, 0].max
       offset = options[:offset] || (page - 1) * per_page + padding
+      scroll = options[:scroll]
 
       # model and eager loading
       load = options[:load].nil? ? true : options[:load]
@@ -247,9 +272,10 @@ module Searchkick
         should = []
 
         if options[:similar]
+          like = options[:similar] == true ? term : options[:similar]
           query = {
             more_like_this: {
-              like: term,
+              like: like,
               min_doc_freq: 1,
               min_term_freq: 1,
               analyzer: "searchkick_search2"
@@ -337,11 +363,11 @@ module Searchkick
             field_misspellings = misspellings && (!misspellings_fields || misspellings_fields.include?(base_field(field)))
 
             if field == "_all" || field.end_with?(".analyzed")
-              shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false
+              shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false || (!below73? && !track_total_hits?)
               qs << shared_options.merge(analyzer: "searchkick_search")
 
-              # searchkick_search and searchkick_search2 are the same for ukrainian
-              unless %w(japanese korean polish ukrainian vietnamese).include?(searchkick_options[:language])
+              # searchkick_search and searchkick_search2 are the same for some languages
+              unless %w(japanese japanese2 korean polish ukrainian vietnamese).include?(searchkick_options[:language])
                 qs << shared_options.merge(analyzer: "searchkick_search2")
               end
               exclude_analyzer = "searchkick_search2"
@@ -362,11 +388,6 @@ module Searchkick
 
             if field.start_with?("*.")
               q2 = qs.map { |q| {multi_match: q.merge(fields: [field], type: match_type == :match_phrase ? "phrase" : "best_fields")} }
-              if below61?
-                q2.each do |q|
-                  q[:multi_match].delete(:fuzzy_transpositions)
-                end
-              end
             else
               q2 = qs.map { |q| {match_type => {field => q}} }
             end
@@ -418,11 +439,29 @@ module Searchkick
         payload = {}
 
         # type when inheritance
-        where = (options[:where] || {}).dup
+        where = ensure_permitted(options[:where] || {}).dup
         if searchkick_options[:inheritance] && (options[:type] || (klass != searchkick_klass && searchkick_index))
           where[:type] = [options[:type] || klass].flatten.map { |v| searchkick_index.klass_document_type(v, true) }
         end
 
+        models = Array(options[:models])
+        if models.any? { |m| m != m.searchkick_klass }
+          # aliases are not supported with _index in ES below 7.5
+          # see https://github.com/elastic/elasticsearch/pull/46640
+          if below75?
+            Searchkick.warn("Passing child models to models option throws off hits and pagination - use type option instead")
+          else
+            index_type_or =
+              models.map do |m|
+                v = {_index: m.searchkick_index.name}
+                v[:type] = m.searchkick_index.klass_document_type(m, true) if m != m.searchkick_klass
+                v
+              end
+
+            where[:or] = Array(where[:or]) + [index_type_or]
+          end
+        end
+
         # start everything as efficient filters
         # move to post_filters as aggs demand
         filters = where_filters(where)
@@ -480,7 +519,7 @@ module Searchkick
       pagination_options = options[:page] || options[:limit] || options[:per_page] || options[:offset] || options[:padding]
       if !options[:body] || pagination_options
         payload[:size] = per_page
-        payload[:from] = offset
+        payload[:from] = offset if offset > 0
       end
 
       # type
@@ -491,17 +530,28 @@ module Searchkick
       # routing
       @routing = options[:routing] if options[:routing]
 
+      if track_total_hits?
+        payload[:track_total_hits] = true
+      end
+
       # merge more body options
       payload = payload.deep_merge(options[:body_options]) if options[:body_options]
 
       # run block
       options[:block].call(payload) if options[:block]
 
+      # scroll optimization when interating over all docs
+      # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html
+      if options[:scroll] && payload[:query] == {match_all: {}}
+        payload[:sort] ||= ["_doc"]
+      end
+
       @body = payload
       @page = page
       @per_page = per_page
       @padding = padding
       @load = load
+      @scroll = scroll
     end
 
     def set_fields
@@ -532,7 +582,8 @@ module Searchkick
 
     def build_query(query, filters, should, must_not, custom_filters, multiply_filters)
       if filters.any? || must_not.any? || should.any?
-        bool = {must: query}
+        bool = {}
+        bool[:must] = query if query
         bool[:filter] = filters if filters.any?      # where
         bool[:must_not] = must_not if must_not.any?  # exclude
         bool[:should] = should if should.any?        # conversions
@@ -645,9 +696,9 @@ module Searchkick
     def set_boost_by(multiply_filters, custom_filters)
       boost_by = options[:boost_by] || {}
       if boost_by.is_a?(Array)
-        boost_by = Hash[boost_by.map { |f| [f, {factor: 1}] }]
+        boost_by = boost_by.to_h { |f| [f, {factor: 1}] }
       elsif boost_by.is_a?(Hash)
-        multiply_by, boost_by = boost_by.partition { |_, v| v.delete(:boost_mode) == "multiply" }.map { |i| Hash[i] }
+        multiply_by, boost_by = boost_by.partition { |_, v| v.delete(:boost_mode) == "multiply" }.map(&:to_h)
       end
       boost_by[options[:boost]] = {factor: 1} if options[:boost]
 
@@ -712,7 +763,7 @@ module Searchkick
 
     def set_highlights(payload, fields)
       payload[:highlight] = {
-        fields: Hash[fields.map { |f| [f, {}] }],
+        fields: fields.to_h { |f| [f, {}] },
         fragment_size: 0
       }
 
@@ -746,7 +797,7 @@ module Searchkick
       aggs = options[:aggs]
       payload[:aggs] = {}
 
-      aggs = Hash[aggs.map { |f| [f, {}] }] if aggs.is_a?(Array) # convert to more advanced syntax
+      aggs = aggs.to_h { |f| [f, {}] } if aggs.is_a?(Array) # convert to more advanced syntax
       aggs.each do |field, agg_options|
         size = agg_options[:limit] ? agg_options[:limit] : 1_000
         shared_agg_options = agg_options.except(:limit, :field, :ranges, :date_ranges, :where)
@@ -785,8 +836,9 @@ module Searchkick
         end
 
         where = {}
-        where = (options[:where] || {}).reject { |k| k == field } unless options[:smart_aggs] == false
-        agg_filters = where_filters(where.merge(agg_options[:where] || {}))
+        where = ensure_permitted(options[:where] || {}).reject { |k| k == field } unless options[:smart_aggs] == false
+        agg_where = ensure_permitted(agg_options[:where] || {})
+        agg_filters = where_filters(where.merge(agg_where))
 
         # only do one level comparison for simplicity
         filters.select! do |filter|
@@ -821,11 +873,14 @@ module Searchkick
       }
     end
 
-    # TODO id transformation for arrays
     def set_order(payload)
-      order = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
-      id_field = :_id
-      payload[:sort] = order.is_a?(Array) ? order : Hash[order.map { |k, v| [k.to_s == "id" ? id_field : k, v] }]
+      payload[:sort] = options[:order].is_a?(Enumerable) ? options[:order] : {options[:order] => :asc}
+    end
+
+    # provides *very* basic protection from unfiltered parameters
+    # this is not meant to be comprehensive and may be expanded in the future
+    def ensure_permitted(obj)
+      obj.to_h
     end
 
     def where_filters(where)
@@ -843,15 +898,12 @@ module Searchkick
           filters << {bool: {must_not: where_filters(value)}}
         elsif field == :_and
           filters << {bool: {must: value.map { |or_statement| {bool: {filter: where_filters(or_statement)}} }}}
+        # elsif field == :_script
+        #   filters << {script: {script: {source: value, lang: "painless"}}}
         else
           # expand ranges
           if value.is_a?(Range)
-            # infinite? added in Ruby 2.4
-            if value.end.nil? || (value.end.respond_to?(:infinite?) && value.end.infinite?)
-              value = {gte: value.first}
-            else
-              value = {gte: value.first, (value.exclude_end? ? :lt : :lte) => value.last}
-            end
+            value = expand_range(value)
           end
 
           value = {in: value} if value.is_a?(Array)
@@ -903,8 +955,32 @@ module Searchkick
                     }
                   }
                 }
+              when :like, :ilike
+                # based on Postgres
+                # https://www.postgresql.org/docs/current/functions-matching.html
+                # % matches zero or more characters
+                # _ matches one character
+                # \ is escape character
+                # escape Lucene reserved characters
+                # https://www.elastic.co/guide/en/elasticsearch/reference/current/regexp-syntax.html#regexp-optional-operators
+                reserved = %w(\\ . ? + * | { } [ ] ( ) ")
+                regex = op_value.dup
+                reserved.each do |v|
+                  regex.gsub!(v, "\\\\" + v)
+                end
+                regex = regex.gsub(/(?<!\\)%/, ".*").gsub(/(?<!\\)_/, ".").gsub("\\%", "%").gsub("\\_", "_")
+
+                if op == :ilike
+                  if below710?
+                    raise ArgumentError, "ilike requires Elasticsearch 7.10+"
+                  else
+                    filters << {regexp: {field => {value: regex, flags: "NONE", case_insensitive: true}}}
+                  end
+                else
+                  filters << {regexp: {field => {value: regex, flags: "NONE"}}}
+                end
               when :prefix
-                filters << {prefix: {field => op_value}}
+                filters << {prefix: {field => {value: op_value}}}
               when :regexp # support for regexp queries without using a regexp ruby object
                 filters << {regexp: {field => {value: op_value}}}
               when :not, :_not # not equal
@@ -915,6 +991,8 @@ module Searchkick
                 end
               when :in
                 filters << term_filters(field, op_value)
+              when :exists
+                filters << {exists: {field: field}}
               else
                 range_query =
                   case op
@@ -927,7 +1005,7 @@ module Searchkick
                   when :lte
                     {to: op_value, include_upper: true}
                   else
-                    raise "Unknown where operator: #{op.inspect}"
+                    raise ArgumentError, "Unknown where operator: #{op.inspect}"
                   end
                 # issue 132
                 if (existing = filters.find { |f| f[:range] && f[:range][field] })
@@ -955,9 +1033,40 @@ module Searchkick
       elsif value.nil?
         {bool: {must_not: {exists: {field: field}}}}
       elsif value.is_a?(Regexp)
-        {regexp: {field => {value: value.source, flags: "NONE"}}}
+        source = value.source
+
+        # TODO handle other regexp options
+
+        # TODO handle other anchor characters, like ^, $, \Z
+        if source.start_with?("\\A")
+          source = source[2..-1]
+        else
+          source = ".*#{source}"
+        end
+
+        if source.end_with?("\\z")
+          source = source[0..-3]
+        else
+          source = "#{source}.*"
+        end
+
+        if below710?
+          if value.casefold?
+            raise ArgumentError, "Case-insensitive flag does not work with Elasticsearch < 7.10"
+          end
+          {regexp: {field => {value: source, flags: "NONE"}}}
+        else
+          {regexp: {field => {value: source, flags: "NONE", case_insensitive: value.casefold?}}}
+        end
       else
-        {term: {field => value}}
+        # TODO add this for other values
+        if value.as_json.is_a?(Enumerable)
+          # query will fail, but this is better
+          # same message as Active Record
+          raise TypeError, "can't cast #{value.class.name}"
+        end
+
+        {term: {field => {value: value}}}
       end
     end
 
@@ -1017,16 +1126,39 @@ module Searchkick
       end
     end
 
+    def expand_range(range)
+      expanded = {}
+      expanded[:gte] = range.begin if range.begin
+
+      if range.end && !(range.end.respond_to?(:infinite?) && range.end.infinite?)
+        expanded[range.exclude_end? ? :lt : :lte] = range.end
+      end
+
+      expanded
+    end
+
     def base_field(k)
       k.sub(/\.(analyzed|word_start|word_middle|word_end|text_start|text_middle|text_end|exact)\z/, "")
     end
 
-    def below61?
-      Searchkick.server_below?("6.1.0")
+    def track_total_hits?
+      searchkick_options[:deep_paging] || body_options[:track_total_hits]
+    end
+
+    def body_options
+      options[:body_options] || {}
+    end
+
+    def below73?
+      Searchkick.server_below?("7.3.0")
+    end
+
+    def below75?
+      Searchkick.server_below?("7.5.0")
     end
 
-    def below70?
-      Searchkick.server_below?("7.0.0")
+    def below710?
+      Searchkick.server_below?("7.10.0")
     end
   end
 end

data/lib/searchkick/record_data.rb

@@ -39,7 +39,6 @@ module Searchkick
         _index: index.name,
         _id: search_id
       }
-      data[:_type] = document_type if Searchkick.server_below7?
       data[:routing] = record.search_routing if record.respond_to?(:search_routing)
       data
     end