searchkick 5.5.2 → 6.0.1

data/lib/searchkick/query.rb

@@ -17,10 +17,18 @@ module Searchkick
       :with_score, :misspellings?, :scroll_id, :clear_scroll, :missing_records, :with_hit
 
     def initialize(klass, term = "*", **options)
+      if options[:conversions]
+        Searchkick.warn("The `conversions` option is deprecated in favor of `conversions_v2`, which provides much better search performance. Upgrade to `conversions_v2` or rename `conversions` to `conversions_v1`")
+      end
+
+      if options.key?(:conversions_v1)
+        options[:conversions] = options.delete(:conversions_v1)
+      end
+
       unknown_keywords = options.keys - [:aggs, :block, :body, :body_options, :boost,
-        :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_term, :debug, :emoji, :exclude, :explain,
+        :boost_by, :boost_by_distance, :boost_by_recency, :boost_where, :conversions, :conversions_v2, :conversions_term, :debug, :emoji, :exclude, :explain,
         :fields, :highlight, :includes, :index_name, :indices_boost, :knn, :limit, :load,
-        :match, :misspellings, :models, :model_includes, :offset, :operator, :order, :padding, :page, :per_page, :profile,
+        :match, :misspellings, :models, :model_includes, :offset, :opaque_id, :operator, :order, :padding, :page, :per_page, :profile,
         :request_params, :routing, :scope_results, :scroll, :select, :similar, :smart_aggs, :suggest, :total_entries, :track, :type, :where]
       raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any?
 
@@ -86,6 +94,7 @@ module Searchkick
       params[:type] = @type if @type
       params[:routing] = @routing if @routing
       params[:scroll] = @scroll if @scroll
+      params[:opaque_id] = @opaque_id if @opaque_id
       params.merge!(options[:request_params]) if options[:request_params]
       params
     end
@@ -98,34 +107,13 @@ module Searchkick
             prepare
             response = execute_search
           end
-        rescue => e # TODO rescue type
+        rescue => e
           handle_error(e)
         end
         handle_response(response)
       end
     end
 
-    def to_curl
-      query = params
-      type = query[:type]
-      index = query[:index].is_a?(Array) ? query[:index].join(",") : query[:index]
-      request_params = query.except(:index, :type, :body)
-
-      # no easy way to tell which host the client will use
-      host =
-        if Searchkick.client.transport.respond_to?(:transport)
-          Searchkick.client.transport.transport.hosts.first
-        else
-          Searchkick.client.transport.hosts.first
-        end
-      credentials = host[:user] || host[:password] ? "#{host[:user]}:#{host[:password]}@" : nil
-      params = ["pretty"]
-      request_params.each do |k, v|
-        params << "#{CGI.escape(k.to_s)}=#{CGI.escape(v.to_s)}"
-      end
-      "curl #{host[:protocol]}://#{credentials}#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?#{params.join('&')} -H 'Content-Type: application/json' -d '#{query[:body].to_json}'"
-    end
-
     def handle_response(response)
       opts = {
         page: @page,
@@ -144,15 +132,17 @@ module Searchkick
         total_entries: options[:total_entries],
         index_mapping: @index_mapping,
         suggest: options[:suggest],
-        scroll: options[:scroll]
+        scroll: options[:scroll],
+        opaque_id: options[:opaque_id]
       }
 
       if options[:debug]
-        puts "Searchkick Version: #{Searchkick::VERSION}"
-        puts "Elasticsearch Version: #{Searchkick.server_version}"
+        server = Searchkick.opensearch? ? "OpenSearch" : "Elasticsearch"
+        puts "Searchkick #{Searchkick::VERSION}"
+        puts "#{server} #{Searchkick.server_version}"
         puts
 
-        puts "Model Searchkick Options"
+        puts "Model Options"
         pp searchkick_options
         puts
 
@@ -161,7 +151,7 @@ module Searchkick
         puts
 
         if searchkick_index
-          puts "Model Search Data"
+          puts "Record Data"
           begin
             pp klass.limit(3).map { |r| RecordData.new(searchkick_index, r).index_data }
           rescue => e
@@ -169,21 +159,21 @@ module Searchkick
           end
           puts
 
-          puts "Elasticsearch Mapping"
+          puts "Mapping"
           puts JSON.pretty_generate(searchkick_index.mapping)
           puts
 
-          puts "Elasticsearch Settings"
+          puts "Settings"
           puts JSON.pretty_generate(searchkick_index.settings)
           puts
         end
 
-        puts "Elasticsearch Query"
-        puts to_curl
+        puts "Query"
+        puts JSON.pretty_generate(params[:body])
         puts
 
-        puts "Elasticsearch Results"
-        puts JSON.pretty_generate(response)
+        puts "Results"
+        puts JSON.pretty_generate(response.to_h)
       end
 
       # set execute for multi search
@@ -257,6 +247,7 @@ module Searchkick
       padding = [options[:padding].to_i, 0].max
       offset = (options[:offset] || (page - 1) * per_page + padding).to_i
       scroll = options[:scroll]
+      opaque_id = options[:opaque_id]
 
       max_result_window = searchkick_options[:max_result_window]
       original_per_page = per_page
@@ -374,7 +365,6 @@ module Searchkick
             field_misspellings = misspellings && (!misspellings_fields || misspellings_fields.include?(base_field(field)))
 
             if field == "_all" || field.end_with?(".analyzed")
-              shared_options[:cutoff_frequency] = 0.001 unless operator.to_s == "and" || field_misspellings == false || (!below73? && !track_total_hits?) || match_type == :match_phrase || !below80? || Searchkick.opensearch?
               qs << shared_options.merge(analyzer: "searchkick_search")
 
               # searchkick_search and searchkick_search2 are the same for some languages
@@ -442,6 +432,7 @@ module Searchkick
             }
 
             should.concat(set_conversions)
+            should.concat(set_conversions_v2)
           end
 
           query = payload
@@ -457,20 +448,14 @@ module Searchkick
 
         models = Array(options[:models])
         if models.any? { |m| m != m.searchkick_klass }
-          # aliases are not supported with _index in ES below 7.5
-          # see https://github.com/elastic/elasticsearch/pull/46640
-          if below75?
-            Searchkick.warn("Passing child models to models option throws off hits and pagination - use type option instead")
-          else
-            index_type_or =
-              models.map do |m|
-                v = {_index: m.searchkick_index.name}
-                v[:type] = m.searchkick_index.klass_document_type(m, true) if m != m.searchkick_klass
-                v
-              end
+          index_type_or =
+            models.map do |m|
+              v = {_index: m.searchkick_index.name}
+              v[:type] = m.searchkick_index.klass_document_type(m, true) if m != m.searchkick_klass
+              v
+            end
 
-            where[:or] = Array(where[:or]) + [index_type_or]
-          end
+          where[:or] = Array(where[:or]) + [index_type_or]
         end
 
         # start everything as efficient filters
@@ -554,7 +539,7 @@ module Searchkick
       # run block
       options[:block].call(payload) if options[:block]
 
-      # scroll optimization when interating over all docs
+      # scroll optimization when iterating over all docs
       # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html
       if options[:scroll] && payload[:query] == {match_all: {}}
         payload[:sort] ||= ["_doc"]
@@ -566,6 +551,7 @@ module Searchkick
       @padding = padding
       @load = load
       @scroll = scroll
+      @opaque_id = opaque_id
     end
 
     def set_fields
@@ -656,6 +642,43 @@ module Searchkick
       end
     end
 
+    def set_conversions_v2
+      conversions_v2 = options[:conversions_v2]
+      return [] if conversions_v2.nil? && !searchkick_options[:conversions_v2]
+      return [] if conversions_v2 == false
+
+      # disable if searchkick_options[:conversions] to make it easy to upgrade without downtime
+      return [] if conversions_v2.nil? && searchkick_options[:conversions]
+
+      unless conversions_v2.is_a?(Hash)
+        conversions_v2 = {field: conversions_v2}
+      end
+
+      conversions_fields =
+        case conversions_v2[:field]
+        when true, nil
+          Array(searchkick_options[:conversions_v2]).map(&:to_s)
+        else
+          [conversions_v2[:field].to_s]
+        end
+
+      conversions_term = (conversions_v2[:term] || options[:conversions_term] || term).to_s
+      unless searchkick_options[:case_sensitive]
+        conversions_term = conversions_term.downcase
+      end
+      conversions_term = conversions_term.gsub(".", "*")
+
+      conversions_fields.map do |conversions_field|
+        {
+          rank_feature: {
+            field: "#{conversions_field}.#{conversions_term}",
+            linear: {},
+            boost: conversions_v2[:factor] || 1
+          }
+        }
+      end
+    end
+
     def set_exclude(field, analyzer)
       Array(options[:exclude]).map do |phrase|
         {
@@ -712,7 +735,7 @@ module Searchkick
       if boost_by.is_a?(Array)
         boost_by = boost_by.to_h { |f| [f, {factor: 1}] }
       elsif boost_by.is_a?(Hash)
-        multiply_by, boost_by = boost_by.partition { |_, v| v.delete(:boost_mode) == "multiply" }.map(&:to_h)
+        multiply_by, boost_by = boost_by.transform_values(&:dup).partition { |_, v| v.delete(:boost_mode) == "multiply" }.map(&:to_h)
       end
       boost_by[options[:boost]] = {factor: 1} if options[:boost]
 
@@ -935,11 +958,11 @@ module Searchkick
                   space_type: space_type
                 }
               },
-              boost: distance == "cosine" && Searchkick.server_below?("2.19.0", true) ? 0.5 : 1.0
+              boost: distance == "cosine" && Searchkick.server_below?("2.19.0") ? 0.5 : 1.0
             }
           }
         else
-          if ef_search && Searchkick.server_below?("2.16.0", true)
+          if ef_search && Searchkick.server_below?("2.16.0")
             raise Error, "ef_search requires OpenSearch 2.16+"
           end
 
@@ -956,7 +979,7 @@ module Searchkick
       else
         if exact
           # prevent incorrect distances/results with Elasticsearch 9.0.0-rc1
-          if !below90? && field_options[:distance] == "cosine" && distance != "cosine"
+          if !Searchkick.server_below?("9.0.0") && field_options[:distance] == "cosine" && distance != "cosine"
             raise ArgumentError, "distance must match searchkick options"
           end
 
@@ -1113,11 +1136,7 @@ module Searchkick
                 regex = regex.gsub(/(?<!\\)%/, ".*").gsub(/(?<!\\)_/, ".").gsub("\\%", "%").gsub("\\_", "_")
 
                 if op == :ilike
-                  if below710?
-                    raise ArgumentError, "ilike requires Elasticsearch 7.10+"
-                  else
-                    filters << {regexp: {field => {value: regex, flags: "NONE", case_insensitive: true}}}
-                  end
+                  filters << {regexp: {field => {value: regex, flags: "NONE", case_insensitive: true}}}
                 else
                   filters << {regexp: {field => {value: regex, flags: "NONE"}}}
                 end
@@ -1134,27 +1153,25 @@ module Searchkick
               when :in
                 filters << term_filters(field, op_value)
               when :exists
-                # TODO add support for false in Searchkick 6
-                if op_value != true
-                  # TODO raise error in Searchkick 6
-                  Searchkick.warn("Passing a value other than true to exists is not supported")
+                case op_value
+                when true
+                  filters << {exists: {field: field}}
+                when false
+                  filters << {bool: {must_not: {exists: {field: field}}}}
+                else
+                  raise ArgumentError, "Passing a value other than true or false to exists is not supported"
                 end
-                filters << {exists: {field: field}}
               else
                 range_query =
                   case op
                   when :gt
-                    # TODO always use gt in Searchkick 6
-                    below90? ? {from: op_value, include_lower: false} : {gt: op_value}
+                    {gt: op_value}
                   when :gte
-                    # TODO always use gte in Searchkick 6
-                    below90? ? {from: op_value, include_lower: true} : {gte: op_value}
+                    {gte: op_value}
                   when :lt
-                    # TODO always use lt in Searchkick 6
-                    below90? ? {to: op_value, include_upper: false} : {lt: op_value}
+                    {lt: op_value}
                   when :lte
-                    # TODO always use lte in Searchkick 6
-                    below90? ? {to: op_value, include_upper: true} : {lte: op_value}
+                    {lte: op_value}
                   else
                     raise ArgumentError, "Unknown where operator: #{op.inspect}"
                   end
@@ -1201,14 +1218,7 @@ module Searchkick
           source = "#{source}.*"
         end
 
-        if below710?
-          if value.casefold?
-            raise ArgumentError, "Case-insensitive flag does not work with Elasticsearch < 7.10"
-          end
-          {regexp: {field => {value: source, flags: "NONE"}}}
-        else
-          {regexp: {field => {value: source, flags: "NONE", case_insensitive: value.casefold?}}}
-        end
+        {regexp: {field => {value: source, flags: "NONE", case_insensitive: value.casefold?}}}
       else
         # TODO add this for other values
         if value.as_json.is_a?(Enumerable)
@@ -1299,25 +1309,5 @@ module Searchkick
     def body_options
       options[:body_options] || {}
     end
-
-    def below73?
-      Searchkick.server_below?("7.3.0")
-    end
-
-    def below75?
-      Searchkick.server_below?("7.5.0")
-    end
-
-    def below710?
-      Searchkick.server_below?("7.10.0")
-    end
-
-    def below80?
-      Searchkick.server_below?("8.0.0")
-    end
-
-    def below90?
-      Searchkick.server_below?("9.0.0")
-    end
   end
 end

data/lib/searchkick/record_data.rb

@@ -58,6 +58,25 @@ module Searchkick
         end
       end
 
+      index.conversions_v2_fields.each do |conversions_field|
+        key = source.key?(conversions_field) ? conversions_field : conversions_field.to_sym
+        if !partial_reindex || source[key]
+          if index.options[:case_sensitive]
+            source[key] =
+              (source[key] || {}).reduce(Hash.new(0)) do |memo, (k, v)|
+                memo[k.to_s.gsub(".", "*")] += v
+                memo
+              end
+          else
+            source[key] =
+              (source[key] || {}).reduce(Hash.new(0)) do |memo, (k, v)|
+                memo[k.to_s.downcase.gsub(".", "*")] += v
+                memo
+              end
+          end
+        end
+      end
+
       # hack to prevent generator field doesn't exist error
       if !partial_reindex
         index.suggest_fields.each do |field|

data/lib/searchkick/record_indexer.rb

@@ -6,7 +6,7 @@ module Searchkick
       @index = index
     end
 
-    def reindex(records, mode:, method_name:, full: false, single: false)
+    def reindex(records, mode:, method_name:, ignore_missing:, full: false, single: false, job_options: nil)
       # prevents exists? check if records is a relation
       records = records.to_a
       return if records.empty?
@@ -17,6 +17,14 @@ module Searchkick
           raise Error, "Active Job not found"
         end
 
+        job_options ||= {}
+
+        # only add if set for backwards compatibility
+        extra_options = {}
+        if ignore_missing
+          extra_options[:ignore_missing] = ignore_missing
+        end
+
         # we could likely combine ReindexV2Job, BulkReindexJob, and ProcessBatchJob
         # but keep them separate for now
         if single
@@ -28,19 +36,21 @@ module Searchkick
             routing = record.search_routing
           end
 
-          Searchkick::ReindexV2Job.perform_later(
+          Searchkick::ReindexV2Job.set(**job_options).perform_later(
             record.class.name,
             record.id.to_s,
             method_name ? method_name.to_s : nil,
             routing: routing,
-            index_name: index.name
+            index_name: index.name,
+            **extra_options
           )
         else
-          Searchkick::BulkReindexJob.perform_later(
+          Searchkick::BulkReindexJob.set(**job_options).perform_later(
             class_name: records.first.class.searchkick_options[:class_name],
             record_ids: records.map { |r| r.id.to_s },
             index_name: index.name,
-            method_name: method_name ? method_name.to_s : nil
+            method_name: method_name ? method_name.to_s : nil,
+            **extra_options
           )
         end
       when :queue
@@ -51,7 +61,7 @@ module Searchkick
         index.reindex_queue.push_records(records)
       when true, :inline
         index_records, other_records = records.partition { |r| index_record?(r) }
-        import_inline(index_records, !full ? other_records : [], method_name: method_name, single: single)
+        import_inline(index_records, !full ? other_records : [], method_name: method_name, ignore_missing: ignore_missing, single: single)
       else
         raise ArgumentError, "Invalid value for mode"
       end
@@ -60,7 +70,7 @@ module Searchkick
       true
     end
 
-    def reindex_items(klass, items, method_name:, single: false)
+    def reindex_items(klass, items, method_name:, ignore_missing:, single: false)
       routing = items.to_h { |r| [r[:id], r[:routing]] }
       record_ids = routing.keys
 
@@ -76,7 +86,7 @@ module Searchkick
           construct_record(klass, id, routing[id])
         end
 
-      import_inline(records, delete_records, method_name: method_name, single: single)
+      import_inline(records, delete_records, method_name: method_name, ignore_missing: ignore_missing, single: single)
     end
 
     private
@@ -86,13 +96,13 @@ module Searchkick
     end
 
     # import in single request with retries
-    def import_inline(index_records, delete_records, method_name:, single:)
+    def import_inline(index_records, delete_records, method_name:, ignore_missing:, single:)
       return if index_records.empty? && delete_records.empty?
 
       maybe_bulk(index_records, delete_records, method_name, single) do
         if index_records.any?
           if method_name
-            index.bulk_update(index_records, method_name)
+            index.bulk_update(index_records, method_name, ignore_missing: ignore_missing)
           else
             index.bulk_index(index_records)
           end

data/lib/searchkick/reindex_queue.rb

@@ -33,17 +33,7 @@ module Searchkick
 
     # TODO use reliable queuing
     def reserve(limit: 1000)
-      if supports_rpop_with_count?
-        Searchkick.with_redis { |r| r.call("RPOP", redis_key, limit) }.to_a
-      else
-        record_ids = []
-        Searchkick.with_redis do |r|
-          while record_ids.size < limit && (record_id = r.call("RPOP", redis_key))
-            record_ids << record_id
-          end
-        end
-        record_ids
-      end
+      Searchkick.with_redis { |r| r.call("RPOP", redis_key, limit) }.to_a
     end
 
     def clear
@@ -60,19 +50,6 @@ module Searchkick
       "searchkick:reindex_queue:#{name}"
     end
 
-    def supports_rpop_with_count?
-      redis_version >= Gem::Version.new("6.2")
-    end
-
-    def redis_version
-      @redis_version ||=
-        Searchkick.with_redis do |r|
-          info = r.call("INFO")
-          matches = /redis_version:(\S+)/.match(info)
-          Gem::Version.new(matches[1])
-        end
-    end
-
     def escape(value)
       value.to_s.gsub("|", "||")
     end

data/lib/searchkick/reindex_v2_job.rb

@@ -1,8 +1,8 @@
 module Searchkick
-  class ReindexV2Job < ActiveJob::Base
+  class ReindexV2Job < Searchkick.parent_job.constantize
     queue_as { Searchkick.queue_name }
 
-    def perform(class_name, id, method_name = nil, routing: nil, index_name: nil)
+    def perform(class_name, id, method_name = nil, routing: nil, index_name: nil, ignore_missing: nil)
       model = Searchkick.load_model(class_name, allow_child: true)
       index = model.searchkick_index(name: index_name)
       # use should_index? to decide whether to index (not default scope)
@@ -11,7 +11,7 @@ module Searchkick
       # but keep for now for backwards compatibility
       model = model.unscoped if model.respond_to?(:unscoped)
       items = [{id: id, routing: routing}]
-      RecordIndexer.new(index).reindex_items(model, items, method_name: method_name, single: true)
+      RecordIndexer.new(index).reindex_items(model, items, method_name: method_name, ignore_missing: ignore_missing, single: true)
     end
   end
 end