searchkick 5.5.1 → 6.0.0

data/lib/searchkick/bulk_reindex_job.rb

@@ -1,20 +1,18 @@
 module Searchkick
-  class BulkReindexJob < ActiveJob::Base
+  class BulkReindexJob < Searchkick.parent_job.constantize
     queue_as { Searchkick.queue_name }
 
-    # TODO remove min_id and max_id in Searchkick 6
-    def perform(class_name:, record_ids: nil, index_name: nil, method_name: nil, batch_id: nil, min_id: nil, max_id: nil)
+    def perform(class_name:, record_ids: nil, index_name: nil, method_name: nil, batch_id: nil, min_id: nil, max_id: nil, ignore_missing: nil)
       model = Searchkick.load_model(class_name)
       index = model.searchkick_index(name: index_name)
 
-      # legacy
       record_ids ||= min_id..max_id
 
       relation = Searchkick.scope(model)
       relation = Searchkick.load_records(relation, record_ids)
       relation = relation.search_import if relation.respond_to?(:search_import)
 
-      RecordIndexer.new(index).reindex(relation, mode: :inline, method_name: method_name, full: false)
+      RecordIndexer.new(index).reindex(relation, mode: :inline, method_name: method_name, ignore_missing: ignore_missing, full: false)
       RelationIndexer.new(index).batch_completed(batch_id) if batch_id
     end
   end

data/lib/searchkick/hash_wrapper.rb

@@ -1,12 +1,33 @@
 module Searchkick
-  # Subclass of `Hashie::Mash` to wrap Hash-like structures
-  # (responses from Elasticsearch)
-  #
-  # The primary goal of the subclass is to disable the
-  # warning being printed by Hashie for re-defined
-  # methods, such as `sort`.
-  #
-  class HashWrapper < ::Hashie::Mash
-    disable_warnings if respond_to?(:disable_warnings)
+  class HashWrapper
+    def initialize(attributes)
+      @attributes = attributes
+    end
+
+    def [](name)
+      @attributes[name.to_s]
+    end
+
+    def to_h
+      @attributes
+    end
+
+    def method_missing(name, ...)
+      if @attributes.key?(name.to_s)
+        self[name]
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(name, ...)
+      @attributes.key?(name.to_s) || super
+    end
+
+    def inspect
+      attributes = @attributes.reject { |k, v| k[0] == "_" }.map { |k, v| "#{k}: #{v.inspect}" }
+      attributes.unshift(attributes.pop) # move id to start
+      "#<#{self.class.name} #{attributes.join(", ")}>"
+    end
   end
 end

data/lib/searchkick/index.rb

@@ -164,11 +164,11 @@ module Searchkick
     end
     alias_method :import, :bulk_index
 
-    def bulk_update(records, method_name)
+    def bulk_update(records, method_name, ignore_missing: nil)
       return if records.empty?
 
       notify_bulk(records, "Update") do
-        queue_update(records, method_name)
+        queue_update(records, method_name, ignore_missing: ignore_missing)
       end
     end
 
@@ -192,7 +192,6 @@ module Searchkick
       if Searchkick.opensearch?
         client.transport.perform_request "POST", "_plugins/_refresh_search_analyzers/#{CGI.escape(name)}"
       else
-        raise Error, "Requires Elasticsearch 7.3+" if Searchkick.server_below?("7.3.0")
         begin
           client.transport.perform_request("GET", "#{CGI.escape(name)}/_reload_search_analyzers")
         rescue => e
@@ -212,10 +211,14 @@ module Searchkick
 
     # note: this is designed to be used internally
     # so it does not check object matches index class
-    def reindex(object, method_name: nil, full: false, **options)
+    def reindex(object, method_name: nil, ignore_missing: nil, full: false, **options)
+      if @options[:job_options]
+        options[:job_options] = (@options[:job_options] || {}).merge(options[:job_options] || {})
+      end
+
       if object.is_a?(Array)
         # note: purposefully skip full
-        return reindex_records(object, method_name: method_name, **options)
+        return reindex_records(object, method_name: method_name, ignore_missing: ignore_missing, **options)
       end
 
       if !object.respond_to?(:searchkick_klass)
@@ -231,22 +234,22 @@ module Searchkick
 
       if method_name || (scoped && !full)
         mode = options.delete(:mode) || :inline
+        scope = options.delete(:scope)
+        job_options = options.delete(:job_options)
         raise ArgumentError, "unsupported keywords: #{options.keys.map(&:inspect).join(", ")}" if options.any?
 
         # import only
-        import_scope(relation, method_name: method_name, mode: mode)
+        import_scope(relation, method_name: method_name, mode: mode, scope: scope, ignore_missing: ignore_missing, job_options: job_options)
         self.refresh if refresh
         true
       else
         async = options.delete(:async)
         if async
           if async.is_a?(Hash) && async[:wait]
-            # TODO warn in 5.1
-            # Searchkick.warn "async option is deprecated - use mode: :async, wait: true instead"
+            Searchkick.warn "async option is deprecated - use mode: :async, wait: true instead"
             options[:wait] = true unless options.key?(:wait)
           else
-            # TODO warn in 5.1
-            # Searchkick.warn "async option is deprecated - use mode: :async instead"
+            Searchkick.warn "async option is deprecated - use mode: :async instead"
           end
           options[:mode] ||= :async
         end
@@ -291,6 +294,11 @@ module Searchkick
       end
     end
 
+    # private
+    def conversions_v2_fields
+      @conversions_v2_fields ||= Array(options[:conversions_v2]).map(&:to_s)
+    end
+
     # private
     def suggest_fields
       @suggest_fields ||= Array(options[:suggest]).map(&:to_s)
@@ -323,8 +331,10 @@ module Searchkick
       Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(self, r).delete_data })
     end
 
-    def queue_update(records, method_name)
-      Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).update_data(method_name) })
+    def queue_update(records, method_name, ignore_missing:)
+      items = records.map { |r| RecordData.new(self, r).update_data(method_name) }
+      items.each { |i| i.instance_variable_set(:@ignore_missing, true) } if ignore_missing
+      Searchkick.indexer.queue(items)
     end
 
     def relation_indexer
@@ -350,10 +360,9 @@ module Searchkick
 
     # https://gist.github.com/jarosan/3124884
     # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
-    def full_reindex(relation, import: true, resume: false, retain: false, mode: nil, refresh_interval: nil, scope: nil, wait: nil)
+    def full_reindex(relation, import: true, resume: false, retain: false, mode: nil, refresh_interval: nil, scope: nil, wait: nil, job_options: nil)
       raise ArgumentError, "wait only available in :async mode" if !wait.nil? && mode != :async
-      # TODO raise ArgumentError in Searchkick 6
-      Searchkick.warn("Full reindex does not support :queue mode - use :async mode instead") if mode == :queue
+      raise ArgumentError, "Full reindex does not support :queue mode - use :async mode instead" if mode == :queue
 
       if resume
         index_name = all_indices.sort.last
@@ -371,7 +380,8 @@ module Searchkick
         mode: (mode || :inline),
         full: true,
         resume: resume,
-        scope: scope
+        scope: scope,
+        job_options: job_options
       }
 
       uuid = index.uuid

data/lib/searchkick/index_options.rb

@@ -184,12 +184,6 @@ module Searchkick
         end
       end
 
-      if options[:case_sensitive]
-        settings[:analysis][:analyzer].each do |_, analyzer|
-          analyzer[:filter].delete("lowercase")
-        end
-      end
-
       add_synonyms(settings)
       add_search_synonyms(settings)
 
@@ -199,6 +193,12 @@ module Searchkick
         end
       end
 
+      if options[:case_sensitive]
+        settings[:analysis][:analyzer].each do |_, analyzer|
+          analyzer[:filter].delete("lowercase")
+        end
+      end
+
       settings
     end
 
@@ -372,6 +372,16 @@ module Searchkick
         }
       end
 
+      Array(options[:conversions_v2]).each do |conversions_field|
+        mapping[conversions_field] = {
+          type: "rank_features"
+        }
+      end
+
+      if (Array(options[:conversions_v2]).map(&:to_s) & Array(options[:conversions]).map(&:to_s)).any?
+        raise ArgumentError, "Must have separate conversions fields"
+      end
+
       mapping_options =
         [:suggest, :word, :text_start, :text_middle, :text_end, :word_start, :word_middle, :word_end, :highlight, :searchable, :filterable]
           .to_h { |type| [type, (options[type] || []).map(&:to_s)] }
@@ -423,6 +433,7 @@ module Searchkick
 
       (options[:knn] || []).each do |field, knn_options|
         distance = knn_options[:distance]
+        quantization = knn_options[:quantization]
 
         if Searchkick.opensearch?
           if distance.nil?
@@ -448,6 +459,10 @@ module Searchkick
                 raise ArgumentError, "Unknown distance: #{distance}"
               end
 
+            if !quantization.nil?
+              raise ArgumentError, "Quantization not supported yet for OpenSearch"
+            end
+
             vector_options[:method] = {
               name: "hnsw",
               space_type: space_type,
@@ -477,13 +492,18 @@ module Searchkick
                 raise ArgumentError, "Unknown distance: #{distance}"
               end
 
+            type =
+              case quantization
+              when "int8", "int4", "bbq"
+                "#{quantization}_hnsw"
+              when nil
+                "hnsw"
+              else
+                raise ArgumentError, "Unknown quantization: #{quantization}"
+              end
+
             vector_index_options = knn_options.slice(:m, :ef_construction)
-            if vector_index_options.any?
-              # TODO no quantization by default in Searchkick 6
-              # int8_hnsw was made the default in Elasticsearch 8.14.0
-              type = Searchkick.server_below?("8.14.0") ? "hnsw" : "int8_hnsw"
-              vector_options[:index_options] = {type: type}.merge(vector_index_options)
-            end
+            vector_options[:index_options] = {type: type}.merge(vector_index_options)
           end
 
           mapping[field.to_s] = vector_options
@@ -578,9 +598,9 @@ module Searchkick
         if search_synonyms.is_a?(String)
           synonym_graph = {
             type: "synonym_graph",
-            synonyms_path: search_synonyms
+            synonyms_path: search_synonyms,
+            updateable: true
           }
-          synonym_graph[:updateable] = true unless below73?
         else
           synonym_graph = {
             type: "synonym_graph",
@@ -628,9 +648,5 @@ module Searchkick
     def default_analyzer
       :searchkick_index
     end
-
-    def below73?
-      Searchkick.server_below?("7.3.0")
-    end
   end
 end

data/lib/searchkick/indexer.rb

@@ -24,12 +24,20 @@ module Searchkick
         # note: delete does not set error when item not found
         first_with_error = response["items"].map do |item|
           (item["index"] || item["delete"] || item["update"])
-        end.find { |item| item["error"] }
-        raise ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
+        end.find.with_index { |item, i| item["error"] && !ignore_missing?(items[i], item["error"]) }
+        if first_with_error
+          raise ImportError, "#{first_with_error["error"]} on item with id '#{first_with_error["_id"]}'"
+        end
       end
 
       # maybe return response in future
       nil
     end
+
+    private
+
+    def ignore_missing?(item, error)
+      error["type"] == "document_missing_exception" && item.instance_variable_defined?(:@ignore_missing)
+    end
   end
 end

data/lib/searchkick/log_subscriber.rb

@@ -24,7 +24,7 @@ module Searchkick
 
       index = payload[:query][:index].is_a?(Array) ? payload[:query][:index].join(",") : payload[:query][:index]
       type = payload[:query][:type]
-      request_params = payload[:query].except(:index, :type, :body)
+      request_params = payload[:query].except(:index, :type, :body, :opaque_id)
 
       params = []
       request_params.each do |k, v|

data/lib/searchkick/model.rb

@@ -1,10 +1,18 @@
 module Searchkick
   module Model
     def searchkick(**options)
-      options = Searchkick.model_options.merge(options)
+      options = Searchkick.model_options.deep_merge(options)
 
-      unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :case_sensitive, :conversions, :deep_paging, :default_fields,
-        :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :knn, :language,
+      if options[:conversions]
+        Searchkick.warn("The `conversions` option is deprecated in favor of `conversions_v2`, which provides much better search performance. Upgrade to `conversions_v2` or rename `conversions` to `conversions_v1`")
+      end
+
+      if options.key?(:conversions_v1)
+        options[:conversions] = options.delete(:conversions_v1)
+      end
+
+      unknown_keywords = options.keys - [:_all, :_type, :batch_size, :callbacks, :callback_options, :case_sensitive, :conversions, :conversions_v2, :deep_paging, :default_fields,
+        :filterable, :geo_shape, :highlight, :ignore_above, :index_name, :index_prefix, :inheritance, :job_options, :knn, :language,
         :locations, :mappings, :match, :max_result_window, :merge_mappings, :routing, :searchable, :search_synonyms, :settings, :similarity,
         :special_characters, :stem, :stemmer, :stem_conversions, :stem_exclusion, :stemmer_override, :suggest, :synonyms, :text_end,
         :text_middle, :text_start, :unscope, :word, :word_end, :word_middle, :word_start]
@@ -21,14 +29,16 @@ module Searchkick
       unless [:inline, true, false, :async, :queue].include?(callbacks)
         raise ArgumentError, "Invalid value for callbacks"
       end
+      callback_options = (options[:callback_options] || {}).dup
+      callback_options[:if] = [-> { Searchkick.callbacks?(default: callbacks) }, callback_options[:if]].compact.flatten(1)
 
       base = self
 
       mod = Module.new
       include(mod)
       mod.module_eval do
-        def reindex(method_name = nil, mode: nil, refresh: false)
-          self.class.searchkick_index.reindex([self], method_name: method_name, mode: mode, refresh: refresh, single: true)
+        def reindex(method_name = nil, mode: nil, refresh: false, ignore_missing: nil, job_options: nil)
+          self.class.searchkick_index.reindex([self], method_name: method_name, mode: mode, refresh: refresh, ignore_missing: ignore_missing, job_options: job_options, single: true)
         end unless base.method_defined?(:reindex)
 
         def similar(**options)
@@ -99,10 +109,10 @@ module Searchkick
         # always add callbacks, even when callbacks is false
         # so Model.callbacks block can be used
         if respond_to?(:after_commit)
-          after_commit :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
+          after_commit :reindex, **callback_options
         elsif respond_to?(:after_save)
-          after_save :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
-          after_destroy :reindex, if: -> { Searchkick.callbacks?(default: callbacks) }
+          after_save :reindex, **callback_options
+          after_destroy :reindex, **callback_options
         end
       end
     end

data/lib/searchkick/multi_search.rb

@@ -2,8 +2,9 @@ module Searchkick
   class MultiSearch
     attr_reader :queries
 
-    def initialize(queries)
+    def initialize(queries, opaque_id: nil)
       @queries = queries
+      @opaque_id = opaque_id
     end
 
     def perform
@@ -15,7 +16,11 @@ module Searchkick
     private
 
     def perform_search(search_queries, perform_retry: true)
-      responses = client.msearch(body: search_queries.flat_map { |q| [q.params.except(:body), q.body] })["responses"]
+      params = {
+        body: search_queries.flat_map { |q| [q.params.except(:body), q.body] }
+      }
+      params[:opaque_id] = @opaque_id if @opaque_id
+      responses = client.msearch(params)["responses"]
 
       retry_queries = []
       search_queries.each_with_index do |query, i|

data/lib/searchkick/process_batch_job.rb

@@ -1,5 +1,5 @@
 module Searchkick
-  class ProcessBatchJob < ActiveJob::Base
+  class ProcessBatchJob < Searchkick.parent_job.constantize
     queue_as { Searchkick.queue_name }
 
     def perform(class_name:, record_ids:, index_name: nil)
@@ -14,7 +14,7 @@ module Searchkick
         end
 
       relation = Searchkick.scope(model)
-      RecordIndexer.new(index).reindex_items(relation, items, method_name: nil)
+      RecordIndexer.new(index).reindex_items(relation, items, method_name: nil, ignore_missing: nil)
     end
   end
 end

data/lib/searchkick/process_queue_job.rb

@@ -1,11 +1,12 @@
 module Searchkick
-  class ProcessQueueJob < ActiveJob::Base
+  class ProcessQueueJob < Searchkick.parent_job.constantize
     queue_as { Searchkick.queue_name }
 
-    def perform(class_name:, index_name: nil, inline: false)
+    def perform(class_name:, index_name: nil, inline: false, job_options: nil)
       model = Searchkick.load_model(class_name)
       index = model.searchkick_index(name: index_name)
       limit = model.searchkick_options[:batch_size] || 1000
+      job_options = (model.searchkick_options[:job_options] || {}).merge(job_options || {})
 
       loop do
         record_ids = index.reindex_queue.reserve(limit: limit)
@@ -20,7 +21,7 @@ module Searchkick
             # use new.perform to avoid excessive logging
             Searchkick::ProcessBatchJob.new.perform(**batch_options)
           else
-            Searchkick::ProcessBatchJob.perform_later(**batch_options)
+            Searchkick::ProcessBatchJob.set(job_options).perform_later(**batch_options)
           end
 
           # TODO when moving to reliable queuing, mark as complete