searchkick 2.3.2 → 5.2.1

data/lib/searchkick/index.rb

@@ -1,7 +1,5 @@
 module Searchkick
   class Index
-    include IndexOptions
-
     attr_reader :name, :options
 
     def initialize(name, options = {})
@@ -10,12 +8,16 @@ module Searchkick
       @klass_document_type = {} # cache
     end
 
+    def index_options
+      IndexOptions.new(self).index_options
+    end
+
     def create(body = {})
       client.indices.create index: name, body: body
     end
 
     def delete
-      if !Searchkick.server_below?("6.0.0-alpha1") && alias_exists?
+      if alias_exists?
         # can't call delete directly on aliases in ES 6
         indices = client.indices.get_alias(name: name).keys
         client.indices.delete index: indices
@@ -36,25 +38,45 @@ module Searchkick
       client.indices.exists_alias name: name
     end
 
+    # call to_h for consistent results between elasticsearch gem 7 and 8
+    # could do for all API calls, but just do for ones where return value is focus for now
     def mapping
-      client.indices.get_mapping index: name
+      client.indices.get_mapping(index: name).to_h
     end
 
+    # call to_h for consistent results between elasticsearch gem 7 and 8
     def settings
-      client.indices.get_settings index: name
+      client.indices.get_settings(index: name).to_h
     end
 
     def refresh_interval
-      settings.values.first["settings"]["index"]["refresh_interval"]
+      index_settings["refresh_interval"]
     end
 
     def update_settings(settings)
       client.indices.put_settings index: name, body: settings
     end
 
+    def tokens(text, options = {})
+      client.indices.analyze(body: {text: text}.merge(options), index: name)["tokens"].map { |t| t["token"] }
+    end
+
+    def total_docs
+      response =
+        client.search(
+          index: name,
+          body: {
+            query: {match_all: {}},
+            size: 0
+          }
+        )
+
+      Results.new(nil, response).total_count
+    end
+
     def promote(new_name, update_refresh_interval: false)
       if update_refresh_interval
-        new_index = Searchkick::Index.new(new_name)
+        new_index = Index.new(new_name, @options)
         settings = options[:settings] || {}
         refresh_interval = (settings[:index] && settings[:index][:refresh_interval]) || "1s"
         new_index.update_settings(index: {refresh_interval: refresh_interval})
@@ -63,7 +85,8 @@ module Searchkick
       old_indices =
         begin
           client.indices.get_alias(name: name).keys
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
+        rescue => e
+          raise e unless Searchkick.not_found_error?(e)
           {}
         end
       actions = old_indices.map { |old_name| {remove: {index: old_name, alias: name}} } + [{add: {index: new_name, alias: name}}]
@@ -71,176 +94,293 @@ module Searchkick
     end
     alias_method :swap, :promote
 
-    # record based
-    # use helpers for notifications
+    def retrieve(record)
+      record_data = RecordData.new(self, record).record_data
+
+      # remove underscore
+      get_options = record_data.to_h { |k, v| [k.to_s.sub(/\A_/, "").to_sym, v] }
+
+      client.get(get_options)["_source"]
+    end
+
+    def all_indices(unaliased: false)
+      indices =
+        begin
+          if client.indices.respond_to?(:get_alias)
+            client.indices.get_alias(index: "#{name}*")
+          else
+            client.indices.get_aliases
+          end
+        rescue => e
+          raise e unless Searchkick.not_found_error?(e)
+          {}
+        end
+      indices = indices.select { |_k, v| v.empty? || v["aliases"].empty? } if unaliased
+      indices.select { |k, _v| k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys
+    end
+
+    # remove old indices that start w/ index_name
+    def clean_indices
+      indices = all_indices(unaliased: true)
+      indices.each do |index|
+        Index.new(index).delete
+      end
+      indices
+    end
 
     def store(record)
-      bulk_index_helper([record])
+      notify(record, "Store") do
+        queue_index([record])
+      end
     end
 
     def remove(record)
-      bulk_delete_helper([record])
+      notify(record, "Remove") do
+        queue_delete([record])
+      end
     end
 
     def update_record(record, method_name)
-      bulk_update_helper([record], method_name)
+      notify(record, "Update") do
+        queue_update([record], method_name)
+      end
     end
 
     def bulk_delete(records)
-      bulk_delete_helper(records)
+      return if records.empty?
+
+      notify_bulk(records, "Delete") do
+        queue_delete(records)
+      end
     end
 
     def bulk_index(records)
-      bulk_index_helper(records)
+      return if records.empty?
+
+      notify_bulk(records, "Import") do
+        queue_index(records)
+      end
     end
     alias_method :import, :bulk_index
 
     def bulk_update(records, method_name)
-      bulk_update_helper(records, method_name)
-    end
+      return if records.empty?
 
-    def record_data(r)
-      data = {
-        _index: name,
-        _id: search_id(r),
-        _type: document_type(r)
-      }
-      data[:_routing] = r.search_routing if r.respond_to?(:search_routing)
-      data
-    end
-
-    def retrieve(record)
-      client.get(
-        index: name,
-        type: document_type(record),
-        id: search_id(record)
-      )["_source"]
+      notify_bulk(records, "Update") do
+        queue_update(records, method_name)
+      end
     end
 
-    def reindex_record(record)
-      if record.destroyed? || !record.should_index?
-        begin
-          remove(record)
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          # do nothing
-        end
-      else
-        store(record)
-      end
+    def search_id(record)
+      RecordData.new(self, record).search_id
     end
 
-    def reindex_record_async(record)
-      if Searchkick.callbacks_value.nil?
-        if defined?(Searchkick::ReindexV2Job)
-          Searchkick::ReindexV2Job.perform_later(record.class.name, record.id.to_s)
-        else
-          raise Searchkick::Error, "Active Job not found"
-        end
-      else
-        reindex_record(record)
-      end
+    def document_type(record)
+      RecordData.new(self, record).document_type
     end
 
     def similar_record(record, **options)
-      like_text = retrieve(record).to_hash
-        .keep_if { |k, _| !options[:fields] || options[:fields].map(&:to_s).include?(k) }
-        .values.compact.join(" ")
-
-      # TODO deep merge method
-      options[:where] ||= {}
-      options[:where][:_id] ||= {}
-      options[:where][:_id][:not] = record.id.to_s
       options[:per_page] ||= 10
-      options[:similar] = true
+      options[:similar] = [RecordData.new(self, record).record_data]
+      options[:models] ||= [record.class] unless options.key?(:model)
 
-      # TODO use index class instead of record class
-      search_model(record.class, like_text, options)
+      Searchkick.search("*", **options)
+    end
+
+    def reload_synonyms
+      if Searchkick.opensearch?
+        client.transport.perform_request "POST", "_plugins/_refresh_search_analyzers/#{CGI.escape(name)}"
+      else
+        raise Error, "Requires Elasticsearch 7.3+" if Searchkick.server_below?("7.3.0")
+        begin
+          client.transport.perform_request("GET", "#{CGI.escape(name)}/_reload_search_analyzers")
+        rescue => e
+          raise Error, "Requires non-OSS version of Elasticsearch" if Searchkick.not_allowed_error?(e)
+          raise e
+        end
+      end
     end
 
     # queue
 
     def reindex_queue
-      Searchkick::ReindexQueue.new(name)
+      ReindexQueue.new(name)
     end
 
-    # search
+    # reindex
+
+    # note: this is designed to be used internally
+    # so it does not check object matches index class
+    def reindex(object, method_name: nil, full: false, **options)
+      if object.is_a?(Array)
+        # note: purposefully skip full
+        return reindex_records(object, method_name: method_name, **options)
+      end
+
+      if !object.respond_to?(:searchkick_klass)
+        raise Error, "Cannot reindex object"
+      end
+
+      scoped = Searchkick.relation?(object)
+      # call searchkick_klass for inheritance
+      relation = scoped ? object.all : Searchkick.scope(object.searchkick_klass).all
+
+      refresh = options.fetch(:refresh, !scoped)
+      options.delete(:refresh)
 
-    # TODO remove in next major version
-    def search_model(searchkick_klass, term = "*", **options, &block)
-      query = Searchkick::Query.new(searchkick_klass, term, options)
-      yield(query.body) if block
-      if options[:execute] == false
-        query
+      if method_name || (scoped && !full)
+        mode = options.delete(:mode) || :inline
+        raise ArgumentError, "unsupported keywords: #{options.keys.map(&:inspect).join(", ")}" if options.any?
+
+        # import only
+        import_scope(relation, method_name: method_name, mode: mode)
+        self.refresh if refresh
+        true
       else
-        query.execute
+        async = options.delete(:async)
+        if async
+          if async.is_a?(Hash) && async[:wait]
+            # TODO warn in 5.1
+            # Searchkick.warn "async option is deprecated - use mode: :async, wait: true instead"
+            options[:wait] = true unless options.key?(:wait)
+          else
+            # TODO warn in 5.1
+            # Searchkick.warn "async option is deprecated - use mode: :async instead"
+          end
+          options[:mode] ||= :async
+        end
+
+        full_reindex(relation, **options)
       end
     end
 
-    # reindex
-
     def create_index(index_options: nil)
       index_options ||= self.index_options
-      index = Searchkick::Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options)
+      index = Index.new("#{name}_#{Time.now.strftime('%Y%m%d%H%M%S%L')}", @options)
       index.create(index_options)
       index
     end
 
-    def all_indices(unaliased: false)
-      indices =
-        begin
-          client.indices.get_aliases
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound
-          {}
+    def import_scope(relation, **options)
+      relation_indexer.reindex(relation, **options)
+    end
+
+    def batches_left
+      relation_indexer.batches_left
+    end
+
+    # private
+    def klass_document_type(klass, ignore_type = false)
+      @klass_document_type[[klass, ignore_type]] ||= begin
+        if !ignore_type && klass.searchkick_klass.searchkick_options[:_type]
+          type = klass.searchkick_klass.searchkick_options[:_type]
+          type = type.call if type.respond_to?(:call)
+          type
+        else
+          klass.model_name.to_s.underscore
         end
-      indices = indices.select { |_k, v| v.empty? || v["aliases"].empty? } if unaliased
-      indices.select { |k, _v| k =~ /\A#{Regexp.escape(name)}_\d{14,17}\z/ }.keys
+      end
     end
 
-    # remove old indices that start w/ index_name
-    def clean_indices
-      indices = all_indices(unaliased: true)
-      indices.each do |index|
-        Searchkick::Index.new(index).delete
+    # private
+    def conversions_fields
+      @conversions_fields ||= begin
+        conversions = Array(options[:conversions])
+        conversions.map(&:to_s) + conversions.map(&:to_sym)
       end
-      indices
     end
 
-    def total_docs
-      response =
-        client.search(
-          index: name,
-          body: {
-            query: {match_all: {}},
-            size: 0
-          }
-        )
+    # private
+    def suggest_fields
+      @suggest_fields ||= Array(options[:suggest]).map(&:to_s)
+    end
+
+    # private
+    def locations_fields
+      @locations_fields ||= begin
+        locations = Array(options[:locations])
+        locations.map(&:to_s) + locations.map(&:to_sym)
+      end
+    end
+
+    # private
+    def uuid
+      index_settings["uuid"]
+    end
 
-      response["hits"]["total"]
+    protected
+
+    def client
+      Searchkick.client
+    end
+
+    def queue_index(records)
+      Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).index_data })
+    end
+
+    def queue_delete(records)
+      Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(self, r).delete_data })
+    end
+
+    def queue_update(records, method_name)
+      Searchkick.indexer.queue(records.map { |r| RecordData.new(self, r).update_data(method_name) })
+    end
+
+    def relation_indexer
+      @relation_indexer ||= RelationIndexer.new(self)
+    end
+
+    def index_settings
+      settings.values.first["settings"]["index"]
+    end
+
+    def import_before_promotion(index, relation, **import_options)
+      index.import_scope(relation, **import_options)
+    end
+
+    def reindex_records(object, mode: nil, refresh: false, **options)
+      mode ||= Searchkick.callbacks_value || @options[:callbacks] || :inline
+      mode = :inline if mode == :bulk
+
+      result = RecordIndexer.new(self).reindex(object, mode: mode, full: false, **options)
+      self.refresh if refresh
+      result
     end
 
     # https://gist.github.com/jarosan/3124884
     # http://www.elasticsearch.org/blog/changing-mapping-with-zero-downtime/
-    def reindex_scope(scope, import: true, resume: false, retain: false, async: false, refresh_interval: nil)
+    def full_reindex(relation, import: true, resume: false, retain: false, mode: nil, refresh_interval: nil, scope: nil, wait: nil)
+      raise ArgumentError, "wait only available in :async mode" if !wait.nil? && mode != :async
+
       if resume
         index_name = all_indices.sort.last
-        raise Searchkick::Error, "No index to resume" unless index_name
-        index = Searchkick::Index.new(index_name)
+        raise Error, "No index to resume" unless index_name
+        index = Index.new(index_name, @options)
       else
         clean_indices unless retain
 
-        index_options = scope.searchkick_index_options
+        index_options = relation.searchkick_index_options
         index_options.deep_merge!(settings: {index: {refresh_interval: refresh_interval}}) if refresh_interval
         index = create_index(index_options: index_options)
       end
 
+      import_options = {
+        mode: (mode || :inline),
+        full: true,
+        resume: resume,
+        scope: scope
+      }
+
+      uuid = index.uuid
+
       # check if alias exists
       alias_exists = alias_exists?
       if alias_exists
-        # import before promotion
-        index.import_scope(scope, resume: resume, async: async, full: true) if import
+        import_before_promotion(index, relation, **import_options) if import
 
         # get existing indices to remove
-        unless async
+        unless mode == :async
+          check_uuid(uuid, index.uuid)
           promote(index.name, update_refresh_interval: !refresh_interval.nil?)
           clean_indices unless retain
         end
@@ -249,11 +389,11 @@ module Searchkick
         promote(index.name, update_refresh_interval: !refresh_interval.nil?)
 
         # import after promotion
-        index.import_scope(scope, resume: resume, async: async, full: true) if import
+        index.import_scope(relation, **import_options) if import
       end
 
-      if async
-        if async.is_a?(Hash) && async[:wait]
+      if mode == :async
+        if wait
           puts "Created index: #{index.name}"
           puts "Jobs queued. Waiting..."
           loop do
@@ -265,6 +405,7 @@ module Searchkick
           # already promoted if alias didn't exist
           if alias_exists
             puts "Jobs complete. Promoting..."
+            check_uuid(uuid, index.uuid)
             promote(index.name, update_refresh_interval: !refresh_interval.nil?)
           end
           clean_indices unless retain
@@ -276,267 +417,51 @@ module Searchkick
         index.refresh
         true
       end
-    end
-
-    def import_scope(scope, resume: false, method_name: nil, async: false, batch: false, batch_id: nil, full: false)
-      # use scope for import
-      scope = scope.search_import if scope.respond_to?(:search_import)
-
-      if batch
-        import_or_update scope.to_a, method_name, async
-        Searchkick.with_redis { |r| r.srem(batches_key, batch_id) } if batch_id
-      elsif full && async
-        full_reindex_async(scope)
-      elsif scope.respond_to?(:find_in_batches)
-        if resume
-          # use total docs instead of max id since there's not a great way
-          # to get the max _id without scripting since it's a string
-
-          # TODO use primary key and prefix with table name
-          scope = scope.where("id > ?", total_docs)
-        end
-
-        scope = scope.select("id").except(:includes, :preload) if async
-
-        scope.find_in_batches batch_size: batch_size do |items|
-          import_or_update items, method_name, async
-        end
-      else
-        each_batch(scope) do |items|
-          import_or_update items, method_name, async
-        end
-      end
-    end
-
-    def batches_left
-      Searchkick.with_redis { |r| r.scard(batches_key) }
-    end
-
-    # other
-
-    def tokens(text, options = {})
-      client.indices.analyze(body: {text: text}.merge(options), index: name)["tokens"].map { |t| t["token"] }
-    end
-
-    def klass_document_type(klass)
-      @klass_document_type[klass] ||= begin
-        if klass.respond_to?(:document_type)
-          klass.document_type
-        else
-          klass.model_name.to_s.underscore
-        end
+    rescue => e
+      if Searchkick.transport_error?(e) && (e.message.include?("No handler for type [text]") || e.message.include?("class java.util.ArrayList cannot be cast to class java.util.Map"))
+        raise UnsupportedVersionError
       end
-    end
-
-    protected
 
-    def client
-      Searchkick.client
+      raise e
     end
 
-    def document_type(record)
-      if record.respond_to?(:search_document_type)
-        record.search_document_type
-      else
-        klass_document_type(record.class)
+    # safety check
+    # still a chance for race condition since its called before promotion
+    # ideal is for user to disable automatic index creation
+    # https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#index-creation
+    def check_uuid(old_uuid, new_uuid)
+      if old_uuid != new_uuid
+        raise Error, "Safety check failed - only run one Model.reindex per model at a time"
       end
     end
 
-    def search_id(record)
-      id = record.respond_to?(:search_document_id) ? record.search_document_id : record.id
-      id.is_a?(Numeric) ? id : id.to_s
-    end
-
-    EXCLUDED_ATTRIBUTES = ["_id", "_type"]
-
-    def search_data(record, method_name = nil)
-      partial_reindex = !method_name.nil?
-      options = record.class.searchkick_options
-
-      # remove _id since search_id is used instead
-      source = record.send(method_name || :search_data).each_with_object({}) { |(k, v), memo| memo[k.to_s] = v; memo }.except(*EXCLUDED_ATTRIBUTES)
-
-      # conversions
-      if options[:conversions]
-        Array(options[:conversions]).map(&:to_s).each do |conversions_field|
-          if source[conversions_field]
-            source[conversions_field] = source[conversions_field].map { |k, v| {query: k, count: v} }
-          end
-        end
-      end
-
-      # hack to prevent generator field doesn't exist error
-      if options[:suggest]
-        options[:suggest].map(&:to_s).each do |field|
-          source[field] = nil if !source[field] && !partial_reindex
-        end
-      end
-
-      # locations
-      if options[:locations]
-        options[:locations].map(&:to_s).each do |field|
-          if source[field]
-            if !source[field].is_a?(Hash) && (source[field].first.is_a?(Array) || source[field].first.is_a?(Hash))
-              # multiple locations
-              source[field] = source[field].map { |a| location_value(a) }
-            else
-              source[field] = location_value(source[field])
-            end
-          end
-        end
-      end
-
-      cast_big_decimal(source)
-
-      source
-    end
-
-    def location_value(value)
-      if value.is_a?(Array)
-        value.map(&:to_f).reverse
-      elsif value.is_a?(Hash)
-        {lat: value[:lat].to_f, lon: value[:lon].to_f}
-      else
-        value
-      end
-    end
-
-    # change all BigDecimal values to floats due to
-    # https://github.com/rails/rails/issues/6033
-    # possible loss of precision :/
-    def cast_big_decimal(obj)
-      case obj
-      when BigDecimal
-        obj.to_f
-      when Hash
-        obj.each do |k, v|
-          obj[k] = cast_big_decimal(v)
-        end
-      when Enumerable
-        obj.map do |v|
-          cast_big_decimal(v)
-        end
-      else
-        obj
-      end
-    end
-
-    def import_or_update(records, method_name, async)
-      if records.any?
-        if async
-          Searchkick::BulkReindexJob.perform_later(
-            class_name: records.first.class.name,
-            record_ids: records.map(&:id),
-            index_name: name,
-            method_name: method_name ? method_name.to_s : nil
-          )
-        else
-          records = records.select(&:should_index?)
-          if records.any?
-            with_retries do
-              method_name ? bulk_update(records, method_name) : import(records)
-            end
-          end
-        end
-      end
-    end
-
-    def full_reindex_async(scope)
-      if scope.respond_to?(:primary_key)
-        # TODO expire Redis key
-        primary_key = scope.primary_key
-
-        starting_id =
-          begin
-            scope.minimum(primary_key)
-          rescue ActiveRecord::StatementInvalid
-            false
-          end
-
-        if starting_id.nil?
-          # no records, do nothing
-        elsif starting_id.is_a?(Numeric)
-          max_id = scope.maximum(primary_key)
-          batches_count = ((max_id - starting_id + 1) / batch_size.to_f).ceil
-
-          batches_count.times do |i|
-            batch_id = i + 1
-            min_id = starting_id + (i * batch_size)
-            bulk_reindex_job scope, batch_id, min_id: min_id, max_id: min_id + batch_size - 1
-          end
-        else
-          scope.find_in_batches(batch_size: batch_size).each_with_index do |batch, i|
-            batch_id = i + 1
-
-            bulk_reindex_job scope, batch_id, record_ids: batch.map { |record| record.id.to_s }
-          end
-        end
+    def notify(record, name)
+      if Searchkick.callbacks_value == :bulk
+        yield
       else
-        batch_id = 1
-        # TODO remove any eager loading
-        scope = scope.only(:_id) if scope.respond_to?(:only)
-        each_batch(scope) do |items|
-          bulk_reindex_job scope, batch_id, record_ids: items.map { |i| i.id.to_s }
-          batch_id += 1
-        end
-      end
-    end
-
-    def each_batch(scope)
-      # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
-      # use cursor for Mongoid
-      items = []
-      scope.all.each do |item|
-        items << item
-        if items.length == batch_size
-          yield items
-          items = []
+        name = "#{record.class.searchkick_klass.name} #{name}" if record && record.class.searchkick_klass
+        event = {
+          name: name,
+          id: search_id(record)
+        }
+        ActiveSupport::Notifications.instrument("request.searchkick", event) do
+          yield
         end
       end
-      yield items if items.any?
     end
 
-    def bulk_reindex_job(scope, batch_id, options)
-      Searchkick::BulkReindexJob.perform_later({
-        class_name: scope.model_name.name,
-        index_name: name,
-        batch_id: batch_id
-      }.merge(options))
-      Searchkick.with_redis { |r| r.sadd(batches_key, batch_id) }
-    end
-
-    def batch_size
-      @batch_size ||= @options[:batch_size] || 1000
-    end
-
-    def with_retries
-      retries = 0
-
-      begin
+    def notify_bulk(records, name)
+      if Searchkick.callbacks_value == :bulk
         yield
-      rescue Faraday::ClientError => e
-        if retries < 1
-          retries += 1
-          retry
+      else
+        event = {
+          name: "#{records.first.class.searchkick_klass.name} #{name}",
+          count: records.size
+        }
+        ActiveSupport::Notifications.instrument("request.searchkick", event) do
+          yield
         end
-        raise e
       end
     end
-
-    def bulk_index_helper(records)
-      Searchkick.indexer.queue(records.map { |r| {index: record_data(r).merge(data: search_data(r))} })
-    end
-
-    def bulk_delete_helper(records)
-      Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| {delete: record_data(r)} })
-    end
-
-    def bulk_update_helper(records, method_name)
-      Searchkick.indexer.queue(records.map { |r| {update: record_data(r).merge(data: {doc: search_data(r, method_name)})} })
-    end
-
-    def batches_key
-      "searchkick:reindex:#{name}:batches"
-    end
   end
 end