searchkick 4.6.3 → 5.5.2

data/lib/searchkick/record_indexer.rb

@@ -1,79 +1,163 @@
 module Searchkick
   class RecordIndexer
-    attr_reader :record, :index
+    attr_reader :index
 
-    def initialize(record)
-      @record = record
-      @index = record.class.searchkick_index
+    def initialize(index)
+      @index = index
     end
 
-    def reindex(method_name = nil, refresh: false, mode: nil)
-      unless [:inline, true, nil, :async, :queue].include?(mode)
-        raise ArgumentError, "Invalid value for mode"
-      end
-
-      mode ||= Searchkick.callbacks_value || index.options[:callbacks] || true
+    def reindex(records, mode:, method_name:, full: false, single: false)
+      # prevents exists? check if records is a relation
+      records = records.to_a
+      return if records.empty?
 
       case mode
+      when :async
+        unless defined?(ActiveJob)
+          raise Error, "Active Job not found"
+        end
+
+        # we could likely combine ReindexV2Job, BulkReindexJob, and ProcessBatchJob
+        # but keep them separate for now
+        if single
+          record = records.first
+
+          # always pass routing in case record is deleted
+          # before the async job runs
+          if record.respond_to?(:search_routing)
+            routing = record.search_routing
+          end
+
+          Searchkick::ReindexV2Job.perform_later(
+            record.class.name,
+            record.id.to_s,
+            method_name ? method_name.to_s : nil,
+            routing: routing,
+            index_name: index.name
+          )
+        else
+          Searchkick::BulkReindexJob.perform_later(
+            class_name: records.first.class.searchkick_options[:class_name],
+            record_ids: records.map { |r| r.id.to_s },
+            index_name: index.name,
+            method_name: method_name ? method_name.to_s : nil
+          )
+        end
       when :queue
         if method_name
-          raise Searchkick::Error, "Partial reindex not supported with queue option"
+          raise Error, "Partial reindex not supported with queue option"
         end
 
-        # always pass routing in case record is deleted
-        # before the queue job runs
-        if record.respond_to?(:search_routing)
-          routing = record.search_routing
-        end
+        index.reindex_queue.push_records(records)
+      when true, :inline
+        index_records, other_records = records.partition { |r| index_record?(r) }
+        import_inline(index_records, !full ? other_records : [], method_name: method_name, single: single)
+      else
+        raise ArgumentError, "Invalid value for mode"
+      end
 
-        # escape pipe with double pipe
-        value = queue_escape(record.id.to_s)
-        value = "#{value}|#{queue_escape(routing)}" if routing
-        index.reindex_queue.push(value)
-      when :async
-        unless defined?(ActiveJob)
-          raise Searchkick::Error, "Active Job not found"
-        end
+      # return true like model and relation reindex for now
+      true
+    end
 
-        # always pass routing in case record is deleted
-        # before the async job runs
-        if record.respond_to?(:search_routing)
-          routing = record.search_routing
-        end
+    def reindex_items(klass, items, method_name:, single: false)
+      routing = items.to_h { |r| [r[:id], r[:routing]] }
+      record_ids = routing.keys
 
-        Searchkick::ReindexV2Job.perform_later(
-          record.class.name,
-          record.id.to_s,
-          method_name ? method_name.to_s : nil,
-          routing: routing
-        )
-      else # bulk, inline/true/nil
-        reindex_record(method_name)
+      relation = Searchkick.load_records(klass, record_ids)
+      # call search_import even for single records for nested associations
+      relation = relation.search_import if relation.respond_to?(:search_import)
+      records = relation.select(&:should_index?)
 
-        index.refresh if refresh
-      end
+      # determine which records to delete
+      delete_ids = record_ids - records.map { |r| r.id.to_s }
+      delete_records =
+        delete_ids.map do |id|
+          construct_record(klass, id, routing[id])
+        end
+
+      import_inline(records, delete_records, method_name: method_name, single: single)
     end
 
     private
 
-    def queue_escape(value)
-      value.gsub("|", "||")
+    def index_record?(record)
+      record.persisted? && !record.destroyed? && record.should_index?
     end
 
-    def reindex_record(method_name)
-      if record.destroyed? || !record.persisted? || !record.should_index?
-        begin
-          index.remove(record)
-        rescue => e
-          raise e unless Searchkick.not_found_error?(e)
-          # do nothing if not found
+    # import in single request with retries
+    def import_inline(index_records, delete_records, method_name:, single:)
+      return if index_records.empty? && delete_records.empty?
+
+      maybe_bulk(index_records, delete_records, method_name, single) do
+        if index_records.any?
+          if method_name
+            index.bulk_update(index_records, method_name)
+          else
+            index.bulk_index(index_records)
+          end
         end
+
+        if delete_records.any?
+          index.bulk_delete(delete_records)
+        end
+      end
+    end
+
+    def maybe_bulk(index_records, delete_records, method_name, single)
+      if Searchkick.callbacks_value == :bulk
+        yield
       else
-        if method_name
-          index.update_record(record, method_name)
-        else
-          index.store(record)
+        # set action and data
+        action =
+          if single && index_records.empty?
+            "Remove"
+          elsif method_name
+            "Update"
+          else
+            single ? "Store" : "Import"
+          end
+        record = index_records.first || delete_records.first
+        name = record.class.searchkick_klass.name
+        message = lambda do |event|
+          event[:name] = "#{name} #{action}"
+          if single
+            event[:id] = index.search_id(record)
+          else
+            event[:count] = index_records.size + delete_records.size
+          end
+        end
+
+        with_retries do
+          Searchkick.callbacks(:bulk, message: message) do
+            yield
+          end
+        end
+      end
+    end
+
+    def construct_record(klass, id, routing)
+      record = klass.new
+      record.id = id
+      if routing
+        record.define_singleton_method(:search_routing) do
+          routing
+        end
+      end
+      record
+    end
+
+    def with_retries
+      retries = 0
+
+      begin
+        yield
+      rescue Faraday::ClientError => e
+        if retries < 1
+          retries += 1
+          retry
         end
+        raise e
       end
     end
   end

data/lib/searchkick/reindex_queue.rb

@@ -5,21 +5,40 @@ module Searchkick
     def initialize(name)
       @name = name
 
-      raise Searchkick::Error, "Searchkick.redis not set" unless Searchkick.redis
+      raise Error, "Searchkick.redis not set" unless Searchkick.redis
     end
 
-    def push(record_id)
-      Searchkick.with_redis { |r| r.lpush(redis_key, record_id) }
+    # supports single and multiple ids
+    def push(record_ids)
+      Searchkick.with_redis { |r| r.call("LPUSH", redis_key, record_ids) }
+    end
+
+    def push_records(records)
+      record_ids =
+        records.map do |record|
+          # always pass routing in case record is deleted
+          # before the queue job runs
+          if record.respond_to?(:search_routing)
+            routing = record.search_routing
+          end
+
+          # escape pipe with double pipe
+          value = escape(record.id.to_s)
+          value = "#{value}|#{escape(routing)}" if routing
+          value
+        end
+
+      push(record_ids)
     end
 
     # TODO use reliable queuing
     def reserve(limit: 1000)
       if supports_rpop_with_count?
-        Searchkick.with_redis { |r| r.call("rpop", redis_key, limit) }.to_a
+        Searchkick.with_redis { |r| r.call("RPOP", redis_key, limit) }.to_a
       else
         record_ids = []
         Searchkick.with_redis do |r|
-          while record_ids.size < limit && (record_id = r.rpop(redis_key))
+          while record_ids.size < limit && (record_id = r.call("RPOP", redis_key))
             record_ids << record_id
           end
         end
@@ -28,11 +47,11 @@ module Searchkick
     end
 
     def clear
-      Searchkick.with_redis { |r| r.del(redis_key) }
+      Searchkick.with_redis { |r| r.call("DEL", redis_key) }
     end
 
     def length
-      Searchkick.with_redis { |r| r.llen(redis_key) }
+      Searchkick.with_redis { |r| r.call("LLEN", redis_key) }
     end
 
     private
@@ -46,7 +65,16 @@ module Searchkick
     end
 
     def redis_version
-      @redis_version ||= Searchkick.with_redis { |r| Gem::Version.new(r.info["redis_version"]) }
+      @redis_version ||=
+        Searchkick.with_redis do |r|
+          info = r.call("INFO")
+          matches = /redis_version:(\S+)/.match(info)
+          Gem::Version.new(matches[1])
+        end
+    end
+
+    def escape(value)
+      value.to_s.gsub("|", "||")
     end
   end
 end

data/lib/searchkick/reindex_v2_job.rb

@@ -1,41 +1,17 @@
 module Searchkick
   class ReindexV2Job < ActiveJob::Base
-    RECORD_NOT_FOUND_CLASSES = [
-      "ActiveRecord::RecordNotFound",
-      "Mongoid::Errors::DocumentNotFound",
-      "NoBrainer::Error::DocumentNotFound",
-      "Cequel::Record::RecordNotFound"
-    ]
-
     queue_as { Searchkick.queue_name }
 
-    def perform(klass, id, method_name = nil, routing: nil)
-      model = klass.constantize
-      record =
-        begin
-          if model.respond_to?(:unscoped)
-            model.unscoped.find(id)
-          else
-            model.find(id)
-          end
-        rescue => e
-          # check by name rather than rescue directly so we don't need
-          # to determine which classes are defined
-          raise e unless RECORD_NOT_FOUND_CLASSES.include?(e.class.name)
-          nil
-        end
-
-      unless record
-        record = model.new
-        record.id = id
-        if routing
-          record.define_singleton_method(:search_routing) do
-            routing
-          end
-        end
-      end
-
-      RecordIndexer.new(record).reindex(method_name, mode: :inline)
+    def perform(class_name, id, method_name = nil, routing: nil, index_name: nil)
+      model = Searchkick.load_model(class_name, allow_child: true)
+      index = model.searchkick_index(name: index_name)
+      # use should_index? to decide whether to index (not default scope)
+      # just like saving inline
+      # could use Searchkick.scope() in future
+      # but keep for now for backwards compatibility
+      model = model.unscoped if model.respond_to?(:unscoped)
+      items = [{id: id, routing: routing}]
+      RecordIndexer.new(index).reindex_items(model, items, method_name: method_name, single: true)
     end
   end
 end

data/lib/searchkick/relation.rb

@@ -0,0 +1,247 @@
+module Searchkick
+  class Relation
+    NO_DEFAULT_VALUE = Object.new
+
+    # note: modifying body directly is not supported
+    # and has no impact on query after being executed
+    # TODO freeze body object?
+    delegate :body, :params, to: :query
+    delegate_missing_to :private_execute
+
+    attr_reader :model
+    alias_method :klass, :model
+
+    def initialize(model, term = "*", **options)
+      @model = model
+      @term = term
+      @options = options
+
+      # generate query to validate options
+      query
+    end
+
+    # same as Active Record
+    def inspect
+      entries = results.first(11).map!(&:inspect)
+      entries[10] = "..." if entries.size == 11
+      "#<#{self.class.name} [#{entries.join(', ')}]>"
+    end
+
+    def execute
+      Searchkick.warn("The execute method is no longer needed")
+      load
+    end
+
+    # experimental
+    def limit(value)
+      clone.limit!(value)
+    end
+
+    # experimental
+    def limit!(value)
+      check_loaded
+      @options[:limit] = value
+      self
+    end
+
+    # experimental
+    def offset(value = NO_DEFAULT_VALUE)
+      # TODO remove in Searchkick 6
+      if value == NO_DEFAULT_VALUE
+        private_execute.offset
+      else
+        clone.offset!(value)
+      end
+    end
+
+    # experimental
+    def offset!(value)
+      check_loaded
+      @options[:offset] = value
+      self
+    end
+
+    # experimental
+    def page(value)
+      clone.page!(value)
+    end
+
+    # experimental
+    def page!(value)
+      check_loaded
+      @options[:page] = value
+      self
+    end
+
+    # experimental
+    def per_page(value = NO_DEFAULT_VALUE)
+      # TODO remove in Searchkick 6
+      if value == NO_DEFAULT_VALUE
+        private_execute.per_page
+      else
+        clone.per_page!(value)
+      end
+    end
+
+    # experimental
+    def per_page!(value)
+      check_loaded
+      @options[:per_page] = value
+      self
+    end
+
+    # experimental
+    def where(value = NO_DEFAULT_VALUE)
+      if value == NO_DEFAULT_VALUE
+        Where.new(self)
+      else
+        clone.where!(value)
+      end
+    end
+
+    # experimental
+    def where!(value)
+      check_loaded
+      if @options[:where]
+        @options[:where] = {_and: [@options[:where], ensure_permitted(value)]}
+      else
+        @options[:where] = ensure_permitted(value)
+      end
+      self
+    end
+
+    # experimental
+    def rewhere(value)
+      clone.rewhere!(value)
+    end
+
+    # experimental
+    def rewhere!(value)
+      check_loaded
+      @options[:where] = ensure_permitted(value)
+      self
+    end
+
+    # experimental
+    def order(*values)
+      clone.order!(*values)
+    end
+
+    # experimental
+    def order!(*values)
+      values = values.first if values.size == 1 && values.first.is_a?(Array)
+      check_loaded
+      (@options[:order] ||= []).concat(values)
+      self
+    end
+
+    # experimental
+    def reorder(*values)
+      clone.reorder!(*values)
+    end
+
+    # experimental
+    def reorder!(*values)
+      check_loaded
+      @options[:order] = values
+      self
+    end
+
+    # experimental
+    def select(*values, &block)
+      if block_given?
+        private_execute.select(*values, &block)
+      else
+        clone.select!(*values)
+      end
+    end
+
+    # experimental
+    def select!(*values)
+      check_loaded
+      (@options[:select] ||= []).concat(values)
+      self
+    end
+
+    # experimental
+    def reselect(*values)
+      clone.reselect!(*values)
+    end
+
+    # experimental
+    def reselect!(*values)
+      check_loaded
+      @options[:select] = values
+      self
+    end
+
+    # experimental
+    def includes(*values)
+      clone.includes!(*values)
+    end
+
+    # experimental
+    def includes!(*values)
+      check_loaded
+      (@options[:includes] ||= []).concat(values)
+      self
+    end
+
+    # experimental
+    def only(*keys)
+      Relation.new(@model, @term, **@options.slice(*keys))
+    end
+
+    # experimental
+    def except(*keys)
+      Relation.new(@model, @term, **@options.except(*keys))
+    end
+
+    # experimental
+    def load
+      private_execute
+      self
+    end
+
+    def loaded?
+      !@execute.nil?
+    end
+
+    def respond_to_missing?(method_name, include_all)
+      Results.new(nil, nil, nil).respond_to?(method_name, include_all) || super
+    end
+
+    # TODO uncomment in 6.0
+    # def to_yaml
+    #   private_execute.to_a.to_yaml
+    # end
+
+    private
+
+    def private_execute
+      @execute ||= query.execute
+    end
+
+    def query
+      @query ||= Query.new(@model, @term, **@options)
+    end
+
+    def check_loaded
+      raise Error, "Relation loaded" if loaded?
+
+      # reset query since options will change
+      @query = nil
+    end
+
+    # provides *very* basic protection from unfiltered parameters
+    # this is not meant to be comprehensive and may be expanded in the future
+    def ensure_permitted(obj)
+      obj.to_h
+    end
+
+    def initialize_copy(other)
+      super
+      @execute = nil
+    end
+  end
+end