searchkick 4.0.0 → 5.0.0

data/lib/searchkick/bulk_indexer.rb

@@ -1,171 +0,0 @@
-module Searchkick
-  class BulkIndexer
-    attr_reader :index
-
-    def initialize(index)
-      @index = index
-    end
-
-    def import_scope(relation, resume: false, method_name: nil, async: false, batch: false, batch_id: nil, full: false, scope: nil)
-      if scope
-        relation = relation.send(scope)
-      elsif relation.respond_to?(:search_import)
-        relation = relation.search_import
-      end
-
-      if batch
-        import_or_update relation.to_a, method_name, async
-        Searchkick.with_redis { |r| r.srem(batches_key, batch_id) } if batch_id
-      elsif full && async
-        full_reindex_async(relation)
-      elsif relation.respond_to?(:find_in_batches)
-        if resume
-          # use total docs instead of max id since there's not a great way
-          # to get the max _id without scripting since it's a string
-
-          # TODO use primary key and prefix with table name
-          relation = relation.where("id > ?", index.total_docs)
-        end
-
-        relation = relation.select("id").except(:includes, :preload) if async
-
-        relation.find_in_batches batch_size: batch_size do |items|
-          import_or_update items, method_name, async
-        end
-      else
-        each_batch(relation) do |items|
-          import_or_update items, method_name, async
-        end
-      end
-    end
-
-    def bulk_index(records)
-      Searchkick.indexer.queue(records.map { |r| RecordData.new(index, r).index_data })
-    end
-
-    def bulk_delete(records)
-      Searchkick.indexer.queue(records.reject { |r| r.id.blank? }.map { |r| RecordData.new(index, r).delete_data })
-    end
-
-    def bulk_update(records, method_name)
-      Searchkick.indexer.queue(records.map { |r| RecordData.new(index, r).update_data(method_name) })
-    end
-
-    def batches_left
-      Searchkick.with_redis { |r| r.scard(batches_key) }
-    end
-
-    private
-
-    def import_or_update(records, method_name, async)
-      if records.any?
-        if async
-          Searchkick::BulkReindexJob.perform_later(
-            class_name: records.first.class.name,
-            record_ids: records.map(&:id),
-            index_name: index.name,
-            method_name: method_name ? method_name.to_s : nil
-          )
-        else
-          records = records.select(&:should_index?)
-          if records.any?
-            with_retries do
-              # call out to index for ActiveSupport notifications
-              if method_name
-                index.bulk_update(records, method_name)
-              else
-                index.bulk_index(records)
-              end
-            end
-          end
-        end
-      end
-    end
-
-    def full_reindex_async(scope)
-      if scope.respond_to?(:primary_key)
-        # TODO expire Redis key
-        primary_key = scope.primary_key
-
-        starting_id =
-          begin
-            scope.minimum(primary_key)
-          rescue ActiveRecord::StatementInvalid
-            false
-          end
-
-        if starting_id.nil?
-          # no records, do nothing
-        elsif starting_id.is_a?(Numeric)
-          max_id = scope.maximum(primary_key)
-          batches_count = ((max_id - starting_id + 1) / batch_size.to_f).ceil
-
-          batches_count.times do |i|
-            batch_id = i + 1
-            min_id = starting_id + (i * batch_size)
-            bulk_reindex_job scope, batch_id, min_id: min_id, max_id: min_id + batch_size - 1
-          end
-        else
-          scope.find_in_batches(batch_size: batch_size).each_with_index do |batch, i|
-            batch_id = i + 1
-
-            bulk_reindex_job scope, batch_id, record_ids: batch.map { |record| record.id.to_s }
-          end
-        end
-      else
-        batch_id = 1
-        # TODO remove any eager loading
-        scope = scope.only(:_id) if scope.respond_to?(:only)
-        each_batch(scope) do |items|
-          bulk_reindex_job scope, batch_id, record_ids: items.map { |i| i.id.to_s }
-          batch_id += 1
-        end
-      end
-    end
-
-    def each_batch(scope)
-      # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb
-      # use cursor for Mongoid
-      items = []
-      scope.all.each do |item|
-        items << item
-        if items.length == batch_size
-          yield items
-          items = []
-        end
-      end
-      yield items if items.any?
-    end
-
-    def bulk_reindex_job(scope, batch_id, options)
-      Searchkick.with_redis { |r| r.sadd(batches_key, batch_id) }
-      Searchkick::BulkReindexJob.perform_later({
-        class_name: scope.model_name.name,
-        index_name: index.name,
-        batch_id: batch_id
-      }.merge(options))
-    end
-
-    def with_retries
-      retries = 0
-
-      begin
-        yield
-      rescue Faraday::ClientError => e
-        if retries < 1
-          retries += 1
-          retry
-        end
-        raise e
-      end
-    end
-
-    def batches_key
-      "searchkick:reindex:#{index.name}:batches"
-    end
-
-    def batch_size
-      @batch_size ||= index.options[:batch_size] || 1000
-    end
-  end
-end

data/lib/searchkick/logging.rb

@@ -1,243 +0,0 @@
-# based on https://gist.github.com/mnutt/566725
-require "active_support/core_ext/module/attr_internal"
-
-module Searchkick
-  module QueryWithInstrumentation
-    def execute_search
-      name = searchkick_klass ? "#{searchkick_klass.name} Search" : "Search"
-      event = {
-        name: name,
-        query: params
-      }
-      ActiveSupport::Notifications.instrument("search.searchkick", event) do
-        super
-      end
-    end
-  end
-
-  module IndexWithInstrumentation
-    def store(record)
-      event = {
-        name: "#{record.searchkick_klass.name} Store",
-        id: search_id(record)
-      }
-      if Searchkick.callbacks_value == :bulk
-        super
-      else
-        ActiveSupport::Notifications.instrument("request.searchkick", event) do
-          super
-        end
-      end
-    end
-
-    def remove(record)
-      name = record && record.searchkick_klass ? "#{record.searchkick_klass.name} Remove" : "Remove"
-      event = {
-        name: name,
-        id: search_id(record)
-      }
-      if Searchkick.callbacks_value == :bulk
-        super
-      else
-        ActiveSupport::Notifications.instrument("request.searchkick", event) do
-          super
-        end
-      end
-    end
-
-    def update_record(record, method_name)
-      event = {
-        name: "#{record.searchkick_klass.name} Update",
-        id: search_id(record)
-      }
-      if Searchkick.callbacks_value == :bulk
-        super
-      else
-        ActiveSupport::Notifications.instrument("request.searchkick", event) do
-          super
-        end
-      end
-    end
-
-    def bulk_index(records)
-      if records.any?
-        event = {
-          name: "#{records.first.searchkick_klass.name} Import",
-          count: records.size
-        }
-        event[:id] = search_id(records.first) if records.size == 1
-        if Searchkick.callbacks_value == :bulk
-          super
-        else
-          ActiveSupport::Notifications.instrument("request.searchkick", event) do
-            super
-          end
-        end
-      end
-    end
-    alias_method :import, :bulk_index
-
-    def bulk_update(records, *args)
-      if records.any?
-        event = {
-          name: "#{records.first.searchkick_klass.name} Update",
-          count: records.size
-        }
-        event[:id] = search_id(records.first) if records.size == 1
-        if Searchkick.callbacks_value == :bulk
-          super
-        else
-          ActiveSupport::Notifications.instrument("request.searchkick", event) do
-            super
-          end
-        end
-      end
-    end
-
-    def bulk_delete(records)
-      if records.any?
-        event = {
-          name: "#{records.first.searchkick_klass.name} Delete",
-          count: records.size
-        }
-        event[:id] = search_id(records.first) if records.size == 1
-        if Searchkick.callbacks_value == :bulk
-          super
-        else
-          ActiveSupport::Notifications.instrument("request.searchkick", event) do
-            super
-          end
-        end
-      end
-    end
-  end
-
-  module IndexerWithInstrumentation
-    def perform
-      if Searchkick.callbacks_value == :bulk
-        event = {
-          name: "Bulk",
-          count: queued_items.size
-        }
-        ActiveSupport::Notifications.instrument("request.searchkick", event) do
-          super
-        end
-      else
-        super
-      end
-    end
-  end
-
-  module SearchkickWithInstrumentation
-    def multi_search(searches)
-      event = {
-        name: "Multi Search",
-        body: searches.flat_map { |q| [q.params.except(:body).to_json, q.body.to_json] }.map { |v| "#{v}\n" }.join
-      }
-      ActiveSupport::Notifications.instrument("multi_search.searchkick", event) do
-        super
-      end
-    end
-  end
-
-  # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/log_subscriber.rb
-  class LogSubscriber < ActiveSupport::LogSubscriber
-    def self.runtime=(value)
-      Thread.current[:searchkick_runtime] = value
-    end
-
-    def self.runtime
-      Thread.current[:searchkick_runtime] ||= 0
-    end
-
-    def self.reset_runtime
-      rt = runtime
-      self.runtime = 0
-      rt
-    end
-
-    def search(event)
-      self.class.runtime += event.duration
-      return unless logger.debug?
-
-      payload = event.payload
-      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
-      type = payload[:query][:type]
-      index = payload[:query][:index].is_a?(Array) ? payload[:query][:index].join(",") : payload[:query][:index]
-
-      # no easy way to tell which host the client will use
-      host = Searchkick.client.transport.hosts.first
-      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/#{CGI.escape(index)}#{type ? "/#{type.map { |t| CGI.escape(t) }.join(',')}" : ''}/_search?pretty -H 'Content-Type: application/json' -d '#{payload[:query][:body].to_json}'"
-    end
-
-    def request(event)
-      self.class.runtime += event.duration
-      return unless logger.debug?
-
-      payload = event.payload
-      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
-
-      debug "  #{color(name, YELLOW, true)}  #{payload.except(:name).to_json}"
-    end
-
-    def multi_search(event)
-      self.class.runtime += event.duration
-      return unless logger.debug?
-
-      payload = event.payload
-      name = "#{payload[:name]} (#{event.duration.round(1)}ms)"
-
-      # no easy way to tell which host the client will use
-      host = Searchkick.client.transport.hosts.first
-      debug "  #{color(name, YELLOW, true)}  curl #{host[:protocol]}://#{host[:host]}:#{host[:port]}/_msearch?pretty -H 'Content-Type: application/json' -d '#{payload[:body]}'"
-    end
-  end
-
-  # https://github.com/rails/rails/blob/master/activerecord/lib/active_record/railties/controller_runtime.rb
-  module ControllerRuntime
-    extend ActiveSupport::Concern
-
-    protected
-
-    attr_internal :searchkick_runtime
-
-    def process_action(action, *args)
-      # We also need to reset the runtime before each action
-      # because of queries in middleware or in cases we are streaming
-      # and it won't be cleaned up by the method below.
-      Searchkick::LogSubscriber.reset_runtime
-      super
-    end
-
-    def cleanup_view_runtime
-      searchkick_rt_before_render = Searchkick::LogSubscriber.reset_runtime
-      runtime = super
-      searchkick_rt_after_render = Searchkick::LogSubscriber.reset_runtime
-      self.searchkick_runtime = searchkick_rt_before_render + searchkick_rt_after_render
-      runtime - searchkick_rt_after_render
-    end
-
-    def append_info_to_payload(payload)
-      super
-      payload[:searchkick_runtime] = (searchkick_runtime || 0) + Searchkick::LogSubscriber.reset_runtime
-    end
-
-    module ClassMethods
-      def log_process_action(payload)
-        messages = super
-        runtime = payload[:searchkick_runtime]
-        messages << ("Searchkick: %.1fms" % runtime.to_f) if runtime.to_f > 0
-        messages
-      end
-    end
-  end
-end
-
-Searchkick::Query.prepend(Searchkick::QueryWithInstrumentation)
-Searchkick::Index.prepend(Searchkick::IndexWithInstrumentation)
-Searchkick::Indexer.prepend(Searchkick::IndexerWithInstrumentation)
-Searchkick.singleton_class.prepend(Searchkick::SearchkickWithInstrumentation)
-Searchkick::LogSubscriber.attach_to :searchkick
-ActiveSupport.on_load(:action_controller) do
-  include Searchkick::ControllerRuntime
-end