chewy 0.8.4 → 7.3.4

data/lib/chewy/strategy/delayed_sidekiq/scheduler.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require_relative '../../index'
+
+# The class is responsible for accumulating in redis [type, ids]
+# that were requested to be reindexed during `latency` seconds.
+# The reindex job is going to be scheduled after a `latency` seconds.
+# that job is going to read accumulated [type, ids] from the redis
+# and reindex all them at once.
+module Chewy
+  class Strategy
+    class DelayedSidekiq
+      require_relative 'worker'
+
+      class Scheduler
+        DEFAULT_TTL = 60 * 60 * 24 # in seconds
+        DEFAULT_LATENCY = 10
+        DEFAULT_MARGIN = 2
+        DEFAULT_QUEUE = 'chewy'
+        KEY_PREFIX = 'chewy:delayed_sidekiq'
+        FALLBACK_FIELDS = 'all'
+        FIELDS_IDS_SEPARATOR = ';'
+        IDS_SEPARATOR = ','
+
+        def initialize(type, ids, options = {})
+          @type = type
+          @ids = ids
+          @options = options
+        end
+
+        # the diagram:
+        #
+        #  inputs:
+        #  latency == 2
+        #  reindex_time = Time.current
+        #
+        #  Parallel OR Sequential triggers of reindex:          |  What is going on in reindex store (Redis):
+        #  --------------------------------------------------------------------------------------------------
+        #                                                       |
+        #  process 1 (reindex_time):                            |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1]
+        #    Schedule.new(CitiesIndex, [1]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & schedule a DelayedSidekiq::Worker at 1679347869 (at + 3)
+        #                                                       |    it will zpop chewy:delayed_sidekiq:timechunks up to 1679347866 score and reindex all ids with zpoped keys
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347866
+        #                                                       |
+        #                                                       |
+        #  process 2 (reindex_time):                            |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2]
+        #    Schedule.new(CitiesIndex, [2]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & do not schedule a new worker
+        #                                                       |
+        #                                                       |
+        #  process 1 (reindex_time + (latency - 1).seconds):    |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
+        #    Schedule.new(CitiesIndex, [3]).postpone            |  chewy:delayed_sidekiq:timechunks = [{ score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}]
+        #                                                       |  & do not schedule a new worker
+        #                                                       |
+        #                                                       |
+        #  process 2 (reindex_time + (latency + 1).seconds):    |  chewy:delayed_sidekiq:CitiesIndex:1679347866 = [1, 2, 3]
+        #    Schedule.new(CitiesIndex, [4]).postpone            |  chewy:delayed_sidekiq:CitiesIndex:1679347868 = [4]
+        #                                                       |  chewy:delayed_sidekiq:timechunks = [
+        #                                                       |    { score: 1679347866, "chewy:delayed_sidekiq:CitiesIndex:1679347866"}
+        #                                                       |    { score: 1679347868, "chewy:delayed_sidekiq:CitiesIndex:1679347868"}
+        #                                                       |  ]
+        #                                                       |  & schedule a DelayedSidekiq::Worker at 1679347871 (at + 3)
+        #                                                       |    it will zpop chewy:delayed_sidekiq:timechunks up to 1679347868 score and reindex all ids with zpoped keys
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347866 (in case of failed previous reindex),
+        #                                                       |      chewy:delayed_sidekiq:CitiesIndex:1679347868
+        def postpone
+          ::Sidekiq.redis do |redis|
+            # warning: Redis#sadd will always return an Integer in Redis 5.0.0. Use Redis#sadd? instead
+            if redis.respond_to?(:sadd?)
+              redis.sadd?(timechunk_key, serialize_data)
+            else
+              redis.sadd(timechunk_key, serialize_data)
+            end
+
+            redis.expire(timechunk_key, ttl)
+
+            unless redis.zrank(timechunks_key, timechunk_key)
+              redis.zadd(timechunks_key, at, timechunk_key)
+              redis.expire(timechunks_key, ttl)
+
+              ::Sidekiq::Client.push(
+                'queue' => sidekiq_queue,
+                'at' => at + margin,
+                'class' => Chewy::Strategy::DelayedSidekiq::Worker,
+                'args' => [type_name, at]
+              )
+            end
+          end
+        end
+
+      private
+
+        attr_reader :type, :ids, :options
+
+        # this method returns predictable value that jumps by latency value
+        # another words each latency seconds it return the same value
+        def at
+          @at ||= begin
+            schedule_at = latency.seconds.from_now.to_f
+
+            (schedule_at - (schedule_at % latency)).to_i
+          end
+        end
+
+        def fields
+          options[:update_fields].presence || [FALLBACK_FIELDS]
+        end
+
+        def timechunks_key
+          "#{KEY_PREFIX}:#{type_name}:timechunks"
+        end
+
+        def timechunk_key
+          "#{KEY_PREFIX}:#{type_name}:#{at}"
+        end
+
+        def serialize_data
+          [ids.join(IDS_SEPARATOR), fields.join(IDS_SEPARATOR)].join(FIELDS_IDS_SEPARATOR)
+        end
+
+        def type_name
+          type.name
+        end
+
+        def latency
+          strategy_config.latency || DEFAULT_LATENCY
+        end
+
+        def margin
+          strategy_config.margin || DEFAULT_MARGIN
+        end
+
+        def ttl
+          strategy_config.ttl || DEFAULT_TTL
+        end
+
+        def sidekiq_queue
+          Chewy.settings.dig(:sidekiq, :queue) || DEFAULT_QUEUE
+        end
+
+        def strategy_config
+          type.strategy_config.delayed_sidekiq
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/delayed_sidekiq/worker.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class DelayedSidekiq
+      class Worker
+        include ::Sidekiq::Worker
+
+        def perform(type, score, options = {})
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
+
+          ::Sidekiq.redis do |redis|
+            timechunks_key = "#{Scheduler::KEY_PREFIX}:#{type}:timechunks"
+            timechunk_keys = redis.zrangebyscore(timechunks_key, -1, score)
+            members = timechunk_keys.flat_map { |timechunk_key| redis.smembers(timechunk_key) }.compact
+
+            # extract ids and fields & do the reset of records
+            ids, fields = extract_ids_and_fields(members)
+            options[:update_fields] = fields if fields
+
+            index = type.constantize
+            index.strategy_config.delayed_sidekiq.reindex_wrapper.call do
+              options.any? ? index.import!(ids, **options) : index.import!(ids)
+            end
+
+            redis.del(timechunk_keys)
+            redis.zremrangebyscore(timechunks_key, -1, score)
+          end
+        end
+
+      private
+
+        def extract_ids_and_fields(members)
+          ids = []
+          fields = []
+
+          members.each do |member|
+            member_ids, member_fields = member.split(Scheduler::FIELDS_IDS_SEPARATOR).map do |v|
+              v.split(Scheduler::IDS_SEPARATOR)
+            end
+            ids |= member_ids
+            fields |= member_fields
+          end
+
+          fields = nil if fields.include?(Scheduler::FALLBACK_FIELDS)
+
+          [ids.map(&:to_i), fields]
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/delayed_sidekiq.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Chewy
+  class Strategy
+    class DelayedSidekiq < Sidekiq
+      require_relative 'delayed_sidekiq/scheduler'
+
+      def leave
+        @stash.each do |type, ids|
+          next if ids.empty?
+
+          DelayedSidekiq::Scheduler.new(type, ids).postpone
+        end
+      end
+    end
+  end
+end

data/lib/chewy/strategy/lazy_sidekiq.rb

@@ -0,0 +1,64 @@
+module Chewy
+  class Strategy
+    # The strategy works the same way as sidekiq, but performs
+    # async evaluation of all index callbacks on model create and update
+    # driven by sidekiq
+    #
+    #   Chewy.strategy(:lazy_sidekiq) do
+    #     User.all.map(&:save) # Does nothing here
+    #     Post.all.map(&:save) # And here
+    #     # It schedules import of all the changed users and posts right here
+    #   end
+    #
+    class LazySidekiq < Sidekiq
+      class IndicesUpdateWorker
+        include ::Sidekiq::Worker
+
+        def perform(models)
+          Chewy.strategy(strategy) do
+            models.each do |model_type, model_ids|
+              model_type.constantize.where(id: model_ids).each(&:run_chewy_callbacks)
+            end
+          end
+        end
+
+      private
+
+        def strategy
+          Chewy.disable_refresh_async ? :atomic_no_refresh : :atomic
+        end
+      end
+
+      def initialize
+        # Use parent's @stash to store destroyed records, since callbacks for them have to
+        # be run immediately on the strategy block end because we won't be able to fetch
+        # records further in IndicesUpdateWorker. This will be done by avoiding of
+        # LazySidekiq#update_chewy_indices call and calling LazySidekiq#update instead.
+        super
+
+        # @lazy_stash is used to store all the lazy evaluated callbacks with call of
+        # strategy's #update_chewy_indices.
+        @lazy_stash = {}
+      end
+
+      def leave
+        # Fallback to Sidekiq#leave implementation for destroyed records stored in @stash.
+        super
+
+        # Proceed with other records stored in @lazy_stash
+        return if @lazy_stash.empty?
+
+        ::Sidekiq::Client.push(
+          'queue' => sidekiq_queue,
+          'class' => Chewy::Strategy::LazySidekiq::IndicesUpdateWorker,
+          'args'  => [@lazy_stash]
+        )
+      end
+
+      def update_chewy_indices(object)
+        @lazy_stash[object.class.name] ||= []
+        @lazy_stash[object.class.name] |= Array.wrap(object.id)
+      end
+    end
+  end
+end

data/lib/chewy/strategy/sidekiq.rb

@@ -14,15 +14,28 @@ module Chewy
         include ::Sidekiq::Worker
 
         def perform(type, ids, options = {})
-          type.constantize.import!(ids, options)
+          options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async
+          type.constantize.import!(ids, **options)
         end
       end
 
       def leave
         @stash.each do |type, ids|
-          Chewy::Strategy::Sidekiq::Worker.perform_async(type.name, ids) unless ids.empty?
+          next if ids.empty?
+
+          ::Sidekiq::Client.push(
+            'queue' => sidekiq_queue,
+            'class' => Chewy::Strategy::Sidekiq::Worker,
+            'args'  => [type.name, ids]
+          )
         end
       end
+
+    private
+
+      def sidekiq_queue
+        Chewy.settings.dig(:sidekiq, :queue) || 'chewy'
+      end
     end
   end
 end

data/lib/chewy/strategy/urgent.rb

@@ -9,7 +9,7 @@ module Chewy
     #   end
     #
     class Urgent < Base
-      def update type, objects, options = {}
+      def update(type, objects, _options = {})
         type.import!(Array.wrap(objects))
       end
     end

data/lib/chewy/strategy.rb

@@ -2,23 +2,22 @@ require 'chewy/strategy/base'
 require 'chewy/strategy/bypass'
 require 'chewy/strategy/urgent'
 require 'chewy/strategy/atomic'
-
-begin
-  require 'resque'
-  require 'chewy/strategy/resque'
-rescue LoadError
-end
+require 'chewy/strategy/atomic_no_refresh'
 
 begin
   require 'sidekiq'
   require 'chewy/strategy/sidekiq'
+  require 'chewy/strategy/lazy_sidekiq'
+  require 'chewy/strategy/delayed_sidekiq'
 rescue LoadError
+  nil
 end
 
 begin
   require 'active_job'
   require 'chewy/strategy/active_job'
 rescue LoadError
+  nil
 end
 
 module Chewy
@@ -42,20 +41,21 @@ module Chewy
       @stack.last
     end
 
-    def push name
+    def push(name)
       result = @stack.push resolve(name).new
-      debug "[#{@stack.size}] <- #{current.name}"
+      debug "[#{@stack.size - 1}] <- #{current.name}" if @stack.size > 2
       result
     end
 
     def pop
       raise "Can't pop root strategy" if @stack.one?
-      debug "[#{@stack.size}] -> #{current.name}"
+
       result = @stack.pop.tap(&:leave)
+      debug "[#{@stack.size}] -> #{result.name}, now #{current.name}" if @stack.size > 1
       result
     end
 
-    def wrap name
+    def wrap(name)
       stack = push(name)
       yield
     ensure
@@ -64,19 +64,15 @@ module Chewy
 
   private
 
-    def debug string
-      if Chewy.logger && Chewy.logger.debug?
-        line = caller.detect { |line| line !~ %r{lib/chewy/strategy.rb:|lib/chewy.rb:} }
-        Chewy.logger.debug(["DEBUG: Chewy strategies stack: #{string}", line.sub(/:in\s.+$/, '')].join(' @ '))
-      end
+    def debug(string)
+      return unless Chewy.logger&.debug?
+
+      line = caller.detect { |l| l !~ %r{lib/chewy/strategy.rb:|lib/chewy.rb:} }
+      Chewy.logger.debug(["Chewy strategies stack: #{string}", line.sub(/:in\s.+$/, '')].join(' @ '))
     end
 
-    def resolve name
+    def resolve(name)
       "Chewy::Strategy::#{name.to_s.camelize}".safe_constantize or raise "Can't find update strategy `#{name}`"
-    rescue NameError => ex
-      # WORKAROUND: Strange behavior of `safe_constantize` with mongoid gem
-      raise "Can't find update strategy `#{name}`" if ex.name.to_s.demodulize == name.to_s.camelize
-      raise
     end
   end
 end

data/lib/chewy/version.rb

@@ -1,3 +1,3 @@
 module Chewy
-  VERSION = '0.8.4'
+  VERSION = '7.3.4'.freeze
 end

data/lib/chewy.rb

@@ -1,24 +1,40 @@
+require 'active_support/version'
 require 'active_support/concern'
 require 'active_support/deprecation'
 require 'active_support/json'
 require 'active_support/log_subscriber'
 
+require 'active_support/isolated_execution_state' if ActiveSupport::VERSION::MAJOR >= 7
 require 'active_support/core_ext/array/access'
 require 'active_support/core_ext/array/wrap'
 require 'active_support/core_ext/enumerable'
 require 'active_support/core_ext/hash/reverse_merge'
+require 'active_support/core_ext/hash/keys'
 require 'active_support/core_ext/numeric/time'
 require 'active_support/core_ext/numeric/bytes'
 require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/object/inclusion'
 require 'active_support/core_ext/string/inflections'
 
-require 'i18n/core_ext/hash'
-require 'chewy/backports/deep_dup' unless Object.respond_to?(:deep_dup)
 require 'singleton'
+require 'base64'
 
 require 'elasticsearch'
 
+def try_require(path)
+  require path
+rescue LoadError
+  nil
+end
+
+try_require 'kaminari'
+try_require 'kaminari/core'
+try_require 'parallel'
+
+ActiveSupport.on_load(:active_record) do
+  try_require 'kaminari/activerecord'
+end
+
 require 'chewy/version'
 require 'chewy/errors'
 require 'chewy/config'
@@ -28,102 +44,59 @@ require 'chewy/runtime'
 require 'chewy/log_subscriber'
 require 'chewy/strategy'
 require 'chewy/index'
-require 'chewy/type'
 require 'chewy/fields/base'
 require 'chewy/fields/root'
-require 'chewy/railtie' if defined?(::Rails)
-
-begin
-  require 'kaminari'
-  require 'chewy/query/pagination/kaminari'
-rescue LoadError
-end
-
-begin
-  require 'will_paginate'
-  require 'will_paginate/collection'
-  require 'chewy/query/pagination/will_paginate'
-rescue LoadError
-end
+require 'chewy/journal'
+require 'chewy/railtie' if defined?(::Rails::Railtie)
 
 ActiveSupport.on_load(:active_record) do
-  extend Chewy::Type::Observe::ActiveRecordMethods
-
-  begin
-    require 'will_paginate/active_record'
-  rescue LoadError
-  end
-end
-
-ActiveSupport.on_load(:mongoid) do
-  module Mongoid::Document::ClassMethods
-    include Chewy::Type::Observe::MongoidMethods
-  end
-
-  begin
-    require 'will_paginate/mongoid'
-    require 'chewy/query/pagination/will_paginate'
-  rescue LoadError
-  end
+  include Chewy::Index::Observe::ActiveRecordMethods
 end
 
 module Chewy
-
   @adapters = [
-    Chewy::Type::Adapter::ActiveRecord,
-    Chewy::Type::Adapter::Mongoid,
-    Chewy::Type::Adapter::Sequel,
-    Chewy::Type::Adapter::Object
+    Chewy::Index::Adapter::ActiveRecord,
+    Chewy::Index::Adapter::Object
   ]
 
   class << self
     attr_accessor :adapters
 
-    # Derives type from string `index#type` representation:
-    #
-    #   Chewy.derive_type('users#user') # => UsersIndex::User
-    #
-    # If index has only one type - it is possible to derive it without specification:
-    #
-    #   Chewy.derive_type('users') # => UsersIndex::User
-    #
-    # If index has more then one type - it raises Chewy::UnderivableType.
-    #
-    def derive_type name
-      return name if name.is_a?(Class) && name < Chewy::Type
-
-      index_name, type_name = name.split('#', 2)
-      class_name = "#{index_name.camelize}Index"
-      index = class_name.safe_constantize
-      raise Chewy::UnderivableType.new("Can not find index named `#{class_name}`") unless index && index < Chewy::Index
-      type = if type_name.present?
-        index.type_hash[type_name] or raise Chewy::UnderivableType.new("Index `#{class_name}` doesn`t have type named `#{type_name}`")
-      elsif index.types.one?
-        index.types.first
-      else
-        raise Chewy::UnderivableType.new("Index `#{class_name}` has more than one type, please specify type via `#{index_name}#type_name`")
+    # A thread-local variables accessor
+    # @return [Hash]
+    def current
+      unless Thread.current.thread_variable?(:chewy)
+        Thread.current.thread_variable_set(:chewy, {})
       end
+
+      Thread.current.thread_variable_get(:chewy)
     end
 
-    # Creates Chewy::Type ancestor defining index and adapter methods.
+    # Derives an index for the passed string identifier if possible.
+    #
+    # @example
+    #   Chewy.derive_name(UsersIndex) # => UsersIndex
+    #   Chewy.derive_name('namespace/users') # => Namespace::UsersIndex
+    #   Chewy.derive_name('missing') # => raises Chewy::UndefinedIndex
     #
-    def create_type index, target, options = {}, &block
-      type = Class.new(Chewy::Type)
+    # @param index_name [String, Chewy::Index] index identifier or class
+    # @raise [Chewy::UndefinedIndex] in cases when it is impossible to find index
+    # @return [Chewy::Index]
+    def derive_name(index_name)
+      return index_name if index_name.is_a?(Class) && index_name < Chewy::Index
 
-      adapter = adapters.find { |adapter| adapter.accepts?(target) }.new(target, options)
+      class_name = "#{index_name.camelize.gsub(/Index\z/, '')}Index"
+      index = class_name.safe_constantize
 
-      index.const_set(adapter.name, type)
-      type.send(:define_singleton_method, :index) { index }
-      type.send(:define_singleton_method, :adapter) { adapter }
+      return index if index && index < Chewy::Index
 
-      type.class_eval &block if block
-      type
+      raise Chewy::UndefinedIndex, "Can not find index named `#{class_name}`"
     end
 
     # Main elasticsearch-ruby client instance
     #
     def client
-      Thread.current[:chewy_client] ||= begin
+      Chewy.current[:chewy_client] ||= begin
         client_configuration = configuration.deep_dup
         client_configuration.delete(:prefix) # used by Chewy, not relevant to Elasticsearch::Client
         block = client_configuration[:transport_options].try(:delete, :proc)
@@ -137,14 +110,16 @@ module Chewy
     # Does nothing in case of config `wait_for_status` is undefined.
     #
     def wait_for_status
-      client.cluster.health wait_for_status: Chewy.configuration[:wait_for_status] if Chewy.configuration[:wait_for_status].present?
+      if Chewy.configuration[:wait_for_status].present?
+        client.cluster.health wait_for_status: Chewy.configuration[:wait_for_status]
+      end
     end
 
     # Deletes all corresponding indexes with current prefix from ElasticSearch.
     # Be careful, if current prefix is blank, this will destroy all the indexes.
     #
     def massacre
-      Chewy.client.indices.delete(index: [Chewy.configuration[:prefix], '*'].delete_if(&:blank?).join(?_))
+      Chewy.client.indices.delete(index: [Chewy.configuration[:prefix], '*'].reject(&:blank?).join('_'))
       Chewy.wait_for_status
     end
     alias_method :delete_all, :massacre
@@ -172,27 +147,51 @@ module Chewy
     #   Chewy.strategy.pop
     #   city3.do_update! # index updated again
     #
-    def strategy name = nil, &block
-      Thread.current[:chewy_strategy] ||= Chewy::Strategy.new
+    def strategy(name = nil, &block)
+      Chewy.current[:chewy_strategy] ||= Chewy::Strategy.new
       if name
         if block
-          Thread.current[:chewy_strategy].wrap name, &block
+          Chewy.current[:chewy_strategy].wrap name, &block
         else
-          Thread.current[:chewy_strategy].push name
+          Chewy.current[:chewy_strategy].push name
         end
       else
-        Thread.current[:chewy_strategy]
+        Chewy.current[:chewy_strategy]
       end
     end
 
     def config
       Chewy::Config.instance
     end
-    delegate *Chewy::Config.delegated, to: :config
+    delegate(*Chewy::Config.delegated, to: :config)
 
     def repository
       Chewy::Repository.instance
     end
-    delegate *Chewy::Repository.delegated, to: :repository
+    delegate(*Chewy::Repository.delegated, to: :repository)
+
+    def create_indices
+      Chewy::Index.descendants.each(&:create)
+    end
+
+    def create_indices!
+      Chewy::Index.descendants.each(&:create!)
+    end
+
+    def eager_load!
+      return unless defined?(Chewy::Railtie)
+
+      dirs = Chewy::Railtie.all_engines.map do |engine|
+        engine.paths[Chewy.configuration[:indices_path]]
+      end.compact.map(&:existent).flatten.uniq
+
+      dirs.each do |dir|
+        Dir.glob(File.join(dir, '**/*.rb')).each do |file|
+          require_dependency file
+        end
+      end
+    end
   end
 end
+
+require 'chewy/stash'

data/lib/generators/chewy/install_generator.rb

@@ -1,11 +1,11 @@
 module Chewy
   module Generators
     class InstallGenerator < Rails::Generators::Base
-      source_root File.expand_path("../../templates", __FILE__)
+      source_root File.expand_path('../templates', __dir__)
 
       def copy_configuration
-        template "chewy.yml", "config/chewy.yml"
+        template 'chewy.yml', 'config/chewy.yml'
       end
     end
   end
-end
+end