sidekiq-throttled 0.18.0 → 1.1.0

data/gemfiles/sidekiq_6.4.gemfile

@@ -1,33 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "appraisal"
-gem "rake"
-gem "rspec"
-gem "sidekiq", "~> 6.4.0"
-
-group :development do
-  gem "byebug"
-  gem "guard", require: false
-  gem "guard-rspec", require: false
-  gem "guard-rubocop", require: false
-end
-
-group :test do
-  gem "apparition"
-  gem "capybara"
-  gem "puma"
-  gem "rack-test"
-  gem "sinatra"
-  gem "timecop"
-end
-
-group :lint do
-  gem "rubocop", require: false
-  gem "rubocop-performance", require: false
-  gem "rubocop-rake", require: false
-  gem "rubocop-rspec", require: false
-end
-
-gemspec path: "../"

data/gemfiles/sidekiq_6.5.gemfile

@@ -1,33 +0,0 @@
-# This file was generated by Appraisal
-
-source "https://rubygems.org"
-
-gem "appraisal"
-gem "rake"
-gem "rspec"
-gem "sidekiq", "~> 6.5.0"
-
-group :development do
-  gem "byebug"
-  gem "guard", require: false
-  gem "guard-rspec", require: false
-  gem "guard-rubocop", require: false
-end
-
-group :test do
-  gem "apparition"
-  gem "capybara"
-  gem "puma"
-  gem "rack-test"
-  gem "sinatra"
-  gem "timecop"
-end
-
-group :lint do
-  gem "rubocop", require: false
-  gem "rubocop-performance", require: false
-  gem "rubocop-rake", require: false
-  gem "rubocop-rspec", require: false
-end
-
-gemspec path: "../"

data/lib/sidekiq/throttled/communicator/callbacks.rb

@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-
-require "fiber"
-
-require "sidekiq/throttled/communicator/exception_handler"
-
-module Sidekiq
-  module Throttled
-    class Communicator
-      # Callbacks registry and runner. Runs registered callbacks in dedicated
-      # Fiber solving issue with ConnectionPool and Redis client in subscriber
-      # mode.
-      #
-      # Once Redis entered subscriber mode `#subscribe` method, it can't be used
-      # for any command but pub/sub or quit, making it impossible to use for
-      # anything else. ConnectionPool binds reserved client to Thread, thus
-      # nested `#with` calls inside same thread result into a same connection.
-      # That makes it impossible to issue any normal Redis commands from
-      # within listener Thread.
-      #
-      # @private
-      class Callbacks
-        include ExceptionHandler
-
-        # Initializes callbacks registry.
-        def initialize
-          @mutex    = Mutex.new
-          @handlers = Hash.new { |h, k| h[k] = [] }
-        end
-
-        # Registers handler of given event.
-        #
-        # @example
-        #
-        #   callbacks.on "and out comes wolves" do |who|
-        #     puts "#{who} let the dogs out?!"
-        #   end
-        #
-        # @param [#to_s] event
-        # @raise [ArgumentError] if no handler block given
-        # @yield [*args] Runs given block upon `event`
-        # @yieldreturn [void]
-        # @return [self]
-        def on(event, &handler)
-          raise ArgumentError, "No block given" unless handler
-
-          @mutex.synchronize { @handlers[event.to_s] << handler }
-          self
-        end
-
-        # Runs event handlers with given args.
-        #
-        # @param [#to_s] event
-        # @param [Object] payload
-        # @return [void]
-        def run(event, payload = nil)
-          @mutex.synchronize do
-            fiber = Fiber.new do
-              @handlers[event.to_s].each do |callback|
-                callback.call(payload)
-              rescue => e
-                handle_exception(e, :context => "sidekiq:throttled")
-              end
-            end
-
-            fiber.resume
-          end
-        end
-      end
-    end
-  end
-end

data/lib/sidekiq/throttled/communicator/exception_handler.rb

@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-require "sidekiq"
-require "sidekiq/version"
-
-module Sidekiq
-  module Throttled
-    class Communicator
-      if Sidekiq::VERSION >= "6.5.0"
-        module ExceptionHandler
-          def handle_exception(*args)
-            Sidekiq.handle_exception(*args)
-          end
-        end
-
-        # NOTE: `Sidekiq.default_error_handler` is private API
-        Sidekiq.error_handlers << Sidekiq.method(:default_error_handler)
-      else
-        require "sidekiq/exception_handler"
-
-        ExceptionHandler = ::Sidekiq::ExceptionHandler
-      end
-    end
-  end
-end

data/lib/sidekiq/throttled/communicator/listener.rb

@@ -1,109 +0,0 @@
-# frozen_string_literal: true
-
-require "sidekiq/throttled/communicator/exception_handler"
-
-module Sidekiq
-  module Throttled
-    class Communicator
-      # Redis subscription listener thread.
-      #
-      # @private
-      class Listener < Thread
-        include ExceptionHandler
-
-        # Starts listener thread.
-        #
-        # @param [String] channel Redis pub/sub channel to listen
-        # @param [Callbacks] callbacks Message callbacks registry
-        def initialize(channel, callbacks)
-          @channel    = channel
-          @callbacks  = callbacks
-          @terminated = false
-          @subscribed = false
-
-          super { listen until @terminated }
-        end
-
-        # Whenever underlying redis client subscribed to pub/sup channel.
-        #
-        # @return [Boolean]
-        def ready?
-          @subscribed
-        end
-
-        # Whenever main loop is still running.
-        #
-        # @return [Boolean]
-        def listening?
-          !@terminated
-        end
-
-        # Stops listener.
-        #
-        # @return [void]
-        def stop
-          # Raising exception while client is in subscription mode makes
-          # redis close connection and thus causing ConnectionPool reopen
-          # it (normal mode). Otherwise subscription mode client will be
-          # pushed back to ConnectionPool causing problems.
-          raise Sidekiq::Shutdown
-        end
-
-        private
-
-        # Wraps {#subscribe} with exception handlers:
-        #
-        # - `Sidekiq::Shutdown` exception marks listener as stopped and returns
-        #   making `while` loop of listener thread terminate.
-        #
-        # - `StandardError` got recorded to the log and swallowed,
-        #   making `while` loop of the listener thread restart.
-        #
-        # - `Exception` is recorded to the log and re-raised.
-        #
-        # @return [void]
-        def listen # rubocop:disable Metrics/MethodLength
-          subscribe
-        rescue Sidekiq::Shutdown
-          @terminated = true
-          @subscribed = false
-        rescue StandardError => e # rubocop:disable Style/RescueStandardError
-          @subscribed = false
-          handle_exception(e, { :context => "sidekiq:throttled" })
-          sleep 1
-        rescue Exception => e # rubocop:disable Lint/RescueException
-          @terminated = true
-          @subscribed = false
-          handle_exception(e, { :context => "sidekiq:throttled" })
-          raise
-        end
-
-        # Subscribes to channel and triggers all registerd handlers for
-        # received messages.
-        #
-        # @note Puts thread's Redis connection to subscription mode and
-        #   locks thread.
-        #
-        # @see http://redis.io/topics/pubsub
-        # @see http://redis.io/commands/subscribe
-        # @see Callbacks#run
-        # @return [void]
-        def subscribe # rubocop:disable Metrics/MethodLength
-          Sidekiq.redis do |conn|
-            conn.subscribe @channel do |on|
-              on.subscribe do
-                @subscribed = true
-                @callbacks.run("ready")
-              end
-
-              on.message do |_channel, data|
-                message, payload = Marshal.load(data) # rubocop:disable Security/MarshalLoad:
-                @callbacks.run("message:#{message}", payload)
-              end
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/sidekiq/throttled/communicator.rb

@@ -1,116 +0,0 @@
-# frozen_string_literal: true
-
-require "singleton"
-
-require "sidekiq/throttled/communicator/exception_handler"
-require "sidekiq/throttled/communicator/listener"
-require "sidekiq/throttled/communicator/callbacks"
-
-module Sidekiq
-  module Throttled
-    # Inter-process communication for sidekiq. It starts listener thread on
-    # sidekiq server and listens for incoming messages.
-    #
-    # @example
-    #
-    #   # Add incoming message handler for server
-    #   Communicator.instance.receive "knock" do |who|
-    #     puts "#{who}'s knocking on the door"
-    #   end
-    #
-    #   # Emit message from console
-    #   Sidekiq.redis do |conn|
-    #     Communicator.instance.transmit(conn, "knock", "ixti")
-    #   end
-    class Communicator
-      include Singleton
-      include ExceptionHandler
-
-      # Redis PUB/SUB channel name
-      #
-      # @see http://redis.io/topics/pubsub
-      CHANNEL_NAME = "sidekiq:throttled"
-      private_constant :CHANNEL_NAME
-
-      # Initializes singleton instance.
-      def initialize
-        @callbacks = Callbacks.new
-        @listener  = nil
-        @mutex     = Mutex.new
-      end
-
-      # Starts listener thread.
-      #
-      # @return [void]
-      def start_listener
-        @mutex.synchronize do
-          @listener ||= Listener.new(CHANNEL_NAME, @callbacks)
-        end
-      end
-
-      # Stops listener thread.
-      #
-      # @return [void]
-      def stop_listener
-        @mutex.synchronize do
-          @listener&.stop
-          @listener = nil
-        end
-      end
-
-      # Configures Sidekiq server to start/stop listener thread.
-      #
-      # @private
-      # @return [void]
-      def setup!
-        Sidekiq.configure_server do |config|
-          config.on(:startup) { start_listener }
-          config.on(:quiet) { stop_listener }
-        end
-      end
-
-      # Transmit message to listeners.
-      #
-      # @example
-      #
-      #   Sidekiq.redis do |conn|
-      #     Communicator.instance.transmit(conn, "knock")
-      #   end
-      #
-      # @param [Redis] redis Redis client
-      # @param [#to_s] message
-      # @param [Object] payload
-      # @return [void]
-      def transmit(redis, message, payload = nil)
-        redis.publish(CHANNEL_NAME, Marshal.dump([message.to_s, payload]))
-      end
-
-      # Add incoming message handler.
-      #
-      # @example
-      #
-      #   Communicator.instance.receive "knock" do |payload|
-      #     # do something upon `knock` message
-      #   end
-      #
-      # @param [#to_s] message
-      # @yield [payload] Runs given block everytime `message` being received.
-      # @yieldparam [Object, nil] payload Payload that was transmitted
-      # @yieldreturn [void]
-      # @return [void]
-      def receive(message, &handler)
-        @callbacks.on("message:#{message}", &handler)
-      end
-
-      # Communicator readiness hook.
-      #
-      # @yield Runs given block every time listener thread subscribes
-      #   to Redis pub/sub channel.
-      # @return [void]
-      def ready(&handler)
-        @callbacks.on("ready", &handler)
-        yield if @listener&.ready?
-      end
-    end
-  end
-end

data/lib/sidekiq/throttled/configuration.rb

@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-module Sidekiq
-  module Throttled
-    # Configuration holder.
-    class Configuration
-      # Class constructor.
-      def initialize
-        reset!
-      end
-
-      # Reset configuration to defaults.
-      #
-      # @return [self]
-      def reset!
-        @inherit_strategies = false
-
-        self
-      end
-
-      # Instructs throttler to lookup strategies in parent classes, if there's
-      # no own strategy:
-      #
-      #     class FooJob
-      #       include Sidekiq::Job
-      #       include Sidekiq::Throttled::Job
-      #
-      #       sidekiq_throttle :concurrency => { :limit => 42 }
-      #     end
-      #
-      #     class BarJob < FooJob
-      #     end
-      #
-      # By default in the example above, `Bar` won't have throttling options.
-      # Set this flag to `true` to enable this lookup in initializer, after
-      # that `Bar` will use `Foo` throttling bucket.
-      def inherit_strategies=(value)
-        @inherit_strategies = value ? true : false
-      end
-
-      # Whenever throttled workers should inherit parent's strategies or not.
-      # Default: `false`.
-      #
-      # @return [Boolean]
-      def inherit_strategies?
-        @inherit_strategies
-      end
-    end
-  end
-end

data/lib/sidekiq/throttled/expirable_list.rb

@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-require "monitor"
-
-module Sidekiq
-  module Throttled
-    # List that tracks when elements were added and enumerates over those not
-    # older than `ttl` seconds ago.
-    #
-    # ## Implementation
-    #
-    # Internally list holds an array of arrays. Thus each element is a tuple of
-    # monotonic timestamp (when element was added) and element itself:
-    #
-    #     [
-    #       [ 123456.7890, "default" ],
-    #       [ 123456.7891, "urgent" ],
-    #       [ 123457.9621, "urgent" ],
-    #       ...
-    #     ]
-    #
-    # It does not deduplicates elements. Eviction happens only upon elements
-    # retrieval (see {#each}).
-    #
-    # @see https://ruby-doc.org/core/Process.html#method-c-clock_gettime
-    # @see https://linux.die.net/man/3/clock_gettime
-    #
-    # @private
-    class ExpirableList
-      include Enumerable
-
-      # @param ttl [Float] elements time-to-live in seconds
-      def initialize(ttl)
-        @ttl = ttl.to_f
-        @arr = []
-        @mon = Monitor.new
-      end
-
-      # Pushes given element into the list.
-      #
-      # @params element [Object]
-      # @return [ExpirableList] self
-      def <<(element)
-        @mon.synchronize { @arr << [::Process.clock_gettime(::Process::CLOCK_MONOTONIC), element] }
-        self
-      end
-
-      # Evicts expired elements and calls the given block once for each element
-      # left, passing that element as a parameter.
-      #
-      # @yield [element]
-      # @return [Enumerator] if no block given
-      # @return [ExpirableList] self if block given
-      def each
-        return to_enum __method__ unless block_given?
-
-        @mon.synchronize do
-          horizon = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @ttl
-
-          # drop all elements older than horizon
-          @arr.shift while @arr[0] && @arr[0][0] < horizon
-
-          @arr.each { |x| yield x[1] }
-        end
-
-        self
-      end
-    end
-  end
-end

data/lib/sidekiq/throttled/fetch/unit_of_work.rb

@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-
-require "sidekiq"
-
-require "sidekiq/throttled/queue_name"
-
-module Sidekiq
-  module Throttled
-    class Fetch
-      # BRPOP response envelope.
-      #
-      # @see Throttled::Fetch
-      # @private
-      class UnitOfWork
-        # @return [String] Redis key where job was pulled from
-        attr_reader :queue
-
-        # @return [String] Job's JSON payload
-        attr_reader :job
-
-        # @param [String] queue Redis key where job was pulled from
-        # @param [String] job Job's JSON payload
-        def initialize(queue, job)
-          @queue = queue
-          @job   = job
-        end
-
-        # Callback that is called by `Sidekiq::Processor` when job was
-        # succeccfully processed. Most likely this is used by `ReliableFetch`
-        # of Sidekiq Pro/Enterprise to remove job from running queue.
-        #
-        # @return [void]
-        def acknowledge
-          # do nothing
-        end
-
-        # Normalized `queue` name.
-        #
-        # @see QueueName.normalize
-        # @return [String]
-        def queue_name
-          @queue_name ||= QueueName.normalize queue
-        end
-
-        # Pushes job back to the tail of the queue, so that it will be popped
-        # first next time fetcher will pull job.
-        #
-        # @note This is triggered when job was not finished and Sidekiq server
-        #   process was terminated. It is a reverse of whatever fetcher was
-        #   doing to pull the job out of queue.
-        #
-        # @param [Redis] pipelined connection for requeing via Redis#pipelined
-        # @return [void]
-        def requeue(pipeline = nil)
-          if pipeline
-            pipeline.rpush(QueueName.expand(queue_name), job)
-          else
-            Sidekiq.redis { |conn| conn.rpush(QueueName.expand(queue_name), job) }
-          end
-        end
-
-        # Pushes job back to the head of the queue, so that job won't be tried
-        # immediately after it was requeued (in most cases).
-        #
-        # @note This is triggered when job is throttled. So it is same operation
-        #   Sidekiq performs upon `Sidekiq::Worker.perform_async` call.
-        #
-        # @return [void]
-        def requeue_throttled
-          Sidekiq.redis { |conn| conn.lpush(QueueName.expand(queue_name), job) }
-        end
-
-        # Tells whenever job should be pushed back to queue (throttled) or not.
-        #
-        # @see Sidekiq::Throttled.throttled?
-        # @return [Boolean]
-        def throttled?
-          Throttled.throttled? job
-        end
-      end
-    end
-  end
-end

data/lib/sidekiq/throttled/fetch.rb

@@ -1,94 +0,0 @@
-# frozen_string_literal: true
-
-require "sidekiq"
-require "sidekiq/throttled/expirable_list"
-require "sidekiq/throttled/fetch/unit_of_work"
-require "sidekiq/throttled/queues_pauser"
-require "sidekiq/throttled/queue_name"
-
-module Sidekiq
-  module Throttled
-    # Throttled fetch strategy.
-    #
-    # @private
-    class Fetch
-      # Timeout to sleep between fetch retries in case of no job received,
-      # as well as timeout to wait for redis to give us something to work.
-      TIMEOUT = 2
-
-      # Initializes fetcher instance.
-      # @param options [Hash]
-      # @option options [Integer] :throttled_queue_cooldown (TIMEOUT)
-      #   Min delay in seconds before queue will be polled again after
-      #   throttled job.
-      # @option options [Boolean] :strict (false)
-      # @option options [Array<#to_s>] :queue
-      def initialize(options)
-        @paused = ExpirableList.new(options.fetch(:throttled_queue_cooldown, TIMEOUT))
-
-        @strict = options.fetch(:strict, false)
-        @queues = options.fetch(:queues).map { |q| QueueName.expand q }
-
-        raise ArgumentError, "empty :queues" if @queues.empty?
-
-        @queues.uniq! if @strict
-      end
-
-      # Retrieves job from redis.
-      #
-      # @return [Sidekiq::Throttled::UnitOfWork, nil]
-      def retrieve_work
-        work = brpop
-        return unless work
-
-        work = UnitOfWork.new(*work)
-        return work unless work.throttled?
-
-        work.requeue_throttled
-        @paused << QueueName.expand(work.queue_name)
-
-        nil
-      end
-
-      def bulk_requeue(units, _options)
-        return if units.empty?
-
-        Sidekiq.logger.debug { "Re-queueing terminated jobs" }
-        Sidekiq.redis do |conn|
-          conn.pipelined do |pipeline|
-            units.each { |unit| unit.requeue(pipeline) }
-          end
-        end
-        Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
-      rescue => e
-        Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
-      end
-
-      private
-
-      # Tries to pop pair of `queue` and job `message` out of sidekiq queues.
-      #
-      # @see http://redis.io/commands/brpop
-      # @return [Array(String, String), nil]
-      def brpop
-        queues = filter_queues(@strict ? @queues : @queues.shuffle.uniq)
-
-        if queues.empty?
-          sleep TIMEOUT
-          return
-        end
-
-        Sidekiq.redis { |conn| conn.brpop(*queues, :timeout => TIMEOUT) }
-      end
-
-      # Returns list of queues to try to fetch jobs from.
-      #
-      # @note It may return an empty array.
-      # @param [Array<String>] queues
-      # @return [Array<String>]
-      def filter_queues(queues)
-        QueuesPauser.instance.filter(queues) - @paused.to_a
-      end
-    end
-  end
-end