sidekiq-ultimate 0.0.1.alpha.19 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f588d8dd927a0eaba4910ffbded5a8a24f69b6e8aae7c45b4bb28092373e6825
-  data.tar.gz: e2701097c8fbc5c99054bdb36a3bb4ae1089ee91abd1da74dfab4cab94f12ef7
+  metadata.gz: 3fe29c4188e4bcc73a818ac756cc6fa0eaa97588c2373920d0dc9cf36ba8859e
+  data.tar.gz: bb650bdf494eab357b0d042243cb42d9e5c1a0914e70971576766a55c84508d7
 SHA512:
-  metadata.gz: f5a37773ca7013b24a5e7abb15aab91f7667f157b7e5d4bf43085c8268546a50d51319d351fad4db478428ee437d4acd43f762b33212a96d0d5d3bea57ad7d77
-  data.tar.gz: 7ceeb9436521f998f9f34e642f22aee1e8762e60c6ae9f5689b82902b95a27d6c1578aee89bd984ca81dca91fcfb4ef227eae170c9bfb1e854a1f17cd9ba08f6
+  metadata.gz: 9389d82c1071237e11955ea3ad62ada63c084d4a41d412c87ab8b034982dd0104a4ee4701dbf09e563322150234cc06c18150be65ea5d51aeec3a38b03a55964
+  data.tar.gz: 6a0d8801f2959479e77ec201d377dfac040eacf5a366237962622faf4b92e9696d3cfed40ee00f8b03893798da848e5b42bfa48af27e3493020c87bcb20c7821

data/.github/workflows/rspec.yml

@@ -0,0 +1,58 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# GitHub recommends pinning actions to a commit SHA.
+# To get a newer version, you will need to update the SHA.
+# You can also reference a tag or branch, but the action may change without warning.
+
+name: Gem Rspec testing
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+permissions:
+  contents: read
+
+jobs:
+  rubocop:
+    name: Rubocop
+    runs-on: ubuntu-latest
+    env:
+      BUNDLE_WITHOUT: development
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 2.7.6
+          bundler-cache: true
+      - run: bundle exec rubocop --config .rubocop.yml
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version: ['2.7.6']
+        redis: ['4.8.0']
+        sidekiq: ['6.5.0']
+    env:
+      BUNDLE_GEMFILE: gemfiles/redis_${{ matrix.redis }}_sidekiq_${{ matrix.sidekiq }}.gemfile
+      BUNDLE_WITHOUT: development
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - run: docker-compose up -d
+      - name: Run specs and generate coverage report
+        run: bundle exec rake spec
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v3

data/.gitignore

@@ -1,4 +1,6 @@
 /Gemfile.lock
+/gemfiles/*.gemfile.lock
+/gemfiles/.bundle/config
 
 /.autoenv.zsh
 /.bundle

data/.rubocop.yml

@@ -1,6 +1,15 @@
+require:
+  - rubocop-performance
+  - rubocop-rake
+  - rubocop-rspec
+
 AllCops:
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.7
   DisplayCopNames: true
+  NewCops: enable
+  Exclude:
+    - "gemfiles/**/*"
+    - "vendor/**/*"
 
 
 ## Layout ######################################################################
@@ -8,12 +17,17 @@ AllCops:
 Layout/DotPosition:
   EnforcedStyle: trailing
 
-Layout/IndentArray:
+Layout/FirstArrayElementIndentation:
   EnforcedStyle: consistent
 
-Layout/IndentHash:
+Layout/FirstHashElementIndentation:
   EnforcedStyle: consistent
 
+Layout/LineLength:
+  Max: 120
+
+Layout/HashAlignment:
+  EnforcedHashRocketStyle: table
 
 ## Metrics #####################################################################
 
@@ -21,11 +35,13 @@ Metrics/BlockLength:
   Exclude:
     - "spec/**/*"
 
+Metrics/MethodLength:
+  CountAsOne: ['array', 'hash', 'heredoc', 'method_call']
 
-## Style #######################################################################
+Metrics/ModuleLength:
+  CountAsOne: ['array', 'hash', 'heredoc', 'method_call']
 
-Style/BracesAroundHashParameters:
-  Enabled: false
+## Style #######################################################################
 
 Style/HashSyntax:
   EnforcedStyle: hash_rockets
@@ -44,3 +60,23 @@ Style/StringLiterals:
 
 Style/YodaCondition:
   Enabled: false
+
+## RSpec ########################################################################
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/MultipleExpectations:
+  Enabled: false
+
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+
+RSpec/NestedGroups:
+  Enabled: false
+
+Lint/AmbiguousBlockAssociation:
+  Exclude:
+    - "spec/**/*"
+
+RSpec/IndexedLet:
+  Max: 5

data/ARCHITECTURE.md

@@ -6,7 +6,7 @@ redis documentation. In short fetch will look like this:
 
 ``` ruby
 COOLDOWN = 2
-IDENTITY = Object.new.tap { |o| o.extend Sidekiq::Util }.identity
+IDENTITY = Object.new.tap { |o| o.extend Sidekiq::Component }.identity
 
 def retrieve
   Sidekiq.redis do

data/Appraisals

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+REDIS_VERSIONS = %w[4.8.0].freeze
+SIDEKIQ_VERSIONS = %w[6.5.0].freeze
+
+version_combinations = REDIS_VERSIONS.product(SIDEKIQ_VERSIONS)
+
+version_combinations.each do |redis_version, sidekiq_version|
+  appraise "redis_#{redis_version}_sidekiq_#{sidekiq_version}" do
+    gem "redis", "~> #{redis_version}"
+    gem "sidekiq", "~> #{sidekiq_version}"
+  end
+end

data/CHANGES.md

@@ -0,0 +1,6 @@
+## 2.0.0 (2023-04-10)
+
+* Move to Sidekiq 6.5
+* Move to sidekiq-throttled 0.18.0
+* Drop redis namespaces support
+* Fix deprecation warnings for redis gem

data/Gemfile

@@ -3,9 +3,13 @@
 source "https://rubygems.org"
 ruby RUBY_VERSION
 
+gem "appraisal"
 gem "rake"
 gem "rspec"
-gem "rubocop", "~> 0.52.0", :require => false
+gem "rubocop", "~> 1.50.2", :require => false
+gem "rubocop-performance", "~> 1.17.1", :require => false
+gem "rubocop-rake", "~> 0.6.0", :require => false
+gem "rubocop-rspec", "~> 2.20.0", :require => false
 
 group :development do
   gem "guard",         :require => false
@@ -19,10 +23,4 @@ group :test do
   gem "simplecov",  :require => false
 end
 
-group :doc do
-  gem "redcarpet"
-  gem "yard"
-end
-
-# Specify your gem's dependencies in redis-prescription.gemspec
 gemspec

data/README.md

@@ -18,7 +18,7 @@ extension one will need. :D
 Add this line to your application's Gemfile:
 
 ```ruby
-gem "sidekiq-ultimate", ">= 0.0.1.alpha"
+gem "sidekiq-ultimate"
 ```
 
 And then execute:
@@ -40,6 +40,80 @@ require "sidekiq/ultimate"
 Sidekiq::Ultimate.setup!
 ```
 
+## Configuration
+
+### Resurrection event handler
+
+An event handler can be called when a job is resurrected.
+
+```ruby
+Sidekiq::Ultimate.setup! do |config|
+  config.on_resurrection = ->(queue_name, jobs_count) do
+    puts "Resurrected #{jobs_count} jobs from #{queue_name}"
+  end
+end
+```
+
+### Resurrection counter
+
+A resurrection counter can be enabled to count how many times a job was resurrected. If `enable_resurrection_counter` setting is enabled, on each resurrection event, a counter is increased. Counter value is stored in redis and has expiration time 24 hours. 
+
+For example this can be used in the `ServerMiddleware` later on to early return resurrected jobs based on the counter value.
+
+`enable_resurrection_counter` can be either a `Proc` or a constant. 
+
+Having a `Proc` is useful if you want to enable or disable resurrection counter in run time. It will be called on each 
+resurrection event to decide whether to increase the counter or not.
+
+```ruby
+Sidekiq::Ultimate.setup! do |config|
+  config.enable_resurrection_counter = -> do
+    DynamicSettings.get("enable_resurrection_counter")
+  end
+end
+
+Sidekiq::Ultimate.setup! do |config|
+  config.enable_resurrection_counter = true
+end
+```
+
+#### Read the value
+
+Resurrection counter value can be read using `Sidekiq::Ultimate::Resurrector::Count.read` method.
+
+```ruby
+Sidekiq::Ultimate::Resurrector::Count.read(:job_id => "2647c4fe13acc692326bd4c2")
+=> 1
+```
+
+### Empty Queues Cache Refresh Interval
+
+```ruby
+Sidekiq::Ultimate.setup! do |config|
+  config.empty_queues_cache_refresh_interval_sec = 42
+end
+```
+
+Specifies how often the cache of empty queues should be refreshed.
+In a nutshell, this sets the maximum possible delay between when a job was pushed to previously empty queue and earliest the moment when that new job could be picked up.
+
+**Note:** every sidekiq process maintains its own local cache of empty queues.
+Setting this interval to a low value will increase the number of Redis calls needed to check for empty queues, increasing the total load on Redis.
+
+This setting helps manage the tradeoff between performance penalties and latency needed for reliable fetch.
+Under the hood, Sidekiq's default fetch occurs with [a single Redis `BRPOP` call](https://redis.io/commands/brpop/) which is passes list of all queues to pluck work from.
+In contrast, [reliable fetch uses `LPOPRPUSH`](https://redis.io/commands/rpoplpush/) (or the equivalent `LMOVE` in later Redis versions) to place in progress work into a WIP queue.
+However, `LPOPRPUSH` can only check one source queue to pop from at once, and [no multi-key alternative is available](https://github.com/redis/redis/issues/1785), so multiple Redis calls are needed to pluck work if an empty queue is checked.
+In order to avoid performance penalties for repeated calls to empty queues, Sidekiq Ultimate therefore maintains a list of recently know empty queues which it will avoid polling for work.
+
+Therefore:
+- If your Sidekiq architecture has *a low number of total queues*, the worst case penalty for polling empty queues will be bounded, and it is reasonable to **set a shorter refresh period**.
+- If your Sidekiq architecture has a *high number of total queues*, the worst case penalty for polling empty queues is large, and it is recommended to **set a longer refresh period**.
+- When adjusting this setting:
+    - Check that work is consumed appropriately quickly from high priority queues after they bottom out (after increasing the refresh interval)
+    - Check that backlog work does not accumulate in low priority queues (after decreasing the refresh interval)
+
+
 ---
 
 **NOTICE**
@@ -54,20 +128,13 @@ Thus look up it's README for throttling configuration details.
 
 ## Supported Ruby Versions
 
-This library aims to support and is [tested against][travis-ci] the following
-Ruby and Redis client versions:
+This library aims to support and is tested against the following Ruby and Redis client versions:
 
 * Ruby
-  * 2.3.x
-  * 2.4.x
-  * 2.5.x
+  * 2.7.x
 
 * [redis-rb](https://github.com/redis/redis-rb)
-  * 4.x
-
-* [redis-namespace](https://github.com/resque/redis-namespace)
-  * 1.6
-
+  * 4.8
 
 If something doesn't work on one of these versions, it's a bug.
 
@@ -106,11 +173,10 @@ push git commits and tags, and push the `.gem` file to [rubygems.org][].
 
 ## Copyright
 
-Copyright (c) 2018 SensorTower Inc.<br>
+Copyright (c) 2018-23 SensorTower Inc.<br>
 See [LICENSE.md][] for further details.
 
 
-[travis.ci]: http://travis-ci.org/sensortower/sidekiq-ultimate
 [rubygems.org]: https://rubygems.org
 [LICENSE.md]: https://github.com/sensortower/sidekiq-ultimate/blob/master/LICENSE.txt
-[sidekiq-throttled]: http://travis-ci.org/sensortower/sidekiq-throttled
+[sidekiq-throttled]: https://github.com/ixti/sidekiq-throttled

data/Rakefile

@@ -8,8 +8,4 @@ RSpec::Core::RakeTask.new
 require "rubocop/rake_task"
 RuboCop::RakeTask.new
 
-if ENV["CI"]
-  task :default => :spec
-else
-  task :default => %i[rubocop spec]
-end
+task :default => %i[rubocop spec]

data/docker-compose.yml

@@ -0,0 +1,12 @@
+version: '3.1'
+
+services:
+  redis:
+    image: redis:6.0.6-alpine
+    volumes:
+      - redis-data:/data
+    ports:
+      - '6379:6379'
+
+volumes:
+  redis-data:

data/gemfiles/redis_4.8.0_sidekiq_6.5.0.gemfile

@@ -0,0 +1,29 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+ruby "2.7.6"
+
+gem "appraisal"
+gem "rake"
+gem "rspec"
+gem "rubocop", "~> 1.50.2", require: false
+gem "rubocop-performance", "~> 1.17.1", require: false
+gem "rubocop-rake", "~> 0.6.0", require: false
+gem "rubocop-rspec", "~> 2.20.0", require: false
+gem "redis", "~> 4.8.0"
+gem "sidekiq", "~> 6.5.0"
+
+group :development do
+  gem "guard", require: false
+  gem "guard-rspec", require: false
+  gem "guard-rubocop", require: false
+  gem "pry", require: false
+end
+
+group :test do
+  gem "codecov", require: false
+  gem "simplecov", require: false
+end
+
+gemspec path: "../"

data/lib/sidekiq/ultimate/configuration.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module Sidekiq
+  module Ultimate
+    # Configuration options.
+    class Configuration
+      include Singleton
+
+      # @return [Proc] callback to be called when job is resurrected
+      # @yieldparam queue_name [String] name of the queue
+      # @yieldparam jobs_count [Integer] number of jobs resurrected
+      # @yieldreturn [void]
+      # @example
+      #  Sidekiq::Ultimate::Configuration.instance.on_resurrection = ->(queue_name, jobs_count) do
+      #    puts "Resurrected #{jobs_count} jobs from #{queue_name}"
+      #  end
+      attr_accessor :on_resurrection
+
+      # If `enable_resurrection_counter` setting is enabled, on each resurrection event, a counter is increased.
+      # This is useful for telemetry purposes in order to understand how often jobs are resurrected
+      # Counter value is stored in redis by jid and has expiration time 24 hours.
+      # @return [Boolean]
+      attr_accessor :enable_resurrection_counter
+
+      # It specifies how often the cache of empty queues should be refreshed.
+      # In a nutshell, it specifies the maximum possible delay between a job was pushed to previously empty queue and
+      # the moment when that new job is picked up.
+      # Note that every sidekiq process needs to maintain its own local cache of empty queues. Setting this interval
+      # to a low values will increase the number of redis calls and will increase the load on redis.
+      # @return [Numeric] interval in seconds to refresh the cache of empty queues
+      attr_reader :empty_queues_cache_refresh_interval_sec
+
+      DEFAULT_EMPTY_QUEUES_CACHE_REFRESH_INTERVAL_SEC = 30
+
+      # If fetching attempt from a queue was throttled, it puts the queue to the exhausted list for this amount of time
+      # to avoid throttling for the same queue
+      # @return [Float] timeout in seconds
+      attr_writer :throttled_fetch_timeout_sec
+
+      DEFAULT_THROTTLED_FETCH_TIMEOUT_SEC = 15
+
+      def initialize
+        @empty_queues_cache_refresh_interval_sec = DEFAULT_EMPTY_QUEUES_CACHE_REFRESH_INTERVAL_SEC
+        @throttled_fetch_timeout_sec = DEFAULT_THROTTLED_FETCH_TIMEOUT_SEC
+        super
+      end
+
+      def empty_queues_cache_refresh_interval_sec=(value)
+        unless value.is_a?(Numeric)
+          raise ArgumentError, "Invalid 'empty_queues_cache_refresh_interval_sec' value: #{value}. Must be Numeric"
+        end
+
+        @empty_queues_cache_refresh_interval_sec = value
+      end
+
+      def throttled_fetch_timeout_sec
+        if @throttled_fetch_timeout_sec.respond_to?(:call)
+          @throttled_fetch_timeout_sec.call.to_f
+        else
+          @throttled_fetch_timeout_sec.to_f
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/empty_queues/refresh_timer_task.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "concurrent/timer_task"
+require "sidekiq/ultimate/interval_with_jitter"
+
+module Sidekiq
+  module Ultimate
+    class EmptyQueues
+      # Timer task that periodically refreshes empty queues. Also adds jitter to the execution interval.
+      class RefreshTimerTask
+        TASK_CLASS = Class.new(Concurrent::TimerTask)
+
+        class << self
+          def setup!(empty_queues_class)
+            interval = Sidekiq::Ultimate::Configuration.instance.empty_queues_cache_refresh_interval_sec
+            task = TASK_CLASS.new({
+              :run_now            => true,
+              :execution_interval => Sidekiq::Ultimate::IntervalWithJitter.call(interval)
+            }) { empty_queues_class.instance.refresh! }
+            task.execute
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/empty_queues.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "redlock"
+require "singleton"
+
+require "sidekiq/ultimate/configuration"
+require "sidekiq/ultimate/queue_name"
+require "sidekiq/ultimate/empty_queues/refresh_timer_task"
+
+module Sidekiq
+  module Ultimate
+    # Maintains a cache of empty queues. It has a global cache and a local cache.
+    # The global cache is stored in redis and updated periodically. The local cache is updated either by using the fresh
+    # cache fetched after global cache update or by using existing global cache.
+    # Only one process can update the global cache at a time.
+    class EmptyQueues
+      include Singleton
+
+      LOCK_KEY = "ultimate:empty_queues_updater:lock"
+      LAST_RUN_KEY = "ultimate:empty_queues_updater:last_run"
+      KEY = "ultimate:empty_queues"
+
+      attr_reader :queues, :local_lock
+
+      def initialize
+        @local_lock = Mutex.new
+        @queues = []
+
+        super
+      end
+
+      # Sets up automatic empty queues cache updater.
+      # It will call #refresh! every
+      # `Sidekiq::Ultimate::Configuration.instance.empty_queues_cache_refresh_interval_sec` seconds
+      def self.setup!
+        refresher_timer_task = nil
+
+        Sidekiq.on(:startup) do
+          refresher_timer_task&.shutdown
+          refresher_timer_task = RefreshTimerTask.setup!(self)
+        end
+
+        Sidekiq.on(:shutdown) { refresher_timer_task&.shutdown }
+      end
+
+      # Attempts to update the global cache of empty queues by first acquiring a global lock
+      # If the lock is acquired, it brute force generates an accurate list of currently empty queues and
+      # then writes the updated list to the global cache
+      # The local queue cache is always updated as a result of this operation, either by using the recently generated
+      # list or fetching the most recent list from the global cache
+      #
+      # @return [Boolean] true if local cache was updated
+      def refresh!
+        return false unless local_lock.try_lock
+
+        begin
+          refresh_global_cache! || refresh_local_cache
+        ensure
+          local_lock.unlock
+        end
+      rescue => e
+        Sidekiq.logger.error { "Empty queues cache update failed: #{e}" }
+        raise
+      end
+
+      private
+
+      # Automatically updates local cache if global cache was updated
+      # @return [Boolean] true if cache was updated
+      def refresh_global_cache!
+        Sidekiq.logger.debug { "Refreshing global cache" }
+
+        global_lock do
+          Sidekiq.redis do |redis|
+            empty_queues = generate_empty_queues(redis)
+
+            update_global_cache(redis, empty_queues)
+            update_local_cache(empty_queues)
+          end
+        end
+      end
+
+      def generate_empty_queues(redis)
+        # Cursor is not atomic, so there may be duplicates because of concurrent update operations
+        queues = Sidekiq.redis { |r| r.sscan_each("queues").to_a.uniq }
+
+        queues_statuses =
+          redis.pipelined do |p|
+            queues.each do |queue|
+              p.exists?(QueueName.new(queue).pending)
+            end
+          end
+
+        queues.zip(queues_statuses).reject { |(_, exists)| exists }.map(&:first)
+      end
+
+      def refresh_local_cache
+        Sidekiq.logger.debug { "Refreshing local cache" }
+
+        # Cursor is not atomic, so there may be duplicates because of concurrent update operations
+        list = Sidekiq.redis { |redis| redis.sscan_each(KEY).to_a.uniq }
+        update_local_cache(list)
+      end
+
+      def update_global_cache(redis, list)
+        Sidekiq.logger.debug { "Setting global cache: #{list}" }
+
+        redis.multi do |multi|
+          multi.del(KEY)
+          multi.sadd(KEY, list) if list.any?
+        end
+      end
+
+      def update_local_cache(list)
+        Sidekiq.logger.debug { "Setting local cache: #{list}" }
+
+        @queues = list
+      end
+
+      # @return [Boolean] true if lock was acquired
+      def global_lock
+        Sidekiq.redis do |redis|
+          break false if skip_update?(redis) # Cheap check since lock will not be free most of the time
+
+          Redlock::Client.new([redis], :retry_count => 0).lock(LOCK_KEY, 30_000) do |locked|
+            break false unless locked
+            break false if skip_update?(redis)
+
+            yield
+
+            redis.set(LAST_RUN_KEY, redis.time.first)
+          end
+        end
+      end
+
+      def skip_update?(redis)
+        results = redis.pipelined { |pipeline| [pipeline.time, pipeline.get(LAST_RUN_KEY)] }
+        last_run_distance = results[0][0] - results[1].to_i
+
+        last_run_distance < Sidekiq::Ultimate::Configuration.instance.empty_queues_cache_refresh_interval_sec
+      end
+    end
+  end
+end

data/lib/sidekiq/ultimate/expirable_set.rb

@@ -2,8 +2,6 @@
 
 require "monitor"
 
-require "concurrent/utility/monotonic_time"
-
 module Sidekiq
   module Ultimate
     # List that tracks when elements were added and enumerates over those not
@@ -24,7 +22,6 @@ module Sidekiq
     # It does not deduplicates elements. Eviction happens only upon elements
     # retrieval (see {#each}).
     #
-    # @see http://ruby-concurrency.github.io/concurrent-ruby/Concurrent.html#monotonic_time-class_method
     # @see https://ruby-doc.org/core/Process.html#method-c-clock_gettime
     # @see https://linux.die.net/man/3/clock_gettime
     #
@@ -52,7 +49,7 @@ module Sidekiq
       # @return [ExpirableSet] self
       def add(element, ttl:)
         @mon.synchronize do
-          expires_at = Concurrent.monotonic_time + ttl
+          expires_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + ttl
 
           # do not allow decrease element's expiry
           @set[element] = expires_at if @set[element] < expires_at
@@ -71,7 +68,7 @@ module Sidekiq
         return to_enum __method__ unless block_given?
 
         @mon.synchronize do
-          horizon = Concurrent.monotonic_time
+          horizon = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
           @set.each { |k, v| yield(k) if horizon <= v }
         end