sidekiq 6.4.2 → 6.5.0

data/lib/sidekiq/redis_connection.rb

@@ -5,8 +5,81 @@ require "redis"
 require "uri"
 
 module Sidekiq
-  class RedisConnection
+  module RedisConnection
+    class RedisAdapter
+      BaseError = Redis::BaseError
+      CommandError = Redis::CommandError
+
+      def initialize(options)
+        warn("Usage of the 'redis' gem within Sidekiq itself is deprecated, Sidekiq 7.0 will only use the new, simpler 'redis-client' gem", caller) if ENV["SIDEKIQ_REDIS_CLIENT"] == "1"
+        @options = options
+      end
+
+      def new_client
+        namespace = @options[:namespace]
+
+        client = Redis.new client_opts(@options)
+        if namespace
+          begin
+            require "redis/namespace"
+            Redis::Namespace.new(namespace, redis: client)
+          rescue LoadError
+            Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
+                                 "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
+            exit(-127)
+          end
+        else
+          client
+        end
+      end
+
+      private
+
+      def client_opts(options)
+        opts = options.dup
+        if opts[:namespace]
+          opts.delete(:namespace)
+        end
+
+        if opts[:network_timeout]
+          opts[:timeout] = opts[:network_timeout]
+          opts.delete(:network_timeout)
+        end
+
+        opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
+
+        # Issue #3303, redis-rb will silently retry an operation.
+        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
+        # is performed twice but I believe this is much, much rarer
+        # than the reconnect silently fixing a problem; we keep it
+        # on by default.
+        opts[:reconnect_attempts] ||= 1
+
+        opts
+      end
+    end
+
+    @adapter = RedisAdapter
+
     class << self
+      attr_reader :adapter
+
+      # RedisConnection.adapter = :redis
+      # RedisConnection.adapter = :redis_client
+      def adapter=(adapter)
+        raise "no" if adapter == self
+        result = case adapter
+        when :redis
+          RedisAdapter
+        when Class
+          adapter
+        else
+          require "sidekiq/#{adapter}_adapter"
+          nil
+        end
+        @adapter = result if result
+      end
+
       def create(options = {})
         symbolized_options = options.transform_keys(&:to_sym)
 
@@ -19,20 +92,21 @@ module Sidekiq
         elsif Sidekiq.server?
           # Give ourselves plenty of connections.  pool is lazy
           # so we won't create them until we need them.
-          Sidekiq.options[:concurrency] + 5
+          Sidekiq[:concurrency] + 5
         elsif ENV["RAILS_MAX_THREADS"]
           Integer(ENV["RAILS_MAX_THREADS"])
         else
           5
         end
 
-        verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
+        verify_sizing(size, Sidekiq[:concurrency]) if Sidekiq.server?
 
         pool_timeout = symbolized_options[:pool_timeout] || 1
         log_info(symbolized_options)
 
+        redis_config = adapter.new(symbolized_options)
         ConnectionPool.new(timeout: pool_timeout, size: size) do
-          build_client(symbolized_options)
+          redis_config.new_client
         end
       end
 
@@ -50,47 +124,6 @@ module Sidekiq
         raise ArgumentError, "Your Redis connection pool is too small for Sidekiq. Your pool has #{size} connections but must have at least #{concurrency + 2}" if size < (concurrency + 2)
       end
 
-      def build_client(options)
-        namespace = options[:namespace]
-
-        client = Redis.new client_opts(options)
-        if namespace
-          begin
-            require "redis/namespace"
-            Redis::Namespace.new(namespace, redis: client)
-          rescue LoadError
-            Sidekiq.logger.error("Your Redis configuration uses the namespace '#{namespace}' but the redis-namespace gem is not included in the Gemfile." \
-                                 "Add the gem to your Gemfile to continue using a namespace. Otherwise, remove the namespace parameter.")
-            exit(-127)
-          end
-        else
-          client
-        end
-      end
-
-      def client_opts(options)
-        opts = options.dup
-        if opts[:namespace]
-          opts.delete(:namespace)
-        end
-
-        if opts[:network_timeout]
-          opts[:timeout] = opts[:network_timeout]
-          opts.delete(:network_timeout)
-        end
-
-        opts[:driver] ||= Redis::Connection.drivers.last || "ruby"
-
-        # Issue #3303, redis-rb will silently retry an operation.
-        # This can lead to duplicate jobs if Sidekiq::Client's LPUSH
-        # is performed twice but I believe this is much, much rarer
-        # than the reconnect silently fixing a problem; we keep it
-        # on by default.
-        opts[:reconnect_attempts] ||= 1
-
-        opts
-      end
-
       def log_info(options)
         redacted = "REDACTED"
 
@@ -110,9 +143,9 @@ module Sidekiq
           sentinel[:password] = redacted if sentinel[:password]
         end
         if Sidekiq.server?
-          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with redis options #{scrubbed_options}")
+          Sidekiq.logger.info("Booting Sidekiq #{Sidekiq::VERSION} with #{adapter.name} options #{scrubbed_options}")
         else
-          Sidekiq.logger.debug("#{Sidekiq::NAME} client with redis options #{scrubbed_options}")
+          Sidekiq.logger.debug("#{Sidekiq::NAME} client with #{adapter.name} options #{scrubbed_options}")
         end
       end
 

data/lib/sidekiq/ring_buffer.rb

@@ -0,0 +1,29 @@
+require "forwardable"
+
+module Sidekiq
+  class RingBuffer
+    include Enumerable
+    extend Forwardable
+    def_delegators :@buf, :[], :each, :size
+
+    def initialize(size, default = 0)
+      @size = size
+      @buf = Array.new(size, default)
+      @index = 0
+    end
+
+    def <<(element)
+      @buf[@index % @size] = element
+      @index += 1
+      element
+    end
+
+    def buffer
+      @buf
+    end
+
+    def reset(default = 0)
+      @buf.fill(default)
+    end
+  end
+end

data/lib/sidekiq/scheduled.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require "sidekiq"
-require "sidekiq/util"
 require "sidekiq/api"
+require "sidekiq/component"
 
 module Sidekiq
   module Scheduled
@@ -52,8 +52,8 @@ module Sidekiq
           @lua_zpopbyscore_sha = raw_conn.script(:load, LUA_ZPOPBYSCORE)
         end
 
-        conn.evalsha(@lua_zpopbyscore_sha, keys: keys, argv: argv)
-      rescue Redis::CommandError => e
+        conn.evalsha(@lua_zpopbyscore_sha, keys, argv)
+      rescue RedisConnection.adapter::CommandError => e
         raise unless e.message.start_with?("NOSCRIPT")
 
         @lua_zpopbyscore_sha = nil
@@ -67,12 +67,13 @@ module Sidekiq
     # just pops the job back onto its original queue so the
     # workers can pick it up like any other job.
     class Poller
-      include Util
+      include Sidekiq::Component
 
       INITIAL_WAIT = 10
 
-      def initialize
-        @enq = (Sidekiq.options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
+      def initialize(options)
+        @config = options
+        @enq = (options[:scheduled_enq] || Sidekiq::Scheduled::Enq).new
         @sleeper = ConnectionPool::TimedStack.new
         @done = false
         @thread = nil
@@ -100,7 +101,7 @@ module Sidekiq
             enqueue
             wait
           end
-          Sidekiq.logger.info("Scheduler exiting...")
+          logger.info("Scheduler exiting...")
         }
       end
 
@@ -171,14 +172,14 @@ module Sidekiq
       #
       # We only do this if poll_interval_average is unset (the default).
       def poll_interval_average
-        Sidekiq.options[:poll_interval_average] ||= scaled_poll_interval
+        @config[:poll_interval_average] ||= scaled_poll_interval
       end
 
       # Calculates an average poll interval based on the number of known Sidekiq processes.
       # This minimizes a single point of failure by dispersing check-ins but without taxing
       # Redis if you run many Sidekiq processes.
       def scaled_poll_interval
-        process_count * Sidekiq.options[:average_scheduled_poll_interval]
+        process_count * @config[:average_scheduled_poll_interval]
       end
 
       def process_count
@@ -197,7 +198,7 @@ module Sidekiq
         # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
         # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
         total = 0
-        total += INITIAL_WAIT unless Sidekiq.options[:poll_interval_average]
+        total += INITIAL_WAIT unless @config[:poll_interval_average]
         total += (5 * rand)
 
         @sleeper.pop(total)

data/lib/sidekiq/testing.rb

@@ -188,7 +188,7 @@ module Sidekiq
       end
 
       def clear_for(queue, klass)
-        jobs_by_queue[queue].clear
+        jobs_by_queue[queue.to_s].clear
         jobs_by_class[klass].clear
       end
 

data/lib/sidekiq/transaction_aware_client.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "sidekiq/client"
+
+module Sidekiq
+  class TransactionAwareClient
+    def initialize(redis_pool)
+      @redis_client = Client.new(redis_pool)
+    end
+
+    def push(item)
+      # pre-allocate the JID so we can return it immediately and
+      # save it to the database as part of the transaction.
+      item["jid"] ||= SecureRandom.hex(12)
+      AfterCommitEverywhere.after_commit { @redis_client.push(item) }
+      item["jid"]
+    end
+
+    ##
+    # We don't provide transactionality for push_bulk because we don't want
+    # to hold potentially hundreds of thousands of job records in memory due to
+    # a long running enqueue process.
+    def push_bulk(items)
+      @redis_client.push_bulk(items)
+    end
+  end
+end
+
+##
+# Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
+module Sidekiq
+  def self.transactional_push!
+    begin
+      require "after_commit_everywhere"
+    rescue LoadError
+      Sidekiq.logger.error("You need to add after_commit_everywhere to your Gemfile to use Sidekiq's transactional client")
+      raise
+    end
+
+    default_job_options["client_class"] = Sidekiq::TransactionAwareClient
+    Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
+    true
+  end
+end

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.4.2"
+  VERSION = "6.5.0"
 end

data/lib/sidekiq/web/helpers.rb

@@ -301,7 +301,7 @@ module Sidekiq
     end
 
     def environment_title_prefix
-      environment = Sidekiq.options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      environment = Sidekiq[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
 
       "[#{environment.upcase}] " unless environment == "production"
     end

data/lib/sidekiq/worker.rb

@@ -359,7 +359,8 @@ module Sidekiq
 
       def build_client # :nodoc:
         pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
-        Sidekiq::Client.new(pool)
+        client_class = get_sidekiq_options["client_class"] || Sidekiq::Client
+        client_class.new(pool)
       end
     end
   end

data/lib/sidekiq.rb

@@ -5,6 +5,7 @@ fail "Sidekiq #{Sidekiq::VERSION} does not support Ruby versions below 2.5.0." i
 
 require "sidekiq/logger"
 require "sidekiq/client"
+require "sidekiq/transaction_aware_client"
 require "sidekiq/worker"
 require "sidekiq/job"
 require "sidekiq/redis_connection"
@@ -52,19 +53,79 @@ module Sidekiq
     puts "Calm down, yo."
   end
 
+  # config.concurrency = 5
+  def self.concurrency=(val)
+    self[:concurrency] = Integer(val)
+  end
+
+  # config.queues = %w( high default low )                 # strict
+  # config.queues = %w( high,3 default,2 low,1 )           # weighted
+  # config.queues = %w( feature1,1 feature2,1 feature3,1 ) # random
+  #
+  # With weighted priority, queue will be checked first (weight / total) of the time.
+  # high will be checked first (3/6) or 50% of the time.
+  # I'd recommend setting weights between 1-10. Weights in the hundreds or thousands
+  # are ridiculous and unnecessarily expensive. You can get random queue ordering
+  # by explicitly setting all weights to 1.
+  def self.queues=(val)
+    self[:queues] = Array(val).each_with_object([]) do |qstr, memo|
+      name, weight = qstr.split(",")
+      self[:strict] = false if weight.to_i > 0
+      [weight.to_i, 1].max.times do
+        memo << name
+      end
+    end
+  end
+
+  ### Private APIs
+  def self.default_error_handler(ex, ctx)
+    logger.warn(dump_json(ctx)) unless ctx.empty?
+    logger.warn("#{ex.class.name}: #{ex.message}")
+    logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
+  end
+
+  @config = DEFAULTS.dup
   def self.options
-    @options ||= DEFAULTS.dup
+    logger.warn "`config.options[:key] = value` is deprecated, use `config[:key] = value`: #{caller(1..2)}"
+    @config
   end
 
   def self.options=(opts)
-    @options = opts
+    logger.warn "config.options = hash` is deprecated, use `config.merge!(hash)`: #{caller(1..2)}"
+    @config = opts
+  end
+
+  def self.[](key)
+    @config[key]
+  end
+
+  def self.[]=(key, val)
+    @config[key] = val
+  end
+
+  def self.merge!(hash)
+    @config.merge!(hash)
+  end
+
+  def self.fetch(*args, &block)
+    @config.fetch(*args, &block)
+  end
+
+  def self.handle_exception(ex, ctx = {})
+    self[:error_handlers].each do |handler|
+      handler.call(ex, ctx)
+    rescue => ex
+      logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
+      logger.error ex
+      logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
+    end
   end
+  ###
 
   ##
   # Configuration for Sidekiq server, use like:
   #
   #   Sidekiq.configure_server do |config|
-  #     config.redis = { :namespace => 'myapp', :size => 25, :url => 'redis://myhost:8877/0' }
   #     config.server_middleware do |chain|
   #       chain.add MyServerHook
   #     end
@@ -77,7 +138,7 @@ module Sidekiq
   # Configuration for Sidekiq client, use like:
   #
   #   Sidekiq.configure_client do |config|
-  #     config.redis = { :namespace => 'myapp', :size => 1, :url => 'redis://myhost:8877/0' }
+  #     config.redis = { size: 1, url: 'redis://myhost:8877/0' }
   #   end
   def self.configure_client
     yield self unless server?
@@ -93,7 +154,7 @@ module Sidekiq
       retryable = true
       begin
         yield conn
-      rescue Redis::BaseError => ex
+      rescue RedisConnection.adapter::BaseError => ex
         # 2550 Failover can cause the server to become a replica, need
         # to disconnect and reopen the socket to get back to the primary.
         # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
@@ -118,7 +179,7 @@ module Sidekiq
       else
         conn.info
       end
-    rescue Redis::CommandError => ex
+    rescue RedisConnection.adapter::CommandError => ex
       # 2850 return fake version when INFO command has (probably) been renamed
       raise unless /unknown command/.match?(ex.message)
       FAKE_INFO
@@ -126,19 +187,19 @@ module Sidekiq
   end
 
   def self.redis_pool
-    @redis ||= Sidekiq::RedisConnection.create
+    @redis ||= RedisConnection.create
   end
 
   def self.redis=(hash)
     @redis = if hash.is_a?(ConnectionPool)
       hash
     else
-      Sidekiq::RedisConnection.create(hash)
+      RedisConnection.create(hash)
     end
   end
 
   def self.client_middleware
-    @client_chain ||= Middleware::Chain.new
+    @client_chain ||= Middleware::Chain.new(self)
     yield @client_chain if block_given?
     @client_chain
   end
@@ -150,7 +211,7 @@ module Sidekiq
   end
 
   def self.default_server_middleware
-    Middleware::Chain.new
+    Middleware::Chain.new(self)
   end
 
   def self.default_worker_options=(hash) # deprecated
@@ -179,7 +240,7 @@ module Sidekiq
   #   end
   # end
   def self.death_handlers
-    options[:death_handlers]
+    self[:death_handlers]
   end
 
   def self.load_json(string)
@@ -209,7 +270,7 @@ module Sidekiq
 
   def self.logger=(logger)
     if logger.nil?
-      self.logger.fatal!
+      self.logger.level = Logger::FATAL
       return self.logger
     end
 
@@ -232,7 +293,7 @@ module Sidekiq
   #
   # See sidekiq/scheduled.rb for an in-depth explanation of this value
   def self.average_scheduled_poll_interval=(interval)
-    options[:average_scheduled_poll_interval] = interval
+    self[:average_scheduled_poll_interval] = interval
   end
 
   # Register a proc to handle any error which occurs within the Sidekiq process.
@@ -243,7 +304,7 @@ module Sidekiq
   #
   # The default error handler logs errors to Sidekiq.logger.
   def self.error_handlers
-    options[:error_handlers]
+    self[:error_handlers]
   end
 
   # Register a block to run at a point in the Sidekiq lifecycle.
@@ -256,12 +317,12 @@ module Sidekiq
   #   end
   def self.on(event, &block)
     raise ArgumentError, "Symbols only please: #{event}" unless event.is_a?(Symbol)
-    raise ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
-    options[:lifecycle_events][event] << block
+    raise ArgumentError, "Invalid event name: #{event}" unless self[:lifecycle_events].key?(event)
+    self[:lifecycle_events][event] << block
   end
 
   def self.strict_args!(mode = :raise)
-    options[:on_complex_arguments] = mode
+    self[:on_complex_arguments] = mode
   end
 
   # We are shutting down Sidekiq but what about threads that

data/web/locales/pt-br.yml

@@ -8,6 +8,7 @@
   History: Histórico
   Busy: Ocupados
   Processed: Processados
+  Utilization: Utilização
   Failed: Falhas
   Scheduled: Agendados
   Retries: Tentativas
@@ -26,18 +27,20 @@
   Delete: Apagar
   AddToQueue: Adicionar à fila
   AreYouSureDeleteJob: Deseja deletar esta tarefa?
-  AreYouSureDeleteQueue: Deseja deletar a %{queue} fila?
+  AreYouSureDeleteQueue: Deseja deletar a fila %{queue}? Isso irá deletar todas as tarefas desta fila.
   Queues: Filas
   Size: Tamanho
   Actions: Ações
   NextRetry: Próxima Tentativa
   RetryCount: Número de Tentativas
   RetryNow: Tentar novamente agora
+  Kill: Matar
   LastRetry: Última tentativa
   OriginallyFailed: Falhou originalmente
   AreYouSure: Tem certeza?
   DeleteAll: Apagar tudo
   RetryAll: Tentar tudo novamente
+  KillAll: Matar todas
   NoRetriesFound: Nenhuma tentativa encontrada
   Error: Erro
   ErrorClass: Classe de erro
@@ -58,11 +61,26 @@
   OneMonth: 1 mês
   ThreeMonths: 3 meses
   SixMonths: 6 meses
-  Failures : Falhas
-  DeadJobs : Tarefas mortas
-  NoDeadJobsFound : Nenhuma tarefa morta foi encontrada
-  Dead : Morta
-  Processes : Processos
-  Thread : Thread
-  Threads : Threads
-  Jobs : Tarefas
+  Failures: Falhas
+  DeadJobs: Tarefas mortas
+  NoDeadJobsFound: Nenhuma tarefa morta foi encontrada
+  Dead: Morta
+  Process: Processo
+  Processes: Processos
+  Name: Nome
+  Thread: Thread
+  Threads: Threads
+  Jobs: Tarefas
+  Paused: Pausado
+  Stop: Parar
+  Quiet: Silenciar
+  StopAll: Parar Todos
+  QuietAll: Silenciar Todos
+  PollingInterval: Intervalo de Polling
+  Plugins: Plug-ins
+  NotYetEnqueued: Ainda não enfileirado
+  CreatedAt: Criado em
+  BackToApp: De volta ao aplicativo
+  Latency: Latência
+  Pause: Pausar
+  Unpause: Despausar

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 6.4.2
+  version: 6.5.0
 platform: ruby
 authors:
 - Mike Perham
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-19 00:00:00.000000000 Z
+date: 2022-06-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -75,8 +75,8 @@ files:
 - lib/sidekiq/api.rb
 - lib/sidekiq/cli.rb
 - lib/sidekiq/client.rb
+- lib/sidekiq/component.rb
 - lib/sidekiq/delay.rb
-- lib/sidekiq/exception_handler.rb
 - lib/sidekiq/extensions/action_mailer.rb
 - lib/sidekiq/extensions/active_record.rb
 - lib/sidekiq/extensions/class_methods.rb
@@ -92,17 +92,20 @@ files:
 - lib/sidekiq/middleware/chain.rb
 - lib/sidekiq/middleware/current_attributes.rb
 - lib/sidekiq/middleware/i18n.rb
+- lib/sidekiq/middleware/modules.rb
 - lib/sidekiq/monitor.rb
 - lib/sidekiq/paginator.rb
 - lib/sidekiq/processor.rb
 - lib/sidekiq/rails.rb
+- lib/sidekiq/redis_client_adapter.rb
 - lib/sidekiq/redis_connection.rb
+- lib/sidekiq/ring_buffer.rb
 - lib/sidekiq/scheduled.rb
 - lib/sidekiq/sd_notify.rb
 - lib/sidekiq/systemd.rb
 - lib/sidekiq/testing.rb
 - lib/sidekiq/testing/inline.rb
-- lib/sidekiq/util.rb
+- lib/sidekiq/transaction_aware_client.rb
 - lib/sidekiq/version.rb
 - lib/sidekiq/web.rb
 - lib/sidekiq/web/action.rb

data/lib/sidekiq/exception_handler.rb

@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-require "sidekiq"
-
-module Sidekiq
-  module ExceptionHandler
-    class Logger
-      def call(ex, ctx)
-        Sidekiq.logger.warn(Sidekiq.dump_json(ctx)) unless ctx.empty?
-        Sidekiq.logger.warn("#{ex.class.name}: #{ex.message}")
-        Sidekiq.logger.warn(ex.backtrace.join("\n")) unless ex.backtrace.nil?
-      end
-
-      Sidekiq.error_handlers << Sidekiq::ExceptionHandler::Logger.new
-    end
-
-    def handle_exception(ex, ctx = {})
-      Sidekiq.error_handlers.each do |handler|
-        handler.call(ex, ctx)
-      rescue => ex
-        Sidekiq.logger.error "!!! ERROR HANDLER THREW AN ERROR !!!"
-        Sidekiq.logger.error ex
-        Sidekiq.logger.error ex.backtrace.join("\n") unless ex.backtrace.nil?
-      end
-    end
-  end
-end