sidekiq 6.0.5 → 6.1.0

data/lib/sidekiq/logger.rb

@@ -23,7 +23,7 @@ module Sidekiq
       "info" => 1,
       "warn" => 2,
       "error" => 3,
-      "fatal" => 4,
+      "fatal" => 4
     }
     LEVELS.default_proc = proc do |_, level|
       Sidekiq.logger.warn("Invalid log level: #{level.inspect}")
@@ -31,23 +31,23 @@ module Sidekiq
     end
 
     def debug?
-      level >= 0
+      level <= 0
     end
 
     def info?
-      level >= 1
+      level <= 1
     end
 
     def warn?
-      level >= 2
+      level <= 2
     end
 
     def error?
-      level >= 3
+      level <= 3
     end
 
     def fatal?
-      level >= 4
+      level <= 4
     end
 
     def local_level
@@ -152,7 +152,7 @@ module Sidekiq
             pid: ::Process.pid,
             tid: tid,
             lvl: severity,
-            msg: message,
+            msg: message
           }
           c = ctx
           hash["ctx"] = c unless c.empty?

data/lib/sidekiq/manager.rb

@@ -35,7 +35,7 @@ module Sidekiq
       @done = false
       @workers = Set.new
       @count.times do
-        @workers << Processor.new(self)
+        @workers << Processor.new(self, options)
       end
       @plock = Mutex.new
     end
@@ -90,7 +90,7 @@ module Sidekiq
       @plock.synchronize do
         @workers.delete(processor)
         unless @done
-          p = Processor.new(self)
+          p = Processor.new(self, options)
           @workers << p
           p.start
         end
@@ -123,7 +123,7 @@ module Sidekiq
         # contract says that jobs are run AT LEAST once. Process termination
         # is delayed until we're certain the jobs are back in Redis because
         # it is worse to lose a job than to run it twice.
-        strategy = (@options[:fetch] || Sidekiq::BasicFetch)
+        strategy = @options[:fetch]
         strategy.bulk_requeue(jobs, @options)
       end
 

data/lib/sidekiq/monitor.rb

@@ -62,7 +62,7 @@ class Sidekiq::Monitor
       columns = {
         name: [:ljust, (["name"] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
         size: [:rjust, (["size"] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
-        latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD],
+        latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
       }
       columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
       puts
@@ -101,7 +101,7 @@ class Sidekiq::Monitor
       tags = [
         process["tag"],
         process["labels"],
-        (process["quiet"] == "true" ? "quiet" : nil),
+        (process["quiet"] == "true" ? "quiet" : nil)
       ].flatten.compact
       tags.any? ? "[#{tags.join("] [")}]" : nil
     end

data/lib/sidekiq/processor.rb

@@ -28,15 +28,15 @@ module Sidekiq
     attr_reader :thread
     attr_reader :job
 
-    def initialize(mgr)
+    def initialize(mgr, options)
       @mgr = mgr
       @down = false
       @done = false
       @job = nil
       @thread = nil
-      @strategy = (mgr.options[:fetch] || Sidekiq::BasicFetch).new(mgr.options)
-      @reloader = Sidekiq.options[:reloader]
-      @job_logger = (mgr.options[:job_logger] || Sidekiq::JobLogger).new
+      @strategy = options[:fetch]
+      @reloader = options[:reloader] || proc { |&block| block.call }
+      @job_logger = (options[:job_logger] || Sidekiq::JobLogger).new
       @retrier = Sidekiq::JobRetry.new
     end
 

data/lib/sidekiq/rails.rb

@@ -4,6 +4,22 @@ require "sidekiq/worker"
 
 module Sidekiq
   class Rails < ::Rails::Engine
+    class Reloader
+      def initialize(app = ::Rails.application)
+        @app = app
+      end
+
+      def call
+        @app.reloader.wrap do
+          yield
+        end
+      end
+
+      def inspect
+        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
+      end
+    end
+
     # By including the Options module, we allow AJs to directly control sidekiq features
     # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
     # AJ retries don't show up in the Sidekiq UI Retries tab, save any error data, can't be
@@ -23,8 +39,6 @@ module Sidekiq
 
     # This hook happens after all initializers are run, just before returning
     # from config/environment.rb back to sidekiq/cli.rb.
-    # We have to add the reloader after initialize to see if cache_classes has
-    # been turned on.
     #
     # None of this matters on the client-side, only within the Sidekiq process itself.
     config.after_initialize do
@@ -32,21 +46,5 @@ module Sidekiq
         Sidekiq.options[:reloader] = Sidekiq::Rails::Reloader.new
       end
     end
-
-    class Reloader
-      def initialize(app = ::Rails.application)
-        @app = app
-      end
-
-      def call
-        @app.reloader.wrap do
-          yield
-        end
-      end
-
-      def inspect
-        "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
-      end
-    end
   end
 end

data/lib/sidekiq/redis_connection.rb

@@ -8,15 +8,14 @@ module Sidekiq
   class RedisConnection
     class << self
       def create(options = {})
-        options.keys.each do |key|
-          options[key.to_sym] = options.delete(key)
-        end
+        symbolized_options = options.transform_keys(&:to_sym)
 
-        options[:id] = "Sidekiq-#{Sidekiq.server? ? "server" : "client"}-PID-#{::Process.pid}" unless options.key?(:id)
-        options[:url] ||= determine_redis_provider
+        if !symbolized_options[:url] && (u = determine_redis_provider)
+          symbolized_options[:url] = u
+        end
 
-        size = if options[:size]
-          options[:size]
+        size = if symbolized_options[:size]
+          symbolized_options[:size]
         elsif Sidekiq.server?
           # Give ourselves plenty of connections.  pool is lazy
           # so we won't create them until we need them.
@@ -29,11 +28,11 @@ module Sidekiq
 
         verify_sizing(size, Sidekiq.options[:concurrency]) if Sidekiq.server?
 
-        pool_timeout = options[:pool_timeout] || 1
-        log_info(options)
+        pool_timeout = symbolized_options[:pool_timeout] || 1
+        log_info(symbolized_options)
 
         ConnectionPool.new(timeout: pool_timeout, size: size) do
-          build_client(options)
+          build_client(symbolized_options)
         end
       end
 
@@ -93,9 +92,15 @@ module Sidekiq
       end
 
       def log_info(options)
-        # Don't log Redis AUTH password
         redacted = "REDACTED"
-        scrubbed_options = options.dup
+
+        # deep clone so we can muck with these options all we want
+        #
+        # exclude SSL params from dump-and-load because some information isn't
+        # safely dumpable in current Rubies
+        keys = options.keys
+        keys.delete(:ssl_params)
+        scrubbed_options = Marshal.load(Marshal.dump(options.slice(*keys)))
         if scrubbed_options[:url] && (uri = URI.parse(scrubbed_options[:url])) && uri.password
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
@@ -122,7 +127,7 @@ module Sidekiq
         # initialization code at all.
         #
         p = ENV["REDIS_PROVIDER"]
-        if p && p =~ /\:/
+        if p && p =~ /:/
           raise <<~EOM
             REDIS_PROVIDER should be set to the name of the variable which contains the Redis URL, not a URL itself.
             Platforms like Heroku will sell addons that publish a *_URL variable.  You need to tell Sidekiq with REDIS_PROVIDER, e.g.:

data/lib/sidekiq/sd_notify.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+# The MIT License
+#
+# Copyright (c) 2017, 2018, 2019, 2020 Agis Anastasopoulos
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# This is a copy of https://github.com/agis/ruby-sdnotify as of commit a7d52ee
+# The only changes made was "rehoming" it within the Sidekiq module to avoid
+# namespace collisions and applying standard's code formatting style.
+
+require "socket"
+
+# SdNotify is a pure-Ruby implementation of sd_notify(3). It can be used to
+# notify systemd about state changes. Methods of this package are no-op on
+# non-systemd systems (eg. Darwin).
+#
+# The API maps closely to the original implementation of sd_notify(3),
+# therefore be sure to check the official man pages prior to using SdNotify.
+#
+# @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
+module Sidekiq
+  module SdNotify
+    # Exception raised when there's an error writing to the notification socket
+    class NotifyError < RuntimeError; end
+
+    READY = "READY=1"
+    RELOADING = "RELOADING=1"
+    STOPPING = "STOPPING=1"
+    STATUS = "STATUS="
+    ERRNO = "ERRNO="
+    MAINPID = "MAINPID="
+    WATCHDOG = "WATCHDOG=1"
+    FDSTORE = "FDSTORE=1"
+
+    def self.ready(unset_env = false)
+      notify(READY, unset_env)
+    end
+
+    def self.reloading(unset_env = false)
+      notify(RELOADING, unset_env)
+    end
+
+    def self.stopping(unset_env = false)
+      notify(STOPPING, unset_env)
+    end
+
+    # @param status [String] a custom status string that describes the current
+    #   state of the service
+    def self.status(status, unset_env = false)
+      notify("#{STATUS}#{status}", unset_env)
+    end
+
+    # @param errno [Integer]
+    def self.errno(errno, unset_env = false)
+      notify("#{ERRNO}#{errno}", unset_env)
+    end
+
+    # @param pid [Integer]
+    def self.mainpid(pid, unset_env = false)
+      notify("#{MAINPID}#{pid}", unset_env)
+    end
+
+    def self.watchdog(unset_env = false)
+      notify(WATCHDOG, unset_env)
+    end
+
+    def self.fdstore(unset_env = false)
+      notify(FDSTORE, unset_env)
+    end
+
+    # @return [Boolean] true if the service manager expects watchdog keep-alive
+    #   notification messages to be sent from this process.
+    #
+    # If the $WATCHDOG_USEC environment variable is set,
+    # and the $WATCHDOG_PID variable is unset or set to the PID of the current
+    # process
+    #
+    # @note Unlike sd_watchdog_enabled(3), this method does not mutate the
+    #   environment.
+    def self.watchdog?
+      wd_usec = ENV["WATCHDOG_USEC"]
+      wd_pid = ENV["WATCHDOG_PID"]
+
+      return false unless wd_usec
+
+      begin
+        wd_usec = Integer(wd_usec)
+      rescue
+        return false
+      end
+
+      return false if wd_usec <= 0
+      return true if !wd_pid || wd_pid == $$.to_s
+
+      false
+    end
+
+    # Notify systemd with the provided state, via the notification socket, if
+    # any.
+    #
+    # Generally this method will be used indirectly through the other methods
+    # of the library.
+    #
+    # @param state [String]
+    # @param unset_env [Boolean]
+    #
+    # @return [Fixnum, nil] the number of bytes written to the notification
+    #   socket or nil if there was no socket to report to (eg. the program wasn't
+    #   started by systemd)
+    #
+    # @raise [NotifyError] if there was an error communicating with the systemd
+    #   socket
+    #
+    # @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
+    def self.notify(state, unset_env = false)
+      sock = ENV["NOTIFY_SOCKET"]
+
+      return nil unless sock
+
+      ENV.delete("NOTIFY_SOCKET") if unset_env
+
+      begin
+        Addrinfo.unix(sock, :DGRAM).connect do |s|
+          s.close_on_exec = true
+          s.write(state)
+        end
+      rescue => e
+        raise NotifyError, "#{e.class}: #{e.message}", e.backtrace
+      end
+    end
+  end
+end

data/lib/sidekiq/systemd.rb

@@ -0,0 +1,24 @@
+#
+# Sidekiq's systemd integration allows Sidekiq to inform systemd:
+#  1. when it has successfully started
+#  2. when it is starting shutdown
+#  3. periodically for a liveness check with a watchdog thread
+#
+module Sidekiq
+  def self.start_watchdog
+    usec = Integer(ENV["WATCHDOG_USEC"])
+    return Sidekiq.logger.error("systemd Watchdog too fast: " + usec) if usec < 1_000_000
+
+    sec_f = usec / 1_000_000.0
+    # "It is recommended that a daemon sends a keep-alive notification message
+    # to the service manager every half of the time returned here."
+    ping_f = sec_f / 2
+    Sidekiq.logger.info "Pinging systemd watchdog every #{ping_f.round(1)} sec"
+    Thread.new do
+      loop do
+        sleep ping_f
+        Sidekiq::SdNotify.watchdog
+      end
+    end
+  end
+end

data/lib/sidekiq/testing.rb

@@ -337,7 +337,7 @@ module Sidekiq
   Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
 end
 
-if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?
+if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test? && !$TESTING
   puts("**************************************************")
   puts("⛔️ WARNING: Sidekiq testing API enabled, but this is not the test environment.  Your jobs will not go to Redis.")
   puts("**************************************************")

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.0.5"
+  VERSION = "6.1.0"
 end

data/lib/sidekiq/web.rb

@@ -10,8 +10,9 @@ require "sidekiq/web/helpers"
 require "sidekiq/web/router"
 require "sidekiq/web/action"
 require "sidekiq/web/application"
+require "sidekiq/web/csrf_protection"
 
-require "rack/protection"
+require "rack/content_length"
 
 require "rack/builder"
 require "rack/file"
@@ -31,7 +32,7 @@ module Sidekiq
       "Queues" => "queues",
       "Retries" => "retries",
       "Scheduled" => "scheduled",
-      "Dead" => "morgue",
+      "Dead" => "morgue"
     }
 
     class << self
@@ -154,14 +155,14 @@ module Sidekiq
     def build_sessions
       middlewares = self.middlewares
 
-      unless using?(::Rack::Protection) || ENV["RACK_ENV"] == "test"
-        middlewares.unshift [[::Rack::Protection, {use: :authenticity_token}], nil]
-      end
-
       s = sessions
-      return unless s
 
-      unless using? ::Rack::Session::Cookie
+      # turn on CSRF protection if sessions are enabled and this is not the test env
+      if s && !using?(CsrfProtection) && ENV["RACK_ENV"] != "test"
+        middlewares.unshift [[CsrfProtection], nil]
+      end
+
+      if s && !using?(::Rack::Session::Cookie)
         unless (secret = Web.session_secret)
           require "securerandom"
           secret = SecureRandom.hex(64)
@@ -172,6 +173,13 @@ module Sidekiq
 
         middlewares.unshift [[::Rack::Session::Cookie, options], nil]
       end
+
+      # Since Sidekiq::WebApplication no longer calculates its own
+      # Content-Length response header, we must ensure that the Rack middleware
+      # that does this is loaded
+      unless using? ::Rack::ContentLength
+        middlewares.unshift [[::Rack::ContentLength], nil]
+      end
     end
 
     def build