sidekiq 6.1.3 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eafcca9d632a83e39c7e69e47d104493f32b07974d58724347181ea1d9c47511
-  data.tar.gz: '08b987ddd7686f813f29a24d9d9027093f97307e7a22e24c355ed8b3d4c49362'
+  metadata.gz: 990d3e553aed906265ffa0cafb74fa4c79e0ccde957f21ccc3d09531d01e75bf
+  data.tar.gz: 92a68ab1ea824dc78b91610e98c3d10ce4a5794e7aebd66803dc6f29db419420
 SHA512:
-  metadata.gz: 615b18ce20c57301cb4117e6fbeab847b6ba41377ed1ba22d98e32e2df579f1cef1138b0c2757a51a988b0ecef83e47992754fbdb5b96a6f6d653a294755e36c
-  data.tar.gz: c3f3a44a85fc7a1c9405f59bba34842892c8a23e49a3ebac8dce5ba35b48738a9c5ba71635b671d2c6b9b711e5692d780a71ebf4a0b3c66ba8ac95c8ab9f319b
+  metadata.gz: 37664695dd79557af0395fe20855db1d250753dd8c874023a866c27d69e78bfa36b3f70501dad18a6ac64673e62fa8fdf413c77d0052a9ca0d59f25a17df41f9
+  data.tar.gz: 4d6ef75d6eb4be49e8b8d21d56bd0e300a284a4a6db9a0f0357724206b5e1b81b8bf75b53d8442319fd5fe5a9f08a467d75c59289d82d6aa70a00aa0d5102445

data/Changes.md

@@ -2,6 +2,37 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
+6.2.0
+---------
+
+- Store Redis RTT and log if poor [#4824]
+- Add process/thread stats to Busy page [#4806]
+- Improve Web UI on mobile devices [#4840]
+- **Refactor Web UI session usage** [#4804]
+  Numerous people have hit "Forbidden" errors and struggled with Sidekiq's
+  Web UI session requirement. If you have code in your initializer for
+  Web sessions, it's quite possible it will need to be removed. Here's
+  an overview:
+```
+Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
+make sure you mount Sidekiq::Web *inside* your routes in `config/routes.rb` so
+Sidekiq can reuse the Rails session:
+
+  Rails.application.routes.draw do
+    mount Sidekiq::Web => "/sidekiq"
+    ....
+  end
+
+If this is a bare Rack app, use a session middleware before Sidekiq::Web:
+
+  # first, use IRB to create a shared secret key for sessions and commit it
+  require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
+
+  # now, update your Rack app to include the secret with a session cookie middleware
+  use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
+  run Sidekiq::Web
+```
+
 6.1.3
 ---------
 

data/lib/sidekiq/api.rb

@@ -791,12 +791,12 @@ module Sidekiq
         # you'll be happier this way
         conn.pipelined do
           procs.each do |key|
-            conn.hmget(key, "info", "busy", "beat", "quiet", "rss")
+            conn.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
           end
         end
       }
 
-      result.each do |info, busy, at_s, quiet, rss|
+      result.each do |info, busy, at_s, quiet, rss, rtt|
         # If a process is stopped between when we query Redis for `procs` and
         # when we query for `result`, we will have an item in `result` that is
         # composed of `nil` values.
@@ -806,7 +806,8 @@ module Sidekiq
         yield Process.new(hash.merge("busy" => busy.to_i,
                                      "beat" => at_s.to_f,
                                      "quiet" => quiet,
-                                     "rss" => rss))
+                                     "rss" => rss.to_i,
+                                     "rtt_us" => rtt.to_i))
       end
     end
 
@@ -818,6 +819,17 @@ module Sidekiq
       Sidekiq.redis { |conn| conn.scard("processes") }
     end
 
+    # Total number of threads available to execute jobs.
+    # For Sidekiq Enterprise customers this number (in production) must be
+    # less than or equal to your licensed concurrency.
+    def total_concurrency
+      sum { |x| x["concurrency"] }
+    end
+
+    def total_rss
+      sum { |x| x["rss"] || 0 }
+    end
+
     # Returns the identity of the current cluster leader or "" if no leader.
     # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
     # or Sidekiq Pro.

data/lib/sidekiq/fetch.rb

@@ -36,7 +36,15 @@ module Sidekiq
     end
 
     def retrieve_work
-      work = Sidekiq.redis { |conn| conn.brpop(*queues_cmd) }
+      qs = queues_cmd
+      # 4825 Sidekiq Pro with all queues paused will return an
+      # empty set of queues with a trailing TIMEOUT value.
+      if qs.size <= 1
+        sleep(2)
+        return nil
+      end
+
+      work = Sidekiq.redis { |conn| conn.brpop(*qs) }
       UnitOfWork.new(*work) if work
     end
 

data/lib/sidekiq/launcher.rb

@@ -153,6 +153,8 @@ module Sidekiq
           end
         end
 
+        rtt = check_rtt
+
         fails = procd = 0
         kb = memory_usage(::Process.pid)
 
@@ -163,6 +165,7 @@ module Sidekiq
             conn.hmset(key, "info", to_json,
               "busy", curstate.size,
               "beat", Time.now.to_f,
+              "rtt_us", rtt,
               "quiet", @done,
               "rss", kb)
             conn.expire(key, 60)
@@ -185,6 +188,29 @@ module Sidekiq
       end
     end
 
+    RTT_WARNING_LEVEL = 50_000
+
+    def check_rtt
+      a = b = 0
+      Sidekiq.redis do |x|
+        a = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
+        x.ping
+        b = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :microsecond)
+      end
+      rtt = b - a
+      # Ideal RTT for Redis is < 1000µs
+      # Workable is < 10,000µs
+      # Log a warning if it's a disaster.
+      if rtt > RTT_WARNING_LEVEL
+        Sidekiq.logger.warn <<-EOM
+          Your Redis network connection is performing extremely poorly.
+          Current RTT is #{rtt} µs, ideally this should be < 1000.
+          Ensure Redis is running in the same AZ or datacenter as Sidekiq.
+        EOM
+      end
+      rtt
+    end
+
     MEMORY_GRABBER = case RUBY_PLATFORM
     when /linux/
       ->(pid) {

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.1.3"
+  VERSION = "6.2.0"
 end

data/lib/sidekiq/web.rb

@@ -13,10 +13,8 @@ require "sidekiq/web/application"
 require "sidekiq/web/csrf_protection"
 
 require "rack/content_length"
-
 require "rack/builder"
-require "rack/file"
-require "rack/session/cookie"
+require "rack/static"
 
 module Sidekiq
   class Web
@@ -40,14 +38,6 @@ module Sidekiq
         self
       end
 
-      def middlewares
-        @middlewares ||= []
-      end
-
-      def use(*middleware_args, &block)
-        middlewares << [middleware_args, block]
-      end
-
       def default_tabs
         DEFAULT_TABS
       end
@@ -73,32 +63,45 @@ module Sidekiq
         opts.each { |key| set(key, false) }
       end
 
-      # Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
+      def middlewares
+        @middlewares ||= []
+      end
+
+      def use(*args, &block)
+        middlewares << [args, block]
+      end
+
       def set(attribute, value)
         send(:"#{attribute}=", value)
       end
 
-      attr_accessor :app_url, :session_secret, :redis_pool, :sessions
+      def sessions=(val)
+        puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+      end
+
+      def session_secret=(val)
+        puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+      end
+
+      attr_accessor :app_url, :redis_pool
       attr_writer :locales, :views
     end
 
     def self.inherited(child)
       child.app_url = app_url
-      child.session_secret = session_secret
       child.redis_pool = redis_pool
-      child.sessions = sessions
     end
 
     def settings
       self.class.settings
     end
 
-    def use(*middleware_args, &block)
-      middlewares << [middleware_args, block]
+    def middlewares
+      @middlewares ||= self.class.middlewares
     end
 
-    def middlewares
-      @middlewares ||= Web.middlewares.dup
+    def use(*args, &block)
+      middlewares << [args, block]
     end
 
     def call(env)
@@ -126,18 +129,8 @@ module Sidekiq
       send(:"#{attribute}=", value)
     end
 
-    # Default values
-    set :sessions, true
-
-    attr_writer :sessions
-
-    def sessions
-      unless instance_variable_defined?("@sessions")
-        @sessions = self.class.sessions
-        @sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
-      end
-
-      @sessions
+    def sessions=(val)
+      puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
     end
 
     def self.register(extension)
@@ -146,57 +139,19 @@ module Sidekiq
 
     private
 
-    def using?(middleware)
-      middlewares.any? do |(m, _)|
-        m.is_a?(Array) && (m[0] == middleware || m[0].is_a?(middleware))
-      end
-    end
-
-    def build_sessions
-      middlewares = self.middlewares
-
-      s = sessions
-
-      # turn on CSRF protection if sessions are enabled and this is not the test env
-      if s && !using?(CsrfProtection) && ENV["RACK_ENV"] != "test"
-        middlewares.unshift [[CsrfProtection], nil]
-      end
-
-      if s && !using?(::Rack::Session::Cookie)
-        unless (secret = Web.session_secret)
-          require "securerandom"
-          secret = SecureRandom.hex(64)
-        end
-
-        options = {secret: secret}
-        options = options.merge(s.to_hash) if s.respond_to? :to_hash
-
-        middlewares.unshift [[::Rack::Session::Cookie, options], nil]
-      end
-
-      # Since Sidekiq::WebApplication no longer calculates its own
-      # Content-Length response header, we must ensure that the Rack middleware
-      # that does this is loaded
-      unless using? ::Rack::ContentLength
-        middlewares.unshift [[::Rack::ContentLength], nil]
-      end
-    end
-
     def build
-      build_sessions
-
-      middlewares = self.middlewares
       klass = self.class
+      m = middlewares
 
       ::Rack::Builder.new do
-        %w[stylesheets javascripts images].each do |asset_dir|
-          map "/#{asset_dir}" do
-            run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {"Cache-Control" => "public, max-age=86400"})
-          end
-        end
-
-        middlewares.each { |middleware, block| use(*middleware, &block) }
-
+        use Rack::Static, :urls => ["/stylesheets", "/images", "/javascripts"],
+                          :root => ASSETS,
+                          :cascade => true,
+                          :header_rules => [
+                            [:all, {'Cache-Control' => 'public, max-age=86400'}],
+                          ]
+        m.each { |middleware, block| use(*middleware, &block) }
+        use Sidekiq::Web::CsrfProtection unless $TESTING
         run WebApplication.new(klass)
       end
     end

data/lib/sidekiq/web/application.rb

@@ -4,7 +4,6 @@ module Sidekiq
   class WebApplication
     extend WebRouter
 
-    CONTENT_LENGTH = "Content-Length"
     REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
     CSP_HEADER = [
       "default-src 'self' https: http:",
@@ -42,6 +41,13 @@ module Sidekiq
       # nothing, backwards compatibility
     end
 
+    head "/" do
+      # HEAD / is the cheapest heartbeat possible,
+      # it hits Redis to ensure connectivity
+      Sidekiq.redis { |c| c.llen("queue:default") }
+      ""
+    end
+
     get "/" do
       @redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
       stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)

data/lib/sidekiq/web/csrf_protection.rb

@@ -66,7 +66,28 @@ module Sidekiq
       end
 
       def session(env)
-        env["rack.session"] || fail("you need to set up a session middleware *before* #{self.class}")
+        env["rack.session"] || fail(<<~EOM)
+          Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
+          make sure you mount Sidekiq::Web *inside* your application routes:
+
+
+          Rails.application.routes.draw do
+            mount Sidekiq::Web => "/sidekiq"
+            ....
+          end
+
+
+          If this is a bare Rack app, use a session middleware before Sidekiq::Web:
+
+
+          # first, use IRB to create a shared secret key for sessions and commit it
+          require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
+
+
+          # now use the secret with a session cookie middleware
+          use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
+          run Sidekiq::Web
+        EOM
       end
 
       def accept?(env)

data/lib/sidekiq/web/helpers.rb

@@ -22,6 +22,14 @@ module Sidekiq
       end
     end
 
+    def singularize(str, count)
+      if count == 1 && str.respond_to?(:singularize) # rails
+        str.singularize
+      else
+        str
+      end
+    end
+
     def clear_caches
       @strings = nil
       @locale_files = nil
@@ -158,8 +166,7 @@ module Sidekiq
 
     def redis_connection
       Sidekiq.redis do |conn|
-        c = conn.connection
-        "redis://#{c[:location]}/#{c[:db]}"
+        conn.connection[:id]
       end
     end
 
@@ -259,12 +266,14 @@ module Sidekiq
     end
 
     def format_memory(rss_kb)
-      return "" if rss_kb.nil? || rss_kb == 0
+      return "0" if rss_kb.nil? || rss_kb == 0
 
       if rss_kb < 100_000
         "#{number_with_delimiter(rss_kb)} KB"
-      else
+      elsif rss_kb < 10_000_000
         "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
+      else
+        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
       end
     end
 

data/lib/sidekiq/web/router.rb

@@ -15,6 +15,10 @@ module Sidekiq
     REQUEST_METHOD = "REQUEST_METHOD"
     PATH_INFO = "PATH_INFO"
 
+    def head(path, &block)
+      route(HEAD, path, &block)
+    end
+
     def get(path, &block)
       route(GET, path, &block)
     end
@@ -39,7 +43,6 @@ module Sidekiq
       @routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
 
       @routes[method] << WebRoute.new(method, path, block)
-      @routes[HEAD] << WebRoute.new(method, path, block) if method == GET
     end
 
     def match(env)