sidekiq 6.0.6 → 6.2.2

data/lib/sidekiq/util.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "forwardable"
 require "socket"
 require "securerandom"
 require "sidekiq/exception_handler"
@@ -8,6 +9,33 @@ module Sidekiq
   ##
   # This module is part of Sidekiq core and not intended for extensions.
   #
+
+  class RingBuffer
+    include Enumerable
+    extend Forwardable
+    def_delegators :@buf, :[], :each, :size
+
+    def initialize(size, default = 0)
+      @size = size
+      @buf = Array.new(size, default)
+      @index = 0
+    end
+
+    def <<(element)
+      @buf[@index % @size] = element
+      @index += 1
+      element
+    end
+
+    def buffer
+      @buf
+    end
+
+    def reset(default = 0)
+      @buf.fill(default)
+    end
+  end
+
   module Util
     include ExceptionHandler
 

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.0.6"
+  VERSION = "6.2.2"
 end

data/lib/sidekiq/web/action.rb

@@ -15,7 +15,7 @@ module Sidekiq
     end
 
     def halt(res)
-      throw :halt, res
+      throw :halt, [res, {"Content-Type" => "text/plain"}, [res.to_s]]
     end
 
     def redirect(location)
@@ -68,7 +68,7 @@ module Sidekiq
     end
 
     def json(payload)
-      [200, {"Content-Type" => "application/json", "Cache-Control" => "no-cache"}, [Sidekiq.dump_json(payload)]]
+      [200, {"Content-Type" => "application/json", "Cache-Control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
     end
 
     def initialize(env, block)

data/lib/sidekiq/web/application.rb

@@ -4,7 +4,6 @@ module Sidekiq
   class WebApplication
     extend WebRouter
 
-    CONTENT_LENGTH = "Content-Length"
     REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
     CSP_HEADER = [
       "default-src 'self' https: http:",
@@ -42,6 +41,13 @@ module Sidekiq
       # nothing, backwards compatibility
     end
 
+    head "/" do
+      # HEAD / is the cheapest heartbeat possible,
+      # it hits Redis to ensure connectivity
+      Sidekiq.redis { |c| c.llen("queue:default") }
+      ""
+    end
+
     get "/" do
       @redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
       stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)
@@ -76,15 +82,17 @@ module Sidekiq
       erb(:queues)
     end
 
+    QUEUE_NAME = /\A[a-z_:.\-0-9]+\z/i
+
     get "/queues/:name" do
       @name = route_params[:name]
 
-      halt(404) unless @name
+      halt(404) if !@name || @name !~ QUEUE_NAME
 
       @count = (params["count"] || 25).to_i
       @queue = Sidekiq::Queue.new(@name)
       (@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
-      @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
+      @messages = @messages.map { |msg| Sidekiq::JobRecord.new(msg, @name) }
 
       erb(:queue)
     end
@@ -105,7 +113,7 @@ module Sidekiq
 
     post "/queues/:name/delete" do
       name = route_params[:name]
-      Sidekiq::Job.new(params["key_val"], name).delete
+      Sidekiq::JobRecord.new(params["key_val"], name).delete
 
       redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
     end
@@ -298,7 +306,7 @@ module Sidekiq
         self.class.run_afters(app, action)
       end
 
-      resp = case resp
+      case resp
       when Array
         # redirects go here
         resp
@@ -306,19 +314,17 @@ module Sidekiq
         # rendered content goes here
         headers = {
           "Content-Type" => "text/html",
-          "Cache-Control" => "no-cache",
+          "Cache-Control" => "private, no-store",
           "Content-Language" => action.locale,
           "Content-Security-Policy" => CSP_HEADER
         }
         # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]
       end
-
-      resp
     end
 
     def self.helpers(mod = nil, &block)
-      if block_given?
+      if block
         WebAction.class_eval(&block)
       else
         WebAction.send(:include, mod)

data/lib/sidekiq/web/csrf_protection.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+# this file originally based on authenticity_token.rb from the sinatra/rack-protection project
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2011-2017 Konstantin Haase
+# Copyright (c) 2015-2017 Zachary Scott
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require "securerandom"
+require "base64"
+require "rack/request"
+
+module Sidekiq
+  class Web
+    class CsrfProtection
+      def initialize(app, options = nil)
+        @app = app
+      end
+
+      def call(env)
+        accept?(env) ? admit(env) : deny(env)
+      end
+
+      private
+
+      def admit(env)
+        # On each successful request, we create a fresh masked token
+        # which will be used in any forms rendered for this request.
+        s = session(env)
+        s[:csrf] ||= SecureRandom.base64(TOKEN_LENGTH)
+        env[:csrf_token] = mask_token(s[:csrf])
+        @app.call(env)
+      end
+
+      def safe?(env)
+        %w[GET HEAD OPTIONS TRACE].include? env["REQUEST_METHOD"]
+      end
+
+      def logger(env)
+        @logger ||= (env["rack.logger"] || ::Logger.new(env["rack.errors"]))
+      end
+
+      def deny(env)
+        logger(env).warn "attack prevented by #{self.class}"
+        [403, {"Content-Type" => "text/plain"}, ["Forbidden"]]
+      end
+
+      def session(env)
+        env["rack.session"] || fail(<<~EOM)
+          Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
+          make sure you mount Sidekiq::Web *inside* your application routes:
+
+
+          Rails.application.routes.draw do
+            mount Sidekiq::Web => "/sidekiq"
+            ....
+          end
+
+
+          If this is a Rails app in API mode, you need to enable sessions.
+
+            https://guides.rubyonrails.org/api_app.html#using-session-middlewares
+
+          If this is a bare Rack app, use a session middleware before Sidekiq::Web:
+
+            # first, use IRB to create a shared secret key for sessions and commit it
+            require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
+
+            # now use the secret with a session cookie middleware
+            use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
+            run Sidekiq::Web
+
+        EOM
+      end
+
+      def accept?(env)
+        return true if safe?(env)
+
+        giventoken = ::Rack::Request.new(env).params["authenticity_token"]
+        valid_token?(env, giventoken)
+      end
+
+      TOKEN_LENGTH = 32
+
+      # Checks that the token given to us as a parameter matches
+      # the token stored in the session.
+      def valid_token?(env, giventoken)
+        return false if giventoken.nil? || giventoken.empty?
+
+        begin
+          token = decode_token(giventoken)
+        rescue ArgumentError # client input is invalid
+          return false
+        end
+
+        sess = session(env)
+        localtoken = sess[:csrf]
+
+        # Checks that Rack::Session::Cookie actualy contains the csrf toekn
+        return false if localtoken.nil?
+
+        # Rotate the session token after every use
+        sess[:csrf] = SecureRandom.base64(TOKEN_LENGTH)
+
+        # See if it's actually a masked token or not. We should be able
+        # to handle any unmasked tokens that we've issued without error.
+
+        if unmasked_token?(token)
+          compare_with_real_token token, localtoken
+        elsif masked_token?(token)
+          unmasked = unmask_token(token)
+          compare_with_real_token unmasked, localtoken
+        else
+          false # Token is malformed
+        end
+      end
+
+      # Creates a masked version of the authenticity token that varies
+      # on each request. The masking is used to mitigate SSL attacks
+      # like BREACH.
+      def mask_token(token)
+        token = decode_token(token)
+        one_time_pad = SecureRandom.random_bytes(token.length)
+        encrypted_token = xor_byte_strings(one_time_pad, token)
+        masked_token = one_time_pad + encrypted_token
+        Base64.strict_encode64(masked_token)
+      end
+
+      # Essentially the inverse of +mask_token+.
+      def unmask_token(masked_token)
+        # Split the token into the one-time pad and the encrypted
+        # value and decrypt it
+        token_length = masked_token.length / 2
+        one_time_pad = masked_token[0...token_length]
+        encrypted_token = masked_token[token_length..-1]
+        xor_byte_strings(one_time_pad, encrypted_token)
+      end
+
+      def unmasked_token?(token)
+        token.length == TOKEN_LENGTH
+      end
+
+      def masked_token?(token)
+        token.length == TOKEN_LENGTH * 2
+      end
+
+      def compare_with_real_token(token, local)
+        ::Rack::Utils.secure_compare(token.to_s, decode_token(local).to_s)
+      end
+
+      def decode_token(token)
+        Base64.strict_decode64(token)
+      end
+
+      def xor_byte_strings(s1, s2)
+        s1.bytes.zip(s2.bytes).map { |(c1, c2)| c1 ^ c2 }.pack("c*")
+      end
+    end
+  end
+end

data/lib/sidekiq/web/helpers.rb

@@ -10,18 +10,25 @@ module Sidekiq
   module WebHelpers
     def strings(lang)
       @strings ||= {}
-      @strings[lang] ||= begin
-        # Allow sidekiq-web extensions to add locale paths
-        # so extensions can be localized
-        settings.locales.each_with_object({}) do |path, global|
-          find_locale_files(lang).each do |file|
-            strs = YAML.load(File.open(file))
-            global.merge!(strs[lang])
-          end
+
+      # Allow sidekiq-web extensions to add locale paths
+      # so extensions can be localized
+      @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
+        find_locale_files(lang).each do |file|
+          strs = YAML.load(File.open(file))
+          global.merge!(strs[lang])
         end
       end
     end
 
+    def singularize(str, count)
+      if count == 1 && str.respond_to?(:singularize) # rails
+        str.singularize
+      else
+        str
+      end
+    end
+
     def clear_caches
       @strings = nil
       @locale_files = nil
@@ -118,7 +125,7 @@ module Sidekiq
     # within is used by Sidekiq Pro
     def display_tags(job, within = nil)
       job.tags.map { |tag|
-        "<span class='jobtag label label-info'>#{::Rack::Utils.escape_html(tag)}</span>"
+        "<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
       }.join(" ")
     end
 
@@ -158,8 +165,7 @@ module Sidekiq
 
     def redis_connection
       Sidekiq.redis do |conn|
-        c = conn.connection
-        "redis://#{c[:location]}/#{c[:db]}"
+        conn.connection[:id]
       end
     end
 
@@ -201,12 +207,9 @@ module Sidekiq
 
     # Merge options with current params, filter safe params, and stringify to query string
     def qparams(options)
-      # stringify
-      options.keys.each do |key|
-        options[key.to_s] = options.delete(key)
-      end
+      stringified_options = options.transform_keys(&:to_s)
 
-      to_query_string(params.merge(options))
+      to_query_string(params.merge(stringified_options))
     end
 
     def to_query_string(params)
@@ -233,7 +236,7 @@ module Sidekiq
     end
 
     def csrf_tag
-      "<input type='hidden' name='authenticity_token' value='#{session[:csrf]}'/>"
+      "<input type='hidden' name='authenticity_token' value='#{env[:csrf_token]}'/>"
     end
 
     def to_display(arg)
@@ -261,7 +264,21 @@ module Sidekiq
       end
     end
 
+    def format_memory(rss_kb)
+      return "0" if rss_kb.nil? || rss_kb == 0
+
+      if rss_kb < 100_000
+        "#{number_with_delimiter(rss_kb)} KB"
+      elsif rss_kb < 10_000_000
+        "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
+      else
+        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
+      end
+    end
+
     def number_with_delimiter(number)
+      return "" if number.nil?
+
       begin
         Float(number)
       rescue ArgumentError, TypeError

data/lib/sidekiq/web/router.rb

@@ -15,6 +15,10 @@ module Sidekiq
     REQUEST_METHOD = "REQUEST_METHOD"
     PATH_INFO = "PATH_INFO"
 
+    def head(path, &block)
+      route(HEAD, path, &block)
+    end
+
     def get(path, &block)
       route(GET, path, &block)
     end
@@ -39,7 +43,6 @@ module Sidekiq
       @routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
 
       @routes[method] << WebRoute.new(method, path, block)
-      @routes[HEAD] << WebRoute.new(method, path, block) if method == GET
     end
 
     def match(env)
@@ -66,7 +69,7 @@ module Sidekiq
   class WebRoute
     attr_accessor :request_method, :pattern, :block, :name
 
-    NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^\.:$\/]+)/
+    NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^.:$\/]+)/
 
     def initialize(request_method, pattern, block)
       @request_method = request_method

data/lib/sidekiq/web.rb

@@ -10,12 +10,11 @@ require "sidekiq/web/helpers"
 require "sidekiq/web/router"
 require "sidekiq/web/action"
 require "sidekiq/web/application"
+require "sidekiq/web/csrf_protection"
 
-require "rack/protection"
-
+require "rack/content_length"
 require "rack/builder"
-require "rack/file"
-require "rack/session/cookie"
+require "rack/static"
 
 module Sidekiq
   class Web
@@ -39,14 +38,6 @@ module Sidekiq
         self
       end
 
-      def middlewares
-        @middlewares ||= []
-      end
-
-      def use(*middleware_args, &block)
-        middlewares << [middleware_args, block]
-      end
-
       def default_tabs
         DEFAULT_TABS
       end
@@ -72,32 +63,45 @@ module Sidekiq
         opts.each { |key| set(key, false) }
       end
 
-      # Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
+      def middlewares
+        @middlewares ||= []
+      end
+
+      def use(*args, &block)
+        middlewares << [args, block]
+      end
+
       def set(attribute, value)
         send(:"#{attribute}=", value)
       end
 
-      attr_accessor :app_url, :session_secret, :redis_pool, :sessions
+      def sessions=(val)
+        puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+      end
+
+      def session_secret=(val)
+        puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+      end
+
+      attr_accessor :app_url, :redis_pool
       attr_writer :locales, :views
     end
 
     def self.inherited(child)
       child.app_url = app_url
-      child.session_secret = session_secret
       child.redis_pool = redis_pool
-      child.sessions = sessions
     end
 
     def settings
       self.class.settings
     end
 
-    def use(*middleware_args, &block)
-      middlewares << [middleware_args, block]
+    def middlewares
+      @middlewares ||= self.class.middlewares
     end
 
-    def middlewares
-      @middlewares ||= Web.middlewares.dup
+    def use(*args, &block)
+      middlewares << [args, block]
     end
 
     def call(env)
@@ -125,18 +129,8 @@ module Sidekiq
       send(:"#{attribute}=", value)
     end
 
-    # Default values
-    set :sessions, true
-
-    attr_writer :sessions
-
-    def sessions
-      unless instance_variable_defined?("@sessions")
-        @sessions = self.class.sessions
-        @sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
-      end
-
-      @sessions
+    def sessions=(val)
+      puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
     end
 
     def self.register(extension)
@@ -145,50 +139,20 @@ module Sidekiq
 
     private
 
-    def using?(middleware)
-      middlewares.any? do |(m, _)|
-        m.is_a?(Array) && (m[0] == middleware || m[0].is_a?(middleware))
-      end
-    end
-
-    def build_sessions
-      middlewares = self.middlewares
-
-      unless using?(::Rack::Protection) || ENV["RACK_ENV"] == "test"
-        middlewares.unshift [[::Rack::Protection, {use: :authenticity_token}], nil]
-      end
-
-      s = sessions
-      return unless s
-
-      unless using? ::Rack::Session::Cookie
-        unless (secret = Web.session_secret)
-          require "securerandom"
-          secret = SecureRandom.hex(64)
-        end
-
-        options = {secret: secret}
-        options = options.merge(s.to_hash) if s.respond_to? :to_hash
-
-        middlewares.unshift [[::Rack::Session::Cookie, options], nil]
-      end
-    end
-
     def build
-      build_sessions
-
-      middlewares = self.middlewares
       klass = self.class
+      m = middlewares
 
-      ::Rack::Builder.new do
-        %w[stylesheets javascripts images].each do |asset_dir|
-          map "/#{asset_dir}" do
-            run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {"Cache-Control" => "public, max-age=86400"})
-          end
-        end
-
-        middlewares.each { |middleware, block| use(*middleware, &block) }
+      rules = []
+      rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
+      ::Rack::Builder.new do
+        use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
+                          root: ASSETS,
+                          cascade: true,
+                          header_rules: rules
+        m.each { |middleware, block| use(*middleware, &block) }
+        use Sidekiq::Web::CsrfProtection unless $TESTING
         run WebApplication.new(klass)
       end
     end

data/lib/sidekiq/worker.rb

@@ -235,12 +235,9 @@ module Sidekiq
 
       def client_push(item) # :nodoc:
         pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
-        # stringify
-        item.keys.each do |key|
-          item[key.to_s] = item.delete(key)
-        end
+        stringified_item = item.transform_keys(&:to_s)
 
-        Sidekiq::Client.new(pool).push(item)
+        Sidekiq::Client.new(pool).push(stringified_item)
       end
     end
   end

data/lib/sidekiq.rb

@@ -20,6 +20,7 @@ module Sidekiq
     labels: [],
     concurrency: 10,
     require: ".",
+    strict: true,
     environment: nil,
     timeout: 25,
     poll_interval_average: nil,
@@ -95,10 +96,11 @@ module Sidekiq
       retryable = true
       begin
         yield conn
-      rescue Redis::CommandError => ex
+      rescue Redis::BaseError => ex
         # 2550 Failover can cause the server to become a replica, need
         # to disconnect and reopen the socket to get back to the primary.
-        if retryable && ex.message =~ /READONLY/
+        # 4495 Use the same logic if we have a "Not enough replicas" error from the primary
+        if retryable && ex.message =~ /READONLY|NOREPLICAS/
           conn.disconnect!
           retryable = false
           retry
@@ -196,7 +198,7 @@ module Sidekiq
   end
 
   def self.logger
-    @logger ||= Sidekiq::Logger.new(STDOUT, level: Logger::INFO)
+    @logger ||= Sidekiq::Logger.new($stdout, level: Logger::INFO)
   end
 
   def self.logger=(logger)

data/sidekiq.gemspec

@@ -5,17 +5,24 @@ Gem::Specification.new do |gem|
   gem.email = ["mperham@gmail.com"]
   gem.summary = "Simple, efficient background processing for Ruby"
   gem.description = "Simple, efficient background processing for Ruby."
-  gem.homepage = "http://sidekiq.org"
+  gem.homepage = "https://sidekiq.org"
   gem.license = "LGPL-3.0"
 
   gem.executables = ["sidekiq", "sidekiqmon"]
-  gem.files = `git ls-files | grep -Ev '^(test|myapp|examples)'`.split("\n")
+  gem.files = ["sidekiq.gemspec", "README.md", "Changes.md", "LICENSE"] + `git ls-files | grep -E '^(bin|lib|web)'`.split("\n")
   gem.name = "sidekiq"
   gem.version = Sidekiq::VERSION
   gem.required_ruby_version = ">= 2.5.0"
 
-  gem.add_dependency "redis", ">= 4.1.0"
+  gem.metadata = {
+    "homepage_uri" => "https://sidekiq.org",
+    "bug_tracker_uri" => "https://github.com/mperham/sidekiq/issues",
+    "documentation_uri" => "https://github.com/mperham/sidekiq/wiki",
+    "changelog_uri" => "https://github.com/mperham/sidekiq/blob/master/Changes.md",
+    "source_code_uri" => "https://github.com/mperham/sidekiq"
+  }
+
+  gem.add_dependency "redis", ">= 4.2.0"
   gem.add_dependency "connection_pool", ">= 2.2.2"
   gem.add_dependency "rack", "~> 2.0"
-  gem.add_dependency "rack-protection", ">= 2.0.0"
 end