sidekiq 6.0.0 → 6.3.0

data/lib/sidekiq/web/application.rb

@@ -4,8 +4,6 @@ module Sidekiq
   class WebApplication
     extend WebRouter
 
-    CONTENT_LENGTH = "Content-Length"
-    CONTENT_TYPE = "Content-Type"
     REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
     CSP_HEADER = [
       "default-src 'self' https: http:",
@@ -20,7 +18,7 @@ module Sidekiq
       "script-src 'self' https: http: 'unsafe-inline'",
       "style-src 'self' https: http: 'unsafe-inline'",
       "worker-src 'self'",
-      "base-uri 'self'",
+      "base-uri 'self'"
     ].join("; ").freeze
 
     def initialize(klass)
@@ -43,6 +41,13 @@ module Sidekiq
       # nothing, backwards compatibility
     end
 
+    head "/" do
+      # HEAD / is the cheapest heartbeat possible,
+      # it hits Redis to ensure connectivity
+      Sidekiq.redis { |c| c.llen("queue:default") }
+      ""
+    end
+
     get "/" do
       @redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
       stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)
@@ -77,28 +82,38 @@ module Sidekiq
       erb(:queues)
     end
 
+    QUEUE_NAME = /\A[a-z_:.\-0-9]+\z/i
+
     get "/queues/:name" do
       @name = route_params[:name]
 
-      halt(404) unless @name
+      halt(404) if !@name || @name !~ QUEUE_NAME
 
       @count = (params["count"] || 25).to_i
       @queue = Sidekiq::Queue.new(@name)
-      (@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count)
-      @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
+      (@current_page, @total_size, @jobs) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
+      @jobs = @jobs.map { |msg| Sidekiq::JobRecord.new(msg, @name) }
 
       erb(:queue)
     end
 
     post "/queues/:name" do
-      Sidekiq::Queue.new(route_params[:name]).clear
+      queue = Sidekiq::Queue.new(route_params[:name])
+
+      if Sidekiq.pro? && params["pause"]
+        queue.pause!
+      elsif Sidekiq.pro? && params["unpause"]
+        queue.unpause!
+      else
+        queue.clear
+      end
 
       redirect "#{root_path}queues"
     end
 
     post "/queues/:name/delete" do
       name = route_params[:name]
-      Sidekiq::Job.new(params["key_val"], name).delete
+      Sidekiq::JobRecord.new(params["key_val"], name).delete
 
       redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
     end
@@ -268,7 +283,7 @@ module Sidekiq
           scheduled: sidekiq_stats.scheduled_size,
           retries: sidekiq_stats.retry_size,
           dead: sidekiq_stats.dead_size,
-          default_latency: sidekiq_stats.default_queue_latency,
+          default_latency: sidekiq_stats.default_queue_latency
         },
         redis: redis_stats,
         server_utc_time: server_utc_time
@@ -283,41 +298,33 @@ module Sidekiq
       action = self.class.match(env)
       return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
 
-      resp = catch(:halt) {
-        app = @klass
+      app = @klass
+      resp = catch(:halt) do # rubocop:disable Standard/SemanticBlocks
         self.class.run_befores(app, action)
-        begin
-          resp = action.instance_exec env, &action.block
-        ensure
-          self.class.run_afters(app, action)
-        end
-
-        resp
-      }
+        action.instance_exec env, &action.block
+      ensure
+        self.class.run_afters(app, action)
+      end
 
-      resp = case resp
+      case resp
       when Array
+        # redirects go here
         resp
       else
+        # rendered content goes here
         headers = {
           "Content-Type" => "text/html",
-          "Cache-Control" => "no-cache",
+          "Cache-Control" => "private, no-store",
           "Content-Language" => action.locale,
-          "Content-Security-Policy" => CSP_HEADER,
+          "Content-Security-Policy" => CSP_HEADER
         }
-
+        # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]
       end
-
-      resp[1] = resp[1].dup
-
-      resp[1][CONTENT_LENGTH] = resp[2].inject(0) { |l, p| l + p.bytesize }.to_s
-
-      resp
     end
 
     def self.helpers(mod = nil, &block)
-      if block_given?
+      if block
         WebAction.class_eval(&block)
       else
         WebAction.send(:include, mod)

data/lib/sidekiq/web/csrf_protection.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+# this file originally based on authenticity_token.rb from the sinatra/rack-protection project
+#
+# The MIT License (MIT)
+#
+# Copyright (c) 2011-2017 Konstantin Haase
+# Copyright (c) 2015-2017 Zachary Scott
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# 'Software'), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+require "securerandom"
+require "base64"
+require "rack/request"
+
+module Sidekiq
+  class Web
+    class CsrfProtection
+      def initialize(app, options = nil)
+        @app = app
+      end
+
+      def call(env)
+        accept?(env) ? admit(env) : deny(env)
+      end
+
+      private
+
+      def admit(env)
+        # On each successful request, we create a fresh masked token
+        # which will be used in any forms rendered for this request.
+        s = session(env)
+        s[:csrf] ||= SecureRandom.base64(TOKEN_LENGTH)
+        env[:csrf_token] = mask_token(s[:csrf])
+        @app.call(env)
+      end
+
+      def safe?(env)
+        %w[GET HEAD OPTIONS TRACE].include? env["REQUEST_METHOD"]
+      end
+
+      def logger(env)
+        @logger ||= (env["rack.logger"] || ::Logger.new(env["rack.errors"]))
+      end
+
+      def deny(env)
+        logger(env).warn "attack prevented by #{self.class}"
+        [403, {"Content-Type" => "text/plain"}, ["Forbidden"]]
+      end
+
+      def session(env)
+        env["rack.session"] || fail(<<~EOM)
+          Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
+          make sure you mount Sidekiq::Web *inside* your application routes:
+
+
+          Rails.application.routes.draw do
+            mount Sidekiq::Web => "/sidekiq"
+            ....
+          end
+
+
+          If this is a Rails app in API mode, you need to enable sessions.
+
+            https://guides.rubyonrails.org/api_app.html#using-session-middlewares
+
+          If this is a bare Rack app, use a session middleware before Sidekiq::Web:
+
+            # first, use IRB to create a shared secret key for sessions and commit it
+            require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
+
+            # now use the secret with a session cookie middleware
+            use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
+            run Sidekiq::Web
+
+        EOM
+      end
+
+      def accept?(env)
+        return true if safe?(env)
+
+        giventoken = ::Rack::Request.new(env).params["authenticity_token"]
+        valid_token?(env, giventoken)
+      end
+
+      TOKEN_LENGTH = 32
+
+      # Checks that the token given to us as a parameter matches
+      # the token stored in the session.
+      def valid_token?(env, giventoken)
+        return false if giventoken.nil? || giventoken.empty?
+
+        begin
+          token = decode_token(giventoken)
+        rescue ArgumentError # client input is invalid
+          return false
+        end
+
+        sess = session(env)
+        localtoken = sess[:csrf]
+
+        # Checks that Rack::Session::Cookie actualy contains the csrf toekn
+        return false if localtoken.nil?
+
+        # Rotate the session token after every use
+        sess[:csrf] = SecureRandom.base64(TOKEN_LENGTH)
+
+        # See if it's actually a masked token or not. We should be able
+        # to handle any unmasked tokens that we've issued without error.
+
+        if unmasked_token?(token)
+          compare_with_real_token token, localtoken
+        elsif masked_token?(token)
+          unmasked = unmask_token(token)
+          compare_with_real_token unmasked, localtoken
+        else
+          false # Token is malformed
+        end
+      end
+
+      # Creates a masked version of the authenticity token that varies
+      # on each request. The masking is used to mitigate SSL attacks
+      # like BREACH.
+      def mask_token(token)
+        token = decode_token(token)
+        one_time_pad = SecureRandom.random_bytes(token.length)
+        encrypted_token = xor_byte_strings(one_time_pad, token)
+        masked_token = one_time_pad + encrypted_token
+        Base64.strict_encode64(masked_token)
+      end
+
+      # Essentially the inverse of +mask_token+.
+      def unmask_token(masked_token)
+        # Split the token into the one-time pad and the encrypted
+        # value and decrypt it
+        token_length = masked_token.length / 2
+        one_time_pad = masked_token[0...token_length]
+        encrypted_token = masked_token[token_length..-1]
+        xor_byte_strings(one_time_pad, encrypted_token)
+      end
+
+      def unmasked_token?(token)
+        token.length == TOKEN_LENGTH
+      end
+
+      def masked_token?(token)
+        token.length == TOKEN_LENGTH * 2
+      end
+
+      def compare_with_real_token(token, local)
+        ::Rack::Utils.secure_compare(token.to_s, decode_token(local).to_s)
+      end
+
+      def decode_token(token)
+        Base64.strict_decode64(token)
+      end
+
+      def xor_byte_strings(s1, s2)
+        s1.bytes.zip(s2.bytes).map { |(c1, c2)| c1 ^ c2 }.pack("c*")
+      end
+    end
+  end
+end

data/lib/sidekiq/web/helpers.rb

@@ -10,18 +10,25 @@ module Sidekiq
   module WebHelpers
     def strings(lang)
       @strings ||= {}
-      @strings[lang] ||= begin
-        # Allow sidekiq-web extensions to add locale paths
-        # so extensions can be localized
-        settings.locales.each_with_object({}) do |path, global|
-          find_locale_files(lang).each do |file|
-            strs = YAML.load(File.open(file))
-            global.merge!(strs[lang])
-          end
+
+      # Allow sidekiq-web extensions to add locale paths
+      # so extensions can be localized
+      @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
+        find_locale_files(lang).each do |file|
+          strs = YAML.load(File.open(file))
+          global.merge!(strs[lang])
         end
       end
     end
 
+    def singularize(str, count)
+      if count == 1 && str.respond_to?(:singularize) # rails
+        str.singularize
+      else
+        str
+      end
+    end
+
     def clear_caches
       @strings = nil
       @locale_files = nil
@@ -63,14 +70,6 @@ module Sidekiq
       @head_html.join if defined?(@head_html)
     end
 
-    def poll_path
-      if current_path != "" && params["poll"]
-        root_path + current_path
-      else
-        ""
-      end
-    end
-
     def text_direction
       get_locale["TextDirection"] || "ltr"
     end
@@ -112,6 +111,13 @@ module Sidekiq
       end
     end
 
+    # within is used by Sidekiq Pro
+    def display_tags(job, within = nil)
+      job.tags.map { |tag|
+        "<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
+      }.join(" ")
+    end
+
     # mperham/sidekiq#3243
     def unfiltered?
       yield unless env["PATH_INFO"].start_with?("/filter/")
@@ -130,6 +136,10 @@ module Sidekiq
       end
     end
 
+    def sort_direction_label
+      params[:direction] == "asc" ? "&uarr;" : "&darr;"
+    end
+
     def workers
       @workers ||= Sidekiq::Workers.new
     end
@@ -142,16 +152,9 @@ module Sidekiq
       @stats ||= Sidekiq::Stats.new
     end
 
-    def retries_with_score(score)
-      Sidekiq.redis { |conn|
-        conn.zrangebyscore("retry", score, score)
-      }.map { |msg| Sidekiq.load_json(msg) }
-    end
-
     def redis_connection
       Sidekiq.redis do |conn|
-        c = conn.connection
-        "redis://#{c[:location]}/#{c[:db]}"
+        conn.connection[:id]
       end
     end
 
@@ -189,16 +192,17 @@ module Sidekiq
       [score.to_f, jid]
     end
 
-    SAFE_QPARAMS = %w[page poll]
+    SAFE_QPARAMS = %w[page direction]
 
     # Merge options with current params, filter safe params, and stringify to query string
     def qparams(options)
-      # stringify
-      options.keys.each do |key|
-        options[key.to_s] = options.delete(key)
-      end
+      stringified_options = options.transform_keys(&:to_s)
+
+      to_query_string(params.merge(stringified_options))
+    end
 
-      params.merge(options).map { |key, value|
+    def to_query_string(params)
+      params.map { |key, value|
         SAFE_QPARAMS.include?(key) ? "#{key}=#{CGI.escape(value.to_s)}" : next
       }.compact.join("&")
     end
@@ -221,7 +225,7 @@ module Sidekiq
     end
 
     def csrf_tag
-      "<input type='hidden' name='authenticity_token' value='#{session[:csrf]}'/>"
+      "<input type='hidden' name='authenticity_token' value='#{env[:csrf_token]}'/>"
     end
 
     def to_display(arg)
@@ -238,7 +242,7 @@ module Sidekiq
       queue class args retry_count retried_at failed_at
       jid error_message error_class backtrace
       error_backtrace enqueued_at retry wrapped
-      created_at
+      created_at tags
     ])
 
     def retry_extra_items(retry_job)
@@ -249,7 +253,21 @@ module Sidekiq
       end
     end
 
+    def format_memory(rss_kb)
+      return "0" if rss_kb.nil? || rss_kb == 0
+
+      if rss_kb < 100_000
+        "#{number_with_delimiter(rss_kb)} KB"
+      elsif rss_kb < 10_000_000
+        "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
+      else
+        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
+      end
+    end
+
     def number_with_delimiter(number)
+      return "" if number.nil?
+
       begin
         Float(number)
       rescue ArgumentError, TypeError
@@ -283,7 +301,7 @@ module Sidekiq
     end
 
     def environment_title_prefix
-      environment = Sidekiq.options[:environment] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      environment = Sidekiq.options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
 
       "[#{environment.upcase}] " unless environment == "production"
     end

data/lib/sidekiq/web/router.rb

@@ -15,6 +15,10 @@ module Sidekiq
     REQUEST_METHOD = "REQUEST_METHOD"
     PATH_INFO = "PATH_INFO"
 
+    def head(path, &block)
+      route(HEAD, path, &block)
+    end
+
     def get(path, &block)
       route(GET, path, &block)
     end
@@ -39,7 +43,6 @@ module Sidekiq
       @routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
 
       @routes[method] << WebRoute.new(method, path, block)
-      @routes[HEAD] << WebRoute.new(method, path, block) if method == GET
     end
 
     def match(env)
@@ -66,7 +69,7 @@ module Sidekiq
   class WebRoute
     attr_accessor :request_method, :pattern, :block, :name
 
-    NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^\.:$\/]+)/
+    NAMED_SEGMENTS_PATTERN = /\/([^\/]*):([^.:$\/]+)/
 
     def initialize(request_method, pattern, block)
       @request_method = request_method
@@ -94,9 +97,7 @@ module Sidekiq
         {} if path == matcher
       else
         path_match = path.match(matcher)
-        if path_match
-          Hash[path_match.names.map(&:to_sym).zip(path_match.captures)]
-        end
+        path_match&.named_captures&.transform_keys(&:to_sym)
       end
     end
   end

data/lib/sidekiq/web.rb

@@ -10,12 +10,11 @@ require "sidekiq/web/helpers"
 require "sidekiq/web/router"
 require "sidekiq/web/action"
 require "sidekiq/web/application"
+require "sidekiq/web/csrf_protection"
 
-require "rack/protection"
-
+require "rack/content_length"
 require "rack/builder"
-require "rack/file"
-require "rack/session/cookie"
+require "rack/static"
 
 module Sidekiq
   class Web
@@ -31,7 +30,7 @@ module Sidekiq
       "Queues" => "queues",
       "Retries" => "retries",
       "Scheduled" => "scheduled",
-      "Dead" => "morgue",
+      "Dead" => "morgue"
     }
 
     class << self
@@ -39,14 +38,6 @@ module Sidekiq
         self
       end
 
-      def middlewares
-        @middlewares ||= []
-      end
-
-      def use(*middleware_args, &block)
-        middlewares << [middleware_args, block]
-      end
-
       def default_tabs
         DEFAULT_TABS
       end
@@ -72,32 +63,45 @@ module Sidekiq
         opts.each { |key| set(key, false) }
       end
 
-      # Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
+      def middlewares
+        @middlewares ||= []
+      end
+
+      def use(*args, &block)
+        middlewares << [args, block]
+      end
+
       def set(attribute, value)
         send(:"#{attribute}=", value)
       end
 
-      attr_accessor :app_url, :session_secret, :redis_pool, :sessions
+      def sessions=(val)
+        puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+      end
+
+      def session_secret=(val)
+        puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+      end
+
+      attr_accessor :app_url, :redis_pool
       attr_writer :locales, :views
     end
 
     def self.inherited(child)
       child.app_url = app_url
-      child.session_secret = session_secret
       child.redis_pool = redis_pool
-      child.sessions = sessions
     end
 
     def settings
       self.class.settings
     end
 
-    def use(*middleware_args, &block)
-      middlewares << [middleware_args, block]
+    def middlewares
+      @middlewares ||= self.class.middlewares
     end
 
-    def middlewares
-      @middlewares ||= Web.middlewares.dup
+    def use(*args, &block)
+      middlewares << [args, block]
     end
 
     def call(env)
@@ -125,18 +129,8 @@ module Sidekiq
       send(:"#{attribute}=", value)
     end
 
-    # Default values
-    set :sessions, true
-
-    attr_writer :sessions
-
-    def sessions
-      unless instance_variable_defined?("@sessions")
-        @sessions = self.class.sessions
-        @sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
-      end
-
-      @sessions
+    def sessions=(val)
+      puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
     end
 
     def self.register(extension)
@@ -145,50 +139,20 @@ module Sidekiq
 
     private
 
-    def using?(middleware)
-      middlewares.any? do |(m, _)|
-        m.is_a?(Array) && (m[0] == middleware || m[0].is_a?(middleware))
-      end
-    end
-
-    def build_sessions
-      middlewares = self.middlewares
-
-      unless using?(::Rack::Protection) || ENV["RACK_ENV"] == "test"
-        middlewares.unshift [[::Rack::Protection, {use: :authenticity_token}], nil]
-      end
-
-      s = sessions
-      return unless s
-
-      unless using? ::Rack::Session::Cookie
-        unless (secret = Web.session_secret)
-          require "securerandom"
-          secret = SecureRandom.hex(64)
-        end
-
-        options = {secret: secret}
-        options = options.merge(s.to_hash) if s.respond_to? :to_hash
-
-        middlewares.unshift [[::Rack::Session::Cookie, options], nil]
-      end
-    end
-
     def build
-      build_sessions
-
-      middlewares = self.middlewares
       klass = self.class
+      m = middlewares
 
-      ::Rack::Builder.new do
-        %w[stylesheets javascripts images].each do |asset_dir|
-          map "/#{asset_dir}" do
-            run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {"Cache-Control" => "public, max-age=86400"})
-          end
-        end
-
-        middlewares.each { |middleware, block| use(*middleware, &block) }
+      rules = []
+      rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
+      ::Rack::Builder.new do
+        use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
+                          root: ASSETS,
+                          cascade: true,
+                          header_rules: rules
+        m.each { |middleware, block| use(*middleware, &block) }
+        use Sidekiq::Web::CsrfProtection unless $TESTING
         run WebApplication.new(klass)
       end
     end