sidekiq 6.5.1 → 7.3.6

data/lib/sidekiq/web/csrf_protection.rb

@@ -27,7 +27,6 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 require "securerandom"
-require "base64"
 require "rack/request"
 
 module Sidekiq
@@ -57,12 +56,12 @@ module Sidekiq
       end
 
       def logger(env)
-        @logger ||= (env["rack.logger"] || ::Logger.new(env["rack.errors"]))
+        @logger ||= env["rack.logger"] || ::Logger.new(env["rack.errors"])
       end
 
       def deny(env)
         logger(env).warn "attack prevented by #{self.class}"
-        [403, {"Content-Type" => "text/plain"}, ["Forbidden"]]
+        [403, {Rack::CONTENT_TYPE => "text/plain"}, ["Forbidden"]]
       end
 
       def session(env)
@@ -116,7 +115,7 @@ module Sidekiq
         sess = session(env)
         localtoken = sess[:csrf]
 
-        # Checks that Rack::Session::Cookie actualy contains the csrf toekn
+        # Checks that Rack::Session::Cookie actually contains the csrf token
         return false if localtoken.nil?
 
         # Rotate the session token after every use
@@ -143,7 +142,7 @@ module Sidekiq
         one_time_pad = SecureRandom.random_bytes(token.length)
         encrypted_token = xor_byte_strings(one_time_pad, token)
         masked_token = one_time_pad + encrypted_token
-        Base64.urlsafe_encode64(masked_token)
+        encode_token(masked_token)
       end
 
       # Essentially the inverse of +mask_token+.
@@ -152,7 +151,7 @@ module Sidekiq
         # value and decrypt it
         token_length = masked_token.length / 2
         one_time_pad = masked_token[0...token_length]
-        encrypted_token = masked_token[token_length..-1]
+        encrypted_token = masked_token[token_length..]
         xor_byte_strings(one_time_pad, encrypted_token)
       end
 
@@ -168,8 +167,12 @@ module Sidekiq
         ::Rack::Utils.secure_compare(token.to_s, decode_token(local).to_s)
       end
 
+      def encode_token(token)
+        [token].pack("m0").tr("+/", "-_")
+      end
+
       def decode_token(token)
-        Base64.urlsafe_decode64(token)
+        token.tr("-_", "+/").unpack1("m0")
       end
 
       def xor_byte_strings(s1, s2)

data/lib/sidekiq/web/helpers.rb

@@ -6,8 +6,51 @@ require "yaml"
 require "cgi"
 
 module Sidekiq
-  # This is not a public API
+  # These methods are available to pages within the Web UI and UI extensions.
+  # They are not public APIs for applications to use.
   module WebHelpers
+    def style_tag(location, **kwargs)
+      global = location.match?(/:\/\//)
+      location = root_path + location if !global && !location.start_with?(root_path)
+      attrs = {
+        type: "text/css",
+        media: "screen",
+        rel: "stylesheet",
+        nonce: csp_nonce,
+        href: location
+      }
+      html_tag(:link, attrs.merge(kwargs))
+    end
+
+    def script_tag(location, **kwargs)
+      global = location.match?(/:\/\//)
+      location = root_path + location if !global && !location.start_with?(root_path)
+      attrs = {
+        type: "text/javascript",
+        nonce: csp_nonce,
+        src: location
+      }
+      html_tag(:script, attrs.merge(kwargs)) {}
+    end
+
+    # NB: keys and values are not escaped; do not allow user input
+    # in the attributes
+    private def html_tag(tagname, attrs)
+      s = +"<#{tagname}"
+      attrs.each_pair do |k, v|
+        next unless v
+        s << " #{k}=\"#{v}\""
+      end
+      if block_given?
+        s << ">"
+        yield s
+        s << "</#{tagname}>"
+      else
+        s << " />"
+      end
+      s
+    end
+
     def strings(lang)
       @strings ||= {}
 
@@ -15,12 +58,16 @@ module Sidekiq
       # so extensions can be localized
       @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
         find_locale_files(lang).each do |file|
-          strs = YAML.load(File.open(file))
+          strs = YAML.safe_load(File.read(file))
           global.merge!(strs[lang])
         end
       end
     end
 
+    def to_json(x)
+      Sidekiq.dump_json(x)
+    end
+
     def singularize(str, count)
       if count == 1 && str.respond_to?(:singularize) # rails
         str.singularize
@@ -42,15 +89,36 @@ module Sidekiq
     end
 
     def available_locales
-      @available_locales ||= locale_files.map { |path| File.basename(path, ".yml") }.uniq
+      @available_locales ||= Set.new(locale_files.map { |path| File.basename(path, ".yml") })
     end
 
     def find_locale_files(lang)
       locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
     end
 
-    # This is a hook for a Sidekiq Pro feature.  Please don't touch.
-    def filtering(*)
+    def search(jobset, substr)
+      resultset = jobset.scan(substr).to_a
+      @current_page = 1
+      @count = @total_size = resultset.size
+      resultset
+    end
+
+    def filtering(which)
+      erb(:filtering, locals: {which: which})
+    end
+
+    def filter_link(jid, within = "retries")
+      if within.nil?
+        ::Rack::Utils.escape_html(jid)
+      else
+        "<a href='#{root_path}filter/#{within}?substr=#{jid}'>#{::Rack::Utils.escape_html(jid)}</a>"
+      end
+    end
+
+    def display_tags(job, within = "retries")
+      job.tags.map { |tag|
+        "<span class='label label-info jobtag'>#{filter_link(tag, within)}</span>"
+      }.join(" ")
     end
 
     # This view helper provide ability display you html code in
@@ -96,7 +164,10 @@ module Sidekiq
     #
     # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
     def locale
-      @locale ||= begin
+      # session[:locale] is set via the locale selector from the footer
+      @locale ||= if (l = session&.fetch(:locale, nil)) && available_locales.include?(l)
+        l
+      else
         matched_locale = user_preferred_languages.map { |preferred|
           preferred_language = preferred.split("-", 2).first
 
@@ -111,14 +182,7 @@ module Sidekiq
       end
     end
 
-    # within is used by Sidekiq Pro
-    def display_tags(job, within = nil)
-      job.tags.map { |tag|
-        "<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
-      }.join(" ")
-    end
-
-    # mperham/sidekiq#3243
+    # sidekiq/sidekiq#3243
     def unfiltered?
       yield unless env["PATH_INFO"].start_with?("/filter/")
     end
@@ -137,7 +201,7 @@ module Sidekiq
     end
 
     def sort_direction_label
-      params[:direction] == "asc" ? "&uarr;" : "&darr;"
+      (params[:direction] == "asc") ? "&uarr;" : "&darr;"
     end
 
     def workset
@@ -148,22 +212,39 @@ module Sidekiq
       @processes ||= Sidekiq::ProcessSet.new
     end
 
+    # Sorts processes by hostname following the natural sort order
+    def sorted_processes
+      @sorted_processes ||= begin
+        return processes unless processes.all? { |p| p["hostname"] }
+
+        processes.to_a.sort_by do |process|
+          # Kudos to `shurikk` on StackOverflow
+          # https://stackoverflow.com/a/15170063/575547
+          process["hostname"].split(/(\d+)/).map { |a| /\d+/.match?(a) ? a.to_i : a }
+        end
+      end
+    end
+
+    def busy_weights(capsule_weights)
+      # backwards compat with 7.0.0, remove in 7.1
+      cw = [capsule_weights].flatten
+      cw.map { |hash|
+        hash.map { |name, weight| (weight > 0) ? +name << ": " << weight.to_s : name }.join(", ")
+      }.join("; ")
+    end
+
     def stats
       @stats ||= Sidekiq::Stats.new
     end
 
-    def redis_connection
+    def redis_url
       Sidekiq.redis do |conn|
-        conn.connection[:id]
+        conn.config.server_url
       end
     end
 
-    def namespace
-      @ns ||= Sidekiq.redis { |conn| conn.respond_to?(:namespace) ? conn.namespace : nil }
-    end
-
     def redis_info
-      Sidekiq.redis_info
+      Sidekiq.default_configuration.redis_info
     end
 
     def root_path
@@ -175,7 +256,7 @@ module Sidekiq
     end
 
     def current_status
-      workset.size == 0 ? "idle" : "active"
+      (workset.size == 0) ? "idle" : "active"
     end
 
     def relative_time(time)
@@ -208,7 +289,7 @@ module Sidekiq
     end
 
     def truncate(text, truncate_after_chars = 2000)
-      truncate_after_chars && text.size > truncate_after_chars ? "#{text[0..truncate_after_chars]}..." : text
+      (truncate_after_chars && text.size > truncate_after_chars) ? "#{text[0..truncate_after_chars]}..." : text
     end
 
     def display_args(args, truncate_after_chars = 2000)
@@ -228,6 +309,10 @@ module Sidekiq
       "<input type='hidden' name='authenticity_token' value='#{env[:csrf_token]}'/>"
     end
 
+    def csp_nonce
+      env[:csp_nonce]
+    end
+
     def to_display(arg)
       arg.inspect
     rescue
@@ -261,27 +346,17 @@ module Sidekiq
       elsif rss_kb < 10_000_000
         "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
       else
-        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
+        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)), precision: 1)} GB"
       end
     end
 
-    def number_with_delimiter(number)
-      return "" if number.nil?
-
-      begin
-        Float(number)
-      rescue ArgumentError, TypeError
-        return number
-      end
-
-      options = {delimiter: ",", separator: "."}
-      parts = number.to_s.to_str.split(".")
-      parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{options[:delimiter]}")
-      parts.join(options[:separator])
+    def number_with_delimiter(number, options = {})
+      precision = options[:precision] || 0
+      %(<span data-nwp="#{precision}">#{number.round(precision)}</span>)
     end
 
     def h(text)
-      ::Rack::Utils.escape_html(text)
+      ::Rack::Utils.escape_html(text.to_s)
     rescue ArgumentError => e
       raise unless e.message.eql?("invalid byte sequence in UTF-8")
       text.encode!("UTF-16", "UTF-8", invalid: :replace, replace: "").encode!("UTF-8", "UTF-16")
@@ -301,7 +376,7 @@ module Sidekiq
     end
 
     def environment_title_prefix
-      environment = Sidekiq[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      environment = Sidekiq.default_configuration[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
 
       "[#{environment.upcase}] " unless environment == "production"
     end
@@ -314,11 +389,9 @@ module Sidekiq
       Time.now.utc.strftime("%H:%M:%S UTC")
     end
 
-    def redis_connection_and_namespace
-      @redis_connection_and_namespace ||= begin
-        namespace_suffix = namespace.nil? ? "" : "##{namespace}"
-        "#{redis_connection}#{namespace_suffix}"
-      end
+    def pollable?
+      # there's no point to refreshing the metrics pages every N seconds
+      !(current_path == "" || current_path.index("metrics"))
     end
 
     def retry_or_delete_or_kill(job, params)

data/lib/sidekiq/web/router.rb

@@ -39,10 +39,13 @@ module Sidekiq
       route(DELETE, path, &block)
     end
 
-    def route(method, path, &block)
+    def route(*methods, path, &block)
       @routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
 
-      @routes[method] << WebRoute.new(method, path, block)
+      methods.each do |method|
+        method = method.to_s.upcase
+        @routes[method] << WebRoute.new(method, path, block)
+      end
     end
 
     def match(env)

data/lib/sidekiq/web.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "erb"
+require "securerandom"
 
 require "sidekiq"
 require "sidekiq/api"
@@ -30,9 +31,24 @@ module Sidekiq
       "Queues" => "queues",
       "Retries" => "retries",
       "Scheduled" => "scheduled",
-      "Dead" => "morgue"
+      "Dead" => "morgue",
+      "Metrics" => "metrics"
     }
 
+    if Gem::Version.new(Rack::RELEASE) < Gem::Version.new("3")
+      CONTENT_LANGUAGE = "Content-Language"
+      CONTENT_SECURITY_POLICY = "Content-Security-Policy"
+      LOCATION = "Location"
+      X_CASCADE = "X-Cascade"
+      X_CONTENT_TYPE_OPTIONS = "X-Content-Type-Options"
+    else
+      CONTENT_LANGUAGE = "content-language"
+      CONTENT_SECURITY_POLICY = "content-security-policy"
+      LOCATION = "location"
+      X_CASCADE = "x-cascade"
+      X_CONTENT_TYPE_OPTIONS = "x-content-type-options"
+    end
+
     class << self
       def settings
         self
@@ -47,6 +63,10 @@ module Sidekiq
       end
       alias_method :tabs, :custom_tabs
 
+      def custom_job_info_rows
+        @custom_job_info_rows ||= []
+      end
+
       def locales
         @locales ||= LOCALES
       end
@@ -75,14 +95,6 @@ module Sidekiq
         send(:"#{attribute}=", value)
       end
 
-      def sessions=(val)
-        puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
-      end
-
-      def session_secret=(val)
-        puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
-      end
-
       attr_accessor :app_url, :redis_pool
       attr_writer :locales, :views
     end
@@ -105,6 +117,7 @@ module Sidekiq
     end
 
     def call(env)
+      env[:csp_nonce] = SecureRandom.base64(16)
       app.call(env)
     end
 
@@ -129,11 +142,50 @@ module Sidekiq
       send(:"#{attribute}=", value)
     end
 
-    def sessions=(val)
-      puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
-    end
+    # Register a class as a Sidekiq Web UI extension. The class should
+    # provide one or more tabs which map to an index route. Options:
+    #
+    # @param extension [Class] Class which contains the HTTP actions, required
+    # @param name [String] the name of the extension, used to namespace assets
+    # @param tab [String | Array] labels(s) of the UI tabs
+    # @param index [String | Array] index route(s) for each tab
+    # @param root_dir [String] directory location to find assets, locales and views, typically `web/` within the gemfile
+    # @param asset_paths [Array] one or more directories under {root}/assets/{name} to be publicly served, e.g. ["js", "css", "img"]
+    # @param cache_for [Integer] amount of time to cache assets, default one day
+    #
+    # TODO name, tab and index will be mandatory in 8.0
+    #
+    # Web extensions will have a root `web/` directory with `locales/`, `assets/`
+    # and `views/` subdirectories.
+    def self.register(extension, name: nil, tab: nil, index: nil, root_dir: nil, cache_for: 86400, asset_paths: nil)
+      tab = Array(tab)
+      index = Array(index)
+      tab.zip(index).each do |tab, index|
+        tabs[tab] = index
+      end
+      if root_dir
+        locdir = File.join(root_dir, "locales")
+        locales << locdir if File.directory?(locdir)
+
+        if asset_paths && name
+          # if you have {root}/assets/{name}/js/scripts.js
+          # and {root}/assets/{name}/css/styles.css
+          # you would pass in:
+          #   asset_paths: ["js", "css"]
+          # See script_tag and style_tag in web/helpers.rb
+          assdir = File.join(root_dir, "assets")
+          assurls = Array(asset_paths).map { |x| "/#{name}/#{x}" }
+          assetprops = {
+            urls: assurls,
+            root: assdir,
+            cascade: true
+          }
+          assetprops[:header_rules] = [[:all, {Rack::CACHE_CONTROL => "private, max-age=#{cache_for.to_i}"}]] if cache_for
+          middlewares << [[Rack::Static, assetprops], nil]
+        end
+      end
 
-    def self.register(extension)
+      yield self if block_given?
       extension.registered(WebApplication)
     end
 
@@ -144,7 +196,7 @@ module Sidekiq
       m = middlewares
 
       rules = []
-      rules = [[:all, {"Cache-Control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
+      rules = [[:all, {Rack::CACHE_CONTROL => "private, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
       ::Rack::Builder.new do
         use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
@@ -161,7 +213,7 @@ module Sidekiq
   Sidekiq::WebApplication.helpers WebHelpers
   Sidekiq::WebApplication.helpers Sidekiq::Paginator
 
-  Sidekiq::WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+  Sidekiq::WebAction.class_eval <<-RUBY, Web::LAYOUT, -1 # standard:disable Style/EvalWithLocation
     def _render
       #{ERB.new(File.read(Web::LAYOUT)).src}
     end

data/lib/sidekiq/worker_compatibility_alias.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  # Sidekiq::Job is a new alias for Sidekiq::Worker as of Sidekiq 6.3.0.
+  # Use `include Sidekiq::Job` rather than `include Sidekiq::Worker`.
+  #
+  # The term "worker" is too generic and overly confusing, used in several
+  # different contexts meaning different things. Many people call a Sidekiq
+  # process a "worker". Some people call the thread that executes jobs a
+  # "worker". This change brings Sidekiq closer to ActiveJob where your job
+  # classes extend ApplicationJob.
+  Worker = Job
+end