sidekiq 6.1.1 → 6.5.7

data/lib/sidekiq/transaction_aware_client.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "securerandom"
+require "sidekiq/client"
+
+module Sidekiq
+  class TransactionAwareClient
+    def initialize(redis_pool)
+      @redis_client = Client.new(redis_pool)
+    end
+
+    def push(item)
+      # pre-allocate the JID so we can return it immediately and
+      # save it to the database as part of the transaction.
+      item["jid"] ||= SecureRandom.hex(12)
+      AfterCommitEverywhere.after_commit { @redis_client.push(item) }
+      item["jid"]
+    end
+
+    ##
+    # We don't provide transactionality for push_bulk because we don't want
+    # to hold potentially hundreds of thousands of job records in memory due to
+    # a long running enqueue process.
+    def push_bulk(items)
+      @redis_client.push_bulk(items)
+    end
+  end
+end
+
+##
+# Use `Sidekiq.transactional_push!` in your sidekiq.rb initializer
+module Sidekiq
+  def self.transactional_push!
+    begin
+      require "after_commit_everywhere"
+    rescue LoadError
+      Sidekiq.logger.error("You need to add after_commit_everywhere to your Gemfile to use Sidekiq's transactional client")
+      raise
+    end
+
+    default_job_options["client_class"] = Sidekiq::TransactionAwareClient
+    Sidekiq::JobUtil::TRANSIENT_ATTRIBUTES << "client_class"
+    true
+  end
+end

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.1.1"
+  VERSION = "6.5.7"
 end

data/lib/sidekiq/web/action.rb

@@ -15,11 +15,11 @@ module Sidekiq
     end
 
     def halt(res)
-      throw :halt, res
+      throw :halt, [res, {"content-type" => "text/plain"}, [res.to_s]]
     end
 
     def redirect(location)
-      throw :halt, [302, {"Location" => "#{request.base_url}#{location}"}, []]
+      throw :halt, [302, {"location" => "#{request.base_url}#{location}"}, []]
     end
 
     def params
@@ -68,7 +68,7 @@ module Sidekiq
     end
 
     def json(payload)
-      [200, {"Content-Type" => "application/json", "Cache-Control" => "no-cache"}, [Sidekiq.dump_json(payload)]]
+      [200, {"content-type" => "application/json", "cache-control" => "private, no-store"}, [Sidekiq.dump_json(payload)]]
     end
 
     def initialize(env, block)

data/lib/sidekiq/web/application.rb

@@ -4,7 +4,6 @@ module Sidekiq
   class WebApplication
     extend WebRouter
 
-    CONTENT_LENGTH = "Content-Length"
     REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
     CSP_HEADER = [
       "default-src 'self' https: http:",
@@ -42,15 +41,38 @@ module Sidekiq
       # nothing, backwards compatibility
     end
 
+    head "/" do
+      # HEAD / is the cheapest heartbeat possible,
+      # it hits Redis to ensure connectivity
+      Sidekiq.redis { |c| c.llen("queue:default") }
+      ""
+    end
+
     get "/" do
       @redis_info = redis_info.select { |k, v| REDIS_KEYS.include? k }
-      stats_history = Sidekiq::Stats::History.new((params["days"] || 30).to_i)
+      days = (params["days"] || 30).to_i
+      return halt(401) if days < 1 || days > 180
+
+      stats_history = Sidekiq::Stats::History.new(days)
       @processed_history = stats_history.processed
       @failed_history = stats_history.failed
 
       erb(:dashboard)
     end
 
+    get "/metrics" do
+      q = Sidekiq::Metrics::Query.new
+      @query_result = q.top_jobs
+      erb(:metrics)
+    end
+
+    get "/metrics/:name" do
+      @name = route_params[:name]
+      q = Sidekiq::Metrics::Query.new
+      @query_result = q.for_job(@name)
+      erb(:metrics_for_job)
+    end
+
     get "/busy" do
       erb(:busy)
     end
@@ -76,15 +98,17 @@ module Sidekiq
       erb(:queues)
     end
 
+    QUEUE_NAME = /\A[a-z_:.\-0-9]+\z/i
+
     get "/queues/:name" do
       @name = route_params[:name]
 
-      halt(404) unless @name
+      halt(404) if !@name || @name !~ QUEUE_NAME
 
       @count = (params["count"] || 25).to_i
       @queue = Sidekiq::Queue.new(@name)
-      (@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
-      @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
+      (@current_page, @total_size, @jobs) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
+      @jobs = @jobs.map { |msg| Sidekiq::JobRecord.new(msg, @name) }
 
       erb(:queue)
     end
@@ -105,7 +129,7 @@ module Sidekiq
 
     post "/queues/:name/delete" do
       name = route_params[:name]
-      Sidekiq::Job.new(params["key_val"], name).delete
+      Sidekiq::JobRecord.new(params["key_val"], name).delete
 
       redirect_with_query("#{root_path}queues/#{CGI.escape(name)}")
     end
@@ -288,37 +312,35 @@ module Sidekiq
 
     def call(env)
       action = self.class.match(env)
-      return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
+      return [404, {"content-type" => "text/plain", "x-cascade" => "pass"}, ["Not Found"]] unless action
 
       app = @klass
-      resp = catch(:halt) do # rubocop:disable Standard/SemanticBlocks
+      resp = catch(:halt) do
         self.class.run_befores(app, action)
         action.instance_exec env, &action.block
       ensure
         self.class.run_afters(app, action)
       end
 
-      resp = case resp
+      case resp
       when Array
         # redirects go here
         resp
       else
         # rendered content goes here
         headers = {
-          "Content-Type" => "text/html",
-          "Cache-Control" => "no-cache",
-          "Content-Language" => action.locale,
-          "Content-Security-Policy" => CSP_HEADER
+          "content-type" => "text/html",
+          "cache-control" => "private, no-store",
+          "content-language" => action.locale,
+          "content-security-policy" => CSP_HEADER
         }
         # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]
       end
-
-      resp
     end
 
     def self.helpers(mod = nil, &block)
-      if block_given?
+      if block
         WebAction.class_eval(&block)
       else
         WebAction.send(:include, mod)

data/lib/sidekiq/web/csrf_protection.rb

@@ -66,13 +66,37 @@ module Sidekiq
       end
 
       def session(env)
-        env["rack.session"] || fail("you need to set up a session middleware *before* #{self.class}")
+        env["rack.session"] || fail(<<~EOM)
+          Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
+          make sure you mount Sidekiq::Web *inside* your application routes:
+
+
+          Rails.application.routes.draw do
+            mount Sidekiq::Web => "/sidekiq"
+            ....
+          end
+
+
+          If this is a Rails app in API mode, you need to enable sessions.
+
+            https://guides.rubyonrails.org/api_app.html#using-session-middlewares
+
+          If this is a bare Rack app, use a session middleware before Sidekiq::Web:
+
+            # first, use IRB to create a shared secret key for sessions and commit it
+            require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
+
+            # now use the secret with a session cookie middleware
+            use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
+            run Sidekiq::Web
+
+        EOM
       end
 
       def accept?(env)
         return true if safe?(env)
 
-        giventoken = Rack::Request.new(env).params["authenticity_token"]
+        giventoken = ::Rack::Request.new(env).params["authenticity_token"]
         valid_token?(env, giventoken)
       end
 
@@ -92,6 +116,9 @@ module Sidekiq
         sess = session(env)
         localtoken = sess[:csrf]
 
+        # Checks that Rack::Session::Cookie actualy contains the csrf toekn
+        return false if localtoken.nil?
+
         # Rotate the session token after every use
         sess[:csrf] = SecureRandom.base64(TOKEN_LENGTH)
 
@@ -116,7 +143,7 @@ module Sidekiq
         one_time_pad = SecureRandom.random_bytes(token.length)
         encrypted_token = xor_byte_strings(one_time_pad, token)
         masked_token = one_time_pad + encrypted_token
-        Base64.strict_encode64(masked_token)
+        Base64.urlsafe_encode64(masked_token)
       end
 
       # Essentially the inverse of +mask_token+.
@@ -138,11 +165,11 @@ module Sidekiq
       end
 
       def compare_with_real_token(token, local)
-        Rack::Utils.secure_compare(token.to_s, decode_token(local).to_s)
+        ::Rack::Utils.secure_compare(token.to_s, decode_token(local).to_s)
       end
 
       def decode_token(token)
-        Base64.strict_decode64(token)
+        Base64.urlsafe_decode64(token)
       end
 
       def xor_byte_strings(s1, s2)

data/lib/sidekiq/web/helpers.rb

@@ -10,18 +10,25 @@ module Sidekiq
   module WebHelpers
     def strings(lang)
       @strings ||= {}
-      @strings[lang] ||= begin
-        # Allow sidekiq-web extensions to add locale paths
-        # so extensions can be localized
-        settings.locales.each_with_object({}) do |path, global|
-          find_locale_files(lang).each do |file|
-            strs = YAML.load(File.open(file))
-            global.merge!(strs[lang])
-          end
+
+      # Allow sidekiq-web extensions to add locale paths
+      # so extensions can be localized
+      @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
+        find_locale_files(lang).each do |file|
+          strs = YAML.safe_load(File.open(file))
+          global.merge!(strs[lang])
         end
       end
     end
 
+    def singularize(str, count)
+      if count == 1 && str.respond_to?(:singularize) # rails
+        str.singularize
+      else
+        str
+      end
+    end
+
     def clear_caches
       @strings = nil
       @locale_files = nil
@@ -63,17 +70,6 @@ module Sidekiq
       @head_html.join if defined?(@head_html)
     end
 
-    def poll_path
-      if current_path != "" && params["poll"]
-        path = root_path + current_path
-        query_string = to_query_string(params.slice(*params.keys - %w[page poll]))
-        path += "?#{query_string}" unless query_string.empty?
-        path
-      else
-        ""
-      end
-    end
-
     def text_direction
       get_locale["TextDirection"] || "ltr"
     end
@@ -118,7 +114,7 @@ module Sidekiq
     # within is used by Sidekiq Pro
     def display_tags(job, within = nil)
       job.tags.map { |tag|
-        "<span class='jobtag label label-info'>#{::Rack::Utils.escape_html(tag)}</span>"
+        "<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
       }.join(" ")
     end
 
@@ -144,22 +140,44 @@ module Sidekiq
       params[:direction] == "asc" ? "&uarr;" : "&darr;"
     end
 
-    def workers
-      @workers ||= Sidekiq::Workers.new
+    def workset
+      @work ||= Sidekiq::WorkSet.new
     end
 
     def processes
       @processes ||= Sidekiq::ProcessSet.new
     end
 
+    # Sorts processes by hostname following the natural sort order so that
+    # 'worker.1' < 'worker.2' < 'worker.10' < 'worker.20'
+    # '2.1.1.1' < '192.168.0.2' < '192.168.0.10'
+    def sorted_processes
+      @sorted_processes ||= begin
+        return processes unless processes.all? { |p| p["hostname"] }
+
+        split_characters = /[._-]+/
+
+        padding = processes.flat_map { |p| p["hostname"].split(split_characters) }.map(&:size).max
+
+        processes.to_a.sort_by do |process|
+          process["hostname"].split(split_characters).map do |substring|
+            # Left-pad the substring with '0' if it starts with a number or 'a'
+            # otherwise, so that '25' < 192' < 'a' ('025' < '192' < 'aaa')
+            padding_char = substring[0].match?(/\d/) ? "0" : "a"
+
+            substring.rjust(padding, padding_char)
+          end
+        end
+      end
+    end
+
     def stats
       @stats ||= Sidekiq::Stats.new
     end
 
     def redis_connection
       Sidekiq.redis do |conn|
-        c = conn.connection
-        "redis://#{c[:location]}/#{c[:db]}"
+        conn.connection[:id]
       end
     end
 
@@ -180,7 +198,7 @@ module Sidekiq
     end
 
     def current_status
-      workers.size == 0 ? "idle" : "active"
+      workset.size == 0 ? "idle" : "active"
     end
 
     def relative_time(time)
@@ -197,7 +215,7 @@ module Sidekiq
       [score.to_f, jid]
     end
 
-    SAFE_QPARAMS = %w[page poll direction]
+    SAFE_QPARAMS = %w[page direction]
 
     # Merge options with current params, filter safe params, and stringify to query string
     def qparams(options)
@@ -247,7 +265,7 @@ module Sidekiq
       queue class args retry_count retried_at failed_at
       jid error_message error_class backtrace
       error_backtrace enqueued_at retry wrapped
-      created_at tags
+      created_at tags display_class
     ])
 
     def retry_extra_items(retry_job)
@@ -258,7 +276,21 @@ module Sidekiq
       end
     end
 
+    def format_memory(rss_kb)
+      return "0" if rss_kb.nil? || rss_kb == 0
+
+      if rss_kb < 100_000
+        "#{number_with_delimiter(rss_kb)} KB"
+      elsif rss_kb < 10_000_000
+        "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
+      else
+        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
+      end
+    end
+
     def number_with_delimiter(number)
+      return "" if number.nil?
+
       begin
         Float(number)
       rescue ArgumentError, TypeError
@@ -292,7 +324,7 @@ module Sidekiq
     end
 
     def environment_title_prefix
-      environment = Sidekiq.options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      environment = Sidekiq[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
 
       "[#{environment.upcase}] " unless environment == "production"
     end

data/lib/sidekiq/web/router.rb

@@ -15,6 +15,10 @@ module Sidekiq
     REQUEST_METHOD = "REQUEST_METHOD"
     PATH_INFO = "PATH_INFO"
 
+    def head(path, &block)
+      route(HEAD, path, &block)
+    end
+
     def get(path, &block)
       route(GET, path, &block)
     end
@@ -39,7 +43,6 @@ module Sidekiq
       @routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
 
       @routes[method] << WebRoute.new(method, path, block)
-      @routes[HEAD] << WebRoute.new(method, path, block) if method == GET
     end
 
     def match(env)

data/lib/sidekiq/web.rb

@@ -13,10 +13,8 @@ require "sidekiq/web/application"
 require "sidekiq/web/csrf_protection"
 
 require "rack/content_length"
-
 require "rack/builder"
-require "rack/file"
-require "rack/session/cookie"
+require "rack/static"
 
 module Sidekiq
   class Web
@@ -35,19 +33,15 @@ module Sidekiq
       "Dead" => "morgue"
     }
 
+    if ENV["SIDEKIQ_METRICS_BETA"] == "1"
+      DEFAULT_TABS["Metrics"] = "metrics"
+    end
+
     class << self
       def settings
         self
       end
 
-      def middlewares
-        @middlewares ||= []
-      end
-
-      def use(*middleware_args, &block)
-        middlewares << [middleware_args, block]
-      end
-
       def default_tabs
         DEFAULT_TABS
       end
@@ -73,32 +67,45 @@ module Sidekiq
         opts.each { |key| set(key, false) }
       end
 
-      # Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
+      def middlewares
+        @middlewares ||= []
+      end
+
+      def use(*args, &block)
+        middlewares << [args, block]
+      end
+
       def set(attribute, value)
         send(:"#{attribute}=", value)
       end
 
-      attr_accessor :app_url, :session_secret, :redis_pool, :sessions
+      def sessions=(val)
+        puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+      end
+
+      def session_secret=(val)
+        puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
+      end
+
+      attr_accessor :app_url, :redis_pool
       attr_writer :locales, :views
     end
 
     def self.inherited(child)
       child.app_url = app_url
-      child.session_secret = session_secret
       child.redis_pool = redis_pool
-      child.sessions = sessions
     end
 
     def settings
       self.class.settings
     end
 
-    def use(*middleware_args, &block)
-      middlewares << [middleware_args, block]
+    def middlewares
+      @middlewares ||= self.class.middlewares
     end
 
-    def middlewares
-      @middlewares ||= Web.middlewares.dup
+    def use(*args, &block)
+      middlewares << [args, block]
     end
 
     def call(env)
@@ -126,18 +133,8 @@ module Sidekiq
       send(:"#{attribute}=", value)
     end
 
-    # Default values
-    set :sessions, true
-
-    attr_writer :sessions
-
-    def sessions
-      unless instance_variable_defined?("@sessions")
-        @sessions = self.class.sessions
-        @sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
-      end
-
-      @sessions
+    def sessions=(val)
+      puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
     end
 
     def self.register(extension)
@@ -146,57 +143,20 @@ module Sidekiq
 
     private
 
-    def using?(middleware)
-      middlewares.any? do |(m, _)|
-        m.is_a?(Array) && (m[0] == middleware || m[0].is_a?(middleware))
-      end
-    end
-
-    def build_sessions
-      middlewares = self.middlewares
-
-      s = sessions
-
-      # turn on CSRF protection if sessions are enabled and this is not the test env
-      if s && !using?(CsrfProtection) && ENV["RACK_ENV"] != "test"
-        middlewares.unshift [[CsrfProtection], nil]
-      end
-
-      if s && !using?(::Rack::Session::Cookie)
-        unless (secret = Web.session_secret)
-          require "securerandom"
-          secret = SecureRandom.hex(64)
-        end
-
-        options = {secret: secret}
-        options = options.merge(s.to_hash) if s.respond_to? :to_hash
-
-        middlewares.unshift [[::Rack::Session::Cookie, options], nil]
-      end
-
-      # Since Sidekiq::WebApplication no longer calculates its own
-      # Content-Length response header, we must ensure that the Rack middleware
-      # that does this is loaded
-      unless using? ::Rack::ContentLength
-        middlewares.unshift [[::Rack::ContentLength], nil]
-      end
-    end
-
     def build
-      build_sessions
-
-      middlewares = self.middlewares
       klass = self.class
+      m = middlewares
 
-      ::Rack::Builder.new do
-        %w[stylesheets javascripts images].each do |asset_dir|
-          map "/#{asset_dir}" do
-            run ::Rack::File.new("#{ASSETS}/#{asset_dir}", {"Cache-Control" => "public, max-age=86400"})
-          end
-        end
-
-        middlewares.each { |middleware, block| use(*middleware, &block) }
+      rules = []
+      rules = [[:all, {"cache-control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
+      ::Rack::Builder.new do
+        use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
+          root: ASSETS,
+          cascade: true,
+          header_rules: rules
+        m.each { |middleware, block| use(*middleware, &block) }
+        use Sidekiq::Web::CsrfProtection unless $TESTING
         run WebApplication.new(klass)
       end
     end