sidekiq 7.2.4 → 7.3.9

data/lib/sidekiq/web/application.rb

@@ -5,7 +5,7 @@ module Sidekiq
     extend WebRouter
 
     REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
-    CSP_HEADER = [
+    CSP_HEADER_TEMPLATE = [
       "default-src 'self' https: http:",
       "child-src 'self'",
       "connect-src 'self' https: http: wss: ws:",
@@ -15,8 +15,8 @@ module Sidekiq
       "manifest-src 'self'",
       "media-src 'self'",
       "object-src 'none'",
-      "script-src 'self' https: http:",
-      "style-src 'self' https: http: 'unsafe-inline'",
+      "script-src 'self' 'nonce-!placeholder!'",
+      "style-src 'self' https: http: 'unsafe-inline'", # TODO Nonce in 8.0
       "worker-src 'self'",
       "base-uri 'self'"
     ].join("; ").freeze
@@ -67,11 +67,15 @@ module Sidekiq
     end
 
     get "/metrics" do
+      x = params[:substr]
+      class_filter = (x.nil? || x == "") ? nil : Regexp.new(Regexp.escape(x), Regexp::IGNORECASE)
+
       q = Sidekiq::Metrics::Query.new
       @period = h((params[:period] || "")[0..1])
       @periods = METRICS_PERIODS
       minutes = @periods.fetch(@period, @periods.values.first)
-      @query_result = q.top_jobs(minutes: minutes)
+      @query_result = q.top_jobs(minutes: minutes, class_filter: class_filter)
+
       erb(:metrics)
     end
 
@@ -153,9 +157,15 @@ module Sidekiq
     end
 
     get "/morgue" do
-      @count = (params["count"] || 25).to_i
-      (@current_page, @total_size, @dead) = page("dead", params["page"], @count, reverse: true)
-      @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      x = params[:substr]
+
+      if x && x != ""
+        @dead = search(Sidekiq::DeadSet.new, x)
+      else
+        @count = (params["count"] || 25).to_i
+        (@current_page, @total_size, @dead) = page("dead", params["page"], @count, reverse: true)
+        @dead = @dead.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      end
 
       erb(:morgue)
     end
@@ -174,7 +184,7 @@ module Sidekiq
     end
 
     post "/morgue" do
-      redirect(request.path) unless params["key"]
+      redirect(request.path) unless url_params("key")
 
       params["key"].each do |key|
         job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
@@ -197,7 +207,7 @@ module Sidekiq
     end
 
     post "/morgue/:key" do
-      key = route_params[:key]
+      key = route_params(:key)
       halt(404) unless key
 
       job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
@@ -207,9 +217,15 @@ module Sidekiq
     end
 
     get "/retries" do
-      @count = (params["count"] || 25).to_i
-      (@current_page, @total_size, @retries) = page("retry", params["page"], @count)
-      @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      x = url_params("substr")
+
+      if x && x != ""
+        @retries = search(Sidekiq::RetrySet.new, x)
+      else
+        @count = (params["count"] || 25).to_i
+        (@current_page, @total_size, @retries) = page("retry", params["page"], @count)
+        @retries = @retries.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      end
 
       erb(:retries)
     end
@@ -262,9 +278,15 @@ module Sidekiq
     end
 
     get "/scheduled" do
-      @count = (params["count"] || 25).to_i
-      (@current_page, @total_size, @scheduled) = page("schedule", params["page"], @count)
-      @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      x = params[:substr]
+
+      if x && x != ""
+        @scheduled = search(Sidekiq::ScheduledSet.new, x)
+      else
+        @count = (params["count"] || 25).to_i
+        (@current_page, @total_size, @scheduled) = page("schedule", params["page"], @count)
+        @scheduled = @scheduled.map { |msg, score| Sidekiq::SortedEntry.new(nil, score, msg) }
+      end
 
       erb(:scheduled)
     end
@@ -328,72 +350,6 @@ module Sidekiq
       json Sidekiq::Stats.new.queues
     end
 
-    ########
-    # Filtering
-
-    get "/filter/metrics" do
-      redirect "#{root_path}metrics"
-    end
-
-    post "/filter/metrics" do
-      x = params[:substr]
-      q = Sidekiq::Metrics::Query.new
-      @period = h((params[:period] || "")[0..1])
-      @periods = METRICS_PERIODS
-      minutes = @periods.fetch(@period, @periods.values.first)
-      @query_result = q.top_jobs(minutes: minutes, class_filter: Regexp.new(Regexp.escape(x), Regexp::IGNORECASE))
-
-      erb :metrics
-    end
-
-    get "/filter/retries" do
-      x = params[:substr]
-      return redirect "#{root_path}retries" unless x && x != ""
-
-      @retries = search(Sidekiq::RetrySet.new, params[:substr])
-      erb :retries
-    end
-
-    post "/filter/retries" do
-      x = params[:substr]
-      return redirect "#{root_path}retries" unless x && x != ""
-
-      @retries = search(Sidekiq::RetrySet.new, params[:substr])
-      erb :retries
-    end
-
-    get "/filter/scheduled" do
-      x = params[:substr]
-      return redirect "#{root_path}scheduled" unless x && x != ""
-
-      @scheduled = search(Sidekiq::ScheduledSet.new, params[:substr])
-      erb :scheduled
-    end
-
-    post "/filter/scheduled" do
-      x = params[:substr]
-      return redirect "#{root_path}scheduled" unless x && x != ""
-
-      @scheduled = search(Sidekiq::ScheduledSet.new, params[:substr])
-      erb :scheduled
-    end
-
-    get "/filter/dead" do
-      x = params[:substr]
-      return redirect "#{root_path}morgue" unless x && x != ""
-
-      @dead = search(Sidekiq::DeadSet.new, params[:substr])
-      erb :morgue
-    end
-
-    post "/filter/dead" do
-      x = params[:substr]
-      return redirect "#{root_path}morgue" unless x && x != ""
-
-      @dead = search(Sidekiq::DeadSet.new, params[:substr])
-      erb :morgue
-    end
-
     post "/change_locale" do
       locale = params["locale"]
 
@@ -428,13 +384,18 @@ module Sidekiq
           Rack::CONTENT_TYPE => "text/html",
           Rack::CACHE_CONTROL => "private, no-store",
           Web::CONTENT_LANGUAGE => action.locale,
-          Web::CONTENT_SECURITY_POLICY => CSP_HEADER
+          Web::CONTENT_SECURITY_POLICY => process_csp(env, CSP_HEADER_TEMPLATE),
+          Web::X_CONTENT_TYPE_OPTIONS => "nosniff"
         }
         # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]
       end
     end
 
+    def process_csp(env, input)
+      input.gsub("!placeholder!", env[:csp_nonce])
+    end
+
     def self.helpers(mod = nil, &block)
       if block
         WebAction.class_eval(&block)

data/lib/sidekiq/web/helpers.rb

@@ -6,14 +6,57 @@ require "yaml"
 require "cgi"
 
 module Sidekiq
-  # This is not a public API
+  # These methods are available to pages within the Web UI and UI extensions.
+  # They are not public APIs for applications to use.
   module WebHelpers
+    def style_tag(location, **kwargs)
+      global = location.match?(/:\/\//)
+      location = root_path + location if !global && !location.start_with?(root_path)
+      attrs = {
+        type: "text/css",
+        media: "screen",
+        rel: "stylesheet",
+        nonce: csp_nonce,
+        href: location
+      }
+      html_tag(:link, attrs.merge(kwargs))
+    end
+
+    def script_tag(location, **kwargs)
+      global = location.match?(/:\/\//)
+      location = root_path + location if !global && !location.start_with?(root_path)
+      attrs = {
+        type: "text/javascript",
+        nonce: csp_nonce,
+        src: location
+      }
+      html_tag(:script, attrs.merge(kwargs)) {}
+    end
+
+    # NB: keys and values are not escaped; do not allow user input
+    # in the attributes
+    private def html_tag(tagname, attrs)
+      s = +"<#{tagname}"
+      attrs.each_pair do |k, v|
+        next unless v
+        s << " #{k}=\"#{v}\""
+      end
+      if block_given?
+        s << ">"
+        yield s
+        s << "</#{tagname}>"
+      else
+        s << " />"
+      end
+      s
+    end
+
     def strings(lang)
-      @strings ||= {}
+      @@strings ||= {}
 
       # Allow sidekiq-web extensions to add locale paths
       # so extensions can be localized
-      @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
+      @@strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
         find_locale_files(lang).each do |file|
           strs = YAML.safe_load(File.read(file))
           global.merge!(strs[lang])
@@ -34,19 +77,19 @@ module Sidekiq
     end
 
     def clear_caches
-      @strings = nil
-      @locale_files = nil
-      @available_locales = nil
+      @@strings = nil
+      @@locale_files = nil
+      @@available_locales = nil
     end
 
     def locale_files
-      @locale_files ||= settings.locales.flat_map { |path|
+      @@locale_files ||= settings.locales.flat_map { |path|
         Dir["#{path}/*.yml"]
       }
     end
 
     def available_locales
-      @available_locales ||= locale_files.map { |path| File.basename(path, ".yml") }.uniq
+      @@available_locales ||= Set.new(locale_files.map { |path| File.basename(path, ".yml") })
     end
 
     def find_locale_files(lang)
@@ -68,7 +111,7 @@ module Sidekiq
       if within.nil?
         ::Rack::Utils.escape_html(jid)
       else
-        "<a href='#{root_path}filter/#{within}?substr=#{jid}'>#{::Rack::Utils.escape_html(jid)}</a>"
+        "<a href='#{root_path}#{within}?substr=#{jid}'>#{::Rack::Utils.escape_html(jid)}</a>"
       end
     end
 
@@ -122,10 +165,9 @@ module Sidekiq
     # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
     def locale
       # session[:locale] is set via the locale selector from the footer
-      # defined?(session) && session are used to avoid exceptions when running tests
-      return session[:locale] if defined?(session) && session&.[](:locale)
-
-      @locale ||= begin
+      @locale ||= if (l = session&.fetch(:locale, nil)) && available_locales.include?(l)
+        l
+      else
         matched_locale = user_preferred_languages.map { |preferred|
           preferred_language = preferred.split("-", 2).first
 
@@ -142,7 +184,8 @@ module Sidekiq
 
     # sidekiq/sidekiq#3243
     def unfiltered?
-      yield unless env["PATH_INFO"].start_with?("/filter/")
+      s = url_params("substr")
+      yield unless s && s.size > 0
     end
 
     def get_locale
@@ -267,6 +310,10 @@ module Sidekiq
       "<input type='hidden' name='authenticity_token' value='#{env[:csrf_token]}'/>"
     end
 
+    def csp_nonce
+      env[:csp_nonce]
+    end
+
     def to_display(arg)
       arg.inspect
     rescue
@@ -310,7 +357,7 @@ module Sidekiq
     end
 
     def h(text)
-      ::Rack::Utils.escape_html(text)
+      ::Rack::Utils.escape_html(text.to_s)
     rescue ArgumentError => e
       raise unless e.message.eql?("invalid byte sequence in UTF-8")
       text.encode!("UTF-16", "UTF-8", invalid: :replace, replace: "").encode!("UTF-8", "UTF-16")

data/lib/sidekiq/web/router.rb

@@ -39,10 +39,13 @@ module Sidekiq
       route(DELETE, path, &block)
     end
 
-    def route(method, path, &block)
+    def route(*methods, path, &block)
       @routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
 
-      @routes[method] << WebRoute.new(method, path, block)
+      methods.each do |method|
+        method = method.to_s.upcase
+        @routes[method] << WebRoute.new(method, path, block)
+      end
     end
 
     def match(env)

data/lib/sidekiq/web.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "erb"
+require "securerandom"
 
 require "sidekiq"
 require "sidekiq/api"
@@ -39,14 +40,21 @@ module Sidekiq
       CONTENT_SECURITY_POLICY = "Content-Security-Policy"
       LOCATION = "Location"
       X_CASCADE = "X-Cascade"
+      X_CONTENT_TYPE_OPTIONS = "X-Content-Type-Options"
     else
       CONTENT_LANGUAGE = "content-language"
       CONTENT_SECURITY_POLICY = "content-security-policy"
       LOCATION = "location"
       X_CASCADE = "x-cascade"
+      X_CONTENT_TYPE_OPTIONS = "x-content-type-options"
     end
 
     class << self
+      # Forward compatibility with 8.0
+      def configure
+        yield self
+      end
+
       def settings
         self
       end
@@ -114,6 +122,7 @@ module Sidekiq
     end
 
     def call(env)
+      env[:csp_nonce] = SecureRandom.base64(16)
       app.call(env)
     end
 
@@ -138,7 +147,50 @@ module Sidekiq
       send(:"#{attribute}=", value)
     end
 
-    def self.register(extension)
+    # Register a class as a Sidekiq Web UI extension. The class should
+    # provide one or more tabs which map to an index route. Options:
+    #
+    # @param extension [Class] Class which contains the HTTP actions, required
+    # @param name [String] the name of the extension, used to namespace assets
+    # @param tab [String | Array] labels(s) of the UI tabs
+    # @param index [String | Array] index route(s) for each tab
+    # @param root_dir [String] directory location to find assets, locales and views, typically `web/` within the gemfile
+    # @param asset_paths [Array] one or more directories under {root}/assets/{name} to be publicly served, e.g. ["js", "css", "img"]
+    # @param cache_for [Integer] amount of time to cache assets, default one day
+    #
+    # TODO name, tab and index will be mandatory in 8.0
+    #
+    # Web extensions will have a root `web/` directory with `locales/`, `assets/`
+    # and `views/` subdirectories.
+    def self.register(extension, name: nil, tab: nil, index: nil, root_dir: nil, cache_for: 86400, asset_paths: nil)
+      tab = Array(tab)
+      index = Array(index)
+      tab.zip(index).each do |tab, index|
+        tabs[tab] = index
+      end
+      if root_dir
+        locdir = File.join(root_dir, "locales")
+        locales << locdir if File.directory?(locdir)
+
+        if asset_paths && name
+          # if you have {root}/assets/{name}/js/scripts.js
+          # and {root}/assets/{name}/css/styles.css
+          # you would pass in:
+          #   asset_paths: ["js", "css"]
+          # See script_tag and style_tag in web/helpers.rb
+          assdir = File.join(root_dir, "assets")
+          assurls = Array(asset_paths).map { |x| "/#{name}/#{x}" }
+          assetprops = {
+            urls: assurls,
+            root: assdir,
+            cascade: true
+          }
+          assetprops[:header_rules] = [[:all, {Rack::CACHE_CONTROL => "private, max-age=#{cache_for.to_i}"}]] if cache_for
+          middlewares << [[Rack::Static, assetprops], nil]
+        end
+      end
+
+      yield self if block_given?
       extension.registered(WebApplication)
     end
 
@@ -166,7 +218,7 @@ module Sidekiq
   Sidekiq::WebApplication.helpers WebHelpers
   Sidekiq::WebApplication.helpers Sidekiq::Paginator
 
-  Sidekiq::WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+  Sidekiq::WebAction.class_eval <<-RUBY, Web::LAYOUT, -1 # standard:disable Style/EvalWithLocation
     def _render
       #{ERB.new(File.read(Web::LAYOUT)).src}
     end

data/lib/sidekiq.rb

@@ -32,6 +32,7 @@ require "sidekiq/logger"
 require "sidekiq/client"
 require "sidekiq/transaction_aware_client"
 require "sidekiq/job"
+require "sidekiq/iterable_job"
 require "sidekiq/worker_compatibility_alias"
 require "sidekiq/redis_client_adapter"
 
@@ -101,18 +102,19 @@ module Sidekiq
   def self.freeze!
     @frozen = true
     @config_blocks = nil
+    default_configuration.freeze!
   end
 
   # Creates a Sidekiq::Config instance that is more tuned for embedding
   # within an arbitrary Ruby process. Notably it reduces concurrency by
   # default so there is less contention for CPU time with other threads.
   #
-  #   inst = Sidekiq.configure_embed do |config|
+  #   instance = Sidekiq.configure_embed do |config|
   #     config.queues = %w[critical default low]
   #   end
-  #   inst.run
+  #   instance.run
   #   sleep 10
-  #   inst.terminate
+  #   instance.stop
   #
   # NB: it is really easy to overload a Ruby process with threads due to the GIL.
   # I do not recommend setting concurrency higher than 2-3.

data/sidekiq.gemspec

@@ -23,8 +23,9 @@ Gem::Specification.new do |gem|
     "rubygems_mfa_required" => "true"
   }
 
-  gem.add_dependency "redis-client", ">= 0.19.0"
+  gem.add_dependency "redis-client", ">= 0.22.2"
   gem.add_dependency "connection_pool", ">= 2.3.0"
   gem.add_dependency "rack", ">= 2.2.4"
-  gem.add_dependency "concurrent-ruby", "< 2"
+  gem.add_dependency "logger"
+  gem.add_dependency "base64"
 end

data/web/assets/javascripts/application.js

@@ -34,6 +34,7 @@ function addListeners() {
   addShiftClickListeners()
   updateFuzzyTimes();
   updateNumbers();
+  updateProgressBars();
   setLivePollFromUrl();
 
   var buttons = document.querySelectorAll(".live-poll");
@@ -180,4 +181,8 @@ function showError(error) {
 
 function updateLocale(event) {
   event.target.form.submit();
-};
+}
+
+function updateProgressBars() {
+  document.querySelectorAll('.progress-bar').forEach(bar => { bar.style.width = bar.dataset.width + "%"})
+}

data/web/assets/javascripts/dashboard-charts.js

@@ -83,6 +83,8 @@ class RealtimeChart extends DashboardChart {
     this.chart.data.datasets[1].data.push(failed);
     this.chart.update();
 
+    updateScreenReaderDashboardValues(processed, failed);
+
     updateStatsSummary(this.stats.sidekiq);
     updateRedisStats(this.stats.redis);
     updateFooterUTCTime(this.stats.server_utc_time);
@@ -108,17 +110,27 @@ class RealtimeChart extends DashboardChart {
   }
 
   renderLegend(dp) {
-    this.legend.innerHTML = `
-      <span>
-        <span class="swatch" style="background-color: ${dp[0].dataset.borderColor};"></span>
-        <span>${dp[0].dataset.label}: ${dp[0].formattedValue}</span>
-      </span>
-      <span>
-        <span class="swatch" style="background-color: ${dp[1].dataset.borderColor};"></span>
-        <span>${dp[1].dataset.label}: ${dp[1].formattedValue}</span>
-      </span>
-      <span class="time">${dp[0].label}</span>
-    `;
+    const entry1 = this.legendEntry(dp[0]);
+    const entry2 = this.legendEntry(dp[1]);
+    const time = document.createElement("span");
+    time.classList.add("time");
+    time.innerText = dp[0].label;
+
+    this.legend.replaceChildren(entry1, entry2, time)
+  }
+
+  legendEntry(dp) {
+    const wrapper = document.createElement("span");
+
+    const swatch = document.createElement("span");
+    swatch.classList.add("swatch");
+    swatch.style.backgroundColor = dp.dataset.borderColor;
+    wrapper.appendChild(swatch)
+    
+    const label = document.createElement("span");
+    label.innerText = `${dp.dataset.label}: ${dp.formattedValue}`;
+    wrapper.appendChild(label)
+    return wrapper;
   }
 
   renderCursor(dp) {
@@ -179,4 +191,4 @@ class RealtimeChart extends DashboardChart {
   if (hc != null) {
     var htc = new DashboardChart(hc, JSON.parse(hc.textContent))
     window.historyChart = htc
-  }
+  }

data/web/assets/javascripts/dashboard.js

@@ -28,7 +28,7 @@ var pulseBeacon = function() {
 }
 
 var setSliderLabel = function(val) {
-  document.getElementById('sldr-text').innerText = Math.round(parseFloat(val) / 1000) + ' sec';
+  document.getElementById('sldr-text').innerText = Math.round(parseFloat(val) / 1000) + ' s';
 }
 
 var ready = (callback) => {
@@ -36,6 +36,12 @@ var ready = (callback) => {
   else document.addEventListener("DOMContentLoaded", callback);
 }
 
+var updateScreenReaderDashboardValues = function(processed, failed) {
+  let lastDashboardUpdateSpan = document.getElementById("sr-last-dashboard-update");
+  var updateText = document.getElementById("sr-last-dashboard-update-template").innerText;
+  lastDashboardUpdateSpan.innerText = updateText.replace("PROCESSED_COUNT", processed).replace("FAILED_COUNT", failed);
+}
+
 ready(() => {
   var sldr = document.getElementById('sldr');
   if (typeof localStorage.sidekiqTimeInterval !== 'undefined') {

data/web/assets/stylesheets/application.css

@@ -72,6 +72,14 @@ h1, h2, h3 {
   line-height: 45px;
 }
 
+.progress {
+  margin-bottom: 0;
+}
+
+.w-50 {
+  width: 50%;
+}
+
 .header-container, .header-container .page-title-container {
   display: flex;
   justify-content: space-between;
@@ -640,18 +648,23 @@ div.interval-slider input {
 
 @media (min-width: 768px) {
   .redis-url {
-    max-width: 250px;
+    max-width: 160px;
+  }
+
+  .navbar-fixed-bottom .nav {
+    margin-left: -15px;
+    margin-right: -15px;
   }
 }
 
 @media (min-width: 992px) {
   .redis-url {
-    max-width: 490px;
+    max-width: 380px;
   }
 }
 @media (min-width: 1200px) {
   .redis-url {
-    max-width: 600px;
+    max-width: 580px;
   }
 }
 

data/web/locales/en.yml

@@ -34,6 +34,8 @@ en:
   Jobs: Jobs
   Kill: Kill
   KillAll: Kill All
+  Language: Language
+  LastDashboardUpdateTemplateLiteral: "Latest poll: Processed: PROCESSED_COUNT. Failed: FAILED_COUNT."
   LastRetry: Last Retry
   Latency: Latency
   LivePoll: Live Poll
@@ -53,6 +55,7 @@ en:
   PeakMemoryUsage: Peak Memory Usage
   Plugins: Plugins
   PollingInterval: Polling interval
+  PollingIntervalMilliseconds: Polling interval milliseconds
   Process: Process
   Processed: Processed
   Processes: Processes
@@ -95,7 +98,6 @@ en:
   TotalExecutionTime: Total Execution Time
   AvgExecutionTime: Average Execution Time
   Context: Context
-  Bucket: Bucket
   NoJobMetricsFound: No recent job metrics were found
   Filter: Filter
   AnyJobContent: Any job content

data/web/locales/fr.yml

@@ -95,5 +95,4 @@ fr:
   TotalExecutionTime: Temps d'exécution total
   AvgExecutionTime: Temps d'exécution moyen
   Context: Contexte
-  Bucket: Bucket
   NoJobMetricsFound: Aucune statistique de tâche récente n'a été trouvée

data/web/locales/gd.yml

@@ -95,5 +95,4 @@ gd:
   TotalExecutionTime: Ùine iomlan nan gnìomhan
   AvgExecutionTime: Ùine cuibheasach nan gnìomhan
   Context: Co-theacsa
-  Bucket: Bucaid
   NoJobMetricsFound: Cha deach meatraigeachd o chionn goirid air obair a lorg