sidekiq 7.0.9 → 7.3.10

data/lib/sidekiq/web/helpers.rb

@@ -6,14 +6,57 @@ require "yaml"
 require "cgi"
 
 module Sidekiq
-  # This is not a public API
+  # These methods are available to pages within the Web UI and UI extensions.
+  # They are not public APIs for applications to use.
   module WebHelpers
+    def style_tag(location, **kwargs)
+      global = location.match?(/:\/\//)
+      location = root_path + location if !global && !location.start_with?(root_path)
+      attrs = {
+        type: "text/css",
+        media: "screen",
+        rel: "stylesheet",
+        nonce: csp_nonce,
+        href: location
+      }
+      html_tag(:link, attrs.merge(kwargs))
+    end
+
+    def script_tag(location, **kwargs)
+      global = location.match?(/:\/\//)
+      location = root_path + location if !global && !location.start_with?(root_path)
+      attrs = {
+        type: "text/javascript",
+        nonce: csp_nonce,
+        src: location
+      }
+      html_tag(:script, attrs.merge(kwargs)) {}
+    end
+
+    # NB: keys and values are not escaped; do not allow user input
+    # in the attributes
+    private def html_tag(tagname, attrs)
+      s = +"<#{tagname}"
+      attrs.each_pair do |k, v|
+        next unless v
+        s << " #{k}=\"#{v}\""
+      end
+      if block_given?
+        s << ">"
+        yield s
+        s << "</#{tagname}>"
+      else
+        s << " />"
+      end
+      s
+    end
+
     def strings(lang)
-      @strings ||= {}
+      @@strings ||= {}
 
       # Allow sidekiq-web extensions to add locale paths
       # so extensions can be localized
-      @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
+      @@strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
         find_locale_files(lang).each do |file|
           strs = YAML.safe_load(File.read(file))
           global.merge!(strs[lang])
@@ -21,6 +64,10 @@ module Sidekiq
       end
     end
 
+    def to_json(x)
+      Sidekiq.dump_json(x)
+    end
+
     def singularize(str, count)
       if count == 1 && str.respond_to?(:singularize) # rails
         str.singularize
@@ -30,27 +77,48 @@ module Sidekiq
     end
 
     def clear_caches
-      @strings = nil
-      @locale_files = nil
-      @available_locales = nil
+      @@strings = nil
+      @@locale_files = nil
+      @@available_locales = nil
     end
 
     def locale_files
-      @locale_files ||= settings.locales.flat_map { |path|
+      @@locale_files ||= settings.locales.flat_map { |path|
         Dir["#{path}/*.yml"]
       }
     end
 
     def available_locales
-      @available_locales ||= locale_files.map { |path| File.basename(path, ".yml") }.uniq
+      @@available_locales ||= Set.new(locale_files.map { |path| File.basename(path, ".yml") })
     end
 
     def find_locale_files(lang)
       locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
     end
 
-    # This is a hook for a Sidekiq Pro feature.  Please don't touch.
-    def filtering(*)
+    def search(jobset, substr)
+      resultset = jobset.scan(substr).to_a
+      @current_page = 1
+      @count = @total_size = resultset.size
+      resultset
+    end
+
+    def filtering(which)
+      erb(:filtering, locals: {which: which})
+    end
+
+    def filter_link(jid, within = "retries")
+      if within.nil?
+        ::Rack::Utils.escape_html(jid)
+      else
+        "<a href='#{root_path}#{within}?substr=#{jid}'>#{::Rack::Utils.escape_html(jid)}</a>"
+      end
+    end
+
+    def display_tags(job, within = "retries")
+      job.tags.map { |tag|
+        "<span class='label label-info jobtag'>#{filter_link(tag, within)}</span>"
+      }.join(" ")
     end
 
     # This view helper provide ability display you html code in
@@ -96,7 +164,10 @@ module Sidekiq
     #
     # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
     def locale
-      @locale ||= begin
+      # session[:locale] is set via the locale selector from the footer
+      @locale ||= if (l = session&.fetch(:locale, nil)) && available_locales.include?(l)
+        l
+      else
         matched_locale = user_preferred_languages.map { |preferred|
           preferred_language = preferred.split("-", 2).first
 
@@ -111,16 +182,10 @@ module Sidekiq
       end
     end
 
-    # within is used by Sidekiq Pro
-    def display_tags(job, within = nil)
-      job.tags.map { |tag|
-        "<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
-      }.join(" ")
-    end
-
     # sidekiq/sidekiq#3243
     def unfiltered?
-      yield unless env["PATH_INFO"].start_with?("/filter/")
+      s = url_params("substr")
+      yield unless s && s.size > 0
     end
 
     def get_locale
@@ -245,6 +310,10 @@ module Sidekiq
       "<input type='hidden' name='authenticity_token' value='#{env[:csrf_token]}'/>"
     end
 
+    def csp_nonce
+      env[:csp_nonce]
+    end
+
     def to_display(arg)
       arg.inspect
     rescue
@@ -278,27 +347,17 @@ module Sidekiq
       elsif rss_kb < 10_000_000
         "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
       else
-        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
+        "#{number_with_delimiter(rss_kb / (1024.0 * 1024.0), precision: 1)} GB"
       end
     end
 
-    def number_with_delimiter(number)
-      return "" if number.nil?
-
-      begin
-        Float(number)
-      rescue ArgumentError, TypeError
-        return number
-      end
-
-      options = {delimiter: ",", separator: "."}
-      parts = number.to_s.to_str.split(".")
-      parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{options[:delimiter]}")
-      parts.join(options[:separator])
+    def number_with_delimiter(number, options = {})
+      precision = options[:precision] || 0
+      %(<span data-nwp="#{precision}">#{number.round(precision)}</span>)
     end
 
     def h(text)
-      ::Rack::Utils.escape_html(text)
+      ::Rack::Utils.escape_html(text.to_s)
     rescue ArgumentError => e
       raise unless e.message.eql?("invalid byte sequence in UTF-8")
       text.encode!("UTF-16", "UTF-8", invalid: :replace, replace: "").encode!("UTF-8", "UTF-16")
@@ -332,7 +391,8 @@ module Sidekiq
     end
 
     def pollable?
-      !(current_path == "" || current_path.start_with?("metrics"))
+      # there's no point to refreshing the metrics pages every N seconds
+      !(current_path == "" || current_path.index("metrics"))
     end
 
     def retry_or_delete_or_kill(job, params)

data/lib/sidekiq/web/router.rb

@@ -39,10 +39,13 @@ module Sidekiq
       route(DELETE, path, &block)
     end
 
-    def route(method, path, &block)
+    def route(*methods, path, &block)
       @routes ||= {GET => [], POST => [], PUT => [], PATCH => [], DELETE => [], HEAD => []}
 
-      @routes[method] << WebRoute.new(method, path, block)
+      methods.each do |method|
+        method = method.to_s.upcase
+        @routes[method] << WebRoute.new(method, path, block)
+      end
     end
 
     def match(env)

data/lib/sidekiq/web.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "erb"
+require "securerandom"
 
 require "sidekiq"
 require "sidekiq/api"
@@ -34,7 +35,26 @@ module Sidekiq
       "Metrics" => "metrics"
     }
 
+    if Gem::Version.new(Rack::RELEASE) < Gem::Version.new("3")
+      CONTENT_LANGUAGE = "Content-Language"
+      CONTENT_SECURITY_POLICY = "Content-Security-Policy"
+      LOCATION = "Location"
+      X_CASCADE = "X-Cascade"
+      X_CONTENT_TYPE_OPTIONS = "X-Content-Type-Options"
+    else
+      CONTENT_LANGUAGE = "content-language"
+      CONTENT_SECURITY_POLICY = "content-security-policy"
+      LOCATION = "location"
+      X_CASCADE = "x-cascade"
+      X_CONTENT_TYPE_OPTIONS = "x-content-type-options"
+    end
+
     class << self
+      # Forward compatibility with 8.0
+      def configure
+        yield self
+      end
+
       def settings
         self
       end
@@ -102,6 +122,7 @@ module Sidekiq
     end
 
     def call(env)
+      env[:csp_nonce] = SecureRandom.base64(16)
       app.call(env)
     end
 
@@ -126,7 +147,50 @@ module Sidekiq
       send(:"#{attribute}=", value)
     end
 
-    def self.register(extension)
+    # Register a class as a Sidekiq Web UI extension. The class should
+    # provide one or more tabs which map to an index route. Options:
+    #
+    # @param extension [Class] Class which contains the HTTP actions, required
+    # @param name [String] the name of the extension, used to namespace assets
+    # @param tab [String | Array] labels(s) of the UI tabs
+    # @param index [String | Array] index route(s) for each tab
+    # @param root_dir [String] directory location to find assets, locales and views, typically `web/` within the gemfile
+    # @param asset_paths [Array] one or more directories under {root}/assets/{name} to be publicly served, e.g. ["js", "css", "img"]
+    # @param cache_for [Integer] amount of time to cache assets, default one day
+    #
+    # TODO name, tab and index will be mandatory in 8.0
+    #
+    # Web extensions will have a root `web/` directory with `locales/`, `assets/`
+    # and `views/` subdirectories.
+    def self.register(extension, name: nil, tab: nil, index: nil, root_dir: nil, cache_for: 86400, asset_paths: nil)
+      tab = Array(tab)
+      index = Array(index)
+      tab.zip(index).each do |tab, index|
+        tabs[tab] = index
+      end
+      if root_dir
+        locdir = File.join(root_dir, "locales")
+        locales << locdir if File.directory?(locdir)
+
+        if asset_paths && name
+          # if you have {root}/assets/{name}/js/scripts.js
+          # and {root}/assets/{name}/css/styles.css
+          # you would pass in:
+          #   asset_paths: ["js", "css"]
+          # See script_tag and style_tag in web/helpers.rb
+          assdir = File.join(root_dir, "assets")
+          assurls = Array(asset_paths).map { |x| "/#{name}/#{x}" }
+          assetprops = {
+            urls: assurls,
+            root: assdir,
+            cascade: true
+          }
+          assetprops[:header_rules] = [[:all, {Rack::CACHE_CONTROL => "private, max-age=#{cache_for.to_i}"}]] if cache_for
+          middlewares << [[Rack::Static, assetprops], nil]
+        end
+      end
+
+      yield self if block_given?
       extension.registered(WebApplication)
     end
 
@@ -137,7 +201,7 @@ module Sidekiq
       m = middlewares
 
       rules = []
-      rules = [[:all, {"cache-control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
+      rules = [[:all, {Rack::CACHE_CONTROL => "private, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
       ::Rack::Builder.new do
         use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
@@ -154,7 +218,7 @@ module Sidekiq
   Sidekiq::WebApplication.helpers WebHelpers
   Sidekiq::WebApplication.helpers Sidekiq::Paginator
 
-  Sidekiq::WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
+  Sidekiq::WebAction.class_eval <<-RUBY, Web::LAYOUT, -1 # standard:disable Style/EvalWithLocation
     def _render
       #{ERB.new(File.read(Web::LAYOUT)).src}
     end

data/lib/sidekiq.rb

@@ -32,6 +32,7 @@ require "sidekiq/logger"
 require "sidekiq/client"
 require "sidekiq/transaction_aware_client"
 require "sidekiq/job"
+require "sidekiq/iterable_job"
 require "sidekiq/worker_compatibility_alias"
 require "sidekiq/redis_client_adapter"
 
@@ -101,18 +102,19 @@ module Sidekiq
   def self.freeze!
     @frozen = true
     @config_blocks = nil
+    default_configuration.freeze!
   end
 
   # Creates a Sidekiq::Config instance that is more tuned for embedding
   # within an arbitrary Ruby process. Notably it reduces concurrency by
   # default so there is less contention for CPU time with other threads.
   #
-  #   inst = Sidekiq.configure_embed do |config|
+  #   instance = Sidekiq.configure_embed do |config|
   #     config.queues = %w[critical default low]
   #   end
-  #   inst.run
+  #   instance.run
   #   sleep 10
-  #   inst.terminate
+  #   instance.stop
   #
   # NB: it is really easy to overload a Ruby process with threads due to the GIL.
   # I do not recommend setting concurrency higher than 2-3.

data/sidekiq.gemspec

@@ -23,17 +23,9 @@ Gem::Specification.new do |gem|
     "rubygems_mfa_required" => "true"
   }
 
-  gem.add_dependency "redis-client", ">= 0.11.0"
-  gem.add_dependency "connection_pool", ">= 2.3.0"
-  gem.add_dependency "rack", ">= 2.2.4"
-  gem.add_dependency "concurrent-ruby", "< 2"
-  gem.post_install_message = <<~EOM
-    
-    Welcome to Sidekiq 7.0!
-    
-    1. Use `gem 'sidekiq', '<7'` in your Gemfile if you don't want this new version.
-    2. Read the release notes at https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md
-    3. If you have problems, search for open/closed issues at https://github.com/sidekiq/sidekiq/issues/
-
-  EOM
+  gem.add_dependency "redis-client", ">= 0.23.0", "<1"
+  gem.add_dependency "connection_pool", ">= 2.3.0", "<3"
+  gem.add_dependency "rack", ">= 2.2.4", "<3.3"
+  gem.add_dependency "logger"
+  gem.add_dependency "base64"
 end

data/web/assets/javascripts/application.js

@@ -33,6 +33,8 @@ function addListeners() {
 
   addShiftClickListeners()
   updateFuzzyTimes();
+  updateNumbers();
+  updateProgressBars();
   setLivePollFromUrl();
 
   var buttons = document.querySelectorAll(".live-poll");
@@ -46,6 +48,8 @@ function addListeners() {
       scheduleLivePoll();
     }
   }
+
+  document.getElementById("locale-select").addEventListener("change", updateLocale);
 }
 
 function addPollingListeners(_event)  {
@@ -102,6 +106,20 @@ function updateFuzzyTimes() {
   t.cancel();
 }
 
+function updateNumbers() {
+  document.querySelectorAll("[data-nwp]").forEach(node => {
+    let number = parseFloat(node.textContent);
+    let precision = parseInt(node.dataset["nwp"] || 0);
+    if (typeof number === "number") {
+      let formatted = number.toLocaleString(undefined, {
+        minimumFractionDigits: precision,
+        maximumFractionDigits: precision,
+      });
+      node.textContent = formatted;
+    }
+  });
+}
+
 function setLivePollFromUrl() {
   var url_params = new URL(window.location.href).searchParams
 
@@ -140,6 +158,7 @@ function checkResponse(resp) {
 
 function scheduleLivePoll() {
   let ti = parseInt(localStorage.sidekiqTimeInterval) || 5000;
+  if (ti < 2000) { ti = 2000 }
   livePollTimer = setTimeout(livePollCallback, ti);
 }
 
@@ -159,3 +178,11 @@ function replacePage(text) {
 function showError(error) {
   console.error(error)
 }
+
+function updateLocale(event) {
+  event.target.form.submit();
+}
+
+function updateProgressBars() {
+  document.querySelectorAll('.progress-bar').forEach(bar => { bar.style.width = bar.dataset.width + "%"})
+}

data/web/assets/javascripts/dashboard-charts.js

@@ -57,7 +57,9 @@ class DashboardChart extends BaseChart {
 class RealtimeChart extends DashboardChart {
   constructor(el, options) {
     super(el, options);
-    this.delay = parseInt(localStorage.sidekiqTimeInterval) || 5000;
+    let d = parseInt(localStorage.sidekiqTimeInterval) || 5000;
+    if (d < 2000) { d = 2000; }
+    this.delay = d
     this.startPolling();
     document.addEventListener("interval:update", this.handleUpdate.bind(this));
   }
@@ -81,9 +83,12 @@ class RealtimeChart extends DashboardChart {
     this.chart.data.datasets[1].data.push(failed);
     this.chart.update();
 
+    updateScreenReaderDashboardValues(processed, failed);
+
     updateStatsSummary(this.stats.sidekiq);
     updateRedisStats(this.stats.redis);
     updateFooterUTCTime(this.stats.server_utc_time);
+    updateNumbers();
     pulseBeacon();
 
     this.stats = stats;
@@ -105,17 +110,27 @@ class RealtimeChart extends DashboardChart {
   }
 
   renderLegend(dp) {
-    this.legend.innerHTML = `
-      <span>
-        <span class="swatch" style="background-color: ${dp[0].dataset.borderColor};"></span>
-        <span>${dp[0].dataset.label}: ${dp[0].formattedValue}</span>
-      </span>
-      <span>
-        <span class="swatch" style="background-color: ${dp[1].dataset.borderColor};"></span>
-        <span>${dp[1].dataset.label}: ${dp[1].formattedValue}</span>
-      </span>
-      <span class="time">${dp[0].label}</span>
-    `;
+    const entry1 = this.legendEntry(dp[0]);
+    const entry2 = this.legendEntry(dp[1]);
+    const time = document.createElement("span");
+    time.classList.add("time");
+    time.innerText = dp[0].label;
+
+    this.legend.replaceChildren(entry1, entry2, time)
+  }
+
+  legendEntry(dp) {
+    const wrapper = document.createElement("span");
+
+    const swatch = document.createElement("span");
+    swatch.classList.add("swatch");
+    swatch.style.backgroundColor = dp.dataset.borderColor;
+    wrapper.appendChild(swatch)
+    
+    const label = document.createElement("span");
+    label.innerText = `${dp.dataset.label}: ${dp.formattedValue}`;
+    wrapper.appendChild(label)
+    return wrapper;
   }
 
   renderCursor(dp) {
@@ -164,3 +179,16 @@ class RealtimeChart extends DashboardChart {
     };
   }
 }
+
+  var rc = document.getElementById("realtime-chart")
+  if (rc != null) {
+    var rtc = new RealtimeChart(rc, JSON.parse(rc.textContent))
+    rtc.registerLegend(document.getElementById("realtime-legend"))
+    window.realtimeChart = rtc
+  }
+
+  var hc = document.getElementById("history-chart")
+  if (hc != null) {
+    var htc = new DashboardChart(hc, JSON.parse(hc.textContent))
+    window.historyChart = htc
+  }

data/web/assets/javascripts/dashboard.js

@@ -1,15 +1,13 @@
 Sidekiq = {};
 
-var nf = new Intl.NumberFormat();
-
 var updateStatsSummary = function(data) {
-  document.getElementById("txtProcessed").innerText = nf.format(data.processed);
-  document.getElementById("txtFailed").innerText = nf.format(data.failed);
-  document.getElementById("txtBusy").innerText = nf.format(data.busy);
-  document.getElementById("txtScheduled").innerText = nf.format(data.scheduled);
-  document.getElementById("txtRetries").innerText = nf.format(data.retries);
-  document.getElementById("txtEnqueued").innerText = nf.format(data.enqueued);
-  document.getElementById("txtDead").innerText = nf.format(data.dead);
+  document.getElementById("txtProcessed").innerText = data.processed;
+  document.getElementById("txtFailed").innerText = data.failed;
+  document.getElementById("txtBusy").innerText = data.busy;
+  document.getElementById("txtScheduled").innerText = data.scheduled;
+  document.getElementById("txtRetries").innerText = data.retries;
+  document.getElementById("txtEnqueued").innerText = data.enqueued;
+  document.getElementById("txtDead").innerText = data.dead;
 }
 
 var updateRedisStats = function(data) {
@@ -30,7 +28,7 @@ var pulseBeacon = function() {
 }
 
 var setSliderLabel = function(val) {
-  document.getElementById('sldr-text').innerText = Math.round(parseFloat(val) / 1000) + ' sec';
+  document.getElementById('sldr-text').innerText = Math.round(parseFloat(val) / 1000) + ' s';
 }
 
 var ready = (callback) => {
@@ -38,6 +36,12 @@ var ready = (callback) => {
   else document.addEventListener("DOMContentLoaded", callback);
 }
 
+var updateScreenReaderDashboardValues = function(processed, failed) {
+  let lastDashboardUpdateSpan = document.getElementById("sr-last-dashboard-update");
+  var updateText = document.getElementById("sr-last-dashboard-update-template").innerText;
+  lastDashboardUpdateSpan.innerText = updateText.replace("PROCESSED_COUNT", processed).replace("FAILED_COUNT", failed);
+}
+
 ready(() => {
   var sldr = document.getElementById('sldr');
   if (typeof localStorage.sidekiqTimeInterval !== 'undefined') {

data/web/assets/javascripts/metrics.js

@@ -262,3 +262,37 @@ class HistBubbleChart extends BaseChart {
     };
   }
 }
+
+var ch = document.getElementById("job-metrics-overview-chart");
+if (ch != null) {
+  var jm = new JobMetricsOverviewChart(ch, JSON.parse(ch.textContent));
+  document.querySelectorAll(".metrics-swatch-wrapper > input[type=checkbox]").forEach((imp) => {
+    jm.registerSwatch(imp.id)
+  });
+  window.jobMetricsChart = jm;
+}
+
+var htc = document.getElementById("hist-totals-chart");
+if (htc != null) {
+  var tc = new HistTotalsChart(htc, JSON.parse(htc.textContent));
+  window.histTotalsChart = tc
+}
+
+var hbc = document.getElementById("hist-bubble-chart");
+if (hbc != null) {
+  var bc = new HistBubbleChart(hbc, JSON.parse(hbc.textContent));
+  window.histBubbleChart = bc
+}
+
+var form = document.getElementById("metrics-form")
+document.querySelectorAll("#period-selector").forEach(node => {
+  node.addEventListener("input", debounce(() => form.submit()))
+})
+
+function debounce(func, timeout = 300) {
+  let timer;
+  return (...args) => {
+    clearTimeout(timer);
+    timer = setTimeout(() => { func.apply(this, args); }, timeout);
+  };
+}

data/web/assets/stylesheets/application-rtl.css

@@ -151,3 +151,13 @@ div.interval-slider {
     padding-left: 5px;
   }
 }
+
+#locale-select {
+  float: right;
+}
+
+@media (max-width: 767px) {
+  #locale-select {
+    float: none;
+  }
+}