sidekiq 7.0.0 → 7.3.0

data/lib/sidekiq/web/helpers.rb

@@ -6,8 +6,51 @@ require "yaml"
 require "cgi"
 
 module Sidekiq
-  # This is not a public API
+  # These methods are available to pages within the Web UI and UI extensions.
+  # They are not public APIs for applications to use.
   module WebHelpers
+    def style_tag(location, **kwargs)
+      global = location.match?(/:\/\//)
+      location = root_path + location if !global && !location.start_with?(root_path)
+      attrs = {
+        type: "text/css",
+        media: "screen",
+        rel: "stylesheet",
+        nonce: csp_nonce,
+        href: location
+      }
+      html_tag(:link, attrs.merge(kwargs))
+    end
+
+    def script_tag(location, **kwargs)
+      global = location.match?(/:\/\//)
+      location = root_path + location if !global && !location.start_with?(root_path)
+      attrs = {
+        type: "text/javascript",
+        nonce: csp_nonce,
+        src: location
+      }
+      html_tag(:script, attrs.merge(kwargs)) {}
+    end
+
+    # NB: keys and values are not escaped; do not allow user input
+    # in the attributes
+    private def html_tag(tagname, attrs)
+      s = "<#{tagname}"
+      attrs.each_pair do |k, v|
+        next unless v
+        s << " #{k}=\"#{v}\""
+      end
+      if block_given?
+        s << ">"
+        yield s
+        s << "</#{tagname}>"
+      else
+        s << " />"
+      end
+      s
+    end
+
     def strings(lang)
       @strings ||= {}
 
@@ -15,12 +58,16 @@ module Sidekiq
       # so extensions can be localized
       @strings[lang] ||= settings.locales.each_with_object({}) do |path, global|
         find_locale_files(lang).each do |file|
-          strs = YAML.safe_load(File.open(file))
+          strs = YAML.safe_load(File.read(file))
           global.merge!(strs[lang])
         end
       end
     end
 
+    def to_json(x)
+      Sidekiq.dump_json(x)
+    end
+
     def singularize(str, count)
       if count == 1 && str.respond_to?(:singularize) # rails
         str.singularize
@@ -42,15 +89,36 @@ module Sidekiq
     end
 
     def available_locales
-      @available_locales ||= locale_files.map { |path| File.basename(path, ".yml") }.uniq
+      @available_locales ||= Set.new(locale_files.map { |path| File.basename(path, ".yml") })
     end
 
     def find_locale_files(lang)
       locale_files.select { |file| file =~ /\/#{lang}\.yml$/ }
     end
 
-    # This is a hook for a Sidekiq Pro feature.  Please don't touch.
-    def filtering(*)
+    def search(jobset, substr)
+      resultset = jobset.scan(substr).to_a
+      @current_page = 1
+      @count = @total_size = resultset.size
+      resultset
+    end
+
+    def filtering(which)
+      erb(:filtering, locals: {which: which})
+    end
+
+    def filter_link(jid, within = "retries")
+      if within.nil?
+        ::Rack::Utils.escape_html(jid)
+      else
+        "<a href='#{root_path}filter/#{within}?substr=#{jid}'>#{::Rack::Utils.escape_html(jid)}</a>"
+      end
+    end
+
+    def display_tags(job, within = "retries")
+      job.tags.map { |tag|
+        "<span class='label label-info jobtag'>#{filter_link(tag, within)}</span>"
+      }.join(" ")
     end
 
     # This view helper provide ability display you html code in
@@ -96,7 +164,10 @@ module Sidekiq
     #
     # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
     def locale
-      @locale ||= begin
+      # session[:locale] is set via the locale selector from the footer
+      @locale ||= if (l = session&.fetch(:locale, nil)) && available_locales.include?(l)
+        l
+      else
         matched_locale = user_preferred_languages.map { |preferred|
           preferred_language = preferred.split("-", 2).first
 
@@ -111,14 +182,7 @@ module Sidekiq
       end
     end
 
-    # within is used by Sidekiq Pro
-    def display_tags(job, within = nil)
-      job.tags.map { |tag|
-        "<span class='label label-info jobtag'>#{::Rack::Utils.escape_html(tag)}</span>"
-      }.join(" ")
-    end
-
-    # mperham/sidekiq#3243
+    # sidekiq/sidekiq#3243
     def unfiltered?
       yield unless env["PATH_INFO"].start_with?("/filter/")
     end
@@ -137,7 +201,7 @@ module Sidekiq
     end
 
     def sort_direction_label
-      params[:direction] == "asc" ? "&uarr;" : "&darr;"
+      (params[:direction] == "asc") ? "&uarr;" : "&darr;"
     end
 
     def workset
@@ -161,13 +225,21 @@ module Sidekiq
       end
     end
 
+    def busy_weights(capsule_weights)
+      # backwards compat with 7.0.0, remove in 7.1
+      cw = [capsule_weights].flatten
+      cw.map { |hash|
+        hash.map { |name, weight| (weight > 0) ? +name << ": " << weight.to_s : name }.join(", ")
+      }.join("; ")
+    end
+
     def stats
       @stats ||= Sidekiq::Stats.new
     end
 
     def redis_url
       Sidekiq.redis do |conn|
-        conn._config.server_url
+        conn.config.server_url
       end
     end
 
@@ -184,7 +256,7 @@ module Sidekiq
     end
 
     def current_status
-      workset.size == 0 ? "idle" : "active"
+      (workset.size == 0) ? "idle" : "active"
     end
 
     def relative_time(time)
@@ -217,7 +289,7 @@ module Sidekiq
     end
 
     def truncate(text, truncate_after_chars = 2000)
-      truncate_after_chars && text.size > truncate_after_chars ? "#{text[0..truncate_after_chars]}..." : text
+      (truncate_after_chars && text.size > truncate_after_chars) ? "#{text[0..truncate_after_chars]}..." : text
     end
 
     def display_args(args, truncate_after_chars = 2000)
@@ -237,6 +309,10 @@ module Sidekiq
       "<input type='hidden' name='authenticity_token' value='#{env[:csrf_token]}'/>"
     end
 
+    def csp_nonce
+      env[:csp_nonce]
+    end
+
     def to_display(arg)
       arg.inspect
     rescue
@@ -270,27 +346,17 @@ module Sidekiq
       elsif rss_kb < 10_000_000
         "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
       else
-        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
+        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)), precision: 1)} GB"
       end
     end
 
-    def number_with_delimiter(number)
-      return "" if number.nil?
-
-      begin
-        Float(number)
-      rescue ArgumentError, TypeError
-        return number
-      end
-
-      options = {delimiter: ",", separator: "."}
-      parts = number.to_s.to_str.split(".")
-      parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{options[:delimiter]}")
-      parts.join(options[:separator])
+    def number_with_delimiter(number, options = {})
+      precision = options[:precision] || 0
+      %(<span data-nwp="#{precision}">#{number.round(precision)}</span>)
     end
 
     def h(text)
-      ::Rack::Utils.escape_html(text)
+      ::Rack::Utils.escape_html(text.to_s)
     rescue ArgumentError => e
       raise unless e.message.eql?("invalid byte sequence in UTF-8")
       text.encode!("UTF-16", "UTF-8", invalid: :replace, replace: "").encode!("UTF-8", "UTF-16")
@@ -323,6 +389,11 @@ module Sidekiq
       Time.now.utc.strftime("%H:%M:%S UTC")
     end
 
+    def pollable?
+      # there's no point to refreshing the metrics pages every N seconds
+      !(current_path == "" || current_path.index("metrics"))
+    end
+
     def retry_or_delete_or_kill(job, params)
       if params["retry"]
         job.retry

data/lib/sidekiq/web.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "erb"
+require "securerandom"
 
 require "sidekiq"
 require "sidekiq/api"
@@ -34,6 +35,18 @@ module Sidekiq
       "Metrics" => "metrics"
     }
 
+    if Gem::Version.new(Rack::RELEASE) < Gem::Version.new("3")
+      CONTENT_LANGUAGE = "Content-Language"
+      CONTENT_SECURITY_POLICY = "Content-Security-Policy"
+      LOCATION = "Location"
+      X_CASCADE = "X-Cascade"
+    else
+      CONTENT_LANGUAGE = "content-language"
+      CONTENT_SECURITY_POLICY = "content-security-policy"
+      LOCATION = "location"
+      X_CASCADE = "x-cascade"
+    end
+
     class << self
       def settings
         self
@@ -48,6 +61,10 @@ module Sidekiq
       end
       alias_method :tabs, :custom_tabs
 
+      def custom_job_info_rows
+        @custom_job_info_rows ||= []
+      end
+
       def locales
         @locales ||= LOCALES
       end
@@ -98,6 +115,7 @@ module Sidekiq
     end
 
     def call(env)
+      env[:csp_nonce] = SecureRandom.base64(16)
       app.call(env)
     end
 
@@ -122,7 +140,50 @@ module Sidekiq
       send(:"#{attribute}=", value)
     end
 
-    def self.register(extension)
+    # Register a class as a Sidekiq Web UI extension. The class should
+    # provide one or more tabs which map to an index route. Options:
+    #
+    # @param extension [Class] Class which contains the HTTP actions, required
+    # @param name [String] the name of the extension, used to namespace assets
+    # @param tab [String | Array] labels(s) of the UI tabs
+    # @param index [String | Array] index route(s) for each tab
+    # @param root_dir [String] directory location to find assets, locales and views, typically `web/` within the gemfile
+    # @param asset_paths [Array] one or more directories under {root}/assets/{name} to be publicly served, e.g. ["js", "css", "img"]
+    # @param cache_for [Integer] amount of time to cache assets, default one day
+    #
+    # TODO name, tab and index will be mandatory in 8.0
+    #
+    # Web extensions will have a root `web/` directory with `locales/`, `assets/`
+    # and `views/` subdirectories.
+    def self.register(extension, name: nil, tab: nil, index: nil, root_dir: nil, cache_for: 86400, asset_paths: nil)
+      tab = Array(tab)
+      index = Array(index)
+      tab.zip(index).each do |tab, index|
+        tabs[tab] = index
+      end
+      if root_dir
+        locdir = File.join(root_dir, "locales")
+        locales << locdir if File.directory?(locdir)
+
+        if asset_paths && name
+          # if you have {root}/assets/{name}/js/scripts.js
+          # and {root}/assets/{name}/css/styles.css
+          # you would pass in:
+          #   asset_paths: ["js", "css"]
+          # See script_tag and style_tag in web/helpers.rb
+          assdir = File.join(root_dir, "assets")
+          assurls = Array(asset_paths).map { |x| "/#{name}/#{x}" }
+          assetprops = {
+            urls: assurls,
+            root: assdir,
+            cascade: true
+          }
+          assetprops[:header_rules] = [[:all, {Rack::CACHE_CONTROL => "private, max-age=#{cache_for.to_i}"}]] if cache_for
+          middlewares << [[Rack::Static, assetprops], nil]
+        end
+      end
+
+      yield self if block_given?
       extension.registered(WebApplication)
     end
 
@@ -133,7 +194,7 @@ module Sidekiq
       m = middlewares
 
       rules = []
-      rules = [[:all, {"cache-control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
+      rules = [[:all, {Rack::CACHE_CONTROL => "private, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
 
       ::Rack::Builder.new do
         use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],

data/lib/sidekiq.rb

@@ -32,6 +32,7 @@ require "sidekiq/logger"
 require "sidekiq/client"
 require "sidekiq/transaction_aware_client"
 require "sidekiq/job"
+require "sidekiq/iterable_job"
 require "sidekiq/worker_compatibility_alias"
 require "sidekiq/redis_client_adapter"
 
@@ -112,7 +113,7 @@ module Sidekiq
   #   end
   #   inst.run
   #   sleep 10
-  #   inst.terminate
+  #   inst.stop
   #
   # NB: it is really easy to overload a Ruby process with threads due to the GIL.
   # I do not recommend setting concurrency higher than 2-3.

data/sidekiq.gemspec

@@ -2,7 +2,7 @@ require_relative "lib/sidekiq/version"
 
 Gem::Specification.new do |gem|
   gem.authors = ["Mike Perham"]
-  gem.email = ["mperham@gmail.com"]
+  gem.email = ["info@contribsys.com"]
   gem.summary = "Simple, efficient background processing for Ruby"
   gem.description = "Simple, efficient background processing for Ruby."
   gem.homepage = "https://sidekiq.org"
@@ -16,37 +16,16 @@ Gem::Specification.new do |gem|
 
   gem.metadata = {
     "homepage_uri" => "https://sidekiq.org",
-    "bug_tracker_uri" => "https://github.com/mperham/sidekiq/issues",
-    "documentation_uri" => "https://github.com/mperham/sidekiq/wiki",
-    "changelog_uri" => "https://github.com/mperham/sidekiq/blob/main/Changes.md",
-    "source_code_uri" => "https://github.com/mperham/sidekiq"
+    "bug_tracker_uri" => "https://github.com/sidekiq/sidekiq/issues",
+    "documentation_uri" => "https://github.com/sidekiq/sidekiq/wiki",
+    "changelog_uri" => "https://github.com/sidekiq/sidekiq/blob/main/Changes.md",
+    "source_code_uri" => "https://github.com/sidekiq/sidekiq",
+    "rubygems_mfa_required" => "true"
   }
 
-  gem.add_dependency "redis-client", ">= 0.9.0"
+  gem.add_dependency "redis-client", ">= 0.22.2"
   gem.add_dependency "connection_pool", ">= 2.3.0"
   gem.add_dependency "rack", ">= 2.2.4"
   gem.add_dependency "concurrent-ruby", "< 2"
-  gem.post_install_message = <<~EOM
-    
-    ####################################################
-    
-    
-      █████████  █████ ██████████   ██████████ █████   ████ █████    ██████       ██████████       █████
-     ███░░░░░███░░███ ░░███░░░░███ ░░███░░░░░█░░███   ███░ ░░███   ███░░░░███    ░███░░░░███     ███░░░███
-    ░███    ░░░  ░███  ░███   ░░███ ░███  █ ░  ░███  ███    ░███  ███    ░░███   ░░░    ███     ███   ░░███
-    ░░█████████  ░███  ░███    ░███ ░██████    ░███████     ░███ ░███     ░███         ███     ░███    ░███
-     ░░░░░░░░███ ░███  ░███    ░███ ░███░░█    ░███░░███    ░███ ░███   ██░███        ███      ░███    ░███
-     ███    ░███ ░███  ░███    ███  ░███ ░   █ ░███ ░░███   ░███ ░░███ ░░████        ███       ░░███   ███
-    ░░█████████  █████ ██████████   ██████████ █████ ░░████ █████ ░░░██████░██      ███      ██ ░░░█████░
-     ░░░░░░░░░  ░░░░░ ░░░░░░░░░░   ░░░░░░░░░░ ░░░░░   ░░░░ ░░░░░    ░░░░░░ ░░      ░░░      ░░    ░░░░░░
-    
-    
-    WARNING: This is a beta release, expect breakage!
-    
-    1. Use `gem 'sidekiq', '<7'` in your Gemfile if you don't want to be a beta tester.
-    2. Read the release notes at https://github.com/mperham/sidekiq/blob/main/docs/7.0-Upgrade.md
-    3. Search for open/closed issues at https://github.com/mperham/sidekiq/issues/
-    
-    ####################################################
-  EOM
+  gem.add_dependency "logger"
 end

data/web/assets/javascripts/application.js

@@ -31,7 +31,10 @@ function addListeners() {
     node.addEventListener("click", addDataToggleListeners)
   })
 
+  addShiftClickListeners()
   updateFuzzyTimes();
+  updateNumbers();
+  updateProgressBars();
   setLivePollFromUrl();
 
   var buttons = document.querySelectorAll(".live-poll");
@@ -45,6 +48,8 @@ function addListeners() {
       scheduleLivePoll();
     }
   }
+
+  document.getElementById("locale-select").addEventListener("change", updateLocale);
 }
 
 function addPollingListeners(_event)  {
@@ -71,6 +76,23 @@ function addDataToggleListeners(event) {
   }
 }
 
+function addShiftClickListeners() {
+  let checkboxes = Array.from(document.querySelectorAll(".shift_clickable"));
+  let lastChecked = null;
+  checkboxes.forEach(checkbox => {
+    checkbox.addEventListener("click", (e) => {
+      if (e.shiftKey && lastChecked) {
+        let myIndex = checkboxes.indexOf(checkbox);
+        let lastIndex = checkboxes.indexOf(lastChecked);
+        let [min, max] = [myIndex, lastIndex].sort();
+        let newState = checkbox.checked;
+        checkboxes.slice(min, max).forEach(c => c.checked = newState);
+      }
+      lastChecked = checkbox;
+    });
+  });
+}
+
 function updateFuzzyTimes() {
   var locale = document.body.getAttribute("data-locale");
   var parts = locale.split('-');
@@ -84,6 +106,20 @@ function updateFuzzyTimes() {
   t.cancel();
 }
 
+function updateNumbers() {
+  document.querySelectorAll("[data-nwp]").forEach(node => {
+    let number = parseFloat(node.textContent);
+    let precision = parseInt(node.dataset["nwp"] || 0);
+    if (typeof number === "number") {
+      let formatted = number.toLocaleString(undefined, {
+        minimumFractionDigits: precision,
+        maximumFractionDigits: precision,
+      });
+      node.textContent = formatted;
+    }
+  });
+}
+
 function setLivePollFromUrl() {
   var url_params = new URL(window.location.href).searchParams
 
@@ -122,6 +158,7 @@ function checkResponse(resp) {
 
 function scheduleLivePoll() {
   let ti = parseInt(localStorage.sidekiqTimeInterval) || 5000;
+  if (ti < 2000) { ti = 2000 }
   livePollTimer = setTimeout(livePollCallback, ti);
 }
 
@@ -141,3 +178,11 @@ function replacePage(text) {
 function showError(error) {
   console.error(error)
 }
+
+function updateLocale(event) {
+  event.target.form.submit();
+}
+
+function updateProgressBars() {
+  document.querySelectorAll('.progress-bar').forEach(bar => { bar.style.width = bar.dataset.width + "%"})
+}

data/web/assets/javascripts/dashboard-charts.js

@@ -57,7 +57,9 @@ class DashboardChart extends BaseChart {
 class RealtimeChart extends DashboardChart {
   constructor(el, options) {
     super(el, options);
-    this.delay = parseInt(localStorage.sidekiqTimeInterval) || 5000;
+    let d = parseInt(localStorage.sidekiqTimeInterval) || 5000;
+    if (d < 2000) { d = 2000; }
+    this.delay = d
     this.startPolling();
     document.addEventListener("interval:update", this.handleUpdate.bind(this));
   }
@@ -84,6 +86,7 @@ class RealtimeChart extends DashboardChart {
     updateStatsSummary(this.stats.sidekiq);
     updateRedisStats(this.stats.redis);
     updateFooterUTCTime(this.stats.server_utc_time);
+    updateNumbers();
     pulseBeacon();
 
     this.stats = stats;
@@ -105,17 +108,27 @@ class RealtimeChart extends DashboardChart {
   }
 
   renderLegend(dp) {
-    this.legend.innerHTML = `
-      <span>
-        <span class="swatch" style="background-color: ${dp[0].dataset.borderColor};"></span>
-        <span>${dp[0].dataset.label}: ${dp[0].formattedValue}</span>
-      </span>
-      <span>
-        <span class="swatch" style="background-color: ${dp[1].dataset.borderColor};"></span>
-        <span>${dp[1].dataset.label}: ${dp[1].formattedValue}</span>
-      </span>
-      <span class="time">${dp[0].label}</span>
-    `;
+    const entry1 = this.legendEntry(dp[0]);
+    const entry2 = this.legendEntry(dp[1]);
+    const time = document.createElement("span");
+    time.classList.add("time");
+    time.innerText = dp[0].label;
+
+    this.legend.replaceChildren(entry1, entry2, time)
+  }
+
+  legendEntry(dp) {
+    const wrapper = document.createElement("span");
+
+    const swatch = document.createElement("span");
+    swatch.classList.add("swatch");
+    swatch.style.backgroundColor = dp.dataset.borderColor;
+    wrapper.appendChild(swatch)
+    
+    const label = document.createElement("span");
+    label.innerText = `${dp.dataset.label}: ${dp.formattedValue}`;
+    wrapper.appendChild(label)
+    return wrapper;
   }
 
   renderCursor(dp) {
@@ -164,3 +177,16 @@ class RealtimeChart extends DashboardChart {
     };
   }
 }
+
+  var rc = document.getElementById("realtime-chart")
+  if (rc != null) {
+    var rtc = new RealtimeChart(rc, JSON.parse(rc.textContent))
+    rtc.registerLegend(document.getElementById("realtime-legend"))
+    window.realtimeChart = rtc
+  }
+
+  var hc = document.getElementById("history-chart")
+  if (hc != null) {
+    var htc = new DashboardChart(hc, JSON.parse(hc.textContent))
+    window.historyChart = htc
+  }

data/web/assets/javascripts/dashboard.js

@@ -1,15 +1,13 @@
 Sidekiq = {};
 
-var nf = new Intl.NumberFormat();
-
 var updateStatsSummary = function(data) {
-  document.getElementById("txtProcessed").innerText = nf.format(data.processed);
-  document.getElementById("txtFailed").innerText = nf.format(data.failed);
-  document.getElementById("txtBusy").innerText = nf.format(data.busy);
-  document.getElementById("txtScheduled").innerText = nf.format(data.scheduled);
-  document.getElementById("txtRetries").innerText = nf.format(data.retries);
-  document.getElementById("txtEnqueued").innerText = nf.format(data.enqueued);
-  document.getElementById("txtDead").innerText = nf.format(data.dead);
+  document.getElementById("txtProcessed").innerText = data.processed;
+  document.getElementById("txtFailed").innerText = data.failed;
+  document.getElementById("txtBusy").innerText = data.busy;
+  document.getElementById("txtScheduled").innerText = data.scheduled;
+  document.getElementById("txtRetries").innerText = data.retries;
+  document.getElementById("txtEnqueued").innerText = data.enqueued;
+  document.getElementById("txtDead").innerText = data.dead;
 }
 
 var updateRedisStats = function(data) {
@@ -30,7 +28,7 @@ var pulseBeacon = function() {
 }
 
 var setSliderLabel = function(val) {
-  document.getElementById('sldr-text').innerText = Math.round(parseFloat(val) / 1000) + ' sec';
+  document.getElementById('sldr-text').innerText = Math.round(parseFloat(val) / 1000) + ' s';
 }
 
 var ready = (callback) => {