sidekiq 7.1.6 → 7.2.4

data/lib/sidekiq/metrics/query.rb

@@ -20,7 +20,8 @@ module Sidekiq
       end
 
       # Get metric data for all jobs from the last hour
-      def top_jobs(minutes: 60)
+      #  +class_filter+: return only results for classes matching filter
+      def top_jobs(class_filter: nil, minutes: 60)
         result = Result.new
 
         time = @time
@@ -39,6 +40,7 @@ module Sidekiq
         redis_results.each do |hash|
           hash.each do |k, v|
             kls, metric = k.split("|")
+            next if class_filter && !class_filter.match?(kls)
             result.job_results[kls].add_metric metric, time, v.to_i
           end
           time -= 60
@@ -117,6 +119,7 @@ module Sidekiq
 
         def total_avg(metric = "ms")
           completed = totals["p"] - totals["f"]
+          return 0 if completed.zero?
           totals[metric].to_f / completed
         end
 

data/lib/sidekiq/metrics/tracking.rb

@@ -103,12 +103,16 @@ module Sidekiq
       def reset
         @lock.synchronize {
           array = [@totals, @jobs, @grams]
-          @totals = Hash.new(0)
-          @jobs = Hash.new(0)
-          @grams = Hash.new { |hash, key| hash[key] = Histogram.new(key) }
+          reset_instance_variables
           array
         }
       end
+
+      def reset_instance_variables
+        @totals = Hash.new(0)
+        @jobs = Hash.new(0)
+        @grams = Hash.new { |hash, key| hash[key] = Histogram.new(key) }
+      end
     end
 
     class Middleware

data/lib/sidekiq/middleware/current_attributes.rb

@@ -54,7 +54,7 @@ module Sidekiq
             cattrs_to_reset << constklass
 
             job[key].each do |(attribute, value)|
-              constklass.public_send("#{attribute}=", value)
+              constklass.public_send(:"#{attribute}=", value)
             end
           end
         end

data/lib/sidekiq/paginator.rb

@@ -19,9 +19,9 @@ module Sidekiq
           total_size, items = conn.multi { |transaction|
             transaction.zcard(key)
             if rev
-              transaction.zrange(key, starting, ending, "REV", withscores: true)
+              transaction.zrange(key, starting, ending, "REV", "withscores")
             else
-              transaction.zrange(key, starting, ending, withscores: true)
+              transaction.zrange(key, starting, ending, "withscores")
             end
           }
           [current_page, total_size, items]

data/lib/sidekiq/processor.rb

@@ -187,7 +187,7 @@ module Sidekiq
           # we didn't properly finish it.
         rescue Sidekiq::JobRetry::Handled => h
           # this is the common case: job raised error and Sidekiq::JobRetry::Handled
-          # signals that we created a retry successfully.  We can acknowlege the job.
+          # signals that we created a retry successfully.  We can acknowledge the job.
           ack = true
           e = h.cause || h
           handle_exception(e, {context: "Job raised exception", job: job_hash})

data/lib/sidekiq/rails.rb

@@ -20,6 +20,10 @@ module Sidekiq
       def inspect
         "#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
       end
+
+      def to_hash
+        {app: @app.class.name}
+      end
     end
 
     # By including the Options module, we allow AJs to directly control sidekiq features
@@ -56,10 +60,10 @@ module Sidekiq
         # This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
         # it will appear in the Sidekiq console with all of the job context.
         unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
-          if ::Rails::VERSION::STRING < "7.1"
-            ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
-          else
+          if ::Rails.logger.respond_to?(:broadcast_to)
             ::Rails.logger.broadcast_to(config.logger)
+          else
+            ::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
           end
         end
       end

data/lib/sidekiq/redis_client_adapter.rb

@@ -21,6 +21,22 @@ module Sidekiq
         @client.call("EVALSHA", sha, keys.size, *keys, *argv)
       end
 
+      # this is the set of Redis commands used by Sidekiq. Not guaranteed
+      # to be comprehensive, we use this as a performance enhancement to
+      # avoid calling method_missing on most commands
+      USED_COMMANDS = %w[bitfield bitfield_ro del exists expire flushdb
+        get hdel hget hgetall hincrby hlen hmget hset hsetnx incr incrby
+        lindex llen lmove lpop lpush lrange lrem mget mset ping pttl
+        publish rpop rpush sadd scard script set sismember smembers
+        srem ttl type unlink zadd zcard zincrby zrange zrem
+        zremrangebyrank zremrangebyscore]
+
+      USED_COMMANDS.each do |name|
+        define_method(name) do |*args, **kwargs|
+          @client.call(name, *args, **kwargs)
+        end
+      end
+
       private
 
       # this allows us to use methods like `conn.hmset(...)` instead of having to use

data/lib/sidekiq/redis_connection.rb

@@ -39,9 +39,8 @@ module Sidekiq
           uri.password = redacted
           scrubbed_options[:url] = uri.to_s
         end
-        if scrubbed_options[:password]
-          scrubbed_options[:password] = redacted
-        end
+        scrubbed_options[:password] = redacted if scrubbed_options[:password]
+        scrubbed_options[:sentinel_password] = redacted if scrubbed_options[:sentinel_password]
         scrubbed_options[:sentinels]&.each do |sentinel|
           sentinel[:password] = redacted if sentinel[:password]
         end
@@ -67,9 +66,7 @@ module Sidekiq
           EOM
         end
 
-        ENV[
-          p || "REDIS_URL"
-        ]
+        ENV[p.to_s] || ENV["REDIS_URL"]
       end
     end
   end

data/lib/sidekiq/scheduled.rb

@@ -144,7 +144,7 @@ module Sidekiq
         # In the example above, each process should schedule every 10 seconds on average. We special
         # case smaller clusters to add 50% so they would sleep somewhere between 5 and 15 seconds.
         # As we run more processes, the scheduling interval average will approach an even spread
-        # between 0 and poll interval so we don't need this artifical boost.
+        # between 0 and poll interval so we don't need this artificial boost.
         #
         count = process_count
         interval = poll_interval_average(count)
@@ -193,7 +193,7 @@ module Sidekiq
       # should never depend on sidekiq/api.
       def cleanup
         # dont run cleanup more than once per minute
-        return 0 unless redis { |conn| conn.set("process_cleanup", "1", nx: true, ex: 60) }
+        return 0 unless redis { |conn| conn.set("process_cleanup", "1", "NX", "EX", "60") }
 
         count = 0
         redis do |conn|

data/lib/sidekiq/testing.rb

@@ -5,6 +5,7 @@ require "sidekiq"
 
 module Sidekiq
   class Testing
+    class TestModeAlreadySetError < RuntimeError; end
     class << self
       attr_accessor :__global_test_mode
 
@@ -12,8 +13,13 @@ module Sidekiq
       # all threads. Calling with a block only affects the current Thread.
       def __set_test_mode(mode)
         if block_given?
+          # Reentrant testing modes will lead to a rat's nest of code which is
+          # hard to reason about. You can set the testing mode once globally and
+          # you can override that global setting once per-thread.
+          raise TestModeAlreadySetError, "Nesting test modes is not supported" if __local_test_mode
+
+          self.__local_test_mode = mode
           begin
-            self.__local_test_mode = mode
             yield
           ensure
             self.__local_test_mode = nil
@@ -106,7 +112,7 @@ module Sidekiq
     # The Queues class is only for testing the fake queue implementation.
     # There are 2 data structures involved in tandem. This is due to the
     # Rspec syntax of change(HardJob.jobs, :size). It keeps a reference
-    # to the array. Because the array was dervied from a filter of the total
+    # to the array. Because the array was derived from a filter of the total
     # jobs enqueued, it appeared as though the array didn't change.
     #
     # To solve this, we'll keep 2 hashes containing the jobs. One with keys based
@@ -272,7 +278,7 @@ module Sidekiq
       def perform_one
         raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
         next_job = jobs.first
-        Queues.delete_for(next_job["jid"], queue, to_s)
+        Queues.delete_for(next_job["jid"], next_job["queue"], to_s)
         process_job(next_job)
       end
 

data/lib/sidekiq/transaction_aware_client.rb

@@ -9,7 +9,14 @@ module Sidekiq
       @redis_client = Client.new(pool: pool, config: config)
     end
 
+    def batching?
+      Thread.current[:sidekiq_batch]
+    end
+
     def push(item)
+      # 6160 we can't support both Sidekiq::Batch and transactions.
+      return @redis_client.push(item) if batching?
+
       # pre-allocate the JID so we can return it immediately and
       # save it to the database as part of the transaction.
       item["jid"] ||= SecureRandom.hex(12)

data/lib/sidekiq/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "7.1.6"
+  VERSION = "7.2.4"
   MAJOR = 7
 end

data/lib/sidekiq/web/action.rb

@@ -22,6 +22,11 @@ module Sidekiq
       throw :halt, [302, {Web::LOCATION => "#{request.base_url}#{location}"}, []]
     end
 
+    def reload_page
+      current_location = request.referer.gsub(request.base_url, "")
+      redirect current_location
+    end
+
     def params
       indifferent_hash = Hash.new { |hash, key| hash[key.to_s] if Symbol === key }
 

data/lib/sidekiq/web/application.rb

@@ -15,7 +15,7 @@ module Sidekiq
       "manifest-src 'self'",
       "media-src 'self'",
       "object-src 'none'",
-      "script-src 'self' https: http: 'unsafe-inline'",
+      "script-src 'self' https: http:",
       "style-src 'self' https: http: 'unsafe-inline'",
       "worker-src 'self'",
       "base-uri 'self'"
@@ -49,9 +49,9 @@ module Sidekiq
 
     head "/" do
       # HEAD / is the cheapest heartbeat possible,
-      # it hits Redis to ensure connectivity
-      Sidekiq.redis { |c| c.llen("queue:default") }
-      ""
+      # it hits Redis to ensure connectivity and returns
+      # the size of the default queue
+      Sidekiq.redis { |c| c.llen("queue:default") }.to_s
     end
 
     get "/" do
@@ -330,6 +330,22 @@ module Sidekiq
 
     ########
     # Filtering
+
+    get "/filter/metrics" do
+      redirect "#{root_path}metrics"
+    end
+
+    post "/filter/metrics" do
+      x = params[:substr]
+      q = Sidekiq::Metrics::Query.new
+      @period = h((params[:period] || "")[0..1])
+      @periods = METRICS_PERIODS
+      minutes = @periods.fetch(@period, @periods.values.first)
+      @query_result = q.top_jobs(minutes: minutes, class_filter: Regexp.new(Regexp.escape(x), Regexp::IGNORECASE))
+
+      erb :metrics
+    end
+
     get "/filter/retries" do
       x = params[:substr]
       return redirect "#{root_path}retries" unless x && x != ""
@@ -378,6 +394,18 @@ module Sidekiq
       erb :morgue
     end
 
+    post "/change_locale" do
+      locale = params["locale"]
+
+      match = available_locales.find { |available|
+        locale == available
+      }
+
+      session[:locale] = match if match
+
+      reload_page
+    end
+
     def call(env)
       action = self.class.match(env)
       return [404, {Rack::CONTENT_TYPE => "text/plain", Web::X_CASCADE => "pass"}, ["Not Found"]] unless action

data/lib/sidekiq/web/csrf_protection.rb

@@ -27,7 +27,6 @@
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 require "securerandom"
-require "base64"
 require "rack/request"
 
 module Sidekiq
@@ -57,7 +56,7 @@ module Sidekiq
       end
 
       def logger(env)
-        @logger ||= (env["rack.logger"] || ::Logger.new(env["rack.errors"]))
+        @logger ||= env["rack.logger"] || ::Logger.new(env["rack.errors"])
       end
 
       def deny(env)
@@ -116,7 +115,7 @@ module Sidekiq
         sess = session(env)
         localtoken = sess[:csrf]
 
-        # Checks that Rack::Session::Cookie actualy contains the csrf toekn
+        # Checks that Rack::Session::Cookie actually contains the csrf token
         return false if localtoken.nil?
 
         # Rotate the session token after every use
@@ -143,7 +142,7 @@ module Sidekiq
         one_time_pad = SecureRandom.random_bytes(token.length)
         encrypted_token = xor_byte_strings(one_time_pad, token)
         masked_token = one_time_pad + encrypted_token
-        Base64.urlsafe_encode64(masked_token)
+        encode_token(masked_token)
       end
 
       # Essentially the inverse of +mask_token+.
@@ -168,8 +167,12 @@ module Sidekiq
         ::Rack::Utils.secure_compare(token.to_s, decode_token(local).to_s)
       end
 
+      def encode_token(token)
+        [token].pack("m0").tr("+/", "-_")
+      end
+
       def decode_token(token)
-        Base64.urlsafe_decode64(token)
+        token.tr("-_", "+/").unpack1("m0")
       end
 
       def xor_byte_strings(s1, s2)

data/lib/sidekiq/web/helpers.rb

@@ -21,6 +21,10 @@ module Sidekiq
       end
     end
 
+    def to_json(x)
+      Sidekiq.dump_json(x)
+    end
+
     def singularize(str, count)
       if count == 1 && str.respond_to?(:singularize) # rails
         str.singularize
@@ -117,6 +121,10 @@ module Sidekiq
     #
     # Inspiration taken from https://github.com/iain/http_accept_language/blob/master/lib/http_accept_language/parser.rb
     def locale
+      # session[:locale] is set via the locale selector from the footer
+      # defined?(session) && session are used to avoid exceptions when running tests
+      return session[:locale] if defined?(session) && session&.[](:locale)
+
       @locale ||= begin
         matched_locale = user_preferred_languages.map { |preferred|
           preferred_language = preferred.split("-", 2).first
@@ -292,23 +300,13 @@ module Sidekiq
       elsif rss_kb < 10_000_000
         "#{number_with_delimiter((rss_kb / 1024.0).to_i)} MB"
       else
-        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)).round(1))} GB"
+        "#{number_with_delimiter((rss_kb / (1024.0 * 1024.0)), precision: 1)} GB"
       end
     end
 
-    def number_with_delimiter(number)
-      return "" if number.nil?
-
-      begin
-        Float(number)
-      rescue ArgumentError, TypeError
-        return number
-      end
-
-      options = {delimiter: ",", separator: "."}
-      parts = number.to_s.to_str.split(".")
-      parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{options[:delimiter]}")
-      parts.join(options[:separator])
+    def number_with_delimiter(number, options = {})
+      precision = options[:precision] || 0
+      %(<span data-nwp="#{precision}">#{number.round(precision)}</span>)
     end
 
     def h(text)
@@ -346,7 +344,8 @@ module Sidekiq
     end
 
     def pollable?
-      !(current_path == "" || current_path.start_with?("metrics"))
+      # there's no point to refreshing the metrics pages every N seconds
+      !(current_path == "" || current_path.index("metrics"))
     end
 
     def retry_or_delete_or_kill(job, params)

data/sidekiq.gemspec

@@ -23,7 +23,7 @@ Gem::Specification.new do |gem|
     "rubygems_mfa_required" => "true"
   }
 
-  gem.add_dependency "redis-client", ">= 0.14.0"
+  gem.add_dependency "redis-client", ">= 0.19.0"
   gem.add_dependency "connection_pool", ">= 2.3.0"
   gem.add_dependency "rack", ">= 2.2.4"
   gem.add_dependency "concurrent-ruby", "< 2"

data/web/assets/javascripts/application.js

@@ -33,6 +33,7 @@ function addListeners() {
 
   addShiftClickListeners()
   updateFuzzyTimes();
+  updateNumbers();
   setLivePollFromUrl();
 
   var buttons = document.querySelectorAll(".live-poll");
@@ -46,6 +47,8 @@ function addListeners() {
       scheduleLivePoll();
     }
   }
+
+  document.getElementById("locale-select").addEventListener("change", updateLocale);
 }
 
 function addPollingListeners(_event)  {
@@ -102,6 +105,20 @@ function updateFuzzyTimes() {
   t.cancel();
 }
 
+function updateNumbers() {
+  document.querySelectorAll("[data-nwp]").forEach(node => {
+    let number = parseFloat(node.textContent);
+    let precision = parseInt(node.dataset["nwp"] || 0);
+    if (typeof number === "number") {
+      let formatted = number.toLocaleString(undefined, {
+        minimumFractionDigits: precision,
+        maximumFractionDigits: precision,
+      });
+      node.textContent = formatted;
+    }
+  });
+}
+
 function setLivePollFromUrl() {
   var url_params = new URL(window.location.href).searchParams
 
@@ -160,3 +177,7 @@ function replacePage(text) {
 function showError(error) {
   console.error(error)
 }
+
+function updateLocale(event) {
+  event.target.form.submit();
+};

data/web/assets/javascripts/dashboard-charts.js

@@ -86,6 +86,7 @@ class RealtimeChart extends DashboardChart {
     updateStatsSummary(this.stats.sidekiq);
     updateRedisStats(this.stats.redis);
     updateFooterUTCTime(this.stats.server_utc_time);
+    updateNumbers();
     pulseBeacon();
 
     this.stats = stats;
@@ -166,3 +167,16 @@ class RealtimeChart extends DashboardChart {
     };
   }
 }
+
+  var rc = document.getElementById("realtime-chart")
+  if (rc != null) {
+    var rtc = new RealtimeChart(rc, JSON.parse(rc.textContent))
+    rtc.registerLegend(document.getElementById("realtime-legend"))
+    window.realtimeChart = rtc
+  }
+
+  var hc = document.getElementById("history-chart")
+  if (hc != null) {
+    var htc = new DashboardChart(hc, JSON.parse(hc.textContent))
+    window.historyChart = htc
+  }

data/web/assets/javascripts/dashboard.js

@@ -1,15 +1,13 @@
 Sidekiq = {};
 
-var nf = new Intl.NumberFormat();
-
 var updateStatsSummary = function(data) {
-  document.getElementById("txtProcessed").innerText = nf.format(data.processed);
-  document.getElementById("txtFailed").innerText = nf.format(data.failed);
-  document.getElementById("txtBusy").innerText = nf.format(data.busy);
-  document.getElementById("txtScheduled").innerText = nf.format(data.scheduled);
-  document.getElementById("txtRetries").innerText = nf.format(data.retries);
-  document.getElementById("txtEnqueued").innerText = nf.format(data.enqueued);
-  document.getElementById("txtDead").innerText = nf.format(data.dead);
+  document.getElementById("txtProcessed").innerText = data.processed;
+  document.getElementById("txtFailed").innerText = data.failed;
+  document.getElementById("txtBusy").innerText = data.busy;
+  document.getElementById("txtScheduled").innerText = data.scheduled;
+  document.getElementById("txtRetries").innerText = data.retries;
+  document.getElementById("txtEnqueued").innerText = data.enqueued;
+  document.getElementById("txtDead").innerText = data.dead;
 }
 
 var updateRedisStats = function(data) {

data/web/assets/javascripts/metrics.js

@@ -262,3 +262,37 @@ class HistBubbleChart extends BaseChart {
     };
   }
 }
+
+var ch = document.getElementById("job-metrics-overview-chart");
+if (ch != null) {
+  var jm = new JobMetricsOverviewChart(ch, JSON.parse(ch.textContent));
+  document.querySelectorAll(".metrics-swatch-wrapper > input[type=checkbox]").forEach((imp) => {
+    jm.registerSwatch(imp.id)
+  });
+  window.jobMetricsChart = jm;
+}
+
+var htc = document.getElementById("hist-totals-chart");
+if (htc != null) {
+  var tc = new HistTotalsChart(htc, JSON.parse(htc.textContent));
+  window.histTotalsChart = tc
+}
+
+var hbc = document.getElementById("hist-bubble-chart");
+if (hbc != null) {
+  var bc = new HistBubbleChart(hbc, JSON.parse(hbc.textContent));
+  window.histBubbleChart = bc
+}
+
+var form = document.getElementById("metrics-form")
+document.querySelectorAll("#period-selector").forEach(node => {
+  node.addEventListener("input", debounce(() => form.submit()))
+})
+
+function debounce(func, timeout = 300) {
+  let timer;
+  return (...args) => {
+    clearTimeout(timer);
+    timer = setTimeout(() => { func.apply(this, args); }, timeout);
+  };
+}

data/web/assets/stylesheets/application-rtl.css

@@ -151,3 +151,13 @@ div.interval-slider {
     padding-left: 5px;
   }
 }
+
+#locale-select {
+  float: right;
+}
+
+@media (max-width: 767px) {
+  #locale-select {
+    float: none;
+  }
+}

data/web/assets/stylesheets/application.css

@@ -370,6 +370,15 @@ img.smallogo {
 .stat p {
   font-size: 0.9em;
 }
+
+.num {
+  font-family: monospace;
+}
+
+td.num {
+  text-align: right;
+}
+
 @media (max-width: 767px) {
   .stats-container {
     display: block;
@@ -722,3 +731,16 @@ div.interval-slider input {
 canvas {
   margin: 20px 0 30px;
 }
+
+#locale-select {
+  float: left;
+  margin: 8px 15px;
+}
+
+@media (max-width: 767px) {
+  #locale-select {
+    float: none;
+    width: auto;
+    margin: 15px auto;
+  }
+}

data/web/views/_footer.erb

@@ -15,7 +15,19 @@
             <p class="navbar-text"><a rel=help href="https://github.com/sidekiq/sidekiq/wiki">docs</a></p>
           </li>
           <li>
-            <p class="navbar-text"><a rel=external href="https://github.com/sidekiq/sidekiq/tree/main/web/locales"><%= locale %></a></p>
+            <form id="locale-form" class="form-inline" action="<%= root_path %>change_locale" method="post">
+              <%= csrf_tag %>
+              <label class="sr-only" for="locale">Language</label>
+              <select id="locale-select" class="form-control" name="locale">
+                <% available_locales.each do |locale_option| %>
+                  <% if locale_option == locale %>
+                    <option selected value="<%= locale_option %>"><%= locale_option %></option>
+                  <% else %>
+                    <option value="<%= locale_option %>"><%= locale_option %></option>
+                  <% end %>
+                <% end %>
+              </select>
+            </form>
           </li>
         </ul>
     </div>

data/web/views/_metrics_period_select.erb

@@ -1,5 +1,5 @@
 <div>
-  <select class="form-control" onchange="window.location.href = '<%= path %>?period=' + event.target.value">
+  <select class="form-control" data-metric-period="<%= path %>">
     <% periods.each_key do |code| %>
 
       <% if code == period %>