sidekiq 5.2.2 → 5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 985b2f1c0778b1e4d2587eb2f182fc58723277c7
-  data.tar.gz: 2d20a6e15e248bea802b1ca9438c9507a1f94ab4
+  metadata.gz: 0f67e98eb8e3fff8ee4d57f08d5668e41b320fa4
+  data.tar.gz: a11f770fd95fd45b3c0be43beee5c83417811fd4
 SHA512:
-  metadata.gz: 4e4be1e23c16eb53d43424a0a07e6b69ab2001ffbbaf28c7c09db1dac89d9f1e9ac3513c70ac5abd398efb12c8a712da20f563df5fd055c6855eb5d2d7abea05
-  data.tar.gz: 8947fc9287e9a1336ba57b6ef380e5f76d7ea7382fff3930acd914c6c08fed9c08940780c1537bbedf349ed29257b9c249f8356eed095d8537a4195e7426b676
+  metadata.gz: 3e6739681028126924c049e41bbfcdc92bea0d1ee4fb965f4e332da4cca9f10352f5cf18e05345a843d19b6111510f55c88aa97f4e21585bf66b479b7ae2f9bc
+  data.tar.gz: 07f0432c69395be2933f152acb09c8fa4fe43be104011df237b0b4db56073b4725eb3fa3073872376f09d072ea15608f79a32b3dc28104ba18665082ad64adf4

data/Changes.md

@@ -2,6 +2,17 @@
 
 [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
 
+5.2.3
+---------
+
+- Warning message on invalid REDIS\_PROVIDER [#3970]
+- Add `sidekiqctl status` command [#4003, dzunk]
+- Update elapsed time calculatons to use monotonic clock [#3999]
+- Fix a few issues with mobile Web UI styling [#3973, navied]
+- Jobs with `retry: false` now go through the global `death_handlers`,
+  meaning you can take action on failed ephemeral jobs. [#3980, Benjamin-Dobell]
+- Fix race condition in defining Workers. [#3997, mattbooks]
+
 5.2.2
 ---------
 

data/Ent-Changes.md

@@ -7,6 +7,9 @@ Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how t
 HEAD
 -------------
 
+- Fix elapsed time calculations to use monotonic clock [#4000, sj26]
+- Fix edge case where flapping leadership would cause old periodic
+  jobs to be fired once [#3974]
 - Add support for sidekiqswarm memory monitoring on FreeBSD [#3884]
 
 1.7.1

data/Pro-Changes.md

@@ -4,6 +4,11 @@
 
 Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
 
+HEAD
+---------
+
+- Add Japanese and Russian translations, see [issue #3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language
+
 4.0.4
 ---------
 

data/bin/sidekiqctl

@@ -1,20 +1,27 @@
 #!/usr/bin/env ruby
 
 require 'fileutils'
+require 'sidekiq/api'
 
 class Sidekiqctl
   DEFAULT_KILL_TIMEOUT = 10
+  CMD = File.basename($0)
 
   attr_reader :stage, :pidfile, :kill_timeout
 
   def self.print_usage
-    puts "#{File.basename($0)} - stop a Sidekiq process from the command line."
+    puts "#{CMD} - control Sidekiq from the command line."
+    puts
+    puts "Usage: #{CMD} quiet <pidfile> <kill_timeout>"
+    puts "       #{CMD} stop <pidfile> <kill_timeout>"
+    puts "       #{CMD} status <section>"
     puts
-    puts "Usage: #{File.basename($0)} <command> <pidfile> <kill_timeout>"
-    puts " where <command> is either 'quiet' or 'stop'"
     puts "       <pidfile> is path to a pidfile"
     puts "       <kill_timeout> is number of seconds to wait until Sidekiq exits"
     puts "       (default: #{Sidekiqctl::DEFAULT_KILL_TIMEOUT}), after which Sidekiq will be KILL'd"
+    puts 
+    puts "       <section> (optional) view a specific section of the status output"
+    puts "       Valid sections are: #{Sidekiqctl::Status::VALID_SECTIONS.join(', ')}"
     puts
     puts "Be sure to set the kill_timeout LONGER than Sidekiq's -t timeout.  If you want"
     puts "to wait 60 seconds for jobs to finish, use `sidekiq -t 60` and `sidekiqctl stop"
@@ -85,6 +92,137 @@ class Sidekiqctl
     done 'Sidekiq shut down forcefully.'
   end
   alias_method :shutdown, :stop
+
+  class Status
+    VALID_SECTIONS = %w[all version overview processes queues]
+    def display(section = nil)
+      section ||= 'all'
+      unless VALID_SECTIONS.include? section
+        puts "I don't know how to check the status of '#{section}'!"
+        puts "Try one of these: #{VALID_SECTIONS.join(', ')}"
+        return
+      end
+      send(section)
+    rescue StandardError => e
+      puts "Couldn't get status: #{e}"
+    end
+
+    def all
+      version
+      puts
+      overview
+      puts
+      processes
+      puts
+      queues
+    end
+
+    def version
+      puts "Sidekiq #{Sidekiq::VERSION}"
+      puts Time.now
+    end
+
+    def overview
+      puts '---- Overview ----'
+      puts "  Processed: #{delimit stats.processed}"
+      puts "     Failed: #{delimit stats.failed}"
+      puts "       Busy: #{delimit stats.workers_size}"
+      puts "   Enqueued: #{delimit stats.enqueued}"
+      puts "    Retries: #{delimit stats.retry_size}"
+      puts "  Scheduled: #{delimit stats.scheduled_size}"
+      puts "       Dead: #{delimit stats.dead_size}"
+    end
+
+    def processes
+      puts "---- Processes (#{process_set.size}) ----"
+      process_set.each_with_index do |process, index|
+        puts "#{process['identity']} #{tags_for(process)}"
+        puts "  Started: #{Time.at(process['started_at'])} (#{time_ago(process['started_at'])})"
+        puts "  Threads: #{process['concurrency']} (#{process['busy']} busy)"
+        puts "   Queues: #{split_multiline(process['queues'].sort, pad: 11)}"
+        puts '' unless (index+1) == process_set.size
+      end
+    end
+
+    COL_PAD = 2
+    def queues
+      puts "---- Queues (#{queue_data.size}) ----"
+      columns = {
+        name: [:ljust, (['name'] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
+        size: [:rjust, (['size'] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
+        latency: [:rjust, (['latency'] + queue_data.map(&:latency)).map(&:length).max + COL_PAD]
+      }
+      columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
+      puts
+      queue_data.each do |q|
+        columns.each do |col, (dir, width)|
+          print q.send(col).public_send(dir, width)
+        end
+        puts
+      end
+    end
+
+    private
+
+    def delimit(number)
+      number.to_s.reverse.scan(/.{1,3}/).join(',').reverse
+    end
+
+    def split_multiline(values, opts = {})
+      return 'none' unless values
+      pad = opts[:pad] || 0
+      max_length = opts[:max_length] || (80 - pad)
+      out = []
+      line = ''
+      values.each do |value|
+        if (line.length + value.length) > max_length
+          out << line
+          line = ' ' * pad
+        end
+        line << value + ', '
+      end
+      out << line[0..-3]
+      out.join("\n")
+    end
+
+    def tags_for(process)
+      tags = [
+        process['tag'],
+        process['labels'],
+        (process['quiet'] == 'true' ? 'quiet' : nil)
+      ].flatten.compact
+      tags.any? ? "[#{tags.join('] [')}]" : nil
+    end
+
+    def time_ago(timestamp)
+      seconds = Time.now - Time.at(timestamp)
+      return 'just now' if seconds < 60
+      return 'a minute ago' if seconds < 120
+      return "#{seconds.floor / 60} minutes ago" if seconds < 3600
+      return 'an hour ago' if seconds < 7200
+      "#{seconds.floor / 60 / 60} hours ago"
+    end
+
+    QUEUE_STRUCT = Struct.new(:name, :size, :latency)
+    def queue_data
+      @queue_data ||= Sidekiq::Queue.all.map do |q|
+        QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf('%#.2f', q.latency))
+      end
+    end
+
+    def process_set
+      @process_set ||= Sidekiq::ProcessSet.new
+    end
+
+    def stats
+      @stats ||= Sidekiq::Stats.new
+    end
+  end
+end
+
+if ARGV[0] == 'status'
+  Sidekiqctl::Status.new.display(ARGV[1])
+  exit
 end
 
 if ARGV.length < 2

data/lib/sidekiq/job_logger.rb

@@ -3,7 +3,7 @@ module Sidekiq
   class JobLogger
 
     def call(item, queue)
-      start = Time.now
+      start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       logger.info("start")
       yield
       logger.info("done: #{elapsed(start)} sec")
@@ -15,7 +15,7 @@ module Sidekiq
     private
 
     def elapsed(start)
-      (Time.now - start).round(3)
+      (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start).round(3)
     end
 
     def logger

data/lib/sidekiq/job_retry.rb

@@ -80,8 +80,18 @@ module Sidekiq
       # ignore, will be pushed back onto queue during hard_shutdown
       raise Sidekiq::Shutdown if exception_caused_by_shutdown?(e)
 
-      raise e unless msg['retry']
-      attempt_retry(nil, msg, queue, e)
+      if msg['retry']
+        attempt_retry(nil, msg, queue, e)
+      else
+        Sidekiq.death_handlers.each do |handler|
+          begin
+            handler.call(msg, e)
+          rescue => handler_ex
+            handle_exception(handler_ex, { context: "Error calling death handler", job: msg })
+          end
+        end
+      end
+
       raise e
     end
 
@@ -158,7 +168,8 @@ module Sidekiq
 
       if count < max_retry_attempts
         delay = delay_for(worker, count, exception)
-        logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+        # Logging here can break retries if the logging device raises ENOSPC #3979
+        #logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
         retry_at = Time.now.to_f + delay
         payload = Sidekiq.dump_json(msg)
         Sidekiq.redis do |conn|
@@ -171,7 +182,6 @@ module Sidekiq
     end
 
     def retries_exhausted(worker, msg, exception)
-      logger.debug { "Retries exhausted for job" }
       begin
         block = worker && worker.sidekiq_retries_exhausted_block
         block.call(msg, exception) if block
@@ -191,7 +201,7 @@ module Sidekiq
     end
 
     def send_to_morgue(msg)
-      Sidekiq.logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
+      logger.info { "Adding dead #{msg['class']} job #{msg['jid']}" }
       payload = Sidekiq.dump_json(msg)
       DeadSet.new.kill(payload, notify_failure: false)
     end

data/lib/sidekiq/launcher.rb

@@ -40,7 +40,7 @@ module Sidekiq
     # return until all work is complete and cleaned up.
     # It can take up to the timeout to complete.
     def stop
-      deadline = Time.now + @options[:timeout]
+      deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + @options[:timeout]
 
       @done = true
       @manager.quiet

data/lib/sidekiq/manager.rb

@@ -30,7 +30,7 @@ module Sidekiq
     def initialize(options={})
       logger.debug { options.inspect }
       @options = options
-      @count = options[:concurrency] || 25
+      @count = options[:concurrency] || 10
       raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
 
       @done = false
@@ -70,11 +70,11 @@ module Sidekiq
       return if @workers.empty?
 
       logger.info { "Pausing to allow workers to finish..." }
-      remaining = deadline - Time.now
+      remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       while remaining > PAUSE_TIME
         return if @workers.empty?
         sleep PAUSE_TIME
-        remaining = deadline - Time.now
+        remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
       end
       return if @workers.empty?
 

data/lib/sidekiq/processor.rb

@@ -87,7 +87,7 @@ module Sidekiq
     def get_one
       begin
         work = @strategy.retrieve_work
-        (logger.info { "Redis is online, #{Time.now - @down} sec downtime" }; @down = nil) if @down
+        (logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }; @down = nil) if @down
         work
       rescue Sidekiq::Shutdown
       rescue => ex
@@ -107,7 +107,7 @@ module Sidekiq
 
     def handle_fetch_exception(ex)
       if !@down
-        @down = Time.now
+        @down = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
         logger.error("Error fetching job: #{ex}")
         handle_exception(ex)
       end

data/lib/sidekiq/redis_connection.rb

@@ -115,6 +115,25 @@ module Sidekiq
         # REDIS_PROVIDER=MY_REDIS_URL
         # and Sidekiq will find your custom URL variable with no custom
         # initialization code at all.
+        p = ENV['REDIS_PROVIDER']
+        if p && p =~ /\:/
+          Sidekiq.logger.error <<-EOM
+
+#################################################################################
+
+REDIS_PROVIDER should be set to the **name** of the variable which contains the Redis URL, not a URL itself.
+Platforms like Heroku sell addons that publish a *_URL variable.  You tell Sidekiq with REDIS_PROVIDER, e.g.:
+
+  REDIS_PROVIDER=REDISTOGO_URL
+  REDISTOGO_URL=redis://somehost.example.com:6379/4
+
+Use REDIS_URL if you wish to point Sidekiq to a URL directly.
+
+This configuration error will crash starting in Sidekiq 5.3.
+
+#################################################################################
+EOM
+        end
         ENV[
           ENV['REDIS_PROVIDER'] || 'REDIS_URL'
         ]

data/lib/sidekiq/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Sidekiq
-  VERSION = "5.2.2"
+  VERSION = "5.2.3"
 end

data/lib/sidekiq/worker.rb

@@ -66,6 +66,7 @@ module Sidekiq
     end
 
     module ClassMethods
+      ACCESSOR_MUTEX = Mutex.new
 
       def delay(*args)
         raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
@@ -148,10 +149,18 @@ module Sidekiq
         instance_writer = true
 
         attrs.each do |name|
+          synchronized_getter = "__synchronized_#{name}"
+
           singleton_class.instance_eval do
             undef_method(name) if method_defined?(name) || private_method_defined?(name)
           end
-          define_singleton_method(name) { nil }
+
+          define_singleton_method(synchronized_getter) { nil }
+          singleton_class.class_eval do
+            private(synchronized_getter)
+          end
+
+          define_singleton_method(name) { ACCESSOR_MUTEX.synchronize { send synchronized_getter } }
 
           ivar = "@#{name}"
 
@@ -161,8 +170,10 @@ module Sidekiq
           end
           define_singleton_method("#{name}=") do |val|
             singleton_class.class_eval do
-              undef_method(name) if method_defined?(name) || private_method_defined?(name)
-              define_method(name) { val }
+              ACCESSOR_MUTEX.synchronize do
+                undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
+                define_method(synchronized_getter) { val }
+              end
             end
 
             if singleton_class?

data/web/assets/javascripts/dashboard.js

@@ -298,8 +298,18 @@ var debounce = function(fn, timeout)
   }
 };
 
-window.onresize = debounce(function() {
-    clearInterval(poller);
-    resetGraphs();
-    renderGraphs();
-}, 125);
+window.onresize = function() {
+  var prevWidth = window.innerWidth;
+  
+  return debounce(function () {
+    var currWidth = window.innerWidth;
+    
+    if (prevWidth !== currWidth) {
+      prevWidth = currWidth;
+      
+      clearInterval(poller);
+      resetGraphs();
+      renderGraphs();
+    }
+  }, 125);
+}();

data/web/assets/stylesheets/application.css

@@ -126,7 +126,7 @@ header.row .pagination {
   text-align: center;
   width: 14%;
 }
-@media (max-width: 767px) and (min-width: 400px) {
+@media (max-width: 767px) and (min-width: 200px) {
   .summary_bar ul li {
     width: 100%;
   }
@@ -219,6 +219,29 @@ table .table-checkbox label {
   color: #585454;
 }
 
+
+.nav.navbar-nav{
+  display: flex;
+  width: 100%;
+}
+
+.navbar-livereload{
+  margin-left: auto;
+  white-space: nowrap;
+}
+
+.navbar-livereload .poll-wrapper a:last-child{
+	margin-left: 8px;
+}
+
+.navbar-right{
+  margin-right: 0;
+}
+
+.navbar-collapse.collapse{
+  overflow-x: auto !important;
+}
+
 @media (max-width: 768px) {
   .navbar .navbar-header .navbar-livereload {
     border: none;
@@ -234,13 +257,19 @@ table .table-checkbox label {
     display: none;
   }
 
+  .nav.navbar-nav{
+  	display: block;
+  	width: auto;
+  }
+
   .navbar.navbar-fixed-top ul {
     margin-right: -15px!important;
   }
 
   .navbar .nav a {
     text-align: center;
-  }
+  }  
+
 }
 
 @media (width: 768px) {
@@ -645,6 +674,10 @@ div.interval-slider input {
     margin-right: 0;
   }
 
+  .navbar #navbar-menu{
+  	display: none;
+  }
+
   .poll-wrapper {
     width: 100%;
     text-align: center;

data/web/views/_nav.erb

@@ -9,7 +9,7 @@
       <div class="navbar-toggle collapsed navbar-livereload">
         <%= erb :_poll_link %>
         <% if Sidekiq::Web.app_url %>
-          <a class="btn btn-inverse" href="<%= Sidekiq::Web.app_url %>">Back to App</a>
+          <a class="btn btn-inverse" href="<%= Sidekiq::Web.app_url %>"><%= t('BackToApp') %></a>
         <% end %>
       </div>
       <a class="navbar-brand" href="<%= root_path %>">
@@ -32,27 +32,13 @@
           <% end %>
         <% end %>
 
-        <li class="dropdown" data-navbar="dropdown">
-          <a href="#" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">
-            <%= t('Plugins') %> <span class="caret"></span>
-          </a>
-          <ul class="dropdown-menu" role="menu">
-            <% Sidekiq::Web.custom_tabs.each do |title, url| %>
-              <li>
-                <a href="<%= root_path %><%= url %>"><%= t(title) %></a>
-              </li>
-            <% end %>
-          </ul>
-        </li>
-
         <% Sidekiq::Web.custom_tabs.each do |title, url| %>
           <li class="<%= current_path.start_with?(url) ? 'active' : '' %>" data-navbar="custom-tab">
             <a href="<%= root_path %><%= url %>"><%= t(title) %></a>
           </li>
         <% end %>
-      </ul>
-      <ul class="nav navbar-nav navbar-right navbar-livereload" data-navbar="static">
-        <li>
+
+        <li class="navbar-livereload">
           <div class="poll-wrapper">
             <%= erb :_poll_link %>
             <% if Sidekiq::Web.app_url %>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 5.2.2
+  version: 5.2.3
 platform: ruby
 authors:
 - Mike Perham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-17 00:00:00.000000000 Z
+date: 2018-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis