sidekiq 3.3.4 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a6ca011daf81025502d9f9abfdf4e38fafd05d83
-  data.tar.gz: 3687865623e0ad65b98fcd7e5e576f540d80df23
+  metadata.gz: a49959a367b719b42c86d617948251db3a840b9b
+  data.tar.gz: c82ac2415a18b96ff62e0b66a399c0ba27602f08
 SHA512:
-  metadata.gz: 87bf6bf803ce2c24d2c424348fe83681c77dafda7206300c13ebda6c9aca5d36d1551642c68a0d2f37558540a64e8bd5aa5cfb6d95baaa7ce1fd08d62000ffdb
-  data.tar.gz: ddf4ccef0e503b5e59b874d8ec14ecb6d94721cf674f19385cb0a6e3d180c3605da63b03bb4752fd11c4da60be2d41f63f08c2f2a413e623d15bd929bc9210ff
+  metadata.gz: a09bdb137a14e2a24683edd6eac08bf60c1bcfa071bfc0dd1827bdbf8b06538083775b7cef09a01088383603ea8d994b277a9fd34fdbf2585e8fb8f104147a7c
+  data.tar.gz: 4e6417eadd49df60666dd944a55d8e515feaa3475209a9f69e77145e28d361d1664368b7cb5805f84dd6a5c2febd835b148bf28813f42a84c2a122f5e28439ca

data/3.0-Upgrade.md

@@ -60,7 +60,7 @@ if Sidekiq::VERSION < '3'
   end
 else
   Sidekiq.configure_server do |config|
-    config.error_handlers << Proc.new {|ex,context| MyErrorService.notify(ex, context) }
+    config.error_handlers << proc {|ex,context| MyErrorService.notify(ex, context) }
   end
 end
 ```

data/Changes.md

@@ -1,3 +1,20 @@
+3.4.0
+-----------
+
+- Set a `created_at` attribute when jobs are created, set `enqueued_at` only
+  when they go into a queue. Fixes invalid latency calculations with scheduled jobs.
+  [#2373, mrsimo]
+- Don't log timestamp on Heroku [#2343]
+- Run `shutdown` event handlers in reverse order of definition [#2374]
+- Rename and rework `poll_interval` to be simpler, more predictable [#2317, cainlevy]
+  The new setting is `average_scheduled_poll_interval`.  To configure
+  Sidekiq to look for scheduled jobs every 5 seconds, just set it to 5.
+```ruby
+Sidekiq.configure_server do |config|
+  config.average_scheduled_poll_interval = 5
+end
+```
+
 3.3.4
 -----------
 
@@ -205,7 +222,7 @@ end
   occur anywhere within Sidekiq, not just within middleware.
 ```ruby
 Sidekiq.configure_server do |config|
-  config.error_handlers << Proc.new {|ex,ctx| ... }
+  config.error_handlers << proc {|ex,ctx| ... }
 end
 ```
 - **Process Heartbeat** - each Sidekiq process will ping Redis every 5

data/Gemfile

@@ -12,7 +12,7 @@ platforms :ruby do
   gem 'sqlite3'
 end
 
-platforms :mri_21 do
+platforms :mri do
   gem 'pry-byebug'
 end
 

data/Pro-Changes.md

@@ -3,6 +3,18 @@ Sidekiq Pro Changelog
 
 Please see [http://sidekiq.org/pro](http://sidekiq.org/pro) for more details and how to buy.
 
+HEAD
+-----------
+
+- Reliable push now only catches Redis exceptions [#2307]
+
+2.0.3
+-----------
+
+- Display Batch callback data on the Batch details page. [#2347]
+- Fix incompatibility with Pro Web and Rack middleware. [#2344] Thank
+  you to Jason Clark for the tip on how to fix it.
+
 2.0.2
 -----------
 
@@ -12,9 +24,12 @@ Please see [http://sidekiq.org/pro](http://sidekiq.org/pro) for more details and
 POOL1 = ConnectionPool.new { Redis.new(:url => "redis://localhost:6379/0") }
 POOL2 = ConnectionPool.new { Redis.new(:url => "redis://localhost:6378/0") }
 
-mount Sidekiq::Pro::Web.with(redis_pool: POOL1) => '/sidekiq1'
-mount Sidekiq::Pro::Web.with(redis_pool: POOL2) => '/sidekiq2'
+mount Sidekiq::Pro::Web => '/sidekiq' # default
+mount Sidekiq::Pro::Web.with(redis_pool: POOL1), at: '/sidekiq1', as: 'sidekiq1' # shard1
+mount Sidekiq::Pro::Web.with(redis_pool: POOL2), at: '/sidekiq2', as: 'sidekiq2' # shard2
 ```
+- **SECURITY** Fix batch XSS in error data.  Thanks to moneybird.com for
+  reporting the issue.
 
 2.0.1
 -----------

data/lib/sidekiq.rb

@@ -20,6 +20,8 @@ module Sidekiq
     require: '.',
     environment: nil,
     timeout: 8,
+    poll_interval_average: nil,
+    average_scheduled_poll_interval: 15,
     error_handlers: [],
     lifecycle_events: {
       startup: [],
@@ -36,7 +38,7 @@ module Sidekiq
   }
 
   def self.❨╯°□°❩╯︵┻━┻
-    puts "Calm down, bro"
+    puts "Calm down, yo."
   end
 
   def self.options
@@ -127,15 +129,29 @@ module Sidekiq
     Sidekiq::Logging.logger = log
   end
 
+  # When set, overrides Sidekiq.options[:average_scheduled_poll_interval] and sets
+  # the average interval that this process will delay before checking for
+  # scheduled jobs or job retries that are ready to run.
+  #
   # See sidekiq/scheduled.rb for an in-depth explanation of this value
   def self.poll_interval=(interval)
-    self.options[:poll_interval] = interval
+    $stderr.puts "DEPRECATION: `config.poll_interval = #{interval}` will be removed in Sidekiq 4. Please update to `config.average_scheduled_poll_interval = #{interval}`."
+    self.options[:poll_interval_average] = interval
+  end
+
+  # How frequently Redis should be checked by a random Sidekiq process for
+  # scheduled and retriable jobs. Each individual process will take turns by
+  # waiting some multiple of this value.
+  #
+  # See sidekiq/scheduled.rb for an in-depth explanation of this value
+  def self.average_scheduled_poll_interval=(interval)
+    self.options[:average_scheduled_poll_interval] = interval
   end
 
   # Register a proc to handle any error which occurs within the Sidekiq process.
   #
   #   Sidekiq.configure_server do |config|
-  #     config.error_handlers << Proc.new {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
+  #     config.error_handlers << proc {|ex,ctx_hash| MyErrorService.notify(ex, ctx_hash) }
   #   end
   #
   # The default error handler logs errors to Sidekiq.logger.

data/lib/sidekiq/api.rb

@@ -315,12 +315,16 @@ module Sidekiq
       Time.at(@item['enqueued_at'] || 0).utc
     end
 
+    def created_at
+      Time.at(@item['created_at'] || @item['enqueued_at'] || 0).utc
+    end
+
     def queue
       @queue
     end
 
     def latency
-      Time.now.to_f - @item['enqueued_at']
+      Time.now.to_f - (@item['enqueued_at'] || @item['created_at'])
     end
 
     ##

data/lib/sidekiq/cli.rb

@@ -90,7 +90,7 @@ module Sidekiq
       rescue Interrupt
         logger.info 'Shutting down'
         launcher.stop
-        fire_event(:shutdown)
+        fire_event(:shutdown, true)
         # Explicitly exit so busy Processor threads can't block
         # process shutdown.
         exit(0)
@@ -125,7 +125,7 @@ module Sidekiq
       when 'USR1'
         Sidekiq.logger.info "Received USR1, no longer accepting new work"
         launcher.manager.async.stop
-        fire_event(:quiet)
+        fire_event(:quiet, true)
       when 'USR2'
         if Sidekiq.options[:logfile]
           Sidekiq.logger.info "Received USR2, reopening log file"
@@ -269,6 +269,10 @@ module Sidekiq
         logger.info @parser
         die(1)
       end
+
+      [:concurrency, :timeout].each do |opt|
+        raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.has_key?(opt) && options[opt].to_i <= 0
+      end
     end
 
     def parse_options(argv)

data/lib/sidekiq/client.rb

@@ -191,7 +191,11 @@ module Sidekiq
         end)
       else
         q = payloads.first['queue']
-        to_push = payloads.map { |entry| Sidekiq.dump_json(entry) }
+        now = Time.now.to_f
+        to_push = payloads.map do |entry|
+          entry['enqueued_at'.freeze] = now
+          Sidekiq.dump_json(entry)
+        end
         conn.sadd('queues'.freeze, q)
         conn.lpush("queue:#{q}", to_push)
       end
@@ -217,7 +221,7 @@ module Sidekiq
       item['class'.freeze] = item['class'.freeze].to_s
       item['queue'.freeze] = item['queue'.freeze].to_s
       item['jid'.freeze] ||= SecureRandom.hex(12)
-      item['enqueued_at'.freeze] ||= Time.now.to_f
+      item['created_at'.freeze] ||= Time.now.to_f
       item
     end
 

data/lib/sidekiq/exception_handler.rb

@@ -6,7 +6,7 @@ module Sidekiq
     class Logger
       def call(ex, ctxHash)
         Sidekiq.logger.warn(ctxHash) if !ctxHash.empty?
-        Sidekiq.logger.warn ex
+        Sidekiq.logger.warn "#{ex.class.name}: #{ex.message}"
         Sidekiq.logger.warn ex.backtrace.join("\n") unless ex.backtrace.nil?
       end
 

data/lib/sidekiq/logging.rb

@@ -18,6 +18,12 @@ module Sidekiq
       end
     end
 
+    class WithoutTimestamp < Pretty
+      def call(severity, time, program_name, message)
+        "#{::Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+      end
+    end
+
     def self.with_context(msg)
       Thread.current[:sidekiq_context] ||= []
       Thread.current[:sidekiq_context] << msg
@@ -30,7 +36,7 @@ module Sidekiq
       oldlogger = defined?(@logger) ? @logger : nil
       @logger = Logger.new(log_target)
       @logger.level = Logger::INFO
-      @logger.formatter = Pretty.new
+      @logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
       oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
       @logger
     end

data/lib/sidekiq/manager.rb

@@ -134,12 +134,18 @@ module Sidekiq
       @threads[proxy_id] = thr
     end
 
+    PROCTITLES = [
+      proc { 'sidekiq'.freeze },
+      proc { Sidekiq::VERSION },
+      proc { |mgr, data| data['tag'] },
+      proc { |mgr, data| "[#{mgr.busy.size} of #{data['concurrency']} busy]" },
+      proc { |mgr, data| "stopping" if mgr.stopped? },
+    ]
+
     def heartbeat(key, data, json)
-      proctitle = ['sidekiq', Sidekiq::VERSION]
-      proctitle << data['tag'] unless data['tag'].empty?
-      proctitle << "[#{@busy.size} of #{data['concurrency']} busy]"
-      proctitle << 'stopping' if stopped?
-      $0 = proctitle.join(' ')
+      results = PROCTITLES.map {|x| x.(self, data) }
+      results.compact!
+      $0 = results.join(' ')
 
       ❤(key, json)
       after(5) do
@@ -147,6 +153,10 @@ module Sidekiq
       end
     end
 
+    def stopped?
+      @done
+    end
+
     private
 
     def ❤(key, json)
@@ -206,10 +216,6 @@ module Sidekiq
       @fetcher.async.fetch
     end
 
-    def stopped?
-      @done
-    end
-
     def shutdown
       requeue
       @finished.signal

data/lib/sidekiq/processor.rb

@@ -132,7 +132,7 @@ module Sidekiq
 
     # If an exception occurs in the block passed to this method, that block will be retried up to max_retries times.
     # All exceptions will be swallowed and logged.
-    def retry_and_suppress_exceptions(max_retries = 2)
+    def retry_and_suppress_exceptions(max_retries = 5)
       retry_count = 0
       begin
         yield
@@ -140,12 +140,16 @@ module Sidekiq
         retry_count += 1
         if retry_count <= max_retries
           Sidekiq.logger.debug {"Suppressing and retrying error: #{e.inspect}"}
-          sleep(1)
+          pause_for_recovery(retry_count)
           retry
         else
           handle_exception(e, { :message => "Exhausted #{max_retries} retries"})
         end
       end
     end
+
+    def pause_for_recovery(retry_count)
+      sleep(retry_count)
+    end
   end
 end

data/lib/sidekiq/scheduled.rb

@@ -60,35 +60,41 @@ module Sidekiq
             logger.error ex.backtrace.first
           end
 
-          # Randomizing scales the interval by half since
-          # on average calling `rand` returns 0.5.
-          # We make up for this by doubling the interval
-          after(poll_interval * 2 * rand) { poll }
+          after(random_poll_interval) { poll }
         end
       end
 
       private
 
-      # We do our best to tune poll_interval to the size of the active Sidekiq
+      # Calculates a random interval that is ±50% the desired average.
+      def random_poll_interval
+        poll_interval_average * rand + poll_interval_average.to_f / 2
+      end
+
+      # We do our best to tune the poll interval to the size of the active Sidekiq
       # cluster.  If you have 30 processes and poll every 15 seconds, that means one
       # Sidekiq is checking Redis every 0.5 seconds - way too often for most people
       # and really bad if the retry or scheduled sets are large.
       #
       # Instead try to avoid polling more than once every 15 seconds.  If you have
-      # 30 Sidekiq processes, we'll set poll_interval to 30 * 15 * 2 or 900 seconds.
+      # 30 Sidekiq processes, we'll poll every 30 * 15 or 450 seconds.
       # To keep things statistically random, we'll sleep a random amount between
-      # 0 and 900 seconds for each poll or 450 seconds on average.  Otherwise restarting
+      # 225 and 675 seconds for each poll or 450 seconds on average.  Otherwise restarting
       # all your Sidekiq processes at the same time will lead to them all polling at
       # the same time: the thundering herd problem.
       #
       # We only do this if poll_interval is unset (the default).
-      def poll_interval
-        Sidekiq.options[:poll_interval] ||= begin
-          ps = Sidekiq::ProcessSet.new
-          pcount = ps.size
-          pcount = 1 if pcount == 0
-          pcount * 15
-        end
+      def poll_interval_average
+        Sidekiq.options[:poll_interval_average] ||= scaled_poll_interval
+      end
+
+      # Calculates an average poll interval based on the number of known Sidekiq processes.
+      # This minimizes a single point of failure by dispersing check-ins but without taxing
+      # Redis if you run many Sidekiq processes.
+      def scaled_poll_interval
+        pcount = Sidekiq::ProcessSet.new.size
+        pcount = 1 if pcount == 0
+        pcount * Sidekiq.options[:average_scheduled_poll_interval]
       end
 
       def initial_wait
@@ -96,7 +102,7 @@ module Sidekiq
           # Have all processes sleep between 5-15 seconds.  10 seconds
           # to give time for the heartbeat to register (if the poll interval is going to be calculated by the number
           # of workers), and 5 random seconds to ensure they don't all hit Redis at the same time.
-          sleep(INITIAL_WAIT) unless Sidekiq.options[:poll_interval]
+          sleep(INITIAL_WAIT) unless Sidekiq.options[:poll_interval_average]
           sleep(5 * rand)
         rescue Celluloid::Task::TerminatedError
           # Hit Ctrl-C when Sidekiq is finished booting and we have a chance

data/lib/sidekiq/testing.rb

@@ -152,6 +152,7 @@ module Sidekiq
         while job = jobs.shift do
           worker = new
           worker.jid = job['jid']
+          worker.bid = job['bid'] if worker.respond_to?(:bid=)
           execute_job(worker, job['args'])
         end
       end
@@ -162,6 +163,7 @@ module Sidekiq
         job = jobs.shift
         worker = new
         worker.jid = job['jid']
+        worker.bid = job['bid'] if worker.respond_to?(:bid=)
         execute_job(worker, job['args'])
       end
 

data/lib/sidekiq/util.rb

@@ -39,8 +39,10 @@ module Sidekiq
       @@identity ||= "#{hostname}:#{$$}:#{process_nonce}"
     end
 
-    def fire_event(event)
-      Sidekiq.options[:lifecycle_events][event].each do |block|
+    def fire_event(event, reverse=false)
+      arr = Sidekiq.options[:lifecycle_events][event]
+      arr.reverse! if reverse
+      arr.each do |block|
         begin
           block.call
         rescue => ex

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "3.3.4"
+  VERSION = "3.4.0"
 end

data/lib/sidekiq/web.rb

@@ -12,8 +12,8 @@ module Sidekiq
     include Sidekiq::Paginator
 
     set :root, File.expand_path(File.dirname(__FILE__) + "/../../web")
-    set :public_folder, Proc.new { "#{root}/assets" }
-    set :views, Proc.new { "#{root}/views" }
+    set :public_folder, proc { "#{root}/assets" }
+    set :views, proc { "#{root}/views" }
     set :locales, ["#{root}/locales"]
 
     helpers WebHelpers

data/lib/sidekiq/web_helpers.rb

@@ -65,6 +65,7 @@ module Sidekiq
         locale = 'en'.freeze
         languages = request.env['HTTP_ACCEPT_LANGUAGE'.freeze] || 'en'.freeze
         languages.downcase.split(','.freeze).each do |lang|
+          next if lang == '*'.freeze
           lang = lang.split(';'.freeze)[0]
           break locale = lang if find_locale_files(lang).any?
         end

data/test/test_manager.rb

@@ -110,11 +110,13 @@ class TestManager < Sidekiq::Test
 
       describe 'when manager is active' do
         before do
+          Sidekiq::Manager::PROCTITLES << proc { "xyz" }
           @mgr.heartbeat('identity', heartbeat_data, Sidekiq.dump_json(heartbeat_data))
+          Sidekiq::Manager::PROCTITLES.pop
         end
 
         it 'sets useful info to proctitle' do
-          assert_equal "sidekiq #{Sidekiq::VERSION} myapp [1 of 3 busy]", $0
+          assert_equal "sidekiq #{Sidekiq::VERSION} myapp [1 of 3 busy] xyz", $0
         end
 
         it 'stores process info in redis' do