sidekiq 3.1.3 → 3.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4adde5768b38bd2da507698bc57358f36ce64041
-  data.tar.gz: 82b200a66bfcc8fcc46532b5b0d055ea87f91370
+  metadata.gz: 842995b5a36bd3651ba6ed63d3f0c2c5f99a2c1b
+  data.tar.gz: 4f362b15d3748d3df41dad8ba1364d967a5e37cd
 SHA512:
-  metadata.gz: 111778ad7a808b0a9b093969846037c03d88440d5dd9e6bcec4cb5196c8831b24a4e6d2bf425e11fbed02b4b61053c15f039d0392d4c9ee67a8278a18edf40c0
-  data.tar.gz: 055b58470ee8899a33e0467d6a4d11c202eed48db7c41d740aa469646ad5b075917f2f7bf11e320062d4d3c718065f15a66fbd44d67779e3d925b676259fdaff
+  metadata.gz: dc001192b00a22ad94893693543234b2542a6f91af74abc0a9963bf19bfd96c33babe084efcad2ce454f26bd78bdbbb356364fc86ba7d63bc5e0fb441a514a09
+  data.tar.gz: 7ea2ab5908163f70dc386b9ab89ce84321f3e93bf5bc4d20f0152ecc3a1ae67dba100abf088bcea4c9c163cf904623208877c16b993127d3e927038a0dafbfde

data/.travis.yml

@@ -4,9 +4,9 @@ services:
 rvm:
   - 1.9.3
   - jruby-19mode
-  - rbx
+  - rbx-2
   - 2.0.0
   - 2.1
 matrix:
   allow_failures:
-    - rvm: rbx
+    - rvm: rbx-2

data/3.0-Upgrade.md

@@ -29,7 +29,10 @@ changes a few data elements in Redis.  To upgrade cleanly:
 * Redis-to-Go is no longer transparently activated on Heroku so as to not play
   favorites with any particular Redis service. You need to set a config option
   for your app:
-  `heroku config:set REDIS_PROVIDER=REDISTOGO_URL`
+  `heroku config:set REDIS_PROVIDER=REDISTOGO_URL`.  You may also use 
+  the generic `REDIS_URL`.  See
+  [Advanced Options: Setting the Location of your Redis server][1]
+  for details.
 * Anyone using Airbrake, Honeybadger, Exceptional or ExceptionNotifier
   will need to update their error gem version to the latest to pull in
   Sidekiq support.  Sidekiq will not provide explicit support for these
@@ -63,3 +66,5 @@ end
 ```
 
 Your error handler must respond to `call(exception, context_hash)`.
+
+[1]: https://github.com/mperham/sidekiq/wiki/Advanced-Options#via-env-variable

data/Changes.md

@@ -1,3 +1,12 @@
+3.1.4
+-----------
+
+- Happy π release!
+- Self-tuning Scheduler polling, we use heartbeat info to better tune poll\_interval [#1630]
+- Remove all table column width rules, hopefully get better column formatting [#1747]
+- Handle edge case where YAML can't be decoded in dev mode [#1761]
+- Fix lingering jobs in Busy page on Heroku [#1764]
+
 3.1.3
 -----------
 

data/Rakefile

@@ -1,3 +1,4 @@
+require 'bundler/gem_tasks'
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |test|
   test.libs << 'test'

data/lib/sidekiq.rb

@@ -118,9 +118,7 @@ module Sidekiq
     Sidekiq::Logging.logger = log
   end
 
-  # The default poll interval is 15 seconds.  This should generally be set to
-  # (Sidekiq process count * 5).  So if you have a dozen Sidekiq processes, set
-  # poll_interval to 60.
+  # See sidekiq/scheduled.rb for an in-depth explanation of this value
   def self.poll_interval=(interval)
     self.options[:poll_interval] = interval
   end
@@ -149,20 +147,6 @@ module Sidekiq
     raise ArgumentError, "Invalid event name: #{event}" if !options[:lifecycle_events].keys.include?(event)
     options[:lifecycle_events][event] << block
   end
-
-  BANNER = %q{         s
-        ss
-   sss  sss         ss
-   s  sss s   ssss sss   ____  _     _      _    _
-   s     sssss ssss     / ___|(_) __| | ___| | _(_) __ _
-  s         sss         \___ \| |/ _` |/ _ \ |/ / |/ _` |
-  s sssss  s             ___) | | (_| |  __/   <| | (_| |
-  ss    s  s            |____/|_|\__,_|\___|_|\_\_|\__, |
-  s     s s                                           |_|
-        s s
-       sss
-       sss }
-
 end
 
 require 'sidekiq/extensions/class_methods'

data/lib/sidekiq/api.rb

@@ -206,8 +206,9 @@ module Sidekiq
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
       @klass ||= case klass
                  when /\ASidekiq::Extensions::Delayed/
-                   (target, method, _) = YAML.load(args[0])
-                   "#{target}.#{method}"
+                   safe_load(args[0], klass) do |target, method, _|
+                     "#{target}.#{method}"
+                   end
                  when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
                    args[0]
                  else
@@ -219,8 +220,9 @@ module Sidekiq
       # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
       @args ||= case klass
                 when /\ASidekiq::Extensions::Delayed/
-                  (_, _, arg) = YAML.load(args[0])
-                  arg
+                  safe_load(args[0], args) do |_, _, arg|
+                    arg
+                  end
                 when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
                   args[1..-1]
                 else
@@ -260,6 +262,19 @@ module Sidekiq
     def [](name)
       @item.__send__(:[], name)
     end
+
+    private
+
+    def safe_load(content, default)
+      begin
+        yield *YAML.load(content)
+      rescue ::ArgumentError => ex
+        # #1761 in dev mode, it's possible to have jobs enqueued which haven't been loaded into
+        # memory yet so the YAML can't be loaded.
+        Sidekiq.logger.warn "Unable to load YAML: #{ex.message}" unless Sidekiq.options[:environment] == 'development'
+        default
+      end
+    end
   end
 
   class SortedEntry < Job
@@ -286,32 +301,56 @@ module Sidekiq
     end
 
     def add_to_queue
-      Sidekiq.redis do |conn|
-        results = conn.multi do
-          conn.zrangebyscore('schedule', score, score)
-          conn.zremrangebyscore('schedule', score, score)
-        end.first
-        results.map do |message|
-          msg = Sidekiq.load_json(message)
-          Sidekiq::Client.push(msg)
-        end
+      remove_job do |message|
+        msg = Sidekiq.load_json(message)
+        Sidekiq::Client.push(msg)
       end
     end
 
     def retry
       raise "Retry not available on jobs which have not failed" unless item["failed_at"]
+      remove_job do |message|
+        msg = Sidekiq.load_json(message)
+        msg['retry_count'] = msg['retry_count'] - 1
+        Sidekiq::Client.push(msg)
+      end
+    end
+
+    private
+
+    def remove_job
       Sidekiq.redis do |conn|
         results = conn.multi do
           conn.zrangebyscore(parent.name, score, score)
           conn.zremrangebyscore(parent.name, score, score)
         end.first
-        results.map do |message|
-          msg = Sidekiq.load_json(message)
-          msg['retry_count'] = msg['retry_count'] - 1
-          Sidekiq::Client.push(msg)
+
+        if results.size == 1
+          yield results.first
+        else
+          # multiple jobs with the same score
+          # find the one with the right JID and push it
+          hash = results.group_by do |message|
+            if message.index(jid)
+              msg = Sidekiq.load_json(message)
+              msg['jid'] == jid
+            else
+              false
+            end
+          end
+          message = hash[true].first
+          yield message
+
+          # push the rest back onto the sorted set
+          conn.multi do
+            hash[false].each do |message|
+              conn.zadd(parent.name, score.to_f.to_s, message)
+            end
+          end
         end
       end
     end
+
   end
 
   class SortedSet

data/lib/sidekiq/cli.rb

@@ -91,13 +91,28 @@ module Sidekiq
       end
     end
 
+    def self.banner
+%q{         s
+          ss
+     sss  sss         ss
+     s  sss s   ssss sss   ____  _     _      _    _
+     s     sssss ssss     / ___|(_) __| | ___| | _(_) __ _
+    s         sss         \___ \| |/ _` |/ _ \ |/ / |/ _` |
+    s sssss  s             ___) | | (_| |  __/   <| | (_| |
+    ss    s  s            |____/|_|\__,_|\___|_|\_\_|\__, |
+    s     s s                                           |_|
+          s s
+         sss
+         sss }
+    end
+
     private
 
     def print_banner
       # Print logo and banner for development
       if environment == 'development' && $stdout.tty?
         puts "\e[#{31}m"
-        puts Sidekiq::BANNER
+        puts Sidekiq::CLI.banner
         puts "\e[0m"
       end
     end

data/lib/sidekiq/launcher.rb

@@ -83,7 +83,10 @@ module Sidekiq
 
     def stop_heartbeat
       Sidekiq.redis do |conn|
-        conn.srem('processes', identity)
+        conn.pipelined do
+          conn.srem('processes', identity)
+          conn.del("#{identity}:workers")
+        end
       end
     end
   end

data/lib/sidekiq/processor.rb

@@ -85,7 +85,7 @@ module Sidekiq
         Sidekiq.redis do |conn|
           conn.multi do
             conn.hmset("#{identity}:workers", thread_identity, hash)
-            conn.expire("#{identity}:workers", 60*60)
+            conn.expire("#{identity}:workers", 60*60*4)
           end
         end
       end

data/lib/sidekiq/scheduled.rb

@@ -5,7 +5,7 @@ require 'sidekiq/actor'
 module Sidekiq
   module Scheduled
 
-    POLL_INTERVAL = 15
+    INITIAL_WAIT = 10
 
     ##
     # The Poller checks Redis every N seconds for messages in the retry or scheduled
@@ -20,7 +20,7 @@ module Sidekiq
 
       def poll(first_time=false)
         watchdog('scheduling poller thread died!') do
-          add_jitter if first_time
+          initial_wait if first_time
 
           begin
             # A message's "score" in Redis is the time at which it should be processed.
@@ -51,19 +51,39 @@ module Sidekiq
             logger.error ex.backtrace.first
           end
 
-          after(poll_interval) { poll }
+          after(poll_interval * rand) { poll }
         end
       end
 
       private
 
+      # We do our best to tune poll_interval to the size of the active Sidekiq
+      # cluster.  If you have 30 processes and poll every 15 seconds, that means one
+      # Sidekiq is checking Redis every 0.5 seconds - way too often for most people
+      # and really bad if the retry or scheduled sets are large.
+      #
+      # Instead try to avoid polling more than once every 15 seconds.  If you have
+      # 30 Sidekiq processes, we'll set poll_interval to 30 * 15 * 2 or 900 seconds.
+      # To keep things statistically random, we'll sleep a random amount between
+      # 0 and 900 seconds for each poll or 450 seconds on average.  Otherwise restarting
+      # all your Sidekiq processes at the same time will lead to them all polling at
+      # the same time: the thundering herd problem.
+      #
+      # We only do this if poll_interval is unset (the default).
       def poll_interval
-        Sidekiq.options[:poll_interval] || POLL_INTERVAL
+        Sidekiq.options[:poll_interval] ||= begin
+          pcount = Sidekiq.redis {|c| c.scard('processes') } || 1
+          pcount * 15 * 2
+        end
       end
 
-      def add_jitter
+      def initial_wait
         begin
-          sleep(poll_interval * rand)
+          # Have all processes sleep between 10-15 seconds.  10 seconds
+          # to give time for the heartbeat to register and 5 random seconds
+          # to ensure they don't all hit Redis at the same time.
+          sleep(INITIAL_WAIT)
+          sleep(5 * rand)
         rescue Celluloid::Task::TerminatedError
           # Hit Ctrl-C when Sidekiq is finished booting and we have a chance
           # to get here.

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "3.1.3"
+  VERSION = "3.1.4"
 end

data/test/helper.rb

@@ -31,9 +31,9 @@ Sidekiq.logger.level = Logger::ERROR
 Sidekiq::Test = Minitest::Test
 
 require 'sidekiq/redis_connection'
-redis_url = ENV['REDIS_URL'] || 'redis://localhost/15'
-REDIS = Sidekiq::RedisConnection.create(:url => redis_url, :namespace => 'testy')
+REDIS_URL = ENV['REDIS_URL'] || 'redis://localhost/15'
+REDIS = Sidekiq::RedisConnection.create(:url => REDIS_URL, :namespace => 'testy')
 
 Sidekiq.configure_client do |config|
-  config.redis = { :url => redis_url, :namespace => 'testy' }
+  config.redis = { :url => REDIS_URL, :namespace => 'testy' }
 end

data/test/test_api.rb

@@ -219,6 +219,7 @@ class TestApi < Sidekiq::Test
     end
 
     it "can move scheduled job to queue" do
+      remain_id = ApiWorker.perform_in(100, 1, 'jason')
       job_id = ApiWorker.perform_in(100, 1, 'jason')
       job = Sidekiq::ScheduledSet.new.find_job(job_id)
       q = Sidekiq::Queue.new
@@ -226,8 +227,24 @@ class TestApi < Sidekiq::Test
       queued_job = q.find_job(job_id)
       refute_nil queued_job
       assert_equal queued_job.jid, job_id
+      assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
+      refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
+    end
+
+    it "handles multiple scheduled jobs when moving to queue" do
+      jids = Sidekiq::Client.push_bulk('class' => ApiWorker,
+                                       'args' => [[1, 'jason'], [2, 'jason']],
+                                       'at' => Time.now.to_f)
+      assert_equal 2, jids.size
+      (remain_id, job_id) = jids
       job = Sidekiq::ScheduledSet.new.find_job(job_id)
-      assert_nil job
+      q = Sidekiq::Queue.new
+      job.add_to_queue
+      queued_job = q.find_job(job_id)
+      refute_nil queued_job
+      assert_equal queued_job.jid, job_id
+      assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
+      refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
     end
 
     it 'can find job by id in sorted sets' do

data/test/test_fetch.rb

@@ -4,7 +4,7 @@ require 'sidekiq/fetch'
 class TestFetcher < Sidekiq::Test
   describe 'fetcher' do
     before do
-      Sidekiq.redis = { :namespace => 'fuzzy' }
+      Sidekiq.redis = { :url => REDIS_URL, :namespace => 'fuzzy' }
       Sidekiq.redis do |conn|
         conn.flushdb
         conn.rpush('queue:basic', 'msg')

data/test/test_manager.rb

@@ -115,7 +115,7 @@ class TestManager < Sidekiq::Test
           info = Sidekiq.redis { |c| c.hmget('identity', 'busy') }
           assert_equal ["1"], info
           expires = Sidekiq.redis { |c| c.pttl('identity') }
-          assert_in_delta 60000, expires, 2
+          assert_in_delta 60000, expires, 10
         end
       end
 

data/test/test_redis_connection.rb

@@ -70,6 +70,14 @@ class TestRedisConnection < Sidekiq::Test
 
   describe ".determine_redis_provider" do
 
+    before do
+      @old_env = ENV.to_hash
+    end
+
+    after do
+      ENV.update(@old_env)
+    end
+
     def with_env_var(var, uri, skip_provider=false)
       vars = ['REDISTOGO_URL', 'REDIS_PROVIDER', 'REDIS_URL'] - [var]
       vars.each do |v|

data/web/assets/stylesheets/application.css

@@ -197,18 +197,6 @@ td form {
   padding: 0px;
 }
 
-.table td {
-  /* Non standard for webkit */
-  word-break: break-word;
-
-  -webkit-hyphens: auto;
-  -moz-hyphens: auto;
-  hyphens: auto;
-
-  -ms-word-break: break-all;
-  word-break: break-all;
-}
-
 table .table-checkbox label {
   height: 100%;
   width: 100%;

data/web/views/morgue.erb

@@ -20,7 +20,7 @@
               <input type="checkbox" class="check_all" />
             </label>
           </th>
-          <th width="25%"><%= t('LastRetry') %></th>
+          <th><%= t('LastRetry') %></th>
           <th><%= t('Queue') %></th>
           <th><%= t('Job') %></th>
           <th><%= t('Arguments') %></th>

data/web/views/retries.erb

@@ -20,8 +20,8 @@
               <input type="checkbox" class="check_all" />
             </label>
           </th>
-          <th width="25%"><%= t('NextRetry') %></th>
-          <th width="11%"><%= t('RetryCount') %></th>
+          <th><%= t('NextRetry') %></th>
+          <th><%= t('RetryCount') %></th>
           <th><%= t('Queue') %></th>
           <th><%= t('Job') %></th>
           <th><%= t('Arguments') %></th>

data/web/views/scheduled.erb

@@ -19,8 +19,8 @@
           <th width="20px">
             <input type="checkbox" class="check_all" />
           </th>
-          <th width="25%"><%= t('When') %></th>
-          <th width="10%"><%= t('Queue') %></th>
+          <th><%= t('When') %></th>
+          <th><%= t('Queue') %></th>
           <th><%= t('Job') %></th>
           <th><%= t('Arguments') %></th>
         </tr>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 3.1.3
+  version: 3.1.4
 platform: ruby
 authors:
 - Mike Perham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-01 00:00:00.000000000 Z
+date: 2014-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis