sidekiq 2.17.6 → 2.17.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d42c2e9a872cd5b00e30147f9d72aa587dc2a33
-  data.tar.gz: 0d8a3223a321ae8aff8da8c468ca43a14bd0a851
+  metadata.gz: 37ea727f6e186178793472075b5b8c98aa1ff6fa
+  data.tar.gz: d95a324293b763a29f291f69d257e79f37468ba8
 SHA512:
-  metadata.gz: 042f1408d57e3327c9391543e2bbf0e4d6f02ba7d7f3db9f7beccb4d91ef54d165d14b210b0b0d2b3b6be69e6212e1f72fc51648d3a71fb17959dbfd78d08d29
-  data.tar.gz: e663acb2a1fbf05ef5ca4ccae9adc2d9704610f3e281be4072a1621538b2b0507f9bc37958bb397eb3af45af6df12d4b4db76dd7da605513adb10857f1f2863a
+  metadata.gz: ac8aaddac1fbc4312444129640e4137b5a93772f7246241af77e05a10d05a509ec342cbce8ae1351689fac0b5fdaffbbde36331b936c5fd2f62684c6bb2b1cde
+  data.tar.gz: 7c4ee6d1f8d8cf1c6aba0f74ef2db62a9f59e17847094d648af5b8fc75e5e31aeba0d6e1c1d529118f79f55adf66ce56e2915da4e7758d64ef8afcfc8b1d425e

data/Changes.md

@@ -1,3 +1,11 @@
+2.17.7
+-----------
+
+- Auto-prune jobs older than one hour from the Workers page [#1508]
+- Add Sidekiq::Workers#prune which can perform the auto-pruning.
+- Fix issue where a job could be lost when an exception occurs updating
+  Redis stats before the job executes [#1511]
+
 2.17.6
 -----------
 

data/lib/sidekiq/api.rb

@@ -424,9 +424,10 @@ module Sidekiq
       Sidekiq.redis do |conn|
         workers = conn.smembers("workers")
         workers.each do |w|
-          msg, time = conn.mget("worker:#{w}", "worker:#{w}:started")
-          next unless msg
-          block.call(w, Sidekiq.load_json(msg), time)
+          json = conn.get("worker:#{w}")
+          next unless json
+          msg = Sidekiq.load_json(json)
+          block.call(w, msg, Time.at(msg['run_at']).to_s)
         end
       end
     end
@@ -436,6 +437,36 @@ module Sidekiq
         conn.scard("workers")
       end.to_i
     end
+
+    # Prune old worker entries from the Busy set.  Worker entries
+    # can be orphaned if Sidekiq hard crashes while processing jobs.
+    # Default is to delete worker entries older than one hour.
+    #
+    # Returns the number of records removed.
+    def prune(older_than=60*60)
+      to_rem = []
+      Sidekiq.redis do |conn|
+        conn.smembers('workers').each do |w|
+          msg = conn.get("worker:#{w}")
+          if !msg
+            to_rem << w
+          else
+            m = Sidekiq.load_json(msg)
+            run_at = Time.at(m['run_at'])
+            # prune jobs older than one hour
+            if run_at < (Time.now - older_than)
+              to_rem << w
+            else
+            end
+          end
+        end
+      end
+
+      if to_rem.size > 0
+        Sidekiq.redis { |conn| conn.srem('workers', to_rem) }
+      end
+      to_rem.size
+    end
   end
 
 end

data/lib/sidekiq/processor.rb

@@ -92,38 +92,45 @@ module Sidekiq
     end
 
     def stats(worker, msg, queue)
-      redis do |conn|
-        conn.multi do
-          conn.sadd('workers', identity)
-          conn.setex("worker:#{identity}:started", EXPIRY, Time.now.to_s)
-          hash = {:queue => queue, :payload => msg, :run_at => Time.now.to_i }
-          conn.setex("worker:#{identity}", EXPIRY, Sidekiq.dump_json(hash))
+      # Do not conflate errors from the job with errors caused by updating stats so calling code can react appropriately
+      retry_and_suppress_exceptions do
+        redis do |conn|
+          conn.multi do
+            conn.sadd('workers', identity)
+            conn.setex("worker:#{identity}:started", EXPIRY, Time.now.to_s)
+            hash = {:queue => queue, :payload => msg, :run_at => Time.now.to_i }
+            conn.setex("worker:#{identity}", EXPIRY, Sidekiq.dump_json(hash))
+          end
         end
       end
 
       begin
         yield
       rescue Exception
-        redis do |conn|
-          failed = "stat:failed:#{Time.now.utc.to_date}"
-          result = conn.multi do
-            conn.incrby("stat:failed", 1)
-            conn.incrby(failed, 1)
+        retry_and_suppress_exceptions do
+          redis do |conn|
+            failed = "stat:failed:#{Time.now.utc.to_date}"
+            result = conn.multi do
+              conn.incrby("stat:failed", 1)
+              conn.incrby(failed, 1)
+            end
+            conn.expire(failed, STATS_TIMEOUT) if result.last == 1
           end
-          conn.expire(failed, STATS_TIMEOUT) if result.last == 1
         end
         raise
       ensure
-        redis do |conn|
-          processed = "stat:processed:#{Time.now.utc.to_date}"
-          result = conn.multi do
-            conn.srem("workers", identity)
-            conn.del("worker:#{identity}")
-            conn.del("worker:#{identity}:started")
-            conn.incrby("stat:processed", 1)
-            conn.incrby(processed, 1)
+        retry_and_suppress_exceptions do
+          redis do |conn|
+            processed = "stat:processed:#{Time.now.utc.to_date}"
+            result = conn.multi do
+              conn.srem("workers", identity)
+              conn.del("worker:#{identity}")
+              conn.del("worker:#{identity}:started")
+              conn.incrby("stat:processed", 1)
+              conn.incrby(processed, 1)
+            end
+            conn.expire(processed, STATS_TIMEOUT) if result.last == 1
           end
-          conn.expire(processed, STATS_TIMEOUT) if result.last == 1
         end
       end
     end
@@ -137,5 +144,23 @@ module Sidekiq
     def cloned(ary)
       Marshal.load(Marshal.dump(ary))
     end
+
+    # If an exception occurs in the block passed to this method, that block will be retried up to max_retries times.
+    # All exceptions will be swallowed and logged.
+    def retry_and_suppress_exceptions(max_retries = 2)
+      retry_count = 0
+      begin
+        yield
+      rescue => e
+        retry_count += 1
+        if retry_count <= max_retries
+          Sidekiq.logger.debug {"Suppressing and retrying error: #{e.inspect}"}
+          sleep(1)
+          retry
+        else
+          Sidekiq.logger.info {"Exhausted #{max_retries} retries due to Redis timeouts: #{e.inspect}"}
+        end
+      end
+    end
   end
 end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.17.6"
+  VERSION = "2.17.7"
 end

data/lib/sidekiq/web.rb

@@ -38,7 +38,7 @@ module Sidekiq
     end
 
     get "/workers" do
-      erb :index
+      erb :workers
     end
 
     get "/queues" do

data/lib/sidekiq/web_helpers.rb

@@ -49,21 +49,9 @@ module Sidekiq
 
     def workers
       @workers ||= begin
-        to_rem = []
-        workers = Sidekiq.redis do |conn|
-          conn.smembers('workers').map do |w|
-            msg = conn.get("worker:#{w}")
-            msg ? [w, Sidekiq.load_json(msg)] : (to_rem << w; nil)
-          end.compact.sort { |x| x[1] ? -1 : 1 }
+        Sidekiq::Workers.new.tap do |w|
+          w.prune
         end
-
-        # Detect and clear out any orphaned worker records.
-        # These can be left in Redis if Sidekiq crashes hard
-        # while processing jobs.
-        if to_rem.size > 0
-          Sidekiq.redis { |conn| conn.srem('workers', to_rem) }
-        end
-        workers
       end
     end
 

data/test/test_api.rb

@@ -362,6 +362,22 @@ class TestApi < Sidekiq::Test
         assert_equal 'default', y['queue']
         assert_equal Time.now.year, DateTime.parse(z).year
       end
+
+      s = '12346'
+      data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => (Time.now.to_i - 2*60*60) })
+      Sidekiq.redis do |c|
+        c.multi do
+          c.sadd('workers', s)
+          c.set("worker:#{s}", data)
+          c.set("worker:#{s}:started", Time.now.to_s)
+          c.sadd('workers', '123457')
+        end
+      end
+
+      assert_equal 3, w.size
+      count = w.prune
+      assert_equal 1, w.size
+      assert_equal 2, count
     end
 
     it 'can reschedule jobs' do

data/web/views/_workers.erb

@@ -6,7 +6,7 @@
     <th><%= t('Arguments') %></th>
     <th><%= t('Started') %></th>
   </thead>
-  <% workers.each_with_index do |(worker, msg), index| %>
+  <% workers.each_with_index do |(worker, msg, run_at), index| %>
     <tr>
       <td><%= worker %></td>
       <td>
@@ -16,7 +16,7 @@
       <td>
         <div class="args"><%= display_args(msg['payload']['args']) %></div>
       </td>
-      <td><%= relative_time(msg['run_at'].is_a?(Numeric) ? Time.at(msg['run_at']) : Time.parse(msg['run_at'])) %></td>
+      <td><%= relative_time(run_at.is_a?(String) ? DateTime.parse(run_at) : run_at) %></td>
     </tr>
   <% end %>
 </table>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 2.17.6
+  version: 2.17.7
 platform: ruby
 authors:
 - Mike Perham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-19 00:00:00.000000000 Z
+date: 2014-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -310,7 +310,6 @@ files:
 - web/views/_summary.erb
 - web/views/_workers.erb
 - web/views/dashboard.erb
-- web/views/index.erb
 - web/views/layout.erb
 - web/views/queue.erb
 - web/views/queues.erb
@@ -318,6 +317,7 @@ files:
 - web/views/retry.erb
 - web/views/scheduled.erb
 - web/views/scheduled_job_info.erb
+- web/views/workers.erb
 homepage: http://sidekiq.org
 licenses:
 - LGPL-3.0