sidekiq 1.1.3 → 1.1.4

data/Changes.md

@@ -1,3 +1,11 @@
+1.1.4
+-----------
+
+- Add 24 hr expiry for basic keys set in Redis, to avoid any possible leaking.
+- Only register workers in Redis while working, to avoid lingering
+  workers [#156]
+- Speed up shutdown significantly.
+
 1.1.3
 -----------
 

data/lib/sidekiq/manager.rb

@@ -39,7 +39,7 @@ module Sidekiq
         timeout = options[:timeout]
 
         @done = true
-        @fetcher.terminate if @fetcher.alive?
+        @fetcher.terminate! if @fetcher.alive?
 
         logger.info { "Shutting down #{@ready.size} quiet workers" }
         @ready.each { |x| x.terminate if x.alive? }
@@ -155,7 +155,7 @@ module Sidekiq
     end
 
     def procline
-      $0 = "sidekiq #{Sidekiq::VERSION} [#{@busy.size} of #{@count} busy]"
+      $0 = "sidekiq #{Sidekiq::VERSION} [#{@busy.size} of #{@count} busy]#{stopped? ? ' stopping' : ''}"
       after(5) { procline }
     end
   end

data/lib/sidekiq/processor.rb

@@ -23,7 +23,6 @@ module Sidekiq
 
     def initialize(boss)
       @boss = boss
-      redis {|x| x.sadd('workers', self) }
     end
 
     def process(msg, queue)
@@ -53,9 +52,10 @@ module Sidekiq
     def stats(worker, msg, queue)
       redis do |conn|
         conn.multi do
-          conn.set("worker:#{self}:started", Time.now.to_s)
+          conn.sadd('workers', self)
+          conn.setex("worker:#{self}:started", DEFAULT_EXPIRY, Time.now.to_s)
           hash = {:queue => queue, :payload => msg, :run_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z")}
-          conn.set("worker:#{self}", Sidekiq.dump_json(hash))
+          conn.setex("worker:#{self}", DEFAULT_EXPIRY, Sidekiq.dump_json(hash))
         end
       end
 
@@ -69,13 +69,13 @@ module Sidekiq
           conn.multi do
             conn.incrby("stat:failed", 1)
             conn.del("stat:processed:#{self}")
-            conn.srem("workers", self)
           end
         end
         raise
       ensure
         redis do |conn|
           conn.multi do
+            conn.srem("workers", self)
             conn.del("worker:#{self}")
             conn.del("worker:#{self}:started")
             conn.incrby("stat:processed", 1)

data/lib/sidekiq/util.rb

@@ -7,6 +7,8 @@ module Sidekiq
   #
   module Util
 
+    DEFAULT_EXPIRY = 24 * 60 * 60
+
     class Pretty < Logger::Formatter
       # Provide a call() method that returns the formatted message.
       def call(severity, time, program_name, message)

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "1.1.3"
+  VERSION = "1.1.4"
 end

data/lib/sidekiq/web.rb

@@ -52,9 +52,8 @@ module Sidekiq
           Sidekiq.redis do |conn|
             conn.smembers('workers').map do |w|
               msg = conn.get("worker:#{w}")
-              msg = Sidekiq.load_json(msg) if msg
-              [w, msg]
-            end.sort { |x| x[1] ? -1 : 1 }
+              msg ? [w, Sidekiq.load_json(msg)] : nil
+            end.compact.sort { |x| x[1] ? -1 : 1 }
           end
         end
       end
@@ -102,8 +101,7 @@ module Sidekiq
       end
 
       def current_status
-        return 'down' if workers.size == 0
-        return 'idle' if workers.size > 0 && workers.map { |x| x[1] }.compact.size == 0
+        return 'idle' if workers.size == 0
         return 'active'
       end
 

data/test/test_manager.rb

@@ -38,6 +38,13 @@ class TestManager < MiniTest::Unit::TestCase
       result = q.timed_pop(1.0)
       assert_equal 'done', result
       mgr.stop
+      mgr.terminate
+
+      # Gross bloody hack because I can't get the actor threads
+      # to shut down cleanly in the test.  Need @bascule's help here.
+      (Thread.list - [Thread.current]).each do |t|
+        t.raise Interrupt
+      end
     end
   end
 end

data/test/test_stats.rb

@@ -29,11 +29,6 @@ class TestStats < MiniTest::Unit::TestCase
         processor = Sidekiq::Processor.new(boss)
         boss.expect(:processor_done!, nil, [processor])
 
-        # adds to the workers set upon initialize
-        set = conn.smembers('workers')
-        assert_equal 1, set.size
-        assert_match(/#{Regexp.escape(`hostname`.strip)}/, set.first)
-
         assert_equal 0, conn.get('stat:failed').to_i
         assert_equal 0, conn.get('stat:processed').to_i
         assert_equal 0, conn.get("stat:processed:#{processor}").to_i
@@ -42,9 +37,6 @@ class TestStats < MiniTest::Unit::TestCase
         processor.process(msg, 'xyzzy')
         processor.process(msg, 'xyzzy')
 
-        set = conn.smembers('workers')
-        assert_equal 1, set.size
-        assert_match(/#{Regexp.escape(`hostname`.strip)}/, set.first)
         assert_equal 0, conn.get('stat:failed').to_i
         assert_equal 3, conn.get('stat:processed').to_i
         assert_equal 3, conn.get("stat:processed:#{processor}").to_i
@@ -61,15 +53,12 @@ class TestStats < MiniTest::Unit::TestCase
         assert_equal 0, conn.get('stat:processed').to_i
 
         processor = Sidekiq::Processor.new(boss)
-        assert_equal 1, conn.smembers('workers').size
 
         pstr = processor.to_s
         assert_raises RuntimeError do
           processor.process(msg, 'xyzzy')
         end
 
-        set = conn.smembers('workers')
-        assert_equal 0, set.size
         assert_equal 1, conn.get('stat:failed').to_i
         assert_equal 1, conn.get('stat:processed').to_i
         assert_equal nil, conn.get("stat:processed:#{pstr}")

data/test/test_web.rb

@@ -27,14 +27,14 @@ class TestWeb < MiniTest::Unit::TestCase
     it 'shows active queues' do
       get '/'
       assert_equal 200, last_response.status
-      assert_match last_response.body, /Sidekiq is down/
+      assert_match last_response.body, /Sidekiq is idle/
       refute_match last_response.body, /default/
 
       assert WebWorker.perform_async(1, 2)
 
       get '/'
       assert_equal 200, last_response.status
-      assert_match last_response.body, /Sidekiq is down/
+      assert_match last_response.body, /Sidekiq is idle/
       assert_match last_response.body, /default/
       refute_match last_response.body, /foo/
 
@@ -42,7 +42,7 @@ class TestWeb < MiniTest::Unit::TestCase
 
       get '/'
       assert_equal 200, last_response.status
-      assert_match last_response.body, /Sidekiq is down/
+      assert_match last_response.body, /Sidekiq is idle/
       assert_match last_response.body, /default/
       assert_match last_response.body, /foo/
     end

data/web/views/index.slim

@@ -3,13 +3,13 @@
   h1 Sidekiq is #{current_status}
   p Processed: #{processed}
   p Failed: #{failed}
-  p Workers: #{workers.size}
+  p Busy Workers: #{workers.size}
   p Retries Pending: #{retry_count}
 
 .tabbable
   ul.nav.nav-tabs
     li.active
-      a href="#workers" data-toggle="tab" Workers
+      a href="#workers" data-toggle="tab" Busy Workers
     li
       a href="#queues" data-toggle="tab" Queues
     li
@@ -26,13 +26,10 @@
         - workers.each do |(worker, msg)|
           tr
             td= worker
-            - if msg
-              td= msg['queue']
-              td= msg['payload']['class']
-              td= msg['payload']['args'].inspect[0..100]
-              td== relative_time(Time.parse(msg['run_at']))
-            - else
-              td colspan=4 Idle
+            td= msg['queue']
+            td= msg['payload']['class']
+            td= msg['payload']['args'].inspect[0..100]
+            td== relative_time(Time.parse(msg['run_at']))
       form action="#{root_path}reset" method="post"
         button.btn type="submit" Clear worker list
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.1.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-23 00:00:00.000000000 Z
+date: 2012-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70116788678440 !ruby/object:Gem::Requirement
+  requirement: &70161328001220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70116788678440
+  version_requirements: *70161328001220
 - !ruby/object:Gem::Dependency
   name: redis-namespace
-  requirement: &70116788677560 !ruby/object:Gem::Requirement
+  requirement: &70161328000340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70116788677560
+  version_requirements: *70161328000340
 - !ruby/object:Gem::Dependency
   name: connection_pool
-  requirement: &70116788676520 !ruby/object:Gem::Requirement
+  requirement: &70161327999200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,10 +43,10 @@ dependencies:
         version: 0.9.0
   type: :runtime
   prerelease: false
-  version_requirements: *70116788676520
+  version_requirements: *70161327999200
 - !ruby/object:Gem::Dependency
   name: celluloid
-  requirement: &70116788675940 !ruby/object:Gem::Requirement
+  requirement: &70161327998620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: 0.10.0
   type: :runtime
   prerelease: false
-  version_requirements: *70116788675940
+  version_requirements: *70161327998620
 - !ruby/object:Gem::Dependency
   name: multi_json
-  requirement: &70116788675040 !ruby/object:Gem::Requirement
+  requirement: &70161327997960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,10 +65,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *70116788675040
+  version_requirements: *70161327997960
 - !ruby/object:Gem::Dependency
   name: minitest
-  requirement: &70116788674260 !ruby/object:Gem::Requirement
+  requirement: &70161327996900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70116788674260
+  version_requirements: *70161327996900
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70116788673760 !ruby/object:Gem::Requirement
+  requirement: &70161327996420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70116788673760
+  version_requirements: *70161327996420
 - !ruby/object:Gem::Dependency
   name: slim
-  requirement: &70116788689600 !ruby/object:Gem::Requirement
+  requirement: &70161328012260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70116788689600
+  version_requirements: *70161328012260
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70116788689140 !ruby/object:Gem::Requirement
+  requirement: &70161328011740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70116788689140
+  version_requirements: *70161328011740
 - !ruby/object:Gem::Dependency
   name: actionmailer
-  requirement: &70116788688560 !ruby/object:Gem::Requirement
+  requirement: &70161328011180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -120,10 +120,10 @@ dependencies:
         version: '3'
   type: :development
   prerelease: false
-  version_requirements: *70116788688560
+  version_requirements: *70161328011180
 - !ruby/object:Gem::Dependency
   name: activerecord
-  requirement: &70116788688060 !ruby/object:Gem::Requirement
+  requirement: &70161328010660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -131,7 +131,7 @@ dependencies:
         version: '3'
   type: :development
   prerelease: false
-  version_requirements: *70116788688060
+  version_requirements: *70161328010660
 description: Simple, efficient message processing for Ruby
 email:
 - mperham@gmail.com