sidekiq 2.12.0 → 2.12.1

data/Changes.md

@@ -1,3 +1,16 @@
+2.12.1
+-----------
+
+- Force Celluloid 0.14.1 as 0.14.0 has a serious bug. [#954]
+- Scheduled and Retry jobs now use Sidekiq::Client to push
+  jobs onto the queue, so they use client middleware. [dimko, #948]
+- Record the timestamp when jobs are enqueued. Add
+  Sidekiq::Job#enqueued\_at to query the time. [mariovisic, #944]
+- Add Sidekiq::Queue#latency - calculates diff between now and
+  enqueued\_at for the oldest job in the queue.
+- Add testing method `perform_one` that dequeues and performs a single job.
+  This is mainly to aid testing jobs that spawn other jobs. [fumin, #963]
+
 2.12.0
 -----------
 

data/Pro-Changes.md

@@ -3,7 +3,7 @@ Sidekiq Pro Changelog
 
 Please see http://sidekiq.org/pro for more details and how to buy.
 
-HEAD
+1.1.0
 -----------
 
 - New `sidekiq/pro/reliable_push` which makes Sidekiq::Client resiliant

data/README.md

@@ -27,7 +27,7 @@ Requirements
 I test on Ruby 1.9.3 and JRuby 1.7.x.  Other versions/VMs are
 untested but I will do my best to support them.  Ruby 1.8 is not supported.
 
-Redis 2.0 or greater is required.
+Redis 2.4 or greater is required.
 
 
 Installation

data/lib/sidekiq/api.rb

@@ -128,6 +128,14 @@ module Sidekiq
       Sidekiq.redis { |con| con.llen(@rname) }
     end
 
+    def latency
+      entry = Sidekiq.redis do |conn|
+        conn.lrange(@rname, -1, -1)
+      end.first
+      return 0 unless entry
+      Time.now.to_f - Sidekiq.load_json(entry)['enqueued_at']
+    end
+
     def each(&block)
       page = 0
       page_size = 50
@@ -186,10 +194,18 @@ module Sidekiq
       @item['jid']
     end
 
+    def enqueued_at
+      Time.at(@item['enqueued_at'])
+    end
+
     def queue
       @queue
     end
 
+    def latency
+      Time.now.to_f - @item['enqueued_at']
+    end
+
     ##
     # Remove this job from the queue.
     def delete
@@ -234,7 +250,7 @@ module Sidekiq
         results.map do |message|
           msg = Sidekiq.load_json(message)
           msg['retry_count'] = msg['retry_count'] - 1
-          conn.lpush("queue:#{msg['queue']}", Sidekiq.dump_json(msg))
+          Sidekiq::Client.push(msg)
         end
       end
     end
@@ -372,15 +388,16 @@ module Sidekiq
   # WARNING WARNING WARNING
   #
   # This is live data that can change every millisecond.
-  # If you do #size => 5 and then expect #each to be
+  # If you call #size => 5 and then expect #each to be
   # called 5 times, you're going to have a bad time.
   #
   #    workers = Sidekiq::Workers.new
   #    workers.size => 2
-  #    workers.each do |name, work|
-  #      # name is a unique identifier per Processor instance
+  #    workers.each do |name, work, started_at|
+  #      # name is a unique identifier per worker
   #      # work is a Hash which looks like:
   #      # { 'queue' => name, 'run_at' => timestamp, 'payload' => msg }
+  #      # started_at is a String rep of the time when the worker started working on the job
   #    end
 
   class Workers
@@ -390,9 +407,12 @@ module Sidekiq
       Sidekiq.redis do |conn|
         workers = conn.smembers("workers")
         workers.each do |w|
-          msg = conn.get("worker:#{w}")
+          msg, time = conn.multi do
+            conn.get("worker:#{w}")
+            conn.get("worker:#{w}:started")
+          end
           next unless msg
-          block.call(w, Sidekiq.load_json(msg))
+          block.call(w, Sidekiq.load_json(msg), time)
         end
       end
     end

data/lib/sidekiq/client.rb

@@ -62,7 +62,7 @@ module Sidekiq
         normed = normalize_item(items)
         payloads = items['args'].map do |args|
           raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !args.is_a?(Array)
-          process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12)))
+          process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f))
         end.compact
 
         pushed = false
@@ -94,7 +94,10 @@ module Sidekiq
         pushed = false
         Sidekiq.redis do |conn|
           if payloads.first['at']
-            pushed = conn.zadd('schedule', payloads.map {|hash| [hash['at'].to_s, Sidekiq.dump_json(hash)]})
+            pushed = conn.zadd('schedule', payloads.map do |hash|
+              at = hash.delete('at').to_s
+              [at, Sidekiq.dump_json(hash)]
+            end)
           else
             q = payloads.first['queue']
             to_push = payloads.map { |entry| Sidekiq.dump_json(entry) }
@@ -129,7 +132,8 @@ module Sidekiq
           normalized_item = Sidekiq::Worker::ClassMethods::DEFAULT_OPTIONS.merge(item)
         end
 
-        normalized_item['jid'] = SecureRandom.hex(12)
+        normalized_item['jid'] ||= SecureRandom.hex(12)
+        normalized_item['enqueued_at'] = Time.now.to_f
         normalized_item
       end
 

data/lib/sidekiq/exception_handler.rb

@@ -33,7 +33,7 @@ module Sidekiq
     end
 
     def send_to_exception_notifier(msg, ex)
-      ::ExceptionNotifier.notify_exception(ex, :data => {:message => msg})
+      ::ExceptionNotifier::Notifier.background_exception_notification(ex, :data => { :message => msg }).deliver
     end
   end
 end

data/lib/sidekiq/scheduled.rb

@@ -33,15 +33,12 @@ module Sidekiq
                 # going wrong between the time jobs are popped from the scheduled queue and when
                 # they are pushed onto a work queue and losing the jobs.
                 while message = conn.zrangebyscore(sorted_set, '-inf', now, :limit => [0, 1]).first do
-                  msg = Sidekiq.load_json(message)
+
                   # Pop item off the queue and add it to the work queue. If the job can't be popped from
                   # the queue, it's because another process already popped it so we can move on to the
                   # next one.
                   if conn.zrem(sorted_set, message)
-                    conn.multi do
-                      conn.sadd('queues', msg['queue'])
-                      conn.lpush("queue:#{msg['queue']}", message)
-                    end
+                    Sidekiq::Client.push(Sidekiq.load_json(message))
                     logger.debug { "enqueued #{sorted_set}: #{message}" }
                   end
                 end

data/lib/sidekiq/testing.rb

@@ -1,5 +1,7 @@
 module Sidekiq
 
+  class EmptyQueueError < RuntimeError; end
+
   class Client
     class << self
       alias_method :raw_push_old, :raw_push
@@ -87,6 +89,13 @@ module Sidekiq
           new.perform(*job['args'])
         end
       end
+
+      # Pop out a single job and perform it
+      def perform_one
+        raise(EmptyQueueError, "perform_one called with empty job queue") if jobs.empty?
+        job = jobs.shift
+        new.perform(*job['args'])
+      end
     end
 
     class << self

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.12.0"
+  VERSION = "2.12.1"
 end

data/lib/sidekiq/web.rb

@@ -2,6 +2,8 @@ require 'yaml'
 require 'sinatra/base'
 require 'slim'
 
+raise "The Sidekiq Web UI requires slim 1.3.8 or greater.  You have slim v#{Slim::VERSION}" if Gem::Version.new(Slim::VERSION) < Gem::Version.new('1.3.8')
+
 require 'sidekiq'
 require 'sidekiq/api'
 require 'sidekiq/paginator'
@@ -105,6 +107,20 @@ module Sidekiq
         args.map { |arg| a = arg.inspect; a.size > count ? "#{a[0..count]}..." : a }.join(", ")
       end
 
+      RETRY_JOB_KEYS = Set.new(%w(
+        queue class args retry_count retried_at failed_at
+        jid error_message error_class backtrace
+        error_backtrace enqueued_at
+      ))
+
+      def retry_extra_items(retry_job)
+        @retry_extra_items ||= {}.tap do |extra|
+          retry_job.item.each do |key, value|
+            extra[key] = value unless RETRY_JOB_KEYS.include?(key)
+          end
+        end
+      end
+
       def tabs
         @tabs ||= {
           "Dashboard" => '',

data/sidekiq.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency                  'redis', '>= 3.0'
   gem.add_dependency                  'redis-namespace'
   gem.add_dependency                  'connection_pool', '>= 1.0.0'
-  gem.add_dependency                  'celluloid', '>= 0.14.0'
+  gem.add_dependency                  'celluloid', '>= 0.14.1'
   gem.add_dependency                  'json'
   gem.add_development_dependency      'sinatra'
   gem.add_development_dependency      'slim'

data/test/test_api.rb

@@ -164,6 +164,7 @@ class TestApi < Minitest::Test
     it 'shows queue as empty' do
       q = Sidekiq::Queue.new
       assert_equal 0, q.size
+      assert_equal 0, q.latency
     end
 
     class ApiWorker
@@ -172,12 +173,18 @@ class TestApi < Minitest::Test
 
     it 'can enumerate jobs' do
       q = Sidekiq::Queue.new
-      ApiWorker.perform_async(1, 'mike')
-      assert_equal ['TestApi::ApiWorker'], q.map(&:klass)
+      Time.stub(:now, Time.new(2012, 12, 26)) do
+        ApiWorker.perform_async(1, 'mike')
+        assert_equal ['TestApi::ApiWorker'], q.map(&:klass)
+
+        job = q.first
+        assert_equal 24, job.jid.size
+        assert_equal [1, 'mike'], job.args
+        assert_equal Time.new(2012, 12, 26), job.enqueued_at
 
-      job = q.first
-      assert_equal 24, job.jid.size
-      assert_equal [1, 'mike'], job.args
+      end
+
+      assert q.latency > 10_000_000
 
       q = Sidekiq::Queue.new('other')
       assert_equal 0, q.size
@@ -192,11 +199,11 @@ class TestApi < Minitest::Test
     end
 
     it 'can find job by id in sorted sets' do
-      q = Sidekiq::Queue.new
       job_id = ApiWorker.perform_in(100, 1, 'jason')
       job = Sidekiq::ScheduledSet.new.find_job(job_id)
       refute_nil job
       assert_equal job_id, job.jid
+      assert_in_delta job.latency, 0.0, 0.01
     end
 
     it 'can find job by id in queues' do
@@ -310,13 +317,15 @@ class TestApi < Minitest::Test
         c.multi do
           c.sadd('workers', s)
           c.set("worker:#{s}", data)
+          c.set("worker:#{s}:started", Time.now.to_s)
         end
       end
 
       assert_equal 1, w.size
-      w.each do |x, y|
+      w.each do |x, y, z|
         assert_equal s, x
         assert_equal 'default', y['queue']
+        assert_equal Time.now.year, DateTime.parse(z).year
       end
     end
 

data/test/test_exception_handler.rb

@@ -73,7 +73,8 @@ class TestExceptionHandler < Minitest::Test
 
   describe "with fake ExceptionNotifier" do
     before do
-      ::ExceptionNotifier = Minitest::Mock.new
+      ::ExceptionNotifier = Module.new
+      ::ExceptionNotifier::Notifier = MiniTest::Mock.new
     end
 
     after do
@@ -81,9 +82,12 @@ class TestExceptionHandler < Minitest::Test
     end
 
     it "notifies ExceptionNotifier" do
-      ::ExceptionNotifier.expect(:notify_exception,nil,[TEST_EXCEPTION, :data => { :message => { :b => 2 } }])
+      mail = MiniTest::Mock.new
+      mail.expect(:deliver,nil)
+      ::ExceptionNotifier::Notifier.expect(:background_exception_notification,mail,[TEST_EXCEPTION, :data => { :message => { :b => 2 } }])
       Component.new.invoke_exception(:b => 2)
-      ::ExceptionNotifier.verify
+      ::ExceptionNotifier::Notifier.verify
+      mail.verify
     end
   end
 

data/test/test_scheduled.rb

@@ -14,33 +14,72 @@ class TestScheduled < Minitest::Test
       Sidekiq.redis do |conn|
         conn.flushdb
       end
+
+      @error_1  = { 'class' => ScheduledWorker.name, 'args' => [0], 'queue' => 'queue_1' }
+      @error_2  = { 'class' => ScheduledWorker.name, 'args' => [1], 'queue' => 'queue_2' }
+      @error_3  = { 'class' => ScheduledWorker.name, 'args' => [2], 'queue' => 'queue_3' }
+      @future_1 = { 'class' => ScheduledWorker.name, 'args' => [3], 'queue' => 'queue_4' }
+      @future_2 = { 'class' => ScheduledWorker.name, 'args' => [4], 'queue' => 'queue_5' }
+      @future_3 = { 'class' => ScheduledWorker.name, 'args' => [5], 'queue' => 'queue_6' }
+
+      @retry = Sidekiq::RetrySet.new
+      @scheduled = Sidekiq::ScheduledSet.new
+      @poller = Sidekiq::Scheduled::Poller.new
+    end
+
+    class Stopper
+      def call(worker_class, message, queue)
+        yield if message['args'].first.odd?
+      end
+    end
+
+    it 'executes client middleware' do
+      Sidekiq.client_middleware.add Stopper
+      begin
+        @retry.schedule (Time.now - 60).to_f, @error_1
+        @retry.schedule (Time.now - 60).to_f, @error_2
+        @scheduled.schedule (Time.now - 60).to_f, @future_2
+        @scheduled.schedule (Time.now - 60).to_f, @future_3
+
+        @poller.poll
+
+        Sidekiq.redis do |conn|
+          assert_equal 0, conn.llen("queue:queue_1")
+          assert_equal 1, conn.llen("queue:queue_2")
+          assert_equal 0, conn.llen("queue:queue_5")
+          assert_equal 1, conn.llen("queue:queue_6")
+        end
+      ensure
+        Sidekiq.client_middleware.remove Stopper
+      end
     end
 
     it 'should empty the retry and scheduled queues up to the current time' do
-      Sidekiq.redis do |conn|
-        error_1 = Sidekiq.dump_json('class' => ScheduledWorker.name, 'args' => ["error_1"], 'queue' => 'queue_1')
-        error_2 = Sidekiq.dump_json('class' => ScheduledWorker.name, 'args' => ["error_2"], 'queue' => 'queue_2')
-        error_3 = Sidekiq.dump_json('class' => ScheduledWorker.name, 'args' => ["error_3"], 'queue' => 'queue_3')
-        future_1 = Sidekiq.dump_json('class' => ScheduledWorker.name, 'args' => ["future_1"], 'queue' => 'queue_4')
-        future_2 = Sidekiq.dump_json('class' => ScheduledWorker.name, 'args' => ["future_2"], 'queue' => 'queue_5')
-        future_3 = Sidekiq.dump_json('class' => ScheduledWorker.name, 'args' => ["future_3"], 'queue' => 'queue_6')
-
-        conn.zadd("retry", (Time.now - 60).to_f.to_s, error_1)
-        conn.zadd("retry", (Time.now - 50).to_f.to_s, error_2)
-        conn.zadd("retry", (Time.now + 60).to_f.to_s, error_3)
-        conn.zadd("schedule", (Time.now - 60).to_f.to_s, future_1)
-        conn.zadd("schedule", (Time.now - 50).to_f.to_s, future_2)
-        conn.zadd("schedule", (Time.now + 60).to_f.to_s, future_3)
-
-        poller = Sidekiq::Scheduled::Poller.new
-        poller.poll
-
-        assert_equal [error_1], conn.lrange("queue:queue_1", 0, -1)
-        assert_equal [error_2], conn.lrange("queue:queue_2", 0, -1)
-        assert_equal [error_3], conn.zrange("retry", 0, -1)
-        assert_equal [future_1], conn.lrange("queue:queue_4", 0, -1)
-        assert_equal [future_2], conn.lrange("queue:queue_5", 0, -1)
-        assert_equal [future_3], conn.zrange("schedule", 0, -1)
+      enqueued_time = Time.new(2013, 2, 4)
+
+      Time.stub(:now, enqueued_time) do
+        @retry.schedule (Time.now - 60).to_f, @error_1
+        @retry.schedule (Time.now - 50).to_f, @error_2
+        @retry.schedule (Time.now + 60).to_f, @error_3
+        @scheduled.schedule (Time.now - 60).to_f, @future_1
+        @scheduled.schedule (Time.now - 50).to_f, @future_2
+        @scheduled.schedule (Time.now + 60).to_f, @future_3
+
+        @poller.poll
+
+        Sidekiq.redis do |conn|
+          assert_equal 1, conn.llen("queue:queue_1")
+          assert_equal enqueued_time.to_f, Sidekiq.load_json(conn.lrange("queue:queue_1", 0, -1)[0])['enqueued_at']
+          assert_equal 1, conn.llen("queue:queue_2")
+          assert_equal enqueued_time.to_f, Sidekiq.load_json(conn.lrange("queue:queue_2", 0, -1)[0])['enqueued_at']
+          assert_equal 1, conn.llen("queue:queue_4")
+          assert_equal enqueued_time.to_f, Sidekiq.load_json(conn.lrange("queue:queue_4", 0, -1)[0])['enqueued_at']
+          assert_equal 1, conn.llen("queue:queue_5")
+          assert_equal enqueued_time.to_f, Sidekiq.load_json(conn.lrange("queue:queue_5", 0, -1)[0])['enqueued_at']
+        end
+
+        assert_equal 1, @retry.size
+        assert_equal 1, @scheduled.size
       end
     end
   end

data/test/test_testing.rb

@@ -119,6 +119,23 @@ class TestTesting < Minitest::Test
       StoredWorker.clear
     end
 
+    it 'perform_one runs only one job' do
+      DirectWorker.perform_async(1, 2)
+      DirectWorker.perform_async(3, 4)
+      assert_equal 2, DirectWorker.jobs.size
+
+      DirectWorker.perform_one
+      assert_equal 1, DirectWorker.jobs.size
+
+      DirectWorker.clear
+    end
+
+    it 'perform_one raise error upon empty queue' do
+      assert_raises Sidekiq::EmptyQueueError do
+        DirectWorker.perform_one
+      end
+    end
+
     class FirstWorker
       include Sidekiq::Worker
       class_attribute :count

data/test/test_web.rb

@@ -264,7 +264,6 @@ class TestWeb < Minitest::Test
       score = Time.now.to_f
       msg = { 'class' => 'HardWorker',
               'args' => ['bob', 1, Time.now.to_f],
-              'at' => score,
               'jid' => 'f39af2a05e8f4b24dbc0f1e4' }
       Sidekiq.redis do |conn|
         conn.zadd('schedule', score, Sidekiq.dump_json(msg))

data/web/locales/en.yml

@@ -21,6 +21,7 @@ en: # <---- change this to your locale code
   Class: Class
   Job: Job
   Arguments: Arguments
+  Extras: Extras
   Started: Started
   ShowAll: Show All
   CurrentMessagesInQueue: Current messages in <span class='title'>%{queue}</span>

data/web/locales/pt-br.yml

@@ -20,6 +20,7 @@
   Class: Classe
   Job: Tarefa
   Arguments: Argumentos
+  Extras: Extras
   Started: Iniciados
   ShowAll: Mostrar todos
   CurrentMessagesInQueue: Mensagens atualmenta na <span class='title'>%{queue}</span>

data/web/views/retry.slim

@@ -19,6 +19,11 @@ table class="retry table table-bordered table-striped"
       th JID
       td
         code= @retry.jid
+    - unless retry_extra_items(@retry).empty?
+      tr
+        th = t('Extras')
+        td
+          code= display_args(retry_extra_items(@retry), 1000)
     - if @retry['retry_count'] > 0
       tr
         th = t('RetryCount')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 2.12.0
+  version: 2.12.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-15 00:00:00.000000000 Z
+date: 2013-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -66,7 +66,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: 0.14.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +74,7 @@ dependencies:
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: 0.14.1
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement