sidekiq 2.2.1 → 2.3.0

data/Changes.md

@@ -1,3 +1,12 @@
+2.3.0
+-----------
+
+- Upgrade Celluloid to 0.12
+- Upgrade Twitter Bootstrap to 2.1.0
+- Rescue more Exceptions
+- Change Job ID to be Hex, rather than Base64, for HTTP safety
+- Use `Airbrake#notify_or_ignore`
+
 2.2.1
 -----------
 

data/Gemfile

@@ -1,9 +1,8 @@
 source 'http://rubygems.org'
 gemspec
 
-gem 'celluloid'
+gem 'celluloid', "~> 0.12.0"
 gem 'slim'
-gem 'sprockets'
 gem 'sass'
 gem 'rails', '3.2.8'
 gem 'sqlite3'

data/README.md

@@ -7,8 +7,8 @@ Sidekiq
 Simple, efficient message processing for Ruby.
 
 Sidekiq uses threads to handle many messages at the same time in the
-same process.  It integrates tightly with Rails 3 to make background
-message processing dead simple.
+same process.  It does not require Rails but will integrate tightly with
+Rails 3 to make background message processing dead simple.
 
 Sidekiq is compatible with Resque.  It uses the exact same
 message format as Resque so it can integrate into an existing Resque processing farm.

data/examples/sinkiq.rb

@@ -19,8 +19,8 @@ class SinatraWorker
 end
 
 get '/' do
-  @failed = $redis.get('stat:failed')
-  @processed = $redis.get('stat:processed')
+  @failed = Sidekiq::Stats.failed
+  @processed = Sidekiq::Stats.processed
   @messages = $redis.lrange('sinkiq-example-messages', 0, -1)
   erb :index
 end

data/lib/sidekiq.rb

@@ -4,6 +4,7 @@ require 'sidekiq/client'
 require 'sidekiq/worker'
 require 'sidekiq/redis_connection'
 require 'sidekiq/util'
+require 'sidekiq/stats'
 
 require 'sidekiq/extensions/class_methods'
 require 'sidekiq/extensions/action_mailer'
@@ -13,6 +14,7 @@ require 'sidekiq/rails' if defined?(::Rails::Engine)
 require 'multi_json'
 
 module Sidekiq
+  LICENSE = 'See LICENSE and the LGPL-3.0 for licensing details.'
 
   DEFAULTS = {
     :queues => [],

data/lib/sidekiq/cli.rb

@@ -66,6 +66,10 @@ module Sidekiq
     end
 
     def run
+      logger.info "Booting Sidekiq #{Sidekiq::VERSION} with Redis at #{redis {|x| x.client.id}}"
+      logger.info "Running in #{RUBY_DESCRIPTION}"
+      logger.info Sidekiq::LICENSE
+
       @manager = Sidekiq::Manager.new(options)
       poller = Sidekiq::Scheduled::Poller.new
       begin

data/lib/sidekiq/client.rb

@@ -1,3 +1,5 @@
+require 'securerandom'
+
 require 'sidekiq/middleware/chain'
 
 module Sidekiq
@@ -44,7 +46,7 @@ module Sidekiq
       item = worker_class.get_sidekiq_options.merge(item)
       item['retry'] = !!item['retry']
       queue = item['queue']
-      item['jid'] = SecureRandom.base64
+      item['jid'] = SecureRandom.hex(12)
 
       pushed = false
       Sidekiq.client_middleware.invoke(worker_class, item, queue) do
@@ -63,14 +65,22 @@ module Sidekiq
       pushed ? item['jid'] : nil
     end
 
-    # Redis compatibility helper.  Example usage:
+    # Resque compatibility helpers.
     #
+    # Example usage:
     #   Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
     #
     # Messages are enqueued to the 'default' queue.
     #
     def self.enqueue(klass, *args)
-      klass.perform_async(*args)
+      klass.client_push('class' => klass, 'args' => args)
+    end
+
+    # Example usage:
+    #   Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
+    #
+    def self.enqueue_to(queue, klass, *args)
+      klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
     end
   end
 end

data/lib/sidekiq/exception_handler.rb

@@ -13,7 +13,7 @@ module Sidekiq
     private
 
     def send_to_airbrake(msg, ex)
-      ::Airbrake.notify(ex, :parameters => msg)
+      ::Airbrake.notify_or_ignore(ex, :parameters => msg)
     end
 
     def send_to_exceptional(msg, ex)

data/lib/sidekiq/manager.rb

@@ -18,8 +18,6 @@ module Sidekiq
     trap_exit :processor_died
 
     def initialize(options={})
-      logger.info "Booting sidekiq #{Sidekiq::VERSION} with Redis at #{redis {|x| x.client.id}}"
-      logger.info "Running in #{RUBY_DESCRIPTION}"
       logger.debug { options.inspect }
       @count = options[:concurrency] || 25
       @done_callback = nil
@@ -129,6 +127,7 @@ module Sidekiq
               # processor is an actor proxy and we can't call any methods
               # that would go to the actor (since it's busy).  Instead
               # we'll use the object_id to track the worker's data here.
+              processor.terminate if processor.alive?
               msg, queue = @in_progress[processor.object_id]
               conn.lpush("queue:#{queue}", msg)
             end

data/lib/sidekiq/middleware/server/logging.rb

@@ -10,7 +10,7 @@ module Sidekiq
               logger.info { "start" }
               yield
               logger.info { "done: #{elapsed(start)} sec" }
-            rescue
+            rescue Exception
               logger.info { "fail: #{elapsed(start)} sec" }
               raise
             end

data/lib/sidekiq/middleware/server/retry_jobs.rb

@@ -40,8 +40,8 @@ module Sidekiq
 
         def call(worker, msg, queue)
           yield
-        rescue => e
-          raise unless msg['retry']
+        rescue Exception => e
+          raise e unless msg['retry']
 
           msg['queue'] = queue
           msg['error_message'] = e.message
@@ -72,7 +72,7 @@ module Sidekiq
             # Goodbye dear message, you (re)tried your best I'm sure.
             logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
           end
-          raise
+          raise e
         end
 
       end

data/lib/sidekiq/processor.rb

@@ -15,6 +15,8 @@ module Sidekiq
     include Util
     include Celluloid
 
+#    exclusive :process
+
     def self.default_middleware
       Middleware::Chain.new do |m|
         m.add Middleware::Server::Logging
@@ -29,24 +31,19 @@ module Sidekiq
     end
 
     def process(msgstr, queue)
-      # Defer worker execution to Celluloid's thread pool since all actor
-      # invocations are run within a Fiber, which dramatically limits
-      # our stack size.
-      defer do
-        begin
-          msg = Sidekiq.load_json(msgstr)
-          klass  = constantize(msg['class'])
-          worker = klass.new
+      begin
+        msg = Sidekiq.load_json(msgstr)
+        klass  = constantize(msg['class'])
+        worker = klass.new
 
-          stats(worker, msg, queue) do
-            Sidekiq.server_middleware.invoke(worker, msg, queue) do
-              worker.perform(*cloned(msg['args']))
-            end
+        stats(worker, msg, queue) do
+          Sidekiq.server_middleware.invoke(worker, msg, queue) do
+            worker.perform(*cloned(msg['args']))
           end
-        rescue Exception => ex
-          handle_exception(ex, msg || { :message => msgstr })
-          raise
         end
+      rescue Exception => ex
+        handle_exception(ex, msg || { :message => msgstr })
+        raise
       end
       @boss.processor_done!(current_actor)
     end

data/lib/sidekiq/stats.rb

@@ -0,0 +1,36 @@
+module Sidekiq
+  module_function
+
+  def info
+    results = {}
+    processed, failed, queues = Sidekiq.redis { |conn|
+      conn.multi do
+        conn.get('stat:processed')
+        conn.get('stat:failed')
+        conn.smembers('queues')
+      end
+    }
+    results[:queues_with_sizes] = Sidekiq.redis do |conn|
+      queues.inject({}) { |memo, q|
+        memo[q] = conn.llen("queue:#{q}")
+        memo
+      }.sort_by { |_, size| size }
+    end
+    results[:processed] = (processed || 0).to_i
+    results[:failed] = (failed || 0).to_i
+    results[:backlog] = results[:queues_with_sizes].
+                          map {|_, size| size }.
+                          inject(0) {|memo, val| memo + val }
+    results
+  end
+
+  def size(*queues)
+    return info[:backlog] if queues.empty?
+
+    Sidekiq.redis { |conn|
+      conn.multi {
+        queues.map { |q| conn.llen("queue:#{q}") }
+      }
+    }.inject(0) { |memo, count| memo += count }
+  end
+end

data/lib/sidekiq/testing/inline.rb

@@ -1,8 +1,8 @@
 module Sidekiq
-  module Worker
+  class Client
 
     ##
-    # The Sidekiq inline infrastructure overrides the perform_async so that it
+    # The Sidekiq inline infrastructure overrides perform_async so that it
     # actually calls perform instead. This allows workers to be run inline in a
     # testing environment.
     #
@@ -26,10 +26,10 @@ module Sidekiq
     #   ExternalWorker.perform_async
     #   assert_equal 1, $external_variable
     #
-    module ClassMethods
-      alias_method :perform_async_old, :perform_async
-      def perform_async(*args)
-        new.perform(*Sidekiq.load_json(Sidekiq.dump_json(args)))
+    singleton_class.class_eval do
+      alias_method :push_old, :push
+      def push(hash)
+        hash['class'].new.perform(*Sidekiq.load_json(Sidekiq.dump_json(hash['args'])))
         true
       end
     end

data/lib/sidekiq/util.rb

@@ -7,7 +7,7 @@ module Sidekiq
   module Util
     include ExceptionHandler
 
-    EXPIRY = 60 * 60
+    EXPIRY = 60 * 60 * 24
 
     def constantize(camel_cased_word)
       names = camel_cased_word.split('::')
@@ -22,7 +22,7 @@ module Sidekiq
 
     def watchdog(last_words)
       yield
-    rescue => ex
+    rescue Exception => ex
       handle_exception(ex, { :context => last_words })
     end
 

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.2.1"
+  VERSION = "2.3.0"
 end

data/lib/sidekiq/web.rb

@@ -60,12 +60,16 @@ module Sidekiq
         end
       end
 
+      def info
+        @info ||= Sidekiq.info
+      end
+
       def processed
-        Sidekiq.redis { |conn| conn.get('stat:processed') } || 0
+        info[:processed]
       end
 
       def failed
-        Sidekiq.redis { |conn| conn.get('stat:failed') } || 0
+        info[:failed]
       end
 
       def zcard(name)
@@ -73,15 +77,11 @@ module Sidekiq
       end
 
       def queues
-        @queues ||= Sidekiq.redis do |conn|
-          conn.smembers('queues').map do |q|
-            [q, conn.llen("queue:#{q}") || 0]
-          end.sort { |x,y| x[1] <=> y[1] }
-        end
+        @queues ||= Sidekiq.info[:queues_with_sizes]
       end
 
       def backlog
-        queues.map {|name, size| size }.inject(0) {|memo, val| memo + val }
+        info[:backlog]
       end
 
       def retries_with_score(score)
@@ -115,6 +115,19 @@ module Sidekiq
       def tabs
         self.class.tabs
       end
+
+      def number_with_delimiter(number)
+        begin
+          Float(number)
+        rescue ArgumentError, TypeError
+          return number
+        end
+
+        options = {:delimiter => ',', :separator => '.'}
+        parts = number.to_s.to_str.split('.')
+        parts[0].gsub!(/(\d)(?=(\d\d\d)+(?!\d))/, "\\1#{options[:delimiter]}")
+        parts.join(options[:separator])
+      end
     end
 
     get "/" do

data/sidekiq.gemspec

@@ -9,7 +9,7 @@ Gem::Specification.new do |gem|
   gem.license       = "LGPL-3.0"
 
   gem.executables   = ['sidekiq', 'sidekiqctl']
-  gem.files         = `git ls-files`.split("\n")
+  gem.files         = `git ls-files | grep -v myapp`.split("\n")
   gem.test_files    = `git ls-files -- test/*`.split("\n")
   gem.name          = "sidekiq"
   gem.require_paths = ["lib"]
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency                  'redis', '~> 3'
   gem.add_dependency                  'redis-namespace'
   gem.add_dependency                  'connection_pool', '~> 0.9.2'
-  gem.add_dependency                  'celluloid', '~> 0.11.1'
+  gem.add_dependency                  'celluloid', '~> 0.12.0'
   gem.add_dependency                  'multi_json', '~> 1'
   gem.add_development_dependency      'minitest', '~> 3'
   gem.add_development_dependency      'sinatra'
@@ -25,4 +25,5 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency      'rake'
   gem.add_development_dependency      'actionmailer', '~> 3'
   gem.add_development_dependency      'activerecord', '~> 3'
+  gem.add_development_dependency      'pry'
 end

data/test/helper.rb

@@ -4,6 +4,8 @@ if ENV.has_key?("SIMPLECOV")
   SimpleCov.start
 end
 
+require 'pry'
+
 require 'minitest/unit'
 require 'minitest/pride'
 require 'minitest/autorun'

data/test/test_client.rb

@@ -69,6 +69,13 @@ class TestClient < MiniTest::Unit::TestCase
       @redis.verify
     end
 
+    it 'enqueues messages to redis' do
+      @redis.expect :rpush, 1, ['queue:custom_queue', String]
+      pushed = Sidekiq::Client.enqueue_to(:custom_queue, MyWorker, 1, 2)
+      assert pushed
+      @redis.verify
+    end
+
     class QueuedWorker
       include Sidekiq::Worker
       sidekiq_options :queue => :flimflam, :timeout => 1

data/test/test_exception_handler.rb

@@ -49,7 +49,7 @@ class TestExceptionHandler < MiniTest::Unit::TestCase
     end
 
     it "notifies Airbrake" do
-      ::Airbrake.expect(:notify,nil,[TEST_EXCEPTION,:parameters => { :a => 1 }])
+      ::Airbrake.expect(:notify_or_ignore,nil,[TEST_EXCEPTION,:parameters => { :a => 1 }])
       Component.new.invoke_exception(:a => 1)
       ::Airbrake.verify
     end

data/test/test_stats.rb

@@ -11,6 +11,7 @@ class TestStats < MiniTest::Unit::TestCase
 
     class DumbWorker
       include Sidekiq::Worker
+      sidekiq_options :queue => 'dumbq'
 
       def perform(arg)
         raise 'bang' if arg == nil
@@ -31,15 +32,15 @@ class TestStats < MiniTest::Unit::TestCase
         boss.expect(:processor_done!, nil, [processor])
         boss.expect(:processor_done!, nil, [processor])
 
-        assert_equal 0, conn.get('stat:failed').to_i
-        assert_equal 0, conn.get('stat:processed').to_i
+        assert_equal 0, Sidekiq.info[:failed]
+        assert_equal 0, Sidekiq.info[:processed]
 
         processor.process(msg, 'xyzzy')
         processor.process(msg, 'xyzzy')
         processor.process(msg, 'xyzzy')
 
-        assert_equal 0, conn.get('stat:failed').to_i
-        assert_equal 3, conn.get('stat:processed').to_i
+        assert_equal 0, Sidekiq.info[:failed]
+        assert_equal 3, Sidekiq.info[:processed]
       end
     end
 
@@ -49,8 +50,8 @@ class TestStats < MiniTest::Unit::TestCase
 
       @redis.with do |conn|
         assert_equal [], conn.smembers('workers')
-        assert_equal 0, conn.get('stat:failed').to_i
-        assert_equal 0, conn.get('stat:processed').to_i
+        assert_equal 0, Sidekiq.info[:failed]
+        assert_equal 0, Sidekiq.info[:processed]
 
         processor = Sidekiq::Processor.new(boss)
 
@@ -59,8 +60,46 @@ class TestStats < MiniTest::Unit::TestCase
           processor.process(msg, 'xyzzy')
         end
 
-        assert_equal 1, conn.get('stat:failed').to_i
-        assert_equal 1, conn.get('stat:processed').to_i
+        assert_equal 1, Sidekiq.info[:failed]
+        assert_equal 1, Sidekiq.info[:processed]
+      end
+    end
+
+    describe "info counts" do
+      before do
+        @redis.with do |conn|
+          conn.rpush 'queue:foo', '{}'
+          conn.sadd 'queues', 'foo'
+
+          conn.rpush 'queue:bar', '{}'
+          conn.rpush 'queue:bar', '{}'
+          conn.sadd 'queues', 'bar'
+
+          conn.rpush 'queue:baz', '{}'
+          conn.sadd 'queues', 'baz'
+        end
+      end
+
+      describe "queues_with_sizes" do
+        it "returns queue names and corresponding job counts" do
+          assert_equal [["foo", 1], ["baz", 1], ["bar", 2]], Sidekiq.info[:queues_with_sizes]
+        end
+      end
+
+      describe "backlog" do
+        it "returns count of all jobs yet to be processed" do
+          assert_equal 4, Sidekiq.info[:backlog]
+        end
+      end
+
+      describe "size" do
+        it "returns size of queues" do
+          assert_equal 0, Sidekiq.size("foox")
+          assert_equal 1, Sidekiq.size(:foo)
+          assert_equal 1, Sidekiq.size("foo")
+          assert_equal 3, Sidekiq.size("foo", "bar")
+          assert_equal 4, Sidekiq.size
+        end
       end
     end