sidekiq 0.5.1 → 0.6.0

data/myapp/Gemfile

@@ -1,5 +1,8 @@
 source 'https://rubygems.org'
 
+gem 'jruby-openssl' if defined?(JRUBY_VERSION)
+
 gem 'rails', '3.2.1'
 gem 'sqlite3'
-gem 'sidekiq'#, :path => '..'
+gem 'sidekiq', :path => '..'
+gem 'resque'

data/myapp/Gemfile.lock

@@ -1,3 +1,13 @@
+PATH
+  remote: ..
+  specs:
+    sidekiq (0.6.0)
+      celluloid
+      connection_pool
+      multi_json
+      redis
+      redis-namespace
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -37,6 +47,7 @@ GEM
     i18n (0.6.0)
     journey (1.0.1)
     json (1.6.5)
+    json (1.6.5-java)
     mail (2.4.1)
       i18n (>= 0.4.0)
       mime-types (~> 1.16)
@@ -47,6 +58,8 @@ GEM
     rack (1.4.1)
     rack-cache (1.1)
       rack (>= 0.4)
+    rack-protection (1.2.0)
+      rack
     rack-ssl (1.3.2)
       rack
     rack-test (0.6.1)
@@ -70,11 +83,17 @@ GEM
     rdoc (3.12)
       json (~> 1.4)
     redis (2.2.2)
-    sidekiq (0.5.1)
-      celluloid
-      connection_pool
-      multi_json
-      redis
+    redis-namespace (1.0.3)
+      redis (< 3.0.0)
+    resque (1.19.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.0.2)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    sinatra (1.3.2)
+      rack (~> 1.3, >= 1.3.6)
+      rack-protection (~> 1.2)
+      tilt (~> 1.3, >= 1.3.3)
     sprockets (2.1.2)
       hike (~> 1.2)
       rack (~> 1.0)
@@ -86,11 +105,15 @@ GEM
       polyglot
       polyglot (>= 0.3.1)
     tzinfo (0.3.31)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
 
 PLATFORMS
+  java
   ruby
 
 DEPENDENCIES
   rails (= 3.2.1)
-  sidekiq
+  resque
+  sidekiq!
   sqlite3

data/myapp/app/controllers/work_controller.rb

@@ -0,0 +1,9 @@
+class WorkController < ApplicationController
+  def index
+    @count = rand(100)
+    puts "Adding #{@count} jobs"
+    @count.times do |x|
+      HardWorker.perform_async('bubba', x)
+    end
+  end
+end

data/myapp/app/views/work/index.html.erb

@@ -0,0 +1 @@
+Added <%= @count %> jobs!

data/myapp/app/workers/hard_worker.rb

@@ -1,6 +1,8 @@
 class HardWorker
+  include Sidekiq::Worker
+
   def perform(name, count)
-    sleep 0.01
-    puts 'done'
+    sleep 1
+    print "#{Time.now}\n"
   end
 end

data/myapp/config/environments/development.rb

@@ -34,4 +34,5 @@ Myapp::Application.configure do
 
   # Expands the lines which load the assets
   config.assets.debug = true
+  config.assets.logger = nil
 end

data/myapp/config/initializers/sidekiq.rb

@@ -0,0 +1 @@
+Sidekiq::Client.redis = Sidekiq::RedisConnection.create(:namespace => 'resque')

data/myapp/config/routes.rb

@@ -1,58 +1,6 @@
-Myapp::Application.routes.draw do
-  # The priority is based upon order of creation:
-  # first created -> highest priority.
-
-  # Sample of regular route:
-  #   match 'products/:id' => 'catalog#view'
-  # Keep in mind you can assign values other than :controller and :action
-
-  # Sample of named route:
-  #   match 'products/:id/purchase' => 'catalog#purchase', :as => :purchase
-  # This route can be invoked with purchase_url(:id => product.id)
-
-  # Sample resource route (maps HTTP verbs to controller actions automatically):
-  #   resources :products
-
-  # Sample resource route with options:
-  #   resources :products do
-  #     member do
-  #       get 'short'
-  #       post 'toggle'
-  #     end
-  #
-  #     collection do
-  #       get 'sold'
-  #     end
-  #   end
-
-  # Sample resource route with sub-resources:
-  #   resources :products do
-  #     resources :comments, :sales
-  #     resource :seller
-  #   end
+require 'resque/server'
 
-  # Sample resource route with more complex sub-resources
-  #   resources :products do
-  #     resources :comments
-  #     resources :sales do
-  #       get 'recent', :on => :collection
-  #     end
-  #   end
-
-  # Sample resource route within a namespace:
-  #   namespace :admin do
-  #     # Directs /admin/products/* to Admin::ProductsController
-  #     # (app/controllers/admin/products_controller.rb)
-  #     resources :products
-  #   end
-
-  # You can have the root of your site routed with "root"
-  # just remember to delete public/index.html.
-  # root :to => 'welcome#index'
-
-  # See how all your routes lay out with "rake routes"
-
-  # This is a legacy wild controller route that's not recommended for RESTful applications.
-  # Note: This route will make all actions in every controller accessible via GET requests.
-  # match ':controller(/:action(/:id))(.:format)'
+Myapp::Application.routes.draw do
+  mount Resque::Server.new, :at => '/resque'
+  get "work" => "work#index"
 end

data/sidekiq.gemspec

@@ -14,6 +14,7 @@ Gem::Specification.new do |gem|
   gem.require_paths = ["lib"]
   gem.version       = Sidekiq::VERSION
   gem.add_dependency                  'redis'
+  gem.add_dependency                  'redis-namespace'
   gem.add_dependency                  'connection_pool'
   gem.add_dependency                  'celluloid'
   gem.add_dependency                  'multi_json'

data/test/helper.rb

@@ -7,3 +7,6 @@ end
 require 'minitest/unit'
 require 'minitest/pride'
 require 'minitest/autorun'
+
+require 'sidekiq/util'
+Sidekiq::Util.logger.level = Logger::ERROR

data/test/test_cli.rb

@@ -0,0 +1,49 @@
+require 'helper'
+require 'sidekiq/cli'
+
+class TestCli < MiniTest::Unit::TestCase
+  describe 'with cli' do
+    before do
+      @cli = new_cli
+    end
+
+    it 'blows up with an invalid require' do
+      assert_raises ArgumentError do
+        @cli.parse(['sidekiq', '-r', 'foobar'])
+      end
+    end
+
+    it 'blows up with invalid Ruby' do
+      @cli.parse(['sidekiq', '-r', './test/fake_env.rb'])
+      assert($LOADED_FEATURES.any? { |x| x =~ /fake_env/ })
+      assert @cli.valid?
+    end
+
+    it 'changes concurrency' do
+      @cli.parse(['sidekiq', '-c', '60', '-r', './test/fake_env.rb'])
+      assert_equal 60, @cli.options[:processor_count]
+    end
+
+    it 'changes queues' do
+      @cli.parse(['sidekiq', '-q', 'foo', '-r', './test/fake_env.rb'])
+      assert_equal ['foo'], @cli.options[:queues]
+    end
+
+    it 'handles weights' do
+      @cli.parse(['sidekiq', '-q', 'foo,3', '-q', 'bar', '-r', './test/fake_env.rb'])
+      assert_equal %w(bar foo foo foo), @cli.options[:queues].sort
+    end
+
+    def new_cli
+      cli = Sidekiq::CLI.new
+      def cli.die(code)
+        @code = code
+      end
+
+      def cli.valid?
+        !@code
+      end
+      cli
+    end
+  end
+end

data/test/test_client.rb

@@ -3,9 +3,41 @@ require 'sidekiq/client'
 require 'sidekiq/worker'
 
 class TestClient < MiniTest::Unit::TestCase
+  describe 'with real redis' do
+    before do
+      Sidekiq::Client.redis = Redis.connect(:url => 'redis://localhost/sidekiq_test')
+      Sidekiq::Client.redis.flushdb
+    end
+
+    it 'does not push duplicate messages when configured for unique only' do
+      Sidekiq::Client.middleware.entries.clear
+      Sidekiq::Client.middleware.register do
+        use Sidekiq::Middleware::Client::UniqueJobs, Sidekiq::Client.redis
+        use Sidekiq::Middleware::Client::ResqueWebCompatibility, Sidekiq::Client.redis
+      end
+      10.times { Sidekiq::Client.push('customqueue', 'class' => 'Foo', 'args' => [1, 2]) }
+      assert_equal 1, Sidekiq::Client.redis.llen("queue:customqueue")
+    end
+
+    it 'does push duplicate messages when not configured for unique only' do
+      Sidekiq::Client.middleware.unregister(Sidekiq::Middleware::Client::UniqueJobs)
+      10.times { Sidekiq::Client.push('customqueue2', 'class' => 'Foo', 'args' => [1, 2]) }
+      assert_equal 10, Sidekiq::Client.redis.llen("queue:customqueue2")
+    end
+  end
+
   describe 'with mock redis' do
     before do
       @redis = MiniTest::Mock.new
+      def @redis.multi; yield; end
+      def @redis.set(*); true; end
+      def @redis.sadd(*); true; end
+      def @redis.srem(*); true; end
+      def @redis.get(*); nil; end
+      def @redis.del(*); nil; end
+      def @redis.incrby(*); nil; end
+      def @redis.setex(*); nil; end
+      def @redis.expire(*); true; end
       Sidekiq::Client.redis = @redis
     end
 
@@ -21,29 +53,39 @@ class TestClient < MiniTest::Unit::TestCase
 
     it 'pushes messages to redis' do
       @redis.expect :rpush, 1, ['queue:foo', String]
-      count = Sidekiq::Client.push('foo', 'class' => 'Foo', 'args' => [1, 2])
-      assert count > 0
+      pushed = Sidekiq::Client.push('foo', 'class' => 'Foo', 'args' => [1, 2])
+      assert pushed
       @redis.verify
     end
 
     class MyWorker
       include Sidekiq::Worker
-      def self.queue
-        'foo'
-      end
     end
 
     it 'handles perform_async' do
       @redis.expect :rpush, 1, ['queue:default', String]
-      count = MyWorker.perform_async(1, 2)
-      assert count > 0
+      pushed = MyWorker.perform_async(1, 2)
+      assert pushed
       @redis.verify
     end
 
     it 'enqueues messages to redis' do
-      @redis.expect :rpush, 1, ['queue:foo', String]
-      count = Sidekiq::Client.enqueue(MyWorker, 1, 2)
-      assert count > 0
+      @redis.expect :rpush, 1, ['queue:default', String]
+      pushed = Sidekiq::Client.enqueue(MyWorker, 1, 2)
+      assert pushed
+      @redis.verify
+    end
+
+    class QueuedWorker
+      include Sidekiq::Worker
+
+      queue :flimflam
+    end
+
+    it 'enqueues to the named queue' do
+      @redis.expect :rpush, 1, ['queue:flimflam', String]
+      pushed = QueuedWorker.perform_async(1, 2)
+      assert pushed
       @redis.verify
     end
   end

data/test/test_manager.rb

@@ -1,36 +1,41 @@
 require 'helper'
 require 'sidekiq'
 require 'sidekiq/manager'
-require 'timed_queue'
+
+# for TimedQueue
+require 'connection_pool'
 
 class TestManager < MiniTest::Unit::TestCase
   describe 'with redis' do
     before do
-      Sidekiq::Client.redis = @redis = Redis.connect(:url => 'redis://localhost/sidekiq_test')
+      Sidekiq::Manager.redis = Sidekiq::Client.redis = @redis = Sidekiq::RedisConnection.create(:url => 'redis://localhost/sidekiq_test')
       @redis.flushdb
       $processed = 0
+      $mutex = Mutex.new
     end
 
     class IntegrationWorker
       include Sidekiq::Worker
 
       def perform(a, b)
-        $processed += 1
+        $mutex.synchronize do
+          $processed += 1
+        end
         a + b
       end
     end
 
     it 'processes messages' do
       Sidekiq::Client.push(:foo, 'class' => IntegrationWorker, 'args' => [1, 2])
-      Sidekiq::Client.push(:foo, 'class' => IntegrationWorker, 'args' => [1, 2])
+      Sidekiq::Client.push(:foo, 'class' => IntegrationWorker, 'args' => [1, 3])
 
       q = TimedQueue.new
-      mgr = Sidekiq::Manager.new("redis://localhost/sidekiq_test", :queues => [:foo])
+      mgr = Sidekiq::Manager.new(:queues => [:foo], :processor_count => 2)
       mgr.when_done do |_|
         q << 'done' if $processed == 2
       end
       mgr.start!
-      result = q.timed_pop
+      result = q.timed_pop(1.0)
       assert_equal 'done', result
       mgr.stop
     end

data/test/test_middleware.rb

@@ -1,12 +1,13 @@
 require 'helper'
-require 'sidekiq/middleware'
+require 'sidekiq/middleware/chain'
+require 'sidekiq/middleware/server/unique_jobs'
 require 'sidekiq/processor'
 
 class TestMiddleware < MiniTest::Unit::TestCase
   describe 'middleware chain' do
     before do
-      @boss = MiniTest::Mock.new
-      Celluloid.logger = nil
+      Sidekiq::Manager.redis = @redis = Sidekiq::RedisConnection.create(:url => 'redis://localhost/sidekiq_test')
+      Sidekiq::Client.redis = nil
     end
 
     class CustomMiddleware
@@ -15,48 +16,66 @@ class TestMiddleware < MiniTest::Unit::TestCase
         @recorder = recorder
       end
 
-      def call(worker, msg)
+      def call(*args)
         @recorder << [@name, 'before']
         yield
         @recorder << [@name, 'after']
       end
     end
 
-    it 'configures default middleware' do
-      chain = Sidekiq::Middleware::Chain.chain
-      assert_equal chain, Sidekiq::Middleware::Chain.default
-    end
-
     it 'supports custom middleware' do
-      Sidekiq::Middleware::Chain.register do
+      chain = Sidekiq::Middleware::Chain.new
+      chain.register do
         use CustomMiddleware, 1, []
       end
-      chain = Sidekiq::Middleware::Chain.chain
-      assert_equal chain.last.klass, CustomMiddleware
+
+      assert_equal CustomMiddleware, chain.entries.last.klass
     end
 
     class CustomWorker
+      include Sidekiq::Worker
       def perform(recorder)
         recorder << ['work_performed']
       end
     end
 
+    class NonYieldingMiddleware
+      def call(*args)
+      end
+    end
+
     it 'executes middleware in the proper order' do
+      Sidekiq::Middleware::Server::UniqueJobs.class_eval do
+        def call(*args); yield; end
+      end
+
       recorder = []
       msg = { 'class' => CustomWorker.to_s, 'args' => [recorder] }
 
-      Sidekiq::Middleware::Chain.register do
+      Sidekiq::Processor.middleware.register do
         2.times { |i| use CustomMiddleware, i.to_s, recorder }
       end
 
-      processor = Sidekiq::Processor.new(@boss)
-      @boss.expect(:processor_done!, nil, [processor])
-      processor.process(msg)
-      assert_equal recorder.flatten, %w(0 before 1 before work_performed 1 after 0 after)
+      boss = MiniTest::Mock.new
+      processor = Sidekiq::Processor.new(boss)
+      boss.expect(:processor_done!, nil, [processor])
+      processor.process(msg, 'default')
+      assert_equal %w(0 before work_performed 0 after), recorder.flatten
     end
-  end
-end
-
 
+    it 'allows middleware to abruptly stop processing rest of chain' do
+      recorder = []
+      chain = Sidekiq::Middleware::Chain.new
 
+      chain.register do
+        use NonYieldingMiddleware
+        use CustomMiddleware, 1, recorder
+      end
 
+      final_action = nil
+      chain.invoke { final_action = true }
+      assert_equal nil, final_action
+      assert_equal [], recorder
+    end
+  end
+end