sidekiq 2.3.0 → 2.3.1

data/Changes.md

@@ -1,3 +1,11 @@
+2.3.1
+-----------
+
+- Add Sidekiq::Client.push\_bulk for bulk adding of jobs to Redis.
+  My own simple test case shows pushing 10,000 jobs goes from 5 sec to 1.5 sec.
+- Add support for multiple processes per host to Capistrano recipe
+- Re-enable Celluloid::Actor#defer to fix stack overflow issues [#398]
+
 2.3.0
 -----------
 

data/lib/sidekiq.rb

@@ -14,6 +14,7 @@ require 'sidekiq/rails' if defined?(::Rails::Engine)
 require 'multi_json'
 
 module Sidekiq
+  NAME = "Sidekiq"
   LICENSE = 'See LICENSE and the LGPL-3.0 for licensing details.'
 
   DEFAULTS = {

data/lib/sidekiq/capistrano.rb

@@ -4,26 +4,38 @@ Capistrano::Configuration.instance.load do
   after "deploy:start",   "sidekiq:start"
   after "deploy:restart", "sidekiq:restart"
 
-  _cset(:sidekiq_timeout) { 10 }
-  _cset(:sidekiq_role)    { :app }
-  _cset(:sidekiq_pid)     { "#{current_path}/tmp/pids/sidekiq.pid" }
+  _cset(:sidekiq_timeout)   { 10 }
+  _cset(:sidekiq_role)      { :app }
+  _cset(:sidekiq_pid)       { "#{current_path}/tmp/pids/sidekiq.pid" }
+  _cset(:sidekiq_processes) { 1 }
 
   namespace :sidekiq do
+    def for_each_process(&block)
+      0.upto(fetch(:sidekiq_processes) - 1) do |process|
+        yield process == 0 ? "#{fetch(:sidekiq_pid)}" : "#{fetch(:sidekiq_pid)}-#{process}"
+      end
+    end
 
     desc "Quiet sidekiq (stop accepting new work)"
     task :quiet, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
-      run "if [ -d #{current_path} ] && [ -f #{fetch :sidekiq_pid} ]; then cd #{current_path} && #{fetch(:bundle_cmd, "bundle")} exec sidekiqctl quiet #{fetch :sidekiq_pid} ; fi"
+      for_each_process do |pid_file|
+        run "if [ -d #{current_path} ] && [ -f #{pid_file} ]; then cd #{current_path} && #{fetch(:bundle_cmd, "bundle")} exec sidekiqctl quiet #{pid_file} ; fi"
+      end
     end
 
     desc "Stop sidekiq"
     task :stop, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
-      run "if [ -d #{current_path} ] && [ -f #{fetch :sidekiq_pid} ]; then cd #{current_path} && #{fetch(:bundle_cmd, "bundle")} exec sidekiqctl stop #{fetch :sidekiq_pid} #{fetch :sidekiq_timeout} ; fi"
+      for_each_process do |pid_file|
+        run "if [ -d #{current_path} ] && [ -f #{pid_file} ]; then cd #{current_path} && #{fetch(:bundle_cmd, "bundle")} exec sidekiqctl stop #{pid_file} #{fetch :sidekiq_timeout} ; fi"
+      end
     end
 
     desc "Start sidekiq"
     task :start, :roles => lambda { fetch(:sidekiq_role) }, :on_no_matching_servers => :continue do
       rails_env = fetch(:rails_env, "production")
-      run "cd #{current_path} ; nohup #{fetch(:bundle_cmd, "bundle")} exec sidekiq -e #{rails_env} -C #{current_path}/config/sidekiq.yml -P #{fetch :sidekiq_pid} >> #{current_path}/log/sidekiq.log 2>&1 &", :pty => false
+      for_each_process do |pid_file|
+        run "cd #{current_path} ; nohup #{fetch(:bundle_cmd, "bundle")} exec sidekiq -e #{rails_env} -C #{current_path}/config/sidekiq.yml -P #{pid_file} >> #{current_path}/log/sidekiq.log 2>&1 &", :pty => false
+      end
     end
 
     desc "Restart sidekiq"

data/lib/sidekiq/client.rb

@@ -36,33 +36,52 @@ module Sidekiq
     #   Sidekiq::Client.push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
     #
     def self.push(item)
-      raise(ArgumentError, "Message must be a Hash of the form: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
-      raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item['class'] || !item['args']
-      raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item['class'].ancestors.inspect}") if !item['class'].is_a?(Class) || !item['class'].respond_to?('get_sidekiq_options')
+      normed = normalize_item(item)
+      normed, payload = process_single(item['class'], normed)
 
-      worker_class = item['class']
-      item['class'] = item['class'].to_s
+      pushed = false
+      Sidekiq.redis do |conn|
+        if normed['at']
+          pushed = conn.zadd('schedule', normed['at'].to_s, payload)
+        else
+          _, pushed = conn.multi do
+            conn.sadd('queues', normed['queue'])
+            conn.rpush("queue:#{normed['queue']}", payload)
+          end
+        end
+      end if normed
+      pushed ? normed['jid'] : nil
+    end
 
-      item = worker_class.get_sidekiq_options.merge(item)
-      item['retry'] = !!item['retry']
-      queue = item['queue']
-      item['jid'] = SecureRandom.hex(12)
+    ##
+    # Push a large number of jobs to Redis.  In practice this method is only
+    # useful if you are pushing tens of thousands of jobs or more.  This method
+    # basically cuts down on the redis round trip latency.
+    #
+    # Takes the same arguments as Client.push except that args is expected to be
+    # an Array of Arrays.  All other keys are duplicated for each job.  Each job
+    # is run through the client middleware pipeline and each job gets its own Job ID
+    # as normal.
+    #
+    # Returns the number of jobs pushed or nil if the pushed failed.  The number of jobs
+    # pushed can be less than the number given if the middleware stopped processing for one
+    # or more jobs.
+    def self.push_bulk(items)
+      normed = normalize_item(items)
+      payloads = items['args'].map do |args|
+        _, payload = process_single(items['class'], normed.merge('args' => args, 'jid' => SecureRandom.hex(12)))
+        payload
+      end.compact
 
       pushed = false
-      Sidekiq.client_middleware.invoke(worker_class, item, queue) do
-        payload = Sidekiq.dump_json(item)
-        Sidekiq.redis do |conn|
-          if item['at']
-            pushed = conn.zadd('schedule', item['at'].to_s, payload)
-          else
-            _, pushed = conn.multi do
-              conn.sadd('queues', queue)
-              conn.rpush("queue:#{queue}", payload)
-            end
-          end
+      Sidekiq.redis do |conn|
+        _, pushed = conn.multi do
+          conn.sadd('queues', normed['queue'])
+          conn.rpush("queue:#{normed['queue']}", payloads)
         end
       end
-      pushed ? item['jid'] : nil
+
+      pushed ? payloads.size : nil
     end
 
     # Resque compatibility helpers.
@@ -82,5 +101,30 @@ module Sidekiq
     def self.enqueue_to(queue, klass, *args)
       klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
     end
+
+    private
+
+    def self.process_single(worker_class, item)
+      queue = item['queue']
+
+      Sidekiq.client_middleware.invoke(worker_class, item, queue) do
+        payload = Sidekiq.dump_json(item)
+        return item, payload
+      end
+    end
+
+    def self.normalize_item(item)
+      raise(ArgumentError, "Message must be a Hash of the form: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
+      raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item['class'] || !item['args']
+      raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item['class'].ancestors.inspect}") if !item['class'].is_a?(Class) || !item['class'].respond_to?('get_sidekiq_options')
+
+      normalized_item = item.dup
+      normalized_item['class'] = normalized_item['class'].to_s
+      normalized_item['retry'] = !!normalized_item['retry']
+      normalized_item['jid'] = SecureRandom.hex(12)
+
+      item['class'].get_sidekiq_options.merge normalized_item
+    end
+
   end
 end

data/lib/sidekiq/processor.rb

@@ -1,3 +1,4 @@
+require 'socket'
 require 'celluloid'
 require 'sidekiq/util'
 
@@ -31,19 +32,21 @@ module Sidekiq
     end
 
     def process(msgstr, queue)
-      begin
-        msg = Sidekiq.load_json(msgstr)
-        klass  = constantize(msg['class'])
-        worker = klass.new
+      defer do
+        begin
+          msg = Sidekiq.load_json(msgstr)
+          klass  = constantize(msg['class'])
+          worker = klass.new
 
-        stats(worker, msg, queue) do
-          Sidekiq.server_middleware.invoke(worker, msg, queue) do
-            worker.perform(*cloned(msg['args']))
+          stats(worker, msg, queue) do
+            Sidekiq.server_middleware.invoke(worker, msg, queue) do
+              worker.perform(*cloned(msg['args']))
+            end
           end
+          rescue Exception => ex
+            handle_exception(ex, msg || { :message => msgstr })
+          raise
         end
-      rescue Exception => ex
-        handle_exception(ex, msg || { :message => msgstr })
-        raise
       end
       @boss.processor_done!(current_actor)
     end
@@ -105,7 +108,7 @@ module Sidekiq
     end
 
     def hostname
-      @h ||= `hostname`.strip
+      @h ||= Socket.gethostname
     end
   end
 end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "2.3.0"
+  VERSION = "2.3.1"
 end

data/test/test_client.rb

@@ -99,6 +99,14 @@ class TestClient < MiniTest::Unit::TestCase
     end
   end
 
+  describe 'bulk' do
+    it 'can push a large set of jobs at once' do
+      a = Time.now
+      count = Sidekiq::Client.push_bulk('class' => QueuedWorker, 'args' => (1..1_000).to_a.map { |x| Array(x) })
+      assert_equal 1_000, count
+    end
+  end
+
   class BaseWorker
     include Sidekiq::Worker
     sidekiq_options 'retry' => 'base'
@@ -109,6 +117,26 @@ class TestClient < MiniTest::Unit::TestCase
     sidekiq_options 'retry' => 'b'
   end
 
+  describe 'client middleware' do
+
+    class Stopper
+      def call(worker_class, message, queue)
+        yield if message['args'].first.odd?
+      end
+    end
+
+    it 'can stop some of the jobs from pushing' do
+      Sidekiq.client_middleware.add Stopper
+      begin
+        assert_equal nil, Sidekiq::Client.push('class' => MyWorker, 'args' => [0])
+        assert_match /[0-9a-f]{12}/, Sidekiq::Client.push('class' => MyWorker, 'args' => [1])
+        assert_equal 1, Sidekiq::Client.push_bulk('class' => MyWorker, 'args' => [[0], [1]])
+      ensure
+        Sidekiq.client_middleware.remove Stopper
+      end
+    end
+  end
+
   describe 'inheritance' do
     it 'should inherit sidekiq options' do
       assert_equal 'base', AWorker.get_sidekiq_options['retry']

data/web/views/layout.slim

@@ -2,7 +2,7 @@ doctype html
 html
   head
     link href='#{{root_path}}assets/application.css' media='screen' rel='stylesheet' type='text/css'
-    title Sidekiq
+    title= Sidekiq::NAME
   body
     .navbar.navbar-fixed-top.navbar-inverse
       .navbar-inner
@@ -12,7 +12,7 @@ html
             span.icon-bar
             span.icon-bar
           a.brand href='#{{root_path}}'
-            | Sidekiq
+            = Sidekiq::NAME
           div.nav-collapse
             ul.nav
               li

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq
 version: !ruby/object:Gem::Version
-  version: 2.3.0
+  version: 2.3.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-09 00:00:00.000000000 Z
+date: 2012-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis