sidekiq 0.6.0 → 0.7.0

data/lib/sidekiq/cli.rb

@@ -9,8 +9,9 @@ trap 'TERM' do
   Thread.main.raise Interrupt
 end
 
+require 'yaml'
 require 'optparse'
-require 'sidekiq/version'
+require 'sidekiq'
 require 'sidekiq/util'
 require 'sidekiq/redis_connection'
 require 'sidekiq/manager'
@@ -20,7 +21,7 @@ module Sidekiq
     include Util
 
     # Used for CLI testing
-    attr_accessor :options, :code
+    attr_accessor :code
 
     def initialize
       @code = nil
@@ -28,14 +29,21 @@ module Sidekiq
 
     def parse(args=ARGV)
       Sidekiq::Util.logger
-      parse_options(args)
+
+      cli = parse_options(args)
+      config = parse_config(cli)
+      options.merge!(config.merge(cli))
+
+      set_logger_level_to_debug if options[:verbose]
+
+      write_pid
       validate!
       boot_system
     end
 
     def run
-      Sidekiq::Manager.redis = RedisConnection.create(:url => @options[:server], :namespace => @options[:namespace])
-      manager = Sidekiq::Manager.new(@options)
+      Sidekiq.redis ||= RedisConnection.create(:url => options[:server], :namespace => options[:namespace])
+      manager = Sidekiq::Manager.new(options)
       begin
         logger.info 'Starting processing, hit Ctrl-C to stop'
         manager.start!
@@ -44,7 +52,7 @@ module Sidekiq
         sleep
       rescue Interrupt
         # TODO Need clean shutdown support from Celluloid
-        logger.info 'Shutting down, pausing 5 seconds to let workers finish...'
+        logger.info 'Shutting down'
         manager.stop!
         manager.wait(:shutdown)
       end
@@ -56,29 +64,33 @@ module Sidekiq
       exit(code)
     end
 
+    def options
+      Sidekiq.options
+    end
+
     def detected_environment
-      @options[:environment] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      options[:environment] ||= ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
     end
 
     def boot_system
       ENV['RACK_ENV'] = ENV['RAILS_ENV'] = detected_environment
 
-      raise ArgumentError, "#{@options[:require]} does not exist" if !File.exist?(@options[:require])
+      raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
 
-      if File.directory?(@options[:require])
-        require File.expand_path("#{@options[:require]}/config/environment.rb")
+      if File.directory?(options[:require])
+        require File.expand_path("#{options[:require]}/config/environment.rb")
         ::Rails.application.eager_load!
       else
-        require @options[:require]
+        require options[:require]
       end
     end
 
     def validate!
-      @options[:queues] << 'default' if @options[:queues].empty?
-      @options[:queues].shuffle!
+      options[:queues] << 'default' if options[:queues].empty?
+      options[:queues].shuffle!
 
-      if !File.exist?(@options[:require]) ||
-         (File.directory?(@options[:require]) && !File.exist?("#{@options[:require]}/config/application.rb"))
+      if !File.exist?(options[:require]) ||
+         (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
         logger.info "=================================================================="
         logger.info "  Please point sidekiq to a Rails 3 application or a Ruby file    "
         logger.info "  to load your worker classes with -r [DIR|FILE]."
@@ -89,43 +101,44 @@ module Sidekiq
     end
 
     def parse_options(argv)
-      @options = {
-        :queues => [],
-        :processor_count => 25,
-        :require => '.',
-        :environment => nil,
-      }
+      opts = {}
 
       @parser = OptionParser.new do |o|
         o.on "-q", "--queue QUEUE,WEIGHT", "Queue to process, with optional weight" do |arg|
           (q, weight) = arg.split(",")
-          (weight || 1).to_i.times do
-            @options[:queues] << q
-          end
+          parse_queues(opts, q, weight)
         end
 
         o.on "-v", "--verbose", "Print more verbose output" do
-          Sidekiq::Util.logger.level = Logger::DEBUG
+          set_logger_level_to_debug
         end
 
         o.on "-n", "--namespace NAMESPACE", "namespace worker queues are under" do |arg|
-          @options[:namespace] = arg
+          opts[:namespace] = arg
         end
 
         o.on "-s", "--server LOCATION", "Where to find Redis" do |arg|
-          @options[:server] = arg
+          opts[:server] = arg
         end
 
         o.on '-e', '--environment ENV', "Application environment" do |arg|
-          @options[:environment] = arg
+          opts[:environment] = arg
         end
 
         o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
-          @options[:require] = arg
+          opts[:require] = arg
         end
 
         o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
-          @options[:processor_count] = arg.to_i
+          opts[:concurrency] = arg.to_i
+        end
+
+        o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
+          opts[:pidfile] = arg
+        end
+
+        o.on '-C', '--config PATH', "path to YAML config file" do |arg|
+          opts[:config_file] = arg
         end
       end
 
@@ -135,6 +148,35 @@ module Sidekiq
         die 1
       end
       @parser.parse!(argv)
+      opts
+    end
+
+    def write_pid
+      if path = options[:pidfile]
+        File.open(path, 'w') do |f|
+          f.puts Process.pid
+        end
+      end
+    end
+
+    def parse_config(cli)
+      opts = {}
+      if cli[:config_file] && File.exist?(cli[:config_file])
+        opts = YAML.load_file cli[:config_file]
+        queues = opts.delete(:queues) || []
+        queues.each { |pair| parse_queues(opts, *pair) }
+      end
+      opts
+    end
+
+    def parse_queues(opts, q, weight)
+      (weight || 1).to_i.times do
+        (opts[:queues] ||= []) << q
+      end
+    end
+
+    def set_logger_level_to_debug
+      Sidekiq::Util.logger.level = Logger::DEBUG
     end
 
   end

data/lib/sidekiq/client.rb

@@ -10,26 +10,26 @@ module Sidekiq
   class Client
 
     def self.middleware
-      @middleware ||= begin
-        m = Middleware::Chain.new
-        m.register do
-          use Middleware::Client::UniqueJobs, Client.redis
-          use Middleware::Client::ResqueWebCompatibility, Client.redis
-        end
-        m
+      raise "Sidekiq::Client.middleware is now Sidekiq.client_middleware"
+    end
+
+    def self.default_middleware
+      Middleware::Chain.new do |m|
+        m.add Middleware::Client::UniqueJobs
+        m.add Middleware::Client::ResqueWebCompatibility
       end
     end
 
-    def self.queues
-      @queues ||= {}
+    def self.registered_workers
+      Sidekiq.redis.smembers('workers')
     end
 
-    def self.redis
-      @redis ||= RedisConnection.create
+    def self.registered_queues
+      Sidekiq.redis.smembers('queues')
     end
 
-    def self.redis=(redis)
-      @redis = redis
+    def self.queue_mappings
+      @queue_mappings ||= {}
     end
 
     # Example usage:
@@ -38,13 +38,13 @@ module Sidekiq
       raise(ArgumentError, "Message must be a Hash of the form: { 'class' => SomeClass, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
       raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item['class'] || !item['args']
 
-      queue = queue || queues[item['class'].to_s] || 'default'
+      queue = queue || queue_mappings[item['class'].to_s] || 'default'
 
       item['class'] = item['class'].to_s if !item['class'].is_a?(String)
 
       pushed = false
-      middleware.invoke(item, queue) do
-        redis.rpush("queue:#{queue}", MultiJson.encode(item))
+      Sidekiq.client_middleware.invoke(item, queue) do
+        Sidekiq.redis.rpush("queue:#{queue}", MultiJson.encode(item))
         pushed = true
       end
       pushed

data/lib/sidekiq/extensions/action_mailer.rb

@@ -0,0 +1,27 @@
+require 'sidekiq/extensions/generic_proxy'
+
+module Sidekiq
+  module Extensions
+    ##
+    # Adds a 'delay' method to ActionMailer to offload arbitrary email
+    # delivery to Sidekiq.  Example:
+    #
+    #    UserMailer.delay.send_welcome_email(new_user)
+    class DelayedMailer
+      include Sidekiq::Worker
+
+      def perform(yml)
+        (target, method_name, args) = YAML.load(yml)
+        target.send(method_name, *args).deliver
+      end
+    end
+
+    module ActionMailer
+      def delay
+        Proxy.new(DelayedMailer, self)
+      end
+    end
+
+    ::ActionMailer::Base.extend(ActionMailer)
+  end
+end if defined?(::ActionMailer)

data/lib/sidekiq/extensions/active_record.rb

@@ -0,0 +1,29 @@
+require 'sidekiq/extensions/generic_proxy'
+
+module Sidekiq
+  module Extensions
+    ##
+    # Adds a 'delay' method to ActiveRecord to offload arbitrary method
+    # execution to Sidekiq.  Examples:
+    #
+    # User.delay.delete_inactive
+    # User.recent_signups.each { |user| user.delay.mark_as_awesome }
+    class DelayedModel
+      include Sidekiq::Worker
+
+      def perform(yml)
+        (target, method_name, args) = YAML.load(yml)
+        target.send(method_name, *args)
+      end
+    end
+
+    module ActiveRecord
+      def delay
+        Proxy.new(DelayedModel, self)
+      end
+    end
+
+    ::ActiveRecord::Base.extend(ActiveRecord)
+    ::ActiveRecord::Base.send(:include, ActiveRecord)
+  end
+end if defined?(::ActiveRecord)

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -0,0 +1,21 @@
+module Sidekiq
+  module Extensions
+    class Proxy < ::BasicObject
+      def initialize(performable, target)
+        @performable = performable
+        @target = target
+      end
+
+      def method_missing(name, *args)
+        # Sidekiq has a limitation in that its message must be JSON.
+        # JSON can't round trip real Ruby objects so we use YAML to
+        # serialize the objects to a String.  The YAML will be converted
+        # to JSON and then deserialized on the other side back into a
+        # Ruby object.
+        obj = [@target, name, args]
+        ::Sidekiq::Client.push('class' => @performable.name, 'args' => [::YAML.dump(obj)])
+      end
+    end
+
+  end
+end

data/lib/sidekiq/manager.rb

@@ -18,14 +18,10 @@ module Sidekiq
 
     trap_exit :processor_died
 
-    class << self
-      attr_accessor :redis
-    end
-
     def initialize(options={})
       logger.info "Booting sidekiq #{Sidekiq::VERSION} with Redis at #{redis.client.location}"
       logger.debug { options.inspect }
-      @count = options[:processor_count] || 25
+      @count = options[:concurrency] || 25
       @queues = options[:queues]
       @done_callback = nil
 
@@ -39,15 +35,23 @@ module Sidekiq
       @ready.each(&:terminate)
       @ready.clear
 
-      after(5) do
-        signal(:shutdown)
-      end
-
       redis.with_connection do |conn|
-        conn.smembers('workers').each do |name|
-          conn.srem('workers', name) if name =~ /:#{Process.pid}:/
+        workers = conn.smembers('workers')
+        workers.each do |name|
+          conn.srem('workers', name) if name =~ /:#{process_id}-/
         end
       end
+
+      if @busy.empty?
+        return signal(:shutdown)
+      end
+
+      logger.info("Pausing 5 seconds to allow workers to finish...")
+      after(5) do
+        @busy.each(&:terminate)
+        #@busy.each(&:requeue)
+        signal(:shutdown)
+      end
     end
 
     def start
@@ -63,7 +67,7 @@ module Sidekiq
         @done_callback.call(processor) if @done_callback
         @busy.delete(processor)
         if stopped?
-          processor.terminate
+          processor.terminate if processor.alive?
         else
           @ready << processor
         end

data/lib/sidekiq/middleware/chain.rb

@@ -5,34 +5,37 @@ module Sidekiq
   # (pushing jobs onto the queue) as well as the server
   # side (when jobs are actually processed).
   #
-  # Default middleware for the server side:
+  # To add middleware for the client:
   #
-  # Sidekiq::Processor.middleware.register do
-  #   use Middleware::Server::Airbrake
-  #   use Middleware::Server::ActiveRecord
+  # Sidekiq.client_middleware do |chain|
+  #   chain.add MyClientHook
   # end
   #
-  # To add middleware for the client, do:
+  # To modify middleware for the server, just call
+  # with another block:
   #
-  # Sidekiq::Client.middleware.register do
-  #  use MyClientHook
+  # Sidekiq.server_middleware do |chain|
+  #   chain.add MyServerHook
+  #   chain.remove ActiveRecord
   # end
   #
-  # To add middleware for the server, do:
+  # This is an example of a minimal server middleware:
   #
-  # Sidekiq::Processor.middleware.register do
-  #   use MyServerHook
+  # class MyServerHook
+  #   def call(worker, msg, queue)
+  #     puts "Before work"
+  #     yield
+  #     puts "After work"
+  #   end
   # end
   #
-  # This is an example of a minimal middleware:
+  # This is an example of a minimal client middleware:
   #
-  # class MyHook
-  #   def initialize(options=nil)
-  #   end
-  #   def call(worker, msg)
-  #     puts "Before work"
+  # class MyClientHook
+  #   def call(msg, queue)
+  #     puts "Before push"
   #     yield
-  #     puts "After work"
+  #     puts "After push"
   #   end
   # end
   #
@@ -42,17 +45,14 @@ module Sidekiq
 
       def initialize
         @entries = []
+        yield self if block_given?
       end
 
-      def register(&block)
-        instance_eval(&block)
-      end
-
-      def unregister(klass)
+      def remove(klass)
         entries.delete_if { |entry| entry.klass == klass }
       end
 
-      def use(klass, *args)
+      def add(klass, *args)
         entries << Entry.new(klass, *args) unless exists?(klass)
       end