sidekiq 0.5.1 → 0.6.0

data/lib/sidekiq/middleware/chain.rb

@@ -0,0 +1,92 @@
+module Sidekiq
+  # Middleware is code configured to run before/after
+  # a message is processed.  It is patterned after Rack
+  # middleware. Middleware exists for the client side
+  # (pushing jobs onto the queue) as well as the server
+  # side (when jobs are actually processed).
+  #
+  # Default middleware for the server side:
+  #
+  # Sidekiq::Processor.middleware.register do
+  #   use Middleware::Server::Airbrake
+  #   use Middleware::Server::ActiveRecord
+  # end
+  #
+  # To add middleware for the client, do:
+  #
+  # Sidekiq::Client.middleware.register do
+  #  use MyClientHook
+  # end
+  #
+  # To add middleware for the server, do:
+  #
+  # Sidekiq::Processor.middleware.register do
+  #   use MyServerHook
+  # end
+  #
+  # This is an example of a minimal middleware:
+  #
+  # class MyHook
+  #   def initialize(options=nil)
+  #   end
+  #   def call(worker, msg)
+  #     puts "Before work"
+  #     yield
+  #     puts "After work"
+  #   end
+  # end
+  #
+  module Middleware
+    class Chain
+      attr_reader :entries
+
+      def initialize
+        @entries = []
+      end
+
+      def register(&block)
+        instance_eval(&block)
+      end
+
+      def unregister(klass)
+        entries.delete_if { |entry| entry.klass == klass }
+      end
+
+      def use(klass, *args)
+        entries << Entry.new(klass, *args) unless exists?(klass)
+      end
+
+      def exists?(klass)
+        entries.any? { |entry| entry.klass == klass }
+      end
+
+      def retrieve
+        entries.map(&:make_new)
+      end
+
+      def invoke(*args, &final_action)
+        chain = retrieve.dup
+        traverse_chain = lambda do
+          if chain.empty?
+            final_action.call
+          else
+            chain.shift.call(*args, &traverse_chain)
+          end
+        end
+        traverse_chain.call
+      end
+    end
+
+    class Entry
+      attr_reader :klass
+      def initialize(klass, *args)
+        @klass = klass
+        @args  = args
+      end
+
+      def make_new
+        @klass.new(*@args)
+      end
+    end
+  end
+end

data/lib/sidekiq/middleware/client/resque_web_compatibility.rb

@@ -0,0 +1,17 @@
+module Sidekiq
+  module Middleware
+    module Client
+      class ResqueWebCompatibility
+        def initialize(redis)
+          @redis = redis
+        end
+
+        def call(item, queue)
+          @redis.sadd('queues', queue)
+          yield
+        end
+
+      end
+    end
+  end
+end

data/lib/sidekiq/middleware/client/unique_jobs.rb

@@ -0,0 +1,30 @@
+require 'digest'
+
+module Sidekiq
+  module Middleware
+    module Client
+      class UniqueJobs
+        HASH_KEY_EXPIRATION = 30 * 60
+
+        def initialize(redis)
+          @redis = redis
+        end
+
+        def call(item, queue)
+          payload_hash = Digest::MD5.hexdigest(MultiJson.encode(item))
+          return if already_scheduled?(payload_hash)
+
+          @redis.setex(payload_hash, HASH_KEY_EXPIRATION, 1)
+
+          yield
+        end
+
+        private
+
+        def already_scheduled?(payload_hash)
+          !!@redis.get(payload_hash)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/middleware/server/active_record.rb

@@ -0,0 +1,13 @@
+module Sidekiq
+  module Middleware
+    module Server
+      class ActiveRecord
+        def call(*args)
+          yield
+        ensure
+          ::ActiveRecord::Base.clear_active_connections! if defined?(::ActiveRecord)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/middleware/server/airbrake.rb

@@ -0,0 +1,30 @@
+require 'sidekiq/util'
+
+module Sidekiq
+  module Middleware
+    module Server
+      class Airbrake
+        include Util
+        def call(*args)
+          yield
+        rescue => ex
+          logger.warn ex
+          logger.warn ex.backtrace.join("\n")
+          send_to_airbrake(args[1], ex) if defined?(::Airbrake)
+          raise
+        end
+
+        private
+
+        def send_to_airbrake(msg, ex)
+          ::Airbrake.notify(:error_class   => ex.class.name,
+                            :error_message => "#{ex.class.name}: #{ex.message}",
+                            :parameters    => msg)
+        end
+      end
+    end
+  end
+end
+
+
+

data/lib/sidekiq/middleware/server/unique_jobs.rb

@@ -0,0 +1,17 @@
+module Sidekiq
+  module Middleware
+    module Server
+      class UniqueJobs
+        def initialize(redis)
+          @redis = redis
+        end
+
+        def call(*args)
+          yield
+        ensure
+          @redis.del(Digest::MD5.hexdigest(MultiJson.encode(args[1])))
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/processor.rb

@@ -1,37 +1,95 @@
-require 'sidekiq/util'
-require 'sidekiq/middleware'
 require 'celluloid'
 
+require 'sidekiq/util'
+require 'sidekiq/middleware/chain'
+require 'sidekiq/middleware/server/active_record'
+require 'sidekiq/middleware/server/airbrake'
+require 'sidekiq/middleware/server/unique_jobs'
+
 module Sidekiq
   class Processor
     include Util
     include Celluloid
 
-    def initialize(boss)
-      @boss = boss
+    def self.middleware
+      @middleware ||= begin
+        chain = Middleware::Chain.new
+
+        # default middleware
+        chain.register do
+          use Middleware::Server::Airbrake
+          use Middleware::Server::UniqueJobs, Sidekiq::Manager.redis
+          use Middleware::Server::ActiveRecord
+        end
+        chain
+      end
     end
 
-    def process(msg)
-      klass = constantize(msg['class'])
-      invoke_chain(klass.new, msg)
-      @boss.processor_done!(current_actor)
+    def initialize(boss)
+      @boss = boss
+      redis.sadd('workers', self)
     end
 
-    def invoke_chain(worker, msg)
-      chain = Sidekiq::Middleware::Chain.retrieve.dup
-      traverse_chain = lambda do
-        if chain.empty?
+    def process(msg, queue)
+      klass  = constantize(msg['class'])
+      worker = klass.new
+      stats(worker, msg, queue) do
+        self.class.middleware.invoke(worker, msg, queue) do
           worker.perform(*msg['args'])
-        else
-          chain.shift.call(worker, msg, &traverse_chain)
         end
       end
-      traverse_chain.call
+      @boss.processor_done!(current_actor)
     end
 
     # See http://github.com/tarcieri/celluloid/issues/22
     def inspect
-      "Sidekiq::Processor<#{object_id}>"
+      "#<Processor #{to_s}>"
+    end
+
+    def to_s
+      @str ||= "#{hostname}:#{Process.pid}-#{Thread.current.object_id}:default"
+    end
+
+    private
+
+    def stats(worker, msg, queue)
+      redis.with_connection do |conn|
+        conn.multi do
+          conn.set("worker:#{self}:started", Time.now.to_s)
+          conn.set("worker:#{self}", MultiJson.encode(:queue => queue, :payload => msg,
+                                                   :run_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z")))
+        end
+      end
+
+      dying = false
+      begin
+        yield
+      rescue
+        dying = true
+        # Uh oh, error.  We will die so unregister as much as we can first.
+        redis.with_connection do |conn|
+          conn.multi do
+            conn.incrby("stat:failed", 1)
+            conn.del("stat:processed:#{self}")
+            conn.srem("workers", self)
+          end
+        end
+        raise
+      ensure
+        redis.with_connection do |conn|
+          conn.multi do
+            conn.del("worker:#{self}")
+            conn.del("worker:#{self}:started")
+            conn.incrby("stat:processed", 1)
+            conn.incrby("stat:processed:#{self}", 1) unless dying
+          end
+        end
+      end
+
+    end
+
+    def hostname
+      @h ||= `hostname`.strip
     end
   end
 end

data/lib/sidekiq/redis_connection.rb

@@ -0,0 +1,23 @@
+require 'connection_pool'
+require 'redis/namespace'
+
+module Sidekiq
+  class RedisConnection
+    def self.create(options={})
+      url = options[:url] || ENV['REDISTOGO_URL'] || 'redis://localhost:6379/0'
+      client = build_client(url, options[:namespace])
+      return ConnectionPool.new(:timeout => 1, :size => 25) { client } unless options[:use_pool] == false
+      client
+    end
+
+    def self.build_client(url, namespace)
+      client = Redis.connect(:url => url)
+      if namespace
+        Redis::Namespace.new(namespace, :redis => client)
+      else
+        client
+      end
+    end
+    private_class_method :build_client
+  end
+end

data/lib/sidekiq/testing.rb

@@ -0,0 +1,34 @@
+module Sidekiq
+  module Worker
+
+    ##
+    # The Sidekiq testing infrastructure overrides perform_async
+    # so that it does not actually touch the network.  Instead it
+    # stores the asynchronous jobs in a per-class array so that
+    # their presence/absence can be asserted by your tests.
+    #
+    # This is similar to ActionMailer's :test delivery_method and its
+    # ActionMailer::Base.deliveries array.
+    #
+    # Example:
+    #
+    #   require 'sidekiq/testing'
+    #
+    #   assert_equal 0, HardWorker.jobs.size
+    #   HardWorker.perform_async(:something)
+    #   assert_equal 1, HardWorker.jobs.size
+    #   assert_equal :something, HardWorker.jobs[0]['args'][0]
+    #
+    module ClassMethods
+      alias_method :perform_async_old, :perform_async
+      def perform_async(*args)
+        jobs << { 'class' => self.name, 'args' => args }
+        true
+      end
+
+      def jobs
+        @pushed ||= []
+      end
+    end
+  end
+end

data/lib/sidekiq/util.rb

@@ -1,6 +1,20 @@
+require 'logger'
+
 module Sidekiq
   module Util
 
+    def self.logger
+      @logger ||= begin
+        log = Logger.new(STDERR)
+        log.level = Logger::INFO
+        log
+      end
+    end
+
+    def self.logger=(log)
+      @logger = (log ? log : Logger.new('/dev/null'))
+    end
+
     def constantize(camel_cased_word)
       names = camel_cased_word.split('::')
       names.shift if names.empty? || names.first.empty?
@@ -15,22 +29,17 @@ module Sidekiq
     def watchdog(last_words)
       yield
     rescue => ex
-      STDERR.puts last_words
-      STDERR.puts ex
-      STDERR.puts ex.backtrace.join("\n")
-    end
-
-    def err(msg)
-      STDERR.puts(msg)
+      logger.error last_words
+      logger.error ex
+      logger.error ex.backtrace.join("\n")
     end
 
-    def log(msg)
-      STDOUT.puts(msg) unless $TESTING
+    def logger
+      Sidekiq::Util.logger
     end
 
-    def verbose(msg)
-      STDOUT.puts(msg) if $DEBUG
+    def redis
+      Sidekiq::Manager.redis
     end
-
   end
 end

data/lib/sidekiq/version.rb

@@ -1,3 +1,3 @@
 module Sidekiq
-  VERSION = "0.5.1"
+  VERSION = "0.6.0"
 end

data/lib/sidekiq/worker.rb

@@ -24,19 +24,18 @@ module Sidekiq
       base.extend(ClassMethods)
     end
 
-    def info(msg)
-      print "#{msg}\n"
-    end
-    alias_method :log, :info
-
-    def debug(msg)
-      print "#{msg}\n" if $DEBUG
+    def logger
+      Sidekiq::Util.logger
     end
 
     module ClassMethods
       def perform_async(*args)
         Sidekiq::Client.push('class' => self.name, 'args' => args)
       end
+
+      def queue(name)
+        Sidekiq::Client.queues[self.name] = name.to_s
+      end
     end
   end
 end