kulesa-sidekiq 1.2.2

data/lib/sidekiq/extensions/action_mailer.rb

@@ -0,0 +1,26 @@
+require 'sidekiq/extensions/generic_proxy'
+
+module Sidekiq
+  module Extensions
+    ##
+    # Adds a 'delay' method to ActionMailer to offload arbitrary email
+    # delivery to Sidekiq.  Example:
+    #
+    #    UserMailer.delay.send_welcome_email(new_user)
+    class DelayedMailer
+      include Sidekiq::Worker
+
+      def perform(yml)
+        (target, method_name, args) = YAML.load(yml)
+        target.send(method_name, *args).deliver
+      end
+    end
+
+    module ActionMailer
+      def delay
+        Proxy.new(DelayedMailer, self)
+      end
+    end
+
+  end
+end

data/lib/sidekiq/extensions/active_record.rb

@@ -0,0 +1,27 @@
+require 'sidekiq/extensions/generic_proxy'
+
+module Sidekiq
+  module Extensions
+    ##
+    # Adds a 'delay' method to ActiveRecord to offload arbitrary method
+    # execution to Sidekiq.  Examples:
+    #
+    # User.delay.delete_inactive
+    # User.recent_signups.each { |user| user.delay.mark_as_awesome }
+    class DelayedModel
+      include Sidekiq::Worker
+
+      def perform(yml)
+        (target, method_name, args) = YAML.load(yml)
+        target.send(method_name, *args)
+      end
+    end
+
+    module ActiveRecord
+      def delay
+        Proxy.new(DelayedModel, self)
+      end
+    end
+
+  end
+end

data/lib/sidekiq/extensions/generic_proxy.rb

@@ -0,0 +1,21 @@
+module Sidekiq
+  module Extensions
+    class Proxy < (RUBY_VERSION < '1.9' ? Object : BasicObject)
+      def initialize(performable, target)
+        @performable = performable
+        @target = target
+      end
+
+      def method_missing(name, *args)
+        # Sidekiq has a limitation in that its message must be JSON.
+        # JSON can't round trip real Ruby objects so we use YAML to
+        # serialize the objects to a String.  The YAML will be converted
+        # to JSON and then deserialized on the other side back into a
+        # Ruby object.
+        obj = [@target, name, args]
+        @performable.perform_async(::YAML.dump(obj))
+      end
+    end
+
+  end
+end

data/lib/sidekiq/fetch.rb

@@ -0,0 +1,76 @@
+require 'sidekiq'
+require 'celluloid'
+
+module Sidekiq
+  ##
+  # The Fetcher blocks on Redis, waiting for a message to process
+  # from the queues.  It gets the message and hands it to the Manager
+  # to assign to a ready Processor.
+  class Fetcher
+    include Celluloid
+    include Sidekiq::Util
+
+    TIMEOUT = 1
+
+    def initialize(mgr, queues)
+      @mgr = mgr
+      @queues = queues.map { |q| "queue:#{q}" }
+      @unique_queues = @queues.uniq
+    end
+
+    # Fetching is straightforward: the Manager makes a fetch
+    # request for each idle processor when Sidekiq starts and
+    # then issues a new fetch request every time a Processor
+    # finishes a message.
+    #
+    # Because we have to shut down cleanly, we can't block
+    # forever and we can't loop forever.  Instead we reschedule
+    # a new fetch if the current fetch turned up nothing.
+    def fetch
+      watchdog('Fetcher#fetch died') do
+        return if Sidekiq::Fetcher.done?
+
+        begin
+          queue = nil
+          msg = nil
+          Sidekiq.redis { |conn| queue, msg = conn.blpop(*queues_cmd) }
+
+          if msg
+            @mgr.assign!(msg, queue.gsub(/.*queue:/, ''))
+          else
+            after(0) { fetch }
+          end
+        rescue => ex
+          logger.error("Error fetching message: #{ex}")
+          logger.error(ex.backtrace.first)
+          sleep(TIMEOUT)
+          after(0) { fetch }
+        end
+      end
+    end
+
+    # Ugh.  Say hello to a bloody hack.
+    # Can't find a clean way to get the fetcher to just stop processing
+    # its mailbox when shutdown starts.
+    def self.done!
+      @done = true
+    end
+
+    def self.done?
+      @done
+    end
+
+    private
+
+    # Creating the Redis#blpop command takes into account any
+    # configured queue weights. By default Redis#blpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#blpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      queues = @queues.sample(@unique_queues.size).uniq
+      queues.concat(@unique_queues - queues)
+      queues << TIMEOUT
+    end
+  end
+end

data/lib/sidekiq/logging.rb

@@ -0,0 +1,46 @@
+require 'time'
+require 'logger'
+
+module Sidekiq
+  module Logging
+
+    class Pretty < Logger::Formatter
+      # Provide a call() method that returns the formatted message.
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+      end
+
+      def context
+        c = Thread.current[:sidekiq_context]
+        c ? " #{c}" : ''
+      end
+    end
+
+    def self.with_context(msg)
+      begin
+        Thread.current[:sidekiq_context] = msg
+        yield
+      ensure
+        Thread.current[:sidekiq_context] = nil
+      end
+    end
+
+    def self.logger
+      @logger ||= begin
+        log = Logger.new(STDOUT)
+        log.level = Logger::INFO
+        log.formatter = Pretty.new
+        log
+      end
+    end
+
+    def self.logger=(log)
+      @logger = (log ? log : Logger.new('/dev/null'))
+    end
+
+    def logger
+      Sidekiq::Logging.logger
+    end
+
+  end
+end

data/lib/sidekiq/manager.rb

@@ -0,0 +1,163 @@
+require 'celluloid'
+require 'multi_json'
+
+require 'sidekiq/util'
+require 'sidekiq/processor'
+require 'sidekiq/fetch'
+
+module Sidekiq
+
+  ##
+  # The main router in the system.  This
+  # manages the processor state and accepts messages
+  # from Redis to be dispatched to an idle processor.
+  #
+  class Manager
+    include Util
+    include Celluloid
+
+    trap_exit :processor_died
+
+    def initialize(options={})
+      logger.info "Booting sidekiq #{Sidekiq::VERSION} with Redis at #{redis {|x| x.client.id}}"
+      logger.info "Running in #{RUBY_DESCRIPTION}"
+      logger.debug { options.inspect }
+      @count = options[:concurrency] || 25
+      @done_callback = nil
+
+      @in_progress = {}
+      @done = false
+      @busy = []
+      @fetcher = Fetcher.new(current_actor, options[:queues])
+      @ready = @count.times.map { Processor.new_link(current_actor) }
+      procline
+    end
+
+    def stop(options={})
+      watchdog('Manager#stop died') do
+        shutdown = options[:shutdown]
+        timeout = options[:timeout]
+
+        @done = true
+        Sidekiq::Fetcher.done!
+        @fetcher.terminate! if @fetcher.alive?
+
+        logger.info { "Shutting down #{@ready.size} quiet workers" }
+        @ready.each { |x| x.terminate if x.alive? }
+        @ready.clear
+
+        logger.debug { "Clearing workers in redis" }
+        Sidekiq.redis do |conn|
+          workers = conn.smembers('workers')
+          workers.each do |name|
+            conn.srem('workers', name) if name =~ /:#{process_id}-/
+          end
+        end
+
+        return after(0) { signal(:shutdown) } if @busy.empty?
+        logger.info { "Pausing up to #{timeout} seconds to allow workers to finish..." }
+        hard_shutdown_in timeout if shutdown
+      end
+    end
+
+    def start
+      @ready.each { dispatch }
+    end
+
+    def when_done(&blk)
+      @done_callback = blk
+    end
+
+    def processor_done(processor)
+      watchdog('Manager#processor_done died') do
+        @done_callback.call(processor) if @done_callback
+        @in_progress.delete(processor.object_id)
+        @busy.delete(processor)
+        if stopped?
+          processor.terminate if processor.alive?
+          signal(:shutdown) if @busy.empty?
+        else
+          @ready << processor if processor.alive?
+        end
+        dispatch
+      end
+    end
+
+    def processor_died(processor, reason)
+      watchdog("Manager#processor_died died") do
+        @in_progress.delete(processor.object_id)
+        @busy.delete(processor)
+
+        unless stopped?
+          @ready << Processor.new_link(current_actor)
+          dispatch
+        else
+          signal(:shutdown) if @busy.empty?
+        end
+      end
+    end
+
+    def assign(msg, queue)
+      watchdog("Manager#assign died") do
+        if stopped?
+          # Race condition between Manager#stop if Fetcher
+          # is blocked on redis and gets a message after
+          # all the ready Processors have been stopped.
+          # Push the message back to redis.
+          Sidekiq.redis do |conn|
+            conn.lpush("queue:#{queue}", msg)
+          end
+        else
+          processor = @ready.pop
+          @in_progress[processor.object_id] = [msg, queue]
+          @busy << processor
+          processor.process!(Sidekiq.load_json(msg), queue)
+        end
+      end
+    end
+
+    private
+
+    def hard_shutdown_in(delay)
+      after(delay) do
+        watchdog("Manager#watch_for_shutdown died") do
+          # We've reached the timeout and we still have busy workers.
+          # They must die but their messages shall live on.
+          logger.info("Still waiting for #{@busy.size} busy workers")
+
+          Sidekiq.redis do |conn|
+            @busy.each do |processor|
+              # processor is an actor proxy and we can't call any methods
+              # that would go to the actor (since it's busy).  Instead
+              # we'll use the object_id to track the worker's data here.
+              msg, queue = @in_progress[processor.object_id]
+              conn.lpush("queue:#{queue}", msg)
+            end
+          end
+          logger.info("Pushed #{@busy.size} messages back to Redis")
+
+          after(0) { signal(:shutdown) }
+        end
+      end
+    end
+
+    def dispatch
+      return if stopped?
+      # This is a safety check to ensure we haven't leaked
+      # processors somehow.
+      raise "BUG: No processors, cannot continue!" if @ready.empty? && @busy.empty?
+      raise "No ready processor!?" if @ready.empty?
+
+      @fetcher.fetch!
+    end
+
+    def stopped?
+      @done
+    end
+
+    def procline
+      $0 = "sidekiq #{Sidekiq::VERSION} [#{@busy.size} of #{@count} busy]#{stopped? ? ' stopping' : ''}"
+      after(5) { procline }
+    end
+  end
+end

data/lib/sidekiq/middleware/chain.rb

@@ -0,0 +1,96 @@
+module Sidekiq
+  # Middleware is code configured to run before/after
+  # a message is processed.  It is patterned after Rack
+  # middleware. Middleware exists for the client side
+  # (pushing jobs onto the queue) as well as the server
+  # side (when jobs are actually processed).
+  #
+  # To add middleware for the client:
+  #
+  # Sidekiq.client_middleware do |chain|
+  #   chain.add MyClientHook
+  # end
+  #
+  # To modify middleware for the server, just call
+  # with another block:
+  #
+  # Sidekiq.server_middleware do |chain|
+  #   chain.add MyServerHook
+  #   chain.remove ActiveRecord
+  # end
+  #
+  # This is an example of a minimal server middleware:
+  #
+  # class MyServerHook
+  #   def call(worker, msg, queue)
+  #     puts "Before work"
+  #     yield
+  #     puts "After work"
+  #   end
+  # end
+  #
+  # This is an example of a minimal client middleware:
+  #
+  # class MyClientHook
+  #   def call(msg, queue)
+  #     puts "Before push"
+  #     yield
+  #     puts "After push"
+  #   end
+  # end
+  #
+  module Middleware
+    class Chain
+      attr_reader :entries
+
+      def initialize
+        @entries = []
+        yield self if block_given?
+      end
+
+      def remove(klass)
+        entries.delete_if { |entry| entry.klass == klass }
+      end
+
+      def add(klass, *args)
+        entries << Entry.new(klass, *args) unless exists?(klass)
+      end
+
+      def exists?(klass)
+        entries.any? { |entry| entry.klass == klass }
+      end
+
+      def retrieve
+        entries.map(&:make_new)
+      end
+
+      def clear
+        entries.clear
+      end
+
+      def invoke(*args, &final_action)
+        chain = retrieve.dup
+        traverse_chain = lambda do
+          if chain.empty?
+            final_action.call
+          else
+            chain.shift.call(*args, &traverse_chain)
+          end
+        end
+        traverse_chain.call
+      end
+    end
+
+    class Entry
+      attr_reader :klass
+      def initialize(klass, *args)
+        @klass = klass
+        @args  = args
+      end
+
+      def make_new
+        @klass.new(*@args)
+      end
+    end
+  end
+end