creeper 1.0.9 → 2.0.0

data/lib/creeper/extensions/active_record.rb

@@ -0,0 +1,30 @@
+require 'creeper/extensions/generic_proxy'
+
+module Creeper
+  module Extensions
+    ##
+    # Adds a 'delay' method to ActiveRecord to offload arbitrary method
+    # execution to Creeper.  Examples:
+    #
+    # User.delay.delete_inactive
+    # User.recent_signups.each { |user| user.delay.mark_as_awesome }
+    class DelayedModel
+      include Creeper::Worker
+
+      def perform(yml)
+        (target, method_name, args) = YAML.load(yml)
+        target.send(method_name, *args)
+      end
+    end
+
+    module ActiveRecord
+      def delay
+        Proxy.new(DelayedModel, self)
+      end
+      def delay_for(interval)
+        Proxy.new(DelayedModel, self, Time.now.to_f + interval.to_f)
+      end
+    end
+
+  end
+end

data/lib/creeper/extensions/generic_proxy.rb

@@ -0,0 +1,26 @@
+module Creeper
+  module Extensions
+    class Proxy < BasicObject
+      def initialize(performable, target, at=nil)
+        @performable = performable
+        @target = target
+        @at = at
+      end
+
+      def method_missing(name, *args)
+        # Creeper has a limitation in that its message must be JSON.
+        # JSON can't round trip real Ruby objects so we use YAML to
+        # serialize the objects to a String.  The YAML will be converted
+        # to JSON and then deserialized on the other side back into a
+        # Ruby object.
+        obj = [@target, name, args]
+        if @at
+          @performable.perform_at(@at, ::YAML.dump(obj))
+        else
+          @performable.perform_async(::YAML.dump(obj))
+        end
+      end
+    end
+
+  end
+end

data/lib/creeper/fetch.rb

@@ -0,0 +1,94 @@
+require 'creeper'
+require 'celluloid'
+
+module Creeper
+  ##
+  # The Fetcher blocks on Redis, waiting for a message to process
+  # from the queues.  It gets the message and hands it to the Manager
+  # to assign to a ready Processor.
+  class Fetcher
+    include Celluloid
+    include Creeper::Util
+
+    TIMEOUT = 1
+
+    def initialize(mgr, queues, strict)
+      @mgr = mgr
+      @strictly_ordered_queues = strict
+      @queues = queues.map { |q| "queue:#{q}" }
+      @unique_queues = @queues.uniq
+    end
+
+    # Fetching is straightforward: the Manager makes a fetch
+    # request for each idle processor when Sidekiq starts and
+    # then issues a new fetch request every time a Processor
+    # finishes a message.
+    #
+    # Because we have to shut down cleanly, we can't block
+    # forever and we can't loop forever.  Instead we reschedule
+    # a new fetch if the current fetch turned up nothing.
+    def fetch
+      watchdog('Fetcher#fetch died') do
+        return if Creeper::Fetcher.done?
+
+        begin
+          queue = nil
+          msg   = nil
+          job   = nil
+          conn  = nil
+
+          conn = Creeper::BeanstalkConnection.create
+
+          begin
+            job = conn.reserve(TIMEOUT)
+            queue, msg = Creeper.load_json(job.body)
+          rescue Beanstalk::TimedOut
+            logger.debug("No message fetched after #{TIMEOUT} seconds") if $DEBUG
+            job.release rescue nil
+            conn.close rescue nil
+            sleep(TIMEOUT)
+            return after(0) { fetch }
+          end
+
+          if msg
+            @mgr.assign!(msg, queue.gsub(/.*queue:/, ''), job, conn)
+          else
+            after(0) { fetch }
+          end
+        rescue => ex
+          logger.error("Error fetching message: #{ex}")
+          logger.error(ex.backtrace.first)
+          job.release rescue nil
+          conn.close rescue nil
+          sleep(TIMEOUT)
+          after(0) { fetch }
+        end
+      end
+    end
+
+    # Ugh.  Say hello to a bloody hack.
+    # Can't find a clean way to get the fetcher to just stop processing
+    # its mailbox when shutdown starts.
+    def self.done!
+      @done = true
+    end
+
+    def self.done?
+      @done
+    end
+
+    private
+
+    # Creating the Redis#blpop command takes into account any
+    # configured queue weights. By default Redis#blpop returns
+    # data from the first queue that has pending elements. We
+    # recreate the queue command each time we invoke Redis#blpop
+    # to honor weights and avoid queue starvation.
+    def queues_cmd
+      return @unique_queues.dup << TIMEOUT if @strictly_ordered_queues
+      queues = @queues.sample(@unique_queues.size).uniq
+      queues.concat(@unique_queues - queues)
+      queues << TIMEOUT
+    end
+  end
+end

data/lib/creeper/legacy.rb

@@ -0,0 +1,46 @@
+module Creeper
+  module Legacy
+
+    module ClassMethods
+
+      def job_descriptions
+        @job_descriptions ||= {}
+      end
+
+      def enqueue(job, *args)
+        enqueue!(job, *args)
+      end
+
+      def enqueue!(job, *args)
+        options     = args.last.is_a?(Hash) ? args.last : {}
+        priority    = options[:priority] || options[:pri] || 65536
+        delay       = [ 0, options[:delay].to_i ].max
+        time_to_run = options[:time_to_run] || options[:ttr] || 120
+
+        klass = options[:class] || job_descriptions[job]
+
+        Creeper::Client.push({
+          'queue'       => job,
+          'args'        => args,
+          'class'       => klass,
+          'delay'       => delay,
+          'priority'    => priority,
+          'time_to_run' => time_to_run
+        })
+      end
+
+    end
+
+    module WorkerMethods
+
+      def creeper_legacy_queue(tube = nil)
+        return @creeper_legacy_queue if tube.nil?
+        (@creeper_legacy_queue = tube).tap do
+          Creeper.job_descriptions[@creeper_legacy_queue] = self
+        end
+      end
+
+    end
+
+  end
+end

data/lib/creeper/logging.rb

@@ -0,0 +1,46 @@
+require 'time'
+require 'logger'
+
+module Creeper
+  module Logging
+
+    class Pretty < Logger::Formatter
+      # Provide a call() method that returns the formatted message.
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
+      end
+
+      def context
+        c = Thread.current[:creeper_context]
+        c ? " #{c}" : ''
+      end
+    end
+
+    def self.with_context(msg)
+      begin
+        Thread.current[:creeper_context] = msg
+        yield
+      ensure
+        Thread.current[:creeper_context] = nil
+      end
+    end
+
+    def self.logger
+      @logger ||= begin
+        log = Logger.new(STDOUT)
+        log.level = Logger::INFO
+        log.formatter = Pretty.new
+        log
+      end
+    end
+
+    def self.logger=(log)
+      @logger = (log ? log : Logger.new('/dev/null'))
+    end
+
+    def logger
+      Creeper::Logging.logger
+    end
+
+  end
+end

data/lib/creeper/manager.rb

@@ -0,0 +1,164 @@
+require 'celluloid'
+
+require 'creeper/util'
+require 'creeper/processor'
+require 'creeper/fetch'
+
+module Creeper
+
+  ##
+  # The main router in the system.  This
+  # manages the processor state and accepts messages
+  # from Redis to be dispatched to an idle processor.
+  #
+  class Manager
+    include Util
+    include Celluloid
+
+    trap_exit :processor_died
+
+    def initialize(options={})
+      logger.info "Booting creeper #{Creeper::VERSION} with Beanstalk at #{beanstalk { |x| x.instance_variable_get(:@addrs).join(', ')}} and Redis at #{redis {|x| x.client.id}}"
+      logger.info "Running in #{RUBY_DESCRIPTION}"
+      logger.debug { options.inspect }
+      @count = options[:concurrency] || 25
+      @done_callback = nil
+
+      @in_progress = {}
+      @done = false
+      @busy = []
+      @fetcher = Fetcher.new(current_actor, options[:queues], !!options[:strict])
+      @ready = @count.times.map { Processor.new_link(current_actor) }
+      procline
+    end
+
+    def stop(options={})
+      watchdog('Manager#stop died') do
+        shutdown = options[:shutdown]
+        timeout = options[:timeout]
+
+        @done = true
+        Creeper::Fetcher.done!
+        @fetcher.terminate! if @fetcher.alive?
+
+        logger.info { "Shutting down #{@ready.size} quiet workers" }
+        @ready.each { |x| x.terminate if x.alive? }
+        @ready.clear
+
+        logger.debug { "Clearing workers in redis" }
+        Creeper.redis do |conn|
+          workers = conn.smembers('workers')
+          workers.each do |name|
+            conn.srem('workers', name) if name =~ /:#{process_id}-/
+          end
+        end
+
+        return after(0) { signal(:shutdown) } if @busy.empty?
+        logger.info { "Pausing up to #{timeout} seconds to allow workers to finish..." }
+        hard_shutdown_in timeout if shutdown
+      end
+    end
+
+    def start
+      @ready.each { dispatch }
+    end
+
+    def when_done(&blk)
+      @done_callback = blk
+    end
+
+    def processor_done(processor)
+      watchdog('Manager#processor_done died') do
+        @done_callback.call(processor) if @done_callback
+        @in_progress.delete(processor.object_id)
+        @busy.delete(processor)
+        if stopped?
+          processor.terminate if processor.alive?
+          signal(:shutdown) if @busy.empty?
+        else
+          @ready << processor if processor.alive?
+        end
+        dispatch
+      end
+    end
+
+    def processor_died(processor, reason)
+      watchdog("Manager#processor_died died") do
+        @in_progress.delete(processor.object_id)
+        @busy.delete(processor)
+
+        unless stopped?
+          @ready << Processor.new_link(current_actor)
+          dispatch
+        else
+          signal(:shutdown) if @busy.empty?
+        end
+      end
+    end
+
+    def assign(msg, queue, job, conn)
+      watchdog("Manager#assign died") do
+        if stopped?
+          # Race condition between Manager#stop if Fetcher
+          # is blocked on redis and gets a message after
+          # all the ready Processors have been stopped.
+          # Push the message back to redis.
+          job.release rescue nil
+          conn.close rescue nil
+          # Creeper.redis do |conn|
+          #   conn.lpush("queue:#{queue}", msg)
+          # end
+        else
+          processor = @ready.pop
+          @in_progress[processor.object_id] = [msg, queue, job, conn]
+          @busy << processor
+          processor.process!(msg, queue, job, conn)
+        end
+      end
+    end
+
+    private
+
+    def hard_shutdown_in(delay)
+      after(delay) do
+        watchdog("Manager#watch_for_shutdown died") do
+          # We've reached the timeout and we still have busy workers.
+          # They must die but their messages shall live on.
+          logger.info("Still waiting for #{@busy.size} busy workers")
+
+          @busy.each do |processor|
+            # processor is an actor proxy and we can't call any methods
+            # that would go to the actor (since it's busy).  Instead
+            # we'll use the object_id to track the worker's data here.
+            msg, queue, job, conn = @in_progress[processor.object_id]
+            job.release rescue nil
+            conn.close rescue nil
+            # conn.lpush("queue:#{queue}", msg)
+          end
+          logger.info("Released #{@busy.size} jobs back to Beanstalk")
+
+          after(0) { signal(:shutdown) }
+        end
+      end
+    end
+
+    def dispatch
+      return if stopped?
+      # This is a safety check to ensure we haven't leaked
+      # processors somehow.
+      raise "BUG: No processors, cannot continue!" if @ready.empty? && @busy.empty?
+      raise "No ready processor!?" if @ready.empty?
+
+      @fetcher.fetch!
+    end
+
+    def stopped?
+      @done
+    end
+
+    def procline
+      $0 = "creeper #{Creeper::VERSION} [#{@busy.size} of #{@count} busy]#{stopped? ? ' stopping' : ''}"
+      after(5) { procline }
+    end
+  end
+end

data/lib/creeper/middleware/chain.rb

@@ -0,0 +1,100 @@
+module Creeper
+  # Middleware is code configured to run before/after
+  # a message is processed.  It is patterned after Rack
+  # middleware. Middleware exists for the client side
+  # (pushing jobs onto the queue) as well as the server
+  # side (when jobs are actually processed).
+  #
+  # To add middleware for the client:
+  #
+  # Creeper.configure_client do |config|
+  #   config.client_middleware do |chain|
+  #     chain.add MyClientHook
+  #   end
+  # end
+  #
+  # To modify middleware for the server, just call
+  # with another block:
+  #
+  # Creeper.configure_server do |config|
+  #   config.server_middleware do |chain|
+  #     chain.add MyServerHook
+  #     chain.remove ActiveRecord
+  #   end
+  # end
+  #
+  # This is an example of a minimal server middleware:
+  #
+  # class MyServerHook
+  #   def call(worker_instance, msg, queue)
+  #     puts "Before work"
+  #     yield
+  #     puts "After work"
+  #   end
+  # end
+  #
+  # This is an example of a minimal client middleware:
+  #
+  # class MyClientHook
+  #   def call(worker_class, msg, queue)
+  #     puts "Before push"
+  #     yield
+  #     puts "After push"
+  #   end
+  # end
+  #
+  module Middleware
+    class Chain
+      attr_reader :entries
+
+      def initialize
+        @entries = []
+        yield self if block_given?
+      end
+
+      def remove(klass)
+        entries.delete_if { |entry| entry.klass == klass }
+      end
+
+      def add(klass, *args)
+        entries << Entry.new(klass, *args) unless exists?(klass)
+      end
+
+      def exists?(klass)
+        entries.any? { |entry| entry.klass == klass }
+      end
+
+      def retrieve
+        entries.map(&:make_new)
+      end
+
+      def clear
+        entries.clear
+      end
+
+      def invoke(*args, &final_action)
+        chain = retrieve.dup
+        traverse_chain = lambda do
+          if chain.empty?
+            final_action.call
+          else
+            chain.shift.call(*args, &traverse_chain)
+          end
+        end
+        traverse_chain.call
+      end
+    end
+
+    class Entry
+      attr_reader :klass
+      def initialize(klass, *args)
+        @klass = klass
+        @args  = args
+      end
+
+      def make_new
+        @klass.new(*@args)
+      end
+    end
+  end
+end