creeper 1.0.9 → 2.0.0

data/lib/creeper/middleware/server/active_record.rb

@@ -0,0 +1,13 @@
+module Creeper
+  module Middleware
+    module Server
+      class ActiveRecord
+        def call(*args)
+          yield
+        ensure
+          ::ActiveRecord::Base.clear_active_connections! if defined?(::ActiveRecord)
+        end
+      end
+    end
+  end
+end

data/lib/creeper/middleware/server/logging.rb

@@ -0,0 +1,31 @@
+module Creeper
+  module Middleware
+    module Server
+      class Logging
+
+        def call(worker, msg, queue, job, conn)
+          Creeper::Logging.with_context("#{worker.class.to_s} JOB-#{job.id rescue nil} MSG-#{worker.object_id.to_s(36)}") do
+            begin
+              start = Time.now
+              logger.info { "start" }
+              yield
+              logger.info { "done: #{elapsed(start)} sec" }
+            rescue
+              logger.info { "fail: #{elapsed(start)} sec" }
+              raise
+            end
+          end
+        end
+
+        def elapsed(start)
+          (Time.now - start).to_f.round(3)
+        end
+
+        def logger
+          Creeper.logger
+        end
+      end
+    end
+  end
+end
+

data/lib/creeper/middleware/server/retry_jobs.rb

@@ -0,0 +1,79 @@
+module Creeper
+  module Middleware
+    module Server
+      ##
+      # Automatically retry jobs that fail in Creeper.
+      # Creeper's retry support assumes a typical development lifecycle:
+      # 0. push some code changes with a bug in it
+      # 1. bug causes message processing to fail, creeper's middleware captures
+      #    the message and pushes it onto a retry queue
+      # 2. creeper retries messages in the retry queue multiple times with
+      #    an exponential delay, the message continues to fail
+      # 3. after a few days, a developer deploys a fix.  the message is
+      #    reprocessed successfully.
+      # 4. if 3 never happens, creeper will eventually give up and throw the
+      #    message away.
+      #
+      # A message looks like:
+      #
+      #     { 'class' => 'HardWorker', 'args' => [1, 2, 'foo'] }
+      #
+      # We'll add a bit more data to the message to support retries:
+      #
+      #  * 'queue' - the queue to use
+      #  * 'retry_count' - number of times we've retried so far.
+      #  * 'error_message' - the message from the exception
+      #  * 'error_class' - the exception class
+      #  * 'failed_at' - the first time it failed
+      #  * 'retried_at' - the last time it was retried
+      #
+      # We don't store the backtrace as that can add a lot of overhead
+      # to the message and everyone is using Airbrake, right?
+      class RetryJobs
+        include Creeper::Util
+
+        # delayed_job uses the same basic formula
+        MAX_COUNT = 25
+        DELAY = proc { |count| (count ** 4) + 15 }
+
+        def call(worker, msg, queue, job, conn)
+          yield
+        rescue => e
+          raise unless msg['retry']
+
+          msg['queue'] = queue
+          msg['error_message'] = e.message
+          msg['error_class'] = e.class.name
+          count = if msg['retry_count']
+            msg['retried_at'] = Time.now.utc
+            msg['retry_count'] += 1
+          else
+            msg['failed_at'] = Time.now.utc
+            msg['retry_count'] = 0
+          end
+
+          if msg['backtrace'] == true
+            msg['error_backtrace'] = e.backtrace
+          elsif msg['backtrace'].to_i != 0
+            msg['error_backtrace'] = e.backtrace[0..msg['backtrace'].to_i]
+          end
+
+          if count <= MAX_COUNT
+            delay = DELAY.call(count)
+            logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
+            retry_at = Time.now.to_f + delay
+            payload = Creeper.dump_json(msg)
+            Creeper.redis do |conn|
+              conn.zadd('retry', retry_at.to_s, payload)
+            end
+          else
+            # Goodbye dear message, you (re)tried your best I'm sure.
+            logger.debug { "Dropping message after hitting the retry maximum: #{msg}" }
+          end
+          raise
+        end
+
+      end
+    end
+  end
+end

data/lib/creeper/middleware/server/timeout.rb

@@ -0,0 +1,21 @@
+require 'timeout'
+
+module Creeper
+  module Middleware
+    module Server
+      class Timeout
+
+        def call(worker, msg, queue, job, conn)
+          if msg['timeout'] && msg['timeout'].to_i != 0
+            ::Timeout.timeout(msg['timeout'].to_i) do
+              yield
+            end
+          else
+            yield
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/creeper/paginator.rb

@@ -0,0 +1,31 @@
+module Creeper
+  module Paginator
+    def page(key, pageidx=1, page_size=25)
+      current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
+      pageidx = current_page - 1
+      total_size = 0
+      items = []
+      starting = pageidx * page_size
+      ending = starting + page_size - 1
+
+      Creeper.redis do |conn|
+        type = conn.type(key)
+
+        case type
+        when 'zset'
+          total_size = conn.zcard(key)
+          items = conn.zrange(key, starting, ending, :with_scores => true)
+        when 'list'
+          total_size = conn.llen(key)
+          items = conn.lrange(key, starting, ending)
+        when 'none'
+          return [1, 0, []]
+        else
+          raise "can't page a #{type}"
+        end
+      end
+
+      [current_page, total_size, items]
+    end
+  end
+end

data/lib/creeper/processor.rb

@@ -0,0 +1,116 @@
+require 'celluloid'
+require 'creeper/util'
+
+require 'creeper/middleware/server/active_record'
+require 'creeper/middleware/server/retry_jobs'
+require 'creeper/middleware/server/logging'
+require 'creeper/middleware/server/timeout'
+
+module Creeper
+  ##
+  # The Processor receives a message from the Manager and actually
+  # processes it.  It instantiates the worker, runs the middleware
+  # chain and then calls Creeper::Worker#perform.
+  class Processor
+    include Util
+    include Celluloid
+
+    exclusive :process if ENV['CREEPER_EXCLUSIVE']
+
+    def self.default_middleware
+      Middleware::Chain.new do |m|
+        m.add Middleware::Server::Logging
+        m.add Middleware::Server::RetryJobs
+        m.add Middleware::Server::ActiveRecord
+        m.add Middleware::Server::Timeout
+      end
+    end
+
+    def initialize(boss)
+      @boss = boss
+    end
+
+    def process(msgstr, queue, job, conn)
+      msg     = Creeper.load_json(msgstr) rescue msgstr
+      klass   = Creeper.job_descriptions[queue]
+      klass ||= constantize(msg['class'])
+      worker  = klass.new
+
+      stats(worker, msg, queue) do
+        Creeper.server_middleware.invoke(worker, msg, queue, job, conn) do
+          args   = msg['args']
+          args ||= [msg]
+          worker.perform(*cloned(args))
+        end
+      end
+      job.delete rescue nil
+      @boss.processor_done!(current_actor)
+    rescue => ex
+      job.bury rescue nil
+      handle_exception(ex, msg || { :message => msgstr })
+      raise
+    ensure
+      conn.close rescue nil
+    end
+
+    # See http://github.com/tarcieri/celluloid/issues/22
+    def inspect
+      "#<Processor #{to_s}>"
+    end
+
+    def to_s
+      @str ||= "#{hostname}:#{process_id}-#{Thread.current.object_id}:default"
+    end
+
+    private
+
+    def stats(worker, msg, queue)
+      redis do |conn|
+        conn.multi do
+          conn.sadd('workers', self)
+          conn.setex("worker:#{self}:started", EXPIRY, Time.now.to_s)
+          hash = {:queue => queue, :payload => msg, :run_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z")}
+          conn.setex("worker:#{self}", EXPIRY, Creeper.dump_json(hash))
+        end
+      end
+
+      dying = false
+      begin
+        yield
+      rescue Exception
+        dying = true
+        redis do |conn|
+          conn.multi do
+            conn.incrby("stat:failed", 1)
+          end
+        end
+        raise
+      ensure
+        redis do |conn|
+          conn.multi do
+            conn.srem("workers", self)
+            conn.del("worker:#{self}")
+            conn.del("worker:#{self}:started")
+            conn.incrby("stat:processed", 1)
+          end
+        end
+      end
+    end
+
+    # Singleton classes are not clonable.
+    SINGLETON_CLASSES = [ NilClass, TrueClass, FalseClass, Symbol, Fixnum, Float ].freeze
+
+    # Clone the arguments passed to the worker so that if
+    # the message fails, what is pushed back onto Redis hasn't
+    # been mutated by the worker.
+    def cloned(ary)
+      ary.map do |val|
+        SINGLETON_CLASSES.include?(val.class) ? val : val.clone
+      end
+    end
+
+    def hostname
+      @h ||= `hostname`.strip
+    end
+  end
+end

data/lib/creeper/rails.rb

@@ -0,0 +1,21 @@
+module Creeper
+  def self.hook_rails!
+    return unless Creeper.options[:enable_rails_extensions]
+    if defined?(ActiveRecord)
+      ActiveRecord::Base.extend(Creeper::Extensions::ActiveRecord)
+      ActiveRecord::Base.send(:include, Creeper::Extensions::ActiveRecord)
+    end
+
+    if defined?(ActionMailer)
+      ActionMailer::Base.extend(Creeper::Extensions::ActionMailer)
+    end
+  end
+
+  class Rails < ::Rails::Engine
+    config.autoload_paths << File.expand_path("#{config.root}/app/workers") if File.exist?("#{config.root}/app/workers")
+
+    initializer 'creeper' do
+      Creeper.hook_rails!
+    end
+  end if defined?(::Rails)
+end

data/lib/creeper/redis_connection.rb

@@ -0,0 +1,28 @@
+require 'connection_pool'
+require 'redis'
+require 'redis/namespace'
+
+module Creeper
+  class RedisConnection
+    def self.create(options={})
+      url = options[:url] || ENV['REDISTOGO_URL'] || 'redis://localhost:6379/0'
+      driver = options[:driver] || 'ruby'
+      # need a connection for Fetcher and Retry
+      size = options[:size] || (Creeper.server? ? (Creeper.options[:concurrency] + 2) : 5)
+
+      ConnectionPool.new(:timeout => 1, :size => size) do
+        build_client(url, options[:namespace], driver)
+      end
+    end
+
+    def self.build_client(url, namespace, driver)
+      client = Redis.connect(:url => url, :driver => driver)
+      if namespace
+        Redis::Namespace.new(namespace, :redis => client)
+      else
+        client
+      end
+    end
+    private_class_method :build_client
+  end
+end

data/lib/creeper/testing.rb

@@ -0,0 +1,44 @@
+module Creeper
+  module Worker
+
+    ##
+    # The Creeper testing infrastructure overrides perform_async
+    # so that it does not actually touch the network.  Instead it
+    # stores the asynchronous jobs in a per-class array so that
+    # their presence/absence can be asserted by your tests.
+    #
+    # This is similar to ActionMailer's :test delivery_method and its
+    # ActionMailer::Base.deliveries array.
+    #
+    # Example:
+    #
+    #   require 'creeper/testing'
+    #
+    #   assert_equal 0, HardWorker.jobs.size
+    #   HardWorker.perform_async(:something)
+    #   assert_equal 1, HardWorker.jobs.size
+    #   assert_equal :something, HardWorker.jobs[0]['args'][0]
+    #
+    #   assert_equal 0, Creeper::Extensions::DelayedMailer.jobs.size
+    #   MyMailer.delayed.send_welcome_email('foo@example.com')
+    #   assert_equal 1, Creeper::Extensions::DelayedMailer.jobs.size
+    #
+    module ClassMethods
+      alias_method :client_push_old, :client_push
+      def client_push(opts)
+        jobs << opts
+        true
+      end
+
+      def jobs
+        @pushed ||= []
+      end
+
+      def drain
+        while job = jobs.shift do
+          new.perform(*job['args'])
+        end
+      end
+    end
+  end
+end

data/lib/creeper/util.rb

@@ -0,0 +1,45 @@
+require 'creeper/exception_handler'
+
+module Creeper
+  ##
+  # This module is part of Creeper core and not intended for extensions.
+  #
+  module Util
+    include ExceptionHandler
+
+    EXPIRY = 60 * 60
+
+    def constantize(camel_cased_word)
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      constant = Object
+      names.each do |name|
+        constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
+      end
+      constant
+    end
+
+    def watchdog(last_words)
+      yield
+    rescue => ex
+      handle_exception(ex, { :context => last_words })
+    end
+
+    def logger
+      Creeper.logger
+    end
+
+    def beanstalk(&block)
+      Creeper.beanstalk(&block)
+    end
+
+    def redis(&block)
+      Creeper.redis(&block)
+    end
+
+    def process_id
+      Process.pid
+    end
+  end
+end

data/lib/creeper/version.rb

@@ -1,3 +1,3 @@
 module Creeper
-  VERSION = "1.0.9"
+  VERSION = "2.0.0"
 end