creeper 1.0.9 → 2.0.0

data/lib/creeper/beanstalk_connection.rb

@@ -0,0 +1,35 @@
+require 'beanstalk-client'
+
+module Creeper
+  class BeanstalkConnection
+    def self.create(options={})
+      client_options.merge!(options)
+
+      url     = client_options[:url] || ENV['BEANSTALK_URL'] || 'beanstalk://127.0.0.1:11300/'
+      default = client_options[:default]
+      tubes   = client_options[:tubes] || Creeper.job_descriptions.keys
+
+      build_client(url, default, tubes)
+    end
+
+    def self.build_client(urls, default, tubes)
+      uris = [*urls].flatten.map do |url_string|
+        url_string.split(/[\s,]+/).map do |url|
+          uri = URI.parse(url)
+          "#{uri.host}:#{uri.port || 11300}"
+        end
+      end.flatten
+
+      Beanstalk::Pool.new(uris, default).tap do |client|
+        tubes.each do |tube|
+          client.watch(tube)
+        end
+      end
+    end
+    private_class_method :build_client
+
+    def self.client_options
+      @client_options ||= {}
+    end
+  end
+end

data/lib/creeper/cli.rb

@@ -0,0 +1,225 @@
+trap 'INT' do
+  # Handle Ctrl-C in JRuby like MRI
+  # http://jira.codehaus.org/browse/JRUBY-4637
+  Creeper::CLI.instance.interrupt
+end
+
+trap 'TERM' do
+  # Heroku sends TERM and then waits 10 seconds for process to exit.
+  Creeper::CLI.instance.interrupt
+end
+
+trap 'QUIT' do
+  Creeper.logger.info "Received QUIT, no longer accepting new work"
+  mgr = Creeper::CLI.instance.manager
+  if mgr
+    mgr.stop!(shutdown: true, timeout: Creeper.options[:timeout])
+    mgr.wait(:shutdown)
+    # Explicitly exit so busy Processor threads can't block
+    # process shutdown.
+    exit(0)
+  end
+end
+
+trap 'USR1' do
+  Creeper.logger.info "Received USR1, no longer accepting new work"
+  mgr = Creeper::CLI.instance.manager
+  mgr.stop! if mgr
+end
+
+trap 'TTIN' do
+  Thread.list.each do |thread|
+    Creeper.logger.info "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+    Creeper.logger.info thread.backtrace.join("\n")
+  end
+end
+
+$stdout.sync = true
+
+require 'yaml'
+require 'singleton'
+require 'optparse'
+require 'celluloid'
+
+require 'creeper'
+require 'creeper/util'
+require 'creeper/manager'
+# require 'creeper/scheduled'
+
+module Creeper
+  class CLI
+    include Util
+    include Singleton
+
+    # Used for CLI testing
+    attr_accessor :code
+    attr_accessor :manager
+
+    def initialize
+      @code = nil
+      @interrupt_mutex = Mutex.new
+      @interrupted = false
+    end
+
+    def parse(args=ARGV)
+      @code = nil
+      Creeper.logger
+
+      cli = parse_options(args)
+      config = parse_config(cli)
+      options.merge!(config.merge(cli))
+
+      Creeper.logger.level = Logger::DEBUG if options[:verbose]
+      Celluloid.logger = nil unless options[:verbose]
+
+      validate!
+      write_pid
+      boot_system
+    end
+
+    def run
+      @manager = Creeper::Manager.new(options)
+      begin
+        logger.info 'Starting processing, hit Ctrl-C to stop'
+        @manager.start!
+        sleep
+      rescue Interrupt
+        logger.info 'Shutting down'
+        @manager.stop!(:shutdown => true, :timeout => options[:timeout])
+        @manager.wait(:shutdown)
+        # Explicitly exit so busy Processor threads can't block
+        # process shutdown.
+        exit(0)
+      end
+    end
+
+    def interrupt
+      @interrupt_mutex.synchronize do
+        unless @interrupted
+          @interrupted = true
+          Thread.main.raise Interrupt
+        end
+      end
+    end
+
+    private
+
+    def die(code)
+      exit(code)
+    end
+
+    def options
+      Creeper.options
+    end
+
+    def detected_environment
+      options[:environment] ||= ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+    end
+
+    def boot_system
+      ENV['RACK_ENV'] = ENV['RAILS_ENV'] = detected_environment
+
+      raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
+
+      if File.directory?(options[:require])
+        require 'rails'
+        require 'creeper/rails'
+        require File.expand_path("#{options[:require]}/config/environment.rb")
+        ::Rails.application.eager_load!
+      else
+        require options[:require]
+      end
+    end
+
+    def validate!
+      options[:queues] << 'default' if options[:queues].empty?
+
+      if !File.exist?(options[:require]) ||
+         (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
+        logger.info "=================================================================="
+        logger.info "  Please point creeper to a Rails 3 application or a Ruby file    "
+        logger.info "  to load your worker classes with -r [DIR|FILE]."
+        logger.info "=================================================================="
+        logger.info @parser
+        die(1)
+      end
+    end
+
+    def parse_options(argv)
+      opts = {}
+
+      @parser = OptionParser.new do |o|
+        o.on "-q", "--queue QUEUE[,WEIGHT]...", "Queues to process with optional weights" do |arg|
+          queues_and_weights = arg.scan(/([\w-]+),?(\d*)/)
+          queues_and_weights.each {|queue_and_weight| parse_queues(opts, *queue_and_weight)}
+          opts[:strict] = queues_and_weights.collect(&:last).none? {|weight| weight != ''}
+        end
+
+        o.on "-v", "--verbose", "Print more verbose output" do
+          Creeper.logger.level = ::Logger::DEBUG
+        end
+
+        o.on '-e', '--environment ENV', "Application environment" do |arg|
+          opts[:environment] = arg
+        end
+
+        o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
+          opts[:timeout] = arg.to_i
+        end
+
+        o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
+          opts[:require] = arg
+        end
+
+        o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
+          opts[:concurrency] = arg.to_i
+        end
+
+        o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
+          opts[:pidfile] = arg
+        end
+
+        o.on '-C', '--config PATH', "path to YAML config file" do |arg|
+          opts[:config_file] = arg
+        end
+
+        o.on '-V', '--version', "Print version and exit" do |arg|
+          puts "Creeper #{Creeper::VERSION}"
+          die(0)
+        end
+      end
+
+      @parser.banner = "creeper [options]"
+      @parser.on_tail "-h", "--help", "Show help" do
+        logger.info @parser
+        die 1
+      end
+      @parser.parse!(argv)
+      opts
+    end
+
+    def write_pid
+      if path = options[:pidfile]
+        File.open(path, 'w') do |f|
+          f.puts Process.pid
+        end
+      end
+    end
+
+    def parse_config(cli)
+      opts = {}
+      if cli[:config_file] && File.exist?(cli[:config_file])
+        opts = YAML.load_file cli[:config_file]
+        queues = opts.delete(:queues) || []
+        queues.each { |name, weight| parse_queues(opts, name, weight) }
+      end
+      opts
+    end
+
+    def parse_queues(opts, q, weight)
+      [weight.to_i, 1].max.times do
+       (opts[:queues] ||= []) << q
+      end
+    end
+  end
+end

data/lib/creeper/client.rb

@@ -0,0 +1,93 @@
+require 'creeper/legacy'
+require 'creeper/middleware/chain'
+
+module Creeper
+  class Client
+
+    def self.default_middleware
+      Middleware::Chain.new do |m|
+      end
+    end
+
+    def self.registered_workers
+      Creeper.redis { |x| x.smembers('workers') }
+    end
+
+    def self.registered_queues
+      Creeper.redis { |x| x.smembers('queues') }
+    end
+
+    ##
+    # The main method used to push a job to Redis.  Accepts a number of options:
+    #
+    #   queue - the named queue to use, default 'default'
+    #   class - the worker class to call, required
+    #   args - an array of simple arguments to the perform method, must be JSON-serializable
+    #   retry - whether to retry this job if it fails, true or false, default true
+    #   backtrace - whether to save any error backtrace, default false
+    #
+    # All options must be strings, not symbols.  NB: because we are serializing to JSON, all
+    # symbols in 'args' will be converted to strings.
+    #
+    # Returns nil if not pushed to Redis or a unique Job ID if pushed.
+    #
+    # Example:
+    #   Creeper::Client.push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
+    #
+    def self.push(item)
+      raise(ArgumentError, "Message must be a Hash of the form: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash)
+      raise(ArgumentError, "Message must include a class and set of arguments: #{item.inspect}") if !item['class'] || !item['args']
+      raise(ArgumentError, "Message must include a Creeper::Worker class, not class name: #{item['class'].ancestors.inspect}") if !item['class'].is_a?(Class) || !item['class'].respond_to?('get_creeper_options')
+
+      worker_class = item['class']
+      item['class'] = item['class'].to_s
+
+      item = worker_class.get_creeper_options.merge(item)
+      # item['retry'] = !!item['retry']
+      at          = item['at']
+      queue       = item['queue']
+      priority    = item['priority']
+      delay       = item['delay']
+      delay     ||= at.to_i - Time.now.to_i if at and at.respond_to?(:to_i)
+      time_to_run = item['time_to_run']
+      # item['jid'] = SecureRandom.base64
+
+      pushed = false
+      job = Creeper.client_middleware.invoke(worker_class, item, queue) do
+        payload = Creeper.dump_json([ queue, item ])
+        args    = [ payload, priority, delay, time_to_run ]
+        args.pop while args.last.nil?
+        Creeper.redis do |conn|
+          conn.sadd('queues', queue)
+        end
+        Creeper.beanstalk do |beanstalk|
+          beanstalk.on_tube(queue) do |conn|
+            conn.put(*args)
+          end
+        end
+        # Creeper.redis do |conn|
+        #   if item['at']
+        #     pushed = conn.zadd('schedule', item['at'].to_s, payload)
+        #   else
+        #     _, pushed = conn.multi do
+        #       conn.sadd('queues', queue)
+        #       conn.rpush("queue:#{queue}", payload)
+        #     end
+        #   end
+        # end
+      end
+      pushed = !!job
+      pushed ? job : nil
+    end
+
+    # Redis compatibility helper.  Example usage:
+    #
+    #   Creeper::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
+    #
+    # Messages are enqueued to the 'default' queue.
+    #
+    def self.enqueue(klass, *args)
+      klass.perform_async(*args)
+    end
+  end
+end

data/lib/creeper/core_ext.rb

@@ -0,0 +1,54 @@
+begin
+  require 'active_support/core_ext/class/attribute'
+rescue LoadError
+
+  # A dumbed down version of ActiveSupport's
+  # Class#class_attribute helper.
+  class Class
+    def class_attribute(*attrs)
+      instance_reader = true
+      instance_writer = true
+
+      attrs.each do |name|
+        class_eval <<-RUBY, __FILE__, __LINE__ + 1
+          def self.#{name}() nil end
+          def self.#{name}?() !!#{name} end
+
+          def self.#{name}=(val)
+            singleton_class.class_eval do
+              define_method(:#{name}) { val }
+            end
+
+            if singleton_class?
+              class_eval do
+                def #{name}
+                  defined?(@#{name}) ? @#{name} : singleton_class.#{name}
+                end
+              end
+            end
+            val
+          end
+
+          if instance_reader
+            def #{name}
+              defined?(@#{name}) ? @#{name} : self.class.#{name}
+            end
+
+            def #{name}?
+              !!#{name}
+            end
+          end
+        RUBY
+
+        attr_writer name if instance_writer
+      end
+    end
+
+    private
+    def singleton_class?
+      ancestors.first != self
+    end
+  end
+end
+
+

data/lib/creeper/exception_handler.rb

@@ -0,0 +1,30 @@
+module Creeper
+  module ExceptionHandler
+
+    def handle_exception(ex, msg)
+      Creeper.logger.warn msg
+      Creeper.logger.warn ex
+      Creeper.logger.warn ex.backtrace.join("\n")
+      send_to_airbrake(msg, ex) if defined?(::Airbrake)
+      send_to_exceptional(msg, ex) if defined?(::Exceptional)
+      send_to_exception_notifier(msg, ex) if defined?(::ExceptionNotifier)
+    end
+
+    private
+
+    def send_to_airbrake(msg, ex)
+      ::Airbrake.notify(ex, :parameters => msg)
+    end
+
+    def send_to_exceptional(msg, ex)
+      if ::Exceptional::Config.should_send_to_api?
+        ::Exceptional.context(msg)
+        ::Exceptional::Remote.error(::Exceptional::ExceptionData.new(ex))
+      end
+    end
+
+    def send_to_exception_notifier(msg, ex)
+      ::ExceptionNotifier::Notifier.background_exception_notification(ex, :data => { :message => msg }).deliver
+    end
+  end
+end

data/lib/creeper/extensions/action_mailer.rb

@@ -0,0 +1,33 @@
+require 'creeper/extensions/generic_proxy'
+
+module Creeper
+  module Extensions
+    ##
+    # Adds 'delay' and 'delay_for' to ActionMailer to offload arbitrary email
+    # delivery to Creeper.  Example:
+    #
+    #    UserMailer.delay.send_welcome_email(new_user)
+    #    UserMailer.delay_for(5.days).send_welcome_email(new_user)
+    class DelayedMailer
+      include Creeper::Worker
+      # I think it's reasonable to assume that emails should take less
+      # than 30 seconds to send.
+      creeper_options :timeout => 30
+
+      def perform(yml)
+        (target, method_name, args) = YAML.load(yml)
+        target.send(method_name, *args).deliver
+      end
+    end
+
+    module ActionMailer
+      def delay
+        Proxy.new(DelayedMailer, self)
+      end
+      def delay_for(interval)
+        Proxy.new(DelayedMailer, self, Time.now.to_f + interval.to_f)
+      end
+    end
+
+  end
+end