backburner 0.0.1

data/lib/backburner.rb

@@ -0,0 +1,51 @@
+require 'beanstalk-client'
+require 'json'
+require 'uri'
+require 'timeout'
+require 'backburner/version'
+require 'backburner/helpers'
+require 'backburner/configuration'
+require 'backburner/logger'
+require 'backburner/connection'
+require 'backburner/performable'
+require 'backburner/worker'
+require 'backburner/queue'
+
+module Backburner
+  class << self
+
+    # Enqueues a job to be performed with arguments
+    # Backburner.enqueue NewsletterSender, self.id, user.id
+    def enqueue(job_class, *args)
+      Backburner::Worker.enqueue(job_class, args, {})
+    end
+
+    # Begins working on jobs enqueued with optional tubes specified
+    # Backburner.work('newsletter_sender', 'test_job')
+    def work(*tubes)
+      Backburner::Worker.start(tubes)
+    end
+
+    # Yields a configuration block
+    # Backburner.configure do |config|
+    #  config.beanstalk_url = "beanstalk://..."
+    # end
+    def configure(&block)
+      yield(configuration)
+      configuration
+    end
+
+    # Returns the configuration options set for Backburner
+    # Backburner.configuration.beanstalk_url => false
+    def configuration
+      @_configuration ||= Configuration.new
+    end
+
+    # Returns the queues that are processed by default if none are specified
+    # default_queues << "foo"
+    # default_queues => ["foo", "bar"]
+    def default_queues
+      configuration.default_queues
+    end
+  end
+end

data/lib/backburner/async_proxy.rb

@@ -0,0 +1,22 @@
+module Backburner
+  # BasicObject for 1.8.7
+  class BasicObject
+    instance_methods.each do |m|
+      undef_method(m) if m.to_s !~ /(?:^__|^nil?$|^send$|^object_id$)/
+    end
+  end unless defined?(::BasicObject)
+
+  # Class allows async task to be proxied
+  class AsyncProxy < BasicObject
+    # AsyncProxy(User, 10, :pri => 1000, :ttr => 1000)
+    # Options include `pri` (priority), `delay` (delay in secs), `ttr` (time to respond)
+    def initialize(klazz, id=nil, opts={})
+      @klazz, @id, @opts = klazz, id, opts
+    end
+
+    # Enqueue as job when a method is invoked
+    def method_missing(method, *args, &block)
+      ::Backburner::Worker.enqueue(@klazz, [@id, method, *args], @opts)
+    end
+  end # AsyncProxy
+end # Backburner

data/lib/backburner/configuration.rb

@@ -0,0 +1,19 @@
+module Backburner
+  class Configuration
+    attr_accessor :beanstalk_url      # beanstalk url connection
+    attr_accessor :tube_namespace     # namespace prefix for every queue
+    attr_accessor :default_priority   # default job priority
+    attr_accessor :respond_timeout    # default job timeout
+    attr_accessor :on_error           # error handler
+    attr_accessor :default_queues     # default queues
+
+    def initialize
+      @beanstalk_url     = "beanstalk://localhost"
+      @tube_namespace    = "backburner.worker.queue"
+      @default_priority  = 65536
+      @respond_timeout   = 120
+      @on_error          = nil
+      @default_queues    = []
+    end
+  end # Configuration
+end # Backburner

data/lib/backburner/connection.rb

@@ -0,0 +1,43 @@
+require 'delegate'
+
+module Backburner
+  class Connection < SimpleDelegator
+    class BadURL < RuntimeError; end
+
+    attr_accessor :url, :beanstalk
+
+    def initialize(url)
+      @url = url
+      connect!
+    end
+
+    # Sets the delegator object to the underlying beanstalk connection
+    # self.put(...)
+    def __getobj__
+      __setobj__(@beanstalk)
+      super
+    end
+
+    protected
+
+    # Connects to a beanstalk queue
+    def connect!
+      @beanstalk ||= Beanstalk::Pool.new(beanstalk_addresses)
+    end
+
+    # Returns the beanstalk queue addresses
+    # beanstalk_addresses => ["localhost:11300"]
+    def beanstalk_addresses
+      uris = self.url.split(/[\s,]+/)
+      uris.map {|uri| beanstalk_host_and_port(uri)}
+    end
+
+    # Returns a host and port based on the uri_string given
+    # beanstalk_host_and_port("beanstalk://localhost") => "localhost:11300"
+    def beanstalk_host_and_port(uri_string)
+      uri = URI.parse(uri_string)
+      raise(BadURL, uri_string) if uri.scheme != 'beanstalk'
+      "#{uri.host}:#{uri.port || 11300}"
+    end
+  end # Connection
+end # Backburner

data/lib/backburner/helpers.rb

@@ -0,0 +1,99 @@
+module Backburner
+  module Helpers
+    # Loads in instance and class levels
+    def self.included(base)
+      base.extend self
+    end
+
+    # Prints out exception_message based on specified e
+    def exception_message(e)
+      msg = [ "Exception #{e.class} -> #{e.message}" ]
+
+      base = File.expand_path(Dir.pwd) + '/'
+      e.backtrace.each do |t|
+        msg << "   #{File.expand_path(t).gsub(/#{base}/, '')}"
+      end if e.backtrace
+
+      msg.join("\n")
+    end
+
+    # Given a word with dashes, returns a camel cased version of it.
+    # classify('job-name') # => 'JobName'
+    def classify(dashed_word)
+      dashed_word.to_s.split('-').each { |part| part[0] = part[0].chr.upcase }.join
+    end
+
+    # Given a class, dasherizes the name, used for getting tube names
+    # dasherize('JobName') => "job-name"
+    def dasherize(word)
+      classify(word).to_s.gsub(/::/, '/').
+            gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+            gsub(/([a-z\d])([A-Z])/,'\1_\2').
+            tr("_", "-").downcase
+    end
+
+    # Tries to find a constant with the name specified in the argument string:
+    #
+    # constantize("Module") # => Module
+    # constantize("Test::Unit") # => Test::Unit
+    #
+    # The name is assumed to be the one of a top-level constant, no matter
+    # whether it starts with "::" or not. No lexical context is taken into
+    # account:
+    #
+    # C = 'outside'
+    # module M
+    #   C = 'inside'
+    #   C # => 'inside'
+    #   constantize("C") # => 'outside', same as ::C
+    # end
+    #
+    # NameError is raised when the constant is unknown.
+    def constantize(camel_cased_word)
+      camel_cased_word = camel_cased_word.to_s
+
+      if camel_cased_word.include?('-')
+        camel_cased_word = classify(camel_cased_word)
+      end
+
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      constant = Object
+      names.each do |name|
+        args = Module.method(:const_get).arity != 1 ? [false] : []
+
+        if constant.const_defined?(name, *args)
+          constant = constant.const_get(name)
+        else
+          constant = constant.const_missing(name)
+        end
+      end
+      constant
+    end
+
+    # Returns tube_namespace for backburner
+    # tube_namespace => "some.namespace"
+    def tube_namespace
+      Backburner.configuration.tube_namespace
+    end
+
+    # Expands a tube to include the prefix
+    # expand_tube_name("foo") => <prefix>.foo
+    # expand_tube_name(FooJob) => <prefix>.foo-job
+    def expand_tube_name(tube)
+      prefix = tube_namespace
+      queue_name = if tube.is_a?(String)
+        tube
+      elsif tube.respond_to?(:queue) # use queue name
+        tube.queue
+      elsif tube.is_a?(Class) # no queue name, use job_class
+        tube.name
+      else # turn into a string
+        tube.to_s
+      end
+      [prefix.gsub(/\.$/, ''), dasherize(queue_name).gsub(/^#{prefix}/, '')].join(".")
+    end
+
+  end
+end

data/lib/backburner/logger.rb

@@ -0,0 +1,44 @@
+module Backburner
+  module Logger
+
+    # Loads in instance and class levels
+    def self.included(base)
+      base.extend self
+    end
+
+    # Print out when a job is about to begin
+    def log_job_begin(body)
+      log [ "Working", body ].join(' ')
+      @job_begun = Time.now
+    end
+
+    # Print out when a job completed
+    def log_job_end(name, failed=false)
+      ellapsed = Time.now - @job_begun
+      ms = (ellapsed.to_f * 1000).to_i
+      log "Finished #{name} in #{ms}ms #{failed ? ' (failed)' : ''}"
+    end
+
+    # Prints message about failure when beastalk cannot be connected
+    # failed_connection(ex)
+    def failed_connection(e)
+      log_error exception_message(e)
+      log_error "*** Failed connection to #{connection.url}"
+      log_error "*** Check that beanstalkd is running (or set a different beanstalk url)"
+      exit 1
+    end
+
+    # Print a message to stdout
+    # log("Working on task")
+    def log(msg)
+      puts msg
+    end
+
+    # Print an error to stderr
+    # log_error("Task failed!")
+    def log_error(msg)
+      $stderr.puts msg
+    end
+
+  end
+end

data/lib/backburner/performable.rb

@@ -0,0 +1,40 @@
+require 'backburner/async_proxy'
+
+module Backburner
+  module Performable
+    def self.included(base)
+      base.send(:include, InstanceMethods)
+      base.send(:include, Backburner::Queue)
+      base.extend ClassMethods
+    end
+
+    module InstanceMethods
+      # Return proxy object to enqueue jobs for object
+      # Options: `pri` (priority), `delay` (delay in secs), `ttr` (time to respond), `queue` (queue name)
+      # @model.async(:pri => 1000).do_something("foo")
+      def async(opts={})
+        Backburner::AsyncProxy.new(self.class, self.id, opts)
+      end
+    end # InstanceMethods
+
+    module ClassMethods
+      # Return proxy object to enqueue jobs for object
+      # Options: `pri` (priority), `delay` (delay in secs), `ttr` (time to respond), `queue` (queue name)
+      # Model.async(:ttr => 300).do_something("foo")
+      def async(opts={})
+        Backburner::AsyncProxy.new(self, nil, opts)
+      end
+
+      # Defines perform method for job processing
+      # perform(55, :do_something, "foo", "bar")
+      def perform(id, method, *args)
+        if id # instance
+          find(id).send(method, *args)
+        else # class method
+          send(method, *args)
+        end
+      end # perform
+    end # ClassMethods
+
+  end # Performable
+end # Backburner

data/lib/backburner/queue.rb

@@ -0,0 +1,22 @@
+module Backburner
+  module Queue
+    def self.included(base)
+      base.send(:extend, Backburner::Helpers)
+      base.extend ClassMethods
+      Backburner::Worker.known_queue_classes << base
+    end
+
+    module ClassMethods
+      # Returns or assigns queue name for this job
+      # queue "some.task.name"
+      # queue => "some.task.name"
+      def queue(name=nil)
+        if name
+          @queue_name = name
+        else # accessor
+          @queue_name || dasherize(self.name)
+        end
+      end
+    end # ClassMethods
+  end # Job
+end # Backburner

data/lib/backburner/tasks.rb

@@ -0,0 +1,11 @@
+# require 'backburner/tasks'
+# will give you the backburner tasks
+
+namespace :backburner do
+  # QUEUE=foo,bar,baz rake backburner:work
+  desc "Start an backburner worker"
+  task :work => :environment do
+    queues = (ENV["QUEUE"] ? ENV["QUEUE"].split(',') : nil) rescue nil
+    Backburner.work queues
+  end
+end

data/lib/backburner/version.rb

@@ -0,0 +1,3 @@
+module Backburner
+  VERSION = "0.0.1"
+end

data/lib/backburner/worker.rb

@@ -0,0 +1,140 @@
+module Backburner
+  class Worker
+    include Backburner::Helpers
+    include Backburner::Logger
+
+    class JobNotFound < RuntimeError; end
+    class JobTimeout < RuntimeError; end
+    class JobQueueNotSet < RuntimeError; end
+
+    # Backburner::Worker.known_queue_classes
+    # List of known_queue_classes
+    class << self
+      attr_writer :known_queue_classes
+      def known_queue_classes; @known_queue_classes ||= []; end
+    end
+
+    # Enqueues a job to be processed later by a worker
+    # Options: `pri` (priority), `delay` (delay in secs), `ttr` (time to respond), `queue` (queue name)
+    # Backburner::Worker.enqueue NewsletterSender, [self.id, user.id], :ttr => 1000
+    def self.enqueue(job_class, args=[], opts={})
+      pri   = opts[:pri] || Backburner.configuration.default_priority
+      delay = [0, opts[:delay].to_i].max
+      ttr   = opts[:ttr] || Backburner.configuration.respond_timeout
+      connection.use expand_tube_name(opts[:queue]  || job_class)
+      data = { :class => job_class.name, :args => args }
+      connection.put data.to_json, pri, delay, ttr
+    rescue Beanstalk::NotConnected => e
+      failed_connection(e)
+    end
+
+    # Starts processing jobs in the specified tube_names
+    # Backburner::Worker.start(["foo.tube.name"])
+    def self.start(tube_names=nil)
+      self.new(tube_names).start
+    end
+
+    # Returns the worker connection
+    # Backburner::Worker.connection => <Beanstalk::Pool>
+    def self.connection
+      @connection ||= Connection.new(Backburner.configuration.beanstalk_url)
+    end
+
+    # List of tube names to be watched and processed
+    attr_accessor :tube_names
+
+    # Worker.new(['test.job'])
+    def initialize(tube_names=nil)
+      @tube_names = begin
+        tube_names = tube_names.first if tube_names && tube_names.size == 1 && tube_names.first.is_a?(Array)
+        tube_names = Array(tube_names).compact if tube_names && Array(tube_names).compact.size > 0
+        tube_names = nil if tube_names && tube_names.compact.empty?
+        tube_names
+      end
+    end
+
+    # Starts processing new jobs indefinitely
+    # Primary way to consume and process jobs in specified tubes
+    # @worker.start
+    def start
+      prepare
+      loop { work_one_job }
+    end
+
+    # Setup beanstalk tube_names and watch all specified tubes for jobs.
+    # Used to prepare job queues before processing jobs.
+    # @worker.prepare
+    def prepare
+      self.tube_names ||= Backburner.default_queues.any? ? Backburner.default_queues : all_existing_queues
+      self.tube_names = Array(self.tube_names)
+      self.tube_names.map! { |name| expand_tube_name(name)  }
+      log "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
+      self.tube_names.uniq.each { |name| self.connection.watch(name) }
+      self.connection.list_tubes_watched.each do |server, tubes|
+        tubes.each { |tube| self.connection.ignore(tube) unless self.tube_names.include?(tube) }
+      end
+    rescue Beanstalk::NotConnected => e
+      failed_connection(e)
+    end
+
+    # Reserves one job within the specified queues
+    # Pops the job off and serializes the job to JSON
+    # Each job is performed by invoking `perform` on the job class.
+    # @worker.work_one_job
+    def work_one_job
+      job = self.connection.reserve
+      body = JSON.parse job.body
+      name, args = body["class"], body["args"]
+      self.class.log_job_begin(body)
+      handler = constantize(name)
+      raise(JobNotFound, name) unless handler
+
+      begin
+        Timeout::timeout(job.ttr - 1) do
+          handler.perform(*args)
+        end
+      rescue Timeout::Error
+        raise JobTimeout, "#{name} hit #{job.ttr-1}s timeout"
+      end
+
+      job.delete
+      self.class.log_job_end(name)
+    rescue Beanstalk::NotConnected => e
+      failed_connection(e)
+    rescue SystemExit
+      raise
+    rescue => e
+      job.bury
+      self.class.log_error self.class.exception_message(e)
+      self.class.log_job_end(name, 'failed') if @job_begun
+      handle_error(e, name, args)
+    end
+
+    protected
+
+    # Returns a list of all tubes known within the system
+    # Filtered for tubes that match the known prefix
+    def all_existing_queues
+      known_queues    = Backburner::Worker.known_queue_classes.map(&:queue)
+      existing_tubes  = self.connection.list_tubes.values.flatten.uniq.select { |tube| tube =~ /^#{tube_namespace}/ }
+      known_queues + existing_tubes
+    end
+
+    # Returns a reference to the beanstalk connection
+    def connection
+      self.class.connection
+    end
+
+    # Handles an error according to custom definition
+    # Used when processing a job that errors out
+    def handle_error(e, name, args)
+      if error_handler = Backburner.configuration.on_error
+        if error_handler.arity == 1
+          error_handler.call(e)
+        else
+          error_handler.call(e, name, args)
+        end
+      end
+    end
+  end # Worker
+end # Backburner