backburner-allq 1.0.0

data/lib/backburner/helpers.rb

@@ -0,0 +1,193 @@
+module Backburner
+  module Helpers
+    # Loads in instance and class levels
+    def self.included(base)
+      base.extend self
+    end
+
+    # Prints out exception_message based on specified e
+    def exception_message(e)
+      msg = [ "Exception #{e.class} -> #{e.message}" ]
+
+      base = File.expand_path(Dir.pwd) + '/'
+      e.backtrace.each do |t|
+        msg << "   #{File.expand_path(t).gsub(/#{base}/, '')}"
+      end if e.backtrace
+
+      msg.join("\n")
+    end
+
+    # Given a word with dashes, returns a camel cased version of it.
+    #
+    # @example
+    #   classify('job-name') # => 'JobName'
+    #
+    def classify(dashed_word)
+      dashed_word.to_s.split('-').each { |part| part[0] = part[0].chr.upcase }.join
+    end
+
+    # Given a class, dasherizes the name, used for getting tube names
+    #
+    # @example
+    #   dasherize('JobName') # => "job-name"
+    #
+    def dasherize(word)
+      classify(word).to_s.gsub(/::/, '/').
+            gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+            gsub(/([a-z\d])([A-Z])/,'\1_\2').
+            tr("_", "-").downcase
+    end
+
+    # Tries to find a constant with the name specified in the argument string:
+    #
+    # @example
+    #   constantize("Module") # => Module
+    #   constantize("Test::Unit") # => Test::Unit
+    #
+    # NameError is raised when the constant is unknown.
+    def constantize(camel_cased_word)
+      camel_cased_word = camel_cased_word.to_s
+
+      if camel_cased_word.include?('-')
+        camel_cased_word = classify(camel_cased_word)
+      end
+
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+
+      constant = Object
+      names.each do |name|
+        args = Module.method(:const_get).arity != 1 ? [false] : []
+
+        if constant.const_defined?(name, *args)
+          constant = constant.const_get(name)
+        else
+          constant = constant.const_missing(name)
+        end
+      end
+      constant
+    end
+
+    # Returns configuration options for backburner
+    #
+    # @example
+    #   queue_config.max_job_retries => 3
+    #
+    def queue_config
+      Backburner.configuration
+    end
+
+    # Expands a tube to include the prefix
+    #
+    # @example
+    #   expand_tube_name("foo_with_settings:3:100:6") # => <prefix>.foo_with_settings
+    #   expand_tube_name("foo") # => <prefix>.foo
+    #   expand_tube_name(FooJob) # => <prefix>.foo-job
+    #
+    def expand_tube_name(tube)
+      prefix = queue_config.tube_namespace
+      separator = queue_config.namespace_separator
+      queue_name = if tube.is_a?(String)
+        tube
+      elsif tube.respond_to?(:queue) # use queue name
+        queue = tube.queue
+        queue.is_a?(Proc) ? queue.call(tube) : queue
+      elsif tube.is_a?(Proc)
+        tube.call
+      elsif tube.is_a?(Class) # no queue name, use default
+        queue_config.primary_queue # tube.name
+      else # turn into a string
+        tube.to_s
+      end
+      [prefix.gsub(/\.$/, ''), dasherize(queue_name).gsub(/^#{prefix}/, '')].join(separator).gsub(/#{Regexp::escape(separator)}+/, separator).split(':').first
+    end
+
+    # Resolves job priority based on the value given. Can be integer, a class or nothing
+    #
+    # @example
+    #  resolve_priority(1000) => 1000
+    #  resolve_priority(FooBar) => <queue priority>
+    #  resolve_priority(nil) => <default priority>
+    #
+    def resolve_priority(pri)
+      if pri.respond_to?(:queue_priority)
+        resolve_priority(pri.queue_priority)
+      elsif pri.is_a?(String) || pri.is_a?(Symbol) # named priority
+        resolve_priority(Backburner.configuration.priority_labels[pri.to_sym])
+      elsif pri.is_a?(Integer) # numerical
+        pri
+      else # default
+        Backburner.configuration.default_priority
+      end
+    end
+
+    # Resolves job respond timeout based on the value given. Can be integer, a class or nothing
+    #
+    # @example
+    #  resolve_respond_timeout(1000) => 1000
+    #  resolve_respond_timeout(FooBar) => <queue respond_timeout>
+    #  resolve_respond_timeout(nil) => <default respond_timeout>
+    #
+    def resolve_respond_timeout(ttr)
+      if ttr.respond_to?(:queue_respond_timeout)
+        resolve_respond_timeout(ttr.queue_respond_timeout)
+      elsif ttr.is_a?(Integer) # numerical
+        ttr
+      else # default
+        Backburner.configuration.respond_timeout
+      end
+    end
+
+    # Resolves max retries based on the value given. Can be integer, a class or nothing
+    #
+    # @example
+    #  resolve_max_job_retries(5) => 5
+    #  resolve_max_job_retries(FooBar) => <queue max_job_retries>
+    #  resolve_max_job_retries(nil) => <default max_job_retries>
+    #
+    def resolve_max_job_retries(retries)
+      if retries.respond_to?(:queue_max_job_retries)
+        resolve_max_job_retries(retries.queue_max_job_retries)
+      elsif retries.is_a?(Integer) # numerical
+        retries
+      else # default
+        Backburner.configuration.max_job_retries
+      end
+    end
+
+    # Resolves retry delay based on the value given. Can be integer, a class or nothing
+    #
+    # @example
+    #  resolve_retry_delay(5) => 5
+    #  resolve_retry_delay(FooBar) => <queue retry_delay>
+    #  resolve_retry_delay(nil) => <default retry_delay>
+    #
+    def resolve_retry_delay(delay)
+      if delay.respond_to?(:queue_retry_delay)
+        resolve_retry_delay(delay.queue_retry_delay)
+      elsif delay.is_a?(Integer) # numerical
+        delay
+      else # default
+        Backburner.configuration.retry_delay
+      end
+    end
+
+    # Resolves retry delay proc based on the value given. Can be proc, a class or nothing
+    #
+    # @example
+    #  resolve_retry_delay_proc(proc) => proc
+    #  resolve_retry_delay_proc(FooBar) => <queue retry_delay_proc>
+    #  resolve_retry_delay_proc(nil) => <default retry_delay_proc>
+    #
+    def resolve_retry_delay_proc(proc)
+      if proc.respond_to?(:queue_retry_delay_proc)
+        resolve_retry_delay_proc(proc.queue_retry_delay_proc)
+      elsif proc.is_a?(Proc)
+        proc
+      else # default
+        Backburner.configuration.retry_delay_proc
+      end
+    end
+
+  end # Helpers
+end # Backburner

data/lib/backburner/hooks.rb

@@ -0,0 +1,53 @@
+module Backburner
+  class Hooks
+    class << self
+      # Triggers all method hooks that match the given event type with specified arguments.
+      #
+      # @example
+      #   invoke_hook_events(hookable, :before_enqueue, 'some', 'args')
+      #   invoke_hook_events(hookable, :after_perform, 5)
+      #
+      def invoke_hook_events(hookable, event, *args)
+        res = find_hook_events(hookable, event).map { |e| hookable.send(e, *args) }
+        return false if res.any? { |result| result == false }
+        res
+      end
+
+      # Triggers all method hooks that match given around event type. Used for 'around' hooks
+      # that stack over the original task cumulatively onto one another.
+      #
+      # The final block will be the one that actually invokes the
+      # original task after calling all other around blocks.
+      #
+      # @example
+      #   around_hook_events(hookable, :around_perform) { hookable.perform }
+      #
+      def around_hook_events(hookable, event, *args, &block)
+        raise "Please pass a block to hook events!" unless block_given?
+        around_hooks = find_hook_events(hookable, event).reverse
+        aggregate_filter = Proc.new { |&blk| blk.call }
+        around_hooks.each do |ah|
+          prior_around_filter = aggregate_filter
+          aggregate_filter = Proc.new do |&blk|
+            hookable.method(ah).call(*args) do
+              prior_around_filter.call(&blk)
+            end
+          end
+        end
+        aggregate_filter.call(&block)
+      end
+
+      protected
+
+      # Returns all methods that match given hook type
+      #
+      # @example
+      #   find_hook_events(:before_enqueue)
+      #   # => ['before_enqueue_foo', 'before_enqueue_bar']
+      #
+      def find_hook_events(hookable, event)
+        (hookable.methods - Object.methods).grep(/^#{event}/).sort
+      end
+    end
+  end # Hooks
+end # Backburner

data/lib/backburner/job.rb

@@ -0,0 +1,118 @@
+module Backburner
+  # A single backburner job which can be processed and removed by the worker
+  class Job < SimpleDelegator
+    include Backburner::Helpers
+
+    # Raises when a job times out
+    class JobTimeout < RuntimeError; end
+    class JobNotFound < RuntimeError; end
+    class JobFormatInvalid < RuntimeError; end
+    class RetryJob < RuntimeError; end
+
+    attr_accessor :task, :body, :name, :args
+
+    # Construct a job to be parsed and processed
+    #
+    # task is a reserved object containing the json body in the form of
+    #   { :class => "NewsletterSender", :args => ["foo@bar.com"] }
+    #
+    # @example
+    #  Backburner::Job.new(payload)
+    #
+    def initialize(task)
+      @hooks = Backburner::Hooks
+      @task = task
+      @body = task.body.is_a?(Hash) ? task.body : Backburner.configuration.job_parser_proc.call(task.body)
+      @name = body["class"] || body[:class]
+      @args = body["args"] || body[:args]
+    rescue => ex # Job was not valid format
+      self.bury
+      raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
+    end
+
+    # Sets the delegator object to the underlying beaneater job
+    # self.bury
+    def __getobj__
+      __setobj__(@task)
+      super
+    end
+
+    # Processes a job and handles any failure, deleting the job once complete
+    #
+    # @example
+    #   @task.process
+    #
+    def process
+      # Invoke before hook and stop if false
+      res = @hooks.invoke_hook_events(job_name, :before_perform, *args)
+      return false unless res
+      # Execute the job
+      @hooks.around_hook_events(job_name, :around_perform, *args) do
+        # We subtract one to ensure we timeout before beanstalkd does, except if:
+        #  a) ttr == 0, to support never timing out
+        #  b) ttr == 1, so that we don't accidentally set it to never time out
+        #  NB: A ttr of 1 will likely result in race conditions between
+        #  Backburner and beanstalkd and should probably be avoided
+        timeout_job_after(task.ttr > 1 ? task.ttr - 1 : task.ttr) { job_class.perform(*args) }
+      end
+      task.delete
+      # Invoke after perform hook
+      @hooks.invoke_hook_events(job_name, :after_perform, *args)
+    rescue => e
+      @hooks.invoke_hook_events(job_name, :on_failure, e, *args)
+      raise e
+    end
+
+    def bury
+      @hooks.invoke_hook_events(job_name, :on_bury, *args)
+      task.bury
+    end
+
+    def retry(count, delay)
+      @hooks.invoke_hook_events(job_name, :on_retry, count, delay, *args)
+      task.release(delay: delay)
+    end
+
+    # Returns the class for the job handler
+    #
+    # @example
+    #   job_class # => NewsletterSender
+    #
+    def job_class
+      handler = try_job_class
+      raise(JobNotFound, self.name) unless handler
+      handler
+    end
+
+    protected
+
+    # Attempts to return a constantized job name, otherwise reverts to the name string
+    #
+    # @example
+    #   job_name # => "SomeUnknownJob"
+    def job_name
+      handler = try_job_class
+      handler ? handler : self.name
+    end
+
+    def try_job_class
+      constantize(self.name)
+    rescue NameError
+      nil
+    end
+
+    # Timeout job within specified block after given time.
+    #
+    # @example
+    #   timeout_job_after(3) { do_something! }
+    #
+    def timeout_job_after(secs, &block)
+      begin
+        Timeout::timeout(secs) { yield }
+      rescue Timeout::Error => e
+        raise JobTimeout, "#{name}(#{(@args||[]).join(', ')}) hit #{secs}s timeout.\nbacktrace: #{e.backtrace}"
+      end
+    end
+
+  end # Job
+end # Backburner

data/lib/backburner/logger.rb

@@ -0,0 +1,53 @@
+require 'logger'
+
+module Backburner
+  module Logger
+    # Loads in instance and class levels
+    def self.included(base)
+      base.extend self
+    end
+
+    # Print out when a job is about to begin
+    def log_job_begin(name, args)
+      log_info "Work job #{name} with #{args.inspect}"
+      Thread.current[:job_started_at] = Time.now
+    end
+
+    # Print out when a job completed
+    # If message is nil, job is considered complete
+    def log_job_end(name, message = nil)
+      ellapsed = Time.now - job_started_at
+      ms = (ellapsed.to_f * 1000).to_i
+      action_word = message ? 'Finished' : 'Completed'
+      log_info("#{action_word} #{name} in #{ms}ms #{message}")
+    end
+
+    # Returns true if the job logging started
+    def job_started_at
+      Thread.current[:job_started_at]
+    end
+
+    # Print a message to stdout
+    #
+    # @example
+    #   log_info("Working on task")
+    #
+    def log_info(msg)
+      logger ? logger.info(msg) : puts(msg)
+    end
+
+    # Print an error to stderr
+    #
+    # @example
+    #   log_error("Task failed!")
+    #
+    def log_error(msg)
+      logger ? logger.error(msg) : $stderr.puts(msg)
+    end
+
+    # Return logger if specified
+    def logger
+      Backburner.configuration.logger
+    end
+  end
+end

data/lib/backburner/performable.rb

@@ -0,0 +1,95 @@
+require 'backburner/async_proxy'
+
+module Backburner
+  module Performable
+    def self.included(base)
+      base.send(:include, InstanceMethods)
+      base.send(:include, Backburner::Queue)
+      base.extend ClassMethods
+    end
+
+    module InstanceMethods
+      # Return proxy object to enqueue jobs for object
+      # Options: `pri` (priority), `delay` (delay in secs), `ttr` (time to respond), `queue` (queue name)
+      # @example
+      #   @model.async(:pri => 1000).do_something("foo")
+      #
+      def async(opts={})
+        Backburner::AsyncProxy.new(self.class, self.id, opts)
+      end
+    end # InstanceMethods
+
+    module ClassMethods
+      # Return proxy object to enqueue jobs for object
+      # Options: `pri` (priority), `delay` (delay in secs), `ttr` (time to respond), `queue` (queue name)
+      # @example
+      #   Model.async(:ttr => 300).do_something("foo")
+      def async(opts={})
+        Backburner::AsyncProxy.new(self, nil, opts)
+      end
+
+      # Defines perform method for job processing
+      # @example
+      #   perform(55, :do_something, "foo", "bar")
+      def perform(id, method, *args)
+        if id # instance
+          find(id).send(method, *args)
+        else # class method
+          send(method, *args)
+        end
+      end # perform
+
+      # Always handle an instance method asynchronously
+      # @example
+      #   User.handle_asynchronously :send_welcome_email, queue: 'send-mail', delay: 10
+      def handle_asynchronously(method, opts={})
+        Backburner::Performable.handle_asynchronously(self, method, opts)
+      end
+
+      # Always handle a class method asynchronously
+      # @example
+      #   User.handle_static_asynchronously :update_recent_visitors, ttr: 300
+      def handle_static_asynchronously(method, opts={})
+        Backburner::Performable.handle_static_asynchronously(self, method, opts)
+      end
+    end # ClassMethods
+
+
+    # Make all calls to an instance method asynchronous. The given opts will be passed
+    # to the async method.
+    # @example
+    #   Backburner::Performable.handle_asynchronously(MyObject, :long_task, queue: 'long-tasks')
+    # NB: The method called on the async proxy will be ""#{method}_without_async". This
+    # will also be what's given to the Worker.enqueue method so your workers need
+    # to know about that. It shouldn't be a problem unless the producer and consumer are
+    # from different codebases (or anywhere they don't both call the handle_asynchronously
+    # method when booting up)
+    def self.handle_asynchronously(klass, method, opts={})
+      _handle_asynchronously(klass, klass, method, opts)
+    end
+
+    # Make all calls to a class method asynchronous. The given opts will be passed
+    # to the async method. Please see the NB on #handle_asynchronously
+    def self.handle_static_asynchronously(klass, method, opts={})
+      _handle_asynchronously(klass, klass.singleton_class, method, opts)
+    end
+
+    def self._handle_asynchronously(klass, klass_eval_scope, method, opts={})
+      aliased_method, punctuation = method.to_s.sub(/([?!=])$/, ''), $1
+      with_async_name    = :"#{aliased_method}_with_async#{punctuation}"
+      without_async_name = :"#{aliased_method}_without_async#{punctuation}"
+
+      klass.send(:include, Performable) unless included_modules.include?(Performable)
+      klass_eval_scope.class_eval do
+        define_method with_async_name do |*args|
+          async(opts).__send__ without_async_name, *args
+        end
+        alias_method without_async_name, method.to_sym
+        alias_method method.to_sym, with_async_name
+      end
+    end
+    private_class_method :_handle_asynchronously
+
+
+  end # Performable
+end # Backburner