backburner 1.7.0 → 1.9.0

data/lib/backburner/helpers.rb

@@ -11,7 +11,7 @@ module Backburner
 
       base = File.expand_path(Dir.pwd) + '/'
       e.backtrace.each do |t|
-        msg << "   #{File.expand_path(t).gsub(/#{base}/, '')}"
+        msg << "   #{File.expand_path(t).gsub(base, '')}"
       end if e.backtrace
 
       msg.join("\n")
@@ -44,20 +44,13 @@ module Backburner
     #   constantize("Module") # => Module
     #   constantize("Test::Unit") # => Test::Unit
     #
-    # NameError is raised when the constant is unknown.
+    # NameError is raised when the constant is unknown. Given a block, yields each module and the name
+    # of the constant it is about to look up in it, so a caller can tell which one failed.
     def constantize(camel_cased_word)
-      camel_cased_word = camel_cased_word.to_s
-
-      if camel_cased_word.include?('-')
-        camel_cased_word = classify(camel_cased_word)
-      end
-
-      names = camel_cased_word.split('::')
-      names.shift if names.empty? || names.first.empty?
-
       constant = Object
-      names.each do |name|
+      constant_names(camel_cased_word).each do |name|
         args = Module.method(:const_get).arity != 1 ? [false] : []
+        yield constant, name if block_given?
 
         if constant.const_defined?(name, *args)
           constant = constant.const_get(name)
@@ -68,6 +61,23 @@ module Backburner
       constant
     end
 
+    # The names of the constants constantize looks up in turn
+    #
+    # @example
+    #   constant_names("Test::Unit") # => ["Test", "Unit"]
+    #
+    def constant_names(camel_cased_word)
+      camel_cased_word = camel_cased_word.to_s
+
+      if camel_cased_word.include?('-')
+        camel_cased_word = classify(camel_cased_word)
+      end
+
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+      names
+    end
+
     # Returns configuration options for backburner
     #
     # @example
@@ -85,7 +95,7 @@ module Backburner
     #   expand_tube_name(FooJob) # => <prefix>.foo-job
     #
     def expand_tube_name(tube)
-      prefix = queue_config.tube_namespace
+      prefix = queue_config.tube_namespace # nil or false mean no prefix; '' gives names that start with the separator, as in 1.7.0
       separator = queue_config.namespace_separator
       queue_name = if tube.is_a?(String)
         tube
@@ -99,7 +109,29 @@ module Backburner
       else # turn into a string
         tube.to_s
       end
-      [prefix.gsub(/\.$/, ''), dasherize(queue_name).gsub(/^#{prefix}/, '')].join(separator).gsub(/#{Regexp::escape(separator)}+/, separator).split(':').first
+      tube_name = dasherize(queue_name)
+      tube_name = [separator.empty? ? prefix : prefix.gsub(/\.$/, ''), tube_name.gsub(namespace_pattern(prefix), '')].join(separator) if prefix
+      tube_name = tube_name.gsub(/#{Regexp::escape(separator)}+/, separator) unless separator.empty? # /+/ isn't a regexp
+      tube_name.split(':').first
+    end
+
+    # What every tube name expand_tube_name gives starts with: the namespace and its separator ('' for a nil or false one)
+    #
+    # @example
+    #   expanded_tube_prefix # => "<prefix>."
+    #
+    def expanded_tube_prefix
+      expand_tube_name('').to_s
+    end
+
+    # The namespace as the pattern expand_tube_name strips from the start of a queue name.
+    # ceiling: a namespace that is a valid regexp is still matched as one ("app.prod" also strips "app-prod"), because
+    # matching it literally would move the jobs of the queue names it matches to other tubes; one that isn't ("app(")
+    # used to fail every enqueue and is matched literally.
+    def namespace_pattern(prefix)
+      Regexp.new("^#{prefix}")
+    rescue RegexpError
+      Regexp.new("^#{Regexp.escape(prefix)}")
     end
 
     # Resolves job priority based on the value given. Can be integer, a class or nothing
@@ -110,21 +142,31 @@ module Backburner
     #  resolve_priority(nil) => <default priority>
     #
     def resolve_priority(pri)
+      priority = resolve_priority_value(pri)
+      return priority if priority
+      default = Backburner.configuration.default_priority # can be a label too
+      resolve_priority_value(default) || default # left as configured if it isn't a known label
+    end
+
+    # Follows a job class's queue_priority and priority labels to a number; nil for nothing,
+    # an unknown label or a cycle of labels
+    def resolve_priority_value(pri, seen_labels = [])
       if pri.respond_to?(:queue_priority)
-        resolve_priority(pri.queue_priority)
+        resolve_priority_value(pri.queue_priority, seen_labels)
       elsif pri.is_a?(String) || pri.is_a?(Symbol) # named priority
-        resolve_priority(Backburner.configuration.priority_labels[pri.to_sym])
+        label = pri.to_sym
+        resolve_priority_value(Backburner.configuration.priority_labels[label], seen_labels + [label]) unless seen_labels.include?(label)
       elsif pri.is_a?(Integer) # numerical
         pri
-      else # default
-        Backburner.configuration.default_priority
       end
     end
 
-    # Resolves job respond timeout based on the value given. Can be integer, a class or nothing
+    # Resolves job respond timeout based on the value given, rounded up to whole seconds. Can be a number,
+    # a class or nothing
     #
     # @example
     #  resolve_respond_timeout(1000) => 1000
+    #  resolve_respond_timeout(1.5) => 2
     #  resolve_respond_timeout(FooBar) => <queue respond_timeout>
     #  resolve_respond_timeout(nil) => <default respond_timeout>
     #
@@ -133,8 +175,11 @@ module Backburner
         resolve_respond_timeout(ttr.queue_respond_timeout)
       elsif ttr.is_a?(Integer) # numerical
         ttr
+      elsif ttr.is_a?(Numeric) && ttr.finite? # fractional
+        whole_seconds(ttr)
       else # default
-        Backburner.configuration.respond_timeout
+        ttr = Backburner.configuration.respond_timeout
+        !ttr.is_a?(Integer) && ttr.is_a?(Numeric) && ttr.finite? ? whole_seconds(ttr) : ttr
       end
     end
 
@@ -155,7 +200,7 @@ module Backburner
       end
     end
 
-    # Resolves retry delay based on the value given. Can be integer, a class or nothing
+    # Resolves retry delay based on the value given. Can be a number, a class or nothing
     #
     # @example
     #  resolve_retry_delay(5) => 5
@@ -165,7 +210,7 @@ module Backburner
     def resolve_retry_delay(delay)
       if delay.respond_to?(:queue_retry_delay)
         resolve_retry_delay(delay.queue_retry_delay)
-      elsif delay.is_a?(Integer) # numerical
+      elsif delay.is_a?(Numeric) && delay.finite? # numerical (the worker rounds the delay it computes from it up)
         delay
       else # default
         Backburner.configuration.retry_delay
@@ -189,5 +234,15 @@ module Backburner
       end
     end
 
+    # Rounds a delay up to the whole seconds beanstalkd takes, so a job is never ready early
+    #
+    # @example
+    #  whole_seconds(1.5) => 2
+    #  whole_seconds(nil) => 0
+    #
+    def whole_seconds(delay)
+      [0, delay.to_f.ceil].max
+    end
+
   end # Helpers
 end # Backburner

data/lib/backburner/job.rb

@@ -45,12 +45,15 @@ module Backburner
     def process
       # Invoke before hook and stop if false
       res = @hooks.invoke_hook_events(job_name, :before_perform, *args)
-      return false unless res
+      unless res
+        task.delete # aborted: don't leave it reserved, or beanstalkd hands it out again after its TTR
+        return false
+      end
       # Execute the job
       @hooks.around_hook_events(job_name, :around_perform, *args) do
-        # We subtract one to ensure we timeout before beanstalkd does, except if:
-        #  a) ttr == 0, to support never timing out
-        #  b) ttr == 1, so that we don't accidentally set it to never time out
+        # We subtract one to ensure we timeout before beanstalkd does, except if
+        # ttr == 1 (beanstalkd's minimum: it stores a ttr of 0 as 1), so that we
+        # don't accidentally set it to never time out.
         #  NB: A ttr of 1 will likely result in race conditions between
         #  Backburner and beanstalkd and should probably be avoided
         timeout_job_after(task.ttr > 1 ? task.ttr - 1 : task.ttr) { job_class.perform(*args) }
@@ -80,10 +83,16 @@ module Backburner
     #
     def job_class
       handler = try_job_class
-      raise(JobNotFound, self.name) unless handler
+      raise(JobNotFound, @job_class_error ? "#{name} (#{@job_class_error.class}: #{@job_class_error.message.to_s.lines.first.to_s.chomp})" : name) unless handler
       handler
     end
 
+    # Whether the last lookup of the job's class found it missing (not defined, or an autoload whose
+    # file isn't there) rather than failing to load it (its file raised, or required a missing gem)
+    def job_class_missing?
+      !!@job_class_missing
+    end
+
     protected
 
     # Attempts to return a constantized job name, otherwise reverts to the name string
@@ -96,11 +105,41 @@ module Backburner
     end
 
     def try_job_class
-      constantize(self.name)
-    rescue NameError
+      @job_class_error = nil
+      @job_class_missing = false
+      scope = segment = autoload_path = nil
+      invalid_name = !constant_names(self.name).all? { |name| constant_name?(name) }
+      constantize(self.name) do |mod, name|
+        scope, segment = mod, name
+        autoload_path = mod.autoload?(name) rescue nil # a module, or the value of a constant that isn't one
+      end
+    rescue StandardError, ScriptError => e # its file may raise anything; an autoloader may raise LoadError for a file that isn't there
+      @job_class_error = e
+      @job_class_missing = !invalid_name && missing_constant_error?(e, scope, segment, autoload_path)
       nil
     end
 
+    # Whether a name can be a constant's: one that can't ("not a class", "123") will never be found
+    def constant_name?(name)
+      Module.new.const_defined?(name, false) # raises NameError for one that can't
+      true
+    rescue NameError
+      false
+    end
+
+    # Whether an error raised while looking up `segment` in `scope` says that constant is missing,
+    # rather than coming from inside the file that defines it (a NameError for another constant, or for
+    # one of the same name in another module, a NoMethodError, a require of a missing gem, any other
+    # error or a syntax error)
+    def missing_constant_error?(e, scope, segment, autoload_path)
+      case e
+      when NoMethodError then false
+      when NameError then e.name.to_s == segment.to_s && (e.receiver.equal?(scope) rescue true) # no receiver: by name only
+      when LoadError then e.path.nil? || e.path == autoload_path
+      else false
+      end
+    end
+
     # Timeout job within specified block after given time.
     #
     # @example

data/lib/backburner/logger.rb

@@ -45,6 +45,15 @@ module Backburner
       logger ? logger.error(msg) : $stderr.puts(msg)
     end
 
+    # Print a warning to stderr
+    #
+    # @example
+    #   log_warn("Task released!")
+    #
+    def log_warn(msg)
+      logger ? (logger.respond_to?(:warn) ? logger.warn(msg) : logger.error(msg)) : $stderr.puts(msg)
+    end
+
     # Return logger if specified
     def logger
       Backburner.configuration.logger

data/lib/backburner/performable.rb

@@ -28,14 +28,16 @@ module Backburner
         Backburner::AsyncProxy.new(self, nil, opts)
       end
 
-      # Defines perform method for job processing
+      # Defines perform method for job processing. It calls a method the app defines on the class,
+      # its superclasses or the modules they include, and raises NoMethodError for any other.
       # @example
       #   perform(55, :do_something, "foo", "bar")
       def perform(id, method, *args)
         if id # instance
-          find(id).send(method, *args)
+          record = find(id)
+          Backburner::Performable.perform_job_method(record, record.class, Object, method, args)
         else # class method
-          send(method, *args)
+          Backburner::Performable.perform_job_method(self, singleton_class, Object.singleton_class, method, args)
         end
       end # perform
 
@@ -79,7 +81,7 @@ module Backburner
       with_async_name    = :"#{aliased_method}_with_async#{punctuation}"
       without_async_name = :"#{aliased_method}_without_async#{punctuation}"
 
-      klass.send(:include, Performable) unless included_modules.include?(Performable)
+      klass.send(:include, Performable) unless klass.included_modules.include?(Performable)
       klass_eval_scope.class_eval do
         define_method with_async_name do |*args|
           async(opts).__send__ without_async_name, *args
@@ -90,6 +92,29 @@ module Backburner
     end
     private_class_method :_handle_asynchronously
 
+    # Calls a job's method on `receiver`. It raises NoMethodError when the method isn't one the app
+    # defined: one that `methods_of` (the receiver's class, or the singleton class for a class method)
+    # gets from `core` (Object, or Object's singleton class for a class method) or the modules mixed
+    # into it, from Backburner's own modules, or Ruby's method_missing hook. A method with the same name
+    # as one of core's must be the app's own definition: one that method_missing would handle, or one
+    # from Delegator's copy of Kernel (the module delegate.rb aliases __raise__ in), is refused too.
+    def self.perform_job_method(receiver, methods_of, core, method, args)
+      owner = begin
+        methods_of.instance_method(method).owner
+      rescue NameError
+        nil
+      end
+      refused = if owner && (core.ancestors.include?(owner) || [InstanceMethods, ClassMethods, Backburner::Queue::ClassMethods].include?(owner))
+        true
+      elsif core.method_defined?(method) || core.private_method_defined?(method)
+        owner.nil? || method.to_sym == :method_missing || (defined?(::Delegator) && owner.equal?(::Delegator.instance_method(:__raise__).owner))
+      end
+      if refused
+        kind = methods_of.singleton_class? ? "class method" : "method"
+        raise NoMethodError.new("undefined job #{kind} '#{method}' for #{methods_of.singleton_class? ? receiver : methods_of}", method.to_sym, receiver: receiver)
+      end
+      receiver.__send__(method, *args)
+    end
 
   end # Performable
 end # Backburner

data/lib/backburner/queue.rb

@@ -1,17 +1,14 @@
 module Backburner
   module Queue
     def self.included(base)
-      base.instance_variable_set(:@queue_name, nil)
-      base.instance_variable_set(:@queue_priority, nil)
-      base.instance_variable_set(:@queue_respond_timeout, nil)
-      base.instance_variable_set(:@queue_max_job_retries, nil)
-      base.instance_variable_set(:@queue_retry_delay, nil)
-      base.instance_variable_set(:@queue_retry_delay_proc, nil)
-      base.instance_variable_set(:@queue_jobs_limit, nil)
-      base.instance_variable_set(:@queue_garbage_limit, nil)
-      base.instance_variable_set(:@queue_retry_limit, nil)
+      # Initialize the settings, but keep any already set: including Queue again (as including
+      # Performable does) must not reset them
+      [:@queue_name, :@queue_priority, :@queue_respond_timeout, :@queue_max_job_retries, :@queue_retry_delay,
+       :@queue_retry_delay_proc, :@queue_jobs_limit, :@queue_garbage_limit, :@queue_retry_limit].each do |setting|
+        base.instance_variable_set(setting, nil) unless base.instance_variable_defined?(setting)
+      end
       base.extend ClassMethods
-      Backburner::Worker.known_queue_classes << base
+      Backburner::Worker.known_queue_classes << base unless Backburner::Worker.known_queue_classes.include?(base)
     end
 
     module ClassMethods
@@ -25,7 +22,8 @@ module Backburner
         if name
           @queue_name = name
         else # accessor
-          (@queue_name.is_a?(Proc) ? @queue_name.call(self) : @queue_name) || Backburner.configuration.primary_queue
+          queue_name = inherited_queue_setting(:@queue_name)
+          (queue_name.is_a?(Proc) ? queue_name.call(self) : queue_name) || Backburner.configuration.primary_queue
         end
       end
 
@@ -39,7 +37,7 @@ module Backburner
         if pri
           @queue_priority = pri
         else # accessor
-          @queue_priority
+          inherited_queue_setting(:@queue_priority)
         end
       end
 
@@ -53,7 +51,7 @@ module Backburner
         if ttr
           @queue_respond_timeout = ttr
         else # accessor
-          @queue_respond_timeout
+          inherited_queue_setting(:@queue_respond_timeout)
         end
       end
 
@@ -67,7 +65,7 @@ module Backburner
         if delay
           @queue_max_job_retries = delay
         else # accessor
-          @queue_max_job_retries
+          inherited_queue_setting(:@queue_max_job_retries)
         end
       end
 
@@ -81,7 +79,7 @@ module Backburner
         if delay
           @queue_retry_delay = delay
         else # accessor
-          @queue_retry_delay
+          inherited_queue_setting(:@queue_retry_delay)
         end
       end
 
@@ -95,7 +93,7 @@ module Backburner
         if proc
           @queue_retry_delay_proc = proc
         else # accessor
-          @queue_retry_delay_proc
+          inherited_queue_setting(:@queue_retry_delay_proc)
         end
       end
 
@@ -109,7 +107,7 @@ module Backburner
         if limit
           @queue_jobs_limit = limit
         else #accessor
-          @queue_jobs_limit
+          inherited_queue_setting(:@queue_jobs_limit)
         end
       end
 
@@ -123,7 +121,7 @@ module Backburner
         if limit
           @queue_garbage_limit = limit
         else #accessor
-          @queue_garbage_limit
+          inherited_queue_setting(:@queue_garbage_limit)
         end
       end
 
@@ -137,8 +135,28 @@ module Backburner
         if limit
           @queue_retry_limit = limit
         else #accessor
-          @queue_retry_limit
+          inherited_queue_setting(:@queue_retry_limit)
+        end
+      end
+
+      private
+
+      # Registers subclasses as known queues too (they inherit the queue settings)
+      def inherited(subclass)
+        super
+        Backburner::Worker.known_queue_classes << subclass unless Backburner::Worker.known_queue_classes.include?(subclass)
+      end
+
+      # Returns the setting from this class or else from the closest superclass that has it,
+      # so a subclass shares its parent's queue settings unless it sets its own
+      def inherited_queue_setting(ivar)
+        klass = self
+        while klass.is_a?(ClassMethods)
+          value = klass.instance_variable_get(ivar) if klass.instance_variable_defined?(ivar)
+          return value unless value.nil?
+          klass = klass.respond_to?(:superclass) ? klass.superclass : nil
         end
+        nil
       end
     end # ClassMethods
   end # Queue

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.7.0"
+  VERSION = "1.9.0"
 end