backburner 1.6.1 → 1.8.0

data/lib/active_job/queue_adapters/backburner_adapter.rb

@@ -0,0 +1,41 @@
+# Rails' own adapter files require backburner too, for apps that don't require it themselves
+require 'backburner' unless defined?(Backburner::Worker)
+
+module ActiveJob
+  module QueueAdapters
+    # Rails 7.2 added AbstractAdapter; the adapters of older Rails have no superclass
+    rails_7_2_or_newer = const_defined?(:AbstractAdapter, false)
+
+    # Explicitly remove the implementation rails' 7.2+ ship. Older rails' keep theirs, unless their autoload
+    # loads this file instead (it shadows theirs when this gem's lib comes first on the load path).
+    remove_const(:BackburnerAdapter) if rails_7_2_or_newer && const_defined?(:BackburnerAdapter)
+
+    # = Backburner adapter for Active Job
+    #
+    # To use Backburner set the queue_adapter config to +:backburner+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :backburner
+    class BackburnerAdapter < (rails_7_2_or_newer ? ::ActiveJob::QueueAdapters::AbstractAdapter : Object)
+      def enqueue(job)
+        response = Backburner::Worker.enqueue(JobWrapper, [job.serialize], queue: job.queue_name, pri: job.priority)
+        job.provider_job_id = response[:id] if response.is_a?(Hash)
+        response
+      end
+
+      def enqueue_at(job, timestamp)
+        delay = timestamp - Time.current.to_f
+        response = Backburner::Worker.enqueue(JobWrapper, [job.serialize], queue: job.queue_name, pri: job.priority, delay: delay)
+        job.provider_job_id = response[:id] if response.is_a?(Hash)
+        response
+      end
+
+      class JobWrapper
+        class << self
+          def perform(job_data)
+            Base.execute job_data
+          end
+        end
+      end
+    end
+  end
+end

data/lib/backburner/configuration.rb

@@ -19,6 +19,7 @@ module Backburner
     attr_accessor :reserve_timeout     # duration to wait to reserve on a single server
     attr_accessor :job_serializer_proc # proc to write the job body to a string
     attr_accessor :job_parser_proc     # proc to parse a job body from a string
+    attr_reader   :enqueue_connection_max_idle # seconds an idle enqueue connection is reused for (nil: no limit)
 
     def initialize
       @beanstalk_url       = "beanstalk://127.0.0.1"
@@ -38,6 +39,12 @@ module Backburner
       @reserve_timeout     = nil
       @job_serializer_proc = lambda { |body| body.to_json }
       @job_parser_proc     = lambda { |body| JSON.parse(body) }
+      @enqueue_connection_max_idle = 60
+    end
+
+    # Accepts a number or a numeric string (from ENV, say), or nil for no limit
+    def enqueue_connection_max_idle=(seconds)
+      @enqueue_connection_max_idle = seconds.nil? ? nil : Float(seconds)
     end
 
     def namespace_separator=(val)

data/lib/backburner/connection.rb

@@ -44,7 +44,9 @@ module Backburner
     # completely new connection)
     # @raise [Beaneater::NotConnected] If beanstalk fails to connect
     def reconnect!
-      close
+      # Close the socket but keep the delegate: if connecting fails, the next call
+      # must still reach ensure_connected! and try again (unlike after #close)
+      @beanstalk.close if @beanstalk
       connect!
       @on_reconnect.call(self) if @on_reconnect.respond_to?(:call)
     end
@@ -67,7 +69,7 @@ module Backburner
       begin
         yield
 
-      rescue Beaneater::NotConnected
+      rescue Beaneater::NotConnected => e
         if retry_count > 0
           reconnect!
           retry_count -= 1

data/lib/backburner/enqueue_pool.rb

@@ -0,0 +1,88 @@
+module Backburner
+  # The connections Worker.enqueue puts jobs with, shared by the process's threads and fibers,
+  # so enqueueing doesn't open a TCP connection per job (that ran out of ports, #125).
+  # A connection is checked out for one enqueue and checked back in after it; up to
+  # MAX_IDLE_CONNECTIONS stay open between enqueues and the rest are closed.
+  #
+  # ceiling: idle connections are only closed when they are next checked out (or when the URL
+  # changes or the process exits); a reaper thread would close them sooner.
+  class EnqueuePool
+    MAX_IDLE_CONNECTIONS = 5
+
+    # The pool of this process: a forked child gets a new one and leaves its parent's
+    # sockets (and a lock another thread may have held at the fork) alone.
+    # ceiling: no lock, so threads that race on the first enqueue (or the first after a fork) may
+    # each make a pool; only the last one is kept, and the others' connections stay open until
+    # garbage collected. A class-level lock would need replacing after a fork, as a thread that
+    # doesn't exist in the child may hold it.
+    def self.current
+      pool = @current
+      pool && pool.pid == Process.pid ? pool : (@current = new)
+    end
+
+    attr_reader :pid
+
+    def initialize
+      @pid  = Process.pid
+      @lock = Mutex.new
+      @url  = nil
+      @idle = [] # [connection, idle since], most recently used last
+    end
+
+    # Yields a connection to the configured beanstalk_url. If the block doesn't finish (it raises,
+    # or its thread is killed, or it is left with throw), the connection is closed instead of
+    # checked back in, since an interrupted command may have left a reply unread on it.
+    def with_connection
+      url, connection = checkout
+      finished = false
+      begin
+        result = yield connection
+        finished = true
+        result
+      ensure
+        if finished
+          checkin(url, connection)
+        else
+          connection.close rescue nil
+        end
+      end
+    end
+
+    private
+
+    def checkout
+      url = Backburner.configuration.beanstalk_url
+      max_idle = Backburner.configuration.enqueue_connection_max_idle
+      to_close = []
+      connection = @lock.synchronize do
+        unless @url == url # the URL changed: its old connections go
+          to_close.concat(@idle)
+          @idle = []
+          @url = url
+        end
+        # Connections idle for too long may have been dropped by a firewall or load balancer
+        # without either end knowing, and a put on one would hang until TCP gives up
+        now = monotonic_now
+        expired, @idle = @idle.partition { |_conn, since| max_idle && now - since > max_idle }
+        to_close.concat(expired)
+        conn, _since = @idle.pop
+        conn
+      end
+      to_close.each { |conn, _since| conn.close rescue nil }
+      [url, connection || Backburner::Connection.new(url)]
+    end
+
+    def checkin(url, connection)
+      kept = @lock.synchronize do
+        if @url == url && @idle.size < MAX_IDLE_CONNECTIONS
+          @idle.push([connection, monotonic_now])
+        end
+      end
+      connection.close rescue nil unless kept
+    end
+
+    def monotonic_now
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end # EnqueuePool
+end # Backburner

data/lib/backburner/helpers.rb

@@ -11,7 +11,7 @@ module Backburner
 
       base = File.expand_path(Dir.pwd) + '/'
       e.backtrace.each do |t|
-        msg << "   #{File.expand_path(t).gsub(/#{base}/, '')}"
+        msg << "   #{File.expand_path(t).gsub(base, '')}"
       end if e.backtrace
 
       msg.join("\n")
@@ -44,20 +44,13 @@ module Backburner
     #   constantize("Module") # => Module
     #   constantize("Test::Unit") # => Test::Unit
     #
-    # NameError is raised when the constant is unknown.
+    # NameError is raised when the constant is unknown. Given a block, yields each module and the name
+    # of the constant it is about to look up in it, so a caller can tell which one failed.
     def constantize(camel_cased_word)
-      camel_cased_word = camel_cased_word.to_s
-
-      if camel_cased_word.include?('-')
-        camel_cased_word = classify(camel_cased_word)
-      end
-
-      names = camel_cased_word.split('::')
-      names.shift if names.empty? || names.first.empty?
-
       constant = Object
-      names.each do |name|
+      constant_names(camel_cased_word).each do |name|
         args = Module.method(:const_get).arity != 1 ? [false] : []
+        yield constant, name if block_given?
 
         if constant.const_defined?(name, *args)
           constant = constant.const_get(name)
@@ -68,6 +61,23 @@ module Backburner
       constant
     end
 
+    # The names of the constants constantize looks up in turn
+    #
+    # @example
+    #   constant_names("Test::Unit") # => ["Test", "Unit"]
+    #
+    def constant_names(camel_cased_word)
+      camel_cased_word = camel_cased_word.to_s
+
+      if camel_cased_word.include?('-')
+        camel_cased_word = classify(camel_cased_word)
+      end
+
+      names = camel_cased_word.split('::')
+      names.shift if names.empty? || names.first.empty?
+      names
+    end
+
     # Returns configuration options for backburner
     #
     # @example
@@ -85,7 +95,7 @@ module Backburner
     #   expand_tube_name(FooJob) # => <prefix>.foo-job
     #
     def expand_tube_name(tube)
-      prefix = queue_config.tube_namespace
+      prefix = queue_config.tube_namespace # nil or false mean no prefix; '' gives names that start with the separator, as in 1.7.0
       separator = queue_config.namespace_separator
       queue_name = if tube.is_a?(String)
         tube
@@ -99,7 +109,29 @@ module Backburner
       else # turn into a string
         tube.to_s
       end
-      [prefix.gsub(/\.$/, ''), dasherize(queue_name).gsub(/^#{prefix}/, '')].join(separator).gsub(/#{Regexp::escape(separator)}+/, separator).split(':').first
+      tube_name = dasherize(queue_name)
+      tube_name = [separator.empty? ? prefix : prefix.gsub(/\.$/, ''), tube_name.gsub(namespace_pattern(prefix), '')].join(separator) if prefix
+      tube_name = tube_name.gsub(/#{Regexp::escape(separator)}+/, separator) unless separator.empty? # /+/ isn't a regexp
+      tube_name.split(':').first
+    end
+
+    # What every tube name expand_tube_name gives starts with: the namespace and its separator ('' for a nil or false one)
+    #
+    # @example
+    #   expanded_tube_prefix # => "<prefix>."
+    #
+    def expanded_tube_prefix
+      expand_tube_name('').to_s
+    end
+
+    # The namespace as the pattern expand_tube_name strips from the start of a queue name.
+    # ceiling: a namespace that is a valid regexp is still matched as one ("app.prod" also strips "app-prod"), because
+    # matching it literally would move the jobs of the queue names it matches to other tubes; one that isn't ("app(")
+    # used to fail every enqueue and is matched literally.
+    def namespace_pattern(prefix)
+      Regexp.new("^#{prefix}")
+    rescue RegexpError
+      Regexp.new("^#{Regexp.escape(prefix)}")
     end
 
     # Resolves job priority based on the value given. Can be integer, a class or nothing
@@ -110,21 +142,31 @@ module Backburner
     #  resolve_priority(nil) => <default priority>
     #
     def resolve_priority(pri)
+      priority = resolve_priority_value(pri)
+      return priority if priority
+      default = Backburner.configuration.default_priority # can be a label too
+      resolve_priority_value(default) || default # left as configured if it isn't a known label
+    end
+
+    # Follows a job class's queue_priority and priority labels to a number; nil for nothing,
+    # an unknown label or a cycle of labels
+    def resolve_priority_value(pri, seen_labels = [])
       if pri.respond_to?(:queue_priority)
-        resolve_priority(pri.queue_priority)
+        resolve_priority_value(pri.queue_priority, seen_labels)
       elsif pri.is_a?(String) || pri.is_a?(Symbol) # named priority
-        resolve_priority(Backburner.configuration.priority_labels[pri.to_sym])
+        label = pri.to_sym
+        resolve_priority_value(Backburner.configuration.priority_labels[label], seen_labels + [label]) unless seen_labels.include?(label)
       elsif pri.is_a?(Integer) # numerical
         pri
-      else # default
-        Backburner.configuration.default_priority
       end
     end
 
-    # Resolves job respond timeout based on the value given. Can be integer, a class or nothing
+    # Resolves job respond timeout based on the value given, rounded up to whole seconds. Can be a number,
+    # a class or nothing
     #
     # @example
     #  resolve_respond_timeout(1000) => 1000
+    #  resolve_respond_timeout(1.5) => 2
     #  resolve_respond_timeout(FooBar) => <queue respond_timeout>
     #  resolve_respond_timeout(nil) => <default respond_timeout>
     #
@@ -133,8 +175,11 @@ module Backburner
         resolve_respond_timeout(ttr.queue_respond_timeout)
       elsif ttr.is_a?(Integer) # numerical
         ttr
+      elsif ttr.is_a?(Numeric) && ttr.finite? # fractional
+        whole_seconds(ttr)
       else # default
-        Backburner.configuration.respond_timeout
+        ttr = Backburner.configuration.respond_timeout
+        !ttr.is_a?(Integer) && ttr.is_a?(Numeric) && ttr.finite? ? whole_seconds(ttr) : ttr
       end
     end
 
@@ -155,7 +200,7 @@ module Backburner
       end
     end
 
-    # Resolves retry delay based on the value given. Can be integer, a class or nothing
+    # Resolves retry delay based on the value given. Can be a number, a class or nothing
     #
     # @example
     #  resolve_retry_delay(5) => 5
@@ -165,7 +210,7 @@ module Backburner
     def resolve_retry_delay(delay)
       if delay.respond_to?(:queue_retry_delay)
         resolve_retry_delay(delay.queue_retry_delay)
-      elsif delay.is_a?(Integer) # numerical
+      elsif delay.is_a?(Numeric) && delay.finite? # numerical (the worker rounds the delay it computes from it up)
         delay
       else # default
         Backburner.configuration.retry_delay
@@ -189,5 +234,15 @@ module Backburner
       end
     end
 
+    # Rounds a delay up to the whole seconds beanstalkd takes, so a job is never ready early
+    #
+    # @example
+    #  whole_seconds(1.5) => 2
+    #  whole_seconds(nil) => 0
+    #
+    def whole_seconds(delay)
+      [0, delay.to_f.ceil].max
+    end
+
   end # Helpers
 end # Backburner

data/lib/backburner/job.rb

@@ -45,12 +45,15 @@ module Backburner
     def process
       # Invoke before hook and stop if false
       res = @hooks.invoke_hook_events(job_name, :before_perform, *args)
-      return false unless res
+      unless res
+        task.delete # aborted: don't leave it reserved, or beanstalkd hands it out again after its TTR
+        return false
+      end
       # Execute the job
       @hooks.around_hook_events(job_name, :around_perform, *args) do
-        # We subtract one to ensure we timeout before beanstalkd does, except if:
-        #  a) ttr == 0, to support never timing out
-        #  b) ttr == 1, so that we don't accidentally set it to never time out
+        # We subtract one to ensure we timeout before beanstalkd does, except if
+        # ttr == 1 (beanstalkd's minimum: it stores a ttr of 0 as 1), so that we
+        # don't accidentally set it to never time out.
         #  NB: A ttr of 1 will likely result in race conditions between
         #  Backburner and beanstalkd and should probably be avoided
         timeout_job_after(task.ttr > 1 ? task.ttr - 1 : task.ttr) { job_class.perform(*args) }
@@ -80,10 +83,16 @@ module Backburner
     #
     def job_class
       handler = try_job_class
-      raise(JobNotFound, self.name) unless handler
+      raise(JobNotFound, @job_class_error ? "#{name} (#{@job_class_error.class}: #{@job_class_error.message.to_s.lines.first.to_s.chomp})" : name) unless handler
       handler
     end
 
+    # Whether the last lookup of the job's class found it missing (not defined, or an autoload whose
+    # file isn't there) rather than failing to load it (its file raised, or required a missing gem)
+    def job_class_missing?
+      !!@job_class_missing
+    end
+
     protected
 
     # Attempts to return a constantized job name, otherwise reverts to the name string
@@ -96,11 +105,41 @@ module Backburner
     end
 
     def try_job_class
-      constantize(self.name)
-    rescue NameError
+      @job_class_error = nil
+      @job_class_missing = false
+      scope = segment = autoload_path = nil
+      invalid_name = !constant_names(self.name).all? { |name| constant_name?(name) }
+      constantize(self.name) do |mod, name|
+        scope, segment = mod, name
+        autoload_path = mod.autoload?(name) rescue nil # a module, or the value of a constant that isn't one
+      end
+    rescue StandardError, ScriptError => e # its file may raise anything; an autoloader may raise LoadError for a file that isn't there
+      @job_class_error = e
+      @job_class_missing = !invalid_name && missing_constant_error?(e, scope, segment, autoload_path)
       nil
     end
 
+    # Whether a name can be a constant's: one that can't ("not a class", "123") will never be found
+    def constant_name?(name)
+      Module.new.const_defined?(name, false) # raises NameError for one that can't
+      true
+    rescue NameError
+      false
+    end
+
+    # Whether an error raised while looking up `segment` in `scope` says that constant is missing,
+    # rather than coming from inside the file that defines it (a NameError for another constant, or for
+    # one of the same name in another module, a NoMethodError, a require of a missing gem, any other
+    # error or a syntax error)
+    def missing_constant_error?(e, scope, segment, autoload_path)
+      case e
+      when NoMethodError then false
+      when NameError then e.name.to_s == segment.to_s && (e.receiver.equal?(scope) rescue true) # no receiver: by name only
+      when LoadError then e.path.nil? || e.path == autoload_path
+      else false
+      end
+    end
+
     # Timeout job within specified block after given time.
     #
     # @example

data/lib/backburner/logger.rb

@@ -45,6 +45,15 @@ module Backburner
       logger ? logger.error(msg) : $stderr.puts(msg)
     end
 
+    # Print a warning to stderr
+    #
+    # @example
+    #   log_warn("Task released!")
+    #
+    def log_warn(msg)
+      logger ? (logger.respond_to?(:warn) ? logger.warn(msg) : logger.error(msg)) : $stderr.puts(msg)
+    end
+
     # Return logger if specified
     def logger
       Backburner.configuration.logger

data/lib/backburner/performable.rb

@@ -79,7 +79,7 @@ module Backburner
       with_async_name    = :"#{aliased_method}_with_async#{punctuation}"
       without_async_name = :"#{aliased_method}_without_async#{punctuation}"
 
-      klass.send(:include, Performable) unless included_modules.include?(Performable)
+      klass.send(:include, Performable) unless klass.included_modules.include?(Performable)
       klass_eval_scope.class_eval do
         define_method with_async_name do |*args|
           async(opts).__send__ without_async_name, *args

data/lib/backburner/queue.rb

@@ -1,17 +1,14 @@
 module Backburner
   module Queue
     def self.included(base)
-      base.instance_variable_set(:@queue_name, nil)
-      base.instance_variable_set(:@queue_priority, nil)
-      base.instance_variable_set(:@queue_respond_timeout, nil)
-      base.instance_variable_set(:@queue_max_job_retries, nil)
-      base.instance_variable_set(:@queue_retry_delay, nil)
-      base.instance_variable_set(:@queue_retry_delay_proc, nil)
-      base.instance_variable_set(:@queue_jobs_limit, nil)
-      base.instance_variable_set(:@queue_garbage_limit, nil)
-      base.instance_variable_set(:@queue_retry_limit, nil)
+      # Initialize the settings, but keep any already set: including Queue again (as including
+      # Performable does) must not reset them
+      [:@queue_name, :@queue_priority, :@queue_respond_timeout, :@queue_max_job_retries, :@queue_retry_delay,
+       :@queue_retry_delay_proc, :@queue_jobs_limit, :@queue_garbage_limit, :@queue_retry_limit].each do |setting|
+        base.instance_variable_set(setting, nil) unless base.instance_variable_defined?(setting)
+      end
       base.extend ClassMethods
-      Backburner::Worker.known_queue_classes << base
+      Backburner::Worker.known_queue_classes << base unless Backburner::Worker.known_queue_classes.include?(base)
     end
 
     module ClassMethods
@@ -25,7 +22,8 @@ module Backburner
         if name
           @queue_name = name
         else # accessor
-          (@queue_name.is_a?(Proc) ? @queue_name.call(self) : @queue_name) || Backburner.configuration.primary_queue
+          queue_name = inherited_queue_setting(:@queue_name)
+          (queue_name.is_a?(Proc) ? queue_name.call(self) : queue_name) || Backburner.configuration.primary_queue
         end
       end
 
@@ -39,7 +37,7 @@ module Backburner
         if pri
           @queue_priority = pri
         else # accessor
-          @queue_priority
+          inherited_queue_setting(:@queue_priority)
         end
       end
 
@@ -53,7 +51,7 @@ module Backburner
         if ttr
           @queue_respond_timeout = ttr
         else # accessor
-          @queue_respond_timeout
+          inherited_queue_setting(:@queue_respond_timeout)
         end
       end
 
@@ -67,7 +65,7 @@ module Backburner
         if delay
           @queue_max_job_retries = delay
         else # accessor
-          @queue_max_job_retries
+          inherited_queue_setting(:@queue_max_job_retries)
         end
       end
 
@@ -81,7 +79,7 @@ module Backburner
         if delay
           @queue_retry_delay = delay
         else # accessor
-          @queue_retry_delay
+          inherited_queue_setting(:@queue_retry_delay)
         end
       end
 
@@ -95,7 +93,7 @@ module Backburner
         if proc
           @queue_retry_delay_proc = proc
         else # accessor
-          @queue_retry_delay_proc
+          inherited_queue_setting(:@queue_retry_delay_proc)
         end
       end
 
@@ -109,7 +107,7 @@ module Backburner
         if limit
           @queue_jobs_limit = limit
         else #accessor
-          @queue_jobs_limit
+          inherited_queue_setting(:@queue_jobs_limit)
         end
       end
 
@@ -123,7 +121,7 @@ module Backburner
         if limit
           @queue_garbage_limit = limit
         else #accessor
-          @queue_garbage_limit
+          inherited_queue_setting(:@queue_garbage_limit)
         end
       end
 
@@ -137,8 +135,28 @@ module Backburner
         if limit
           @queue_retry_limit = limit
         else #accessor
-          @queue_retry_limit
+          inherited_queue_setting(:@queue_retry_limit)
+        end
+      end
+
+      private
+
+      # Registers subclasses as known queues too (they inherit the queue settings)
+      def inherited(subclass)
+        super
+        Backburner::Worker.known_queue_classes << subclass unless Backburner::Worker.known_queue_classes.include?(subclass)
+      end
+
+      # Returns the setting from this class or else from the closest superclass that has it,
+      # so a subclass shares its parent's queue settings unless it sets its own
+      def inherited_queue_setting(ivar)
+        klass = self
+        while klass.is_a?(ClassMethods)
+          value = klass.instance_variable_get(ivar) if klass.instance_variable_defined?(ivar)
+          return value unless value.nil?
+          klass = klass.respond_to?(:superclass) ? klass.superclass : nil
         end
+        nil
       end
     end # ClassMethods
   end # Queue

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.6.1"
+  VERSION = "1.8.0"
 end