backburner 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 89f6987a5f81f904f8e2dd8d48bad8deea664a7e
-  data.tar.gz: d0a1cdf8c5cd0087f60d4c29da334f40ebc00a3f
+  metadata.gz: 9f2543596fd5e337e0b4514ca94f8b32959c3901
+  data.tar.gz: eb4133d3c373307e9399636ce8c5c4df7630e74a
 SHA512:
-  metadata.gz: 19fc5c5d7086baf5ca15cc8cf93d0a429cd858ac30fbdfe86719322bd8704edacfb008d3568eae6f5c6c1a029e7831a0ef6052345cb934d7fea45cca90e60103
-  data.tar.gz: 3da11501d42cecb97fae3428cdf9d3f9e7403f0eab8801e581ff90685a79646a12f4ee7d95811efd5b4339995f97ff96bca7d13fe8ca8372b99b52047d28a2b2
+  metadata.gz: 58857e61ce26ff416fa6b3732839adedc2b9230886cff2fafa812c444598b264525f141a48e9eb2294ff3d8b55bd105fb00c28a89fdf1eb7250ca74022a41567
+  data.tar.gz: dcbde0b50cc908dbb6052769a546c72589585a86b9d50ec5a4ceea1c6bdc9977430cfa45321593de52bb0e2f9048dd78b151f41de524be1b730b94bc64765606

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # CHANGELOG
 
+## Version 1.1.0 (September 14 2015)
+
+ * NEW Ability to configure namespace separator (@bfolkens)
+ * NEW Avoid timeouts altogether by setting queue_respond_timeout to 0 (@zacviandier)
+ * NEW Event hooks for on_retry and on_bury (@contentfree)
+ * NEW Support lambdas for queue names (@contentfree)
+ * NEW Allow for control of delay calculation (@contentfree)
+ * NEW Ability to specify environment when running the CLI (@contentfree)
+ * NEW Control default async behavior of methods (@contentfree)
+
 ## Version 1.0.0 (April 26 2015)
 
  * NEW Updating to Beaneater 1.0 (@alup)
@@ -87,4 +97,4 @@ NOTE: This is the start of working with @bradgessler to improve backburner and m
 ## Version 0.1.0 (Nov 4 2012)
 
  * Switch to beaneater as new ruby beanstalkd client
- * Add support for array of connections in `beanstalk_url`
+ * Add support for array of connections in `beanstalk_url`

data/HOOKS.md

@@ -26,6 +26,10 @@ There are a variety of hooks available that are triggered during the lifecycle o
 	to perform the job (but is not required to do so). It may handle exceptions
 	thrown by perform, but uncaught exceptions will be treated like regular job exceptions.
 
+* `on_retry`: Called with the retry count, the delay and the job args whenever a job is retried. 
+
+* `on_bury`: Called with the job args when the job is buried.
+
 * `on_failure`: Called with the exception and job args if any exception occurs
   while performing the job (or hooks).
 

data/README.md

@@ -88,35 +88,39 @@ Backburner is extremely simple to setup. Just configure basic settings for backb
 
 ```ruby
 Backburner.configure do |config|
-  config.beanstalk_url    = ["beanstalk://127.0.0.1", "..."]
-  config.tube_namespace   = "some.app.production"
-  config.on_error         = lambda { |e| puts e }
-  config.max_job_retries  = 3 # default 0 retries
-  config.retry_delay      = 2 # default 5 seconds
-  config.default_priority = 65536
-  config.respond_timeout  = 120
-  config.default_worker   = Backburner::Workers::Simple
-  config.logger           = Logger.new(STDOUT)
-  config.primary_queue    = "backburner-jobs"
-  config.priority_labels  = { :custom => 50, :useless => 1000 }
-  config.reserve_timeout  = nil
+  config.beanstalk_url       = ["beanstalk://127.0.0.1", "..."]
+  config.tube_namespace      = "some.app.production"
+  config.namespace_separator = "."
+  config.on_error            = lambda { |e| puts e }
+  config.max_job_retries     = 3 # default 0 retries
+  config.retry_delay         = 2 # default 5 seconds
+  config.retry_delay_proc    = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) }
+  config.default_priority    = 65536
+  config.respond_timeout     = 120
+  config.default_worker      = Backburner::Workers::Simple
+  config.logger              = Logger.new(STDOUT)
+  config.primary_queue       = "backburner-jobs"
+  config.priority_labels     = { :custom => 50, :useless => 1000 }
+  config.reserve_timeout     = nil
 end
 ```
 
 The key options available are:
 
-| Option            | Description                                                          |
-| ----------------- | -------------------------------                                      |
-| `beanstalk_url`   | Address such as 'beanstalk://127.0.0.1' or an array of addresses.    |
-| `tube_namespace`  | Prefix used for all tubes related to this backburner queue.          |
-| `on_error`        | Lambda invoked with the error whenever any job in the system fails.  |
-| `default_worker`  | Worker class that will be used if no other worker is specified.      |
-| `max_job_retries` | Integer defines how many times to retry a job before burying.        |
-| `retry_delay`     | Integer defines the base time to wait (in secs) between job retries. |
-| `logger`          | Logger recorded to when backburner wants to report info or errors.   |
-| `primary_queue`   | Primary queue used for a job when an alternate queue is not given.   |
-| `priority_labels` | Hash of named priority definitions for your app.                     |
-| `reserve_timeout` | Duration to wait for work from a single server, or nil for forever.  |
+| Option                | Description                                                          |
+| -----------------     | -------------------------------                                      |
+| `beanstalk_url`       | Address such as 'beanstalk://127.0.0.1' or an array of addresses.    |
+| `tube_namespace`      | Prefix used for all tubes related to this backburner queue.          |
+| `namespace_separator` | Separator used for namespace and queue name                          |
+| `on_error`            | Lambda invoked with the error whenever any job in the system fails.  |
+| `default_worker`      | Worker class that will be used if no other worker is specified.      |
+| `max_job_retries`     | Integer defines how many times to retry a job before burying.        |
+| `retry_delay`         | Integer defines the base time to wait (in secs) between job retries. |
+| `retry_delay_proc`    | Lambda calculates the delay used, allowing for exponential back-off. |
+| `logger`              | Logger recorded to when backburner wants to report info or errors.   |
+| `primary_queue`       | Primary queue used for a job when an alternate queue is not given.   |
+| `priority_labels`     | Hash of named priority definitions for your app.                     |
+| `reserve_timeout`     | Duration to wait for work from a single server, or nil for forever.  |
 
 ## Breaking Changes
 
@@ -154,7 +158,7 @@ class NewsletterJob
 
   # optional, defaults to respond_timeout
   def self.queue_respond_timeout
-    300 # number of seconds before job times out
+    300 # number of seconds before job times out, 0 to avoid timeout
   end
 end
 ```
@@ -166,7 +170,7 @@ class NewsletterJob
   include Backburner::Queue
   queue "newsletter-sender"  # defaults to 'backburner-jobs' tube
   queue_priority 1000 # most urgent priority is 0
-  queue_respond_timeout 300 # number of seconds before job times out
+  queue_respond_timeout 300 # number of seconds before job times out, 0 to avoid timeout
 
   def self.perform(email, body)
     NewsletterMailer.deliver_text_to_email(email, body)
@@ -184,6 +188,10 @@ Backburner.enqueue NewsletterJob, 'foo@admin.com', 'lorem ipsum...'
 to that object's `perform` method. The queue name used by default is `{namespace}.backburner-jobs`
 unless otherwise specified.
 
+You may also pass a lambda as the queue name and it will be evaluated when enqueuing a
+job (and passed the Job's class as an argument). This is especially useful when combined
+with "Simple Async Jobs" (see below).
+
 ### Simple Async Jobs ###
 
 In addition to defining custom jobs, a job can also be enqueued by invoking the `async` method on any object which
@@ -194,7 +202,7 @@ class User
   include Backburner::Performable
   queue "user-jobs"  # defaults to 'user'
   queue_priority 500 # most urgent priority is 0
-  queue_respond_timeout 300 # number of seconds before job times out
+  queue_respond_timeout 300 # number of seconds before job times out, 0 to avoid timeout
 
   def activate(device_id)
     @device = Device.find(device_id)
@@ -216,7 +224,55 @@ User.async(:pri => 100, :delay => 10.seconds).reset_password(@user.id)
 This automatically enqueues a job for that user record that will run `activate` with the specified argument.
 Note that you can set the queue name and queue priority at the class level and
 you are also able to pass `pri`, `ttr`, `delay` and `queue` directly as options into `async`.
-The queue name used by default is `{namespace}.backburner-jobs` if not otherwise specified.
+
+The queue name used by default is `{namespace}.backburner-jobs` if not otherwise
+specified.
+
+If a lambda is given for `queue`, then it will be called and given the
+_performable_ object's class as an argument:
+
+```ruby
+# Given the User class above
+User.async(:queue => lambda { |user_klass| ["queue1","queue2"].sample(1).first }).do_hard_work # would add the job to either queue1 or queue2 randomly
+```
+
+### Using Async Asynchronously ###
+
+It's often useful to be able to configure your app in production such that every invocation of a method is asynchronous by default as seen in [delayed_job](https://github.com/collectiveidea/delayed_job#queuing-jobs). To accomplish this, the `Backburner::Performable` module exposes two `handle_asynchronously` convenience methods 
+which accept the same options as the `async` method:
+
+```ruby
+class User
+  include Backburner::Performable
+
+  def send_welcome_email
+    # ...
+  end
+  
+  # ---> For instance methods
+  handle_asynchronously :send_welcome_email, queue: 'send-mail', pri: 5000, ttr: 60
+
+  def self.update_recent_visitors
+    # ...
+  end
+  
+  # ---> For class methods
+  handle_static_asynchronously :update_recent_visitors, queue: 'long-tasks', ttr: 300
+end
+```
+
+Now, all calls to `User.update_recent_visitors` or `User#send_welcome_email` will automatically be handled asynchronously when invoked. Similarly, you can call these methods directly on the `Backburner::Performable` module to apply async behavior outside the class:
+
+```ruby
+# Given the User class above
+Backburner::Performable.handle_asynchronously(User, :activate, ttr: 100, queue: 'activate')
+```
+
+Now all calls to the `activate` method on a `User` instance will be async with the provided options.
+
+#### A Note About Auto-Async
+
+Because an async proxy is injected and used in place of the original method, you must not rely on the return value of the method. Using the example `User` class above, if my `send_welcome_email` returned the status of an email submission and I relied on that to take some further action, I will be surprised after rewiring things with `handle_asynchronously` because the async proxy actually returns the (boolean) result of `Backburner::Worker.enqueue`.
 
 ### Working Jobs
 
@@ -366,7 +422,9 @@ $ QUEUE=newsletter-sender,push-message THREADS=2 GARBAGE=1000 rake backburner:th
 ```
 
 For more information on the threads_on_fork worker, check out the
-[ThreadsOnFork Worker](https://github.com/nesquena/backburner/wiki/ThreadsOnFork-worker) documentation.
+[ThreadsOnFork Worker](https://github.com/nesquena/backburner/wiki/ThreadsOnFork-worker) documentation. Please note that the `ThreadsOnFork` worker does not work on Windows due to its lack of `fork`.
+
+
 Additional workers such as individual `threaded` and `forking` strategies will hopefully be contributed in the future.
 If you are interested in helping out, please let us know.
 
@@ -401,7 +459,7 @@ The `default_queues` stores the specific list of queues that should be processed
 ### Failures
 
 When a job fails in backburner (usually because an exception was raised), the job will be released
-and retried again (with progressive delays in between) until the `max_job_retries` configuration is reached.
+and retried again until the `max_job_retries` configuration is reached.
 
 ```ruby
 Backburner.configure do |config|
@@ -411,6 +469,19 @@ end
 ```
 
 Note the default `max_job_retries` is 0, meaning that by default **jobs are not retried**.
+
+As jobs are retried, a progressively-increasing delay is added to give time for transient
+problems to resolve themselves. This may be configured using `retry_delay_proc`. It expects 
+an object that responds to `#call` and receives the value of `retry_delay` and the number
+of times the job has been retried already. The default is a cubic back-off, eg:
+
+```ruby
+Backburner.configure do |config|
+  config.retry_delay      = 2 # The minimum number of seconds a retry will be delayed
+  config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) }
+end
+```
+
 If continued retry attempts fail, the job will be buried and can be 'kicked' later for inspection.
 
 You can also setup a custom error handler for jobs using configure:
@@ -468,6 +539,39 @@ The best way to deploy these rake tasks is using a monitoring library. We sugges
 which watches processes and ensures their stability. A simple God recipe for Backburner can be found in
 [examples/god](https://github.com/nesquena/backburner/blob/master/examples/god.rb).
 
+#### Command-Line Interface
+
+Instead of using the Rake tasks, you can use Backburner's command-line interface (CLI) – powered by the [Dante gem](https://github.com/nesquena/dante) – to launch daemonized workers. Several flags are available to control the process. Many of these are provided by Dante itself, such as flags for logging (`-l`), the process' PID (`-P`), whether to daemonize (`-d`) or kill a running process (`-k`). Backburner provides a few more:
+
+
+##### Queues (`-q`)
+
+Control which queues the worker will watch with the `-q` flag. Comma-separate multiple queue names and, if you're using the `ThreadsOnFork` worker, colon-separate the settings for thread limit, garbage limit and retries limit (eg. `send_mail:4:10:3`). See its [wiki page](https://github.com/nesquena/backburner/wiki/ThreadsOnFork-worker) for some more details.
+
+```ruby
+backburner -q send_mail,create_thumbnail # You may need to use `bundle exec`
+```
+
+##### Boot an app (`-r`)
+
+Load an app with the `-r` flag. Backburner supports automatic loading for both Rails and Padrino apps when started from the their root folder. However, you may point to a specific app's root using this flag, which is very useful when running workers from a service script.
+
+```ruby
+path="/var/www/my-app/current"
+backburner -r "$path"
+```
+
+##### Load an environment (`-e`)
+
+Use the `-e` flag to control which environment your app should use:
+
+```ruby
+environment="production"  
+backburner -e $environment
+```
+
+#### Reconnecting
+
 In Backburner, if the beanstalkd connection is temporarily severed, several retries to establish the connection will be attempted.
 After several retries, if the connection is still not able to be made, a `Beaneater::NotConnected` exception will be raised.
 You can manually catch this exception, and attempt another manual retry using `Backburner::Worker.retry_connection!`.
@@ -481,6 +585,7 @@ jobs processed by your beanstalk workers. An excellent addition to your Backburn
 ## Acknowledgements
 
  * [Nathan Esquenazi](https://github.com/nesquena) - Project maintainer
+ * [Dave Myron](https://github.com/contentfree) - Multiple features and doc improvements
  * Kristen Tucker - Coming up with the gem name
  * [Tim Lee](https://github.com/timothy1ee), [Josh Hull](https://github.com/joshbuddy), [Nico Taing](https://github.com/Nico-Taing) - Helping me work through the idea
  * [Miso](http://gomiso.com) - Open-source friendly place to work

data/lib/backburner/cli.rb

@@ -13,19 +13,23 @@ module Backburner
         opts.on("-q", "--queues PATH", String, "The specific queues to work.") do |queues|
           options[:queues] = queues
         end
+        opts.on("-e", "--environment ENVIRONMENT", String, "The environment to run Backburner within") do |environment|
+          options[:environment] = environment
+        end
       end
       runner.execute do |opts|
         queues = (opts[:queues] ? opts[:queues].split(',') : nil) rescue nil
-        load_enviroment(opts[:require])
+        load_environment(opts[:require], opts[:environment])
         Backburner.work(queues)
       end
     end
 
     protected
 
-      def self.load_enviroment(file = nil)
+      def self.load_environment(file = nil, environment = nil)
         file ||= "."
         if File.directory?(file) && File.exists?(File.expand_path("#{file}/config/environment.rb"))
+          ENV["RAILS_ENV"] = environment if environment && ENV["RAILS_ENV"].nil?
           require "rails"
           require File.expand_path("#{file}/config/environment.rb")
           if defined?(::Rails) && ::Rails.respond_to?(:application)
@@ -37,11 +41,9 @@ module Backburner
             ::Rails::Initializer.run :load_application_classes
           end
         elsif File.directory?(file) && File.exists?(File.expand_path("#{file}/config/boot.rb"))
-          if defined?(::Padrino)
-            # Padrino
-            require "padrino"
-            require File.expand_path("#{file}/config/boot.rb")
-          end
+          ENV["RACK_ENV"] = environment if environment && ENV["RACK_ENV"].nil?
+          ENV["PADRINO_ROOT"] = file
+          require File.expand_path("#{file}/config/boot.rb")
         elsif File.file?(file)
           require File.expand_path(file)
         end

data/lib/backburner/configuration.rb

@@ -2,34 +2,43 @@ module Backburner
   class Configuration
     PRIORITY_LABELS = { :high => 0, :medium => 100, :low => 200 }
 
-    attr_accessor :beanstalk_url      # beanstalk url connection
-    attr_accessor :tube_namespace     # namespace prefix for every queue
-    attr_accessor :default_priority   # default job priority
-    attr_accessor :respond_timeout    # default job timeout
-    attr_accessor :on_error           # error handler
-    attr_accessor :max_job_retries    # max job retries
-    attr_accessor :retry_delay        # retry delay in seconds
-    attr_accessor :default_queues     # default queues
-    attr_accessor :logger             # logger
-    attr_accessor :default_worker     # default worker class
-    attr_accessor :primary_queue      # the general queue
-    attr_accessor :priority_labels    # priority labels
-    attr_accessor :reserve_timeout    # duration to wait to reserve on a single server
+    attr_accessor :beanstalk_url       # beanstalk url connection
+    attr_accessor :tube_namespace      # namespace prefix for every queue
+    attr_reader   :namespace_separator # namespace separator
+    attr_accessor :default_priority    # default job priority
+    attr_accessor :respond_timeout     # default job timeout
+    attr_accessor :on_error            # error handler
+    attr_accessor :max_job_retries     # max job retries
+    attr_accessor :retry_delay         # (minimum) retry delay in seconds
+    attr_accessor :retry_delay_proc    # proc to calculate delay (and allow for back-off)
+    attr_accessor :default_queues      # default queues
+    attr_accessor :logger              # logger
+    attr_accessor :default_worker      # default worker class
+    attr_accessor :primary_queue       # the general queue
+    attr_accessor :priority_labels     # priority labels
+    attr_accessor :reserve_timeout     # duration to wait to reserve on a single server
 
     def initialize
-      @beanstalk_url     = "beanstalk://localhost"
-      @tube_namespace    = "backburner.worker.queue"
-      @default_priority  = 65536
-      @respond_timeout   = 120
-      @on_error          = nil
-      @max_job_retries   = 0
-      @retry_delay       = 5
-      @default_queues    = []
-      @logger            = nil
-      @default_worker    = Backburner::Workers::Simple
-      @primary_queue     = "backburner-jobs"
-      @priority_labels   = PRIORITY_LABELS
-      @reserve_timeout   = nil
+      @beanstalk_url       = "beanstalk://localhost"
+      @tube_namespace      = "backburner.worker.queue"
+      @namespace_separator = "."
+      @default_priority    = 65536
+      @respond_timeout     = 120
+      @on_error            = nil
+      @max_job_retries     = 0
+      @retry_delay         = 5
+      @retry_delay_proc    = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) }
+      @default_queues      = []
+      @logger              = nil
+      @default_worker      = Backburner::Workers::Simple
+      @primary_queue       = "backburner-jobs"
+      @priority_labels     = PRIORITY_LABELS
+      @reserve_timeout     = nil
+    end
+
+    def namespace_separator=(val)
+      raise 'Namespace separator cannot used reserved queue configuration separator ":"' if val == ':'
+      @namespace_separator = val
     end
   end # Configuration
 end # Backburner

data/lib/backburner/helpers.rb

@@ -86,16 +86,20 @@ module Backburner
     #
     def expand_tube_name(tube)
       prefix = queue_config.tube_namespace
+      separator = queue_config.namespace_separator
       queue_name = if tube.is_a?(String)
         tube
       elsif tube.respond_to?(:queue) # use queue name
-        tube.queue
+        queue = tube.queue
+        queue.is_a?(Proc) ? queue.call(tube) : queue
+      elsif tube.is_a?(Proc)
+        tube.call
       elsif tube.is_a?(Class) # no queue name, use default
         queue_config.primary_queue # tube.name
       else # turn into a string
         tube.to_s
       end
-      [prefix.gsub(/\.$/, ''), dasherize(queue_name).gsub(/^#{prefix}/, '')].join(".").gsub(/\.+/, '.').split(':').first
+      [prefix.gsub(/\.$/, ''), dasherize(queue_name).gsub(/^#{prefix}/, '')].join(separator).gsub(/#{Regexp::escape(separator)}+/, separator).split(':').first
     end
 
     # Resolves job priority based on the value given. Can be integer, a class or nothing
@@ -135,4 +139,4 @@ module Backburner
     end
 
   end # Helpers
-end # Backburner
+end # Backburner

data/lib/backburner/job.rb

@@ -19,10 +19,10 @@ module Backburner
     #  Backburner::Job.new(payload)
     #
     def initialize(task)
+      @hooks = Backburner::Hooks
       @task = task
       @body = task.body.is_a?(Hash) ? task.body : JSON.parse(task.body)
       @name, @args = body["class"], body["args"]
-      @hooks = Backburner::Hooks
     rescue => ex # Job was not valid format
       self.bury
       raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
@@ -46,7 +46,7 @@ module Backburner
       return false unless res
       # Execute the job
       @hooks.around_hook_events(job_class, :around_perform, *args) do
-        timeout_job_after(task.ttr - 1) { job_class.perform(*args) }
+        timeout_job_after(task.ttr) { job_class.perform(*args) }
       end
       task.delete
       # Invoke after perform hook
@@ -56,6 +56,16 @@ module Backburner
       raise e
     end
 
+    def bury
+      @hooks.invoke_hook_events(job_class, :on_bury, *args)
+      task.bury
+    end
+
+    def retry(count, delay)
+      @hooks.invoke_hook_events(job_class, :on_retry, count, delay, *args)
+      task.release(delay: delay)
+    end
+
     protected
 
     # Returns the class for the job handler

data/lib/backburner/performable.rb

@@ -38,7 +38,58 @@ module Backburner
           send(method, *args)
         end
       end # perform
+
+      # Always handle an instance method asynchronously
+      # @example
+      #   User.handle_asynchronously :send_welcome_email, queue: 'send-mail', delay: 10
+      def handle_asynchronously(method, opts={})
+        Backburner::Performable.handle_asynchronously(self, method, opts)
+      end
+
+      # Always handle a class method asynchronously
+      # @example
+      #   User.handle_static_asynchronously :update_recent_visitors, ttr: 300
+      def handle_static_asynchronously(method, opts={})
+        Backburner::Performable.handle_static_asynchronously(self, method, opts)
+      end
     end # ClassMethods
 
+
+    # Make all calls to an instance method asynchronous. The given opts will be passed
+    # to the async method.
+    # @example
+    #   Backburner::Performable.handle_asynchronously(MyObject, :long_task, queue: 'long-tasks')
+    # NB: The method called on the async proxy will be ""#{method}_without_async". This
+    # will also be what's given to the Worker.enqueue method so your workers need
+    # to know about that. It shouldn't be a problem unless the producer and consumer are
+    # from different codebases (or anywhere they don't both call the handle_asynchronously
+    # method when booting up)
+    def self.handle_asynchronously(klass, method, opts={})
+      _handle_asynchronously(klass, klass, method, opts)
+    end
+
+    # Make all calls to a class method asynchronous. The given opts will be passed
+    # to the async method. Please see the NB on #handle_asynchronously
+    def self.handle_static_asynchronously(klass, method, opts={})
+      _handle_asynchronously(klass, klass.singleton_class, method, opts)
+    end
+
+    def self._handle_asynchronously(klass, klass_eval_scope, method, opts={})
+      aliased_method, punctuation = method.to_s.sub(/([?!=])$/, ''), $1
+      with_async_name    = :"#{aliased_method}_with_async#{punctuation}"
+      without_async_name = :"#{aliased_method}_without_async#{punctuation}"
+
+      klass.send(:include, Performable) unless included_modules.include?(Performable)
+      klass_eval_scope.class_eval do
+        define_method with_async_name do |*args|
+          async(opts).__send__ without_async_name, *args
+        end
+        alias_method without_async_name, method.to_sym
+        alias_method method.to_sym, with_async_name
+      end
+    end
+    private_class_method :_handle_asynchronously
+
+
   end # Performable
 end # Backburner

data/lib/backburner/queue.rb

@@ -22,7 +22,7 @@ module Backburner
         if name
           @queue_name = name
         else # accessor
-          @queue_name || Backburner.configuration.primary_queue
+          (@queue_name.is_a?(Proc) ? @queue_name.call(self) : @queue_name) || Backburner.configuration.primary_queue
         end
       end
 

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/lib/backburner/worker.rb

@@ -31,7 +31,8 @@ module Backburner
       return false unless res # stop if hook is false
       data = { :class => job_class.name, :args => args }
       retryable_command do
-        tube  = connection.tubes[expand_tube_name(opts[:queue]  || job_class)]
+        queue = opts[:queue] && (Proc === opts[:queue] ? opts[:queue].call(job_class) : opts[:queue])
+        tube  = connection.tubes[expand_tube_name(queue || job_class)]
         tube.put(data.to_json, :pri => pri, :delay => delay, :ttr => ttr)
       end
       Backburner::Hooks.invoke_hook_events(job_class, :after_enqueue, *args)
@@ -139,8 +140,8 @@ module Backburner
       num_retries = job.stats.releases
       retry_status = "failed: attempt #{num_retries+1} of #{queue_config.max_job_retries+1}"
       if num_retries < queue_config.max_job_retries # retry again
-        delay = queue_config.retry_delay + num_retries ** 3
-        job.release(:delay => delay)
+        delay = queue_config.retry_delay_proc.call(queue_config.retry_delay, num_retries) rescue queue_config.retry_delay
+        job.retry(num_retries + 1, delay)
         self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
       else # retries failed, bury
         job.bury

data/test/back_burner_test.rb

@@ -62,6 +62,16 @@ describe "Backburner module" do
     it "remembers the tube_namespace" do
       assert_equal "demo.test", Backburner.configuration.tube_namespace
     end
+
+    it "remembers the namespace_separator" do
+      assert_equal ".", Backburner.configuration.namespace_separator
+    end
+
+    it "disallows a reserved separator" do
+      assert_raises RuntimeError do
+        Backburner.configuration.namespace_separator = ':'
+      end
+    end
   end # configuration
 
   describe "for default_queues" do

data/test/fixtures/hooked.rb

@@ -100,9 +100,17 @@ class HookedObjectSuccess
     puts "!!on_failure_foo!! #{ex.inspect} #{args.inspect}"
   end
 
+  def self.on_bury_foo(*args)
+    puts "!!on_bury_foo!! #{args.inspect}"
+  end
+
+  def self.on_retry_foo(retry_count, delay, *args)
+    puts "!!on_retry_foo!! #{retry_count} #{delay} #{args.inspect}"
+  end
+
   def self.foo(x)
     $hooked_fail_count += 1
     raise HookFailError, "Fail!" if $hooked_fail_count == 1
     puts "This is the job running successfully!! #{x.inspect}"
   end
-end # HookedObjectSuccess
+end # HookedObjectSuccess

data/test/fixtures/test_jobs.rb

@@ -27,7 +27,22 @@ class TestRetryJob
   end
 end
 
+class TestConfigurableRetryJob
+  include Backburner::Queue
+  def self.perform(retry_count)
+    $worker_test_count += 1
+    raise RuntimeError unless $worker_test_count > retry_count
+    $worker_success = true
+  end
+end
+
 class TestAsyncJob
   include Backburner::Performable
   def self.foo(x, y); $worker_test_count = x * y; end
-end
+end
+
+class TestLambdaQueueJob
+  include Backburner::Queue
+  queue lambda { |klass| klass.calculated_queue_name }
+  def self.calculated_queue_name; 'lambda-queue' end
+end

data/test/helpers_test.rb

@@ -45,7 +45,7 @@ describe "Backburner::Helpers module" do
   end # exception_message
 
   describe "for queue_config" do
-    before { Backburner.expects(:configuration).returns(stub(:tube_namespace => "test.foo.job")) }
+    before { Backburner.expects(:configuration).returns(stub(:tube_namespace => "test.foo.job", :namespace_separator => '.')) }
 
     it "accesses correct value for namespace" do
       assert_equal "test.foo.job", queue_config.tube_namespace
@@ -53,7 +53,7 @@ describe "Backburner::Helpers module" do
   end # config
 
   describe "for expand_tube_name method" do
-    before { Backburner.stubs(:configuration).returns(stub(:tube_namespace => "test.foo.job.", :primary_queue => "backburner-jobs"))  }
+    before { Backburner.stubs(:configuration).returns(stub(:tube_namespace => "test.foo.job.", :namespace_separator => '.', :primary_queue => "backburner-jobs"))  }
 
     it "supports base strings" do
       assert_equal "test.foo.job.email/send-news", expand_tube_name("email/send_news")
@@ -75,8 +75,26 @@ describe "Backburner::Helpers module" do
     it "supports class names" do
       assert_equal "test.foo.job.backburner-jobs", expand_tube_name(RuntimeError)
     end # class names
+
+    it "supports lambda in queue object" do
+      test = stub(:queue => lambda { |job_class| "email/send_news" })
+      assert_equal "test.foo.job.email/send-news", expand_tube_name(test)
+    end # lambdas in queue object
+
+    it "supports lambdas" do
+      test = lambda { "email/send_news" }
+      assert_equal "test.foo.job.email/send-news", expand_tube_name(test)
+    end #lambdas
   end # expand_tube_name
 
+  describe "for alternative namespace separator" do
+    before { Backburner.stubs(:configuration).returns(stub(:tube_namespace => "test", :namespace_separator => '-', :primary_queue => "backburner-jobs"))  }
+
+    it "uses alternative namespace separator" do
+      assert_equal "test-queue-name", expand_tube_name("queue_name")
+    end # simple string
+  end
+
   describe "for resolve_priority method" do
     before do
       @original_queue_priority = Backburner.configuration.default_priority

data/test/hooks_test.rb

@@ -69,6 +69,20 @@ describe "Backburner::Hooks module" do
         assert_match(/!!on_failure_foo!! RuntimeError \[10\]/, out)
       end
     end # on_failure
+
+    describe 'with on_retry' do
+      it "should support successful invocation" do
+        out = silenced { @hooks.invoke_hook_events(HookedObjectSuccess, :on_retry, 1, 0, 10) }
+        assert_match(/!!on_retry_foo!! 1 0 \[10\]/, out)
+      end
+    end
+
+    describe 'with on_bury' do
+      it "should support successful invocation" do
+        out = silenced { @hooks.invoke_hook_events(HookedObjectSuccess, :on_bury, 10) }
+        assert_match(/!!on_bury_foo!! \[10\]/, out)
+      end
+    end
   end # invoke_hook_events
 
   describe "for around_hook_events method" do

data/test/performable_test.rb

@@ -1,7 +1,6 @@
 require File.expand_path('../test_helper', __FILE__)
 
 class TestObj
-  include Backburner::Performable
   ID = 56
   def id; ID; end
   def self.find(id); TestObj.new if id == ID; end
@@ -9,30 +8,81 @@ class TestObj
   def self.bar(state, state2); "baz #{state} #{state2}"; end
 end
 
+class PerformableTestObj < TestObj
+  include Backburner::Performable
+end
+
+class AutomagicTestObj < TestObj
+  # Don't include Backburner::Performable because it should be automagically included
+  def qux(state, state2); "garply #{state} #{state2}" end
+  def self.garply(state, state2); "thud #{state} #{state2}" end
+  def qux?; "garply!" end
+end
+
+class AsyncInstanceMethodsTestObj < PerformableTestObj; end
+class AsyncStaticMethodsTestObj < PerformableTestObj; end
+
+
+
 describe "Backburner::Performable module" do
   after { ENV["TEST"] = nil }
 
   describe "for async instance method" do
     it "should invoke worker enqueue" do
-      Backburner::Worker.expects(:enqueue).with(TestObj, [56, :foo, true, false], has_entries(:pri => 5000, :queue => "foo"))
-      TestObj.new.async(:pri => 5000, :queue => "foo").foo(true, false)
+      Backburner::Worker.expects(:enqueue).with(PerformableTestObj, [56, :foo, true, false], has_entries(:pri => 5000, :queue => "foo"))
+      PerformableTestObj.new.async(:pri => 5000, :queue => "foo").foo(true, false)
     end
   end # async instance
 
   describe "for async class method" do
     it "should invoke worker enqueue" do
-      Backburner::Worker.expects(:enqueue).with(TestObj, [nil, :bar, true, false], has_entries(:pri => 5000, :queue => "foo"))
-      TestObj.async(:pri => 5000, :queue => "foo").bar(true, false)
+      Backburner::Worker.expects(:enqueue).with(PerformableTestObj, [nil, :bar, true, false], has_entries(:pri => 5000, :queue => "foo"))
+      PerformableTestObj.async(:pri => 5000, :queue => "foo").bar(true, false)
     end
   end # async class
 
   describe "for perform class method" do
     it "should work for instance" do
-      assert_equal "bar true false", TestObj.perform(TestObj::ID, :foo, true, false)
+      assert_equal "bar true false", PerformableTestObj.perform(PerformableTestObj::ID, :foo, true, false)
     end # instance
 
     it "should work for class level" do
-      assert_equal "baz false true", TestObj.perform(nil, :bar, false, true)
+      assert_equal "baz false true", PerformableTestObj.perform(nil, :bar, false, true)
     end # class
   end # perform
-end
+
+  describe "for handle_asynchronously class method" do
+    it "should automagically asynchronously proxy calls to the method" do
+      Backburner::Performable.handle_asynchronously(AutomagicTestObj, :qux, :pri => 5000, :queue => "qux")
+
+      Backburner::Worker.expects(:enqueue).with(AutomagicTestObj, [56, :qux_without_async, true, false], has_entries(:pri => 5000, :queue => "qux"))
+      AutomagicTestObj.new.qux(true, false)
+    end
+
+    it "should work for class methods, too" do
+      Backburner::Performable.handle_static_asynchronously(AutomagicTestObj, :garply, :pri => 5000, :queue => "garply")
+
+      Backburner::Worker.expects(:enqueue).with(AutomagicTestObj, [nil, :garply_without_async, true, false], has_entries(:pri => 5000, :queue => "garply"))
+      AutomagicTestObj.garply(true, false)
+    end
+
+    it "should correctly handle punctuation" do
+      Backburner::Performable.handle_asynchronously(AutomagicTestObj, :qux?)
+
+      Backburner::Worker.expects(:enqueue).with(AutomagicTestObj, [56, :qux_without_async?], {})
+      AutomagicTestObj.new.qux?
+    end
+
+    it "should be available for instance methods on any class that includes the Performable module" do
+      AsyncInstanceMethodsTestObj.handle_asynchronously :foo, pri: 5000, queue: 'qux'
+      Backburner::Worker.expects(:enqueue).with(AsyncInstanceMethodsTestObj, [56, :foo_without_async, true, false], has_entries(:pri => 5000, :queue => "qux"))
+      AsyncInstanceMethodsTestObj.new.foo(true, false)
+    end
+
+    it "should be available for class methods on any class that includes the Performable module" do
+      AsyncStaticMethodsTestObj.handle_static_asynchronously :bar, pri: 5000, queue: 'garply'
+      Backburner::Worker.expects(:enqueue).with(AsyncStaticMethodsTestObj, [nil, :bar_without_async, true, false], has_entries(:pri => 5000, :queue => "garply"))
+      AsyncStaticMethodsTestObj.bar(true, false)
+    end
+  end
+end

data/test/queue_test.rb

@@ -24,6 +24,11 @@ describe "Backburner::Queue module" do
       NestedDemo::TestJobB.queue("nested/job")
       assert_equal "nested/job", NestedDemo::TestJobB.queue
     end
+
+    it "should allow lambdas" do
+      NestedDemo::TestJobB.queue(lambda { |klass| klass.name })
+      assert_equal "NestedDemo::TestJobB", NestedDemo::TestJobB.queue
+    end
   end # queue
 
   describe "for queue_priority assignment method" do
@@ -39,4 +44,4 @@ describe "Backburner::Queue module" do
       assert_equal 300, NestedDemo::TestJobB.queue_respond_timeout
     end
   end # queue_respond_timeout
-end # Backburner::Queue
+end # Backburner::Queue

data/test/worker_test.rb

@@ -63,6 +63,14 @@ describe "Backburner::Worker module" do
       assert_equal 100, job.ttr
       assert_equal Backburner.configuration.default_priority, job.pri
     end # async
+
+    it "should support enqueueing job with lambda queue" do
+      expected_queue_name = TestLambdaQueueJob.calculated_queue_name
+      Backburner::Worker.enqueue TestLambdaQueueJob, [6, 7], :queue => lambda { |klass| klass.calculated_queue_name }
+      job, body = pop_one_job(expected_queue_name)
+      assert_equal "TestLambdaQueueJob", body["class"]
+      assert_equal [6, 7], body["args"]
+    end
   end # enqueue
 
   describe "for start class method" do
@@ -117,4 +125,4 @@ describe "Backburner::Worker module" do
       assert_equal ['baz', 'bam'], worker.tube_names
     end
   end # tube_names
-end # Backburner::Worker
+end # Backburner::Worker

data/test/workers/simple_worker_test.rb

@@ -54,7 +54,7 @@ describe "Backburner::Workers::Basic module" do
       out = capture_stdout { worker.prepare }
       assert_contains worker.tube_names, "demo.test.backburner-jobs"
       assert_contains @worker_class.connection.tubes.watched.map(&:name), "demo.test.backburner-jobs"
-      assert_match(/demo\.test\.test-job/, out)
+      assert_match(/demo\.test\.backburner-jobs/, out)
     end # all read
   end # prepare
 
@@ -172,6 +172,58 @@ describe "Backburner::Workers::Basic module" do
       assert_equal true, $worker_success
     end # retrying, succeeds
 
+    it "should back off retries exponentially" do
+      max_job_retries = 3
+      clear_jobs!('foo.bar')
+      Backburner.configure do |config|
+        config.max_job_retries = max_job_retries
+        config.retry_delay = 0
+        #config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) } # default retry_delay_proc
+      end
+      @worker_class.enqueue TestConfigurableRetryJob, [max_job_retries], :queue => 'foo.bar'
+      out = []
+      (max_job_retries + 1).times do
+        out << silenced(10) do
+          worker = @worker_class.new('foo.bar')
+          worker.prepare
+          worker.work_one_job
+        end
+      end
+      assert_match(/attempt 1 of 4, retrying in 0/, out.first)
+      assert_match(/attempt 2 of 4, retrying in 1/, out[1])
+      assert_match(/attempt 3 of 4, retrying in 8/, out[2])
+      assert_match(/Completed TestConfigurableRetryJob/m, out.last)
+      refute_match(/failed/, out.last)
+      assert_equal 4, $worker_test_count
+      assert_equal true, $worker_success
+    end
+
+    it "should allow configurable back off retry delays" do
+      max_job_retries = 3
+      clear_jobs!('foo.bar')
+      Backburner.configure do |config|
+        config.max_job_retries = max_job_retries
+        config.retry_delay = 0
+        config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
+      end
+      @worker_class.enqueue TestConfigurableRetryJob, [max_job_retries], :queue => 'foo.bar'
+      out = []
+      (max_job_retries + 1).times do
+        out << silenced(5) do
+          worker = @worker_class.new('foo.bar')
+          worker.prepare
+          worker.work_one_job
+        end
+      end
+      assert_match(/attempt 1 of 4, retrying in 0/, out.first)
+      assert_match(/attempt 2 of 4, retrying in 1/, out[1])
+      assert_match(/attempt 3 of 4, retrying in 4/, out[2])
+      assert_match(/Completed TestConfigurableRetryJob/m, out.last)
+      refute_match(/failed/, out.last)
+      assert_equal 4, $worker_test_count
+      assert_equal true, $worker_success
+    end
+
     it "should support event hooks without retry" do
       $hooked_fail_count = 0
       clear_jobs!('foo.bar.events')
@@ -188,6 +240,7 @@ describe "Backburner::Workers::Basic module" do
       assert_match(/!!BEGIN around_perform_bar!! \[nil, "foo", 5\]/, out)
       assert_match(/!!BEGIN around_perform_cat!! \[nil, "foo", 5\]/, out)
       assert_match(/!!on_failure_foo!!.*HookFailError/, out)
+      assert_match(/!!on_bury_foo!! \[nil, "foo", 5\]/, out)
       assert_match(/attempt 1 of 1, burying/, out)
     end # event hooks, no retry
 
@@ -211,6 +264,7 @@ describe "Backburner::Workers::Basic module" do
       assert_match(/!!BEGIN around_perform_cat!! \[nil, "foo", 5\]/, out)
       assert_match(/!!on_failure_foo!!.*HookFailError/, out)
       assert_match(/!!on_failure_foo!!.*retrying.*around_perform_bar.*around_perform_cat/m, out)
+      assert_match(/!!on_retry_foo!! 1 0 \[nil, "foo", 5\]/, out)
       assert_match(/attempt 1 of 2, retrying/, out)
       assert_match(/!!before_perform_foo!! \[nil, "foo", 5\]/, out)
       assert_match(/!!END around_perform_bar!! \[nil, "foo", 5\]/, out)
@@ -246,7 +300,11 @@ describe "Backburner::Workers::Basic module" do
     end # stopping perform
 
     after do
-      Backburner.configure { |config| config.max_job_retries = 0; config.retry_delay = 5 }
+      Backburner.configure do |config|
+        config.max_job_retries = 0
+        config.retry_delay = 5
+        config.retry_delay_proc = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) } 
+      end
     end
   end # work_one_job
 end # Worker

metadata

@@ -1,55 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: backburner
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Nathan Esquenazi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-26 00:00:00.000000000 Z
+date: 2015-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beaneater
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: dante
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>'
+    - - ">"
       - !ruby/object:Gem::Version
         version: 0.1.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>'
+    - - ">"
       - !ruby/object:Gem::Version
         version: 0.1.5
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -70,14 +70,14 @@ dependencies:
   name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Beanstalk background job processing made easy
@@ -88,9 +88,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .DS_Store
-- .gitignore
-- .travis.yml
+- ".DS_Store"
+- ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - CONTRIBUTING.md
 - Gemfile
@@ -156,17 +156,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.6
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Reliable beanstalk background job processing made easy for Ruby and Sinatra