backburner 0.4.5 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c49c95fffe9a81e2e0f775b07b3659a8557d465a
-  data.tar.gz: 69446fcc23d3a4a31c4ce9b569ee3fe61f0309be
+  metadata.gz: b2b489b60308b955b398ce8a9936c0bfca96588e
+  data.tar.gz: 6a22f67c5502cd730139c4a857d4417552419c57
 SHA512:
-  metadata.gz: d5968d8bd1d3df5d72cd697fa66509f573bceca12781b26f84d9e8ce66f3265affdd65e98231d5780674a959c0dc9691cdb77af476b07c4b7c637677155f371c
-  data.tar.gz: b48cdf8db9eefa524d6e653af2876551f0c44d5f8e7e9453f68ce312132859be526224ab2ec1e9d43374265983ffbb8fea836434a425e5a015547b79fd598e1d
+  metadata.gz: c751606af667cdeadbfd9d270d5aae80ccc42a9f9f968607f4a02d783207b3db4171ab9cc207e44b38a435024a68b6e6bfcb319973788940f819f6d05b49843a
+  data.tar.gz: 5bacb2a1f9c613addac6a7ad3e4c45d9358fdd523092faf50f696fbbb3aede4d0676bddd875caa4fca84d36995bf9bfbd3aafe43fc33fda6e9458539dd6d3421

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # CHANGELOG
 
+## Version 0.4.6 (October 26 2014)
+
+ * NEW Add job to on_error handler if the handler has a 4th argument (@Nitrodist)
+ * NEW Use a timeout when looking for a job to reserve (@EasyPost)
+ * NEW Support configuring settings on threads on fork class (@silentshade)
+ * FIX queue override by existing queues (@silentshade)
+ * FIX Use thread to log exit message (@silentshade)
+
 ## Version 0.4.5 (December 16 2013)
 
  * FIX #47 Create a backburner connection per thread (Thanks @thcrock)

data/README.md

@@ -99,6 +99,7 @@ Backburner.configure do |config|
   config.logger           = Logger.new(STDOUT)
   config.primary_queue    = "backburner-jobs"
   config.priority_labels  = { :custom => 50, :useless => 1000 }
+  config.reserve_timeout  = nil
 end
 ```
 
@@ -115,6 +116,7 @@ The key options available are:
 | `logger`          | Logger recorded to when backburner wants to report info or errors.   |
 | `primary_queue`   | Primary queue used for a job when an alternate queue is not given.   |
 | `priority_labels` | Hash of named priority definitions for your app.                     |
+| `reserve_timeout` | Duration to wait for work from a single server, or nil for forever.  |
 
 ## Breaking Changes
 
@@ -482,6 +484,8 @@ jobs processed by your beanstalk workers. An excellent addition to your Backburn
  * Kristen Tucker - Coming up with the gem name
  * [Tim Lee](https://github.com/timothy1ee), [Josh Hull](https://github.com/joshbuddy), [Nico Taing](https://github.com/Nico-Taing) - Helping me work through the idea
  * [Miso](http://gomiso.com) - Open-source friendly place to work
+ * [Evgeniy Denisov](https://github.com/silentshade) - Multiple fixes and cleanups
+ * [Andy Bakun](https://github.com/thwarted) - Fixes to how multiple beanstalkd instances are processed
  * [Renan T. Fernandes](https://github.com/ShadowBelmolve) - Added threads_on_fork worker
  * [Daniel Farrell](https://github.com/danielfarrell) - Added forking worker
 

data/lib/backburner/cli.rb

@@ -17,7 +17,6 @@ module Backburner
       runner.execute do |opts|
         queues = (opts[:queues] ? opts[:queues].split(',') : nil) rescue nil
         load_enviroment(opts[:require])
-        p queues
         Backburner.work(queues)
       end
     end

data/lib/backburner/configuration.rb

@@ -14,6 +14,7 @@ module Backburner
     attr_accessor :default_worker     # default worker class
     attr_accessor :primary_queue      # the general queue
     attr_accessor :priority_labels    # priority labels
+    attr_accessor :reserve_timeout    # duration to wait to reserve on a single server
 
     def initialize
       @beanstalk_url     = "beanstalk://localhost"
@@ -28,6 +29,7 @@ module Backburner
       @default_worker    = Backburner::Workers::Simple
       @primary_queue     = "backburner-jobs"
       @priority_labels   = PRIORITY_LABELS
+      @reserve_timeout   = nil
     end
   end # Configuration
-end # Backburner
+end # Backburner

data/lib/backburner/helpers.rb

@@ -80,6 +80,7 @@ module Backburner
     # Expands a tube to include the prefix
     #
     # @example
+    #   expand_tube_name("foo_with_settings:3:100:6") # => <prefix>.foo_with_settings
     #   expand_tube_name("foo") # => <prefix>.foo
     #   expand_tube_name(FooJob) # => <prefix>.foo-job
     #
@@ -94,7 +95,7 @@ module Backburner
       else # turn into a string
         tube.to_s
       end
-      [prefix.gsub(/\.$/, ''), dasherize(queue_name).gsub(/^#{prefix}/, '')].join(".").gsub(/\.+/, '.')
+      [prefix.gsub(/\.$/, ''), dasherize(queue_name).gsub(/^#{prefix}/, '')].join(".").gsub(/\.+/, '.').split(':').first
     end
 
     # Resolves job priority based on the value given. Can be integer, a class or nothing

data/lib/backburner/queue.rb

@@ -47,6 +47,48 @@ module Backburner
           @queue_respond_timeout
         end
       end
+
+      # Returns or assigns queue parallel active jobs limit (only ThreadsOnFork Worker)
+      #
+      # @example
+      #   queue_jobs_limit 5
+      #   @klass.queue_jobs_limit # => 5
+      #
+      def queue_jobs_limit(limit=nil)
+        if limit
+          @queue_jobs_limit = limit
+        else #accessor
+          @queue_jobs_limit
+        end
+      end
+
+      # Returns or assigns queue jobs garbage limit (only ThreadsOnFork Worker)
+      #
+      # @example
+      #   queue_garbage_limit 1000
+      #   @klass.queue_garbage_limit # => 1000
+      #
+      def queue_garbage_limit(limit=nil)
+        if limit
+          @queue_garbage_limit = limit
+        else #accessor
+          @queue_garbage_limit
+        end
+      end
+
+      # Returns or assigns queue retry limit (only ThreadsOnFork Worker)
+      #
+      # @example
+      #   queue_retry_limit 6
+      #   @klass.queue_retry_limit # => 6
+      #
+      def queue_retry_limit(limit=nil)
+        if limit
+          @queue_retry_limit = limit
+        else #accessor
+          @queue_retry_limit
+        end
+      end
     end # ClassMethods
   end # Queue
-end # Backburner
+end # Backburner

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "0.4.5"
+  VERSION = "0.4.6"
 end

data/lib/backburner/worker.rb

@@ -58,22 +58,6 @@ module Backburner
       @connection ||= Connection.new(Backburner.configuration.beanstalk_url)
     end
 
-    # Retries the given command specified in the block several times if there is a connection error
-    # Used to execute beanstalkd commands in a retryable way
-    #
-    # @example
-    #   retryable_command { ... }
-    # @raise [Beaneater::NotConnected] If beanstalk fails to connect multiple times.
-    #
-    def self.retryable_command(max_tries=8, &block)
-      begin
-        yield
-      rescue Beaneater::NotConnected => e
-        retry_connection!(max_tries)
-        yield
-      end
-    end
-
     # List of tube names to be watched and processed
     attr_accessor :tube_names
 
@@ -112,7 +96,9 @@ module Backburner
 
     # Triggers this worker to shutdown
     def shutdown
-      log_info 'Worker exiting...'
+      Thread.new do
+        log_info 'Worker exiting...'
+      end
       Kernel.exit
     end
 
@@ -138,7 +124,11 @@ module Backburner
     #
     def work_one_job(conn = nil)
       conn ||= self.connection
-      job = Backburner::Job.new(conn.tubes.reserve)
+      begin
+        job = Backburner::Job.new(conn.tubes.reserve(Backburner.configuration.reserve_timeout))
+      rescue Beaneater::TimedOutError => e
+        return
+      end
       self.log_job_begin(job.name, job.args)
       job.process
       self.log_job_end(job.name)
@@ -156,7 +146,7 @@ module Backburner
         job.bury
         self.log_job_end(job.name, "#{retry_status}, burying") if job_started_at
       end
-      handle_error(e, job.name, job.args)
+      handle_error(e, job.name, job.args, job)
     end
 
     # Retries the given command specified in the block several times if there is a connection error
@@ -201,7 +191,7 @@ module Backburner
     def all_existing_queues
       known_queues    = Backburner::Worker.known_queue_classes.map(&:queue)
       existing_tubes  = self.connection.tubes.all.map(&:name).select { |tube| tube =~ /^#{queue_config.tube_namespace}/ }
-      known_queues + existing_tubes + [queue_config.primary_queue]
+      existing_tubes + known_queues + [queue_config.primary_queue]
     end
 
     # Returns a reference to the beanstalk connection
@@ -211,12 +201,14 @@ module Backburner
 
     # Handles an error according to custom definition
     # Used when processing a job that errors out
-    def handle_error(e, name, args)
+    def handle_error(e, name, args, job)
       if error_handler = Backburner.configuration.on_error
         if error_handler.arity == 1
           error_handler.call(e)
-        else
+        elsif error_handler.arity == 3
           error_handler.call(e, name, args)
+        else
+          error_handler.call(e, name, args, job)
         end
       end
     end
@@ -238,4 +230,4 @@ module Backburner
       trap('INT')  { shutdown  }
     end
   end # Worker
-end # Backburner
+end # Backburner

data/lib/backburner/workers/threads_on_fork.rb

@@ -70,6 +70,7 @@ module Backburner
       def initialize(*args)
         @tubes_data = {}
         super
+        self.process_tube_options
       end
 
       # Process the special tube_names of ThreadsOnFork worker
@@ -101,11 +102,29 @@ module Backburner
         end
       end
 
+      # Process the tube settings
+      # This overrides @tubes_data set by process_tube_names method. So a tube has name 'super_job:5:20:10'
+      # and the tube class has setting queue_jobs_limit 10, the result limit will be 10
+      # If the tube is known by existing beanstalkd queue, but not by class - skip it
+      #
+      def process_tube_options
+        Backburner::Worker.known_queue_classes.each do |queue|
+          next if @tubes_data[expand_tube_name(queue)].nil?
+          queue_settings = {
+              :threads => queue.queue_jobs_limit,
+              :garbage => queue.queue_garbage_limit,
+              :retries => queue.queue_retry_limit
+          }
+          @tubes_data[expand_tube_name(queue)].merge!(queue_settings){|k, v1, v2| v2.nil? ? v1 : v2 }
+        end
+      end
+
       def prepare
         self.tube_names ||= Backburner.default_queues.any? ? Backburner.default_queues : all_existing_queues
         self.tube_names = Array(self.tube_names)
         tube_names.map! { |name| expand_tube_name(name)  }
-        log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
+        tube_display_names = tube_names.map{|name| "#{name}:#{@tubes_data[name].values}"}
+        log_info "Working #{tube_names.size} queues: [ #{tube_display_names.join(', ')} ]"
       end
 
       # For each tube we will call fork_and_watch to create the fork

data/test/fixtures/test_queue_settings.rb

@@ -0,0 +1,14 @@
+class TestJobSettings
+  include Backburner::Queue
+  queue "job-settings:5:10:6"
+  def self.perform; end
+end
+
+class TestJobSettingsOverride
+  include Backburner::Queue
+  queue "job-settings-override:5:10:12"
+  queue_jobs_limit 10
+  queue_garbage_limit 1000
+  queue_retry_limit 2
+  def self.perform; end
+end

data/test/workers/threads_on_fork_worker_test.rb

@@ -1,5 +1,6 @@
 require File.expand_path('../../test_helper', __FILE__)
 require File.expand_path('../../fixtures/test_fork_jobs', __FILE__)
+require File.expand_path('../../fixtures/test_queue_settings', __FILE__)
 
 describe "Backburner::Workers::ThreadsOnFork module" do
 
@@ -16,7 +17,8 @@ describe "Backburner::Workers::ThreadsOnFork module" do
   after do
     Backburner.configure { |config| config.max_job_retries = 0; config.retry_delay = 5; config.logger = nil }
     unless @ignore_forks
-      if @worker_class.instance_variable_get("@child_pids").length > 0
+      cpids = @worker_class.instance_variable_get("@child_pids")
+      if cpids && cpids.length > 0
         raise "Why is there forks alive?"
       end
     end
@@ -49,6 +51,18 @@ describe "Backburner::Workers::ThreadsOnFork module" do
     end
   end
 
+  describe "for process_tube_settings" do
+    it "should set the settings specified by queue name in class" do
+      worker = @worker_class.new
+      assert_equal(worker.instance_variable_get("@tubes_data")['demo.test.job-settings'],  { :threads => 5,   :garbage => 10,   :retries => 6 })
+    end
+
+    it 'should override the tube settings if they are specified directly at class level' do
+      worker = @worker_class.new
+      assert_equal(worker.instance_variable_get("@tubes_data")['demo.test.job-settings-override'], { :threads => 10,   :garbage => 1000,   :retries => 2 })
+    end
+  end
+
   describe "for prepare method" do
     before do
       Backburner.configure { |config| config.logger = false }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.6
 platform: ruby
 authors:
 - Nathan Esquenazi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-07 00:00:00.000000000 Z
+date: 2014-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beaneater
@@ -132,6 +132,7 @@ files:
 - test/fixtures/test_fork_jobs.rb
 - test/fixtures/test_forking_jobs.rb
 - test/fixtures/test_jobs.rb
+- test/fixtures/test_queue_settings.rb
 - test/helpers/templogger.rb
 - test/helpers_test.rb
 - test/hooks_test.rb
@@ -176,6 +177,7 @@ test_files:
 - test/fixtures/test_fork_jobs.rb
 - test/fixtures/test_forking_jobs.rb
 - test/fixtures/test_jobs.rb
+- test/fixtures/test_queue_settings.rb
 - test/helpers/templogger.rb
 - test/helpers_test.rb
 - test/hooks_test.rb