backburner 1.4.1 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 174517c1affdf586cdd239cbb8ac805e1fcd1f92
-  data.tar.gz: e26c736cdb790119a3a4b58a3cb267d1bad9edb1
+  metadata.gz: 9dfd7f06d2b08cb31ed2cf463506169544e0920d
+  data.tar.gz: 5d7585331fa5ccc082901017e9149d9982cb9de3
 SHA512:
-  metadata.gz: 1c106cf2657f606b65ad6817197650582104bb58d22160bbda9a5bbfdae85f58c6bdd5453ffc0df491a9d88d8d6f3db7de4b4a1a87de2f6a4a73cc8561c53209
-  data.tar.gz: b739580e0c435c2038bea4751f5e1efc964ff23381986bfbd051873f95165cb9fc0089d0e4c203854785b383ee4e17def330b0455060a68dc7ae186263aa3b64
+  metadata.gz: 38f1deb4da6faef87915b8fca99d99209adf76c5059f3478a75883b38a5e7abf89a9c0c9cec439641530f2b3e272362bfb0b8cc3343b9c1f9fcabf256b5bee24
+  data.tar.gz: d881653ea35fa55df59d53c5e104bc6dea13d857cabebacd8ecb5ba8ff845b9e736263ac7d1c386bbd6ff1d062e9abe0137ff8cf995262accd7231c7e4dc5a0b

data/.travis.yml

@@ -2,6 +2,11 @@
 rvm:
   - 1.9.3
   - 2.0.0
+  - 2.1
+  - 2.2
+  - 2.3
+  - 2.4
+  - 2.5
   - rbx-2
 before_install:
   - curl -L https://github.com/kr/beanstalkd/archive/v1.9.tar.gz | tar xz -C /tmp
@@ -11,11 +16,10 @@ before_install:
   - cd $TRAVIS_BUILD_DIR
   - gem update --system
   - gem update bundler
-install:
-  - bundle install
 matrix:
   allow_failures:
     - rvm: rbx-2
+    - rvm: 2.0.0
 script:
   - bundle exec rake test
 gemfile: Gemfile

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## Version 1.5.0 (September 10 2018)
+
+* TBD
+
 ## Version 1.4.1 (June 10 2017)
 
 * Fix warning for constant ::Fixnum is deprecated (@amatsuda)

data/README.md

@@ -1,4 +1,4 @@
-# Backburner
+# Backburner [![Build Status](https://travis-ci.org/nesquena/backburner.svg?branch=master)](https://travis-ci.org/nesquena/backburner)
 
 Backburner is a [beanstalkd](http://kr.github.com/beanstalkd/)-powered job queue that can handle a very high volume of jobs.
 You create background jobs and place them on multiple work queues to be processed later.
@@ -34,7 +34,7 @@ The real question then is... "Why Beanstalk?".
 
 Illya has an excellent blog post
 [Scalable Work Queues with Beanstalk](http://www.igvita.com/2010/05/20/scalable-work-queues-with-beanstalk/) and
-Adam Wiggins posted [an excellent comparison](http://adam.heroku.com/past/2010/4/24/beanstalk_a_simple_and_fast_queueing_backend/).
+Adam Wiggins posted [an excellent comparison](http://adam.herokuapp.com/past/2010/4/24/beanstalk_a_simple_and_fast_queueing_backend/).
 
 You will quickly see that **beanstalkd** is an underrated but incredible project that is extremely well-suited as a job queue.
 Significantly better suited for this task than Redis or a database. Beanstalk is a simple,
@@ -513,6 +513,9 @@ end
 
 Now all backburner queue errors will appear on airbrake for deeper inspection.
 
+If you wish to retry a job without logging an error (for example when handling transient issues in a cloud or service oriented environment), 
+simply raise a `Backburner::Job::RetryJob` error.
+
 ### Logging
 
 Logging in backburner is rather simple. When a job is run, the log records that. When a job

data/backburner.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
 
   s.add_runtime_dependency 'beaneater', '~> 1.0'
   s.add_runtime_dependency 'dante', '> 0.1.5'
-  s.add_runtime_dependency 'concurrent-ruby', '~> 1.0.1'
+  s.add_runtime_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0.1'
 
   s.add_development_dependency 'rake'
   s.add_development_dependency 'minitest', '3.2.0'

data/lib/backburner/connection.rb

@@ -34,7 +34,7 @@ module Backburner
     def connected?
       begin
         !!(@beanstalk && @beanstalk.connection && @beanstalk.connection.connection && !@beanstalk.connection.connection.closed?) # Would be nice if beaneater provided a connected? method
-      rescue => e
+      rescue
         false
       end
     end

data/lib/backburner/job.rb

@@ -7,6 +7,7 @@ module Backburner
     class JobTimeout < RuntimeError; end
     class JobNotFound < RuntimeError; end
     class JobFormatInvalid < RuntimeError; end
+    class RetryJob < RuntimeError; end
 
     attr_accessor :task, :body, :name, :args
 
@@ -43,10 +44,10 @@ module Backburner
     #
     def process
       # Invoke before hook and stop if false
-      res = @hooks.invoke_hook_events(job_class, :before_perform, *args)
+      res = @hooks.invoke_hook_events(job_name, :before_perform, *args)
       return false unless res
       # Execute the job
-      @hooks.around_hook_events(job_class, :around_perform, *args) do
+      @hooks.around_hook_events(job_name, :around_perform, *args) do
         # We subtract one to ensure we timeout before beanstalkd does, except if:
         #  a) ttr == 0, to support never timing out
         #  b) ttr == 1, so that we don't accidentally set it to never time out
@@ -56,19 +57,19 @@ module Backburner
       end
       task.delete
       # Invoke after perform hook
-      @hooks.invoke_hook_events(job_class, :after_perform, *args)
+      @hooks.invoke_hook_events(job_name, :after_perform, *args)
     rescue => e
-      @hooks.invoke_hook_events(job_class, :on_failure, e, *args)
+      @hooks.invoke_hook_events(job_name, :on_failure, e, *args)
       raise e
     end
 
     def bury
-      @hooks.invoke_hook_events(job_class, :on_bury, *args)
+      @hooks.invoke_hook_events(job_name, :on_bury, *args)
       task.bury
     end
 
     def retry(count, delay)
-      @hooks.invoke_hook_events(job_class, :on_retry, count, delay, *args)
+      @hooks.invoke_hook_events(job_name, :on_retry, count, delay, *args)
       task.release(delay: delay)
     end
 
@@ -80,11 +81,26 @@ module Backburner
     #   job_class # => NewsletterSender
     #
     def job_class
-      handler = constantize(self.name) rescue nil
+      handler = try_job_class
       raise(JobNotFound, self.name) unless handler
       handler
     end
 
+    # Attempts to return a constantized job name, otherwise reverts to the name string
+    #
+    # @example
+    #   job_name # => "SomeUnknownJob"
+    def job_name
+      handler = try_job_class
+      handler ? handler : self.name
+    end
+
+    def try_job_class
+      constantize(self.name)
+    rescue NameError
+      nil
+    end
+
     # Timeout job within specified block after given time.
     #
     # @example

data/lib/backburner/logger.rb

@@ -10,7 +10,7 @@ module Backburner
     # Print out when a job is about to begin
     def log_job_begin(name, args)
       log_info "Work job #{name} with #{args.inspect}"
-      @job_started_at = Time.now
+      Thread.current[:job_started_at] = Time.now
     end
 
     # Print out when a job completed
@@ -24,7 +24,7 @@ module Backburner
 
     # Returns true if the job logging started
     def job_started_at
-      @job_started_at
+      Thread.current[:job_started_at]
     end
 
     # Print a message to stdout
@@ -50,4 +50,4 @@ module Backburner
       Backburner.configuration.logger
     end
   end
-end
+end

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.4.1"
+  VERSION = "1.5.0"
 end

data/lib/backburner/worker.rb

@@ -140,7 +140,7 @@ module Backburner
     rescue Backburner::Job::JobFormatInvalid => e
       self.log_error self.exception_message(e)
     rescue => e # Error occurred processing job
-      self.log_error self.exception_message(e)
+      self.log_error self.exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
 
       unless job
         self.log_error "Error occurred before we were able to assign a job. Giving up without retrying!"

data/lib/backburner/workers/threading.rb

@@ -3,9 +3,13 @@ require 'concurrent'
 module Backburner
   module Workers
     class Threading < Worker
+      attr_accessor :self_read, :self_write, :exit_on_shutdown
+
+      @shutdown_timeout = 10
+
       class << self
-        attr_accessor :shutdown
         attr_accessor :threads_number
+        attr_accessor :shutdown_timeout
       end
 
       # Custom initializer just to set @tubes_data
@@ -13,6 +17,7 @@ module Backburner
         @tubes_data = {}
         super
         self.process_tube_options
+        @exit_on_shutdown = true
       end
 
       # Used to prepare job queues before processing jobs.
@@ -48,16 +53,19 @@ module Backburner
             connection.on_reconnect = lambda { |conn| conn.tubes.watch!(tube_name) }
 
             # Make it work jobs using its own connection per thread
-            pool.post(connection) { |connection|
-              loop {
+            pool.post(connection) do |memo_connection|
+              # TODO: use read-write lock?
+              loop do
                 begin
-                  work_one_job(connection)
-
+                  break if @in_shutdown
+                  work_one_job(memo_connection)
                 rescue => e
                   log_error("Exception caught in thread pool loop. Continuing. -> #{e.message}\nBacktrace: #{e.backtrace}")
                 end
-              }
-            }
+              end
+
+              connection.close
+            end
           end
         end
 
@@ -111,18 +119,44 @@ module Backburner
 
       # Wait for the shutdown signel
       def wait_for_shutdown!
-        while !self.class.shutdown do
-          sleep 0.5
+        raise Interrupt while IO.select([self_read])
+      rescue Interrupt
+        shutdown
+      end
+
+      def shutdown_threadpools
+        @thread_pools.each { |_name, pool| pool.shutdown }
+        shutdown_time = Time.now
+        @in_shutdown = true
+        all_shutdown = @thread_pools.all? do |_name, pool|
+          time_to_wait = self.class.shutdown_timeout - (Time.now - shutdown_time).to_i
+          pool.wait_for_termination(time_to_wait) if time_to_wait > 0
         end
+      rescue Interrupt
+        log_info "graceful shutdown aborted, shutting down immediately"
+      ensure
+        kill unless all_shutdown
+      end
 
-        # Shutting down
-        # FIXME: Shut down each thread's connection
-        @thread_pools.each { |name, pool| pool.kill }
+      def kill
+        @thread_pools.each { |_name, pool| pool.kill unless pool.shutdown? }
       end
 
       def shutdown
-        Backburner::Workers::Threading.shutdown = true
-        super
+        log_info "beginning graceful worker shutdown"
+        shutdown_threadpools
+        super if @exit_on_shutdown
+      end
+
+      # Registers signal handlers TERM and INT to trigger
+      def register_signal_handlers!
+        @self_read, @self_write = IO.pipe
+        %w[TERM INT].each do |sig|
+          trap(sig) do
+            raise Interrupt if @in_shutdown
+            self_write.puts(sig)
+          end
+        end
       end
     end # Threading
   end # Workers

data/test/fixtures/test_jobs.rb

@@ -13,6 +13,24 @@ class TestJob
   def self.perform(x, y); $worker_test_count += x + y; end
 end
 
+class TestSlowJob
+  include Backburner::Queue
+  queue_priority :medium
+  queue_respond_timeout 300
+  def self.perform(x, y); sleep 1; $worker_test_count += x + y; end
+end
+
+class TestStuckJob
+  include Backburner::Queue
+  queue_priority :medium
+  queue_respond_timeout 300
+  def self.perform(_x, _y)
+    loop do
+      sleep 0.5
+    end
+  end
+end
+
 class TestFailJob
   include Backburner::Queue
   def self.perform(x, y); raise RuntimeError; end

data/test/job_test.rb

@@ -113,16 +113,41 @@ describe "Backburner::Job module" do
   end # process
 
   describe "for simple delegation method" do
-    before do
-      @task_body =  { "class" => "NestedDemo::TestJobC", "args" => [56] }
-      @task = stub(:body => @task_body, :ttr => 120, :delete => true, :bury => true)
-      @task.expects(:bury).once
+    describe "with valid class" do
+      before do
+        @task_body =  { "class" => "NestedDemo::TestJobC", "args" => [56] }
+        @task = stub(:body => @task_body, :ttr => 120, :delete => true, :bury => true)
+        @task.expects(:bury).once
+      end
+
+      it "should call bury for task" do
+        @job = Backburner::Job.new(@task)
+        @job.bury
+      end # bury
     end
 
-    it "should call bury for task" do
-      @job = Backburner::Job.new(@task)
-      @job.bury
-    end # bury
+    describe "with invalid class" do
+      before do
+        @task_body = { "class" => "AnUnknownClass", "args" => [] }
+        @task = stub(:body => @task_body, :ttr => 120, :delete => true, :bury => true, :release => true)
+      end
+
+      it "should call bury for task" do
+        @task.expects(:bury).once
+        @job = Backburner::Job.new(@task)
+        Backburner::Hooks.expects(:invoke_hook_events)
+          .with("AnUnknownClass", :on_bury, anything)
+        @job.bury
+      end
+
+      it "should call retry for task" do
+        @task.expects(:release).once
+        @job = Backburner::Job.new(@task)
+        Backburner::Hooks.expects(:invoke_hook_events)
+          .with("AnUnknownClass", :on_retry, 0, is_a(Integer), anything)
+        @job.retry(0, 0)
+      end
+    end
   end # simple delegation
 
   describe "timing out for various values of ttr" do

data/test/test_helper.rb

@@ -6,7 +6,7 @@ begin
 rescue LoadError
   require 'mocha'
 end
-$:.unshift File.expand_path("../../lib")
+$LOAD_PATH.unshift File.expand_path("lib")
 require 'backburner'
 require File.expand_path('../helpers/templogger', __FILE__)
 

data/test/workers/forking_worker_test.rb

@@ -11,7 +11,7 @@ describe "Backburner::Workers::Forking module" do
   describe "for prepare method" do
     it "should make tube names array always unique to avoid duplication" do
       worker = @worker_class.new(["foo", "demo.test.foo"])
-      worker.prepare
+      capture_stdout { worker.prepare }
       assert_equal ["demo.test.foo"], worker.tube_names
     end
 

data/test/workers/simple_worker_test.rb

@@ -11,7 +11,7 @@ describe "Backburner::Workers::Simple module" do
   describe "for prepare method" do
     it "should make tube names array always unique to avoid duplication" do
       worker = @worker_class.new(["foo", "demo.test.foo"])
-      worker.prepare
+      capture_stdout { worker.prepare }
       assert_equal ["demo.test.foo"], worker.tube_names
     end
 
@@ -309,7 +309,7 @@ describe "Backburner::Workers::Simple module" do
       worker = @worker_class.new('foo.bar')
       connection = mock('connection')
       worker.expects(:reserve_job).with(connection).returns(stub_everything('job'))
-      worker.work_one_job(connection)
+      capture_stdout { worker.work_one_job(connection) }
     end
 
     after do

data/test/workers/threading_worker_test.rb

@@ -6,24 +6,25 @@ describe "Backburner::Workers::Threading worker" do
   before do
     Backburner.default_queues.clear
     @worker_class = Backburner::Workers::Threading
+    @worker_class.shutdown_timeout = 2
   end
 
   describe "for prepare method" do
     it "should make tube names array always unique to avoid duplication" do
       worker = @worker_class.new(["foo", "demo.test.foo"])
-      worker.prepare
+      capture_stdout { worker.prepare }
       assert_equal ["demo.test.foo"], worker.tube_names
     end
 
     it 'creates a thread pool per queue' do
       worker = @worker_class.new(%w(foo bar))
-      out = capture_stdout { worker.prepare }
+      capture_stdout { worker.prepare }
       assert_equal 2, worker.instance_variable_get("@thread_pools").keys.size
     end
 
     it 'uses Concurrent.processor_count if no custom thread count is provided' do
       worker = @worker_class.new("foo")
-      out = capture_stdout { worker.prepare }
+      capture_stdout { worker.prepare }
       assert_equal ::Concurrent.processor_count, worker.instance_variable_get("@thread_pools")["demo.test.foo"].max_length
     end
   end # prepare
@@ -61,10 +62,43 @@ describe "Backburner::Workers::Threading worker" do
     it 'runs work_on_job per thread' do
       clear_jobs!("foo")
       job_count=10
-      job_count.times { @worker_class.enqueue TestJob, [1, 0], :queue => "foo" } # TestJob adds the given arguments together and then to $worker_test_count
-      @worker.start(false) # don't wait for shutdown
-      sleep 0.5 # Wait for threads to do their work
+      # TestJob adds the given arguments together and then to $worker_test_count
+      job_count.times { @worker_class.enqueue TestJob, [1, 0], :queue => "foo" }
+      capture_stdout do
+        @worker.start(false) # don't wait for shutdown
+        sleep 0.5 # Wait for threads to do their work
+      end
       assert_equal job_count, $worker_test_count
     end
   end # working a queue
+
+  describe 'shutting down' do
+    before do
+      @thread_count = 3
+      @worker = @worker_class.new(["threaded-shutdown:#{@thread_count}"])
+      @worker.exit_on_shutdown = false
+      $worker_test_count = 0
+      clear_jobs!("threaded-shutdown")
+    end
+
+    it 'gracefully exits and completes all in-flight jobs' do
+      6.times { @worker_class.enqueue TestSlowJob, [1, 0], :queue => "threaded-shutdown" }
+      Thread.new { sleep 0.1; @worker.self_write.puts("TERM") }
+      capture_stdout do
+        @worker.start
+      end
+
+      assert_equal @thread_count, $worker_test_count
+    end
+
+    it 'forces an exit when a job is stuck' do
+      6.times { @worker_class.enqueue TestStuckJob, [1, 0], :queue => "threaded-shutdown" }
+      Thread.new { sleep 0.1; @worker.self_write.puts("TERM") }
+      capture_stdout do
+        @worker.start
+      end
+
+      assert_equal 0, $worker_test_count
+    end
+  end
 end

data/test/workers/threads_on_fork_worker_test.rb

@@ -70,7 +70,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
 
     it "should make tube names array always unique to avoid duplication" do
       worker = @worker_class.new(["foo", "demo.test.foo"])
-      worker.prepare
+      capture_stdout { worker.prepare }
       assert_equal ["demo.test.foo"], worker.tube_names
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.5.0
 platform: ruby
 authors:
 - Nathan Esquenazi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-10 00:00:00.000000000 Z
+date: 2018-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beaneater
@@ -43,6 +43,9 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
   type: :runtime
@@ -50,6 +53,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
 - !ruby/object:Gem::Dependency