backburner 0.4.4 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5cefef513a8a0bdd3a88721c01eb503c01f4aa3c
-  data.tar.gz: b9b0506df21468b7496c5a6b7dec90492c0dc899
+  metadata.gz: c49c95fffe9a81e2e0f775b07b3659a8557d465a
+  data.tar.gz: 69446fcc23d3a4a31c4ce9b569ee3fe61f0309be
 SHA512:
-  metadata.gz: 0da929c2d10311de013e5b1943754fdeb5dc75e598cb2e52097510827ebf23df93a8cb788be599fbfc54fbf83f3b15637a0c91825cbeb49c3da995a7f734363e
-  data.tar.gz: 16783da0512636828941b8a141539c67f4a3dc528f2d0a66e065b4c5c2c3ea710a7c4d658829753362e40f4d3981d6fbb329d4df5c0573d8e006721c5f4692e7
+  metadata.gz: d5968d8bd1d3df5d72cd697fa66509f573bceca12781b26f84d9e8ce66f3265affdd65e98231d5780674a959c0dc9691cdb77af476b07c4b7c637677155f371c
+  data.tar.gz: b48cdf8db9eefa524d6e653af2876551f0c44d5f8e7e9453f68ce312132859be526224ab2ec1e9d43374265983ffbb8fea836434a425e5a015547b79fd598e1d

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## Version 0.4.5 (December 16 2013)
+
+ * FIX #47 Create a backburner connection per thread (Thanks @thcrock)
+
 ## Version 0.4.4 (October 27 2013)
 
  * NEW #51 Added ability to set per-queue default ttr's (Thanks @ryanjohns)

data/README.md

@@ -251,6 +251,22 @@ bundle exec backburner -q newsletter-sender,push-notifier -d -P /var/run/backbur
 This will daemonize the worker and store the pid and logs automatically. For Rails and Padrino, the environment should
 load automatically. For other cases, use the `-r` flag to specify a file to require.
 
+### Delaying Jobs
+
+In Backburner, jobs can be delayed by specifying the `delay` option whenever you enqueue a job. If you want to schedule a job for an hour from now, simply add that option while enqueuing the standard job:
+
+```ruby
+Backburner::Worker.enqueue(NewsletterJob, ['foo@admin.com', 'lorem ipsum...'], :delay => 1.hour)
+```
+
+or while you schedule an async method call:
+
+```ruby
+User.async(:delay => 1.hour).reset_password(@user.id)
+```
+
+Backburner will take care of the rest!
+
 ### Persistence
 
 Jobs are persisted to queues as JSON objects. Let's take our `User`

data/lib/backburner/job.rb

@@ -77,10 +77,10 @@ module Backburner
     def timeout_job_after(secs, &block)
       begin
         Timeout::timeout(secs) { yield }
-      rescue Timeout::Error
-        raise JobTimeout, "#{name} hit #{secs}s timeout"
+      rescue Timeout::Error => e
+        raise JobTimeout, "#{name} hit #{secs}s timeout.\nbacktrace: #{e.backtrace}"
       end
     end
 
   end # Job
-end # Backburner
+end # Backburner

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "0.4.4"
+  VERSION = "0.4.5"
 end

data/lib/backburner/worker.rb

@@ -136,8 +136,9 @@ module Backburner
     # @example
     #   @worker.work_one_job
     #
-    def work_one_job
-      job = Backburner::Job.new(self.connection.tubes.reserve)
+    def work_one_job(conn = nil)
+      conn ||= self.connection
+      job = Backburner::Job.new(conn.tubes.reserve)
       self.log_job_begin(job.name, job.args)
       job.process
       self.log_job_end(job.name)

data/lib/backburner/workers/threads_on_fork.rb

@@ -165,12 +165,17 @@ module Backburner
         @runs = 0
 
         if @threads_number == 1
-          run_while_can
+          run_while_can(name)
         else
           threads_count = Thread.list.count
           @threads_number.times do
             create_thread do
-              run_while_can
+              conn = Connection.new(Backburner.configuration.beanstalk_url)
+              begin
+                run_while_can(name, conn)
+              ensure
+                conn.close
+              end
             end
           end
           sleep 0.1 while Thread.list.count > threads_count
@@ -180,16 +185,19 @@ module Backburner
       end
 
       # Run work_one_job while we can
-      def run_while_can
+      def run_while_can(name, conn = nil)
+        conn ||= connection
+        watch_tube(name, conn)
         while @garbage_after.nil? or @garbage_after > @runs
           @runs += 1
-          work_one_job
+          work_one_job(conn)
         end
       end
 
       # Shortcut for watching a tube on beanstalk connection
-      def watch_tube(name)
-        connection.tubes.watch!(name)
+      def watch_tube(name, conn = nil)
+        conn ||= connection
+        conn.tubes.watch!(name)
       end
 
       # Exit with Kernel.exit! to avoid at_exit callbacks that should belongs to
@@ -236,4 +244,4 @@ at_exit do
     Backburner::Workers::ThreadsOnFork.shutdown = true
   end
   Backburner::Workers::ThreadsOnFork.finish_forks
-end
+end

data/test/fixtures/test_forking_jobs.rb

@@ -53,4 +53,4 @@ class TestAsyncJobForking
         :worker_test_count_set => x * y
     }], :queue => 'response'
   end
-end
+end

data/test/workers/threads_on_fork_worker_test.rb

@@ -192,6 +192,33 @@ describe "Backburner::Workers::ThreadsOnFork module" do
         end
       end
 
+      it "should create a connection for each thread" do
+        name = 'demo.test.foo'
+        num_threads = 3
+
+        worker = @worker_class.new(%(foo))
+        @worker_class.expects(:threads_number).returns(num_threads)
+        invocations = Array(1..num_threads).map { |i|
+          conn = OpenStruct.new(:num => i)
+          conn.expects(:close)
+          conn
+        }
+        Backburner::Connection.expects(:new).times(num_threads).returns(*invocations)
+
+        # ensure each invocation of run_while_can is with a different connection
+        num_conns = states('num_conns').starts_as(0)
+        invocations.each do |conn|
+          worker.expects(:run_while_can).with(name, conn).when(num_conns.is(conn.num-1)).then(num_conns.is(conn.num))
+        end
+
+        def worker.create_thread(*args, &block); block.call(*args) end
+        silenced do
+          worker.prepare
+          worker.fork_inner(name)
+        end
+        assert_equal(num_threads, num_conns.current_state)
+      end
+
       it "should set @garbage_after, @threads_number and set retries if needed" do
         worker = @worker_class.new(%W(foo1 foo2:10 foo3:20:30 foo4:40:50:60))
         default_threads = 1
@@ -327,7 +354,7 @@ describe "Backburner::Workers::ThreadsOnFork module" do
         $worker_success = false
         $worker_raise   = false
         clear_jobs!('response')
-        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5')
+        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6')
         @worker_class.threads_number = 1
         @worker_class.garbage_after  = 10
         silenced do
@@ -340,7 +367,8 @@ describe "Backburner::Workers::ThreadsOnFork module" do
       after do
         @templogger.close
         clear_jobs!('response')
-        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5')
+        clear_jobs!('foo.bar.1', 'foo.bar.2', 'foo.bar.3', 'foo.bar.4', 'foo.bar.5', 'foo.bar.6')
+        @worker_class.threads_number = 1
         @worker_class.shutdown = true
         silenced do
           @worker_class.stop_forks
@@ -409,8 +437,24 @@ describe "Backburner::Workers::ThreadsOnFork module" do
         assert_equal true, $worker_success
       end # retrying, succeeds
 
+      it "should support a multithreaded worker without deadlocks" do
+        num_threads = 5
+        num_jobs = 8
+        num_jobs.times do
+          @worker_class.enqueue TestJobFork, [6,2], :queue => 'foo.bar.6'
+        end
+        @worker_class.threads_number = num_threads
+        @worker = @worker_class.new('foo.bar.6')
+        @worker.start(false)
+        silenced(2) do
+          @templogger.wait_for_match(/Completed TestJobFork/m)
+          num_jobs.times { @response_worker.work_one_job }
+        end
+        assert_equal num_jobs, $worker_test_count
+      end # multithreaded
+
     end # practical tests
 
   end # forking and threading
 
-end # Backburner::Workers::ThreadsOnFork module
+end # Backburner::Workers::ThreadsOnFork module

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.4.5
 platform: ruby
 authors:
 - Nathan Esquenazi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-27 00:00:00.000000000 Z
+date: 2013-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beaneater