backburner-allq 1.0.13 → 1.0.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01bd4bf1b3f1e5905844d23996a0529b08baa629cd46883dbb144feca42e527c
-  data.tar.gz: c77846b06501f94b118fc9000a90d98b6fdfa7c7b91bd2af2e36b29fde35f556
+  metadata.gz: 961c84a21c68a6bbe287105030dcbcf73e50177d647321dc410786b417ba653c
+  data.tar.gz: bee092eb96d993ee4b34b4c1e9fa671cc92e4a3316179890be130dbe14cc70f1
 SHA512:
-  metadata.gz: '086c4e08ff55f7c9754f668a97c9003d8d01118c39b527bfc871cf42abbac8a33df9f739ab0836f9f19cbf9f4a8058292170cc8317c18e273cef327129aab7f9'
-  data.tar.gz: 1b3854c17f9c411d0d7584c94c2d3fe8e7b92ea5ea81d3f5ae3439fce27a8f1aad9f582922c009a15facb82a7c780c3123795f94254cf3ca0f44f3300e92d9e9
+  metadata.gz: a09fd305b986a90330434db34e13e961f4f6878bff406348fb246f03ac0a3e0b9b8216862128c62005e5b25a4338371cb341e57133aadaa1e3187066fee8698e
+  data.tar.gz: 12fd13a7430682a5021d16ad32c7f3be1f39767413feeb2a96a7335c465d3639dfbd47905f43249aae27d23030c27a284fce36d098f7b6f396ac789182330efe

data/Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in backburner.gemspec
-gemspec
+gemspec

data/deploy.sh

@@ -1,3 +1,3 @@
 echo "Did you update the version?"
 gem build backburner-allq.gemspec
-gem push backburner-allq-1.0.13.gem
+gem push backburner-allq-1.0.19.gem

data/lib/backburner/allq_wrapper.rb

@@ -13,14 +13,16 @@ module Backburner
 
     def watch
       Thread.new do
-        ran = false
-        job = @allq_wrapper.get(@tube_name)
-        if job.body
-          perform(job)
-          ran = true
+        loop do
+          ran = false
+          job = @allq_wrapper.get(@tube_name)
+          if job.body
+            perform(job)
+            ran = true
+          end
+          # Wait if nothing returned
+          sleep(rand() * 3) unless ran
         end
-        # Wait if nothing returned
-        sleep(rand() * 3) unless ran
       end
     end
   end
@@ -123,6 +125,10 @@ module Backburner
       stats_hash.keys
     end
 
+    def tubes
+      tube_names
+    end
+
     def peek_buried(tube_name = 'default')
       job = nil
       job = @client.peek_get(tube_name, buried: true)
@@ -250,6 +256,11 @@ module Backburner
       result
     end
 
+    def stats(tube)
+      final_stats = stats
+      final_stats[tube]
+    end
+
     def stats
       raw_stats = @admin.stats_get
       final_stats = {}

data/lib/backburner/configuration.rb

@@ -27,7 +27,7 @@ module Backburner
       @default_priority    = 5
       @respond_timeout     = 120
       @on_error            = nil
-      @max_job_retries     = 0
+      @max_job_retries     = 1
       @retry_delay         = 5
       @retry_delay_proc    = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) }
       @default_queues      = []

data/lib/backburner/connection.rb

@@ -38,6 +38,10 @@ module Backburner
       end
     end
 
+    def tubes
+      @allq_wrapper.tube_names if @allq_wrapper
+    end
+
     # Attempt to reconnect to allq. Note: the connection will not be watching
     # or using the tubes it was before it was reconnected (as it's actually a
     # completely new connection)

data/lib/backburner/job.rb

@@ -26,8 +26,8 @@ module Backburner
       @name = body["class"] || body[:class]
       @args = body["args"] || body[:args]
     rescue => ex # Job was not valid format
-      self.bury
-      raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
+#      self.bury
+#      raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
     end
 
     # Sets the delegator object to the underlying beaneater job
@@ -60,12 +60,12 @@ module Backburner
 
     def bury
       @hooks.invoke_hook_events(job_name, :on_bury, *args)
-      task.bury
+      @task.bury
     end
 
     def retry(count, delay)
       @hooks.invoke_hook_events(job_name, :on_retry, count, delay, *args)
-      task.release(delay: delay)
+      @task.release(delay: delay)
     end
 
     # Returns the class for the job handler

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.0.13"
+  VERSION = "1.0.19"
 end

data/lib/backburner/worker.rb

@@ -130,39 +130,50 @@ module Backburner
       begin
         job = reserve_job(conn)
       rescue Exception => e
+        self.log_error "Sleeping"
+        self.log_error "Exception: #{e.full_message}"
         sleep(rand*3)
         return
       end
 
-      self.log_job_begin(job.name, job.args)
-      job.process
-      self.log_job_end(job.name)
-
-    rescue Backburner::Job::JobFormatInvalid => e
-      self.log_error self.exception_message(e)
-    rescue => e # Error occurred processing job
-      self.log_error self.exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
-
-      unless job
-        self.log_error "Error occurred before we were able to assign a job. Giving up without retrying!"
-        return
-      end
-
-      # NB: There's a slight chance here that the connection to allq has
-      # gone down between the time we reserved / processed the job and here.
-      num_retries = job.releases
-      max_job_retries = resolve_max_job_retries(job.job_class)
-      retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
-      retry_delay = resolve_retry_delay(job.job_class)
-      delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
-      if num_retries + 1 > max_job_retries
-        job.bury
+      if job && job.body
+        begin
+          self.log_job_begin(job.name, job.args)
+          job.process
+          self.log_job_end(job.name)
+        rescue Backburner::Job::JobFormatInvalid => e
+          self.log_error self.exception_message(e)
+        rescue => e # Error occurred processing job
+          self.log_error self.exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
+
+          unless job
+            self.log_error "Error occurred before we were able to assign a job. Giving up without retrying!"
+            return
+          end
+
+          # NB: There's a slight chance here that the connection to allq has
+          # gone down between the time we reserved / processed the job and here.
+          num_retries = job.releases
+          max_job_retries = resolve_max_job_retries(job.job_class)
+          retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
+          retry_delay = resolve_retry_delay(job.job_class)
+          delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
+          puts "num_retries = #{num_retries}"
+          puts "max_job_retries = #{max_job_retries}"
+          
+          if num_retries + 1 > max_job_retries
+            job.bury
+          else
+            job.release(delay)
+          end
+          self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
+
+          handle_error(e, job.name, job.args, job)
+        end
       else
-        job.release(delay)
+        sleep(rand*3)
       end
-      self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
-
-      handle_error(e, job.name, job.args, job)
+      job
     end
 
 
@@ -175,7 +186,9 @@ module Backburner
 
     # Reserve a job from the watched queues
     def reserve_job(conn, reserve_timeout = Backburner.configuration.reserve_timeout)
-      Backburner::Job.new(conn.get(@tube_names.sample))
+      job = conn.get(@tube_names.sample)
+      return nil if job.nil? || job.body == nil?
+      Backburner::Job.new(job)
     end
 
     # Returns a list of all tubes known within the system

data/lib/backburner/workers/forking.rb

@@ -11,7 +11,7 @@ module Backburner
       def prepare
         self.tube_names.map! { |name| expand_tube_name(name)  }.uniq!
         log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
-        self.connection.tubes.watch!(*self.tube_names)
+#        self.connection.tubes.watch!(*self.tube_names)
       end
 
       # Starts processing new jobs indefinitely.

data/lib/backburner/workers/simple.rb

@@ -11,7 +11,7 @@ module Backburner
       def prepare
         self.tube_names.map! { |name| expand_tube_name(name)  }.uniq!
         log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
-        self.connection.tubes.watch!(*self.tube_names)
+#        self.connection.tubes.watch!(*self.tube_names)
       end
 
       # Starts processing new jobs indefinitely.

data/lib/backburner/workers/threading.rb

@@ -49,8 +49,8 @@ module Backburner
         @thread_pools.each do |tube_name, pool|
           pool.max_length.times do
             # Create a new connection and set it up to listen on this tube name
-            connection = new_connection.tap{ |conn| conn.tubes.watch!(tube_name) }
-            connection.on_reconnect = lambda { |conn| conn.tubes.watch!(tube_name) }
+            # connection = new_connection.tap{ |conn| conn.tubes.watch!(tube_name) }
+            # connection.on_reconnect = lambda { |conn| conn.tubes.watch!(tube_name) }
 
             # Make it work jobs using its own connection per thread
             pool.post(connection) do |memo_connection|

data/lib/backburner/workers/threads_on_fork.rb

@@ -181,6 +181,8 @@ module Backburner
 
         @runs = 0
 
+        puts "Threads number = #{@threads_number}"
+
         if @threads_number == 1
           watch_tube(name)
           run_while_can
@@ -205,9 +207,15 @@ module Backburner
 
       # Run work_one_job while we can
       def run_while_can(conn = connection)
+        puts "Run while can"
         while @garbage_after.nil? or @garbage_after > @runs
           @runs += 1 # FIXME: Likely race condition
-          work_one_job(conn)
+          ran_job = work_one_job(conn)
+          # Wait a second if we didn't find a job
+          unless ran_job
+            puts "sleeping"
+            sleep(rand() * 3)
+          end
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner-allq
 version: !ruby/object:Gem::Version
-  version: 1.0.13
+  version: 1.0.19
 platform: ruby
 authors:
 - Jason Malcolm
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-23 00:00:00.000000000 Z
+date: 2021-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: allq_rest