backburner-allq 1.0.17 → 1.0.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a73b5d7a8f60e385a3f600bfded5c0579691bbafdd4bc15b0f51e0df7298f4f
-  data.tar.gz: 26000c93ce60b0f53ff666f51f7fcd1549ddde403dd400f7f4c457e056803c92
+  metadata.gz: '069b0228e7eee5979af0afe8d31601cd34400bb932009e86ffdef82e0d460dad'
+  data.tar.gz: 7aa635443fd85b5080dfe3ddf580cb8140d6c02855d7ee8642facf6b9b72300e
 SHA512:
-  metadata.gz: a0d2079019d65e529e6b769ec860456bd13b2a29caf6797adcea9c1feca0f3b397d006ac9cd3e8a85e9d78a86af4ef8964d500d5822b001f730123705d80ada3
-  data.tar.gz: 747c8436582124a553424a1e674ae97a44fee865f74720fbe5e84ca5fc6006a05087a7e1c079a6acdb72c2414c7b94db0642fefbba886734d8b575735411ecbb
+  metadata.gz: 6ad58a808c08fa4a75bf6911923fd82bb8d404cfa7f25ebcfffabddc07176fa1cf6089e0e648e6a99908b0b71fc5cf228e03f82dbdbca0d4820e11eb0d3cc90d
+  data.tar.gz: 3364c59a0d72be8a435a3e65e54a6b997c1576003155edc370d20e3a6dd1d7439420ca389d643b7d0d12c8806bc4bdced1f82c39b45e5b07168b50ad981241c6

data/Gemfile

@@ -1,4 +1,6 @@
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in backburner.gemspec
-gemspec
+gemspec
+
+

data/deploy.sh

@@ -1,3 +1,3 @@
 echo "Did you update the version?"
 gem build backburner-allq.gemspec
-gem push backburner-allq-1.0.17.gem
+gem push backburner-allq-1.0.22.gem

data/lib/backburner/allq_wrapper.rb

@@ -81,6 +81,7 @@ module Backburner
   end
 
   class AllQWrapper
+    DEFAULT_TIMEOUT = 17800
     def initialize(url = 'localhost:8090')
       allq_conf = Allq::Configuration.new do |config|
         config.host = url
@@ -182,7 +183,7 @@ module Backburner
     def build_new_job(body, options)
       adjusted_priority = map_priority(options[:pri] || 5)
 
-      ttl = options[:ttl] || 600
+      ttl = options[:ttl] || options[:ttr] || DEFAULT_TIMEOUT
       tube_name = options[:tube_name] || 'default'
       delay = options[:delay] || 0
       parent_id = options[:parent_id]
@@ -199,7 +200,7 @@ module Backburner
 
     def build_new_parent_job(body, options)
       adjusted_priority = map_priority(options[:pri] || 5)
-      ttl = options[:ttl] || 600
+      ttl = options[:ttl] || options[:ttr] || DEFAULT_TIMEOUT
       tube_name = options[:tube_name] || 'default'
       delay = options[:delay] || 0
       parent_id = options[:parent_id]

data/lib/backburner/configuration.rb

@@ -22,12 +22,12 @@ module Backburner
 
     def initialize
       @allq_url       = "allq://127.0.0.1:8091"
-      @tube_namespace      = "backallq"
+      @tube_namespace      = "backburner"
       @namespace_separator = "."
       @default_priority    = 5
       @respond_timeout     = 120
       @on_error            = nil
-      @max_job_retries     = 0
+      @max_job_retries     = 1
       @retry_delay         = 5
       @retry_delay_proc    = lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 3) }
       @default_queues      = []

data/lib/backburner/connection.rb

@@ -38,6 +38,10 @@ module Backburner
       end
     end
 
+    def tubes
+      @allq_wrapper.tube_names if @allq_wrapper
+    end
+
     # Attempt to reconnect to allq. Note: the connection will not be watching
     # or using the tubes it was before it was reconnected (as it's actually a
     # completely new connection)

data/lib/backburner/job.rb

@@ -25,9 +25,10 @@ module Backburner
       @body = task.body.is_a?(Hash) ? task.body : Backburner.configuration.job_parser_proc.call(task.body)
       @name = body["class"] || body[:class]
       @args = body["args"] || body[:args]
+      @ttr  = body["ttr"] || body[:ttr]
     rescue => ex # Job was not valid format
-      self.bury
-      raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
+#      self.bury
+#      raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
     end
 
     # Sets the delegator object to the underlying beaneater job
@@ -42,13 +43,13 @@ module Backburner
     # @example
     #   @task.process
     #
-    def process
+    def process      
       # Invoke before hook and stop if false
       res = @hooks.invoke_hook_events(job_name, :before_perform, *args)
       return false unless res
       # Execute the job
       @hooks.around_hook_events(job_name, :around_perform, *args) do
-        job_class.perform(*args)
+        timeout_job_after(@ttr > 1 ? @ttr - 1 : @ttr) { job_class.perform(*args) }
       end
       task.delete
       # Invoke after perform hook
@@ -60,12 +61,12 @@ module Backburner
 
     def bury
       @hooks.invoke_hook_events(job_name, :on_bury, *args)
-      task.bury
+      @task.bury
     end
 
     def retry(count, delay)
       @hooks.invoke_hook_events(job_name, :on_retry, count, delay, *args)
-      task.release(delay: delay)
+      @task.release(delay: delay)
     end
 
     # Returns the class for the job handler

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.0.17"
+  VERSION = "1.0.22"
 end

data/lib/backburner/worker.rb

@@ -31,7 +31,7 @@ module Backburner
 
       return nil unless res # stop if hook is false
 
-      data = { :class => job_class.name, :args => args }
+      data = { :class => job_class.name, :args => args, :ttr => ttr }
       queue = opts[:queue] && (Proc === opts[:queue] ? opts[:queue].call(job_class) : opts[:queue])
 
       begin
@@ -126,44 +126,53 @@ module Backburner
     # @example
     #   @worker.work_one_job
     # @raise [Beaneater::NotConnected] If beanstalk fails to connect multiple times.
-    def work_one_job(conn = connection)
+    def work_one_job(conn = connection, tube_name = nil)
+      if tube_name.nil?
+        self.log_error "Sampling tube, this is bad practice for Allq"
+        tube_name = @tube_names.sample 
+      end
+      
       begin
-        job = reserve_job(conn)
+        job = reserve_job(conn, tube_name)
       rescue Exception => e
+        self.log_error "Sleeping"
+        self.log_error "Exception: #{e.full_message}"
         sleep(rand*3)
         return
       end
 
-      if job
-        self.log_job_begin(job.name, job.args)
-        job.process
-        self.log_job_end(job.name)
-
-      rescue Backburner::Job::JobFormatInvalid => e
-        self.log_error self.exception_message(e)
-      rescue => e # Error occurred processing job
-        self.log_error self.exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
-
-        unless job
-          self.log_error "Error occurred before we were able to assign a job. Giving up without retrying!"
-          return
+      if job && job.body
+        begin
+          self.log_job_begin(job.name, job.args)
+          job.process
+          self.log_job_end(job.name)
+        rescue Backburner::Job::JobFormatInvalid => e
+          self.log_error self.exception_message(e)
+        rescue => e # Error occurred processing job
+          self.log_error self.exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
+
+          unless job
+            self.log_error "Error occurred before we were able to assign a job. Giving up without retrying!"
+            return
+          end
+
+          # NB: There's a slight chance here that the connection to allq has
+          # gone down between the time we reserved / processed the job and here.
+          num_retries = job.releases
+          max_job_retries = resolve_max_job_retries(job.job_class)
+          retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
+          retry_delay = resolve_retry_delay(job.job_class)
+          delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
+          
+          if num_retries + 1 > max_job_retries
+            job.bury
+          else
+            job.release(delay)
+          end
+          self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
+
+          handle_error(e, job.name, job.args, job)
         end
-
-        # NB: There's a slight chance here that the connection to allq has
-        # gone down between the time we reserved / processed the job and here.
-        num_retries = job.releases
-        max_job_retries = resolve_max_job_retries(job.job_class)
-        retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
-        retry_delay = resolve_retry_delay(job.job_class)
-        delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
-        if num_retries + 1 > max_job_retries
-          job.bury
-        else
-          job.release(delay)
-        end
-        self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
-
-        handle_error(e, job.name, job.args, job)
       end
       job
     end
@@ -177,9 +186,9 @@ module Backburner
     end
 
     # Reserve a job from the watched queues
-    def reserve_job(conn, reserve_timeout = Backburner.configuration.reserve_timeout)
-      job = conn.get(@tube_names.sample)
-      return nil if job.body == nil?
+    def reserve_job(conn, tube_name, reserve_timeout = Backburner.configuration.reserve_timeout)
+      job = conn.get(tube_name)
+      return nil if job.nil? || job.body == nil?
       Backburner::Job.new(job)
     end
 

data/lib/backburner/workers/forking.rb

@@ -11,7 +11,7 @@ module Backburner
       def prepare
         self.tube_names.map! { |name| expand_tube_name(name)  }.uniq!
         log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
-        self.connection.tubes.watch!(*self.tube_names)
+#        self.connection.tubes.watch!(*self.tube_names)
       end
 
       # Starts processing new jobs indefinitely.

data/lib/backburner/workers/simple.rb

@@ -11,7 +11,7 @@ module Backburner
       def prepare
         self.tube_names.map! { |name| expand_tube_name(name)  }.uniq!
         log_info "Working #{tube_names.size} queues: [ #{tube_names.join(', ')} ]"
-        self.connection.tubes.watch!(*self.tube_names)
+#        self.connection.tubes.watch!(*self.tube_names)
       end
 
       # Starts processing new jobs indefinitely.

data/lib/backburner/workers/threading.rb

@@ -49,8 +49,8 @@ module Backburner
         @thread_pools.each do |tube_name, pool|
           pool.max_length.times do
             # Create a new connection and set it up to listen on this tube name
-            connection = new_connection.tap{ |conn| conn.tubes.watch!(tube_name) }
-            connection.on_reconnect = lambda { |conn| conn.tubes.watch!(tube_name) }
+            # connection = new_connection.tap{ |conn| conn.tubes.watch!(tube_name) }
+            # connection.on_reconnect = lambda { |conn| conn.tubes.watch!(tube_name) }
 
             # Make it work jobs using its own connection per thread
             pool.post(connection) do |memo_connection|
@@ -58,7 +58,7 @@ module Backburner
               loop do
                 begin
                   break if @in_shutdown
-                  work_one_job(memo_connection)
+                  work_one_job(memo_connection, tube_name)
                 rescue => e
                   log_error("Exception caught in thread pool loop. Continuing. -> #{e.message}\nBacktrace: #{e.backtrace}")
                 end

data/lib/backburner/workers/threads_on_fork.rb

@@ -181,6 +181,8 @@ module Backburner
 
         @runs = 0
 
+        puts "Threads number = #{@threads_number}"
+
         if @threads_number == 1
           watch_tube(name)
           run_while_can
@@ -205,16 +207,21 @@ module Backburner
 
       # Run work_one_job while we can
       def run_while_can(conn = connection)
+        puts "Run while can"
         while @garbage_after.nil? or @garbage_after > @runs
           @runs += 1 # FIXME: Likely race condition
-          ran_job = work_one_job(conn)
+          ran_job = work_one_job(conn, @watched_tube_name)
           # Wait a second if we didn't find a job
-          sleep(rand() * 3) unless ran_job
+          unless ran_job
+            puts "sleeping"
+            sleep(rand() * 3)
+          end
         end
       end
 
       # Shortcut for watching a tube on our beanstalk connection
       def watch_tube(name, conn = connection)
+        @watched_tube_name = name
         # No op for allq
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner-allq
 version: !ruby/object:Gem::Version
-  version: 1.0.17
+  version: 1.0.22
 platform: ruby
 authors:
 - Jason Malcolm
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-13 00:00:00.000000000 Z
+date: 2021-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: allq_rest