backburner-allq 1.0.19 → 1.0.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 961c84a21c68a6bbe287105030dcbcf73e50177d647321dc410786b417ba653c
-  data.tar.gz: bee092eb96d993ee4b34b4c1e9fa671cc92e4a3316179890be130dbe14cc70f1
+  metadata.gz: ca025bad38a673fdca1c9649c0eb697f0a25f38e9b9181c609c242e2cbc694e9
+  data.tar.gz: e1a608abe130c76a3d105c5b28d221263c31c264e425306d2fb86b91c63f2b90
 SHA512:
-  metadata.gz: a09fd305b986a90330434db34e13e961f4f6878bff406348fb246f03ac0a3e0b9b8216862128c62005e5b25a4338371cb341e57133aadaa1e3187066fee8698e
-  data.tar.gz: 12fd13a7430682a5021d16ad32c7f3be1f39767413feeb2a96a7335c465d3639dfbd47905f43249aae27d23030c27a284fce36d098f7b6f396ac789182330efe
+  metadata.gz: 6e36f0d3d7c633705f9c51632b34927d1ce790888544690e1c862a1d30f3e8a783209863771a95b6c4fb20556890a1083691d6a5cde317d330248ad1c88f2bc0
+  data.tar.gz: a92844eb1114d9afec02017ce932cfc0ab7510125250a9e38908106369ed1b6613ca72a4cf2a5ce97cf4aaf3f7963a4c468aa7a0a7323dd26e3679678100d8d4

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in backburner.gemspec
 gemspec
+
+

data/deploy.sh

@@ -1,3 +1,3 @@
 echo "Did you update the version?"
 gem build backburner-allq.gemspec
-gem push backburner-allq-1.0.19.gem
+gem push backburner-allq-1.0.24.gem

data/lib/backburner.rb

@@ -29,6 +29,9 @@ module Backburner
       Backburner::Worker.enqueue(job_class, args, {})
     end
 
+    def enqueue(job_class, args, shard_key = nil)
+      enqueue(job_class, args, { shard_key: shard_key.nil? ? "X" : shard_key.to_s })
+    end
     # Begins working on jobs enqueued with optional tubes specified
     #
     # @example

data/lib/backburner/allq_wrapper.rb

@@ -81,6 +81,7 @@ module Backburner
   end
 
   class AllQWrapper
+    DEFAULT_TIMEOUT = 17800
     def initialize(url = 'localhost:8090')
       allq_conf = Allq::Configuration.new do |config|
         config.host = url
@@ -182,7 +183,7 @@ module Backburner
     def build_new_job(body, options)
       adjusted_priority = map_priority(options[:pri] || 5)
 
-      ttl = options[:ttl] || 600
+      ttl = options[:ttl] || options[:ttr] || DEFAULT_TIMEOUT
       tube_name = options[:tube_name] || 'default'
       delay = options[:delay] || 0
       parent_id = options[:parent_id]
@@ -199,7 +200,7 @@ module Backburner
 
     def build_new_parent_job(body, options)
       adjusted_priority = map_priority(options[:pri] || 5)
-      ttl = options[:ttl] || 600
+      ttl = options[:ttl] || options[:ttr] || DEFAULT_TIMEOUT
       tube_name = options[:tube_name] || 'default'
       delay = options[:delay] || 0
       parent_id = options[:parent_id]

data/lib/backburner/configuration.rb

@@ -22,7 +22,7 @@ module Backburner
 
     def initialize
       @allq_url       = "allq://127.0.0.1:8091"
-      @tube_namespace      = "backallq"
+      @tube_namespace      = "backburner"
       @namespace_separator = "."
       @default_priority    = 5
       @respond_timeout     = 120

data/lib/backburner/job.rb

@@ -25,6 +25,7 @@ module Backburner
       @body = task.body.is_a?(Hash) ? task.body : Backburner.configuration.job_parser_proc.call(task.body)
       @name = body["class"] || body[:class]
       @args = body["args"] || body[:args]
+      @ttr  = body["ttr"] || body[:ttr]
     rescue => ex # Job was not valid format
 #      self.bury
 #      raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
@@ -42,13 +43,13 @@ module Backburner
     # @example
     #   @task.process
     #
-    def process
+    def process      
       # Invoke before hook and stop if false
       res = @hooks.invoke_hook_events(job_name, :before_perform, *args)
       return false unless res
       # Execute the job
       @hooks.around_hook_events(job_name, :around_perform, *args) do
-        job_class.perform(*args)
+        timeout_job_after(@ttr > 1 ? @ttr - 1 : @ttr) { job_class.perform(*args) }
       end
       task.delete
       # Invoke after perform hook

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.0.19"
+  VERSION = "1.0.24"
 end

data/lib/backburner/worker.rb

@@ -31,7 +31,7 @@ module Backburner
 
       return nil unless res # stop if hook is false
 
-      data = { :class => job_class.name, :args => args }
+      data = { :class => job_class.name, :args => args, :ttr => ttr }
       queue = opts[:queue] && (Proc === opts[:queue] ? opts[:queue].call(job_class) : opts[:queue])
 
       begin
@@ -126,9 +126,14 @@ module Backburner
     # @example
     #   @worker.work_one_job
     # @raise [Beaneater::NotConnected] If beanstalk fails to connect multiple times.
-    def work_one_job(conn = connection)
+    def work_one_job(conn = connection, tube_name = nil)
+      if tube_name.nil?
+        self.log_error "Sampling tube, this is bad practice for Allq"
+        tube_name = @tube_names.sample 
+      end
+      
       begin
-        job = reserve_job(conn)
+        job = reserve_job(conn, tube_name)
       rescue Exception => e
         self.log_error "Sleeping"
         self.log_error "Exception: #{e.full_message}"
@@ -158,8 +163,6 @@ module Backburner
           retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
           retry_delay = resolve_retry_delay(job.job_class)
           delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
-          puts "num_retries = #{num_retries}"
-          puts "max_job_retries = #{max_job_retries}"
           
           if num_retries + 1 > max_job_retries
             job.bury
@@ -185,8 +188,8 @@ module Backburner
     end
 
     # Reserve a job from the watched queues
-    def reserve_job(conn, reserve_timeout = Backburner.configuration.reserve_timeout)
-      job = conn.get(@tube_names.sample)
+    def reserve_job(conn, tube_name, reserve_timeout = Backburner.configuration.reserve_timeout)
+      job = conn.get(tube_name)
       return nil if job.nil? || job.body == nil?
       Backburner::Job.new(job)
     end

data/lib/backburner/workers/threading.rb

@@ -58,7 +58,7 @@ module Backburner
               loop do
                 begin
                   break if @in_shutdown
-                  work_one_job(memo_connection)
+                  work_one_job(memo_connection, tube_name)
                 rescue => e
                   log_error("Exception caught in thread pool loop. Continuing. -> #{e.message}\nBacktrace: #{e.backtrace}")
                 end

data/lib/backburner/workers/threads_on_fork.rb

@@ -210,17 +210,13 @@ module Backburner
         puts "Run while can"
         while @garbage_after.nil? or @garbage_after > @runs
           @runs += 1 # FIXME: Likely race condition
-          ran_job = work_one_job(conn)
-          # Wait a second if we didn't find a job
-          unless ran_job
-            puts "sleeping"
-            sleep(rand() * 3)
-          end
+          work_one_job(conn, @watched_tube_name)
         end
       end
 
       # Shortcut for watching a tube on our beanstalk connection
       def watch_tube(name, conn = connection)
+        @watched_tube_name = name
         # No op for allq
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner-allq
 version: !ruby/object:Gem::Version
-  version: 1.0.19
+  version: 1.0.24
 platform: ruby
 authors:
 - Jason Malcolm
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-13 00:00:00.000000000 Z
+date: 2021-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: allq_rest