backburner-allq 1.0.35 → 1.0.38

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcdca608a5d89c656e3bac8e9c08a8609bd621c87a65627be816182047f72682
-  data.tar.gz: 5b22e47c0e50d030e6b512b9b592b5e754281b6152f16225539074277458071b
+  metadata.gz: 1d44569a854f2a26375e77a62b485851cc1970bd7bacce5948178c65e17a1f27
+  data.tar.gz: 29d4a2408a0d7b77c6236a2fd716a07c9ddad32e1566d5d5575c99d557fcb7bf
 SHA512:
-  metadata.gz: ef7402c39f84323438ac0f8427943143f3f7e8ab82f56dd60669147098dc38d6fab2ab84a3a12d2fc0a751f270ac6cb3723ef800aa523b67b18f2e18290f76a5
-  data.tar.gz: b5f5ba04f1bebb6e511cb4d32c3bf83f6c2fb5e0960b908a6d1ef90cbf206e39c9a7c4d9e38745ab7734de990313e24ab1142f724f71d94ef8e69f8a95c1ce33
+  metadata.gz: 9ea4a8b0da3978fe8567ecf7edbb4157246ad88da2acf9352038627e665f2240d877511e6f49f3775d42a54d61617c79dbbee2ccb71413a76797f1c77b01218d
+  data.tar.gz: 9e2be902d44f1600e86941c714c8bb09e2e902c636cba0cc34fd26a2fb901e79006e48df357877a3c3cc248966090561b37d4b10d5c6d0f207d5b7ab3be4e0e5

data/deploy.sh

@@ -1,3 +1,3 @@
 echo "Did you update the version?"
 gem build backburner-allq.gemspec
-gem push backburner-allq-1.0.35.gem
+gem push backburner-allq-1.0.38.gem

data/lib/backburner/allq_wrapper.rb

@@ -42,13 +42,17 @@ module Backburner
     # Release count
     attr_accessor :releases
 
-    def initialize(wrapper, job_resposne)
+    # Release count
+    attr_accessor :special
+
+    def initialize(wrapper, job_response)
       @client = wrapper
-      @id = job_resposne.id
-      @body = job_resposne.body
-      @expireds = job_resposne.expireds
-      @releases = job_resposne.releases
-      @tube = job_resposne.tube.to_s
+      @id = job_response.id
+      @body = job_response.body
+      @expireds = job_response.expireds
+      @releases = job_response.releases
+      @tube = job_response.tube.to_s
+      @special = job_response.special.to_s if job_response.special
     end
 
     def done
@@ -93,12 +97,6 @@ module Backburner
       @recent_times = []
     end
 
-    def speed
-      return @recent_times.sum(0.0) / @recent_times.size if @recent_times.size > 0
-
-      0
-    end
-
     def touch(job)
       @client.touch_put(job.id)
     end
@@ -167,12 +165,6 @@ module Backburner
       app_priority > 10 ? 5 : app_priority
     end
 
-    def log_result(job_result)
-      puts("ALLQ-HTTP-JOB-ID=#{job_result.job_id}")
-    rescue StandardError => e
-      puts(e)
-    end
-
     def build_new_job(body, options)
       adjusted_priority = map_priority(options[:pri] || 5)
 
@@ -195,7 +187,6 @@ module Backburner
       ttl = options[:ttl] || options[:ttr] || DEFAULT_TIMEOUT
       tube_name = options[:tube_name] || 'default'
       delay = options[:delay] || 0
-      parent_id = options[:parent_id]
       limit = options[:limit]
       timeout = options[:timeout] || 3_600
       run_on_timeout = options[:run_on_timeout] || false
@@ -254,15 +245,21 @@ module Backburner
       raw_stats = @admin.stats_get
       final_stats = {}
 
-      raw_stats.each do |agg|
+      raw_stats.each_with_index do |agg, i|
         agg.stats.each do |tube_ref|
           name = tube_ref.tube
-          final_stats[name] = {} unless final_stats[name]
+          final_stats[name] = {'avg' => 0, 'tps' => 0} unless final_stats[name]
           final_stats[name]['ready'] = final_stats[name]['ready'].to_i + tube_ref.ready.to_i
           final_stats[name]['reserved'] = final_stats[name]['reserved'].to_i + tube_ref.reserved.to_i
           final_stats[name]['delayed'] = final_stats[name]['delayed'].to_i + tube_ref.delayed.to_i
           final_stats[name]['buried'] = final_stats[name]['buried'].to_i + tube_ref.buried.to_i
           final_stats[name]['parents'] = final_stats[name]['parents'].to_i + tube_ref.parents.to_i
+
+
+          # Fancy math to calculate averages on the fly
+          # https://math.stackexchange.com/questions/106313/regular-average-calculated-accumulatively
+          final_stats[name]['avg'] += (tube_ref.avg.to_f - final_stats[name]['avg'].to_f)/(i+1).to_f
+          final_stats[name]['tps'] += (tube_ref.tps.to_f - final_stats[name]['tps'].to_f)/(i+1).to_f
         end
       end
       final_stats

data/lib/backburner/job.rb

@@ -26,6 +26,11 @@ module Backburner
       @name = body["class"] || body[:class]
       @args = body["args"] || body[:args]
       @ttr  = body["ttr"] || body[:ttr]
+      if @task.special
+        @args ||= {}
+        @args["special"] = @task.special.to_s
+      end
+
     rescue => ex # Job was not valid format
 #      self.bury
 #      raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.0.35"
+  VERSION = "1.0.38"
 end

data/lib/backburner/worker.rb

@@ -13,7 +13,10 @@ module Backburner
     # List of known_queue_classes
     class << self
       attr_writer :known_queue_classes
-      def known_queue_classes; @known_queue_classes ||= []; end
+
+      def known_queue_classes
+        @known_queue_classes ||= []
+      end
     end
 
     # Enqueues a job to be processed later by a worker.
@@ -23,8 +26,13 @@ module Backburner
     # @example
     #   Backburner::Worker.enqueue NewsletterSender, [self.id, user.id], :ttr => 1000
     #
-    def self.enqueue(job_class, args=[], opts={})
-      opts[:shard_key] = opts[:shard_key].nil? ? "X" : opts[:shard_key].to_s
+    def self.enqueue(job_class, args = [], opts = {})
+      # Invoke Procs if they are sent
+      opts.each_key do |k|
+        opts[k] = opts[k].call job_class, args if opts[k].instance_of?(Proc)
+      end
+
+      opts[:shard_key] = opts[:shard_key].nil? ? 'X' : opts[:shard_key].to_s
       pri   = resolve_priority(opts[:pri] || job_class)
       delay = [0, opts[:delay].to_i].max
       ttr   = resolve_respond_timeout(opts[:ttr] || job_class)
@@ -32,8 +40,8 @@ module Backburner
 
       return nil unless res # stop if hook is false
 
-      data = { :class => job_class.name, :args => args, :ttr => ttr }
-      queue = opts[:queue] && (Proc === opts[:queue] ? opts[:queue].call(job_class) : opts[:queue])
+      data = { class: job_class.name, args: args, ttr: ttr }
+      queue = opts[:queue] && (opts[:queue].is_a?(Proc) ? opts[:queue].call(job_class) : opts[:queue])
 
       begin
         response = nil
@@ -62,12 +70,10 @@ module Backburner
     # @example
     #   Backburner::Worker.start(["foo.tube.name"])
     #
-    def self.start(tube_names=nil)
-      begin
-        self.new(tube_names).start
-      rescue SystemExit
-        # do nothing
-      end
+    def self.start(tube_names = nil)
+      new(tube_names).start
+    rescue SystemExit
+      # do nothing
     end
 
     # List of tube names to be watched and processed
@@ -77,9 +83,9 @@ module Backburner
     #
     # @example
     #   Worker.new(['test.job'])
-    def initialize(tube_names=nil)
+    def initialize(tube_names = nil)
       @connection = new_connection
-      @tube_names = self.process_tube_names(tube_names)
+      @tube_names = process_tube_names(tube_names)
       register_signal_handlers!
     end
 
@@ -135,31 +141,30 @@ module Backburner
     # @raise [Beaneater::NotConnected] If beanstalk fails to connect multiple times.
     def work_one_job(conn = connection, tube_name = nil)
       if tube_name.nil?
-        self.log_error "Sampling tube, this is bad practice for Allq"
-        tube_name = @tube_names.sample 
+        log_error 'Sampling tube, this is bad practice for Allq'
+        tube_name = @tube_names.sample
       end
-      
+
       begin
         job = reserve_job(conn, tube_name)
       rescue Exception => e
-        self.log_error "Sleeping"
-        self.log_error "Exception: #{e.full_message}"
-        sleep(rand*3)
+        log_error "Exception: #{e.full_message}"
+        sleep(rand * 3)
         return
       end
 
       if job && job.body
         begin
-          self.log_job_begin(job.name, job.args)
+          log_job_begin(job.name, job.args)
           job.process
-          self.log_job_end(job.name)
+          log_job_end(job.name)
         rescue Backburner::Job::JobFormatInvalid => e
-          self.log_error self.exception_message(e)
-        rescue => e # Error occurred processing job
-          self.log_error self.exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
+          log_error exception_message(e)
+        rescue StandardError => e # Error occurred processing job
+          log_error exception_message(e) unless e.is_a?(Backburner::Job::RetryJob)
 
           unless job
-            self.log_error "Error occurred before we were able to assign a job. Giving up without retrying!"
+            log_error 'Error occurred before we were able to assign a job. Giving up without retrying!'
             return
           end
 
@@ -167,26 +172,29 @@ module Backburner
           # gone down between the time we reserved / processed the job and here.
           num_retries = job.releases
           max_job_retries = resolve_max_job_retries(job.job_class)
-          retry_status = "failed: attempt #{num_retries+1} of #{max_job_retries+1}"
+          retry_status = "failed: attempt #{num_retries + 1} of #{max_job_retries + 1}"
           retry_delay = resolve_retry_delay(job.job_class)
-          delay = resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries) rescue retry_delay
-          
+          delay = begin
+            resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries)
+          rescue StandardError
+            retry_delay
+          end
+
           if num_retries + 1 > max_job_retries
             job.bury
           else
             job.release(delay)
           end
-          self.log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
+          log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
 
           handle_error(e, job.name, job.args, job)
         end
       else
-        sleep(rand*3)
+        sleep(rand * 3)
       end
       job
     end
 
-
     protected
 
     # Return a new connection instance
@@ -195,9 +203,10 @@ module Backburner
     end
 
     # Reserve a job from the watched queues
-    def reserve_job(conn, tube_name, reserve_timeout = Backburner.configuration.reserve_timeout)
+    def reserve_job(conn, tube_name, _reserve_timeout = Backburner.configuration.reserve_timeout)
       job = conn.get(tube_name)
       return nil if job.nil? || job.body == nil?
+
       Backburner::Job.new(job)
     end
 
@@ -205,11 +214,10 @@ module Backburner
     # Filtered for tubes that match the known prefix
     def all_existing_queues
       known_queues    = Backburner::Worker.known_queue_classes.map(&:queue)
-      existing_tubes  = self.connection.tubes.all.map(&:name).select { |tube| tube =~ /^#{queue_config.tube_namespace}/ }
+      existing_tubes  = connection.tubes.all.map(&:name).select { |tube| tube =~ /^#{queue_config.tube_namespace}/ }
       existing_tubes + known_queues + [queue_config.primary_queue]
     end
 
-
     # Handles an error according to custom definition
     # Used when processing a job that errors out
     def handle_error(e, name, args, job)

data/test/performable_test.rb

@@ -29,15 +29,15 @@ describe "Backburner::Performable module" do
 
   describe "for async instance method" do
     it "should invoke worker enqueue" do
-      Backburner::Worker.expects(:enqueue).with(PerformableTestObj, [56, :foo, true, false], has_entries(:pri => 5000, :queue => "foo"))
-      PerformableTestObj.new.async(:pri => 5000, :queue => "foo").foo(true, false)
+      Backburner::Worker.expects(:enqueue).with(PerformableTestObj, [56, :foo, true, false], has_entries(:pri => 5000, :queue => "foo", :shard_key => "abc"))
+      PerformableTestObj.new.async(:pri => 5000, :queue => "foo", :shard_key => "abc").foo(true, false)
     end
   end # async instance
 
   describe "for async class method" do
     it "should invoke worker enqueue" do
-      Backburner::Worker.expects(:enqueue).with(PerformableTestObj, [nil, :bar, true, false], has_entries(:pri => 5000, :queue => "foo"))
-      PerformableTestObj.async(:pri => 5000, :queue => "foo").bar(true, false)
+      Backburner::Worker.expects(:enqueue).with(PerformableTestObj, [nil, :bar, true, false], has_entries(:pri => 5000, :queue => "foo", :shard_key => "abc"))
+      PerformableTestObj.async(:pri => 5000, :queue => "foo", :shard_key => "abc").bar(true, false)
     end
   end # async class
 
@@ -53,16 +53,16 @@ describe "Backburner::Performable module" do
 
   describe "for handle_asynchronously class method" do
     it "should automagically asynchronously proxy calls to the method" do
-      Backburner::Performable.handle_asynchronously(AutomagicTestObj, :qux, :pri => 5000, :queue => "qux")
+      Backburner::Performable.handle_asynchronously(AutomagicTestObj, :qux, :pri => 5000, :queue => "qux", :shard_key => "abc")
 
-      Backburner::Worker.expects(:enqueue).with(AutomagicTestObj, [56, :qux_without_async, true, false], has_entries(:pri => 5000, :queue => "qux"))
+      Backburner::Worker.expects(:enqueue).with(AutomagicTestObj, [56, :qux_without_async, true, false], has_entries(:pri => 5000, :queue => "qux", :shard_key => "abc"))
       AutomagicTestObj.new.qux(true, false)
     end
 
     it "should work for class methods, too" do
-      Backburner::Performable.handle_static_asynchronously(AutomagicTestObj, :garply, :pri => 5000, :queue => "garply")
+      Backburner::Performable.handle_static_asynchronously(AutomagicTestObj, :garply, :pri => 5000, :queue => "garply", :shard_key => "abc")
 
-      Backburner::Worker.expects(:enqueue).with(AutomagicTestObj, [nil, :garply_without_async, true, false], has_entries(:pri => 5000, :queue => "garply"))
+      Backburner::Worker.expects(:enqueue).with(AutomagicTestObj, [nil, :garply_without_async, true, false], has_entries(:pri => 5000, :queue => "garply", :shard_key => "abc"))
       AutomagicTestObj.garply(true, false)
     end
 
@@ -74,14 +74,14 @@ describe "Backburner::Performable module" do
     end
 
     it "should be available for instance methods on any class that includes the Performable module" do
-      AsyncInstanceMethodsTestObj.handle_asynchronously :foo, pri: 5000, queue: 'qux'
-      Backburner::Worker.expects(:enqueue).with(AsyncInstanceMethodsTestObj, [56, :foo_without_async, true, false], has_entries(:pri => 5000, :queue => "qux"))
+      AsyncInstanceMethodsTestObj.handle_asynchronously :foo, pri: 5000, queue: 'qux', shard_key: "abc"
+      Backburner::Worker.expects(:enqueue).with(AsyncInstanceMethodsTestObj, [56, :foo_without_async, true, false], has_entries(:pri => 5000, :queue => "qux", :shard_key => "abc"))
       AsyncInstanceMethodsTestObj.new.foo(true, false)
     end
 
     it "should be available for class methods on any class that includes the Performable module" do
-      AsyncStaticMethodsTestObj.handle_static_asynchronously :bar, pri: 5000, queue: 'garply'
-      Backburner::Worker.expects(:enqueue).with(AsyncStaticMethodsTestObj, [nil, :bar_without_async, true, false], has_entries(:pri => 5000, :queue => "garply"))
+      AsyncStaticMethodsTestObj.handle_static_asynchronously :bar, pri: 5000, queue: 'garply', shard_key: "abc"
+      Backburner::Worker.expects(:enqueue).with(AsyncStaticMethodsTestObj, [nil, :bar_without_async, true, false], has_entries(:pri => 5000, :queue => "garply",:shard_key => "abc"))
       AsyncStaticMethodsTestObj.bar(true, false)
     end
   end

data/test/test_helper.rb

@@ -12,7 +12,7 @@ require File.expand_path('../helpers/templogger', __FILE__)
 
 # Configure Backburner
 Backburner.configure do |config|
-  config.beanstalk_url = "beanstalk://127.0.0.1"
+  config.allq_url = "allq://127.0.0.1:8091"
   config.tube_namespace = "demo.test"
 end
 
@@ -88,7 +88,7 @@ class MiniTest::Spec
   end
 
   def beanstalk_connection
-    Backburner::Connection.new(Backburner.configuration.beanstalk_url)
+    Backburner::Connection.new(Backburner.configuration.allq_url)
   end
 
   # pop_one_job(tube_name)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner-allq
 version: !ruby/object:Gem::Version
-  version: 1.0.35
+  version: 1.0.38
 platform: ruby
 authors:
 - Jason Malcolm
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-29 00:00:00.000000000 Z
+date: 2022-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: allq_rest