backburner-allq 1.0.38 → 1.0.40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1d44569a854f2a26375e77a62b485851cc1970bd7bacce5948178c65e17a1f27
-  data.tar.gz: 29d4a2408a0d7b77c6236a2fd716a07c9ddad32e1566d5d5575c99d557fcb7bf
+  metadata.gz: 682416738c55b9567c39530b31bfc92a8d9115b51c640494b067c9ff23cca65d
+  data.tar.gz: 9239e634bea3dc41131c9a4deb56d5998b6f4745e891c43a90fad4c569008c0c
 SHA512:
-  metadata.gz: 9ea4a8b0da3978fe8567ecf7edbb4157246ad88da2acf9352038627e665f2240d877511e6f49f3775d42a54d61617c79dbbee2ccb71413a76797f1c77b01218d
-  data.tar.gz: 9e2be902d44f1600e86941c714c8bb09e2e902c636cba0cc34fd26a2fb901e79006e48df357877a3c3cc248966090561b37d4b10d5c6d0f207d5b7ab3be4e0e5
+  metadata.gz: 127ae4b11fcc7651aeed81e9070a791460b2256fb2d308b559e70e9c4ba767d6c8cce5feed450e783c22415f84757e248d4b70de3fb27612317fb14283794b60
+  data.tar.gz: 06e8e765a1f8a6fd55ce3c0405bd319e797c59d87f5f54cebdee9bba2804d888cd8f4765bb8e02935d5b96dde7c38a756df073610106d8df8243f5dc16694f7f

data/.gitlab-ci.yml

@@ -0,0 +1,21 @@
+default:
+  image: ruby:3.0
+    
+stages:          # List of stages for jobs, and their order of execution
+  - test
+
+unit-test:  
+  tags: 
+    - allq
+  stage: test
+  script:
+    - docker-compose down || true
+    - docker-compose up -d || true
+    - sleep 10
+    - ruby -v
+    - bundle install
+    - bundle exec rake test TEST=test/allq/back_burner_test.rb DEBUG=true
+    - bundle exec rake test TEST=test/allq/worker_test.rb DEBUG=true
+  after_script:
+    - docker-compose down || true
+    

data/deploy.sh

@@ -1,3 +1,3 @@
 echo "Did you update the version?"
 gem build backburner-allq.gemspec
-gem push backburner-allq-1.0.38.gem
+gem push backburner-allq-1.0.40.gem

data/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '3'
+services:
+  allq-client:
+    networks:
+      - local-net
+    image: "blitline/allq_client"
+    environment:
+      - SERVER_STRING=allq-server:5555
+    depends_on:
+      - allq-server
+    ports:
+      - 8090:8090
+    restart: unless-stopped
+    healthcheck:
+      test: ["CMD-SHELL", "echo '{\"action\":\"peek\",\"params\":{\"tube\":\"default\", \"buried\":true}}' | socat - tcp4-connect:127.0.0.1:7768"]
+      interval: 1m
+      timeout: 10s
+      retries: 3
+  allq-server:
+    networks:
+      - local-net
+    image: "blitline/allq_server"
+    restart: unless-stopped
+networks:
+  local-net:
+

data/lib/backburner/allq_wrapper.rb

@@ -82,6 +82,10 @@ module Backburner
     def stats
       { 'expireds' => expireds, 'releases' => releases }
     end
+
+    def clear(tube)
+      @client.clear(tube)
+    end
   end
 
   class AllQWrapper
@@ -97,6 +101,10 @@ module Backburner
       @recent_times = []
     end
 
+    def clear(tube)
+       @client.tube_delete(tube)
+    end
+
     def touch(job)
       @client.touch_put(job.id)
     end

data/lib/backburner/configuration.rb

@@ -21,7 +21,7 @@ module Backburner
     attr_accessor :job_parser_proc     # proc to parse a job body from a string
 
     def initialize
-      @allq_url       = "allq://127.0.0.1:8091"
+      @allq_url       = "allq://127.0.0.1:8090"
       @tube_namespace      = "backburner"
       @namespace_separator = "."
       @default_priority    = 5

data/lib/backburner/connection.rb

@@ -23,6 +23,10 @@ module Backburner
       connect!
     end
 
+    def clear(tube)
+      @allq_wrapper.clear(tube)
+    end
+
     # Close the connection, if it exists
     def close
       @allq_wrapper.close if @allq_wrapper
@@ -110,6 +114,11 @@ module Backburner
       }
 
       options[:shard_key] = opt[:shard_key] if opt[:shard_key]
+      options[:parent_id] = opt[:parent_id] if opt[:parent_id]
+      options[:timeout] = opt[:timeout] if opt[:timeout]
+      options[:run_on_timeout] = opt[:run_on_timeout] if opt[:run_on_timeout]
+      options[:limit] = opt[:limit] if opt[:limit]
+      options[:is_parent] = opt[:is_parent] if opt[:is_parent]
 
       # Overwrite originals
       opt.merge!(options)

data/lib/backburner/job.rb

@@ -26,11 +26,6 @@ module Backburner
       @name = body["class"] || body[:class]
       @args = body["args"] || body[:args]
       @ttr  = body["ttr"] || body[:ttr]
-      if @task.special
-        @args ||= {}
-        @args["special"] = @task.special.to_s
-      end
-
     rescue => ex # Job was not valid format
 #      self.bury
 #      raise JobFormatInvalid, "Job body could not be parsed: #{ex.inspect}"
@@ -54,9 +49,28 @@ module Backburner
       return false unless res
       # Execute the job
       @hooks.around_hook_events(job_name, :around_perform, *args) do
-        timeout_job_after(@ttr > 1 ? @ttr - 1 : @ttr) { job_class.perform(*args) }
+
+        timeout_job_after(@ttr > 1 ? @ttr - 1 : @ttr) do
+          # Decide if we want access to the root task
+          if job_class.methods.include?(:perform_with_task)
+            job_class.perform_with_task(@task, *args)
+          else
+           job_class.perform(*args)
+          end
+        end
+      end
+
+      retry_count = 0
+      begin
+        task.delete
+      rescue StandardError => allqex
+        retry_count += 1
+
+        puts "Failed to delete task, waiting #{retry_count * 10}secs"
+        sleep(retry_count * 10)
+        retry if retry_count < 3
       end
-      task.delete
+
       # Invoke after perform hook
       @hooks.invoke_hook_events(job_name, :after_perform, *args)
     rescue => e

data/lib/backburner/version.rb

@@ -1,3 +1,3 @@
 module Backburner
-  VERSION = "1.0.38"
+  VERSION = "1.0.40"
 end

data/lib/backburner/worker.rb

@@ -168,25 +168,6 @@ module Backburner
             return
           end
 
-          # NB: There's a slight chance here that the connection to allq has
-          # gone down between the time we reserved / processed the job and here.
-          num_retries = job.releases
-          max_job_retries = resolve_max_job_retries(job.job_class)
-          retry_status = "failed: attempt #{num_retries + 1} of #{max_job_retries + 1}"
-          retry_delay = resolve_retry_delay(job.job_class)
-          delay = begin
-            resolve_retry_delay_proc(job.job_class).call(retry_delay, num_retries)
-          rescue StandardError
-            retry_delay
-          end
-
-          if num_retries + 1 > max_job_retries
-            job.bury
-          else
-            job.release(delay)
-          end
-          log_job_end(job.name, "#{retry_status}, retrying in #{delay}s") if job_started_at
-
           handle_error(e, job.name, job.args, job)
         end
       else

data/test/allq/back_burner_test.rb

@@ -0,0 +1,74 @@
+require File.expand_path('test_helper', __dir__)
+
+$backburner_sum = 0
+$backburner_numbers = []
+
+class TestBackburnerJob
+  include Backburner::Queue
+  queue 'test.jobber'
+
+  def self.perform(value, number)
+    $backburner_sum += value
+    $backburner_numbers << number
+  end
+end
+
+class TestWorker < Backburner::Worker; end
+
+describe 'Backburner module' do
+  before do
+    Backburner.default_queues.clear
+    clear_jobs!(Backburner.configuration.primary_queue, 'test-plain', 'parent-plain', 'bar.baz.foo')
+  end
+
+  describe 'for work method' do
+    it 'invokes worker simple start' do
+      Backburner::Workers::Simple.expects(:start).with(%w[foo bar])
+      Backburner.work('foo', 'bar')
+    end
+
+    it 'invokes other worker if specified in configuration' do
+      Backburner.configure { |config| config.default_worker = TestWorker }
+      TestWorker.expects(:start).with(%w[foo bar])
+      Backburner.work('foo', 'bar')
+    end
+
+    it 'invokes other worker if specified in work method as options' do
+      TestWorker.expects(:start).with(%w[foo bar])
+      Backburner.work('foo', 'bar', worker: TestWorker)
+    end
+
+    it 'invokes worker start with no args' do
+      Backburner::Workers::Simple.expects(:start).with([])
+      Backburner.work
+    end
+  end # work!
+
+  describe 'for configuration' do
+    it 'remembers the tube_namespace' do
+      assert_equal 'demo.test', Backburner.configuration.tube_namespace
+    end
+
+    it 'remembers the namespace_separator' do
+      assert_equal '.', Backburner.configuration.namespace_separator
+    end
+
+    it 'disallows a reserved separator' do
+      assert_raises RuntimeError do
+        Backburner.configuration.namespace_separator = ':'
+      end
+    end
+  end # configuration
+
+  describe 'for default_queues' do
+    it 'supports assignment' do
+      Backburner.default_queues << 'foo'
+      Backburner.default_queues << 'bar'
+      assert_same_elements %w[foo bar], Backburner.default_queues
+    end
+  end
+
+  after do
+    Backburner.configure { |config| config.default_worker = Backburner::Workers::Simple }
+  end
+end # Backburner

data/test/allq/test_helper.rb

@@ -0,0 +1,108 @@
+$LOAD_PATH.unshift File.expand_path("lib")
+require 'backburner'
+require 'tempfile'
+require 'minitest/autorun'
+require_relative '../fixtures/test_jobs'
+
+begin
+  require 'mocha/setup'
+rescue LoadError
+  require 'mocha'
+end
+
+# Configure Backburner
+Backburner.configure do |config|
+  config.allq_url = "allq://127.0.0.1:8090"
+  config.tube_namespace = "demo.test"
+  config.on_error            = lambda { |e| puts e }
+end
+
+def clear_jobs!(*tube_names)
+  connection = allq_connection
+  tube_names.each do |tube_name|
+    expanded_name = [Backburner.configuration.tube_namespace, tube_name].join(".")
+    connection.clear(expanded_name)
+  end
+ensure
+  connection.close if connection
+end
+
+def allq_connection
+  Backburner::Connection.new(Backburner.configuration.allq_url)
+end
+
+module Kernel
+  # Redirect standard out, standard error and the buffered logger for sprinkle to StringIO
+  # capture_stdout { any_commands; you_want } => "all output from the commands"
+  def capture_stdout
+    if ENV['DEBUG'] # Skip if debug mode
+      yield
+      return ""
+    end
+
+    out = StringIO.new
+    $stdout = out
+    $stderr = out
+    yield
+    return out.string
+  ensure
+    $stdout = STDOUT
+    $stderr = STDERR
+  end
+end
+
+def sleeper(duration_in_seconds)
+  1.upto(duration_in_seconds) do |i|
+    sleep(1)
+    print '.'
+  end
+end
+
+# pop_one_job(tube_name)
+def pop_one_job(tube_name=Backburner.configuration.primary_queue)
+  tube_name  = [Backburner.configuration.tube_namespace, tube_name].join(".")
+  connection = allq_connection
+  silenced(3) { @res = connection.get(tube_name)}
+  data = nil
+  data = Backburner.configuration.job_parser_proc.call(@res.body) if @res && @res.body
+  yield @res, data
+ensure
+  connection.close if connection
+end
+
+# def work_one_job(tube_name=Backburner.configuration.primary_queue)
+#   tube_name  = [Backburner.configuration.tube_namespace, tube_name].join(".")
+#   connection = allq_connection
+#   job = Backburner::Worker.work_one_job(connection, tube_name)
+#   job.delete
+# end
+
+def assert_same_elements(a1, a2, msg = nil)
+  [:select, :inject, :size].each do |m|
+    [a1, a2].each {|a| assert_respond_to(a, m, "Are you sure that #{a.inspect} is an array?  It doesn't respond to #{m}.") }
+  end
+
+  assert a1h = a1.inject({}) { |h,e| h[e] ||= a1.select { |i| i == e }.size; h }
+  assert a2h = a2.inject({}) { |h,e| h[e] ||= a2.select { |i| i == e }.size; h }
+
+  assert_equal(a1h, a2h, msg)
+end
+
+#   assert_contains(['a', '1'], /\d/) => passes
+#   assert_contains(['a', '1'], 'a') => passes
+#   assert_contains(['a', '1'], /not there/) => fails
+def assert_contains(collection, x, extra_msg = "")
+  collection = [collection] unless collection.is_a?(Array)
+  msg = "#{x.inspect} not found in #{collection.to_a.inspect} #{extra_msg}"
+  case x
+  when Regexp
+    assert(collection.detect { |e| e =~ x }, msg)
+  else
+    assert(collection.include?(x), msg)
+  end
+end
+
+# silenced(5) { ... }
+def silenced(time=3, &block)
+  Timeout::timeout(time) { capture_stdout(&block) }
+end

data/test/allq/worker_test.rb

@@ -0,0 +1,82 @@
+require File.expand_path('test_helper', __dir__)
+
+describe 'Backburner::Worker module' do
+  before do
+    Backburner.default_queues.clear
+    clear_jobs!(Backburner.configuration.primary_queue, 'test-plain', 'parent-plain', 'bar.baz.foo')
+  end
+
+  it 'should support enqueuing plain job' do
+    Backburner::Worker.enqueue TestPlainJob, [7, 9], { ttr: 144 }
+    pop_one_job('test-plain') do |_job, body|
+      assert_equal 'TestPlainJob', body['class']
+      assert_equal [7, 9], body['args']
+      assert_equal 144, body['ttr'].to_i
+    end
+  end # plain
+
+  it 'should support enqueuing parent job' do
+    options = {
+      is_parent: true,
+      limit: 1,
+      timeout: 3,
+      run_on_timeout: true
+    }
+    # Enqueue parent job
+    response = Backburner::Worker.enqueue TestParentJob, [17, 9], options
+
+    options = {
+      parent_id: response.job_id
+    }
+    # Enqueue child job
+    Backburner::Worker.enqueue TestPlainJob, [7, 9], options
+    pop_one_job('test-plain') do |job, body|
+      assert_equal 'TestPlainJob', body['class']
+      assert_equal [7, 9], body['args']
+      job.delete
+    end
+
+    pop_one_job('parent-plain') do |_job, body|
+      assert_equal [17, 9], body['args']
+    end
+  end
+
+  it 'should support enqueuing parent job with timeout' do
+    options = {
+      is_parent: true,
+      limit: 1,
+      timeout: 3,
+      run_on_timeout: true
+    }
+    # Enqueue parent job
+    response = Backburner::Worker.enqueue TestParentJob, [17, 9], options
+
+    options = {
+      parent_id: response.job_id
+    }
+    # Enqueue child job
+    Backburner::Worker.enqueue TestPlainJob, [7, 9], options
+
+    pop_one_job('parent-plain') do |_job, body|
+      assert_equal body, nil
+      sleeper(10)
+    end
+
+    worker = Backburner::Workers::Simple.new('demo.test.parent-plain')
+    worker.prepare
+
+    TestParentJob.expects(:perform_with_task)
+    job = worker.work_one_job(allq_connection, 'demo.test.parent-plain')
+    assert_equal job.special, "true"
+  end 
+
+
+
+end
+
+
+      # options[:parent_id] = opt[:parent_id] if opt[:parent_id]
+      # options[:timeout] = opt[:timeout] if opt[:timeout]
+      # options[:run_on_timeout] = opt[:run_on_timeout] if opt[:run_on_timeout]
+      # options[:limit] = opt[:limit] if opt[:limit]
+      # options[:is_parent] = opt[:is_parent] if opt[:is_parent]

data/test/fixtures/test_jobs.rb

@@ -6,6 +6,18 @@ class TestPlainJob
   def self.perform(x, y); $worker_test_count += x + y + 1; end
 end
 
+
+class TestParentJob
+  def self.queue; "parent-plain"; end
+  def self.perform(args)
+    puts "plain perform #{args}"
+  end
+  def self.perform_with_task(task, *args)
+    puts "perform_with_task"
+    puts task.inspect
+  end
+end
+
 class TestJob
   include Backburner::Queue
   queue_priority :medium

data/test/worker_test.rb

@@ -1,6 +1,4 @@
 require File.expand_path('../test_helper', __FILE__)
-require File.expand_path('../fixtures/test_jobs', __FILE__)
-require File.expand_path('../fixtures/hooked', __FILE__)
 
 describe "Backburner::Worker module" do
   before do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backburner-allq
 version: !ruby/object:Gem::Version
-  version: 1.0.38
+  version: 1.0.40
 platform: ruby
 authors:
 - Jason Malcolm
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-07-27 00:00:00.000000000 Z
+date: 2022-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: allq_rest
@@ -109,6 +109,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".gitlab-ci.yml"
 - ".travis.yml"
 - CHANGELOG.md
 - CONTRIBUTING.md
@@ -122,6 +123,7 @@ files:
 - bin/backburner
 - circle.yml
 - deploy.sh
+- docker-compose.yml
 - examples/custom.rb
 - examples/demo.rb
 - examples/god.rb
@@ -148,6 +150,9 @@ files:
 - lib/backburner/workers/simple.rb
 - lib/backburner/workers/threading.rb
 - lib/backburner/workers/threads_on_fork.rb
+- test/allq/back_burner_test.rb
+- test/allq/test_helper.rb
+- test/allq/worker_test.rb
 - test/async_proxy_test.rb
 - test/back_burner_test.rb
 - test/connection_test.rb
@@ -193,6 +198,9 @@ signing_key:
 specification_version: 4
 summary: Reliable allq background job processing made easy for Ruby and Sinatra
 test_files:
+- test/allq/back_burner_test.rb
+- test/allq/test_helper.rb
+- test/allq/worker_test.rb
 - test/async_proxy_test.rb
 - test/back_burner_test.rb
 - test/connection_test.rb