backburner 1.8.0 → 1.9.0

data/examples/simple.rb

@@ -1,43 +0,0 @@
-$:.unshift "lib"
-require 'backburner'
-
-class User
-  include Backburner::Performable
-  attr_accessor :id, :name
-
-  def self.first
-    User.find(3, "John")
-  end
-
-  def self.find(id, name="Fetched")
-    User.new(id, name)
-  end
-
-  def initialize(id, name)
-    @id, @name = id, name
-  end
-
-  def hello(x, y)
-    puts "User(id=#{id}) #hello args: [#{x}, #{y}] (Instance method)"
-  end
-
-  def self.foo(x, y)
-    puts "User #foo args [#{x}, #{y}] (Class method)"
-  end
-end
-
-# Configure Backburner
-Backburner.configure do |config|
-  config.beanstalk_url = "beanstalk://127.0.0.1"
-  config.tube_namespace = "demo.production"
-  config.on_error = lambda { |e| puts "HEY!!! #{e.class}" }
-end
-
-# Enqueue tasks
-@user = User.first
-@user.async(:pri => 1000).hello("foo", "bar")
-User.async.foo("bar", "baz")
-
-# Run work
-# Backburner.default_queues << "user"
-Backburner.work

data/examples/stress.rb

@@ -1,31 +0,0 @@
-$:.unshift "lib"
-require 'backburner'
-
-$values = []
-
-# Define ruby job
-class TestJob
-  include Backburner::Queue
-  queue "test-job"
-
-  def self.perform(value)
-    puts "[TestJob] Running perform with args: [#{value}]"
-    $values << value
-    puts "#{$values.size} total jobs processed"
-  end
-end
-
-# Configure Backburner
-Backburner.configure do |config|
-  config.beanstalk_url = "beanstalk://127.0.0.1"
-  config.tube_namespace = "demo.production"
-end
-
-# Enqueue tasks
-1.upto(1000) do |i|
-  Backburner.enqueue TestJob, i
-end
-
-# Work tasks using threads_on_fork worker
-# twitter tube will have 10 threads, garbage after 1000 executions and retry jobs 1 times.
-Backburner.work("test-job:10:100:1", :worker => Backburner::Workers::ThreadsOnFork)

data/test/active_job_adapter_test.rb

@@ -1,24 +0,0 @@
-require File.expand_path('../test_helper', __FILE__)
-require 'open3'
-require 'rbconfig'
-
-describe "Backburner as the Active Job adapter" do
-  %w(active_job_first backburner_first active_job_only).each do |order|
-    it "enqueues and works jobs when the app requires #{order.tr('_', ' ')}" do
-      namespace = "demo.test.active_job.#{Process.pid}.#{order}"
-      app = File.expand_path('../fixtures/active_job_app.rb', __FILE__)
-      out, status = Open3.capture2e(RbConfig.ruby, '-w', '-I', File.expand_path('../../lib', __FILE__), app, namespace, order)
-      assert status.success?, out
-      # only backburner's own files: RubyGems can warn about its own
-      refute_match(/circular require considered harmful - #{Regexp.escape(File.expand_path('../../lib', __FILE__))}\//, out)
-      result = JSON.parse(out.lines.last)
-
-      tube = "#{namespace}.adapter"
-      assert_equal({ 'tube' => tube, 'pri' => 7, 'state' => 'ready', 'delay' => 0 }, result['jobs']['now'])
-      assert_equal({ 'tube' => tube, 'pri' => 3, 'state' => 'ready', 'delay' => 0 }, result['jobs']['urgent'])
-      # wait: 3 reaches the adapter a little under 3 seconds ahead: rounded up, so it doesn't run early
-      assert_equal({ 'tube' => tube, 'pri' => 7, 'state' => 'delayed', 'delay' => 3 }, result['jobs']['later'])
-      assert_equal %w(later now urgent), result['performed']
-    end
-  end
-end

data/test/async_proxy_test.rb

@@ -1,36 +0,0 @@
-require File.expand_path('../test_helper', __FILE__)
-
-class AsyncUser; end
-
-describe "Backburner::AsyncProxy class" do
-  before do
-    Backburner.default_queues.clear
-    clear_jobs!(Backburner.configuration.primary_queue)
-  end
-
-  describe "for method_missing enqueue" do
-    should "enqueue job onto worker with no args" do
-      @async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
-      @async.foo
-      pop_one_job do |job, body|
-        assert_equal "AsyncUser", body["class"]
-        assert_equal [10, "foo"], body["args"]
-        assert_equal 100, job.ttr
-        assert_equal 1000, job.pri
-        job.delete
-      end
-    end
-
-    should "enqueue job onto worker with args" do
-      @async = Backburner::AsyncProxy.new(AsyncUser, 10, :pri => 1000, :ttr => 100)
-      @async.bar(1, 2, 3)
-      pop_one_job do |job, body|
-        assert_equal "AsyncUser", body["class"]
-        assert_equal [10, "bar", 1, 2, 3], body["args"]
-        assert_equal 100, job.ttr
-        assert_equal 1000, job.pri
-        job.delete
-      end
-    end
-  end # method_missing
-end # AsyncProxy

data/test/back_burner_test.rb

@@ -1,101 +0,0 @@
-require File.expand_path('../test_helper', __FILE__)
-
-$backburner_sum = 0
-$backburner_numbers = []
-
-class TestBackburnerJob
-  include Backburner::Queue
-  queue "test.jobber"
-
-  def self.perform(value, number)
-    $backburner_sum += value
-    $backburner_numbers << number
-  end
-end
-
-class TestWorker < Backburner::Worker; end
-
-describe "Backburner module" do
-  before { Backburner.default_queues.clear }
-
-  describe "for enqueue method" do
-    before do
-      Backburner.enqueue TestBackburnerJob, 5, 6
-      Backburner.enqueue TestBackburnerJob, 15, 10
-      silenced(2) do
-        worker = Backburner::Workers::Simple.new('test.jobber')
-        worker.prepare
-        2.times { worker.work_one_job }
-      end
-    end
-
-    it "can run jobs using #run method" do
-      assert_equal 20, $backburner_sum
-      assert_same_elements [6, 10], $backburner_numbers
-    end
-  end # enqueue
-
-  describe "for work method" do
-    it "invokes worker simple start" do
-      Backburner::Workers::Simple.expects(:start).with(["foo", "bar"])
-      Backburner.work("foo", "bar")
-    end
-
-    it "invokes other worker if specified in configuration" do
-      Backburner.configure { |config| config.default_worker = TestWorker }
-      TestWorker.expects(:start).with(["foo", "bar"])
-      Backburner.work("foo", "bar")
-    end
-
-    it "invokes other worker if specified in work method as options" do
-      TestWorker.expects(:start).with(["foo", "bar"])
-      Backburner.work("foo", "bar", :worker => TestWorker)
-    end
-
-    it "invokes worker start with no args" do
-      Backburner::Workers::Simple.expects(:start).with([])
-      Backburner.work
-    end
-  end # work!
-
-  describe "for configuration" do
-    it "remembers the tube_namespace" do
-      assert_equal "demo.test", Backburner.configuration.tube_namespace
-    end
-
-    it "remembers the namespace_separator" do
-      assert_equal ".", Backburner.configuration.namespace_separator
-    end
-
-    it "disallows a reserved separator" do
-      assert_raises RuntimeError do
-        Backburner.configuration.namespace_separator = ':'
-      end
-    end
-
-    it "reads enqueue_connection_max_idle as seconds" do
-      max_idle = Backburner.configuration.enqueue_connection_max_idle
-      Backburner.configuration.enqueue_connection_max_idle = "30"
-      assert_equal 30.0, Backburner.configuration.enqueue_connection_max_idle
-      Backburner.configuration.enqueue_connection_max_idle = 1
-      assert_equal 1.0, Backburner.configuration.enqueue_connection_max_idle
-      Backburner.configuration.enqueue_connection_max_idle = nil
-      assert_nil Backburner.configuration.enqueue_connection_max_idle
-      assert_raises(ArgumentError) { Backburner.configuration.enqueue_connection_max_idle = "a minute" }
-    ensure
-      Backburner.configuration.enqueue_connection_max_idle = max_idle
-    end
-  end # configuration
-
-  describe "for default_queues" do
-    it "supports assignment" do
-      Backburner.default_queues << "foo"
-      Backburner.default_queues << "bar"
-      assert_same_elements ["foo", "bar"], Backburner.default_queues
-    end
-  end
-
-  after do
-    Backburner.configure { |config| config.default_worker = Backburner::Workers::Simple }
-  end
-end # Backburner

data/test/connection_test.rb

@@ -1,201 +0,0 @@
-require File.expand_path('../test_helper', __FILE__)
-
-describe "Backburner::Connection class" do
-  describe "for initialize with single url" do
-    before do
-      @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
-    end
-
-    it "should store url in accessor" do
-      assert_equal "beanstalk://127.0.0.1", @connection.url
-    end
-
-    it "should setup beanstalk connection" do
-      assert_kind_of Beaneater, @connection.beanstalk
-    end
-  end # initialize single connection
-
-  describe "for initialize with url" do
-    it "should delegate the address url correctly" do
-      @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
-      connection = @connection.beanstalk.connection
-      assert_equal '127.0.0.1:11300', connection.address
-    end
-  end # initialize
-
-  describe "for bad uri" do
-    it "should raise a BadUrl" do
-      assert_raises(Backburner::Connection::BadURL) {
-        @connection = Backburner::Connection.new("fake://foo")
-      }
-    end
-  end
-
-  describe "for initialize with on_reconnect block" do
-    it "should store the block for use upon reconnect" do
-      callback = proc {}
-      connection = Backburner::Connection.new('beanstalk://127.0.0.1', &callback)
-      assert_equal callback, connection.on_reconnect
-    end
-  end
-
-  describe "dealing with connecting and reconnecting" do
-    before do
-      @connection = Backburner::Connection.new('beanstalk://127.0.0.1')
-    end
-
-    it "should know if its connection is open" do
-      assert_equal true, @connection.connected?
-      @connection.close
-      assert_equal false, @connection.connected?
-    end
-
-    it "should be able to attempt reconnecting to beanstalk" do
-      @connection.close
-      assert_equal false, @connection.connected?
-      @connection.reconnect!
-      assert_equal true, @connection.connected?
-    end
-
-    it "should allow for retryable commands" do
-      @result = false
-      @connection.close
-      @connection.retryable { @result = true }
-      assert_equal true, @result
-    end
-
-    it "should provide a hook when a retryable command successfully retries" do
-      @result = false
-      @retried = false
-      @connection.close
-      callback = proc { @result = true }
-      @connection.retryable(:on_retry => callback) do
-        unless @retried
-          @retried = true
-          raise Beaneater::NotConnected.new
-        end
-      end
-      assert_equal true, @result
-    end
-
-    it "should provide a hook when the connection successfully reconnects" do
-      reconnected = false
-      retried = false
-      @connection.close
-      @connection.on_reconnect = proc { reconnected = true }
-      @connection.retryable do
-        unless retried
-          retried = true
-          raise Beaneater::NotConnected.new
-        end
-      end
-      assert_equal true, reconnected
-    end
-
-    it "should call the on_reconnect hook before the on_retry hook" do
-      @result = []
-      @retried = false
-      @connection.close
-      @connection.on_reconnect = proc { @result << "reconnect" }
-      on_retry = proc { @result << "retry" }
-      @connection.retryable(:on_retry => on_retry) do
-        unless @retried
-          @retried = true
-          raise Beaneater::NotConnected.new
-        end
-      end
-      assert_equal %w(reconnect retry), @result
-    end
-
-    describe "ensuring the connection is open" do
-      it "should reattempt the connection to beanstalk several times" do
-        stats = @connection.stats
-        simulate_disconnect(@connection)
-        new_connection = Beaneater.new('127.0.0.1:11300')
-        Beaneater.expects(:new).twice.raises(Beaneater::NotConnected).then.returns(new_connection)
-        @connection.tubes
-        assert_equal true, @connection.connected?
-      end
-
-      it "should not attempt reconnecting if the current connection is open" do
-        assert_equal true, @connection.connected?
-        Beaneater.expects(:new).never
-        @connection.tubes
-      end
-
-      describe "when reconnecting is successful" do
-        it "should allow for a callback" do
-          @result = false
-          simulate_disconnect(@connection)
-          @connection.on_reconnect = proc { @result = true }
-          @connection.tubes
-          assert_equal true, @result
-        end
-
-        it "should pass self to the callback" do
-          result = nil
-          simulate_disconnect(@connection)
-          @connection.on_reconnect = lambda { |conn| result = conn }
-          @connection.tubes
-          assert_equal result, @connection
-        end
-      end
-    end
-
-    describe "when unable to ensure its connected" do
-      it "should raise Beaneater::NotConnected" do
-        Beaneater.stubs(:new).raises(Beaneater::NotConnected)
-        simulate_disconnect(@connection, 1) # since we're stubbing Beaneater.new above we only to simlulate the disconnect of our current connection
-        assert_raises Beaneater::NotConnected do
-          @connection.tubes
-        end
-      end
-    end
-
-    describe "when a reconnect fails" do
-      it "should reconnect on the next call once beanstalkd is back" do
-        back_up = Beaneater.new('127.0.0.1:11300')
-        Beaneater.expects(:new).twice.raises(Beaneater::NotConnected).then.returns(back_up)
-        assert_raises(Beaneater::NotConnected) { @connection.reconnect! }
-        assert_equal false, @connection.connected?
-        assert_kind_of Array, @connection.tubes.all
-        assert_equal true, @connection.connected?
-      end
-    end
-
-    describe "when using the retryable method" do
-      it "should raise Beaneater::NotConnected once the retries are used up" do
-        attempts = 0
-        assert_raises Beaneater::NotConnected do
-          @connection.retryable(max_retries: 1, retry_delay: 0) do
-            attempts += 1
-            raise Beaneater::NotConnected
-          end
-        end
-        assert_equal 2, attempts
-      end
-
-      it "should yield to the block multiple times" do
-        expected    = 2
-        retry_count = 0
-        @connection.retryable(max_retries: expected) do
-          if retry_count < 2
-            retry_count += 1
-            raise Beaneater::NotConnected
-          end
-        end
-        assert_equal expected, retry_count
-      end
-    end
-  end
-
-  describe "for delegated methods" do
-    before do
-      @connection = Backburner::Connection.new("beanstalk://127.0.0.1")
-    end
-
-    it "delegate methods to beanstalk connection" do
-      assert_equal "127.0.0.1", @connection.connection.host
-    end
-  end # delegator
-end # Connection

data/test/fixtures/active_job_app.rb

@@ -1,53 +0,0 @@
-# An app using Backburner as its Active Job adapter. active_job_adapter_test.rb runs it in a child process,
-# so Active Job (and Active Support) stay out of the suite's own process.
-# Arguments: the tube namespace, then which of the two the app requires first (or active_job_only).
-require 'json'
-require 'logger' # Active Support 6.1 and 7.0 need Logger loaded first (with concurrent-ruby 1.3.5+)
-
-namespace, order = ARGV
-case order
-when 'active_job_first' # a Rails app: rails/all, then Bundler.require
-  require 'active_job'
-  require 'backburner'
-when 'backburner_first'
-  require 'backburner'
-  require 'active_job'
-when 'active_job_only' # gem 'backburner', require: false: the adapter loads it
-  require 'active_job'
-end
-
-ActiveJob::Base.logger = Logger.new(nil)
-ActiveJob::Base.queue_adapter = :backburner
-Backburner.configure do |config|
-  config.beanstalk_url = "beanstalk://127.0.0.1"
-  config.tube_namespace = namespace
-  config.logger = Logger.new(nil)
-  config.reserve_timeout = 5
-end
-
-$performed = []
-class AdapterTestJob < ActiveJob::Base
-  queue_as :adapter
-  queue_with_priority 7
-  def perform(name)
-    $performed << name
-  end
-end
-
-enqueued = {
-  'now' => AdapterTestJob.perform_later('now'),
-  'urgent' => AdapterTestJob.set(priority: 3).perform_later('urgent'),
-  'later' => AdapterTestJob.set(wait: 3).perform_later('later')
-}
-beanstalk = Beaneater.new('127.0.0.1')
-jobs = enqueued.map do |name, job|
-  stats = job.provider_job_id && beanstalk.jobs.find(job.provider_job_id).stats
-  [name, stats && { 'tube' => stats.tube, 'pri' => stats.pri, 'state' => stats.state, 'delay' => stats.delay }]
-end.to_h
-beanstalk.close
-
-worker = Backburner::Workers::Simple.new(['adapter'])
-worker.prepare
-enqueued.size.times { worker.work_one_job }
-
-puts JSON.generate('jobs' => jobs, 'performed' => $performed.sort)

data/test/fixtures/active_jobs.rb

@@ -1,11 +0,0 @@
-require 'active_job'
-require 'active_job/queue_adapters/backburner_adapter'
-
-ActiveJob::Base.queue_adapter = :backburner
-ActiveJob::Base.logger = nil
-
-class TestJob < ActiveJob::Base
-  def perform(arg)
-    true
-  end
-end

data/test/fixtures/autoload/test_broken_name_job.rb

@@ -1,4 +0,0 @@
-class TestBrokenNameJob
-  include TestMissingConcern # renamed or removed: NameError for another constant while the file loads
-  def self.perform(x); end
-end

data/test/fixtures/autoload/test_broken_no_method_job.rb

@@ -1,4 +0,0 @@
-class TestBrokenNoMethodJob
-  queue_respond_timout 30 # a typo: NoMethodError while the file loads
-  def self.perform(x); end
-end

data/test/fixtures/autoload/test_broken_require_job.rb

@@ -1,4 +0,0 @@
-require 'backburner_test_missing_gem' # a gem left out of the bundle: LoadError while the file loads
-class TestBrokenRequireJob
-  def self.perform(x); end
-end

data/test/fixtures/autoload/test_broken_runtime_job.rb

@@ -1,4 +0,0 @@
-class TestBrokenRuntimeJob
-  raise ArgumentError, "bad config" # an error of another kind while the file loads (ENV.fetch raising KeyError, say)
-  def self.perform(x); end
-end

data/test/fixtures/autoload/test_broken_syntax_job.rb

@@ -1,4 +0,0 @@
-class TestBrokenSyntaxJob
-  def self.perform(x)
-    if # a syntax error: SyntaxError while the file loads
-  end

data/test/fixtures/autoload/test_broken_timeout_job.rb

@@ -1,4 +0,0 @@
-class TestBrokenTimeoutJob
-  raise Timeout::Error, "class body call timed out" # a call in the class body that timed out (Net::OpenTimeout is a Timeout::Error)
-  def self.perform(x); end
-end

data/test/fixtures/autoload/test_same_name_job.rb

@@ -1,5 +0,0 @@
-module TestOtherNamespace; end
-class TestSameNameJob
-  OTHER = TestOtherNamespace::TestSameNameJob # NameError for a constant of the same name in another module
-  def self.perform(x); end
-end

data/test/fixtures/hooked.rb

@@ -1,122 +0,0 @@
-$hooked_fail_count = 0
-class HookFailError < RuntimeError; end
-
-class HookedObjectBeforeEnqueueFail
-  include Backburner::Performable
-
-  def self.before_enqueue_abe(*args)
-    puts "!!before_enqueue_foo!! #{args.inspect}"
-  end
-
-  def self.before_enqueue_bar(*args)
-    return false
-  end
-end
-
-
-class HookedObjectAfterEnqueueFail
-  def self.after_enqueue_abe(*args)
-    puts "!!after_enqueue_foo!! #{args.inspect}"
-  end
-
-  def self.after_enqueue_bar(*args)
-    raise HookFailError, "Fail HookedObjectAfterEnqueueFail"
-  end
-end
-
-class HookedObjectBeforePerformFail
-  include Backburner::Performable
-
-  def self.before_perform_abe(*args)
-    puts "!!before_perform_foo!! #{args.inspect}"
-  end
-
-  def self.before_perform_foo(*args)
-    return false
-  end
-
-  def self.foo(x)
-    puts "Fail ran!!"
-    raise HookFailError, "HookedObjectJobFailure on foo!"
-  end
-end
-
-class HookedObjectAfterPerformFail
-  def self.after_perform_abe(*args)
-    puts "!!after_perform_foo!! #{args.inspect}"
-  end
-
-  def self.after_perform_bar(*args)
-    raise HookFailError, "Fail HookedObjectAfterEnqueueFail"
-  end
-end
-
-class HookedObjectJobFailure
-  def self.foo(x)
-    raise HookFailError, "HookedObjectJobFailure on foo!"
-  end
-end
-
-class HookedObjectSuccess
-  include Backburner::Performable
-
-  def self.before_enqueue_foo(*args)
-    puts "!!before_enqueue_foo!! #{args.inspect}"
-  end
-
-  def self.before_enqueue_bar(*args)
-    puts "!!before_enqueue_bar!! #{args.inspect}"
-  end
-
-  def self.after_enqueue_foo(*args)
-    puts "!!after_enqueue_foo!! #{args.inspect}"
-  end
-
-  def self.after_enqueue_bar(*args)
-    puts "!!after_enqueue_bar!! #{args.inspect}"
-  end
-
-  def self.before_perform_foo(*args)
-    puts "!!before_perform_foo!! #{args.inspect}"
-  end
-
-  def self.after_perform_foo(*args)
-    puts "!!after_perform_foo!! #{args.inspect}"
-  end
-
-  def self.around_perform_bar(*args)
-    puts "!!BEGIN around_perform_bar!! #{args.inspect}"
-    yield
-    puts "!!END around_perform_bar!! #{args.inspect}"
-  end
-
-  def self.around_perform_cat(*args)
-    puts "!!BEGIN around_perform_cat!! #{args.inspect}"
-    yield
-    puts "!!END around_perform_cat!! #{args.inspect}"
-  end
-
-  def self.on_failure_foo(ex, *args)
-    puts "!!on_failure_foo!! #{ex.inspect} #{args.inspect}"
-  end
-
-  def self.on_bury_foo(*args)
-    puts "!!on_bury_foo!! #{args.inspect}"
-  end
-
-  def self.on_retry_foo(retry_count, delay, *args)
-    puts "!!on_retry_foo!! #{retry_count} #{delay} #{args.inspect}"
-  end
-
-  def self.foo(x)
-    $hooked_fail_count += 1
-    raise HookFailError, "Fail!" if $hooked_fail_count == 1
-    puts "This is the job running successfully!! #{x.inspect}"
-  end
-end # HookedObjectSuccess
-
-class HookedWorker < Backburner::Worker
-  def on_reconnect
-    puts "!!on_reconnect!!"
-  end
-end