activejob-retry 0.0.1 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +22 -0
- data/.travis.yml +20 -5
- data/CHANGELOG.md +3 -0
- data/Gemfile +9 -6
- data/Gemfile.lock +63 -51
- data/LICENSE +1 -1
- data/README.md +57 -25
- data/Rakefile +14 -13
- data/activejob-retry.gemspec +9 -12
- data/lib/active_job/retry.rb +69 -90
- data/lib/active_job/retry/constant_backoff_strategy.rb +50 -0
- data/lib/active_job/retry/constant_options_validator.rb +76 -0
- data/lib/active_job/retry/deserialize_monkey_patch.rb +17 -12
- data/lib/active_job/retry/errors.rb +6 -0
- data/lib/active_job/retry/variable_backoff_strategy.rb +34 -0
- data/lib/active_job/retry/variable_options_validator.rb +56 -0
- data/lib/active_job/retry/version.rb +1 -1
- data/spec/retry/constant_backoff_strategy_spec.rb +115 -0
- data/spec/retry/constant_options_validator_spec.rb +81 -0
- data/spec/retry/variable_backoff_strategy_spec.rb +121 -0
- data/spec/retry/variable_options_validator_spec.rb +83 -0
- data/spec/retry_spec.rb +114 -170
- data/spec/spec_helper.rb +3 -2
- data/test/adapters/backburner.rb +3 -0
- data/test/adapters/delayed_job.rb +7 -0
- data/test/adapters/inline.rb +1 -0
- data/test/adapters/qu.rb +3 -0
- data/test/adapters/que.rb +4 -0
- data/test/adapters/queue_classic.rb +2 -0
- data/test/adapters/resque.rb +2 -0
- data/test/adapters/sidekiq.rb +2 -0
- data/test/adapters/sneakers.rb +2 -0
- data/test/adapters/sucker_punch.rb +2 -0
- data/test/cases/adapter_test.rb +8 -0
- data/test/cases/argument_serialization_test.rb +84 -0
- data/test/cases/callbacks_test.rb +23 -0
- data/test/cases/job_serialization_test.rb +15 -0
- data/test/cases/logging_test.rb +114 -0
- data/test/cases/queue_naming_test.rb +102 -0
- data/test/cases/queuing_test.rb +44 -0
- data/test/cases/rescue_test.rb +35 -0
- data/test/cases/test_case_test.rb +14 -0
- data/test/cases/test_helper_test.rb +226 -0
- data/test/helper.rb +21 -0
- data/test/integration/queuing_test.rb +46 -0
- data/test/jobs/callback_job.rb +31 -0
- data/test/jobs/gid_job.rb +10 -0
- data/test/jobs/hello_job.rb +9 -0
- data/test/jobs/logging_job.rb +12 -0
- data/test/jobs/nested_job.rb +12 -0
- data/test/jobs/rescue_job.rb +31 -0
- data/test/models/person.rb +20 -0
- data/test/support/backburner/inline.rb +8 -0
- data/test/support/delayed_job/delayed/backend/test.rb +111 -0
- data/test/support/delayed_job/delayed/serialization/test.rb +0 -0
- data/test/support/integration/adapters/backburner.rb +38 -0
- data/test/support/integration/adapters/delayed_job.rb +20 -0
- data/test/support/integration/adapters/que.rb +37 -0
- data/test/support/integration/adapters/resque.rb +49 -0
- data/test/support/integration/adapters/sidekiq.rb +58 -0
- data/test/support/integration/dummy_app_template.rb +28 -0
- data/test/support/integration/helper.rb +30 -0
- data/test/support/integration/jobs_manager.rb +27 -0
- data/test/support/integration/test_case_helpers.rb +48 -0
- data/test/support/job_buffer.rb +19 -0
- data/test/support/que/inline.rb +9 -0
- metadata +60 -12
- data/lib/active_job-retry.rb +0 -14
- data/lib/active_job/retry/exponential_backoff.rb +0 -92
- data/lib/active_job/retry/exponential_options_validator.rb +0 -57
- data/lib/active_job/retry/invalid_configuration_error.rb +0 -6
- data/lib/active_job/retry/options_validator.rb +0 -84
data/test/helper.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'bundler'
|
2
|
+
Bundler.setup
|
3
|
+
|
4
|
+
require 'active_job'
|
5
|
+
require 'support/job_buffer'
|
6
|
+
|
7
|
+
GlobalID.app = 'aj'
|
8
|
+
|
9
|
+
@adapter = ENV['AJADAPTER'] || 'resque'
|
10
|
+
|
11
|
+
if ENV['AJ_INTEGRATION_TESTS']
|
12
|
+
require 'support/integration/helper'
|
13
|
+
else
|
14
|
+
require "adapters/#{@adapter}"
|
15
|
+
end
|
16
|
+
|
17
|
+
require 'active_job/retry'
|
18
|
+
|
19
|
+
require 'active_support/testing/autorun'
|
20
|
+
|
21
|
+
ActiveSupport::TestCase.test_order = :random
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'helper'
|
2
|
+
require 'jobs/logging_job'
|
3
|
+
require 'active_support/core_ext/numeric/time'
|
4
|
+
|
5
|
+
class QueuingTest < ActiveSupport::TestCase
|
6
|
+
test 'should run jobs enqueued on a listening queue' do
|
7
|
+
TestJob.perform_later @id
|
8
|
+
wait_for_jobs_to_finish_for(5.seconds)
|
9
|
+
assert job_executed
|
10
|
+
end
|
11
|
+
|
12
|
+
test 'should not run jobs queued on a non-listening queue' do
|
13
|
+
old_queue = TestJob.queue_name
|
14
|
+
|
15
|
+
begin
|
16
|
+
TestJob.queue_as :some_other_queue
|
17
|
+
TestJob.perform_later @id
|
18
|
+
wait_for_jobs_to_finish_for(2.seconds)
|
19
|
+
assert_not job_executed
|
20
|
+
ensure
|
21
|
+
TestJob.queue_name = old_queue
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
test 'should not run job enqueued in the future' do
|
26
|
+
TestJob.set(wait: 10.minutes).perform_later @id
|
27
|
+
wait_for_jobs_to_finish_for(5.seconds)
|
28
|
+
assert_not job_executed
|
29
|
+
end
|
30
|
+
|
31
|
+
test 'should run job enqueued in the future at the specified time' do
|
32
|
+
TestJob.set(wait: 3.seconds).perform_later @id
|
33
|
+
wait_for_jobs_to_finish_for(2.seconds)
|
34
|
+
assert_not job_executed
|
35
|
+
wait_for_jobs_to_finish_for(10.seconds)
|
36
|
+
assert job_executed
|
37
|
+
end
|
38
|
+
|
39
|
+
test 'should retry when the job fails' do
|
40
|
+
TestJob.perform_later @id, true
|
41
|
+
wait_for_jobs_to_finish_for(2.seconds)
|
42
|
+
assert_not job_executed
|
43
|
+
wait_for_jobs_to_finish_for(5.seconds)
|
44
|
+
assert job_executed
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
class CallbackJob < ActiveJob::Base
|
2
|
+
include ActiveJob::Retry
|
3
|
+
|
4
|
+
before_perform ->(job) { job.history << "CallbackJob ran before_perform" }
|
5
|
+
after_perform ->(job) { job.history << "CallbackJob ran after_perform" }
|
6
|
+
|
7
|
+
before_enqueue ->(job) { job.history << "CallbackJob ran before_enqueue" }
|
8
|
+
after_enqueue ->(job) { job.history << "CallbackJob ran after_enqueue" }
|
9
|
+
|
10
|
+
around_perform do |job, block|
|
11
|
+
job.history << "CallbackJob ran around_perform_start"
|
12
|
+
block.call
|
13
|
+
job.history << "CallbackJob ran around_perform_stop"
|
14
|
+
end
|
15
|
+
|
16
|
+
around_enqueue do |job, block|
|
17
|
+
job.history << "CallbackJob ran around_enqueue_start"
|
18
|
+
block.call
|
19
|
+
job.history << "CallbackJob ran around_enqueue_stop"
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
def perform(person = "david")
|
24
|
+
# NOTHING!
|
25
|
+
end
|
26
|
+
|
27
|
+
def history
|
28
|
+
@history ||= []
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require_relative '../support/job_buffer'
|
2
|
+
|
3
|
+
class RescueJob < ActiveJob::Base
|
4
|
+
include ActiveJob::Retry
|
5
|
+
constant_retry limit: 2, delay: 0
|
6
|
+
|
7
|
+
class OtherError < StandardError; end
|
8
|
+
|
9
|
+
rescue_from(ArgumentError) do
|
10
|
+
JobBuffer.add('rescued from ArgumentError')
|
11
|
+
arguments[0] = "DIFFERENT!"
|
12
|
+
retry_job
|
13
|
+
end
|
14
|
+
|
15
|
+
rescue_from(ActiveJob::DeserializationError) do |e|
|
16
|
+
JobBuffer.add('rescued from DeserializationError')
|
17
|
+
JobBuffer.add("DeserializationError original exception was #{e.original_exception.class.name}")
|
18
|
+
end
|
19
|
+
|
20
|
+
def perform(person = "david")
|
21
|
+
JobBuffer.add('running')
|
22
|
+
case person
|
23
|
+
when "david"
|
24
|
+
raise ArgumentError, "Hair too good"
|
25
|
+
when "other"
|
26
|
+
raise OtherError
|
27
|
+
else
|
28
|
+
JobBuffer.add('performed beautifully')
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
class Person
|
2
|
+
class RecordNotFound < StandardError; end
|
3
|
+
|
4
|
+
include GlobalID::Identification
|
5
|
+
|
6
|
+
attr_reader :id
|
7
|
+
|
8
|
+
def self.find(id)
|
9
|
+
raise RecordNotFound.new("Cannot find person with ID=404") if id.to_i==404
|
10
|
+
new(id)
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(id)
|
14
|
+
@id = id
|
15
|
+
end
|
16
|
+
|
17
|
+
def ==(other_person)
|
18
|
+
other_person.is_a?(Person) && id.to_s == other_person.id.to_s
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
#copied from https://github.com/collectiveidea/delayed_job/blob/master/spec/delayed/backend/test.rb
|
2
|
+
require 'ostruct'
|
3
|
+
|
4
|
+
# An in-memory backend suitable only for testing. Tries to behave as if it were an ORM.
|
5
|
+
module Delayed
|
6
|
+
module Backend
|
7
|
+
module Test
|
8
|
+
class Job
|
9
|
+
attr_accessor :id
|
10
|
+
attr_accessor :priority
|
11
|
+
attr_accessor :attempts
|
12
|
+
attr_accessor :handler
|
13
|
+
attr_accessor :last_error
|
14
|
+
attr_accessor :run_at
|
15
|
+
attr_accessor :locked_at
|
16
|
+
attr_accessor :locked_by
|
17
|
+
attr_accessor :failed_at
|
18
|
+
attr_accessor :queue
|
19
|
+
|
20
|
+
include Delayed::Backend::Base
|
21
|
+
|
22
|
+
cattr_accessor :id
|
23
|
+
self.id = 0
|
24
|
+
|
25
|
+
def initialize(hash = {})
|
26
|
+
self.attempts = 0
|
27
|
+
self.priority = 0
|
28
|
+
self.id = (self.class.id += 1)
|
29
|
+
hash.each{|k,v| send(:"#{k}=", v)}
|
30
|
+
end
|
31
|
+
|
32
|
+
@jobs = []
|
33
|
+
def self.all
|
34
|
+
@jobs
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.count
|
38
|
+
all.size
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.delete_all
|
42
|
+
all.clear
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.create(attrs = {})
|
46
|
+
new(attrs).tap(&:save)
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.create!(*args); create(*args); end
|
50
|
+
|
51
|
+
def self.clear_locks!(worker_name)
|
52
|
+
all.select{|j| j.locked_by == worker_name}.each {|j| j.locked_by = nil; j.locked_at = nil}
|
53
|
+
end
|
54
|
+
|
55
|
+
# Find a few candidate jobs to run (in case some immediately get locked by others).
|
56
|
+
def self.find_available(worker_name, limit = 5, max_run_time = Worker.max_run_time)
|
57
|
+
jobs = all.select do |j|
|
58
|
+
j.run_at <= db_time_now &&
|
59
|
+
(j.locked_at.nil? || j.locked_at < db_time_now - max_run_time || j.locked_by == worker_name) &&
|
60
|
+
!j.failed?
|
61
|
+
end
|
62
|
+
|
63
|
+
jobs = jobs.select{|j| Worker.queues.include?(j.queue)} if Worker.queues.any?
|
64
|
+
jobs = jobs.select{|j| j.priority >= Worker.min_priority} if Worker.min_priority
|
65
|
+
jobs = jobs.select{|j| j.priority <= Worker.max_priority} if Worker.max_priority
|
66
|
+
jobs.sort_by{|j| [j.priority, j.run_at]}[0..limit-1]
|
67
|
+
end
|
68
|
+
|
69
|
+
# Lock this job for this worker.
|
70
|
+
# Returns true if we have the lock, false otherwise.
|
71
|
+
def lock_exclusively!(max_run_time, worker)
|
72
|
+
now = self.class.db_time_now
|
73
|
+
if locked_by != worker
|
74
|
+
# We don't own this job so we will update the locked_by name and the locked_at
|
75
|
+
self.locked_at = now
|
76
|
+
self.locked_by = worker
|
77
|
+
end
|
78
|
+
|
79
|
+
return true
|
80
|
+
end
|
81
|
+
|
82
|
+
def self.db_time_now
|
83
|
+
Time.current
|
84
|
+
end
|
85
|
+
|
86
|
+
def update_attributes(attrs = {})
|
87
|
+
attrs.each{|k,v| send(:"#{k}=", v)}
|
88
|
+
save
|
89
|
+
end
|
90
|
+
|
91
|
+
def destroy
|
92
|
+
self.class.all.delete(self)
|
93
|
+
end
|
94
|
+
|
95
|
+
def save
|
96
|
+
self.run_at ||= Time.current
|
97
|
+
|
98
|
+
self.class.all << self unless self.class.all.include?(self)
|
99
|
+
true
|
100
|
+
end
|
101
|
+
|
102
|
+
def save!; save; end
|
103
|
+
|
104
|
+
def reload
|
105
|
+
reset
|
106
|
+
self
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
File without changes
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module BackburnerJobsManager
|
2
|
+
def setup
|
3
|
+
ActiveJob::Base.queue_adapter = :backburner
|
4
|
+
Backburner.configure do |config|
|
5
|
+
config.logger = Rails.logger
|
6
|
+
end
|
7
|
+
unless can_run?
|
8
|
+
puts "Cannot run integration tests for backburner. To be able to run integration tests for backburner you need to install and start beanstalkd.\n"
|
9
|
+
exit
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def clear_jobs
|
14
|
+
tube.clear
|
15
|
+
end
|
16
|
+
|
17
|
+
def start_workers
|
18
|
+
@thread = Thread.new { Backburner.work "integration-tests" } # backburner dasherizes the queue name
|
19
|
+
end
|
20
|
+
|
21
|
+
def stop_workers
|
22
|
+
@thread.kill
|
23
|
+
end
|
24
|
+
|
25
|
+
def tube
|
26
|
+
@tube ||= Beaneater::Tube.new(Backburner::Worker.connection, "backburner.worker.queue.integration-tests") # backburner dasherizes the queue name
|
27
|
+
end
|
28
|
+
|
29
|
+
def can_run?
|
30
|
+
begin
|
31
|
+
Backburner::Worker.connection.send :connect!
|
32
|
+
rescue
|
33
|
+
return false
|
34
|
+
end
|
35
|
+
true
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'delayed_job'
|
2
|
+
require 'delayed_job_active_record'
|
3
|
+
|
4
|
+
module DelayedJobJobsManager
|
5
|
+
def setup
|
6
|
+
ActiveJob::Base.queue_adapter = :delayed_job
|
7
|
+
end
|
8
|
+
def clear_jobs
|
9
|
+
Delayed::Job.delete_all
|
10
|
+
end
|
11
|
+
|
12
|
+
def start_workers
|
13
|
+
@worker = Delayed::Worker.new(quiet: true, sleep_delay: 0.5, queues: %w(integration_tests))
|
14
|
+
@thread = Thread.new { @worker.start }
|
15
|
+
end
|
16
|
+
|
17
|
+
def stop_workers
|
18
|
+
@worker.stop
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module QueJobsManager
|
2
|
+
def setup
|
3
|
+
require 'sequel'
|
4
|
+
ActiveJob::Base.queue_adapter = :que
|
5
|
+
que_url = ENV['QUE_DATABASE_URL'] || 'postgres:///dummy_test'
|
6
|
+
uri = URI.parse(que_url)
|
7
|
+
user = uri.user||ENV['USER']
|
8
|
+
pass = uri.password
|
9
|
+
db = uri.path[1..-1]
|
10
|
+
%x{#{"PGPASSWORD=\"#{pass}\"" if pass} psql -c 'drop database if exists "#{db}"' -U #{user} -t template1}
|
11
|
+
%x{#{"PGPASSWORD=\"#{pass}\"" if pass} psql -c 'create database "#{db}"' -U #{user} -t template1}
|
12
|
+
Que.connection = Sequel.connect(que_url)
|
13
|
+
Que.migrate!
|
14
|
+
Que.mode = :off
|
15
|
+
Que.worker_count = 1
|
16
|
+
rescue Sequel::DatabaseConnectionError
|
17
|
+
puts "Cannot run integration tests for que. To be able to run integration tests for que you need to install and start postgresql.\n"
|
18
|
+
exit
|
19
|
+
end
|
20
|
+
|
21
|
+
def clear_jobs
|
22
|
+
Que.clear!
|
23
|
+
end
|
24
|
+
|
25
|
+
def start_workers
|
26
|
+
@thread = Thread.new do
|
27
|
+
loop do
|
28
|
+
Que::Job.work("integration_tests")
|
29
|
+
sleep 0.5
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def stop_workers
|
35
|
+
@thread.kill
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module ResqueJobsManager
|
2
|
+
def setup
|
3
|
+
ActiveJob::Base.queue_adapter = :resque
|
4
|
+
Resque.redis = Redis::Namespace.new 'active_jobs_int_test', redis: Redis.connect(url: "redis://127.0.0.1:6379/12", :thread_safe => true)
|
5
|
+
Resque.logger = Rails.logger
|
6
|
+
unless can_run?
|
7
|
+
puts "Cannot run integration tests for resque. To be able to run integration tests for resque you need to install and start redis.\n"
|
8
|
+
exit
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def clear_jobs
|
13
|
+
Resque.queues.each { |queue_name| Resque.redis.del "queue:#{queue_name}" }
|
14
|
+
Resque.redis.keys("delayed:*").each { |key| Resque.redis.del "#{key}" }
|
15
|
+
Resque.redis.del "delayed_queue_schedule"
|
16
|
+
end
|
17
|
+
|
18
|
+
def start_workers
|
19
|
+
@resque_thread = Thread.new do
|
20
|
+
w = Resque::Worker.new("integration_tests")
|
21
|
+
w.term_child = true
|
22
|
+
w.work(0.5)
|
23
|
+
end
|
24
|
+
@scheduler_thread = Thread.new do
|
25
|
+
Resque::Scheduler.configure do |c|
|
26
|
+
c.poll_sleep_amount = 0.5
|
27
|
+
c.dynamic = true
|
28
|
+
c.quiet = true
|
29
|
+
c.logfile = nil
|
30
|
+
end
|
31
|
+
Resque::Scheduler.master_lock.release!
|
32
|
+
Resque::Scheduler.run
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def stop_workers
|
37
|
+
@resque_thread.kill
|
38
|
+
@scheduler_thread.kill
|
39
|
+
end
|
40
|
+
|
41
|
+
def can_run?
|
42
|
+
begin
|
43
|
+
Resque.redis.client.connect
|
44
|
+
rescue
|
45
|
+
return false
|
46
|
+
end
|
47
|
+
true
|
48
|
+
end
|
49
|
+
end
|