activejob-retry 0.0.1 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +22 -0
  3. data/.travis.yml +20 -5
  4. data/CHANGELOG.md +3 -0
  5. data/Gemfile +9 -6
  6. data/Gemfile.lock +63 -51
  7. data/LICENSE +1 -1
  8. data/README.md +57 -25
  9. data/Rakefile +14 -13
  10. data/activejob-retry.gemspec +9 -12
  11. data/lib/active_job/retry.rb +69 -90
  12. data/lib/active_job/retry/constant_backoff_strategy.rb +50 -0
  13. data/lib/active_job/retry/constant_options_validator.rb +76 -0
  14. data/lib/active_job/retry/deserialize_monkey_patch.rb +17 -12
  15. data/lib/active_job/retry/errors.rb +6 -0
  16. data/lib/active_job/retry/variable_backoff_strategy.rb +34 -0
  17. data/lib/active_job/retry/variable_options_validator.rb +56 -0
  18. data/lib/active_job/retry/version.rb +1 -1
  19. data/spec/retry/constant_backoff_strategy_spec.rb +115 -0
  20. data/spec/retry/constant_options_validator_spec.rb +81 -0
  21. data/spec/retry/variable_backoff_strategy_spec.rb +121 -0
  22. data/spec/retry/variable_options_validator_spec.rb +83 -0
  23. data/spec/retry_spec.rb +114 -170
  24. data/spec/spec_helper.rb +3 -2
  25. data/test/adapters/backburner.rb +3 -0
  26. data/test/adapters/delayed_job.rb +7 -0
  27. data/test/adapters/inline.rb +1 -0
  28. data/test/adapters/qu.rb +3 -0
  29. data/test/adapters/que.rb +4 -0
  30. data/test/adapters/queue_classic.rb +2 -0
  31. data/test/adapters/resque.rb +2 -0
  32. data/test/adapters/sidekiq.rb +2 -0
  33. data/test/adapters/sneakers.rb +2 -0
  34. data/test/adapters/sucker_punch.rb +2 -0
  35. data/test/cases/adapter_test.rb +8 -0
  36. data/test/cases/argument_serialization_test.rb +84 -0
  37. data/test/cases/callbacks_test.rb +23 -0
  38. data/test/cases/job_serialization_test.rb +15 -0
  39. data/test/cases/logging_test.rb +114 -0
  40. data/test/cases/queue_naming_test.rb +102 -0
  41. data/test/cases/queuing_test.rb +44 -0
  42. data/test/cases/rescue_test.rb +35 -0
  43. data/test/cases/test_case_test.rb +14 -0
  44. data/test/cases/test_helper_test.rb +226 -0
  45. data/test/helper.rb +21 -0
  46. data/test/integration/queuing_test.rb +46 -0
  47. data/test/jobs/callback_job.rb +31 -0
  48. data/test/jobs/gid_job.rb +10 -0
  49. data/test/jobs/hello_job.rb +9 -0
  50. data/test/jobs/logging_job.rb +12 -0
  51. data/test/jobs/nested_job.rb +12 -0
  52. data/test/jobs/rescue_job.rb +31 -0
  53. data/test/models/person.rb +20 -0
  54. data/test/support/backburner/inline.rb +8 -0
  55. data/test/support/delayed_job/delayed/backend/test.rb +111 -0
  56. data/test/support/delayed_job/delayed/serialization/test.rb +0 -0
  57. data/test/support/integration/adapters/backburner.rb +38 -0
  58. data/test/support/integration/adapters/delayed_job.rb +20 -0
  59. data/test/support/integration/adapters/que.rb +37 -0
  60. data/test/support/integration/adapters/resque.rb +49 -0
  61. data/test/support/integration/adapters/sidekiq.rb +58 -0
  62. data/test/support/integration/dummy_app_template.rb +28 -0
  63. data/test/support/integration/helper.rb +30 -0
  64. data/test/support/integration/jobs_manager.rb +27 -0
  65. data/test/support/integration/test_case_helpers.rb +48 -0
  66. data/test/support/job_buffer.rb +19 -0
  67. data/test/support/que/inline.rb +9 -0
  68. metadata +60 -12
  69. data/lib/active_job-retry.rb +0 -14
  70. data/lib/active_job/retry/exponential_backoff.rb +0 -92
  71. data/lib/active_job/retry/exponential_options_validator.rb +0 -57
  72. data/lib/active_job/retry/invalid_configuration_error.rb +0 -6
  73. data/lib/active_job/retry/options_validator.rb +0 -84
data/test/helper.rb ADDED
@@ -0,0 +1,21 @@
1
+ require 'bundler'
2
+ Bundler.setup
3
+
4
+ require 'active_job'
5
+ require 'support/job_buffer'
6
+
7
+ GlobalID.app = 'aj'
8
+
9
+ @adapter = ENV['AJADAPTER'] || 'resque'
10
+
11
+ if ENV['AJ_INTEGRATION_TESTS']
12
+ require 'support/integration/helper'
13
+ else
14
+ require "adapters/#{@adapter}"
15
+ end
16
+
17
+ require 'active_job/retry'
18
+
19
+ require 'active_support/testing/autorun'
20
+
21
+ ActiveSupport::TestCase.test_order = :random
@@ -0,0 +1,46 @@
1
+ require 'helper'
2
+ require 'jobs/logging_job'
3
+ require 'active_support/core_ext/numeric/time'
4
+
5
+ class QueuingTest < ActiveSupport::TestCase
6
+ test 'should run jobs enqueued on a listening queue' do
7
+ TestJob.perform_later @id
8
+ wait_for_jobs_to_finish_for(5.seconds)
9
+ assert job_executed
10
+ end
11
+
12
+ test 'should not run jobs queued on a non-listening queue' do
13
+ old_queue = TestJob.queue_name
14
+
15
+ begin
16
+ TestJob.queue_as :some_other_queue
17
+ TestJob.perform_later @id
18
+ wait_for_jobs_to_finish_for(2.seconds)
19
+ assert_not job_executed
20
+ ensure
21
+ TestJob.queue_name = old_queue
22
+ end
23
+ end
24
+
25
+ test 'should not run job enqueued in the future' do
26
+ TestJob.set(wait: 10.minutes).perform_later @id
27
+ wait_for_jobs_to_finish_for(5.seconds)
28
+ assert_not job_executed
29
+ end
30
+
31
+ test 'should run job enqueued in the future at the specified time' do
32
+ TestJob.set(wait: 3.seconds).perform_later @id
33
+ wait_for_jobs_to_finish_for(2.seconds)
34
+ assert_not job_executed
35
+ wait_for_jobs_to_finish_for(10.seconds)
36
+ assert job_executed
37
+ end
38
+
39
+ test 'should retry when the job fails' do
40
+ TestJob.perform_later @id, true
41
+ wait_for_jobs_to_finish_for(2.seconds)
42
+ assert_not job_executed
43
+ wait_for_jobs_to_finish_for(5.seconds)
44
+ assert job_executed
45
+ end
46
+ end
@@ -0,0 +1,31 @@
1
+ class CallbackJob < ActiveJob::Base
2
+ include ActiveJob::Retry
3
+
4
+ before_perform ->(job) { job.history << "CallbackJob ran before_perform" }
5
+ after_perform ->(job) { job.history << "CallbackJob ran after_perform" }
6
+
7
+ before_enqueue ->(job) { job.history << "CallbackJob ran before_enqueue" }
8
+ after_enqueue ->(job) { job.history << "CallbackJob ran after_enqueue" }
9
+
10
+ around_perform do |job, block|
11
+ job.history << "CallbackJob ran around_perform_start"
12
+ block.call
13
+ job.history << "CallbackJob ran around_perform_stop"
14
+ end
15
+
16
+ around_enqueue do |job, block|
17
+ job.history << "CallbackJob ran around_enqueue_start"
18
+ block.call
19
+ job.history << "CallbackJob ran around_enqueue_stop"
20
+ end
21
+
22
+
23
+ def perform(person = "david")
24
+ # NOTHING!
25
+ end
26
+
27
+ def history
28
+ @history ||= []
29
+ end
30
+
31
+ end
@@ -0,0 +1,10 @@
1
+ require_relative '../support/job_buffer'
2
+
3
+ class GidJob < ActiveJob::Base
4
+ include ActiveJob::Retry
5
+
6
+ def perform(person)
7
+ JobBuffer.add("Person with ID: #{person.id}")
8
+ end
9
+ end
10
+
@@ -0,0 +1,9 @@
1
+ require_relative '../support/job_buffer'
2
+
3
+ class HelloJob < ActiveJob::Base
4
+ include ActiveJob::Retry
5
+
6
+ def perform(greeter = "David")
7
+ JobBuffer.add("#{greeter} says hello")
8
+ end
9
+ end
@@ -0,0 +1,12 @@
1
+ class LoggingJob < ActiveJob::Base
2
+ include ActiveJob::Retry
3
+
4
+ def perform(dummy)
5
+ logger.info "Dummy, here is it: #{dummy}"
6
+ end
7
+
8
+ def job_id
9
+ "LOGGING-JOB-ID"
10
+ end
11
+ end
12
+
@@ -0,0 +1,12 @@
1
+ class NestedJob < ActiveJob::Base
2
+ include ActiveJob::Retry
3
+
4
+ def perform
5
+ LoggingJob.perform_later "NestedJob"
6
+ end
7
+
8
+ def job_id
9
+ "NESTED-JOB-ID"
10
+ end
11
+ end
12
+
@@ -0,0 +1,31 @@
1
+ require_relative '../support/job_buffer'
2
+
3
+ class RescueJob < ActiveJob::Base
4
+ include ActiveJob::Retry
5
+ constant_retry limit: 2, delay: 0
6
+
7
+ class OtherError < StandardError; end
8
+
9
+ rescue_from(ArgumentError) do
10
+ JobBuffer.add('rescued from ArgumentError')
11
+ arguments[0] = "DIFFERENT!"
12
+ retry_job
13
+ end
14
+
15
+ rescue_from(ActiveJob::DeserializationError) do |e|
16
+ JobBuffer.add('rescued from DeserializationError')
17
+ JobBuffer.add("DeserializationError original exception was #{e.original_exception.class.name}")
18
+ end
19
+
20
+ def perform(person = "david")
21
+ JobBuffer.add('running')
22
+ case person
23
+ when "david"
24
+ raise ArgumentError, "Hair too good"
25
+ when "other"
26
+ raise OtherError
27
+ else
28
+ JobBuffer.add('performed beautifully')
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,20 @@
1
+ class Person
2
+ class RecordNotFound < StandardError; end
3
+
4
+ include GlobalID::Identification
5
+
6
+ attr_reader :id
7
+
8
+ def self.find(id)
9
+ raise RecordNotFound.new("Cannot find person with ID=404") if id.to_i==404
10
+ new(id)
11
+ end
12
+
13
+ def initialize(id)
14
+ @id = id
15
+ end
16
+
17
+ def ==(other_person)
18
+ other_person.is_a?(Person) && id.to_s == other_person.id.to_s
19
+ end
20
+ end
@@ -0,0 +1,8 @@
1
+ require 'backburner'
2
+
3
+ Backburner::Worker.class_eval do
4
+ class << self; alias_method :original_enqueue, :enqueue; end
5
+ def self.enqueue(job_class, args=[], opts={})
6
+ job_class.perform(*args)
7
+ end
8
+ end
@@ -0,0 +1,111 @@
1
+ #copied from https://github.com/collectiveidea/delayed_job/blob/master/spec/delayed/backend/test.rb
2
+ require 'ostruct'
3
+
4
+ # An in-memory backend suitable only for testing. Tries to behave as if it were an ORM.
5
+ module Delayed
6
+ module Backend
7
+ module Test
8
+ class Job
9
+ attr_accessor :id
10
+ attr_accessor :priority
11
+ attr_accessor :attempts
12
+ attr_accessor :handler
13
+ attr_accessor :last_error
14
+ attr_accessor :run_at
15
+ attr_accessor :locked_at
16
+ attr_accessor :locked_by
17
+ attr_accessor :failed_at
18
+ attr_accessor :queue
19
+
20
+ include Delayed::Backend::Base
21
+
22
+ cattr_accessor :id
23
+ self.id = 0
24
+
25
+ def initialize(hash = {})
26
+ self.attempts = 0
27
+ self.priority = 0
28
+ self.id = (self.class.id += 1)
29
+ hash.each{|k,v| send(:"#{k}=", v)}
30
+ end
31
+
32
+ @jobs = []
33
+ def self.all
34
+ @jobs
35
+ end
36
+
37
+ def self.count
38
+ all.size
39
+ end
40
+
41
+ def self.delete_all
42
+ all.clear
43
+ end
44
+
45
+ def self.create(attrs = {})
46
+ new(attrs).tap(&:save)
47
+ end
48
+
49
+ def self.create!(*args); create(*args); end
50
+
51
+ def self.clear_locks!(worker_name)
52
+ all.select{|j| j.locked_by == worker_name}.each {|j| j.locked_by = nil; j.locked_at = nil}
53
+ end
54
+
55
+ # Find a few candidate jobs to run (in case some immediately get locked by others).
56
+ def self.find_available(worker_name, limit = 5, max_run_time = Worker.max_run_time)
57
+ jobs = all.select do |j|
58
+ j.run_at <= db_time_now &&
59
+ (j.locked_at.nil? || j.locked_at < db_time_now - max_run_time || j.locked_by == worker_name) &&
60
+ !j.failed?
61
+ end
62
+
63
+ jobs = jobs.select{|j| Worker.queues.include?(j.queue)} if Worker.queues.any?
64
+ jobs = jobs.select{|j| j.priority >= Worker.min_priority} if Worker.min_priority
65
+ jobs = jobs.select{|j| j.priority <= Worker.max_priority} if Worker.max_priority
66
+ jobs.sort_by{|j| [j.priority, j.run_at]}[0..limit-1]
67
+ end
68
+
69
+ # Lock this job for this worker.
70
+ # Returns true if we have the lock, false otherwise.
71
+ def lock_exclusively!(max_run_time, worker)
72
+ now = self.class.db_time_now
73
+ if locked_by != worker
74
+ # We don't own this job so we will update the locked_by name and the locked_at
75
+ self.locked_at = now
76
+ self.locked_by = worker
77
+ end
78
+
79
+ return true
80
+ end
81
+
82
+ def self.db_time_now
83
+ Time.current
84
+ end
85
+
86
+ def update_attributes(attrs = {})
87
+ attrs.each{|k,v| send(:"#{k}=", v)}
88
+ save
89
+ end
90
+
91
+ def destroy
92
+ self.class.all.delete(self)
93
+ end
94
+
95
+ def save
96
+ self.run_at ||= Time.current
97
+
98
+ self.class.all << self unless self.class.all.include?(self)
99
+ true
100
+ end
101
+
102
+ def save!; save; end
103
+
104
+ def reload
105
+ reset
106
+ self
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,38 @@
1
+ module BackburnerJobsManager
2
+ def setup
3
+ ActiveJob::Base.queue_adapter = :backburner
4
+ Backburner.configure do |config|
5
+ config.logger = Rails.logger
6
+ end
7
+ unless can_run?
8
+ puts "Cannot run integration tests for backburner. To be able to run integration tests for backburner you need to install and start beanstalkd.\n"
9
+ exit
10
+ end
11
+ end
12
+
13
+ def clear_jobs
14
+ tube.clear
15
+ end
16
+
17
+ def start_workers
18
+ @thread = Thread.new { Backburner.work "integration-tests" } # backburner dasherizes the queue name
19
+ end
20
+
21
+ def stop_workers
22
+ @thread.kill
23
+ end
24
+
25
+ def tube
26
+ @tube ||= Beaneater::Tube.new(Backburner::Worker.connection, "backburner.worker.queue.integration-tests") # backburner dasherizes the queue name
27
+ end
28
+
29
+ def can_run?
30
+ begin
31
+ Backburner::Worker.connection.send :connect!
32
+ rescue
33
+ return false
34
+ end
35
+ true
36
+ end
37
+ end
38
+
@@ -0,0 +1,20 @@
1
+ require 'delayed_job'
2
+ require 'delayed_job_active_record'
3
+
4
+ module DelayedJobJobsManager
5
+ def setup
6
+ ActiveJob::Base.queue_adapter = :delayed_job
7
+ end
8
+ def clear_jobs
9
+ Delayed::Job.delete_all
10
+ end
11
+
12
+ def start_workers
13
+ @worker = Delayed::Worker.new(quiet: true, sleep_delay: 0.5, queues: %w(integration_tests))
14
+ @thread = Thread.new { @worker.start }
15
+ end
16
+
17
+ def stop_workers
18
+ @worker.stop
19
+ end
20
+ end
@@ -0,0 +1,37 @@
1
+ module QueJobsManager
2
+ def setup
3
+ require 'sequel'
4
+ ActiveJob::Base.queue_adapter = :que
5
+ que_url = ENV['QUE_DATABASE_URL'] || 'postgres:///dummy_test'
6
+ uri = URI.parse(que_url)
7
+ user = uri.user||ENV['USER']
8
+ pass = uri.password
9
+ db = uri.path[1..-1]
10
+ %x{#{"PGPASSWORD=\"#{pass}\"" if pass} psql -c 'drop database if exists "#{db}"' -U #{user} -t template1}
11
+ %x{#{"PGPASSWORD=\"#{pass}\"" if pass} psql -c 'create database "#{db}"' -U #{user} -t template1}
12
+ Que.connection = Sequel.connect(que_url)
13
+ Que.migrate!
14
+ Que.mode = :off
15
+ Que.worker_count = 1
16
+ rescue Sequel::DatabaseConnectionError
17
+ puts "Cannot run integration tests for que. To be able to run integration tests for que you need to install and start postgresql.\n"
18
+ exit
19
+ end
20
+
21
+ def clear_jobs
22
+ Que.clear!
23
+ end
24
+
25
+ def start_workers
26
+ @thread = Thread.new do
27
+ loop do
28
+ Que::Job.work("integration_tests")
29
+ sleep 0.5
30
+ end
31
+ end
32
+ end
33
+
34
+ def stop_workers
35
+ @thread.kill
36
+ end
37
+ end
@@ -0,0 +1,49 @@
1
+ module ResqueJobsManager
2
+ def setup
3
+ ActiveJob::Base.queue_adapter = :resque
4
+ Resque.redis = Redis::Namespace.new 'active_jobs_int_test', redis: Redis.connect(url: "redis://127.0.0.1:6379/12", :thread_safe => true)
5
+ Resque.logger = Rails.logger
6
+ unless can_run?
7
+ puts "Cannot run integration tests for resque. To be able to run integration tests for resque you need to install and start redis.\n"
8
+ exit
9
+ end
10
+ end
11
+
12
+ def clear_jobs
13
+ Resque.queues.each { |queue_name| Resque.redis.del "queue:#{queue_name}" }
14
+ Resque.redis.keys("delayed:*").each { |key| Resque.redis.del "#{key}" }
15
+ Resque.redis.del "delayed_queue_schedule"
16
+ end
17
+
18
+ def start_workers
19
+ @resque_thread = Thread.new do
20
+ w = Resque::Worker.new("integration_tests")
21
+ w.term_child = true
22
+ w.work(0.5)
23
+ end
24
+ @scheduler_thread = Thread.new do
25
+ Resque::Scheduler.configure do |c|
26
+ c.poll_sleep_amount = 0.5
27
+ c.dynamic = true
28
+ c.quiet = true
29
+ c.logfile = nil
30
+ end
31
+ Resque::Scheduler.master_lock.release!
32
+ Resque::Scheduler.run
33
+ end
34
+ end
35
+
36
+ def stop_workers
37
+ @resque_thread.kill
38
+ @scheduler_thread.kill
39
+ end
40
+
41
+ def can_run?
42
+ begin
43
+ Resque.redis.client.connect
44
+ rescue
45
+ return false
46
+ end
47
+ true
48
+ end
49
+ end