backburner 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +24 -4
- data/README.md +70 -14
- data/lib/active_job/queue_adapters/backburner_adapter.rb +9 -2
- data/lib/backburner/cli.rb +1 -1
- data/lib/backburner/enqueue_pool.rb +2 -0
- data/lib/backburner/performable.rb +28 -3
- data/lib/backburner/version.rb +1 -1
- data/lib/backburner/worker.rb +38 -4
- data/lib/backburner/workers/threading.rb +16 -0
- data/lib/backburner/workers/threads_on_fork.rb +1 -0
- metadata +8 -82
- data/.github/workflows/ci.yml +0 -45
- data/.github/workflows/github-release.yml +0 -58
- data/.github/workflows/release.yml +0 -221
- data/.gitignore +0 -17
- data/CONTRIBUTING.md +0 -37
- data/Gemfile +0 -7
- data/Rakefile +0 -17
- data/TODO +0 -4
- data/backburner.gemspec +0 -31
- data/examples/custom.rb +0 -25
- data/examples/demo.rb +0 -60
- data/examples/god.rb +0 -46
- data/examples/hooked.rb +0 -87
- data/examples/retried.rb +0 -31
- data/examples/simple.rb +0 -43
- data/examples/stress.rb +0 -31
- data/test/active_job_adapter_test.rb +0 -24
- data/test/async_proxy_test.rb +0 -36
- data/test/back_burner_test.rb +0 -101
- data/test/connection_test.rb +0 -201
- data/test/fixtures/active_job_app.rb +0 -53
- data/test/fixtures/active_jobs.rb +0 -11
- data/test/fixtures/autoload/test_broken_name_job.rb +0 -4
- data/test/fixtures/autoload/test_broken_no_method_job.rb +0 -4
- data/test/fixtures/autoload/test_broken_require_job.rb +0 -4
- data/test/fixtures/autoload/test_broken_runtime_job.rb +0 -4
- data/test/fixtures/autoload/test_broken_syntax_job.rb +0 -4
- data/test/fixtures/autoload/test_broken_timeout_job.rb +0 -4
- data/test/fixtures/autoload/test_same_name_job.rb +0 -5
- data/test/fixtures/hooked.rb +0 -122
- data/test/fixtures/test_fork_jobs.rb +0 -77
- data/test/fixtures/test_forking_jobs.rb +0 -56
- data/test/fixtures/test_jobs.rb +0 -140
- data/test/fixtures/test_queue_settings.rb +0 -54
- data/test/helpers/templogger.rb +0 -22
- data/test/helpers_test.rb +0 -462
- data/test/hooks_test.rb +0 -112
- data/test/job_test.rb +0 -207
- data/test/logger_test.rb +0 -71
- data/test/performable_test.rb +0 -138
- data/test/queue_test.rb +0 -132
- data/test/test_helper.rb +0 -182
- data/test/worker_test.rb +0 -502
- data/test/workers/forking_worker_test.rb +0 -199
- data/test/workers/simple_worker_test.rb +0 -622
- data/test/workers/threading_worker_test.rb +0 -125
- data/test/workers/threads_on_fork_worker_test.rb +0 -751
|
@@ -1,77 +0,0 @@
|
|
|
1
|
-
class ResponseJob
|
|
2
|
-
include Backburner::Queue
|
|
3
|
-
queue_priority 1000
|
|
4
|
-
def self.perform(data)
|
|
5
|
-
$worker_test_count += data['worker_test_count'].to_i if data['worker_test_count']
|
|
6
|
-
$worker_success = data['worker_success'] if data['worker_success']
|
|
7
|
-
$worker_test_count = data['worker_test_count_set'].to_i if data['worker_test_count_set']
|
|
8
|
-
$worker_raise = data['worker_raise'] if data['worker_raise']
|
|
9
|
-
end
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
class TestJobFork
|
|
13
|
-
include Backburner::Queue
|
|
14
|
-
queue "test-job-fork"
|
|
15
|
-
queue_priority 1000
|
|
16
|
-
def self.perform(x, y)
|
|
17
|
-
Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
|
|
18
|
-
:worker_test_count_set => x + y
|
|
19
|
-
}], :queue => 'response'
|
|
20
|
-
end
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
class TestFailJobFork
|
|
24
|
-
include Backburner::Queue
|
|
25
|
-
queue "test-fail-job-fork"
|
|
26
|
-
def self.perform(x, y)
|
|
27
|
-
Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
|
|
28
|
-
:worker_raise => true
|
|
29
|
-
}], :queue => 'response'
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
class TestRetryJobFork
|
|
34
|
-
include Backburner::Queue
|
|
35
|
-
def self.perform(x, y)
|
|
36
|
-
$worker_test_count += 1
|
|
37
|
-
|
|
38
|
-
if $worker_test_count <= 2
|
|
39
|
-
Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
|
|
40
|
-
:worker_test_count => 1
|
|
41
|
-
}], :queue => 'response'
|
|
42
|
-
|
|
43
|
-
raise RuntimeError
|
|
44
|
-
else # succeeds
|
|
45
|
-
Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
|
|
46
|
-
:worker_test_count => 1,
|
|
47
|
-
:worker_success => true
|
|
48
|
-
}], :queue => 'response'
|
|
49
|
-
end
|
|
50
|
-
end
|
|
51
|
-
end
|
|
52
|
-
|
|
53
|
-
class TestAsyncJobFork
|
|
54
|
-
include Backburner::Performable
|
|
55
|
-
def self.foo(x, y)
|
|
56
|
-
Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
|
|
57
|
-
:worker_test_count_set => x * y
|
|
58
|
-
}], :queue => 'response'
|
|
59
|
-
end
|
|
60
|
-
end
|
|
61
|
-
|
|
62
|
-
class TestJobMultithreadFork
|
|
63
|
-
include Backburner::Queue
|
|
64
|
-
queue "test-job-multithread-fork"
|
|
65
|
-
queue_priority 1000
|
|
66
|
-
def self.perform(x, y)
|
|
67
|
-
sleep 1 # simluate work
|
|
68
|
-
Backburner::Workers::ThreadsOnFork.enqueue ResponseJob, [{
|
|
69
|
-
:worker_test_count_set => x + y
|
|
70
|
-
}], :queue => 'response'
|
|
71
|
-
end
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
class TestJobNoopFork
|
|
75
|
-
include Backburner::Queue
|
|
76
|
-
def self.perform; end
|
|
77
|
-
end
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
class ResponseForkingJob
|
|
2
|
-
include Backburner::Queue
|
|
3
|
-
queue_priority 1000
|
|
4
|
-
def self.perform(data)
|
|
5
|
-
$worker_test_count += data['worker_test_count'].to_i if data['worker_test_count']
|
|
6
|
-
$worker_success = data['worker_success'] if data['worker_success']
|
|
7
|
-
$worker_test_count = data['worker_test_count_set'].to_i if data['worker_test_count_set']
|
|
8
|
-
$worker_raise = data['worker_raise'] if data['worker_raise']
|
|
9
|
-
end
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
class TestJobForking
|
|
13
|
-
include Backburner::Queue
|
|
14
|
-
queue_priority 1000
|
|
15
|
-
def self.perform(x, y)
|
|
16
|
-
Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
|
|
17
|
-
:worker_test_count_set => x + y
|
|
18
|
-
}], :queue => 'response'
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
class TestFailJobForking
|
|
23
|
-
include Backburner::Queue
|
|
24
|
-
def self.perform(x, y)
|
|
25
|
-
Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
|
|
26
|
-
:worker_raise => true
|
|
27
|
-
}], :queue => 'response'
|
|
28
|
-
end
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
class TestRetryJobForking
|
|
32
|
-
include Backburner::Queue
|
|
33
|
-
def self.perform(x, y)
|
|
34
|
-
if $worker_test_count <= 2
|
|
35
|
-
Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
|
|
36
|
-
:worker_test_count => 1
|
|
37
|
-
}], :queue => 'response'
|
|
38
|
-
|
|
39
|
-
raise RuntimeError
|
|
40
|
-
else # succeeds
|
|
41
|
-
Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
|
|
42
|
-
:worker_test_count => 1,
|
|
43
|
-
:worker_success => true
|
|
44
|
-
}], :queue => 'response'
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
class TestAsyncJobForking
|
|
50
|
-
include Backburner::Performable
|
|
51
|
-
def self.foo(x, y)
|
|
52
|
-
Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
|
|
53
|
-
:worker_test_count_set => x * y
|
|
54
|
-
}], :queue => 'response'
|
|
55
|
-
end
|
|
56
|
-
end
|
data/test/fixtures/test_jobs.rb
DELETED
|
@@ -1,140 +0,0 @@
|
|
|
1
|
-
$worker_test_count = 0
|
|
2
|
-
$worker_success = false
|
|
3
|
-
|
|
4
|
-
class TestPlainJob
|
|
5
|
-
def self.queue; "test-plain"; end
|
|
6
|
-
def self.perform(x, y); $worker_test_count += x + y + 1; end
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
class TestJob
|
|
10
|
-
include Backburner::Queue
|
|
11
|
-
queue_priority :medium
|
|
12
|
-
queue_respond_timeout 300
|
|
13
|
-
def self.perform(x, y); $worker_test_count += x + y; end
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
class TestParentQueueJob
|
|
17
|
-
include Backburner::Queue
|
|
18
|
-
queue "test-parent"
|
|
19
|
-
queue_priority :high
|
|
20
|
-
queue_respond_timeout 250
|
|
21
|
-
def self.perform(x, y); $worker_test_count += x + y; end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
class TestChildQueueJob < TestParentQueueJob; end
|
|
25
|
-
|
|
26
|
-
class TestNamedQueueJob
|
|
27
|
-
include Backburner::Queue
|
|
28
|
-
queue lambda { |klass| klass.name.downcase }
|
|
29
|
-
def self.perform; end
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
class TestSlowJob
|
|
33
|
-
include Backburner::Queue
|
|
34
|
-
queue_priority :medium
|
|
35
|
-
queue_respond_timeout 300
|
|
36
|
-
def self.perform(x, y); sleep 1; $worker_test_count += x + y; end
|
|
37
|
-
end
|
|
38
|
-
|
|
39
|
-
class TestStuckJob
|
|
40
|
-
include Backburner::Queue
|
|
41
|
-
queue_priority :medium
|
|
42
|
-
queue_respond_timeout 300
|
|
43
|
-
def self.perform(_x, _y)
|
|
44
|
-
loop do
|
|
45
|
-
sleep 0.5
|
|
46
|
-
end
|
|
47
|
-
end
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
class TestFailJob
|
|
51
|
-
include Backburner::Queue
|
|
52
|
-
def self.perform(x, y); raise RuntimeError; end
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
class TestRetryJob
|
|
56
|
-
include Backburner::Queue
|
|
57
|
-
def self.perform(x, y)
|
|
58
|
-
$worker_test_count += 1
|
|
59
|
-
raise RuntimeError unless $worker_test_count > 2
|
|
60
|
-
$worker_success = true
|
|
61
|
-
end
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
class TestFloatRetryDelayJob
|
|
65
|
-
include Backburner::Queue
|
|
66
|
-
queue_retry_delay 1.5
|
|
67
|
-
def self.perform(retry_count)
|
|
68
|
-
raise RuntimeError
|
|
69
|
-
end
|
|
70
|
-
end
|
|
71
|
-
|
|
72
|
-
class TestFloatRespondTimeoutJob
|
|
73
|
-
include Backburner::Queue
|
|
74
|
-
queue "test-plain"
|
|
75
|
-
queue_respond_timeout 1.5
|
|
76
|
-
def self.perform; end
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
class TestConfigurableRetryJob
|
|
80
|
-
include Backburner::Queue
|
|
81
|
-
def self.perform(retry_count)
|
|
82
|
-
$worker_test_count += 1
|
|
83
|
-
raise RuntimeError unless $worker_test_count > retry_count
|
|
84
|
-
$worker_success = true
|
|
85
|
-
end
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
class TestRetryWithQueueOverridesJob
|
|
89
|
-
include Backburner::Queue
|
|
90
|
-
def self.perform(retry_count)
|
|
91
|
-
$worker_test_count += 1
|
|
92
|
-
raise RuntimeError unless $worker_test_count > retry_count
|
|
93
|
-
$worker_success = true
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
def self.queue_max_job_retries
|
|
97
|
-
3
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
def self.queue_retry_delay
|
|
101
|
-
0
|
|
102
|
-
end
|
|
103
|
-
|
|
104
|
-
def self.queue_retry_delay_proc
|
|
105
|
-
lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
|
|
106
|
-
end
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
class TestAsyncJob
|
|
110
|
-
include Backburner::Performable
|
|
111
|
-
def self.foo(x, y); $worker_test_count = x * y; end
|
|
112
|
-
end
|
|
113
|
-
|
|
114
|
-
class TestLambdaQueueJob
|
|
115
|
-
include Backburner::Queue
|
|
116
|
-
queue lambda { |klass| klass.calculated_queue_name }
|
|
117
|
-
def self.calculated_queue_name; 'lambda-queue' end
|
|
118
|
-
end
|
|
119
|
-
|
|
120
|
-
class TestOverTTRJob
|
|
121
|
-
include Backburner::Queue
|
|
122
|
-
queue_respond_timeout 2
|
|
123
|
-
# Outlives its TTR without Timeout being able to interrupt it (like a blocking C call)
|
|
124
|
-
def self.perform; Thread.handle_interrupt(Object => :never) { sleep 3 }; end
|
|
125
|
-
end
|
|
126
|
-
|
|
127
|
-
# Like an autoloader that doesn't find a file for the constant
|
|
128
|
-
module AutoloadingDemo
|
|
129
|
-
def self.const_missing(name); raise LoadError, "cannot load such file -- autoloading_demo/#{name}"; end
|
|
130
|
-
end
|
|
131
|
-
|
|
132
|
-
# Job classes an app autoloads: whose files fail to load, and one whose file isn't there
|
|
133
|
-
autoload :TestBrokenNameJob, File.expand_path('../autoload/test_broken_name_job', __FILE__)
|
|
134
|
-
autoload :TestBrokenRequireJob, File.expand_path('../autoload/test_broken_require_job', __FILE__)
|
|
135
|
-
autoload :TestBrokenNoMethodJob, File.expand_path('../autoload/test_broken_no_method_job', __FILE__)
|
|
136
|
-
autoload :TestNoFileJob, File.expand_path('../autoload/test_no_such_job', __FILE__)
|
|
137
|
-
autoload :TestBrokenRuntimeJob, File.expand_path('../autoload/test_broken_runtime_job', __FILE__)
|
|
138
|
-
autoload :TestSameNameJob, File.expand_path('../autoload/test_same_name_job', __FILE__)
|
|
139
|
-
autoload :TestBrokenSyntaxJob, File.expand_path('../autoload/test_broken_syntax_job', __FILE__)
|
|
140
|
-
autoload :TestBrokenTimeoutJob, File.expand_path('../autoload/test_broken_timeout_job', __FILE__)
|
|
@@ -1,54 +0,0 @@
|
|
|
1
|
-
class TestJobSettings
|
|
2
|
-
include Backburner::Queue
|
|
3
|
-
queue "job-settings:5:10:6"
|
|
4
|
-
def self.perform; end
|
|
5
|
-
end
|
|
6
|
-
|
|
7
|
-
class TestJobSettingsOverride
|
|
8
|
-
include Backburner::Queue
|
|
9
|
-
queue "job-settings-override:5:10:12"
|
|
10
|
-
queue_jobs_limit 10
|
|
11
|
-
queue_garbage_limit 1000
|
|
12
|
-
queue_retry_limit 2
|
|
13
|
-
def self.perform; end
|
|
14
|
-
end
|
|
15
|
-
class TestTubeLimitsJob
|
|
16
|
-
include Backburner::Queue
|
|
17
|
-
queue "tube-limits"
|
|
18
|
-
queue_jobs_limit 5
|
|
19
|
-
queue_garbage_limit 100
|
|
20
|
-
queue_retry_limit 3
|
|
21
|
-
def self.perform; end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
# Shares its parent's tube, whose options stay the parent's
|
|
25
|
-
class TestTubeLimitsChildJob < TestTubeLimitsJob
|
|
26
|
-
queue_jobs_limit 1
|
|
27
|
-
queue_garbage_limit 7
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
class TestTubeLimitsLambdaJob
|
|
31
|
-
include Backburner::Queue
|
|
32
|
-
queue lambda { |klass| "tube-limits-#{klass.name.downcase}" }
|
|
33
|
-
queue_jobs_limit 3
|
|
34
|
-
def self.perform; end
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
# Gets a tube of its own from the inherited lambda, so its options are its own
|
|
38
|
-
class TestTubeLimitsLambdaChildJob < TestTubeLimitsLambdaJob
|
|
39
|
-
queue_jobs_limit 4
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
class TestTubeFewLimitsJob
|
|
43
|
-
include Backburner::Queue
|
|
44
|
-
queue "tube-few-limits"
|
|
45
|
-
queue_retry_limit 2
|
|
46
|
-
def self.perform; end
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
# Shares its parent's tube and fills in the limits the parent leaves unset
|
|
50
|
-
class TestTubeFewLimitsChildJob < TestTubeFewLimitsJob
|
|
51
|
-
queue_jobs_limit 3
|
|
52
|
-
queue_garbage_limit 50
|
|
53
|
-
queue_retry_limit 9
|
|
54
|
-
end
|
data/test/helpers/templogger.rb
DELETED
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
class Templogger
|
|
2
|
-
attr_reader :logger, :log_path
|
|
3
|
-
|
|
4
|
-
def initialize(root_path)
|
|
5
|
-
@file = Tempfile.new('foo', root_path)
|
|
6
|
-
@log_path = @file.path
|
|
7
|
-
@logger = Logger.new(@log_path)
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
# wait_for_match /Completed TestJobFork/m
|
|
11
|
-
def wait_for_match(match_pattern)
|
|
12
|
-
sleep 0.1 until self.body =~ match_pattern
|
|
13
|
-
end
|
|
14
|
-
|
|
15
|
-
def body
|
|
16
|
-
File.read(@log_path)
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
def close
|
|
20
|
-
@file.close! # unlink it now: a Tempfile finalizer left for GC calls Process.pid, which some tests mock
|
|
21
|
-
end
|
|
22
|
-
end
|