backburner-allq 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.travis.yml +29 -0
  4. data/CHANGELOG.md +133 -0
  5. data/CONTRIBUTING.md +37 -0
  6. data/Gemfile +4 -0
  7. data/HOOKS.md +99 -0
  8. data/LICENSE +22 -0
  9. data/README.md +658 -0
  10. data/Rakefile +17 -0
  11. data/TODO +4 -0
  12. data/backburner-allq.gemspec +26 -0
  13. data/bin/backburner +7 -0
  14. data/circle.yml +3 -0
  15. data/deploy.sh +3 -0
  16. data/examples/custom.rb +25 -0
  17. data/examples/demo.rb +60 -0
  18. data/examples/god.rb +46 -0
  19. data/examples/hooked.rb +87 -0
  20. data/examples/retried.rb +31 -0
  21. data/examples/simple.rb +43 -0
  22. data/examples/stress.rb +31 -0
  23. data/lib/backburner.rb +75 -0
  24. data/lib/backburner/allq_wrapper.rb +317 -0
  25. data/lib/backburner/async_proxy.rb +25 -0
  26. data/lib/backburner/cli.rb +53 -0
  27. data/lib/backburner/configuration.rb +48 -0
  28. data/lib/backburner/connection.rb +157 -0
  29. data/lib/backburner/helpers.rb +193 -0
  30. data/lib/backburner/hooks.rb +53 -0
  31. data/lib/backburner/job.rb +118 -0
  32. data/lib/backburner/logger.rb +53 -0
  33. data/lib/backburner/performable.rb +95 -0
  34. data/lib/backburner/queue.rb +145 -0
  35. data/lib/backburner/tasks.rb +54 -0
  36. data/lib/backburner/version.rb +3 -0
  37. data/lib/backburner/worker.rb +221 -0
  38. data/lib/backburner/workers/forking.rb +52 -0
  39. data/lib/backburner/workers/simple.rb +29 -0
  40. data/lib/backburner/workers/threading.rb +163 -0
  41. data/lib/backburner/workers/threads_on_fork.rb +263 -0
  42. data/test/async_proxy_test.rb +36 -0
  43. data/test/back_burner_test.rb +88 -0
  44. data/test/connection_test.rb +179 -0
  45. data/test/fixtures/hooked.rb +122 -0
  46. data/test/fixtures/test_fork_jobs.rb +72 -0
  47. data/test/fixtures/test_forking_jobs.rb +56 -0
  48. data/test/fixtures/test_jobs.rb +87 -0
  49. data/test/fixtures/test_queue_settings.rb +14 -0
  50. data/test/helpers/templogger.rb +22 -0
  51. data/test/helpers_test.rb +278 -0
  52. data/test/hooks_test.rb +112 -0
  53. data/test/job_test.rb +185 -0
  54. data/test/logger_test.rb +44 -0
  55. data/test/performable_test.rb +88 -0
  56. data/test/queue_test.rb +69 -0
  57. data/test/test_helper.rb +128 -0
  58. data/test/worker_test.rb +157 -0
  59. data/test/workers/forking_worker_test.rb +181 -0
  60. data/test/workers/simple_worker_test.rb +350 -0
  61. data/test/workers/threading_worker_test.rb +104 -0
  62. data/test/workers/threads_on_fork_worker_test.rb +484 -0
  63. metadata +217 -0
@@ -0,0 +1,56 @@
1
+ class ResponseForkingJob
2
+ include Backburner::Queue
3
+ queue_priority 1000
4
+ def self.perform(data)
5
+ $worker_test_count += data['worker_test_count'].to_i if data['worker_test_count']
6
+ $worker_success = data['worker_success'] if data['worker_success']
7
+ $worker_test_count = data['worker_test_count_set'].to_i if data['worker_test_count_set']
8
+ $worker_raise = data['worker_raise'] if data['worker_raise']
9
+ end
10
+ end
11
+
12
+ class TestJobForking
13
+ include Backburner::Queue
14
+ queue_priority 1000
15
+ def self.perform(x, y)
16
+ Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
17
+ :worker_test_count_set => x + y
18
+ }], :queue => 'response'
19
+ end
20
+ end
21
+
22
+ class TestFailJobForking
23
+ include Backburner::Queue
24
+ def self.perform(x, y)
25
+ Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
26
+ :worker_raise => true
27
+ }], :queue => 'response'
28
+ end
29
+ end
30
+
31
+ class TestRetryJobForking
32
+ include Backburner::Queue
33
+ def self.perform(x, y)
34
+ if $worker_test_count <= 2
35
+ Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
36
+ :worker_test_count => 1
37
+ }], :queue => 'response'
38
+
39
+ raise RuntimeError
40
+ else # succeeds
41
+ Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
42
+ :worker_test_count => 1,
43
+ :worker_success => true
44
+ }], :queue => 'response'
45
+ end
46
+ end
47
+ end
48
+
49
+ class TestAsyncJobForking
50
+ include Backburner::Performable
51
+ def self.foo(x, y)
52
+ Backburner::Workers::Forking.enqueue ResponseForkingJob, [{
53
+ :worker_test_count_set => x * y
54
+ }], :queue => 'response'
55
+ end
56
+ end
@@ -0,0 +1,87 @@
1
+ $worker_test_count = 0
2
+ $worker_success = false
3
+
4
+ class TestPlainJob
5
+ def self.queue; "test-plain"; end
6
+ def self.perform(x, y); $worker_test_count += x + y + 1; end
7
+ end
8
+
9
+ class TestJob
10
+ include Backburner::Queue
11
+ queue_priority :medium
12
+ queue_respond_timeout 300
13
+ def self.perform(x, y); $worker_test_count += x + y; end
14
+ end
15
+
16
+ class TestSlowJob
17
+ include Backburner::Queue
18
+ queue_priority :medium
19
+ queue_respond_timeout 300
20
+ def self.perform(x, y); sleep 1; $worker_test_count += x + y; end
21
+ end
22
+
23
+ class TestStuckJob
24
+ include Backburner::Queue
25
+ queue_priority :medium
26
+ queue_respond_timeout 300
27
+ def self.perform(_x, _y)
28
+ loop do
29
+ sleep 0.5
30
+ end
31
+ end
32
+ end
33
+
34
+ class TestFailJob
35
+ include Backburner::Queue
36
+ def self.perform(x, y); raise RuntimeError; end
37
+ end
38
+
39
+ class TestRetryJob
40
+ include Backburner::Queue
41
+ def self.perform(x, y)
42
+ $worker_test_count += 1
43
+ raise RuntimeError unless $worker_test_count > 2
44
+ $worker_success = true
45
+ end
46
+ end
47
+
48
+ class TestConfigurableRetryJob
49
+ include Backburner::Queue
50
+ def self.perform(retry_count)
51
+ $worker_test_count += 1
52
+ raise RuntimeError unless $worker_test_count > retry_count
53
+ $worker_success = true
54
+ end
55
+ end
56
+
57
+ class TestRetryWithQueueOverridesJob
58
+ include Backburner::Queue
59
+ def self.perform(retry_count)
60
+ $worker_test_count += 1
61
+ raise RuntimeError unless $worker_test_count > retry_count
62
+ $worker_success = true
63
+ end
64
+
65
+ def self.queue_max_job_retries
66
+ 3
67
+ end
68
+
69
+ def self.queue_retry_delay
70
+ 0
71
+ end
72
+
73
+ def self.queue_retry_delay_proc
74
+ lambda { |min_retry_delay, num_retries| min_retry_delay + (num_retries ** 2) }
75
+ end
76
+ end
77
+
78
+ class TestAsyncJob
79
+ include Backburner::Performable
80
+ def self.foo(x, y); $worker_test_count = x * y; end
81
+ end
82
+
83
+ class TestLambdaQueueJob
84
+ include Backburner::Queue
85
+ queue lambda { |klass| klass.calculated_queue_name }
86
+ def self.calculated_queue_name; 'lambda-queue' end
87
+ end
@@ -0,0 +1,14 @@
1
+ class TestJobSettings
2
+ include Backburner::Queue
3
+ queue "job-settings:5:10:6"
4
+ def self.perform; end
5
+ end
6
+
7
+ class TestJobSettingsOverride
8
+ include Backburner::Queue
9
+ queue "job-settings-override:5:10:12"
10
+ queue_jobs_limit 10
11
+ queue_garbage_limit 1000
12
+ queue_retry_limit 2
13
+ def self.perform; end
14
+ end
@@ -0,0 +1,22 @@
1
+ class Templogger
2
+ attr_reader :logger, :log_path
3
+
4
+ def initialize(root_path)
5
+ @file = Tempfile.new('foo', root_path)
6
+ @log_path = @file.path
7
+ @logger = Logger.new(@log_path)
8
+ end
9
+
10
+ # wait_for_match /Completed TestJobFork/m
11
+ def wait_for_match(match_pattern)
12
+ sleep 0.1 until self.body =~ match_pattern
13
+ end
14
+
15
+ def body
16
+ File.read(@log_path)
17
+ end
18
+
19
+ def close
20
+ @file.close
21
+ end
22
+ end
@@ -0,0 +1,278 @@
1
+ require File.expand_path('../test_helper', __FILE__)
2
+
3
+ describe "Backburner::Helpers module" do
4
+ include Backburner::Helpers
5
+
6
+ describe "for classify method" do
7
+ it "should support simple classify" do
8
+ assert_equal "FooBarBaz", classify("foo-bar-baz")
9
+ end
10
+
11
+ it "should not affect existing classified strings" do
12
+ assert_equal "Foo::BarBaz", classify("Foo::BarBaz")
13
+ end
14
+ end # classify
15
+
16
+ describe "for constantize method" do
17
+ it "should constantize known constant" do
18
+ assert_equal Backburner, constantize("Backburner")
19
+ end
20
+
21
+ it "should properly report when constant is undefined" do
22
+ assert_raises(NameError) { constantize("FakeObject") }
23
+ end
24
+ end # constantize
25
+
26
+ describe "for dasherize method" do
27
+ it "should not harm existing dashed names" do
28
+ assert_equal "foo/bar-baz", dasherize("foo/bar-baz")
29
+ end
30
+
31
+ it "should properly convert simple names to dashes" do
32
+ assert_equal "foo-bar", dasherize("FooBar")
33
+ end
34
+
35
+ it "should properly convert class to dash with namespace" do
36
+ assert_equal "foo/bar-baz", dasherize("Foo::BarBaz")
37
+ end
38
+ end # dasherize
39
+
40
+ describe "for exception_message method" do
41
+ it "prints out message about failure" do
42
+ output = exception_message(RuntimeError.new("test"))
43
+ assert_match(/Exception RuntimeError/, output)
44
+ end
45
+ end # exception_message
46
+
47
+ describe "for queue_config" do
48
+ before { Backburner.expects(:configuration).returns(stub(:tube_namespace => "test.foo.job", :namespace_separator => '.')) }
49
+
50
+ it "accesses correct value for namespace" do
51
+ assert_equal "test.foo.job", queue_config.tube_namespace
52
+ end
53
+ end # config
54
+
55
+ describe "for expand_tube_name method" do
56
+ before { Backburner.stubs(:configuration).returns(stub(:tube_namespace => "test.foo.job.", :namespace_separator => '.', :primary_queue => "backburner-jobs")) }
57
+
58
+ it "supports base strings" do
59
+ assert_equal "test.foo.job.email/send-news", expand_tube_name("email/send_news")
60
+ end # simple string
61
+
62
+ it "supports qualified strings" do
63
+ assert_equal "test.foo.job.email/send-news", expand_tube_name("test.foo.job.email/send_news")
64
+ end # qualified string
65
+
66
+ it "supports base symbols" do
67
+ assert_equal "test.foo.job.email/send-news", expand_tube_name(:"email/send_news")
68
+ end # symbols
69
+
70
+ it "supports queue names" do
71
+ test = stub(:queue => "email/send_news")
72
+ assert_equal "test.foo.job.email/send-news", expand_tube_name(test)
73
+ end # queue names
74
+
75
+ it "supports class names" do
76
+ assert_equal "test.foo.job.backburner-jobs", expand_tube_name(RuntimeError)
77
+ end # class names
78
+
79
+ it "supports lambda in queue object" do
80
+ test = stub(:queue => lambda { |job_class| "email/send_news" })
81
+ assert_equal "test.foo.job.email/send-news", expand_tube_name(test)
82
+ end # lambdas in queue object
83
+
84
+ it "supports lambdas" do
85
+ test = lambda { "email/send_news" }
86
+ assert_equal "test.foo.job.email/send-news", expand_tube_name(test)
87
+ end #lambdas
88
+ end # expand_tube_name
89
+
90
+ describe "for alternative namespace separator" do
91
+ before { Backburner.stubs(:configuration).returns(stub(:tube_namespace => "test", :namespace_separator => '-', :primary_queue => "backburner-jobs")) }
92
+
93
+ it "uses alternative namespace separator" do
94
+ assert_equal "test-queue-name", expand_tube_name("queue_name")
95
+ end # simple string
96
+ end
97
+
98
+ describe "for resolve_priority method" do
99
+ before do
100
+ @original_queue_priority = Backburner.configuration.default_priority
101
+ Backburner.configure { |config| config.default_priority = 1000 }
102
+ end
103
+ after do
104
+ Backburner.configure { |config| config.default_priority = @original_queue_priority }
105
+ Backburner.configure { |config| config.priority_labels = Backburner::Configuration::PRIORITY_LABELS }
106
+ end
107
+
108
+ it "supports fix num priority" do
109
+ assert_equal 500, resolve_priority(500)
110
+ end
111
+
112
+ it "supports baked in priority alias" do
113
+ assert_equal 200, resolve_priority(:low)
114
+ assert_equal 0, resolve_priority(:high)
115
+ end
116
+
117
+ it "supports custom priority alias" do
118
+ Backburner.configure { |config| config.priority_labels = { :foo => 5 } }
119
+ assert_equal 5, resolve_priority(:foo)
120
+ end
121
+
122
+ it "supports aliased priority alias" do
123
+ Backburner.configure { |config| config.priority_labels = { :foo => 5, :bar => 'foo' } }
124
+ assert_equal 5, resolve_priority(:bar)
125
+ end
126
+
127
+ it "supports classes which respond to queue_priority" do
128
+ job = stub(:queue_priority => 600)
129
+ assert_equal 600, resolve_priority(job)
130
+ end
131
+
132
+ it "supports classes which respond to queue_priority with named alias" do
133
+ job = stub(:queue_priority => :low)
134
+ assert_equal 200, resolve_priority(job)
135
+ end
136
+
137
+ it "supports classes which returns null queue_priority" do
138
+ job = stub(:queue_priority => nil)
139
+ assert_equal 1000, resolve_priority(job)
140
+ end
141
+
142
+ it "supports classes which don't respond to queue_priority" do
143
+ job = stub(:fake => true)
144
+ assert_equal 1000, resolve_priority(job)
145
+ end
146
+
147
+ it "supports default pri for null values" do
148
+ assert_equal 1000, resolve_priority(nil)
149
+ end
150
+ end # resolve_priority
151
+
152
+ describe "for resolve_respond_timeout method" do
153
+ before do
154
+ @original_respond_timeout = Backburner.configuration.respond_timeout
155
+ Backburner.configure { |config| config.respond_timeout = 300 }
156
+ end
157
+ after { Backburner.configure { |config| config.respond_timeout = @original_respond_timeout } }
158
+
159
+ it "supports fix num respond_timeout" do
160
+ assert_equal 500, resolve_respond_timeout(500)
161
+ end
162
+
163
+ it "supports classes which respond to queue_respond_timeout" do
164
+ job = stub(:queue_respond_timeout => 600)
165
+ assert_equal 600, resolve_respond_timeout(job)
166
+ end
167
+
168
+ it "supports classes which returns null queue_respond_timeout" do
169
+ job = stub(:queue_respond_timeout => nil)
170
+ assert_equal 300, resolve_respond_timeout(job)
171
+ end
172
+
173
+ it "supports classes which don't respond to queue_respond_timeout" do
174
+ job = stub(:fake => true)
175
+ assert_equal 300, resolve_respond_timeout(job)
176
+ end
177
+
178
+ it "supports default ttr for null values" do
179
+ assert_equal 300, resolve_respond_timeout(nil)
180
+ end
181
+ end # resolve_respond_timeout
182
+
183
+ describe "for resolve_max_job_retries method" do
184
+ before do
185
+ @original_max_job_retries = Backburner.configuration.max_job_retries
186
+ Backburner.configure { |config| config.max_job_retries = 300 }
187
+ end
188
+ after { Backburner.configure { |config| config.max_job_retries = @original_max_job_retries } }
189
+
190
+ it "supports fix num max_job_retries" do
191
+ assert_equal 500, resolve_max_job_retries(500)
192
+ end
193
+
194
+ it "supports classes which respond to queue_max_job_retries" do
195
+ job = stub(:queue_max_job_retries => 600)
196
+ assert_equal 600, resolve_max_job_retries(job)
197
+ end
198
+
199
+ it "supports classes which return null queue_max_job_retries" do
200
+ job = stub(:queue_max_job_retries => nil)
201
+ assert_equal 300, resolve_max_job_retries(job)
202
+ end
203
+
204
+ it "supports classes which don't respond to queue_max_job_retries" do
205
+ job = stub(:fake => true)
206
+ assert_equal 300, resolve_max_job_retries(job)
207
+ end
208
+
209
+ it "supports default max_job_retries for null values" do
210
+ assert_equal 300, resolve_max_job_retries(nil)
211
+ end
212
+ end # resolve_max_job_retries
213
+
214
+ describe "for resolve_retry_delay method" do
215
+ before do
216
+ @original_retry_delay = Backburner.configuration.retry_delay
217
+ Backburner.configure { |config| config.retry_delay = 300 }
218
+ end
219
+ after { Backburner.configure { |config| config.retry_delay = @original_retry_delay } }
220
+
221
+ it "supports fix num retry_delay" do
222
+ assert_equal 500, resolve_retry_delay(500)
223
+ end
224
+
225
+ it "supports classes which respond to queue_retry_delay" do
226
+ job = stub(:queue_retry_delay => 600)
227
+ assert_equal 600, resolve_retry_delay(job)
228
+ end
229
+
230
+ it "supports classes which return null queue_retry_delay" do
231
+ job = stub(:queue_retry_delay => nil)
232
+ assert_equal 300, resolve_retry_delay(job)
233
+ end
234
+
235
+ it "supports classes which don't respond to queue_retry_delay" do
236
+ job = stub(:fake => true)
237
+ assert_equal 300, resolve_retry_delay(job)
238
+ end
239
+
240
+ it "supports default retry_delay for null values" do
241
+ assert_equal 300, resolve_retry_delay(nil)
242
+ end
243
+ end # resolve_retry_delay
244
+
245
+ describe "for resolve_retry_delay_proc method" do
246
+ before do
247
+ @config_retry_delay_proc = lambda { |x, y| x + y } # Default config proc adds two values
248
+ @override_delay_proc = lambda { |x, y| x - y } # Overriden proc subtracts values
249
+ @original_retry_delay_proc = Backburner.configuration.retry_delay_proc
250
+ Backburner.configure { |config| config.retry_delay_proc = @config_retry_delay_proc }
251
+ end
252
+ after { Backburner.configure { |config| config.retry_delay_proc = @original_retry_delay_proc } }
253
+
254
+ # Rather than compare Procs execute them and compare the output
255
+ it "supports proc retry_delay_proc" do
256
+ assert_equal @override_delay_proc.call(2, 1), resolve_retry_delay_proc(@override_delay_proc).call(2, 1)
257
+ end
258
+
259
+ it "supports classes which respond to queue_retry_delay_proc" do
260
+ job = stub(:queue_retry_delay_proc => @override_delay_proc)
261
+ assert_equal @override_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
262
+ end
263
+
264
+ it "supports classes which return null queue_retry_delay_proc" do
265
+ job = stub(:queue_retry_delay_proc => nil)
266
+ assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
267
+ end
268
+
269
+ it "supports classes which don't respond to queue_retry_delay_proc" do
270
+ job = stub(:fake => true)
271
+ assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(job).call(2, 1)
272
+ end
273
+
274
+ it "supports default retry_delay_proc for null values" do
275
+ assert_equal @original_retry_delay_proc.call(2, 1), resolve_retry_delay_proc(nil).call(2, 1)
276
+ end
277
+ end # resolve_retry_delay_proc
278
+ end