proletariat 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +1 -1
- data/lib/proletariat/configuration.rb +13 -0
- data/lib/proletariat/publisher.rb +4 -2
- data/lib/proletariat/runner.rb +3 -2
- data/lib/proletariat/subscriber.rb +134 -9
- data/lib/proletariat/testing/expectation_guarantor.rb +4 -2
- data/lib/proletariat/version.rb +1 -1
- data/lib/proletariat/worker.rb +4 -3
- data/spec/lib/proletariat_spec.rb +30 -5
- data/spec/lib/testing/expectation_guarantor_spec.rb +3 -3
- data/spec/lib/worker_spec.rb +4 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 857a43f0570e09ed05e46f910b5a746474116604
|
4
|
+
data.tar.gz: 4c5a325c9d11bf52e1ac69b5c1c365a291ecfcc7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 00801d41f5a02e885f82a15bcf4d189e3f5ebe80892afe982cec5efe2f3effe3dbbe3e1aca7e8477110907045968bb991ea83ddeac1f15ef844eaba287428049
|
7
|
+
data.tar.gz: b13a39c9cb321ade6df9a2566d00942edd1ddb35339cae3102e2b1d33279a11f585e8aeef0e7a494cb7334d08255b5308c03e4f748fbef9b0a5b632c56937486
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
@@ -41,7 +41,7 @@ Here's a complete example:
|
|
41
41
|
class SendUserIntroductoryEmail < Proletariat::Worker
|
42
42
|
listen_on 'user.created'
|
43
43
|
|
44
|
-
def work(message, key)
|
44
|
+
def work(message, key, headers)
|
45
45
|
params = JSON.parse(message)
|
46
46
|
|
47
47
|
UserMailer.introductory_email(params).deliver!
|
@@ -4,6 +4,9 @@ module Proletariat
|
|
4
4
|
# Public: The default name used for the RabbitMQ topic exchange.
|
5
5
|
DEFAULT_EXCHANGE_NAME = 'proletariat'
|
6
6
|
|
7
|
+
# Public: The default maximum seconds to delay a failed job requeue.
|
8
|
+
DEFAULT_MAX_RETRY_DELAY = (ENV['MAX_RETRY_DELAY'] || 60).to_i
|
9
|
+
|
7
10
|
# Public: The default number of threads to use for publishers.
|
8
11
|
DEFAULT_PUBLISHER_THREADS = (ENV['PUBLISHER_THREADS'] || 2).to_i
|
9
12
|
|
@@ -19,6 +22,9 @@ module Proletariat
|
|
19
22
|
# Internal: Sets the logger.
|
20
23
|
attr_writer :logger
|
21
24
|
|
25
|
+
# Internal: Sets the maximum seconds to delay a failed job requeue.
|
26
|
+
attr_writer :max_retry_delay
|
27
|
+
|
22
28
|
# Internal: Sets the number of threads to use for publishers.
|
23
29
|
attr_writer :publisher_threads
|
24
30
|
|
@@ -67,6 +73,13 @@ module Proletariat
|
|
67
73
|
@logger ||= Logger.new(STDOUT)
|
68
74
|
end
|
69
75
|
|
76
|
+
# Public: Returns the set max delay seconds or a default.
|
77
|
+
#
|
78
|
+
# Returns a Fixnum.
|
79
|
+
def max_retry_delay
|
80
|
+
@max_retry_delay ||= DEFAULT_MAX_RETRY_DELAY
|
81
|
+
end
|
82
|
+
|
70
83
|
# Public: Returns the set number of publisher threads or a default.
|
71
84
|
#
|
72
85
|
# Returns a Fixnum.
|
@@ -44,10 +44,12 @@ module Proletariat
|
|
44
44
|
# with the RabbitMQ convention you can use the '*' character to
|
45
45
|
# replace one word and the '#' to replace many words.
|
46
46
|
# message - The message as a String.
|
47
|
+
# headers - Hash of message headers.
|
47
48
|
#
|
48
49
|
# Returns nil.
|
49
|
-
def work(to, message)
|
50
|
-
exchange.publish message, routing_key: to, persistent: true
|
50
|
+
def work(to, message, headers)
|
51
|
+
exchange.publish message, routing_key: to, persistent: true,
|
52
|
+
headers: headers
|
51
53
|
|
52
54
|
nil
|
53
55
|
end
|
data/lib/proletariat/runner.rb
CHANGED
@@ -25,10 +25,11 @@ module Proletariat
|
|
25
25
|
# with the RabbitMQ convention you can use the '*' character to
|
26
26
|
# replace one word and the '#' to replace many words.
|
27
27
|
# message - The message as a String.
|
28
|
+
# headers - Hash of message headers.
|
28
29
|
#
|
29
30
|
# Returns nil.
|
30
|
-
def publish(to, message)
|
31
|
-
supervisor['publishers'].post to, message
|
31
|
+
def publish(to, message = '', headers = {})
|
32
|
+
supervisor['publishers'].post to, message, headers
|
32
33
|
|
33
34
|
nil
|
34
35
|
end
|
@@ -58,6 +58,8 @@ module Proletariat
|
|
58
58
|
acknowledger.acknowledge_on_channel channel
|
59
59
|
acknowledgers.delete acknowledger
|
60
60
|
end
|
61
|
+
|
62
|
+
completed_retries.each { |r| scheduled_retries.delete r }
|
61
63
|
end
|
62
64
|
|
63
65
|
# Public: Purge the RabbitMQ queue.
|
@@ -102,6 +104,39 @@ module Proletariat
|
|
102
104
|
nil
|
103
105
|
end
|
104
106
|
|
107
|
+
# Internal: Get scheduled retries whose messages have been requeued.
|
108
|
+
#
|
109
|
+
# Returns an Array of Retrys.
|
110
|
+
def completed_retries
|
111
|
+
scheduled_retries.select { |r| r.requeued? }
|
112
|
+
end
|
113
|
+
|
114
|
+
# Internal: Forwards all message bodies to listener#post. Auto-acks
|
115
|
+
# messages not meant for this subscriber's workers.
|
116
|
+
#
|
117
|
+
# Returns nil.
|
118
|
+
def handle_message(info, properties, body)
|
119
|
+
if handles_worker_type? properties.headers['worker']
|
120
|
+
future = listener.post?(body, info.routing_key, properties.headers)
|
121
|
+
acknowledgers << Acknowledger.new(future, info.delivery_tag, {
|
122
|
+
message: body, key: info.routing_key, headers: properties.headers,
|
123
|
+
worker: queue_config.queue_name }, scheduled_retries)
|
124
|
+
else
|
125
|
+
channel.acknowledge info.delivery_tag
|
126
|
+
end
|
127
|
+
|
128
|
+
nil
|
129
|
+
end
|
130
|
+
|
131
|
+
# Internal: Checks if subscriber should handle message for given worker
|
132
|
+
# header.
|
133
|
+
#
|
134
|
+
# Returns true if should be handled or header is nil.
|
135
|
+
# Returns false if should not be handled.
|
136
|
+
def handles_worker_type?(worker_header)
|
137
|
+
[nil, queue_config.queue_name].include? worker_header
|
138
|
+
end
|
139
|
+
|
105
140
|
# Internal: Get acknowledgers for messages whose work has completed.
|
106
141
|
#
|
107
142
|
# Returns an Array of Acknowledgers.
|
@@ -111,14 +146,18 @@ module Proletariat
|
|
111
146
|
end
|
112
147
|
end
|
113
148
|
|
149
|
+
def scheduled_retries
|
150
|
+
@scheduled_retries ||= []
|
151
|
+
end
|
152
|
+
|
114
153
|
# Internal: Starts a consumer on the queue. The consumer forwards all
|
115
|
-
# message bodies to listener#post.
|
154
|
+
# message bodies to listener#post. Auto-acks messages not meant
|
155
|
+
# for this subscriber's workers.
|
116
156
|
#
|
117
157
|
# Returns nil.
|
118
158
|
def start_consumer
|
119
159
|
@consumer = bunny_queue.subscribe ack: true do |info, properties, body|
|
120
|
-
|
121
|
-
acknowledgers << Acknowledger.new(future, info.delivery_tag)
|
160
|
+
handle_message info, properties, body
|
122
161
|
|
123
162
|
nil
|
124
163
|
end
|
@@ -133,6 +172,7 @@ module Proletariat
|
|
133
172
|
def stop_consumer
|
134
173
|
@consumer.cancel if @consumer
|
135
174
|
wait_for_acknowledgers if acknowledgers.any?
|
175
|
+
scheduled_retries.each { |r| r.expedite }
|
136
176
|
|
137
177
|
nil
|
138
178
|
end
|
@@ -160,11 +200,15 @@ module Proletariat
|
|
160
200
|
|
161
201
|
# Public: Creates a new Acknowledger instance.
|
162
202
|
#
|
163
|
-
# future
|
164
|
-
# delivery_tag
|
165
|
-
|
166
|
-
|
167
|
-
|
203
|
+
# future - A future-like object holding the Worker response.
|
204
|
+
# delivery_tag - The RabbitMQ delivery tag for ack/nacking.
|
205
|
+
# properties - The original message properties; for requeuing.
|
206
|
+
# scheduled_retries - An Array to hold any created Retrys.
|
207
|
+
def initialize(future, delivery_tag, properties, scheduled_retries)
|
208
|
+
@future = future
|
209
|
+
@delivery_tag = delivery_tag
|
210
|
+
@properties = properties
|
211
|
+
@scheduled_retries = scheduled_retries
|
168
212
|
end
|
169
213
|
|
170
214
|
# Public: Retrieves the value from the future and sends the relevant
|
@@ -232,7 +276,9 @@ module Proletariat
|
|
232
276
|
# Returns nil.
|
233
277
|
def acknowledge_error(channel)
|
234
278
|
Proletariat.logger.error future.reason
|
235
|
-
|
279
|
+
|
280
|
+
scheduled_retries << Retry.new(properties)
|
281
|
+
channel.acknowledge delivery_tag
|
236
282
|
|
237
283
|
nil
|
238
284
|
end
|
@@ -242,6 +288,85 @@ module Proletariat
|
|
242
288
|
|
243
289
|
# Internal: Returns the future-like object holding the Worker response.
|
244
290
|
attr_reader :future
|
291
|
+
|
292
|
+
# Internal: Returns the original message properties.
|
293
|
+
attr_reader :properties
|
294
|
+
|
295
|
+
# Internal: Returns the Array of Retrys.
|
296
|
+
attr_reader :scheduled_retries
|
297
|
+
|
298
|
+
# Internal: Used publish an exponential delayed requeue for failures.
|
299
|
+
class Retry
|
300
|
+
# Public: Creates a new Retry instance. Sets appropriate headers for
|
301
|
+
# requeue message.
|
302
|
+
#
|
303
|
+
# properties - The original message properties.
|
304
|
+
def initialize(properties)
|
305
|
+
@properties = properties
|
306
|
+
|
307
|
+
properties[:headers]['failures'] = failures
|
308
|
+
properties[:headers]['worker'] = properties[:worker]
|
309
|
+
|
310
|
+
@scheduled_task = Concurrent::ScheduledTask.new(retry_delay) do
|
311
|
+
requeue_message
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
# Public: Attempt to requeue the message immediately if pending or
|
316
|
+
# wait for natural completion.
|
317
|
+
#
|
318
|
+
# Returns nil.
|
319
|
+
def expedite
|
320
|
+
if scheduled_task.cancel
|
321
|
+
requeue_message
|
322
|
+
else
|
323
|
+
scheduled_task.value
|
324
|
+
end
|
325
|
+
|
326
|
+
nil
|
327
|
+
end
|
328
|
+
|
329
|
+
# Public: Tests whether the message has been requeued.
|
330
|
+
#
|
331
|
+
# Returns a Boolean.
|
332
|
+
def requeued?
|
333
|
+
scheduled_task.fulfilled?
|
334
|
+
end
|
335
|
+
|
336
|
+
private
|
337
|
+
|
338
|
+
# Internal: Returns the original message properties.
|
339
|
+
attr_reader :properties
|
340
|
+
|
341
|
+
# Internal: Returns the ScheduledTask which will requeue the message.
|
342
|
+
attr_reader :scheduled_task
|
343
|
+
|
344
|
+
# Internal: Fetches the current number of message failures from the
|
345
|
+
# headers. Defaults to 1.
|
346
|
+
#
|
347
|
+
# Returns a Fixnum.
|
348
|
+
def failures
|
349
|
+
@failures ||= (properties[:headers]['failures'] || 0) + 1
|
350
|
+
end
|
351
|
+
|
352
|
+
# Internal: Performs the actual message requeue.
|
353
|
+
#
|
354
|
+
# Returns nil.
|
355
|
+
def requeue_message
|
356
|
+
Proletariat.publish(properties[:key], properties[:message],
|
357
|
+
properties[:headers])
|
358
|
+
|
359
|
+
nil
|
360
|
+
end
|
361
|
+
|
362
|
+
# Internal: Calculates an exponential retry delay based on the previous
|
363
|
+
# number of failures. Capped with configuration setting.
|
364
|
+
#
|
365
|
+
# Returns the delay in seconds as a Fixnum.
|
366
|
+
def retry_delay
|
367
|
+
[2**failures, Proletariat.max_retry_delay].min
|
368
|
+
end
|
369
|
+
end
|
245
370
|
end
|
246
371
|
end
|
247
372
|
end
|
@@ -125,10 +125,12 @@ module Proletariat
|
|
125
125
|
# Public: Handles message calls from a subscriber and increments the
|
126
126
|
# count. Return value matches interface expected by Subscriber.
|
127
127
|
#
|
128
|
-
# message
|
128
|
+
# message - The contents of the message.
|
129
|
+
# routing_key - Routing key for messages.
|
130
|
+
# headers - Hash of message headers.
|
129
131
|
#
|
130
132
|
# Returns a future-like object holding an :ok Symbol.
|
131
|
-
def post?(message, routing_key)
|
133
|
+
def post?(message, routing_key, headers)
|
132
134
|
self.count = count + 1
|
133
135
|
|
134
136
|
Concurrent::Future.new { :ok }
|
data/lib/proletariat/version.rb
CHANGED
data/lib/proletariat/worker.rb
CHANGED
@@ -38,7 +38,7 @@ module Proletariat
|
|
38
38
|
# message - The incoming message.
|
39
39
|
#
|
40
40
|
# Raises NotImplementedError unless implemented in subclass.
|
41
|
-
def work(message, routing_key)
|
41
|
+
def work(message, routing_key, headers)
|
42
42
|
fail NotImplementedError
|
43
43
|
end
|
44
44
|
|
@@ -71,11 +71,12 @@ module Proletariat
|
|
71
71
|
# with the RabbitMQ convention you can use the '*' character to
|
72
72
|
# replace one word and the '#' to replace many words.
|
73
73
|
# message - The message as a String.
|
74
|
+
# headers - Hash of message headers.
|
74
75
|
#
|
75
76
|
# Returns nil.
|
76
|
-
def publish(to, message = '')
|
77
|
+
def publish(to, message = '', headers = {})
|
77
78
|
log "Publishing to: #{to}"
|
78
|
-
Proletariat.publish to, message
|
79
|
+
Proletariat.publish to, message, headers
|
79
80
|
|
80
81
|
nil
|
81
82
|
end
|
@@ -8,7 +8,7 @@ class PingWorker < Proletariat::Worker
|
|
8
8
|
attr_accessor :pinged
|
9
9
|
end
|
10
10
|
|
11
|
-
def work(message, routing_key)
|
11
|
+
def work(message, routing_key, headers)
|
12
12
|
self.class.pinged = true
|
13
13
|
|
14
14
|
log 'PING'
|
@@ -24,10 +24,15 @@ class PongWorker < Proletariat::Worker
|
|
24
24
|
listen_on :pong
|
25
25
|
|
26
26
|
class << self
|
27
|
+
attr_accessor :fail_mode
|
27
28
|
attr_accessor :ponged
|
28
29
|
end
|
29
30
|
|
30
|
-
def work(message, routing_key)
|
31
|
+
def work(message, routing_key, headers)
|
32
|
+
if self.class.fail_mode == true
|
33
|
+
fail 'Error' unless headers['failures'] == 2
|
34
|
+
end
|
35
|
+
|
31
36
|
self.class.ponged = true
|
32
37
|
|
33
38
|
log 'PONG'
|
@@ -39,18 +44,38 @@ class PongWorker < Proletariat::Worker
|
|
39
44
|
end
|
40
45
|
|
41
46
|
describe Proletariat do
|
42
|
-
|
47
|
+
before do
|
43
48
|
Proletariat.configure do
|
44
49
|
config.logger = Logger.new('/dev/null')
|
45
50
|
config.worker_classes = [PingWorker, PongWorker]
|
46
51
|
end
|
47
52
|
|
53
|
+
PongWorker.fail_mode = false
|
54
|
+
|
55
|
+
Proletariat.purge
|
48
56
|
Proletariat.run!
|
49
57
|
sleep 2
|
50
|
-
|
51
|
-
|
58
|
+
end
|
59
|
+
|
60
|
+
after do
|
52
61
|
Proletariat.stop
|
53
62
|
Proletariat.purge
|
63
|
+
PingWorker.pinged = false
|
64
|
+
PongWorker.ponged = false
|
65
|
+
end
|
66
|
+
|
67
|
+
it 'should roughly work' do
|
68
|
+
Proletariat.publish 'ping', ''
|
69
|
+
sleep 5
|
70
|
+
|
71
|
+
expect(PingWorker.pinged).to be_truthy
|
72
|
+
expect(PongWorker.ponged).to be_truthy
|
73
|
+
end
|
74
|
+
|
75
|
+
it 'should work in error conditions' do
|
76
|
+
PongWorker.fail_mode = true
|
77
|
+
Proletariat.publish 'ping', ''
|
78
|
+
sleep 10
|
54
79
|
|
55
80
|
expect(PingWorker.pinged).to be_truthy
|
56
81
|
expect(PongWorker.ponged).to be_truthy
|
@@ -34,19 +34,19 @@ module Proletariat
|
|
34
34
|
|
35
35
|
it 'should increment the count' do
|
36
36
|
counter = ExpectationGuarantor::MessageCounter.new(1)
|
37
|
-
counter.post?('message', 'key')
|
37
|
+
counter.post?('message', 'key', {})
|
38
38
|
expect(counter.expected_messages_received?).to be_truthy
|
39
39
|
end
|
40
40
|
|
41
41
|
it 'should return a future containing :ok' do
|
42
42
|
counter = ExpectationGuarantor::MessageCounter.new(1)
|
43
43
|
expect(Concurrent::Future).to receive(:new)
|
44
|
-
counter.post?('message', 'key')
|
44
|
+
counter.post?('message', 'key', {})
|
45
45
|
end
|
46
46
|
|
47
47
|
it 'should ensure the returned future contains :ok' do
|
48
48
|
counter = ExpectationGuarantor::MessageCounter.new(1)
|
49
|
-
future = counter.post?('message', 'key')
|
49
|
+
future = counter.post?('message', 'key', {})
|
50
50
|
expect(future.block.call).to eq :ok
|
51
51
|
end
|
52
52
|
end
|
data/spec/lib/worker_spec.rb
CHANGED
@@ -31,7 +31,7 @@ module Proletariat
|
|
31
31
|
|
32
32
|
describe '#work' do
|
33
33
|
it 'should raise NotImplementedError' do
|
34
|
-
expect { Worker.new.work('message', 'key') }.to \
|
34
|
+
expect { Worker.new.work('message', 'key', {}) }.to \
|
35
35
|
raise_exception NotImplementedError
|
36
36
|
end
|
37
37
|
end
|
@@ -53,12 +53,12 @@ module Proletariat
|
|
53
53
|
|
54
54
|
describe '#publish' do
|
55
55
|
it 'should forward the message to the publisher' do
|
56
|
-
expect(Proletariat).to receive(:publish).with('topic', 'message')
|
57
|
-
Worker.new.publish 'topic', 'message'
|
56
|
+
expect(Proletariat).to receive(:publish).with('topic', 'message', {})
|
57
|
+
Worker.new.publish 'topic', 'message', {}
|
58
58
|
end
|
59
59
|
|
60
60
|
it 'should have a blank default message' do
|
61
|
-
expect(Proletariat).to receive(:publish).with('topic', '')
|
61
|
+
expect(Proletariat).to receive(:publish).with('topic', '', {})
|
62
62
|
Worker.new.publish 'topic'
|
63
63
|
end
|
64
64
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: proletariat
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sebastian Edwards
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-03-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|