eventq_rabbitmq 1.17.2 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/eventq_rabbitmq/rabbitmq_queue_worker_v2.rb +310 -0
- data/lib/eventq_rabbitmq/version.rb +1 -1
- data/lib/eventq_rabbitmq.rb +2 -1
- metadata +6 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b46c2fdb9dd907c5a923d1bab7a816017896e8e7
|
4
|
+
data.tar.gz: 921e8ef146205bc884580936a5bb757600956fc7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c5cdb23b764e56bb9615337befd0a7cbd3ee7718c873087470ececb252c1c5bd6ddfbcda067f184dd2435f6fffa443cf2e20208eb0e9e8cfeaf2980a8c570b42
|
7
|
+
data.tar.gz: ea96de7b90757e49eb516853c913433a487506e456d110cd92f9493ea37589dcd24a80990ba5732c6eafe4c36053eb8c29ae3e6d513280c99f3a2b84ba15027e
|
@@ -0,0 +1,310 @@
|
|
1
|
+
module EventQ
|
2
|
+
module RabbitMq
|
3
|
+
class QueueWorkerV2
|
4
|
+
include EventQ::WorkerId
|
5
|
+
|
6
|
+
attr_accessor :is_running
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@threads = []
|
10
|
+
@forks = []
|
11
|
+
@is_running = false
|
12
|
+
|
13
|
+
@retry_exceeded_block = nil
|
14
|
+
@on_retry_block = nil
|
15
|
+
@on_error_block = nil
|
16
|
+
@hash_helper = HashKit::Helper.new
|
17
|
+
@serialization_provider_manager = EventQ::SerializationProviders::Manager.new
|
18
|
+
@signature_provider_manager = EventQ::SignatureProviders::Manager.new
|
19
|
+
@last_gc_flush = Time.now
|
20
|
+
@gc_flush_interval = 10
|
21
|
+
end
|
22
|
+
|
23
|
+
def start(queue, options = {}, &block)
|
24
|
+
|
25
|
+
EventQ.logger.info("[#{self.class}] - Preparing to start listening for messages.")
|
26
|
+
|
27
|
+
configure(queue, options)
|
28
|
+
|
29
|
+
raise "[#{self.class}] - Worker is already running." if running?
|
30
|
+
|
31
|
+
if options[:client] == nil
|
32
|
+
raise "[#{self.class}] - :client (QueueClient) must be specified."
|
33
|
+
end
|
34
|
+
|
35
|
+
EventQ.logger.info("[#{self.class}] - Listening for messages.")
|
36
|
+
EventQ.logger.debug do
|
37
|
+
"[#{self.class} #start] - Listening for messages on queue: #{EventQ.create_queue_name(queue.name)}"
|
38
|
+
end
|
39
|
+
|
40
|
+
@forks = []
|
41
|
+
|
42
|
+
if @fork_count > 1
|
43
|
+
Thread.new do
|
44
|
+
@fork_count.times do
|
45
|
+
pid = fork do
|
46
|
+
start_process(options, queue, block)
|
47
|
+
end
|
48
|
+
@forks.push(pid)
|
49
|
+
end
|
50
|
+
@forks.each { |pid| Process.wait(pid) }
|
51
|
+
end
|
52
|
+
else
|
53
|
+
start_process(options, queue, block)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def start_process(options, queue, block)
|
58
|
+
@is_running = true
|
59
|
+
|
60
|
+
%w'INT TERM'.each do |sig|
|
61
|
+
Signal.trap(sig) {
|
62
|
+
stop
|
63
|
+
exit
|
64
|
+
}
|
65
|
+
end
|
66
|
+
|
67
|
+
if !options.key?(:durable)
|
68
|
+
options[:durable] = true
|
69
|
+
end
|
70
|
+
|
71
|
+
client = options[:client]
|
72
|
+
manager = EventQ::RabbitMq::QueueManager.new
|
73
|
+
manager.durable = options[:durable]
|
74
|
+
@connection = client.get_connection
|
75
|
+
|
76
|
+
@threads = []
|
77
|
+
|
78
|
+
# loop through each thread count
|
79
|
+
@thread_count.times do
|
80
|
+
channel = @connection.create_channel
|
81
|
+
|
82
|
+
q = manager.get_queue(channel, queue)
|
83
|
+
retry_exchange = manager.get_retry_exchange(channel, queue)
|
84
|
+
|
85
|
+
q.subscribe(:manual_ack => true, :consumer_tag => SecureRandom.uuid) do |delivery_info, properties, payload|
|
86
|
+
begin
|
87
|
+
tag_processing_thread
|
88
|
+
process_message(payload, queue, channel, retry_exchange, delivery_info.delivery_tag, block)
|
89
|
+
rescue => e
|
90
|
+
EventQ.logger.error(
|
91
|
+
"[#{self.class}] - An error occurred attempting to process a message. Error: #{e} | "\
|
92
|
+
"Backtrace: #{e.backtrace}"
|
93
|
+
)
|
94
|
+
call_on_error_block(error: e)
|
95
|
+
ensure
|
96
|
+
untag_processing_thread
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
if (options.key?(:wait) && options[:wait] == true) || (options.key?(:fork_count) && options[:fork_count] > 1)
|
102
|
+
while running? do
|
103
|
+
sleep 5
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
return true
|
108
|
+
end
|
109
|
+
|
110
|
+
def call_on_error_block(error:, message: nil)
|
111
|
+
if @on_error_block
|
112
|
+
EventQ.logger.debug { "[#{self.class}] - Executing on_error block." }
|
113
|
+
begin
|
114
|
+
@on_error_block.call(error, message)
|
115
|
+
rescue => e
|
116
|
+
EventQ.logger.error("[#{self.class}] - An error occurred executing the on_error block. Error: #{e}")
|
117
|
+
end
|
118
|
+
else
|
119
|
+
EventQ.logger.debug { "[#{self.class}] - No on_error block specified to execute." }
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def stop
|
124
|
+
EventQ.logger.info { "[#{self.class}] - Stopping..." }
|
125
|
+
@is_running = false
|
126
|
+
|
127
|
+
if @connection != nil
|
128
|
+
begin
|
129
|
+
@connection.close if @connection.open?
|
130
|
+
rescue Timeout::Error
|
131
|
+
EventQ.logger.error { 'Timeout occurred closing connection.' }
|
132
|
+
end
|
133
|
+
end
|
134
|
+
return true
|
135
|
+
end
|
136
|
+
|
137
|
+
def on_retry_exceeded(&block)
|
138
|
+
@retry_exceeded_block = block
|
139
|
+
return nil
|
140
|
+
end
|
141
|
+
|
142
|
+
def on_retry(&block)
|
143
|
+
@on_retry_block = block
|
144
|
+
return nil
|
145
|
+
end
|
146
|
+
|
147
|
+
def on_error(&block)
|
148
|
+
@on_error_block = block
|
149
|
+
return nil
|
150
|
+
end
|
151
|
+
|
152
|
+
def running?
|
153
|
+
return @is_running
|
154
|
+
end
|
155
|
+
|
156
|
+
def deserialize_message(payload)
|
157
|
+
provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
|
158
|
+
return provider.deserialize(payload)
|
159
|
+
end
|
160
|
+
|
161
|
+
def serialize_message(msg)
|
162
|
+
provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
|
163
|
+
return provider.serialize(msg)
|
164
|
+
end
|
165
|
+
|
166
|
+
def call_on_retry_exceeded_block(message)
|
167
|
+
if @retry_exceeded_block != nil
|
168
|
+
EventQ.logger.debug { "[#{self.class}] - Executing on_retry_exceeded block." }
|
169
|
+
begin
|
170
|
+
@retry_exceeded_block.call(message)
|
171
|
+
rescue => e
|
172
|
+
EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry_exceeded block. Error: #{e}")
|
173
|
+
end
|
174
|
+
else
|
175
|
+
EventQ.logger.debug { "[#{self.class}] - No on_retry_exceeded block specified." }
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
def call_on_retry_block(message)
|
180
|
+
if @on_retry_block
|
181
|
+
EventQ.logger.debug { "[#{self.class}] - Executing on_retry block." }
|
182
|
+
begin
|
183
|
+
@on_retry_block.call(message, abort)
|
184
|
+
rescue => e
|
185
|
+
EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry block. Error: #{e}")
|
186
|
+
end
|
187
|
+
else
|
188
|
+
EventQ.logger.debug { "[#{self.class}] - No on_retry block specified." }
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
def reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
|
193
|
+
EventQ.logger.info("[#{self.class}] - Message rejected removing from queue.")
|
194
|
+
# reject the message to remove from queue
|
195
|
+
channel.reject(delivery_tag, false)
|
196
|
+
|
197
|
+
# check if the message retry limit has been exceeded
|
198
|
+
if message.retry_attempts >= queue.max_retry_attempts
|
199
|
+
|
200
|
+
EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded. Msg: #{serialize_message(message)}")
|
201
|
+
|
202
|
+
call_on_retry_exceeded_block(message)
|
203
|
+
|
204
|
+
# check if the message is allowed to be retried
|
205
|
+
elsif queue.allow_retry
|
206
|
+
EventQ.logger.debug { "[#{self.class}] - Incrementing retry attempts count." }
|
207
|
+
message.retry_attempts += 1
|
208
|
+
|
209
|
+
if queue.allow_retry_back_off == true
|
210
|
+
EventQ.logger.debug do
|
211
|
+
"[#{self.class}] - Calculating message back off retry delay. "\
|
212
|
+
"Attempts: #{message.retry_attempts} * Retry Delay: #{queue.retry_delay}"
|
213
|
+
end
|
214
|
+
message_ttl = message.retry_attempts * queue.retry_delay
|
215
|
+
if (message.retry_attempts * queue.retry_delay) > queue.max_retry_delay
|
216
|
+
EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
|
217
|
+
message_ttl = queue.max_retry_delay
|
218
|
+
end
|
219
|
+
else
|
220
|
+
EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
|
221
|
+
message_ttl = queue.retry_delay
|
222
|
+
end
|
223
|
+
|
224
|
+
EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{message_ttl}" }
|
225
|
+
retry_exchange.publish(serialize_message(message), :expiration => message_ttl)
|
226
|
+
EventQ.logger.debug { "[#{self.class}] - Published message to retry exchange." }
|
227
|
+
|
228
|
+
call_on_retry_block(message)
|
229
|
+
end
|
230
|
+
|
231
|
+
return true
|
232
|
+
end
|
233
|
+
|
234
|
+
def configure(queue, options = {})
|
235
|
+
@queue = queue
|
236
|
+
|
237
|
+
# default thread count
|
238
|
+
@thread_count = 1
|
239
|
+
if options.key?(:thread_count)
|
240
|
+
@thread_count = options[:thread_count]
|
241
|
+
end
|
242
|
+
|
243
|
+
@fork_count = 1
|
244
|
+
if options.key?(:fork_count)
|
245
|
+
@fork_count = options[:fork_count]
|
246
|
+
end
|
247
|
+
|
248
|
+
EventQ.logger.info(
|
249
|
+
"[#{self.class}] - Configuring. Process Count: #{@fork_count} | Thread Count: #{@thread_count} | "\
|
250
|
+
"Interval Sleep: #{@sleep}."
|
251
|
+
)
|
252
|
+
|
253
|
+
return true
|
254
|
+
end
|
255
|
+
|
256
|
+
private
|
257
|
+
|
258
|
+
def process_message(payload, queue, channel, retry_exchange, delivery_tag, block)
|
259
|
+
abort = false
|
260
|
+
error = false
|
261
|
+
message = deserialize_message(payload)
|
262
|
+
|
263
|
+
EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{message.retry_attempts}")
|
264
|
+
|
265
|
+
@signature_provider_manager.validate_signature(message: message, queue: queue)
|
266
|
+
|
267
|
+
message_args = EventQ::MessageArgs.new(type: message.type,
|
268
|
+
retry_attempts: message.retry_attempts,
|
269
|
+
context: message.context,
|
270
|
+
content_type: message.content_type)
|
271
|
+
|
272
|
+
if(!EventQ::NonceManager.is_allowed?(message.id))
|
273
|
+
EventQ.logger.info("[#{self.class}] - Duplicate Message received. Dropping message.")
|
274
|
+
channel.acknowledge(delivery_tag, false)
|
275
|
+
return false
|
276
|
+
end
|
277
|
+
|
278
|
+
# begin worker block for queue message
|
279
|
+
begin
|
280
|
+
block.call(message.content, message_args)
|
281
|
+
|
282
|
+
if message_args.abort == true
|
283
|
+
abort = true
|
284
|
+
EventQ.logger.info("[#{self.class}] - Message aborted.")
|
285
|
+
else
|
286
|
+
# accept the message as processed
|
287
|
+
channel.acknowledge(delivery_tag, false)
|
288
|
+
EventQ.logger.info("[#{self.class}] - Message acknowledged.")
|
289
|
+
end
|
290
|
+
|
291
|
+
rescue => e
|
292
|
+
EventQ.logger.error do
|
293
|
+
"[#{self.class}] - An unhandled error happened attempting to process a queue message. "\
|
294
|
+
"Error: #{e} | Backtrace: #{e.backtrace}"
|
295
|
+
end
|
296
|
+
error = true
|
297
|
+
call_on_error_block(error: e, message: message)
|
298
|
+
end
|
299
|
+
|
300
|
+
if error || abort
|
301
|
+
EventQ::NonceManager.failed(message.id)
|
302
|
+
reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
|
303
|
+
else
|
304
|
+
EventQ::NonceManager.complete(message.id)
|
305
|
+
end
|
306
|
+
end
|
307
|
+
end
|
308
|
+
end
|
309
|
+
end
|
310
|
+
|
data/lib/eventq_rabbitmq.rb
CHANGED
@@ -14,6 +14,7 @@ if RUBY_PLATFORM =~ /java/
|
|
14
14
|
require_relative '../lib/eventq_rabbitmq/jruby/rabbitmq_queue_worker'
|
15
15
|
else
|
16
16
|
require_relative '../lib/eventq_rabbitmq/rabbitmq_queue_worker'
|
17
|
+
require_relative '../lib/eventq_rabbitmq/rabbitmq_queue_worker_v2'
|
17
18
|
end
|
18
19
|
|
19
20
|
require_relative '../lib/eventq_rabbitmq/rabbitmq_subscription_manager'
|
@@ -46,4 +47,4 @@ module EventQ
|
|
46
47
|
end
|
47
48
|
return "#{EventQ.namespace}-#{exchange_name}"
|
48
49
|
end
|
49
|
-
end
|
50
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: eventq_rabbitmq
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.18.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- vaughanbrittonsage
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-03-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -82,7 +82,7 @@ dependencies:
|
|
82
82
|
version: '0'
|
83
83
|
description: This is the rabbitmq implementation for EventQ
|
84
84
|
email:
|
85
|
-
-
|
85
|
+
- vaughan.britton@sage.com
|
86
86
|
executables: []
|
87
87
|
extensions: []
|
88
88
|
extra_rdoc_files: []
|
@@ -96,10 +96,11 @@ files:
|
|
96
96
|
- lib/eventq_rabbitmq/rabbitmq_queue_client.rb
|
97
97
|
- lib/eventq_rabbitmq/rabbitmq_queue_manager.rb
|
98
98
|
- lib/eventq_rabbitmq/rabbitmq_queue_worker.rb
|
99
|
+
- lib/eventq_rabbitmq/rabbitmq_queue_worker_v2.rb
|
99
100
|
- lib/eventq_rabbitmq/rabbitmq_status_checker.rb
|
100
101
|
- lib/eventq_rabbitmq/rabbitmq_subscription_manager.rb
|
101
102
|
- lib/eventq_rabbitmq/version.rb
|
102
|
-
homepage: https://github.com/
|
103
|
+
homepage: https://github.com/sage/eventq
|
103
104
|
licenses:
|
104
105
|
- MIT
|
105
106
|
metadata: {}
|
@@ -119,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
119
120
|
version: '0'
|
120
121
|
requirements: []
|
121
122
|
rubyforge_project:
|
122
|
-
rubygems_version: 2.5.1
|
123
|
+
rubygems_version: 2.5.2.1
|
123
124
|
signing_key:
|
124
125
|
specification_version: 4
|
125
126
|
summary: This is the rabbitmq implementation for EventQ
|