eventq_rabbitmq 1.17.2 → 1.18.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7baf5d3e1253a897011d55b808b80bd684d338c3
4
- data.tar.gz: 85ceed3afcfc5f72e2fca7a7bc2d725a215ab891
3
+ metadata.gz: b46c2fdb9dd907c5a923d1bab7a816017896e8e7
4
+ data.tar.gz: 921e8ef146205bc884580936a5bb757600956fc7
5
5
  SHA512:
6
- metadata.gz: 957d801f9111f82b50d7cb7cb869c7621d688906484dc7e13127508205729d757dfc7f381ecdd18c3a3f9a1b060cd5e020aa7857650a10b31031b69c2e8462f7
7
- data.tar.gz: aca60bbb0c6023db233c5807775f181732c8173f7a42b0e07466223ace7591810664db464a93708d64a84105b92a90e745845c1901d80bf2fe5643da2d14e410
6
+ metadata.gz: c5cdb23b764e56bb9615337befd0a7cbd3ee7718c873087470ececb252c1c5bd6ddfbcda067f184dd2435f6fffa443cf2e20208eb0e9e8cfeaf2980a8c570b42
7
+ data.tar.gz: ea96de7b90757e49eb516853c913433a487506e456d110cd92f9493ea37589dcd24a80990ba5732c6eafe4c36053eb8c29ae3e6d513280c99f3a2b84ba15027e
@@ -0,0 +1,310 @@
1
+ module EventQ
2
+ module RabbitMq
3
+ class QueueWorkerV2
4
+ include EventQ::WorkerId
5
+
6
+ attr_accessor :is_running
7
+
8
+ def initialize
9
+ @threads = []
10
+ @forks = []
11
+ @is_running = false
12
+
13
+ @retry_exceeded_block = nil
14
+ @on_retry_block = nil
15
+ @on_error_block = nil
16
+ @hash_helper = HashKit::Helper.new
17
+ @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
18
+ @signature_provider_manager = EventQ::SignatureProviders::Manager.new
19
+ @last_gc_flush = Time.now
20
+ @gc_flush_interval = 10
21
+ end
22
+
23
+ def start(queue, options = {}, &block)
24
+
25
+ EventQ.logger.info("[#{self.class}] - Preparing to start listening for messages.")
26
+
27
+ configure(queue, options)
28
+
29
+ raise "[#{self.class}] - Worker is already running." if running?
30
+
31
+ if options[:client] == nil
32
+ raise "[#{self.class}] - :client (QueueClient) must be specified."
33
+ end
34
+
35
+ EventQ.logger.info("[#{self.class}] - Listening for messages.")
36
+ EventQ.logger.debug do
37
+ "[#{self.class} #start] - Listening for messages on queue: #{EventQ.create_queue_name(queue.name)}"
38
+ end
39
+
40
+ @forks = []
41
+
42
+ if @fork_count > 1
43
+ Thread.new do
44
+ @fork_count.times do
45
+ pid = fork do
46
+ start_process(options, queue, block)
47
+ end
48
+ @forks.push(pid)
49
+ end
50
+ @forks.each { |pid| Process.wait(pid) }
51
+ end
52
+ else
53
+ start_process(options, queue, block)
54
+ end
55
+ end
56
+
57
+ def start_process(options, queue, block)
58
+ @is_running = true
59
+
60
+ %w'INT TERM'.each do |sig|
61
+ Signal.trap(sig) {
62
+ stop
63
+ exit
64
+ }
65
+ end
66
+
67
+ if !options.key?(:durable)
68
+ options[:durable] = true
69
+ end
70
+
71
+ client = options[:client]
72
+ manager = EventQ::RabbitMq::QueueManager.new
73
+ manager.durable = options[:durable]
74
+ @connection = client.get_connection
75
+
76
+ @threads = []
77
+
78
+ # loop through each thread count
79
+ @thread_count.times do
80
+ channel = @connection.create_channel
81
+
82
+ q = manager.get_queue(channel, queue)
83
+ retry_exchange = manager.get_retry_exchange(channel, queue)
84
+
85
+ q.subscribe(:manual_ack => true, :consumer_tag => SecureRandom.uuid) do |delivery_info, properties, payload|
86
+ begin
87
+ tag_processing_thread
88
+ process_message(payload, queue, channel, retry_exchange, delivery_info.delivery_tag, block)
89
+ rescue => e
90
+ EventQ.logger.error(
91
+ "[#{self.class}] - An error occurred attempting to process a message. Error: #{e} | "\
92
+ "Backtrace: #{e.backtrace}"
93
+ )
94
+ call_on_error_block(error: e)
95
+ ensure
96
+ untag_processing_thread
97
+ end
98
+ end
99
+ end
100
+
101
+ if (options.key?(:wait) && options[:wait] == true) || (options.key?(:fork_count) && options[:fork_count] > 1)
102
+ while running? do
103
+ sleep 5
104
+ end
105
+ end
106
+
107
+ return true
108
+ end
109
+
110
+ def call_on_error_block(error:, message: nil)
111
+ if @on_error_block
112
+ EventQ.logger.debug { "[#{self.class}] - Executing on_error block." }
113
+ begin
114
+ @on_error_block.call(error, message)
115
+ rescue => e
116
+ EventQ.logger.error("[#{self.class}] - An error occurred executing the on_error block. Error: #{e}")
117
+ end
118
+ else
119
+ EventQ.logger.debug { "[#{self.class}] - No on_error block specified to execute." }
120
+ end
121
+ end
122
+
123
+ def stop
124
+ EventQ.logger.info { "[#{self.class}] - Stopping..." }
125
+ @is_running = false
126
+
127
+ if @connection != nil
128
+ begin
129
+ @connection.close if @connection.open?
130
+ rescue Timeout::Error
131
+ EventQ.logger.error { 'Timeout occurred closing connection.' }
132
+ end
133
+ end
134
+ return true
135
+ end
136
+
137
+ def on_retry_exceeded(&block)
138
+ @retry_exceeded_block = block
139
+ return nil
140
+ end
141
+
142
+ def on_retry(&block)
143
+ @on_retry_block = block
144
+ return nil
145
+ end
146
+
147
+ def on_error(&block)
148
+ @on_error_block = block
149
+ return nil
150
+ end
151
+
152
+ def running?
153
+ return @is_running
154
+ end
155
+
156
+ def deserialize_message(payload)
157
+ provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
158
+ return provider.deserialize(payload)
159
+ end
160
+
161
+ def serialize_message(msg)
162
+ provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
163
+ return provider.serialize(msg)
164
+ end
165
+
166
+ def call_on_retry_exceeded_block(message)
167
+ if @retry_exceeded_block != nil
168
+ EventQ.logger.debug { "[#{self.class}] - Executing on_retry_exceeded block." }
169
+ begin
170
+ @retry_exceeded_block.call(message)
171
+ rescue => e
172
+ EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry_exceeded block. Error: #{e}")
173
+ end
174
+ else
175
+ EventQ.logger.debug { "[#{self.class}] - No on_retry_exceeded block specified." }
176
+ end
177
+ end
178
+
179
+ def call_on_retry_block(message)
180
+ if @on_retry_block
181
+ EventQ.logger.debug { "[#{self.class}] - Executing on_retry block." }
182
+ begin
183
+ @on_retry_block.call(message, abort)
184
+ rescue => e
185
+ EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry block. Error: #{e}")
186
+ end
187
+ else
188
+ EventQ.logger.debug { "[#{self.class}] - No on_retry block specified." }
189
+ end
190
+ end
191
+
192
+ def reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
193
+ EventQ.logger.info("[#{self.class}] - Message rejected removing from queue.")
194
+ # reject the message to remove from queue
195
+ channel.reject(delivery_tag, false)
196
+
197
+ # check if the message retry limit has been exceeded
198
+ if message.retry_attempts >= queue.max_retry_attempts
199
+
200
+ EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded. Msg: #{serialize_message(message)}")
201
+
202
+ call_on_retry_exceeded_block(message)
203
+
204
+ # check if the message is allowed to be retried
205
+ elsif queue.allow_retry
206
+ EventQ.logger.debug { "[#{self.class}] - Incrementing retry attempts count." }
207
+ message.retry_attempts += 1
208
+
209
+ if queue.allow_retry_back_off == true
210
+ EventQ.logger.debug do
211
+ "[#{self.class}] - Calculating message back off retry delay. "\
212
+ "Attempts: #{message.retry_attempts} * Retry Delay: #{queue.retry_delay}"
213
+ end
214
+ message_ttl = message.retry_attempts * queue.retry_delay
215
+ if (message.retry_attempts * queue.retry_delay) > queue.max_retry_delay
216
+ EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
217
+ message_ttl = queue.max_retry_delay
218
+ end
219
+ else
220
+ EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
221
+ message_ttl = queue.retry_delay
222
+ end
223
+
224
+ EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{message_ttl}" }
225
+ retry_exchange.publish(serialize_message(message), :expiration => message_ttl)
226
+ EventQ.logger.debug { "[#{self.class}] - Published message to retry exchange." }
227
+
228
+ call_on_retry_block(message)
229
+ end
230
+
231
+ return true
232
+ end
233
+
234
+ def configure(queue, options = {})
235
+ @queue = queue
236
+
237
+ # default thread count
238
+ @thread_count = 1
239
+ if options.key?(:thread_count)
240
+ @thread_count = options[:thread_count]
241
+ end
242
+
243
+ @fork_count = 1
244
+ if options.key?(:fork_count)
245
+ @fork_count = options[:fork_count]
246
+ end
247
+
248
+ EventQ.logger.info(
249
+ "[#{self.class}] - Configuring. Process Count: #{@fork_count} | Thread Count: #{@thread_count} | "\
250
+ "Interval Sleep: #{@sleep}."
251
+ )
252
+
253
+ return true
254
+ end
255
+
256
+ private
257
+
258
+ def process_message(payload, queue, channel, retry_exchange, delivery_tag, block)
259
+ abort = false
260
+ error = false
261
+ message = deserialize_message(payload)
262
+
263
+ EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{message.retry_attempts}")
264
+
265
+ @signature_provider_manager.validate_signature(message: message, queue: queue)
266
+
267
+ message_args = EventQ::MessageArgs.new(type: message.type,
268
+ retry_attempts: message.retry_attempts,
269
+ context: message.context,
270
+ content_type: message.content_type)
271
+
272
+ if(!EventQ::NonceManager.is_allowed?(message.id))
273
+ EventQ.logger.info("[#{self.class}] - Duplicate Message received. Dropping message.")
274
+ channel.acknowledge(delivery_tag, false)
275
+ return false
276
+ end
277
+
278
+ # begin worker block for queue message
279
+ begin
280
+ block.call(message.content, message_args)
281
+
282
+ if message_args.abort == true
283
+ abort = true
284
+ EventQ.logger.info("[#{self.class}] - Message aborted.")
285
+ else
286
+ # accept the message as processed
287
+ channel.acknowledge(delivery_tag, false)
288
+ EventQ.logger.info("[#{self.class}] - Message acknowledged.")
289
+ end
290
+
291
+ rescue => e
292
+ EventQ.logger.error do
293
+ "[#{self.class}] - An unhandled error happened attempting to process a queue message. "\
294
+ "Error: #{e} | Backtrace: #{e.backtrace}"
295
+ end
296
+ error = true
297
+ call_on_error_block(error: e, message: message)
298
+ end
299
+
300
+ if error || abort
301
+ EventQ::NonceManager.failed(message.id)
302
+ reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
303
+ else
304
+ EventQ::NonceManager.complete(message.id)
305
+ end
306
+ end
307
+ end
308
+ end
309
+ end
310
+
@@ -1,3 +1,3 @@
1
1
  module EventqRabbitmq
2
- VERSION = "1.17.2"
2
+ VERSION = "1.18.0"
3
3
  end
@@ -14,6 +14,7 @@ if RUBY_PLATFORM =~ /java/
14
14
  require_relative '../lib/eventq_rabbitmq/jruby/rabbitmq_queue_worker'
15
15
  else
16
16
  require_relative '../lib/eventq_rabbitmq/rabbitmq_queue_worker'
17
+ require_relative '../lib/eventq_rabbitmq/rabbitmq_queue_worker_v2'
17
18
  end
18
19
 
19
20
  require_relative '../lib/eventq_rabbitmq/rabbitmq_subscription_manager'
@@ -46,4 +47,4 @@ module EventQ
46
47
  end
47
48
  return "#{EventQ.namespace}-#{exchange_name}"
48
49
  end
49
- end
50
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: eventq_rabbitmq
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.17.2
4
+ version: 1.18.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - vaughanbrittonsage
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-02-16 00:00:00.000000000 Z
11
+ date: 2018-03-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -82,7 +82,7 @@ dependencies:
82
82
  version: '0'
83
83
  description: This is the rabbitmq implementation for EventQ
84
84
  email:
85
- - vaughanbritton@gmail.com
85
+ - vaughan.britton@sage.com
86
86
  executables: []
87
87
  extensions: []
88
88
  extra_rdoc_files: []
@@ -96,10 +96,11 @@ files:
96
96
  - lib/eventq_rabbitmq/rabbitmq_queue_client.rb
97
97
  - lib/eventq_rabbitmq/rabbitmq_queue_manager.rb
98
98
  - lib/eventq_rabbitmq/rabbitmq_queue_worker.rb
99
+ - lib/eventq_rabbitmq/rabbitmq_queue_worker_v2.rb
99
100
  - lib/eventq_rabbitmq/rabbitmq_status_checker.rb
100
101
  - lib/eventq_rabbitmq/rabbitmq_subscription_manager.rb
101
102
  - lib/eventq_rabbitmq/version.rb
102
- homepage: https://github.com/vaughanbrittonsage/eventq
103
+ homepage: https://github.com/sage/eventq
103
104
  licenses:
104
105
  - MIT
105
106
  metadata: {}
@@ -119,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
119
120
  version: '0'
120
121
  requirements: []
121
122
  rubyforge_project:
122
- rubygems_version: 2.5.1
123
+ rubygems_version: 2.5.2.1
123
124
  signing_key:
124
125
  specification_version: 4
125
126
  summary: This is the rabbitmq implementation for EventQ