eventq_rabbitmq 1.17.2 → 1.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7baf5d3e1253a897011d55b808b80bd684d338c3
4
- data.tar.gz: 85ceed3afcfc5f72e2fca7a7bc2d725a215ab891
3
+ metadata.gz: b46c2fdb9dd907c5a923d1bab7a816017896e8e7
4
+ data.tar.gz: 921e8ef146205bc884580936a5bb757600956fc7
5
5
  SHA512:
6
- metadata.gz: 957d801f9111f82b50d7cb7cb869c7621d688906484dc7e13127508205729d757dfc7f381ecdd18c3a3f9a1b060cd5e020aa7857650a10b31031b69c2e8462f7
7
- data.tar.gz: aca60bbb0c6023db233c5807775f181732c8173f7a42b0e07466223ace7591810664db464a93708d64a84105b92a90e745845c1901d80bf2fe5643da2d14e410
6
+ metadata.gz: c5cdb23b764e56bb9615337befd0a7cbd3ee7718c873087470ececb252c1c5bd6ddfbcda067f184dd2435f6fffa443cf2e20208eb0e9e8cfeaf2980a8c570b42
7
+ data.tar.gz: ea96de7b90757e49eb516853c913433a487506e456d110cd92f9493ea37589dcd24a80990ba5732c6eafe4c36053eb8c29ae3e6d513280c99f3a2b84ba15027e
@@ -0,0 +1,310 @@
1
+ module EventQ
2
+ module RabbitMq
3
+ class QueueWorkerV2
4
+ include EventQ::WorkerId
5
+
6
+ attr_accessor :is_running
7
+
8
+ def initialize
9
+ @threads = []
10
+ @forks = []
11
+ @is_running = false
12
+
13
+ @retry_exceeded_block = nil
14
+ @on_retry_block = nil
15
+ @on_error_block = nil
16
+ @hash_helper = HashKit::Helper.new
17
+ @serialization_provider_manager = EventQ::SerializationProviders::Manager.new
18
+ @signature_provider_manager = EventQ::SignatureProviders::Manager.new
19
+ @last_gc_flush = Time.now
20
+ @gc_flush_interval = 10
21
+ end
22
+
23
+ def start(queue, options = {}, &block)
24
+
25
+ EventQ.logger.info("[#{self.class}] - Preparing to start listening for messages.")
26
+
27
+ configure(queue, options)
28
+
29
+ raise "[#{self.class}] - Worker is already running." if running?
30
+
31
+ if options[:client] == nil
32
+ raise "[#{self.class}] - :client (QueueClient) must be specified."
33
+ end
34
+
35
+ EventQ.logger.info("[#{self.class}] - Listening for messages.")
36
+ EventQ.logger.debug do
37
+ "[#{self.class} #start] - Listening for messages on queue: #{EventQ.create_queue_name(queue.name)}"
38
+ end
39
+
40
+ @forks = []
41
+
42
+ if @fork_count > 1
43
+ Thread.new do
44
+ @fork_count.times do
45
+ pid = fork do
46
+ start_process(options, queue, block)
47
+ end
48
+ @forks.push(pid)
49
+ end
50
+ @forks.each { |pid| Process.wait(pid) }
51
+ end
52
+ else
53
+ start_process(options, queue, block)
54
+ end
55
+ end
56
+
57
+ def start_process(options, queue, block)
58
+ @is_running = true
59
+
60
+ %w'INT TERM'.each do |sig|
61
+ Signal.trap(sig) {
62
+ stop
63
+ exit
64
+ }
65
+ end
66
+
67
+ if !options.key?(:durable)
68
+ options[:durable] = true
69
+ end
70
+
71
+ client = options[:client]
72
+ manager = EventQ::RabbitMq::QueueManager.new
73
+ manager.durable = options[:durable]
74
+ @connection = client.get_connection
75
+
76
+ @threads = []
77
+
78
+ # loop through each thread count
79
+ @thread_count.times do
80
+ channel = @connection.create_channel
81
+
82
+ q = manager.get_queue(channel, queue)
83
+ retry_exchange = manager.get_retry_exchange(channel, queue)
84
+
85
+ q.subscribe(:manual_ack => true, :consumer_tag => SecureRandom.uuid) do |delivery_info, properties, payload|
86
+ begin
87
+ tag_processing_thread
88
+ process_message(payload, queue, channel, retry_exchange, delivery_info.delivery_tag, block)
89
+ rescue => e
90
+ EventQ.logger.error(
91
+ "[#{self.class}] - An error occurred attempting to process a message. Error: #{e} | "\
92
+ "Backtrace: #{e.backtrace}"
93
+ )
94
+ call_on_error_block(error: e)
95
+ ensure
96
+ untag_processing_thread
97
+ end
98
+ end
99
+ end
100
+
101
+ if (options.key?(:wait) && options[:wait] == true) || (options.key?(:fork_count) && options[:fork_count] > 1)
102
+ while running? do
103
+ sleep 5
104
+ end
105
+ end
106
+
107
+ return true
108
+ end
109
+
110
+ def call_on_error_block(error:, message: nil)
111
+ if @on_error_block
112
+ EventQ.logger.debug { "[#{self.class}] - Executing on_error block." }
113
+ begin
114
+ @on_error_block.call(error, message)
115
+ rescue => e
116
+ EventQ.logger.error("[#{self.class}] - An error occurred executing the on_error block. Error: #{e}")
117
+ end
118
+ else
119
+ EventQ.logger.debug { "[#{self.class}] - No on_error block specified to execute." }
120
+ end
121
+ end
122
+
123
+ def stop
124
+ EventQ.logger.info { "[#{self.class}] - Stopping..." }
125
+ @is_running = false
126
+
127
+ if @connection != nil
128
+ begin
129
+ @connection.close if @connection.open?
130
+ rescue Timeout::Error
131
+ EventQ.logger.error { 'Timeout occurred closing connection.' }
132
+ end
133
+ end
134
+ return true
135
+ end
136
+
137
+ def on_retry_exceeded(&block)
138
+ @retry_exceeded_block = block
139
+ return nil
140
+ end
141
+
142
+ def on_retry(&block)
143
+ @on_retry_block = block
144
+ return nil
145
+ end
146
+
147
+ def on_error(&block)
148
+ @on_error_block = block
149
+ return nil
150
+ end
151
+
152
+ def running?
153
+ return @is_running
154
+ end
155
+
156
+ def deserialize_message(payload)
157
+ provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
158
+ return provider.deserialize(payload)
159
+ end
160
+
161
+ def serialize_message(msg)
162
+ provider = @serialization_provider_manager.get_provider(EventQ::Configuration.serialization_provider)
163
+ return provider.serialize(msg)
164
+ end
165
+
166
+ def call_on_retry_exceeded_block(message)
167
+ if @retry_exceeded_block != nil
168
+ EventQ.logger.debug { "[#{self.class}] - Executing on_retry_exceeded block." }
169
+ begin
170
+ @retry_exceeded_block.call(message)
171
+ rescue => e
172
+ EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry_exceeded block. Error: #{e}")
173
+ end
174
+ else
175
+ EventQ.logger.debug { "[#{self.class}] - No on_retry_exceeded block specified." }
176
+ end
177
+ end
178
+
179
+ def call_on_retry_block(message)
180
+ if @on_retry_block
181
+ EventQ.logger.debug { "[#{self.class}] - Executing on_retry block." }
182
+ begin
183
+ @on_retry_block.call(message, abort)
184
+ rescue => e
185
+ EventQ.logger.error("[#{self.class}] - An error occurred executing the on_retry block. Error: #{e}")
186
+ end
187
+ else
188
+ EventQ.logger.debug { "[#{self.class}] - No on_retry block specified." }
189
+ end
190
+ end
191
+
192
+ def reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
193
+ EventQ.logger.info("[#{self.class}] - Message rejected removing from queue.")
194
+ # reject the message to remove from queue
195
+ channel.reject(delivery_tag, false)
196
+
197
+ # check if the message retry limit has been exceeded
198
+ if message.retry_attempts >= queue.max_retry_attempts
199
+
200
+ EventQ.logger.info("[#{self.class}] - Message retry attempt limit exceeded. Msg: #{serialize_message(message)}")
201
+
202
+ call_on_retry_exceeded_block(message)
203
+
204
+ # check if the message is allowed to be retried
205
+ elsif queue.allow_retry
206
+ EventQ.logger.debug { "[#{self.class}] - Incrementing retry attempts count." }
207
+ message.retry_attempts += 1
208
+
209
+ if queue.allow_retry_back_off == true
210
+ EventQ.logger.debug do
211
+ "[#{self.class}] - Calculating message back off retry delay. "\
212
+ "Attempts: #{message.retry_attempts} * Retry Delay: #{queue.retry_delay}"
213
+ end
214
+ message_ttl = message.retry_attempts * queue.retry_delay
215
+ if (message.retry_attempts * queue.retry_delay) > queue.max_retry_delay
216
+ EventQ.logger.debug { "[#{self.class}] - Max message back off retry delay reached." }
217
+ message_ttl = queue.max_retry_delay
218
+ end
219
+ else
220
+ EventQ.logger.debug { "[#{self.class}] - Setting fixed retry delay for message." }
221
+ message_ttl = queue.retry_delay
222
+ end
223
+
224
+ EventQ.logger.debug { "[#{self.class}] - Sending message for retry. Message TTL: #{message_ttl}" }
225
+ retry_exchange.publish(serialize_message(message), :expiration => message_ttl)
226
+ EventQ.logger.debug { "[#{self.class}] - Published message to retry exchange." }
227
+
228
+ call_on_retry_block(message)
229
+ end
230
+
231
+ return true
232
+ end
233
+
234
+ def configure(queue, options = {})
235
+ @queue = queue
236
+
237
+ # default thread count
238
+ @thread_count = 1
239
+ if options.key?(:thread_count)
240
+ @thread_count = options[:thread_count]
241
+ end
242
+
243
+ @fork_count = 1
244
+ if options.key?(:fork_count)
245
+ @fork_count = options[:fork_count]
246
+ end
247
+
248
+ EventQ.logger.info(
249
+ "[#{self.class}] - Configuring. Process Count: #{@fork_count} | Thread Count: #{@thread_count} | "\
250
+ "Interval Sleep: #{@sleep}."
251
+ )
252
+
253
+ return true
254
+ end
255
+
256
+ private
257
+
258
+ def process_message(payload, queue, channel, retry_exchange, delivery_tag, block)
259
+ abort = false
260
+ error = false
261
+ message = deserialize_message(payload)
262
+
263
+ EventQ.logger.info("[#{self.class}] - Message received. Retry Attempts: #{message.retry_attempts}")
264
+
265
+ @signature_provider_manager.validate_signature(message: message, queue: queue)
266
+
267
+ message_args = EventQ::MessageArgs.new(type: message.type,
268
+ retry_attempts: message.retry_attempts,
269
+ context: message.context,
270
+ content_type: message.content_type)
271
+
272
+ if(!EventQ::NonceManager.is_allowed?(message.id))
273
+ EventQ.logger.info("[#{self.class}] - Duplicate Message received. Dropping message.")
274
+ channel.acknowledge(delivery_tag, false)
275
+ return false
276
+ end
277
+
278
+ # begin worker block for queue message
279
+ begin
280
+ block.call(message.content, message_args)
281
+
282
+ if message_args.abort == true
283
+ abort = true
284
+ EventQ.logger.info("[#{self.class}] - Message aborted.")
285
+ else
286
+ # accept the message as processed
287
+ channel.acknowledge(delivery_tag, false)
288
+ EventQ.logger.info("[#{self.class}] - Message acknowledged.")
289
+ end
290
+
291
+ rescue => e
292
+ EventQ.logger.error do
293
+ "[#{self.class}] - An unhandled error happened attempting to process a queue message. "\
294
+ "Error: #{e} | Backtrace: #{e.backtrace}"
295
+ end
296
+ error = true
297
+ call_on_error_block(error: e, message: message)
298
+ end
299
+
300
+ if error || abort
301
+ EventQ::NonceManager.failed(message.id)
302
+ reject_message(channel, message, delivery_tag, retry_exchange, queue, abort)
303
+ else
304
+ EventQ::NonceManager.complete(message.id)
305
+ end
306
+ end
307
+ end
308
+ end
309
+ end
310
+
@@ -1,3 +1,3 @@
1
1
  module EventqRabbitmq
2
- VERSION = "1.17.2"
2
+ VERSION = "1.18.0"
3
3
  end
@@ -14,6 +14,7 @@ if RUBY_PLATFORM =~ /java/
14
14
  require_relative '../lib/eventq_rabbitmq/jruby/rabbitmq_queue_worker'
15
15
  else
16
16
  require_relative '../lib/eventq_rabbitmq/rabbitmq_queue_worker'
17
+ require_relative '../lib/eventq_rabbitmq/rabbitmq_queue_worker_v2'
17
18
  end
18
19
 
19
20
  require_relative '../lib/eventq_rabbitmq/rabbitmq_subscription_manager'
@@ -46,4 +47,4 @@ module EventQ
46
47
  end
47
48
  return "#{EventQ.namespace}-#{exchange_name}"
48
49
  end
49
- end
50
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: eventq_rabbitmq
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.17.2
4
+ version: 1.18.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - vaughanbrittonsage
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-02-16 00:00:00.000000000 Z
11
+ date: 2018-03-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -82,7 +82,7 @@ dependencies:
82
82
  version: '0'
83
83
  description: This is the rabbitmq implementation for EventQ
84
84
  email:
85
- - vaughanbritton@gmail.com
85
+ - vaughan.britton@sage.com
86
86
  executables: []
87
87
  extensions: []
88
88
  extra_rdoc_files: []
@@ -96,10 +96,11 @@ files:
96
96
  - lib/eventq_rabbitmq/rabbitmq_queue_client.rb
97
97
  - lib/eventq_rabbitmq/rabbitmq_queue_manager.rb
98
98
  - lib/eventq_rabbitmq/rabbitmq_queue_worker.rb
99
+ - lib/eventq_rabbitmq/rabbitmq_queue_worker_v2.rb
99
100
  - lib/eventq_rabbitmq/rabbitmq_status_checker.rb
100
101
  - lib/eventq_rabbitmq/rabbitmq_subscription_manager.rb
101
102
  - lib/eventq_rabbitmq/version.rb
102
- homepage: https://github.com/vaughanbrittonsage/eventq
103
+ homepage: https://github.com/sage/eventq
103
104
  licenses:
104
105
  - MIT
105
106
  metadata: {}
@@ -119,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
119
120
  version: '0'
120
121
  requirements: []
121
122
  rubyforge_project:
122
- rubygems_version: 2.5.1
123
+ rubygems_version: 2.5.2.1
123
124
  signing_key:
124
125
  specification_version: 4
125
126
  summary: This is the rabbitmq implementation for EventQ