istox 0.1.152.1.test15 → 0.1.152.1.test16
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/istox/helpers/bunny_boot.rb +57 -31
- data/lib/istox/helpers/subscriber.rb +60 -39
- data/lib/istox/helpers/xray/grpc_client_xray_interceptor.rb +1 -1
- data/lib/istox/helpers/xray/rabbitmq_consumer_interceptor.rb +29 -0
- data/lib/istox/helpers/xray/rabbitmq_publisher_interceptor.rb +27 -1
- data/lib/istox/helpers/xray/xray_initializer.rb +3 -0
- data/lib/istox/version.rb +1 -1
- metadata +3 -3
- data/lib/istox/helpers/xray/rabbitmq_subscriber_interceptor.rb +0 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0556d82320982ae7582d13b0a999e8985822eb66d8086e9084866ffe23ea086c
|
4
|
+
data.tar.gz: '0529d8460f758ff329dff82cb49790d3353b6d52f8f9a4a0fe848c70fd0153c7'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5db924c60ebc7e215e7f343f2d34cfdae12016d632857977193742236636e563615260771b5c242ea273f919b12f2b592e9406cb25510a606797c3b51a5dd4e4
|
7
|
+
data.tar.gz: 2b4f17ebfc0dfa40a6a605839ca82b3a2c62667b9efea2875fd739bc5aece73d0315dd0ff11a2e33b6d55fe3ec417daddd45f1d1310ecdd47724d49a5cf969f7
|
@@ -5,6 +5,29 @@ require 'istox/helpers/logger'
|
|
5
5
|
module Istox
|
6
6
|
class BunnyBoot
|
7
7
|
class << self
|
8
|
+
########################################################
|
9
|
+
##
|
10
|
+
## RABBITMQ interceptors
|
11
|
+
##
|
12
|
+
######################################################
|
13
|
+
def add_consumer_interceptor(interceptor)
|
14
|
+
@consumer_interceptors ||= []
|
15
|
+
@consumer_interceptors.push(interceptor)
|
16
|
+
end
|
17
|
+
|
18
|
+
def add_publisher_interceptor(interceptor)
|
19
|
+
@publisher_interceptors ||= []
|
20
|
+
@publisher_interceptors.push(interceptor)
|
21
|
+
end
|
22
|
+
|
23
|
+
def publisher_interceptors
|
24
|
+
@publisher_interceptors || []
|
25
|
+
end
|
26
|
+
|
27
|
+
def consumer_interceptors
|
28
|
+
@consumer_interceptors || []
|
29
|
+
end
|
30
|
+
|
8
31
|
# Create physical connection to RabbitMQ
|
9
32
|
# During failover of RabbitMQ cluster or temporary failure, there may be error and needs retry in loop
|
10
33
|
def connection
|
@@ -36,16 +59,16 @@ module Istox
|
|
36
59
|
def exchange(eid)
|
37
60
|
type = data[:exchanges][eid][:type]
|
38
61
|
name = eid
|
39
|
-
settings = { durable: exchange_durable?(eid)
|
62
|
+
settings = { durable: exchange_durable?(eid) }
|
40
63
|
confirm = data[:exchanges][eid][:confirm] || -1
|
41
64
|
[type, name, settings, confirm]
|
42
|
-
rescue
|
65
|
+
rescue StandardError
|
43
66
|
nil
|
44
67
|
end
|
45
68
|
|
46
69
|
def binding_exchange_id(id)
|
47
70
|
data[:binding][id][:exchange] || :default
|
48
|
-
rescue
|
71
|
+
rescue StandardError
|
49
72
|
nil
|
50
73
|
end
|
51
74
|
|
@@ -56,17 +79,17 @@ module Istox
|
|
56
79
|
else
|
57
80
|
data[:queues][queue][:queue_name]
|
58
81
|
end
|
59
|
-
rescue
|
82
|
+
rescue StandardError
|
60
83
|
nil
|
61
84
|
end
|
62
85
|
|
63
86
|
def confirm_mode(eid)
|
64
87
|
data[:exchanges][eid][:confirm] || -1
|
65
|
-
rescue
|
88
|
+
rescue StandardError
|
66
89
|
nil
|
67
90
|
end
|
68
91
|
|
69
|
-
def queues_keys_for_subscribe
|
92
|
+
def queues_keys_for_subscribe
|
70
93
|
data['queues'].keys
|
71
94
|
end
|
72
95
|
|
@@ -89,7 +112,7 @@ module Istox
|
|
89
112
|
durable = exchange_config!(exchange_name)['durable']
|
90
113
|
durable = true if durable.nil?
|
91
114
|
durable
|
92
|
-
rescue => e
|
115
|
+
rescue StandardError => e
|
93
116
|
raise e
|
94
117
|
end
|
95
118
|
|
@@ -104,7 +127,7 @@ module Istox
|
|
104
127
|
|
105
128
|
def exchange_name(consumer_key)
|
106
129
|
queue_config_from_consumer_key!(consumer_key)['exchange']
|
107
|
-
rescue
|
130
|
+
rescue StandardError
|
108
131
|
nil
|
109
132
|
end
|
110
133
|
|
@@ -119,43 +142,43 @@ module Istox
|
|
119
142
|
name = "#{prefix}#{delimiter}#{name}" unless prefix.nil?
|
120
143
|
name = "#{name}#{delimiter}#{suffix}" unless suffix.nil?
|
121
144
|
name
|
122
|
-
rescue
|
145
|
+
rescue StandardError
|
123
146
|
nil
|
124
147
|
end
|
125
148
|
|
126
149
|
def queue_priority(consumer_key)
|
127
150
|
queue_config_from_consumer_key!(consumer_key)['priority']
|
128
|
-
rescue
|
151
|
+
rescue StandardError
|
129
152
|
nil
|
130
153
|
end
|
131
154
|
|
132
155
|
def queue_worker_param(consumer_key)
|
133
156
|
queue_config_from_consumer_key!(consumer_key)['worker_param']
|
134
|
-
rescue
|
157
|
+
rescue StandardError
|
135
158
|
nil
|
136
159
|
end
|
137
160
|
|
138
161
|
def queue_worker_param_format(consumer_key)
|
139
162
|
queue_config_from_consumer_key!(consumer_key)['worker_param_format'] || 'open_struct'
|
140
|
-
rescue
|
163
|
+
rescue StandardError
|
141
164
|
nil
|
142
165
|
end
|
143
166
|
|
144
167
|
def queue_routing_key(consumer_key)
|
145
168
|
queue_config_from_consumer_key!(consumer_key)['routing_key'] || (queue_name consumer_key)
|
146
|
-
rescue
|
169
|
+
rescue StandardError
|
147
170
|
nil
|
148
171
|
end
|
149
172
|
|
150
173
|
def queue_exclusive(consumer_key)
|
151
174
|
queue_config_from_consumer_key!(consumer_key)['exclusive'] || false
|
152
|
-
rescue
|
175
|
+
rescue StandardError
|
153
176
|
nil
|
154
177
|
end
|
155
178
|
|
156
179
|
def ruby_class(consumer_key)
|
157
180
|
queue_config_from_consumer_key!(consumer_key)['ruby_class']
|
158
|
-
rescue
|
181
|
+
rescue StandardError
|
159
182
|
nil
|
160
183
|
end
|
161
184
|
|
@@ -175,7 +198,7 @@ module Istox
|
|
175
198
|
end
|
176
199
|
|
177
200
|
durable
|
178
|
-
rescue
|
201
|
+
rescue StandardError
|
179
202
|
nil
|
180
203
|
end
|
181
204
|
|
@@ -230,7 +253,7 @@ module Istox
|
|
230
253
|
persistent = e.durable?
|
231
254
|
mandatory = false
|
232
255
|
# Set Mandatory & Persistent flag for non-DLX and non-manual msg
|
233
|
-
unless [
|
256
|
+
unless %w[dlx manual].include? options[:type]
|
234
257
|
if options[:routing_key].present?
|
235
258
|
v1 = data['publish'][eid]
|
236
259
|
v1 = v1[options[:routing_key]] unless v1.nil?
|
@@ -240,10 +263,15 @@ module Istox
|
|
240
263
|
end
|
241
264
|
options.merge!(persistent: persistent)
|
242
265
|
options.merge!(mandatory: mandatory)
|
243
|
-
|
244
266
|
# message.merge!(locale: I18n.locale)
|
245
|
-
message = JSON.dump message
|
246
267
|
|
268
|
+
raise 'Rabbitmq publishing message must be a hash' unless message.is_a? Hash
|
269
|
+
|
270
|
+
publisher_interceptors.each do |interceptor|
|
271
|
+
interceptor.call(message, options)
|
272
|
+
end
|
273
|
+
|
274
|
+
message = JSON.dump message
|
247
275
|
log.debug "Publish options are: #{options}"
|
248
276
|
log.debug "Publish message payload #{message}"
|
249
277
|
e.publish(message, options)
|
@@ -277,7 +305,7 @@ module Istox
|
|
277
305
|
# combination of channel_id:delivery_tag can uniquely identify a msg
|
278
306
|
# For each retry of msg, channel_id and delivery_tag is unchanged
|
279
307
|
# But each retry, there is new delivery_tag that should be updated
|
280
|
-
id = "#{channel_id
|
308
|
+
id = "#{channel_id}:#{delivery_tag}"
|
281
309
|
|
282
310
|
::Istox::RedisBoot.sets("#{id}:payload", JSON.dump(payload), 4)
|
283
311
|
::Istox::RedisBoot.sets("#{id}:eid", eid.to_s, 4)
|
@@ -285,7 +313,7 @@ module Istox
|
|
285
313
|
end
|
286
314
|
|
287
315
|
def find_tracker_on_channel(channel_id, delivery_tag, key)
|
288
|
-
pattern = "#{channel_id
|
316
|
+
pattern = "#{channel_id}:#{delivery_tag}:#{key}"
|
289
317
|
keys = find_trackers pattern
|
290
318
|
get_tracker(keys.first)
|
291
319
|
end
|
@@ -307,9 +335,7 @@ module Istox
|
|
307
335
|
|
308
336
|
def eid(ex)
|
309
337
|
eid = ex.name
|
310
|
-
if eid.empty?
|
311
|
-
eid = :default
|
312
|
-
end
|
338
|
+
eid = :default if eid.empty?
|
313
339
|
|
314
340
|
eid
|
315
341
|
end
|
@@ -327,9 +353,9 @@ module Istox
|
|
327
353
|
def data
|
328
354
|
Hashie.logger.level = 'ERROR'
|
329
355
|
@data = Hashie::Mash.new(
|
330
|
-
|
331
|
-
|
332
|
-
|
356
|
+
YAML.safe_load(
|
357
|
+
ERB.new(File.read(ENV['AMQP_CONFIG'] || 'config/amqp.yml')).result
|
358
|
+
)
|
333
359
|
)
|
334
360
|
end
|
335
361
|
|
@@ -356,11 +382,11 @@ module Istox
|
|
356
382
|
def find_trackers(pattern)
|
357
383
|
cursor = 0
|
358
384
|
all_keys = []
|
359
|
-
loop
|
360
|
-
cursor, keys = ::Istox::RedisBoot.scan(cursor, {:
|
385
|
+
loop do
|
386
|
+
cursor, keys = ::Istox::RedisBoot.scan(cursor, { match: pattern, count: 500 }, 4)
|
361
387
|
all_keys += keys
|
362
|
-
break if cursor ==
|
363
|
-
|
388
|
+
break if cursor == '0'
|
389
|
+
end
|
364
390
|
all_keys.uniq
|
365
391
|
end
|
366
392
|
|
@@ -4,7 +4,6 @@ module Istox
|
|
4
4
|
# Publisher is relying on BunnyBoot to publish message, please make sure BunnyBoot is initalised properly first during runtime.
|
5
5
|
class Subscriber
|
6
6
|
class << self
|
7
|
-
|
8
7
|
# optionally can pass in consumer_key for single subscription / consumer_keys for multiple subcriptions
|
9
8
|
# consumer_key must be defined in amqp.yml
|
10
9
|
# if nothing pass in it will auto subscribe to all available consumers defined in amqp.yml queues key
|
@@ -81,22 +80,22 @@ module Istox
|
|
81
80
|
end
|
82
81
|
|
83
82
|
if manual_ack
|
84
|
-
if exchange.nil?
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
83
|
+
letter_exchange = if exchange.nil?
|
84
|
+
active_channel.default_exchange.name
|
85
|
+
else
|
86
|
+
exchange.name
|
87
|
+
end
|
89
88
|
retry_queue = active_channel.queue("#{queue_name}.retry", arguments: {
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
89
|
+
'x-dead-letter-exchange': letter_exchange,
|
90
|
+
'x-dead-letter-routing-key': queue_name.to_s,
|
91
|
+
'x-message-ttl': (::Istox::BunnyBoot.queue_retry_gap consumer_key)
|
92
|
+
})
|
93
|
+
if exchange_name.nil?
|
94
|
+
exchange_retry = active_channel.default_exchange
|
95
|
+
else
|
95
96
|
exchange_retry_name = "#{exchange_name}.retry"
|
96
97
|
exchange_retry = active_channel.send exchange_type, exchange_retry_name, durable: exchange_durable
|
97
98
|
retry_queue.bind exchange_retry, routing_key: "#{queue_name}.retry" if manual_ack
|
98
|
-
else
|
99
|
-
exchange_retry = active_channel.default_exchange
|
100
99
|
end
|
101
100
|
end
|
102
101
|
|
@@ -117,14 +116,14 @@ module Istox
|
|
117
116
|
@workers = {} if @workers.nil?
|
118
117
|
unless block || @workers[ruby_class]
|
119
118
|
klass = Object.const_get(
|
120
|
-
|
119
|
+
'::' + ruby_class.camelize
|
121
120
|
)
|
122
121
|
param = ::Istox::BunnyBoot.queue_worker_param consumer_key
|
123
|
-
if param.nil?
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
122
|
+
@workers[ruby_class] = if param.nil?
|
123
|
+
klass.new
|
124
|
+
else
|
125
|
+
klass.new param
|
126
|
+
end
|
128
127
|
end
|
129
128
|
# Subscribe queue
|
130
129
|
priority = ::Istox::BunnyBoot.queue_priority consumer_key
|
@@ -150,16 +149,23 @@ module Istox
|
|
150
149
|
|
151
150
|
if process
|
152
151
|
processing_payload = JSON.parse(payload)
|
153
|
-
|
152
|
+
if ::Istox::BunnyBoot.queue_worker_param_format(consumer_key) == 'open_struct'
|
153
|
+
processing_payload = ::Istox::CommonHelper.to_open_struct(processing_payload)
|
154
|
+
end
|
154
155
|
log.info "Processing in consumer: #{klass}, payload: #{processing_payload.to_h.inspect}"
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
156
|
+
|
157
|
+
interceptors = ::Istox::BunnyBoot.consumer_interceptors.dup
|
158
|
+
|
159
|
+
intercept(interceptors, processing_payload, metadata, delivery_info) do
|
160
|
+
if @workers[ruby_class].nil?
|
161
|
+
block&.call(processing_payload, metadata, delivery_info)
|
162
|
+
else
|
163
|
+
@workers[ruby_class].process(processing_payload, metadata, delivery_info)
|
164
|
+
end
|
159
165
|
end
|
160
166
|
else
|
161
167
|
# Instead of cache msg, print log
|
162
|
-
log.fatal "Drop msg at #{Time.now
|
168
|
+
log.fatal "Drop msg at #{Time.now} for queue #{queue.name}, payload is #{JSON.dump(payload)}" if store
|
163
169
|
end
|
164
170
|
# active_channel.ack(delivery_info.delivery_tag) if manual_ack
|
165
171
|
rescue StandardError => e
|
@@ -175,29 +181,44 @@ module Istox
|
|
175
181
|
# here we adopt the unused priority as remaining retry_count
|
176
182
|
if process
|
177
183
|
processing_payload = JSON.parse(payload)
|
178
|
-
|
184
|
+
if ::Istox::BunnyBoot.queue_worker_param_format(consumer_key) == 'open_struct'
|
185
|
+
processing_payload = ::Istox::CommonHelper.to_open_struct(processing_payload)
|
186
|
+
end
|
179
187
|
::Istox::BunnyBoot.publish(exchange_retry, processing_payload, routing_key: "#{queue_name}.retry", priority: retry_count, type: 'dlx')
|
180
188
|
end
|
181
189
|
end
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
end
|
189
|
-
=end
|
190
|
+
# # For redelivered message, call 'reject' not 'nack' in order to reschedule message to tail not head of queue
|
191
|
+
# if delivery_info.redelivered
|
192
|
+
# active_channel.reject(delivery_info.delivery_tag, true)
|
193
|
+
# else
|
194
|
+
# active_channel.nack(delivery_info.delivery_tag, false, true)
|
195
|
+
# end
|
190
196
|
ensure
|
191
197
|
if manual_ack
|
192
|
-
if !multiple.nil? && !result && result == multiple
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
198
|
+
multiple = if !multiple.nil? && !result && result == multiple
|
199
|
+
true
|
200
|
+
else
|
201
|
+
false
|
202
|
+
end
|
197
203
|
active_channel.ack(delivery_info.delivery_tag, multiple)
|
198
204
|
end
|
199
205
|
end
|
200
206
|
end
|
207
|
+
|
208
|
+
def intercept(interceptors, payload, metadata, delivery_info)
|
209
|
+
return yield if interceptors.none?
|
210
|
+
|
211
|
+
i = interceptors.pop
|
212
|
+
return yield unless i
|
213
|
+
|
214
|
+
i.call(payload, metadata, delivery_info) do
|
215
|
+
if interceptors.any?
|
216
|
+
intercept(interceptors, payload, metadata, delivery_info) { yield }
|
217
|
+
else
|
218
|
+
yield
|
219
|
+
end
|
220
|
+
end
|
221
|
+
end
|
201
222
|
end
|
202
223
|
end
|
203
224
|
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'istox/helpers/logger'
|
2
|
+
require 'aws-xray-sdk'
|
3
|
+
|
4
|
+
module Istox
|
5
|
+
module Xray
|
6
|
+
class RabbitmqConsumerInterceptor
|
7
|
+
def call(payload, _metadata, delivery_info)
|
8
|
+
payload_hash = payload.to_h
|
9
|
+
|
10
|
+
trace_id = (payload_hash[:xray_trace_id] if meta.key?(:xray_trace_id))
|
11
|
+
parent_id = (payload_hash[:xray_parent_id] if meta.key?(:xray_parent_id))
|
12
|
+
|
13
|
+
arr = [delivery_info[:exchange], delivery_info[:routing_key], payload_hash[:type]].compact
|
14
|
+
|
15
|
+
XRay.recorder.begin_segment("#{::Istox::Xray::XrayInitializer.service_name}.rabbitmq.#{arr.join('.')}",
|
16
|
+
trace_id: trace_id, parent_id: parent_id)
|
17
|
+
|
18
|
+
result = yield # this returns consumer handler message
|
19
|
+
|
20
|
+
XRay.recorder.end_segment
|
21
|
+
|
22
|
+
result
|
23
|
+
rescue StandardError => e
|
24
|
+
log.error e
|
25
|
+
raise e
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -4,7 +4,33 @@ require 'aws-xray-sdk'
|
|
4
4
|
module Istox
|
5
5
|
module Xray
|
6
6
|
class RabbitmqPublisherInterceptor
|
7
|
-
def call
|
7
|
+
def call(message, options)
|
8
|
+
no_segment = false
|
9
|
+
|
10
|
+
begin
|
11
|
+
no_segment = XRay.recorder.current_segment.blank?
|
12
|
+
rescue XRay::ContextMissingError
|
13
|
+
no_segment = true
|
14
|
+
end
|
15
|
+
|
16
|
+
## usually we have active segment for client interceptor
|
17
|
+
## but in some special case we might not have segment initiated
|
18
|
+
## eg. when running from rails console,
|
19
|
+
## in those case we will initiate the segment here manully
|
20
|
+
XRay.recorder.begin_segment('rabbitmq_publish') if no_segment
|
21
|
+
|
22
|
+
XRay.recorder.begin_subsegment("rabbitmq_publish.#{options[:routing_key]}")
|
23
|
+
|
24
|
+
message[:xray_trace_id] = XRay.recorder.current_segment.trace_id
|
25
|
+
message[:xray_parent_id] = XRay.recorder.current_segment.id
|
26
|
+
|
27
|
+
XRay.recorder.end_subsegment
|
28
|
+
|
29
|
+
XRay.recorder.end_segment if no_segment
|
30
|
+
rescue StandardError => e
|
31
|
+
log.error e
|
32
|
+
raise e
|
33
|
+
end
|
8
34
|
end
|
9
35
|
end
|
10
36
|
end
|
@@ -33,6 +33,9 @@ module Istox
|
|
33
33
|
end
|
34
34
|
|
35
35
|
return unless enable_rabbitmq_trace == true
|
36
|
+
|
37
|
+
::Istox::BunnyBoot.add_publisher_interceptor(::Istox::Xray::RabbitmqPublisherInterceptor.new)
|
38
|
+
::Istox::BunnyBoot.add_consumer_interceptor(::Istox::Xray::RabbitmqConsumerInterceptor.new)
|
36
39
|
end
|
37
40
|
end
|
38
41
|
end
|
data/lib/istox/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: istox
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.152.1.
|
4
|
+
version: 0.1.152.1.test16
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Siong Leng
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-05-
|
11
|
+
date: 2020-05-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: awesome_print
|
@@ -468,8 +468,8 @@ files:
|
|
468
468
|
- lib/istox/helpers/vault.rb
|
469
469
|
- lib/istox/helpers/xray/grpc_client_xray_interceptor.rb
|
470
470
|
- lib/istox/helpers/xray/grpc_server_xray_interceptor.rb
|
471
|
+
- lib/istox/helpers/xray/rabbitmq_consumer_interceptor.rb
|
471
472
|
- lib/istox/helpers/xray/rabbitmq_publisher_interceptor.rb
|
472
|
-
- lib/istox/helpers/xray/rabbitmq_subscriber_interceptor.rb
|
473
473
|
- lib/istox/helpers/xray/xray_initializer.rb
|
474
474
|
- lib/istox/interfaces/chainhub/transaction.rb
|
475
475
|
- lib/istox/models/blockchain_receipt.rb
|