istox 0.1.152.1 → 0.1.153.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +26 -9
- data/istox.gemspec +2 -0
- data/lib/istox.rb +9 -1
- data/lib/istox/helpers/bunny_boot.rb +55 -31
- data/lib/istox/helpers/grpc_client.rb +11 -2
- data/lib/istox/helpers/{redis.rb → redis_boot.rb} +7 -6
- data/lib/istox/helpers/redis_manager.rb +21 -0
- data/lib/istox/helpers/remote_model_cache.rb +2 -4
- data/lib/istox/helpers/subscriber.rb +60 -39
- data/lib/istox/helpers/xray/grpc_client_xray_interceptor.rb +40 -0
- data/lib/istox/helpers/xray/grpc_server_xray_interceptor.rb +26 -0
- data/lib/istox/helpers/xray/rabbitmq_consumer_interceptor.rb +29 -0
- data/lib/istox/helpers/xray/rabbitmq_publisher_interceptor.rb +36 -0
- data/lib/istox/helpers/xray/xray_initializer.rb +58 -0
- data/lib/istox/version.rb +1 -1
- metadata +37 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 65872c83f71e4c64acb4e856541411c2cf06afbb147fb500fe86d3ae647669d9
|
4
|
+
data.tar.gz: 3b6c32750ad37e9efc5412be0ee61186267f906e4e850aa5de4a39d53423f975
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7b4169651c930a4fec4d685b4109388e791e8bf283c57c87754618b80a00ca650f4e128a21117b1a96afb6cdf54f0b232b68d1a8c5fcd71c1bfbec0a446c39c4
|
7
|
+
data.tar.gz: 803b1349d0ed3335dcc9994aeaeb01518c5e6a49f6523cdc904bed6083c87fdf02b69d40d0a48c8b6ed8ce3f9fece35f03ce39556ce0a169f4ab7613b6a56c34
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
istox (0.1.
|
4
|
+
istox (0.1.153.3)
|
5
|
+
amazing_print
|
5
6
|
awesome_print
|
6
7
|
binding_of_caller
|
7
8
|
bunny (>= 2.12.0)
|
@@ -61,12 +62,25 @@ GEM
|
|
61
62
|
i18n (>= 0.7, < 2)
|
62
63
|
minitest (~> 5.1)
|
63
64
|
tzinfo (~> 1.1)
|
65
|
+
amazing_print (1.1.0)
|
64
66
|
amq-protocol (2.3.1)
|
65
67
|
arel (9.0.0)
|
66
68
|
awesome_print (1.8.0)
|
67
69
|
aws-eventstream (1.1.0)
|
68
|
-
aws-
|
70
|
+
aws-partitions (1.318.0)
|
71
|
+
aws-sdk-core (3.96.1)
|
72
|
+
aws-eventstream (~> 1, >= 1.0.2)
|
73
|
+
aws-partitions (~> 1, >= 1.239.0)
|
74
|
+
aws-sigv4 (~> 1.1)
|
75
|
+
jmespath (~> 1.0)
|
76
|
+
aws-sdk-xray (1.4.0)
|
77
|
+
aws-sdk-core (~> 3)
|
78
|
+
aws-sigv4 (~> 1.0)
|
79
|
+
aws-sigv4 (1.1.3)
|
69
80
|
aws-eventstream (~> 1.0, >= 1.0.2)
|
81
|
+
aws-xray-sdk (0.11.4)
|
82
|
+
aws-sdk-xray (~> 1.4.0)
|
83
|
+
multi_json (~> 1)
|
70
84
|
binding_of_caller (0.8.0)
|
71
85
|
debug_inspector (>= 0.0.1)
|
72
86
|
builder (3.2.4)
|
@@ -98,14 +112,14 @@ GEM
|
|
98
112
|
ffi (1.12.2)
|
99
113
|
globalid (0.4.2)
|
100
114
|
activesupport (>= 4.2.0)
|
101
|
-
google-protobuf (3.
|
115
|
+
google-protobuf (3.12.1-universal-darwin)
|
102
116
|
googleapis-common-protos-types (1.0.5)
|
103
117
|
google-protobuf (~> 3.11)
|
104
|
-
graphlient (0.
|
105
|
-
faraday
|
118
|
+
graphlient (0.4.0)
|
119
|
+
faraday (>= 1.0)
|
106
120
|
faraday_middleware
|
107
121
|
graphql-client
|
108
|
-
graphql (1.10.
|
122
|
+
graphql (1.10.10)
|
109
123
|
graphql-client (0.16.0)
|
110
124
|
activesupport (>= 3.0)
|
111
125
|
graphql (~> 1.8)
|
@@ -128,6 +142,7 @@ GEM
|
|
128
142
|
grpc (~> 1.10)
|
129
143
|
grpc-tools (~> 1.10)
|
130
144
|
slop (~> 4.6)
|
145
|
+
jmespath (1.4.0)
|
131
146
|
listen (3.0.8)
|
132
147
|
rb-fsevent (~> 0.9, >= 0.9.4)
|
133
148
|
rb-inotify (~> 0.9, >= 0.9.7)
|
@@ -143,12 +158,13 @@ GEM
|
|
143
158
|
mini_mime (1.0.1)
|
144
159
|
mini_portile2 (2.4.0)
|
145
160
|
minitest (5.14.0)
|
161
|
+
multi_json (1.14.1)
|
146
162
|
multipart-post (2.1.1)
|
147
163
|
nio4r (2.3.1)
|
148
164
|
nokogiri (1.10.9)
|
149
165
|
mini_portile2 (~> 2.4.0)
|
150
166
|
oj (3.10.6)
|
151
|
-
ougai (1.8.
|
167
|
+
ougai (1.8.5)
|
152
168
|
oj (~> 3.10)
|
153
169
|
paranoia (2.4.2)
|
154
170
|
activerecord (>= 4.0, < 6.1)
|
@@ -185,7 +201,7 @@ GEM
|
|
185
201
|
rake (>= 0.8.7)
|
186
202
|
thor (>= 0.19.0, < 2.0)
|
187
203
|
rake (10.5.0)
|
188
|
-
rb-fsevent (0.10.
|
204
|
+
rb-fsevent (0.10.4)
|
189
205
|
rb-inotify (0.10.1)
|
190
206
|
ffi (~> 1.0)
|
191
207
|
redis (4.1.3)
|
@@ -243,7 +259,7 @@ GEM
|
|
243
259
|
tzinfo (1.2.6)
|
244
260
|
thread_safe (~> 0.1)
|
245
261
|
uniform_notifier (1.11.0)
|
246
|
-
vault (0.13.
|
262
|
+
vault (0.13.2)
|
247
263
|
aws-sigv4
|
248
264
|
websocket-driver (0.7.0)
|
249
265
|
websocket-extensions (>= 0.1.0)
|
@@ -253,6 +269,7 @@ PLATFORMS
|
|
253
269
|
ruby
|
254
270
|
|
255
271
|
DEPENDENCIES
|
272
|
+
aws-xray-sdk
|
256
273
|
bullet (~> 5.7.5)
|
257
274
|
bundler (~> 1.16)
|
258
275
|
database_cleaner (~> 1.6.0)
|
data/istox.gemspec
CHANGED
@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
|
|
30
30
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
31
31
|
spec.require_paths = ['lib']
|
32
32
|
|
33
|
+
spec.add_dependency 'amazing_print'
|
33
34
|
spec.add_dependency 'awesome_print'
|
34
35
|
spec.add_dependency 'binding_of_caller'
|
35
36
|
spec.add_dependency 'bunny', '>= 2.12.0'
|
@@ -44,6 +45,7 @@ Gem::Specification.new do |spec|
|
|
44
45
|
spec.add_dependency 'redis-namespace', '>= 1.0.0'
|
45
46
|
spec.add_dependency 'redis-rails', '~> 5.0.2'
|
46
47
|
spec.add_dependency 'vault', '~> 0.1'
|
48
|
+
spec.add_development_dependency 'aws-xray-sdk'
|
47
49
|
spec.add_development_dependency 'bullet', '~> 5.7.5'
|
48
50
|
spec.add_development_dependency 'bundler', '~> 1.16'
|
49
51
|
spec.add_development_dependency 'database_cleaner', '~> 1.6.0'
|
data/lib/istox.rb
CHANGED
@@ -32,9 +32,17 @@ module Istox
|
|
32
32
|
require 'istox/helpers/common_helper'
|
33
33
|
require 'istox/helpers/regex_helper'
|
34
34
|
require 'istox/helpers/result_handler'
|
35
|
-
require 'istox/helpers/
|
35
|
+
require 'istox/helpers/redis_boot'
|
36
|
+
require 'istox/helpers/redis_manager'
|
36
37
|
require 'istox/helpers/dlm'
|
37
38
|
require 'istox/helpers/remote_model_cache'
|
39
|
+
|
40
|
+
require 'istox/helpers/xray/grpc_client_xray_interceptor'
|
41
|
+
require 'istox/helpers/xray/grpc_server_xray_interceptor'
|
42
|
+
require 'istox/helpers/xray/rabbitmq_publisher_interceptor'
|
43
|
+
require 'istox/helpers/xray/rabbitmq_consumer_interceptor'
|
44
|
+
require 'istox/helpers/xray/xray_initializer'
|
45
|
+
|
38
46
|
require 'istox/models/blockchain_receipt'
|
39
47
|
require 'istox/models/concerns/blockchain_receipt_query'
|
40
48
|
require 'istox/consumers/blockchain_status_handler'
|
@@ -5,6 +5,29 @@ require 'istox/helpers/logger'
|
|
5
5
|
module Istox
|
6
6
|
class BunnyBoot
|
7
7
|
class << self
|
8
|
+
########################################################
|
9
|
+
##
|
10
|
+
## RABBITMQ interceptors
|
11
|
+
##
|
12
|
+
######################################################
|
13
|
+
def add_consumer_interceptor(interceptor)
|
14
|
+
@consumer_interceptors ||= []
|
15
|
+
@consumer_interceptors.push(interceptor)
|
16
|
+
end
|
17
|
+
|
18
|
+
def add_publisher_interceptor(interceptor)
|
19
|
+
@publisher_interceptors ||= []
|
20
|
+
@publisher_interceptors.push(interceptor)
|
21
|
+
end
|
22
|
+
|
23
|
+
def publisher_interceptors
|
24
|
+
@publisher_interceptors || []
|
25
|
+
end
|
26
|
+
|
27
|
+
def consumer_interceptors
|
28
|
+
@consumer_interceptors || []
|
29
|
+
end
|
30
|
+
|
8
31
|
# Create physical connection to RabbitMQ
|
9
32
|
# During failover of RabbitMQ cluster or temporary failure, there may be error and needs retry in loop
|
10
33
|
def connection
|
@@ -36,16 +59,16 @@ module Istox
|
|
36
59
|
def exchange(eid)
|
37
60
|
type = data[:exchanges][eid][:type]
|
38
61
|
name = eid
|
39
|
-
settings = { durable: exchange_durable?(eid)
|
62
|
+
settings = { durable: exchange_durable?(eid) }
|
40
63
|
confirm = data[:exchanges][eid][:confirm] || -1
|
41
64
|
[type, name, settings, confirm]
|
42
|
-
rescue
|
65
|
+
rescue StandardError
|
43
66
|
nil
|
44
67
|
end
|
45
68
|
|
46
69
|
def binding_exchange_id(id)
|
47
70
|
data[:binding][id][:exchange] || :default
|
48
|
-
rescue
|
71
|
+
rescue StandardError
|
49
72
|
nil
|
50
73
|
end
|
51
74
|
|
@@ -56,17 +79,17 @@ module Istox
|
|
56
79
|
else
|
57
80
|
data[:queues][queue][:queue_name]
|
58
81
|
end
|
59
|
-
rescue
|
82
|
+
rescue StandardError
|
60
83
|
nil
|
61
84
|
end
|
62
85
|
|
63
86
|
def confirm_mode(eid)
|
64
87
|
data[:exchanges][eid][:confirm] || -1
|
65
|
-
rescue
|
88
|
+
rescue StandardError
|
66
89
|
nil
|
67
90
|
end
|
68
91
|
|
69
|
-
def queues_keys_for_subscribe
|
92
|
+
def queues_keys_for_subscribe
|
70
93
|
data['queues'].keys
|
71
94
|
end
|
72
95
|
|
@@ -89,7 +112,7 @@ module Istox
|
|
89
112
|
durable = exchange_config!(exchange_name)['durable']
|
90
113
|
durable = true if durable.nil?
|
91
114
|
durable
|
92
|
-
rescue => e
|
115
|
+
rescue StandardError => e
|
93
116
|
raise e
|
94
117
|
end
|
95
118
|
|
@@ -104,7 +127,7 @@ module Istox
|
|
104
127
|
|
105
128
|
def exchange_name(consumer_key)
|
106
129
|
queue_config_from_consumer_key!(consumer_key)['exchange']
|
107
|
-
rescue
|
130
|
+
rescue StandardError
|
108
131
|
nil
|
109
132
|
end
|
110
133
|
|
@@ -119,43 +142,43 @@ module Istox
|
|
119
142
|
name = "#{prefix}#{delimiter}#{name}" unless prefix.nil?
|
120
143
|
name = "#{name}#{delimiter}#{suffix}" unless suffix.nil?
|
121
144
|
name
|
122
|
-
rescue
|
145
|
+
rescue StandardError
|
123
146
|
nil
|
124
147
|
end
|
125
148
|
|
126
149
|
def queue_priority(consumer_key)
|
127
150
|
queue_config_from_consumer_key!(consumer_key)['priority']
|
128
|
-
rescue
|
151
|
+
rescue StandardError
|
129
152
|
nil
|
130
153
|
end
|
131
154
|
|
132
155
|
def queue_worker_param(consumer_key)
|
133
156
|
queue_config_from_consumer_key!(consumer_key)['worker_param']
|
134
|
-
rescue
|
157
|
+
rescue StandardError
|
135
158
|
nil
|
136
159
|
end
|
137
160
|
|
138
161
|
def queue_worker_param_format(consumer_key)
|
139
162
|
queue_config_from_consumer_key!(consumer_key)['worker_param_format'] || 'open_struct'
|
140
|
-
rescue
|
163
|
+
rescue StandardError
|
141
164
|
nil
|
142
165
|
end
|
143
166
|
|
144
167
|
def queue_routing_key(consumer_key)
|
145
168
|
queue_config_from_consumer_key!(consumer_key)['routing_key'] || (queue_name consumer_key)
|
146
|
-
rescue
|
169
|
+
rescue StandardError
|
147
170
|
nil
|
148
171
|
end
|
149
172
|
|
150
173
|
def queue_exclusive(consumer_key)
|
151
174
|
queue_config_from_consumer_key!(consumer_key)['exclusive'] || false
|
152
|
-
rescue
|
175
|
+
rescue StandardError
|
153
176
|
nil
|
154
177
|
end
|
155
178
|
|
156
179
|
def ruby_class(consumer_key)
|
157
180
|
queue_config_from_consumer_key!(consumer_key)['ruby_class']
|
158
|
-
rescue
|
181
|
+
rescue StandardError
|
159
182
|
nil
|
160
183
|
end
|
161
184
|
|
@@ -175,7 +198,7 @@ module Istox
|
|
175
198
|
end
|
176
199
|
|
177
200
|
durable
|
178
|
-
rescue
|
201
|
+
rescue StandardError
|
179
202
|
nil
|
180
203
|
end
|
181
204
|
|
@@ -230,7 +253,7 @@ module Istox
|
|
230
253
|
persistent = e.durable?
|
231
254
|
mandatory = false
|
232
255
|
# Set Mandatory & Persistent flag for non-DLX and non-manual msg
|
233
|
-
unless [
|
256
|
+
unless %w[dlx manual].include? options[:type]
|
234
257
|
if options[:routing_key].present?
|
235
258
|
v1 = data['publish'][eid]
|
236
259
|
v1 = v1[options[:routing_key]] unless v1.nil?
|
@@ -240,10 +263,13 @@ module Istox
|
|
240
263
|
end
|
241
264
|
options.merge!(persistent: persistent)
|
242
265
|
options.merge!(mandatory: mandatory)
|
243
|
-
|
244
266
|
# message.merge!(locale: I18n.locale)
|
245
|
-
message = JSON.dump message
|
246
267
|
|
268
|
+
publisher_interceptors.each do |interceptor|
|
269
|
+
interceptor.call(message, options)
|
270
|
+
end
|
271
|
+
|
272
|
+
message = JSON.dump message
|
247
273
|
log.debug "Publish options are: #{options}"
|
248
274
|
log.debug "Publish message payload #{message}"
|
249
275
|
e.publish(message, options)
|
@@ -277,7 +303,7 @@ module Istox
|
|
277
303
|
# combination of channel_id:delivery_tag can uniquely identify a msg
|
278
304
|
# For each retry of msg, channel_id and delivery_tag is unchanged
|
279
305
|
# But each retry, there is new delivery_tag that should be updated
|
280
|
-
id = "#{channel_id
|
306
|
+
id = "#{channel_id}:#{delivery_tag}"
|
281
307
|
|
282
308
|
::Istox::RedisBoot.sets("#{id}:payload", JSON.dump(payload), 4)
|
283
309
|
::Istox::RedisBoot.sets("#{id}:eid", eid.to_s, 4)
|
@@ -285,7 +311,7 @@ module Istox
|
|
285
311
|
end
|
286
312
|
|
287
313
|
def find_tracker_on_channel(channel_id, delivery_tag, key)
|
288
|
-
pattern = "#{channel_id
|
314
|
+
pattern = "#{channel_id}:#{delivery_tag}:#{key}"
|
289
315
|
keys = find_trackers pattern
|
290
316
|
get_tracker(keys.first)
|
291
317
|
end
|
@@ -307,9 +333,7 @@ module Istox
|
|
307
333
|
|
308
334
|
def eid(ex)
|
309
335
|
eid = ex.name
|
310
|
-
if eid.empty?
|
311
|
-
eid = :default
|
312
|
-
end
|
336
|
+
eid = :default if eid.empty?
|
313
337
|
|
314
338
|
eid
|
315
339
|
end
|
@@ -327,9 +351,9 @@ module Istox
|
|
327
351
|
def data
|
328
352
|
Hashie.logger.level = 'ERROR'
|
329
353
|
@data = Hashie::Mash.new(
|
330
|
-
|
331
|
-
|
332
|
-
|
354
|
+
YAML.safe_load(
|
355
|
+
ERB.new(File.read(ENV['AMQP_CONFIG'] || 'config/amqp.yml')).result
|
356
|
+
)
|
333
357
|
)
|
334
358
|
end
|
335
359
|
|
@@ -356,11 +380,11 @@ module Istox
|
|
356
380
|
def find_trackers(pattern)
|
357
381
|
cursor = 0
|
358
382
|
all_keys = []
|
359
|
-
loop
|
360
|
-
cursor, keys = ::Istox::RedisBoot.scan(cursor, {:
|
383
|
+
loop do
|
384
|
+
cursor, keys = ::Istox::RedisBoot.scan(cursor, { match: pattern, count: 500 }, 4)
|
361
385
|
all_keys += keys
|
362
|
-
break if cursor ==
|
363
|
-
|
386
|
+
break if cursor == '0'
|
387
|
+
end
|
364
388
|
all_keys.uniq
|
365
389
|
end
|
366
390
|
|
@@ -9,6 +9,12 @@ module Istox
|
|
9
9
|
@@hosts[host_type] = url
|
10
10
|
end
|
11
11
|
|
12
|
+
def add_interceptors(interceptor)
|
13
|
+
@interceptors = [] unless defined?(@interceptors)
|
14
|
+
|
15
|
+
@interceptors.push(interceptor)
|
16
|
+
end
|
17
|
+
|
12
18
|
def call(host_type, service, method, grpc_retries_count: 1, **keyword_args)
|
13
19
|
execute(host_type, service, method, **keyword_args)
|
14
20
|
rescue Gruf::Client::Errors::Unavailable => e
|
@@ -72,7 +78,7 @@ module Istox
|
|
72
78
|
|
73
79
|
log.info 'Reinitiating to grpc host at ' + host_url
|
74
80
|
t1 = Time.now
|
75
|
-
@@services[get_key(host_type, service)] = ::Gruf::Client.new(service: service, options: { hostname: host_url }, client_options:
|
81
|
+
@@services[get_key(host_type, service)] = ::Gruf::Client.new(service: service, options: { hostname: host_url }, client_options: client_options)
|
76
82
|
log.info "Time taken for reinitiating grpc host: #{Time.now - t1} seconds"
|
77
83
|
end
|
78
84
|
|
@@ -80,8 +86,11 @@ module Istox
|
|
80
86
|
host_type.to_s + service.to_s
|
81
87
|
end
|
82
88
|
|
83
|
-
def
|
89
|
+
def client_options
|
90
|
+
@interceptors = [] unless defined?(@interceptors)
|
91
|
+
|
84
92
|
{
|
93
|
+
interceptors: @interceptors,
|
85
94
|
channel_args: default_channel_options
|
86
95
|
}
|
87
96
|
end
|
@@ -3,9 +3,8 @@ require 'redis'
|
|
3
3
|
module Istox
|
4
4
|
class RedisBoot
|
5
5
|
class << self
|
6
|
-
|
7
|
-
|
8
|
-
redis(db).set(k,v, options)
|
6
|
+
def sets(k, v, options = {}, db)
|
7
|
+
redis(db).set(k, v, options)
|
9
8
|
end
|
10
9
|
|
11
10
|
def scan(cursor, options = {}, db)
|
@@ -44,10 +43,12 @@ module Istox
|
|
44
43
|
|
45
44
|
def redis(db)
|
46
45
|
@redis = {} if @redis.nil?
|
47
|
-
@redis[db]
|
48
|
-
|
46
|
+
if @redis[db].nil?
|
47
|
+
@redis[db] = Redis.new(
|
48
|
+
url: ENV['REDIS_URL'] || 'redis://127.0.0.1',
|
49
49
|
db: db
|
50
|
-
|
50
|
+
)
|
51
|
+
end
|
51
52
|
@redis[db]
|
52
53
|
end
|
53
54
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Istox
|
2
|
+
class RedisManager
|
3
|
+
class << self
|
4
|
+
# for remote model cache
|
5
|
+
def remote_model_redis
|
6
|
+
@remote_model_redis ||= Redis::Namespace.new(:platform_cache, redis: Redis.new(
|
7
|
+
url: ENV['REDIS_URL'] || 'redis://127.0.0.1:16379',
|
8
|
+
db: 5
|
9
|
+
))
|
10
|
+
end
|
11
|
+
|
12
|
+
# cache request to prevent double submission
|
13
|
+
def request_redis
|
14
|
+
@request_redis ||= Redis::Namespace.new(:request_cache, redis: Redis.new(
|
15
|
+
url: ENV['REDIS_URL'] || 'redis://127.0.0.1:16379',
|
16
|
+
db: 6
|
17
|
+
))
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'istox/helpers/logger'
|
2
|
+
require 'istox/helpers/redis_manager'
|
2
3
|
|
3
4
|
module Istox
|
4
5
|
class RemoteModelCache
|
@@ -77,10 +78,7 @@ module Istox
|
|
77
78
|
end
|
78
79
|
|
79
80
|
def redis
|
80
|
-
|
81
|
-
url: ENV['REDIS_URL'] || 'redis://127.0.0.1:16379',
|
82
|
-
db: 5
|
83
|
-
))
|
81
|
+
::Istox::RedisManager.remote_model_redis
|
84
82
|
end
|
85
83
|
end
|
86
84
|
end
|
@@ -4,7 +4,6 @@ module Istox
|
|
4
4
|
# Publisher is relying on BunnyBoot to publish message, please make sure BunnyBoot is initalised properly first during runtime.
|
5
5
|
class Subscriber
|
6
6
|
class << self
|
7
|
-
|
8
7
|
# optionally can pass in consumer_key for single subscription / consumer_keys for multiple subcriptions
|
9
8
|
# consumer_key must be defined in amqp.yml
|
10
9
|
# if nothing pass in it will auto subscribe to all available consumers defined in amqp.yml queues key
|
@@ -81,22 +80,22 @@ module Istox
|
|
81
80
|
end
|
82
81
|
|
83
82
|
if manual_ack
|
84
|
-
if exchange.nil?
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
83
|
+
letter_exchange = if exchange.nil?
|
84
|
+
active_channel.default_exchange.name
|
85
|
+
else
|
86
|
+
exchange.name
|
87
|
+
end
|
89
88
|
retry_queue = active_channel.queue("#{queue_name}.retry", arguments: {
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
89
|
+
'x-dead-letter-exchange': letter_exchange,
|
90
|
+
'x-dead-letter-routing-key': queue_name.to_s,
|
91
|
+
'x-message-ttl': (::Istox::BunnyBoot.queue_retry_gap consumer_key)
|
92
|
+
})
|
93
|
+
if exchange_name.nil?
|
94
|
+
exchange_retry = active_channel.default_exchange
|
95
|
+
else
|
95
96
|
exchange_retry_name = "#{exchange_name}.retry"
|
96
97
|
exchange_retry = active_channel.send exchange_type, exchange_retry_name, durable: exchange_durable
|
97
98
|
retry_queue.bind exchange_retry, routing_key: "#{queue_name}.retry" if manual_ack
|
98
|
-
else
|
99
|
-
exchange_retry = active_channel.default_exchange
|
100
99
|
end
|
101
100
|
end
|
102
101
|
|
@@ -117,14 +116,14 @@ module Istox
|
|
117
116
|
@workers = {} if @workers.nil?
|
118
117
|
unless block || @workers[ruby_class]
|
119
118
|
klass = Object.const_get(
|
120
|
-
|
119
|
+
'::' + ruby_class.camelize
|
121
120
|
)
|
122
121
|
param = ::Istox::BunnyBoot.queue_worker_param consumer_key
|
123
|
-
if param.nil?
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
122
|
+
@workers[ruby_class] = if param.nil?
|
123
|
+
klass.new
|
124
|
+
else
|
125
|
+
klass.new param
|
126
|
+
end
|
128
127
|
end
|
129
128
|
# Subscribe queue
|
130
129
|
priority = ::Istox::BunnyBoot.queue_priority consumer_key
|
@@ -150,16 +149,23 @@ module Istox
|
|
150
149
|
|
151
150
|
if process
|
152
151
|
processing_payload = JSON.parse(payload)
|
153
|
-
|
152
|
+
if ::Istox::BunnyBoot.queue_worker_param_format(consumer_key) == 'open_struct'
|
153
|
+
processing_payload = ::Istox::CommonHelper.to_open_struct(processing_payload)
|
154
|
+
end
|
154
155
|
log.info "Processing in consumer: #{klass}, payload: #{processing_payload.to_h.inspect}"
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
156
|
+
|
157
|
+
interceptors = ::Istox::BunnyBoot.consumer_interceptors.dup
|
158
|
+
|
159
|
+
result = intercept(interceptors, processing_payload, metadata, delivery_info) do
|
160
|
+
if @workers[ruby_class].nil?
|
161
|
+
block&.call(processing_payload, metadata, delivery_info)
|
162
|
+
else
|
163
|
+
@workers[ruby_class].process(processing_payload, metadata, delivery_info)
|
164
|
+
end
|
159
165
|
end
|
160
166
|
else
|
161
167
|
# Instead of cache msg, print log
|
162
|
-
log.fatal "Drop msg at #{Time.now
|
168
|
+
log.fatal "Drop msg at #{Time.now} for queue #{queue.name}, payload is #{JSON.dump(payload)}" if store
|
163
169
|
end
|
164
170
|
# active_channel.ack(delivery_info.delivery_tag) if manual_ack
|
165
171
|
rescue StandardError => e
|
@@ -175,29 +181,44 @@ module Istox
|
|
175
181
|
# here we adopt the unused priority as remaining retry_count
|
176
182
|
if process
|
177
183
|
processing_payload = JSON.parse(payload)
|
178
|
-
|
184
|
+
if ::Istox::BunnyBoot.queue_worker_param_format(consumer_key) == 'open_struct'
|
185
|
+
processing_payload = ::Istox::CommonHelper.to_open_struct(processing_payload)
|
186
|
+
end
|
179
187
|
::Istox::BunnyBoot.publish(exchange_retry, processing_payload, routing_key: "#{queue_name}.retry", priority: retry_count, type: 'dlx')
|
180
188
|
end
|
181
189
|
end
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
end
|
189
|
-
=end
|
190
|
+
# # For redelivered message, call 'reject' not 'nack' in order to reschedule message to tail not head of queue
|
191
|
+
# if delivery_info.redelivered
|
192
|
+
# active_channel.reject(delivery_info.delivery_tag, true)
|
193
|
+
# else
|
194
|
+
# active_channel.nack(delivery_info.delivery_tag, false, true)
|
195
|
+
# end
|
190
196
|
ensure
|
191
197
|
if manual_ack
|
192
|
-
if !multiple.nil? && !result && result == multiple
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
198
|
+
multiple = if !multiple.nil? && !result && result == multiple
|
199
|
+
true
|
200
|
+
else
|
201
|
+
false
|
202
|
+
end
|
197
203
|
active_channel.ack(delivery_info.delivery_tag, multiple)
|
198
204
|
end
|
199
205
|
end
|
200
206
|
end
|
207
|
+
|
208
|
+
def intercept(interceptors, payload, metadata, delivery_info)
|
209
|
+
return yield if interceptors.none?
|
210
|
+
|
211
|
+
i = interceptors.pop
|
212
|
+
return yield unless i
|
213
|
+
|
214
|
+
i.call(payload, metadata, delivery_info) do
|
215
|
+
if interceptors.any?
|
216
|
+
intercept(interceptors, payload, metadata, delivery_info) { yield }
|
217
|
+
else
|
218
|
+
yield
|
219
|
+
end
|
220
|
+
end
|
221
|
+
end
|
201
222
|
end
|
202
223
|
end
|
203
224
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'istox/helpers/logger'
|
2
|
+
require 'aws-xray-sdk'
|
3
|
+
|
4
|
+
module Istox
|
5
|
+
module Xray
|
6
|
+
class GrpcClientXrayInterceptor < ::Gruf::Interceptors::ClientInterceptor
|
7
|
+
def call(request_context:)
|
8
|
+
no_segment = false
|
9
|
+
|
10
|
+
begin
|
11
|
+
no_segment = XRay.recorder.current_segment.blank?
|
12
|
+
rescue XRay::ContextMissingError
|
13
|
+
no_segment = true
|
14
|
+
end
|
15
|
+
|
16
|
+
## usually we have active segment for client interceptor
|
17
|
+
## but in some special case we might not have segment initiated
|
18
|
+
## eg. when running from rails console,
|
19
|
+
## in those case we will initiate the segment here manully
|
20
|
+
XRay.recorder.begin_segment('grpc_request') if no_segment
|
21
|
+
|
22
|
+
XRay.recorder.begin_subsegment("grpc_request.#{request_context.method}")
|
23
|
+
|
24
|
+
request_context.metadata[:xray_trace_id] = XRay.recorder.current_segment.trace_id
|
25
|
+
request_context.metadata[:xray_parent_id] = XRay.recorder.current_segment.id
|
26
|
+
|
27
|
+
result = yield
|
28
|
+
|
29
|
+
XRay.recorder.end_subsegment
|
30
|
+
|
31
|
+
XRay.recorder.end_segment if no_segment
|
32
|
+
|
33
|
+
result
|
34
|
+
rescue StandardError => e
|
35
|
+
log.error e
|
36
|
+
raise e
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'istox/helpers/logger'
|
2
|
+
require 'aws-xray-sdk'
|
3
|
+
|
4
|
+
module Istox
|
5
|
+
module Xray
|
6
|
+
class GrpcServerXrayInterceptor < ::Gruf::Interceptors::ServerInterceptor
|
7
|
+
def call
|
8
|
+
meta = request.active_call.metadata
|
9
|
+
trace_id = (meta['xray_trace_id'] if meta.present? && meta.key?('xray_trace_id'))
|
10
|
+
parent_id = (meta['xray_parent_id'] if meta.present? && meta.key?('xray_parent_id'))
|
11
|
+
|
12
|
+
XRay.recorder.begin_segment("#{::Istox::Xray::XrayInitializer.service_name}.grpc.#{request.method_key}",
|
13
|
+
trace_id: trace_id, parent_id: parent_id)
|
14
|
+
|
15
|
+
result = yield # this returns the protobuf message
|
16
|
+
|
17
|
+
XRay.recorder.end_segment
|
18
|
+
|
19
|
+
result
|
20
|
+
rescue StandardError => e
|
21
|
+
log.error e
|
22
|
+
raise e
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'istox/helpers/logger'
|
2
|
+
require 'aws-xray-sdk'
|
3
|
+
|
4
|
+
module Istox
|
5
|
+
module Xray
|
6
|
+
class RabbitmqConsumerInterceptor
|
7
|
+
def call(payload, _metadata, delivery_info)
|
8
|
+
payload_hash = payload.to_h
|
9
|
+
|
10
|
+
trace_id = (payload_hash[:xray_trace_id] if payload_hash.key?(:xray_trace_id))
|
11
|
+
parent_id = (payload_hash[:xray_parent_id] if payload_hash.key?(:xray_parent_id))
|
12
|
+
|
13
|
+
arr = [delivery_info[:exchange]&.downcase, delivery_info[:routing_key]&.downcase, payload_hash[:type]&.downcase].compact
|
14
|
+
|
15
|
+
XRay.recorder.begin_segment("#{::Istox::Xray::XrayInitializer.service_name}.rabbitmq.#{arr.join('.')}",
|
16
|
+
trace_id: trace_id, parent_id: parent_id)
|
17
|
+
|
18
|
+
result = yield # this returns consumer handler message
|
19
|
+
|
20
|
+
XRay.recorder.end_segment
|
21
|
+
|
22
|
+
result
|
23
|
+
rescue StandardError => e
|
24
|
+
log.error e
|
25
|
+
raise e
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'istox/helpers/logger'
|
2
|
+
require 'aws-xray-sdk'
|
3
|
+
|
4
|
+
module Istox
|
5
|
+
module Xray
|
6
|
+
class RabbitmqPublisherInterceptor
|
7
|
+
def call(message, options)
|
8
|
+
no_segment = false
|
9
|
+
|
10
|
+
begin
|
11
|
+
no_segment = XRay.recorder.current_segment.blank?
|
12
|
+
rescue XRay::ContextMissingError
|
13
|
+
no_segment = true
|
14
|
+
end
|
15
|
+
|
16
|
+
## usually we have active segment for client interceptor
|
17
|
+
## but in some special case we might not have segment initiated
|
18
|
+
## eg. when running from rails console,
|
19
|
+
## in those case we will initiate the segment here manully
|
20
|
+
XRay.recorder.begin_segment('rabbitmq_publish') if no_segment
|
21
|
+
|
22
|
+
XRay.recorder.begin_subsegment("rabbitmq_publish.#{options[:routing_key]}")
|
23
|
+
|
24
|
+
message[:xray_trace_id] = XRay.recorder.current_segment.trace_id
|
25
|
+
message[:xray_parent_id] = XRay.recorder.current_segment.id
|
26
|
+
|
27
|
+
XRay.recorder.end_subsegment
|
28
|
+
|
29
|
+
XRay.recorder.end_segment if no_segment
|
30
|
+
rescue StandardError => e
|
31
|
+
log.error e
|
32
|
+
raise e
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'istox/helpers/logger'
|
2
|
+
require 'istox/helpers/xray/grpc_client_xray_interceptor'
|
3
|
+
require 'istox/helpers/xray/grpc_server_xray_interceptor'
|
4
|
+
require 'istox/helpers/xray/rabbitmq_publisher_interceptor'
|
5
|
+
require 'istox/helpers/xray/rabbitmq_consumer_interceptor'
|
6
|
+
require 'istox/helpers/logger'
|
7
|
+
require 'aws-xray-sdk'
|
8
|
+
|
9
|
+
module Istox
|
10
|
+
module Xray
|
11
|
+
class XrayInitializer
|
12
|
+
class << self
|
13
|
+
attr_reader :service_name
|
14
|
+
|
15
|
+
## input gruf config to enable gruf/grpc tracing, eg. Gruf.configure do |config|
|
16
|
+
## set enable_rabbitmq_trace to enable rabbitmq tracing
|
17
|
+
def init(service_name, gruf_config: nil, enable_rabbitmq_trace: nil)
|
18
|
+
if ENV['AWS_XRAY_ENABLE']&.downcase.to_s == 'true'
|
19
|
+
|
20
|
+
Rails.application.config.xray = {
|
21
|
+
# default segment name generated by XRay middleware
|
22
|
+
daemon_address: ENV['AWS_XRAY_DAEMON_ADDRESS'],
|
23
|
+
name: service_name,
|
24
|
+
patch: %I[net_http aws_sdk],
|
25
|
+
# record db transactions as subsegments
|
26
|
+
active_record: true,
|
27
|
+
context_missing: 'LOG_ERROR'
|
28
|
+
}
|
29
|
+
|
30
|
+
@service_name = service_name
|
31
|
+
|
32
|
+
if gruf_config.present?
|
33
|
+
gruf_config.interceptors.use(::Istox::Xray::GrpcServerXrayInterceptor)
|
34
|
+
::Istox::GrpcClient.add_interceptors(::Istox::Xray::GrpcClientXrayInterceptor.new)
|
35
|
+
end
|
36
|
+
|
37
|
+
return unless enable_rabbitmq_trace == true
|
38
|
+
|
39
|
+
::Istox::BunnyBoot.add_publisher_interceptor(::Istox::Xray::RabbitmqPublisherInterceptor.new)
|
40
|
+
::Istox::BunnyBoot.add_consumer_interceptor(::Istox::Xray::RabbitmqConsumerInterceptor.new)
|
41
|
+
|
42
|
+
else
|
43
|
+
Rails.application.config.xray = {
|
44
|
+
name: service_name,
|
45
|
+
context_missing: 'LOG_ERROR'
|
46
|
+
}
|
47
|
+
|
48
|
+
config = {
|
49
|
+
sampling: false,
|
50
|
+
name: service_name
|
51
|
+
}
|
52
|
+
XRay.recorder.configure(config)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
data/lib/istox/version.rb
CHANGED
metadata
CHANGED
@@ -1,15 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: istox
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.153.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Siong Leng
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-05-
|
11
|
+
date: 2020-05-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: amazing_print
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
13
27
|
- !ruby/object:Gem::Dependency
|
14
28
|
name: awesome_print
|
15
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -206,6 +220,20 @@ dependencies:
|
|
206
220
|
- - "~>"
|
207
221
|
- !ruby/object:Gem::Version
|
208
222
|
version: '0.1'
|
223
|
+
- !ruby/object:Gem::Dependency
|
224
|
+
name: aws-xray-sdk
|
225
|
+
requirement: !ruby/object:Gem::Requirement
|
226
|
+
requirements:
|
227
|
+
- - ">="
|
228
|
+
- !ruby/object:Gem::Version
|
229
|
+
version: '0'
|
230
|
+
type: :development
|
231
|
+
prerelease: false
|
232
|
+
version_requirements: !ruby/object:Gem::Requirement
|
233
|
+
requirements:
|
234
|
+
- - ">="
|
235
|
+
- !ruby/object:Gem::Version
|
236
|
+
version: '0'
|
209
237
|
- !ruby/object:Gem::Dependency
|
210
238
|
name: bullet
|
211
239
|
requirement: !ruby/object:Gem::Requirement
|
@@ -446,12 +474,18 @@ files:
|
|
446
474
|
- lib/istox/helpers/order_book_prorate.rb
|
447
475
|
- lib/istox/helpers/publisher.rb
|
448
476
|
- lib/istox/helpers/rate_limit.rb
|
449
|
-
- lib/istox/helpers/
|
477
|
+
- lib/istox/helpers/redis_boot.rb
|
478
|
+
- lib/istox/helpers/redis_manager.rb
|
450
479
|
- lib/istox/helpers/regex_helper.rb
|
451
480
|
- lib/istox/helpers/remote_model_cache.rb
|
452
481
|
- lib/istox/helpers/result_handler.rb
|
453
482
|
- lib/istox/helpers/subscriber.rb
|
454
483
|
- lib/istox/helpers/vault.rb
|
484
|
+
- lib/istox/helpers/xray/grpc_client_xray_interceptor.rb
|
485
|
+
- lib/istox/helpers/xray/grpc_server_xray_interceptor.rb
|
486
|
+
- lib/istox/helpers/xray/rabbitmq_consumer_interceptor.rb
|
487
|
+
- lib/istox/helpers/xray/rabbitmq_publisher_interceptor.rb
|
488
|
+
- lib/istox/helpers/xray/xray_initializer.rb
|
455
489
|
- lib/istox/interfaces/chainhub/transaction.rb
|
456
490
|
- lib/istox/models/blockchain_receipt.rb
|
457
491
|
- lib/istox/models/concerns/blockchain_receipt_query.rb
|