istox 0.1.152.1.test17 → 0.1.152.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +8 -23
- data/bulk-update-version.py +93 -0
- data/istox.gemspec +0 -1
- data/lib/istox.rb +0 -7
- data/lib/istox/helpers/bunny_boot.rb +31 -57
- data/lib/istox/helpers/grpc_client.rb +2 -11
- data/lib/istox/helpers/publisher.rb +13 -1
- data/lib/istox/helpers/remote_model_cache.rb +2 -2
- data/lib/istox/helpers/subscriber.rb +39 -60
- data/lib/istox/version.rb +1 -1
- metadata +5 -23
- data/lib/istox/helpers/xray/grpc_client_xray_interceptor.rb +0 -40
- data/lib/istox/helpers/xray/grpc_server_xray_interceptor.rb +0 -26
- data/lib/istox/helpers/xray/rabbitmq_consumer_interceptor.rb +0 -29
- data/lib/istox/helpers/xray/rabbitmq_publisher_interceptor.rb +0 -36
- data/lib/istox/helpers/xray/xray_initializer.rb +0 -43
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0ae1c1658b28cacf39ae2e867711f841618737369ec62116f49c3bf443635b06
|
4
|
+
data.tar.gz: 7d120a6c5b570e84f281d82cde6b6e3966ed5cbe80ec7c73bcc73fa491d4cdb7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: efc6c53f12a9b790e2645f3d61ef3df973f56290e0332028dd4a9bc0577f254e13b7f69ce5b2e9277e0429204ff342d783e36705aacbb664ade098a169ed81e8
|
7
|
+
data.tar.gz: ea90504c68e7bf96e5b7680f8fc417d6c1474246e5e624bf3f2a1b06ce0d999447ca12cbe80b09b0535bdd80c890875ce07b6093e908e8653ce22024be17b549
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
istox (0.1.152
|
4
|
+
istox (0.1.152)
|
5
5
|
awesome_print
|
6
6
|
binding_of_caller
|
7
7
|
bunny (>= 2.12.0)
|
@@ -65,20 +65,8 @@ GEM
|
|
65
65
|
arel (9.0.0)
|
66
66
|
awesome_print (1.8.0)
|
67
67
|
aws-eventstream (1.1.0)
|
68
|
-
aws-
|
69
|
-
aws-sdk-core (3.96.1)
|
70
|
-
aws-eventstream (~> 1, >= 1.0.2)
|
71
|
-
aws-partitions (~> 1, >= 1.239.0)
|
72
|
-
aws-sigv4 (~> 1.1)
|
73
|
-
jmespath (~> 1.0)
|
74
|
-
aws-sdk-xray (1.4.0)
|
75
|
-
aws-sdk-core (~> 3)
|
76
|
-
aws-sigv4 (~> 1.0)
|
77
|
-
aws-sigv4 (1.1.3)
|
68
|
+
aws-sigv4 (1.1.1)
|
78
69
|
aws-eventstream (~> 1.0, >= 1.0.2)
|
79
|
-
aws-xray-sdk (0.11.4)
|
80
|
-
aws-sdk-xray (~> 1.4.0)
|
81
|
-
multi_json (~> 1)
|
82
70
|
binding_of_caller (0.8.0)
|
83
71
|
debug_inspector (>= 0.0.1)
|
84
72
|
builder (3.2.4)
|
@@ -110,18 +98,18 @@ GEM
|
|
110
98
|
ffi (1.12.2)
|
111
99
|
globalid (0.4.2)
|
112
100
|
activesupport (>= 4.2.0)
|
113
|
-
google-protobuf (3.
|
101
|
+
google-protobuf (3.11.4-universal-darwin)
|
114
102
|
googleapis-common-protos-types (1.0.5)
|
115
103
|
google-protobuf (~> 3.11)
|
116
104
|
graphlient (0.3.7)
|
117
105
|
faraday
|
118
106
|
faraday_middleware
|
119
107
|
graphql-client
|
120
|
-
graphql (1.10.
|
108
|
+
graphql (1.10.6)
|
121
109
|
graphql-client (0.16.0)
|
122
110
|
activesupport (>= 3.0)
|
123
111
|
graphql (~> 1.8)
|
124
|
-
grpc (1.28.0)
|
112
|
+
grpc (1.28.0-universal-darwin)
|
125
113
|
google-protobuf (~> 3.11)
|
126
114
|
googleapis-common-protos-types (~> 1.0)
|
127
115
|
grpc-tools (1.28.0)
|
@@ -140,7 +128,6 @@ GEM
|
|
140
128
|
grpc (~> 1.10)
|
141
129
|
grpc-tools (~> 1.10)
|
142
130
|
slop (~> 4.6)
|
143
|
-
jmespath (1.4.0)
|
144
131
|
listen (3.0.8)
|
145
132
|
rb-fsevent (~> 0.9, >= 0.9.4)
|
146
133
|
rb-inotify (~> 0.9, >= 0.9.7)
|
@@ -156,13 +143,12 @@ GEM
|
|
156
143
|
mini_mime (1.0.1)
|
157
144
|
mini_portile2 (2.4.0)
|
158
145
|
minitest (5.14.0)
|
159
|
-
multi_json (1.14.1)
|
160
146
|
multipart-post (2.1.1)
|
161
147
|
nio4r (2.3.1)
|
162
148
|
nokogiri (1.10.9)
|
163
149
|
mini_portile2 (~> 2.4.0)
|
164
150
|
oj (3.10.6)
|
165
|
-
ougai (1.8.
|
151
|
+
ougai (1.8.3)
|
166
152
|
oj (~> 3.10)
|
167
153
|
paranoia (2.4.2)
|
168
154
|
activerecord (>= 4.0, < 6.1)
|
@@ -199,7 +185,7 @@ GEM
|
|
199
185
|
rake (>= 0.8.7)
|
200
186
|
thor (>= 0.19.0, < 2.0)
|
201
187
|
rake (10.5.0)
|
202
|
-
rb-fsevent (0.10.
|
188
|
+
rb-fsevent (0.10.3)
|
203
189
|
rb-inotify (0.10.1)
|
204
190
|
ffi (~> 1.0)
|
205
191
|
redis (4.1.3)
|
@@ -257,7 +243,7 @@ GEM
|
|
257
243
|
tzinfo (1.2.6)
|
258
244
|
thread_safe (~> 0.1)
|
259
245
|
uniform_notifier (1.11.0)
|
260
|
-
vault (0.13.
|
246
|
+
vault (0.13.0)
|
261
247
|
aws-sigv4
|
262
248
|
websocket-driver (0.7.0)
|
263
249
|
websocket-extensions (>= 0.1.0)
|
@@ -267,7 +253,6 @@ PLATFORMS
|
|
267
253
|
ruby
|
268
254
|
|
269
255
|
DEPENDENCIES
|
270
|
-
aws-xray-sdk
|
271
256
|
bullet (~> 5.7.5)
|
272
257
|
bundler (~> 1.16)
|
273
258
|
database_cleaner (~> 1.6.0)
|
@@ -0,0 +1,93 @@
|
|
1
|
+
import sys
|
2
|
+
import re
|
3
|
+
from tempfile import mkstemp
|
4
|
+
from shutil import move, copymode
|
5
|
+
from os import fdopen, remove
|
6
|
+
import os
|
7
|
+
|
8
|
+
# HOW TO RUN: python bulk-update-version.py <version updating to, eg. 0.1.150.2>
|
9
|
+
|
10
|
+
SERVICES = ["client-api", "admin-api", "message-api", "auth", "admin-auth"]
|
11
|
+
|
12
|
+
|
13
|
+
def replace(file_path, pattern, subst):
|
14
|
+
# Create temp file
|
15
|
+
fh, abs_path = mkstemp()
|
16
|
+
with fdopen(fh, 'w') as new_file:
|
17
|
+
dirname = os.path.dirname(__file__)
|
18
|
+
file_path = os.path.join(dirname, file_path)
|
19
|
+
if os.path.exists(file_path) == False:
|
20
|
+
return
|
21
|
+
with open(file_path) as old_file:
|
22
|
+
for line in old_file:
|
23
|
+
new_file.write(re.sub(pattern, subst, line))
|
24
|
+
# Copy the file permissions from the old file to the new file
|
25
|
+
copymode(file_path, abs_path)
|
26
|
+
# Remove original file
|
27
|
+
remove(file_path)
|
28
|
+
# Move new file
|
29
|
+
move(abs_path, file_path)
|
30
|
+
|
31
|
+
|
32
|
+
def push_service(file_path, version, hotfix):
|
33
|
+
dirname = os.path.dirname(__file__)
|
34
|
+
file_path = os.path.join(dirname, file_path)
|
35
|
+
|
36
|
+
if os.path.exists(file_path) == False:
|
37
|
+
return
|
38
|
+
|
39
|
+
if hotfix == True:
|
40
|
+
branch_name = "hotfix"
|
41
|
+
else:
|
42
|
+
branch_name = "FIX/update-istox-gem-" + version
|
43
|
+
|
44
|
+
os.chdir(file_path)
|
45
|
+
os.system(
|
46
|
+
"git branch -D %s &>/dev/null" % (branch_name))
|
47
|
+
os.system("echo 'Deleted local branch'")
|
48
|
+
os.system(
|
49
|
+
"git push origin --delete %s &>/dev/null" % (branch_name))
|
50
|
+
os.system("echo 'Deleted remote branch'")
|
51
|
+
os.system("git fetch -a")
|
52
|
+
os.system("git checkout -b %s" % (branch_name))
|
53
|
+
os.system("git add Gemfile")
|
54
|
+
os.system("git commit -m 'Update istox-gem version to %s'" % (version))
|
55
|
+
os.system("git push --set-upstream origin %s" % (branch_name))
|
56
|
+
# os.system("cd " + file_path + " && git branch -D " +
|
57
|
+
# branch_name + " &>/dev/null || echo 'Deleted local branch' || git push origin --delete hotfix &>/dev/null || echo 'Deleted remote branch' || git fetch -a || git checkout -b " +
|
58
|
+
# branch_name + " && git add Gemfile && git commit -m 'Update istox-gem version to "
|
59
|
+
# + version + "' && git push --set-upstream origin " + branch_name)
|
60
|
+
|
61
|
+
|
62
|
+
arguments = sys.argv
|
63
|
+
arguments.pop(0)
|
64
|
+
|
65
|
+
new_version = arguments[0]
|
66
|
+
|
67
|
+
hotfix = False
|
68
|
+
|
69
|
+
while True:
|
70
|
+
result = raw_input("Is this a hotix? Y/N \n")
|
71
|
+
|
72
|
+
if result.lower() == 'y':
|
73
|
+
hotfix = True
|
74
|
+
break
|
75
|
+
elif result.lower() == 'n':
|
76
|
+
hotfix = False
|
77
|
+
break
|
78
|
+
else:
|
79
|
+
print("Please input only Y or N.")
|
80
|
+
|
81
|
+
|
82
|
+
print("Updating services " + str(SERVICES) +
|
83
|
+
" to istox-gem version " + new_version)
|
84
|
+
|
85
|
+
for service in SERVICES:
|
86
|
+
replace("../" + service + "/Gemfile", r"gem 'istox'.+",
|
87
|
+
"gem 'istox', '" + new_version + "'")
|
88
|
+
|
89
|
+
for service in SERVICES:
|
90
|
+
push_service("../" + service, new_version, hotfix)
|
91
|
+
|
92
|
+
print("Services " + str(SERVICES) +
|
93
|
+
" has been updated to istox-gem version " + new_version)
|
data/istox.gemspec
CHANGED
@@ -44,7 +44,6 @@ Gem::Specification.new do |spec|
|
|
44
44
|
spec.add_dependency 'redis-namespace', '>= 1.0.0'
|
45
45
|
spec.add_dependency 'redis-rails', '~> 5.0.2'
|
46
46
|
spec.add_dependency 'vault', '~> 0.1'
|
47
|
-
spec.add_development_dependency 'aws-xray-sdk'
|
48
47
|
spec.add_development_dependency 'bullet', '~> 5.7.5'
|
49
48
|
spec.add_development_dependency 'bundler', '~> 1.16'
|
50
49
|
spec.add_development_dependency 'database_cleaner', '~> 1.6.0'
|
data/lib/istox.rb
CHANGED
@@ -35,13 +35,6 @@ module Istox
|
|
35
35
|
require 'istox/helpers/redis'
|
36
36
|
require 'istox/helpers/dlm'
|
37
37
|
require 'istox/helpers/remote_model_cache'
|
38
|
-
|
39
|
-
require 'istox/helpers/xray/grpc_client_xray_interceptor'
|
40
|
-
require 'istox/helpers/xray/grpc_server_xray_interceptor'
|
41
|
-
require 'istox/helpers/xray/rabbitmq_publisher_interceptor'
|
42
|
-
require 'istox/helpers/xray/rabbitmq_consumer_interceptor'
|
43
|
-
require 'istox/helpers/xray/xray_initializer'
|
44
|
-
|
45
38
|
require 'istox/models/blockchain_receipt'
|
46
39
|
require 'istox/models/concerns/blockchain_receipt_query'
|
47
40
|
require 'istox/consumers/blockchain_status_handler'
|
@@ -5,29 +5,6 @@ require 'istox/helpers/logger'
|
|
5
5
|
module Istox
|
6
6
|
class BunnyBoot
|
7
7
|
class << self
|
8
|
-
########################################################
|
9
|
-
##
|
10
|
-
## RABBITMQ interceptors
|
11
|
-
##
|
12
|
-
######################################################
|
13
|
-
def add_consumer_interceptor(interceptor)
|
14
|
-
@consumer_interceptors ||= []
|
15
|
-
@consumer_interceptors.push(interceptor)
|
16
|
-
end
|
17
|
-
|
18
|
-
def add_publisher_interceptor(interceptor)
|
19
|
-
@publisher_interceptors ||= []
|
20
|
-
@publisher_interceptors.push(interceptor)
|
21
|
-
end
|
22
|
-
|
23
|
-
def publisher_interceptors
|
24
|
-
@publisher_interceptors || []
|
25
|
-
end
|
26
|
-
|
27
|
-
def consumer_interceptors
|
28
|
-
@consumer_interceptors || []
|
29
|
-
end
|
30
|
-
|
31
8
|
# Create physical connection to RabbitMQ
|
32
9
|
# During failover of RabbitMQ cluster or temporary failure, there may be error and needs retry in loop
|
33
10
|
def connection
|
@@ -59,16 +36,16 @@ module Istox
|
|
59
36
|
def exchange(eid)
|
60
37
|
type = data[:exchanges][eid][:type]
|
61
38
|
name = eid
|
62
|
-
settings = { durable: exchange_durable?(eid)
|
39
|
+
settings = { durable: exchange_durable?(eid) }
|
63
40
|
confirm = data[:exchanges][eid][:confirm] || -1
|
64
41
|
[type, name, settings, confirm]
|
65
|
-
rescue
|
42
|
+
rescue
|
66
43
|
nil
|
67
44
|
end
|
68
45
|
|
69
46
|
def binding_exchange_id(id)
|
70
47
|
data[:binding][id][:exchange] || :default
|
71
|
-
rescue
|
48
|
+
rescue
|
72
49
|
nil
|
73
50
|
end
|
74
51
|
|
@@ -79,17 +56,17 @@ module Istox
|
|
79
56
|
else
|
80
57
|
data[:queues][queue][:queue_name]
|
81
58
|
end
|
82
|
-
rescue
|
59
|
+
rescue
|
83
60
|
nil
|
84
61
|
end
|
85
62
|
|
86
63
|
def confirm_mode(eid)
|
87
64
|
data[:exchanges][eid][:confirm] || -1
|
88
|
-
rescue
|
65
|
+
rescue
|
89
66
|
nil
|
90
67
|
end
|
91
68
|
|
92
|
-
def queues_keys_for_subscribe
|
69
|
+
def queues_keys_for_subscribe()
|
93
70
|
data['queues'].keys
|
94
71
|
end
|
95
72
|
|
@@ -112,7 +89,7 @@ module Istox
|
|
112
89
|
durable = exchange_config!(exchange_name)['durable']
|
113
90
|
durable = true if durable.nil?
|
114
91
|
durable
|
115
|
-
rescue
|
92
|
+
rescue => e
|
116
93
|
raise e
|
117
94
|
end
|
118
95
|
|
@@ -127,7 +104,7 @@ module Istox
|
|
127
104
|
|
128
105
|
def exchange_name(consumer_key)
|
129
106
|
queue_config_from_consumer_key!(consumer_key)['exchange']
|
130
|
-
rescue
|
107
|
+
rescue
|
131
108
|
nil
|
132
109
|
end
|
133
110
|
|
@@ -142,43 +119,43 @@ module Istox
|
|
142
119
|
name = "#{prefix}#{delimiter}#{name}" unless prefix.nil?
|
143
120
|
name = "#{name}#{delimiter}#{suffix}" unless suffix.nil?
|
144
121
|
name
|
145
|
-
rescue
|
122
|
+
rescue
|
146
123
|
nil
|
147
124
|
end
|
148
125
|
|
149
126
|
def queue_priority(consumer_key)
|
150
127
|
queue_config_from_consumer_key!(consumer_key)['priority']
|
151
|
-
rescue
|
128
|
+
rescue
|
152
129
|
nil
|
153
130
|
end
|
154
131
|
|
155
132
|
def queue_worker_param(consumer_key)
|
156
133
|
queue_config_from_consumer_key!(consumer_key)['worker_param']
|
157
|
-
rescue
|
134
|
+
rescue
|
158
135
|
nil
|
159
136
|
end
|
160
137
|
|
161
138
|
def queue_worker_param_format(consumer_key)
|
162
139
|
queue_config_from_consumer_key!(consumer_key)['worker_param_format'] || 'open_struct'
|
163
|
-
rescue
|
140
|
+
rescue
|
164
141
|
nil
|
165
142
|
end
|
166
143
|
|
167
144
|
def queue_routing_key(consumer_key)
|
168
145
|
queue_config_from_consumer_key!(consumer_key)['routing_key'] || (queue_name consumer_key)
|
169
|
-
rescue
|
146
|
+
rescue
|
170
147
|
nil
|
171
148
|
end
|
172
149
|
|
173
150
|
def queue_exclusive(consumer_key)
|
174
151
|
queue_config_from_consumer_key!(consumer_key)['exclusive'] || false
|
175
|
-
rescue
|
152
|
+
rescue
|
176
153
|
nil
|
177
154
|
end
|
178
155
|
|
179
156
|
def ruby_class(consumer_key)
|
180
157
|
queue_config_from_consumer_key!(consumer_key)['ruby_class']
|
181
|
-
rescue
|
158
|
+
rescue
|
182
159
|
nil
|
183
160
|
end
|
184
161
|
|
@@ -198,7 +175,7 @@ module Istox
|
|
198
175
|
end
|
199
176
|
|
200
177
|
durable
|
201
|
-
rescue
|
178
|
+
rescue
|
202
179
|
nil
|
203
180
|
end
|
204
181
|
|
@@ -253,7 +230,7 @@ module Istox
|
|
253
230
|
persistent = e.durable?
|
254
231
|
mandatory = false
|
255
232
|
# Set Mandatory & Persistent flag for non-DLX and non-manual msg
|
256
|
-
unless
|
233
|
+
unless ['dlx', 'manual'].include? options[:type]
|
257
234
|
if options[:routing_key].present?
|
258
235
|
v1 = data['publish'][eid]
|
259
236
|
v1 = v1[options[:routing_key]] unless v1.nil?
|
@@ -263,15 +240,10 @@ module Istox
|
|
263
240
|
end
|
264
241
|
options.merge!(persistent: persistent)
|
265
242
|
options.merge!(mandatory: mandatory)
|
266
|
-
# message.merge!(locale: I18n.locale)
|
267
|
-
|
268
|
-
raise 'Rabbitmq publishing message must be a hash' unless message.is_a? Hash
|
269
|
-
|
270
|
-
publisher_interceptors.each do |interceptor|
|
271
|
-
interceptor.call(message, options)
|
272
|
-
end
|
273
243
|
|
244
|
+
# message.merge!(locale: I18n.locale)
|
274
245
|
message = JSON.dump message
|
246
|
+
|
275
247
|
log.debug "Publish options are: #{options}"
|
276
248
|
log.debug "Publish message payload #{message}"
|
277
249
|
e.publish(message, options)
|
@@ -305,7 +277,7 @@ module Istox
|
|
305
277
|
# combination of channel_id:delivery_tag can uniquely identify a msg
|
306
278
|
# For each retry of msg, channel_id and delivery_tag is unchanged
|
307
279
|
# But each retry, there is new delivery_tag that should be updated
|
308
|
-
id = "#{channel_id}:#{delivery_tag}"
|
280
|
+
id = "#{channel_id.to_s}:#{delivery_tag.to_s}"
|
309
281
|
|
310
282
|
::Istox::RedisBoot.sets("#{id}:payload", JSON.dump(payload), 4)
|
311
283
|
::Istox::RedisBoot.sets("#{id}:eid", eid.to_s, 4)
|
@@ -313,7 +285,7 @@ module Istox
|
|
313
285
|
end
|
314
286
|
|
315
287
|
def find_tracker_on_channel(channel_id, delivery_tag, key)
|
316
|
-
pattern = "#{channel_id}:#{delivery_tag}:#{key}"
|
288
|
+
pattern = "#{channel_id.to_s}:#{delivery_tag.to_s}:#{key}"
|
317
289
|
keys = find_trackers pattern
|
318
290
|
get_tracker(keys.first)
|
319
291
|
end
|
@@ -335,7 +307,9 @@ module Istox
|
|
335
307
|
|
336
308
|
def eid(ex)
|
337
309
|
eid = ex.name
|
338
|
-
|
310
|
+
if eid.empty?
|
311
|
+
eid = :default
|
312
|
+
end
|
339
313
|
|
340
314
|
eid
|
341
315
|
end
|
@@ -353,9 +327,9 @@ module Istox
|
|
353
327
|
def data
|
354
328
|
Hashie.logger.level = 'ERROR'
|
355
329
|
@data = Hashie::Mash.new(
|
356
|
-
|
357
|
-
|
358
|
-
|
330
|
+
YAML.safe_load(
|
331
|
+
ERB.new(File.read(ENV['AMQP_CONFIG'] || 'config/amqp.yml')).result
|
332
|
+
)
|
359
333
|
)
|
360
334
|
end
|
361
335
|
|
@@ -382,11 +356,11 @@ module Istox
|
|
382
356
|
def find_trackers(pattern)
|
383
357
|
cursor = 0
|
384
358
|
all_keys = []
|
385
|
-
loop
|
386
|
-
cursor, keys = ::Istox::RedisBoot.scan(cursor, {
|
359
|
+
loop {
|
360
|
+
cursor, keys = ::Istox::RedisBoot.scan(cursor, {:match => pattern, :count => 500}, 4)
|
387
361
|
all_keys += keys
|
388
|
-
break if cursor ==
|
389
|
-
|
362
|
+
break if cursor == "0"
|
363
|
+
}
|
390
364
|
all_keys.uniq
|
391
365
|
end
|
392
366
|
|
@@ -9,12 +9,6 @@ module Istox
|
|
9
9
|
@@hosts[host_type] = url
|
10
10
|
end
|
11
11
|
|
12
|
-
def add_interceptors(interceptor)
|
13
|
-
@interceptors = [] unless defined?(@interceptors)
|
14
|
-
|
15
|
-
@interceptors.push(interceptor)
|
16
|
-
end
|
17
|
-
|
18
12
|
def call(host_type, service, method, grpc_retries_count: 1, **keyword_args)
|
19
13
|
execute(host_type, service, method, **keyword_args)
|
20
14
|
rescue Gruf::Client::Errors::Unavailable => e
|
@@ -78,7 +72,7 @@ module Istox
|
|
78
72
|
|
79
73
|
log.info 'Reinitiating to grpc host at ' + host_url
|
80
74
|
t1 = Time.now
|
81
|
-
@@services[get_key(host_type, service)] = ::Gruf::Client.new(service: service, options: { hostname: host_url }, client_options:
|
75
|
+
@@services[get_key(host_type, service)] = ::Gruf::Client.new(service: service, options: { hostname: host_url }, client_options: channel_options)
|
82
76
|
log.info "Time taken for reinitiating grpc host: #{Time.now - t1} seconds"
|
83
77
|
end
|
84
78
|
|
@@ -86,11 +80,8 @@ module Istox
|
|
86
80
|
host_type.to_s + service.to_s
|
87
81
|
end
|
88
82
|
|
89
|
-
def
|
90
|
-
@interceptors = [] unless defined?(@interceptors)
|
91
|
-
|
83
|
+
def channel_options
|
92
84
|
{
|
93
|
-
interceptors: @interceptors,
|
94
85
|
channel_args: default_channel_options
|
95
86
|
}
|
96
87
|
end
|
@@ -78,6 +78,9 @@ module Istox
|
|
78
78
|
@channel[t]['confirm-1'].close
|
79
79
|
@channel[t]['noconfirm'].close
|
80
80
|
@channel.delete t
|
81
|
+
|
82
|
+
# Remove exchange from @exchange
|
83
|
+
@exchanges.delete t
|
81
84
|
end
|
82
85
|
end
|
83
86
|
end.join
|
@@ -233,7 +236,16 @@ module Istox
|
|
233
236
|
sleep 1
|
234
237
|
do_publish(ex,options,message)
|
235
238
|
rescue => e
|
236
|
-
log.debug "Error happens: #{e}"
|
239
|
+
log.debug "Error happens: #{e.message}"
|
240
|
+
|
241
|
+
# If the error indicates that the channel is already closed
|
242
|
+
# then clear hash @channel and @exchange
|
243
|
+
if e.message.include? "cannot use a closed channel"
|
244
|
+
@channel.delete Thread.current.object_id
|
245
|
+
@exchanges.delete Thread.current.object_id
|
246
|
+
ex = exchange(eid)
|
247
|
+
do_publish(ex, options, message)
|
248
|
+
end
|
237
249
|
end
|
238
250
|
end
|
239
251
|
end
|
@@ -15,9 +15,9 @@ module Istox
|
|
15
15
|
log.debug "Writing data to cache, key: #{key}, fields: #{fields.inspect}"
|
16
16
|
log.debug "Cache data: #{data.inspect}"
|
17
17
|
|
18
|
-
|
19
|
-
data = data.to_json unless data.instance_of?(String)
|
18
|
+
data = data.to_json unless data.instance_of?(String)
|
20
19
|
|
20
|
+
Thread.new do
|
21
21
|
results = fields.map do |f|
|
22
22
|
redis.hset(key, f.to_s, data)
|
23
23
|
end
|
@@ -4,6 +4,7 @@ module Istox
|
|
4
4
|
# Publisher is relying on BunnyBoot to publish message, please make sure BunnyBoot is initalised properly first during runtime.
|
5
5
|
class Subscriber
|
6
6
|
class << self
|
7
|
+
|
7
8
|
# optionally can pass in consumer_key for single subscription / consumer_keys for multiple subcriptions
|
8
9
|
# consumer_key must be defined in amqp.yml
|
9
10
|
# if nothing pass in it will auto subscribe to all available consumers defined in amqp.yml queues key
|
@@ -80,22 +81,22 @@ module Istox
|
|
80
81
|
end
|
81
82
|
|
82
83
|
if manual_ack
|
83
|
-
|
84
|
-
|
85
|
-
else
|
86
|
-
exchange.name
|
87
|
-
end
|
88
|
-
retry_queue = active_channel.queue("#{queue_name}.retry", arguments: {
|
89
|
-
'x-dead-letter-exchange': letter_exchange,
|
90
|
-
'x-dead-letter-routing-key': queue_name.to_s,
|
91
|
-
'x-message-ttl': (::Istox::BunnyBoot.queue_retry_gap consumer_key)
|
92
|
-
})
|
93
|
-
if exchange_name.nil?
|
94
|
-
exchange_retry = active_channel.default_exchange
|
84
|
+
if exchange.nil?
|
85
|
+
letter_exchange = active_channel.default_exchange.name
|
95
86
|
else
|
87
|
+
letter_exchange = exchange.name
|
88
|
+
end
|
89
|
+
retry_queue = active_channel.queue("#{queue_name}.retry", arguments: {
|
90
|
+
'x-dead-letter-exchange': letter_exchange,
|
91
|
+
'x-dead-letter-routing-key': "#{queue_name}",
|
92
|
+
'x-message-ttl': (::Istox::BunnyBoot.queue_retry_gap consumer_key)
|
93
|
+
})
|
94
|
+
unless exchange_name.nil?
|
96
95
|
exchange_retry_name = "#{exchange_name}.retry"
|
97
96
|
exchange_retry = active_channel.send exchange_type, exchange_retry_name, durable: exchange_durable
|
98
97
|
retry_queue.bind exchange_retry, routing_key: "#{queue_name}.retry" if manual_ack
|
98
|
+
else
|
99
|
+
exchange_retry = active_channel.default_exchange
|
99
100
|
end
|
100
101
|
end
|
101
102
|
|
@@ -116,14 +117,14 @@ module Istox
|
|
116
117
|
@workers = {} if @workers.nil?
|
117
118
|
unless block || @workers[ruby_class]
|
118
119
|
klass = Object.const_get(
|
119
|
-
|
120
|
+
'::' + (ruby_class).camelize
|
120
121
|
)
|
121
122
|
param = ::Istox::BunnyBoot.queue_worker_param consumer_key
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
123
|
+
if param.nil?
|
124
|
+
@workers[ruby_class] = klass.new
|
125
|
+
else
|
126
|
+
@workers[ruby_class] = klass.new param
|
127
|
+
end
|
127
128
|
end
|
128
129
|
# Subscribe queue
|
129
130
|
priority = ::Istox::BunnyBoot.queue_priority consumer_key
|
@@ -149,23 +150,16 @@ module Istox
|
|
149
150
|
|
150
151
|
if process
|
151
152
|
processing_payload = JSON.parse(payload)
|
152
|
-
if ::Istox::BunnyBoot.queue_worker_param_format(consumer_key) == 'open_struct'
|
153
|
-
processing_payload = ::Istox::CommonHelper.to_open_struct(processing_payload)
|
154
|
-
end
|
153
|
+
processing_payload = ::Istox::CommonHelper.to_open_struct(processing_payload) if ::Istox::BunnyBoot.queue_worker_param_format(consumer_key) == 'open_struct'
|
155
154
|
log.info "Processing in consumer: #{klass}, payload: #{processing_payload.to_h.inspect}"
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
if @workers[ruby_class].nil?
|
161
|
-
block&.call(processing_payload, metadata, delivery_info)
|
162
|
-
else
|
163
|
-
@workers[ruby_class].process(processing_payload, metadata, delivery_info)
|
164
|
-
end
|
155
|
+
if @workers[ruby_class].nil?
|
156
|
+
result = block.call(processing_payload, metadata, delivery_info) unless block.nil?
|
157
|
+
else
|
158
|
+
result = @workers[ruby_class].process(processing_payload, metadata, delivery_info)
|
165
159
|
end
|
166
160
|
else
|
167
161
|
# Instead of cache msg, print log
|
168
|
-
log.fatal "Drop msg at #{Time.now} for queue #{queue.name}, payload is #{JSON.dump(payload)}" if store
|
162
|
+
log.fatal "Drop msg at #{Time.now.to_s} for queue #{queue.name}, payload is #{JSON.dump(payload)}" if store
|
169
163
|
end
|
170
164
|
# active_channel.ack(delivery_info.delivery_tag) if manual_ack
|
171
165
|
rescue StandardError => e
|
@@ -181,44 +175,29 @@ module Istox
|
|
181
175
|
# here we adopt the unused priority as remaining retry_count
|
182
176
|
if process
|
183
177
|
processing_payload = JSON.parse(payload)
|
184
|
-
if ::Istox::BunnyBoot.queue_worker_param_format(consumer_key) == 'open_struct'
|
185
|
-
processing_payload = ::Istox::CommonHelper.to_open_struct(processing_payload)
|
186
|
-
end
|
178
|
+
processing_payload = ::Istox::CommonHelper.to_open_struct(processing_payload) if ::Istox::BunnyBoot.queue_worker_param_format(consumer_key) == 'open_struct'
|
187
179
|
::Istox::BunnyBoot.publish(exchange_retry, processing_payload, routing_key: "#{queue_name}.retry", priority: retry_count, type: 'dlx')
|
188
180
|
end
|
189
181
|
end
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
182
|
+
=begin
|
183
|
+
# For redelivered message, call 'reject' not 'nack' in order to reschedule message to tail not head of queue
|
184
|
+
if delivery_info.redelivered
|
185
|
+
active_channel.reject(delivery_info.delivery_tag, true)
|
186
|
+
else
|
187
|
+
active_channel.nack(delivery_info.delivery_tag, false, true)
|
188
|
+
end
|
189
|
+
=end
|
196
190
|
ensure
|
197
191
|
if manual_ack
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
192
|
+
if !multiple.nil? && !result && result == multiple
|
193
|
+
multiple = true
|
194
|
+
else
|
195
|
+
multiple = false
|
196
|
+
end
|
203
197
|
active_channel.ack(delivery_info.delivery_tag, multiple)
|
204
198
|
end
|
205
199
|
end
|
206
200
|
end
|
207
|
-
|
208
|
-
def intercept(interceptors, payload, metadata, delivery_info)
|
209
|
-
return yield if interceptors.none?
|
210
|
-
|
211
|
-
i = interceptors.pop
|
212
|
-
return yield unless i
|
213
|
-
|
214
|
-
i.call(payload, metadata, delivery_info) do
|
215
|
-
if interceptors.any?
|
216
|
-
intercept(interceptors, payload, metadata, delivery_info) { yield }
|
217
|
-
else
|
218
|
-
yield
|
219
|
-
end
|
220
|
-
end
|
221
|
-
end
|
222
201
|
end
|
223
202
|
end
|
224
203
|
end
|
data/lib/istox/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: istox
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.152.
|
4
|
+
version: 0.1.152.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Siong Leng
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-06-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: awesome_print
|
@@ -206,20 +206,6 @@ dependencies:
|
|
206
206
|
- - "~>"
|
207
207
|
- !ruby/object:Gem::Version
|
208
208
|
version: '0.1'
|
209
|
-
- !ruby/object:Gem::Dependency
|
210
|
-
name: aws-xray-sdk
|
211
|
-
requirement: !ruby/object:Gem::Requirement
|
212
|
-
requirements:
|
213
|
-
- - ">="
|
214
|
-
- !ruby/object:Gem::Version
|
215
|
-
version: '0'
|
216
|
-
type: :development
|
217
|
-
prerelease: false
|
218
|
-
version_requirements: !ruby/object:Gem::Requirement
|
219
|
-
requirements:
|
220
|
-
- - ">="
|
221
|
-
- !ruby/object:Gem::Version
|
222
|
-
version: '0'
|
223
209
|
- !ruby/object:Gem::Dependency
|
224
210
|
name: bullet
|
225
211
|
requirement: !ruby/object:Gem::Requirement
|
@@ -441,6 +427,7 @@ files:
|
|
441
427
|
- Rakefile
|
442
428
|
- bin/console
|
443
429
|
- bin/setup
|
430
|
+
- bulk-update-version.py
|
444
431
|
- istox.gemspec
|
445
432
|
- lib/istox.rb
|
446
433
|
- lib/istox/constants/error.rb
|
@@ -466,11 +453,6 @@ files:
|
|
466
453
|
- lib/istox/helpers/result_handler.rb
|
467
454
|
- lib/istox/helpers/subscriber.rb
|
468
455
|
- lib/istox/helpers/vault.rb
|
469
|
-
- lib/istox/helpers/xray/grpc_client_xray_interceptor.rb
|
470
|
-
- lib/istox/helpers/xray/grpc_server_xray_interceptor.rb
|
471
|
-
- lib/istox/helpers/xray/rabbitmq_consumer_interceptor.rb
|
472
|
-
- lib/istox/helpers/xray/rabbitmq_publisher_interceptor.rb
|
473
|
-
- lib/istox/helpers/xray/xray_initializer.rb
|
474
456
|
- lib/istox/interfaces/chainhub/transaction.rb
|
475
457
|
- lib/istox/models/blockchain_receipt.rb
|
476
458
|
- lib/istox/models/concerns/blockchain_receipt_query.rb
|
@@ -490,9 +472,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
490
472
|
version: '0'
|
491
473
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
492
474
|
requirements:
|
493
|
-
- - "
|
475
|
+
- - ">="
|
494
476
|
- !ruby/object:Gem::Version
|
495
|
-
version:
|
477
|
+
version: '0'
|
496
478
|
requirements: []
|
497
479
|
rubygems_version: 3.0.6
|
498
480
|
signing_key:
|
@@ -1,40 +0,0 @@
|
|
1
|
-
require 'istox/helpers/logger'
|
2
|
-
require 'aws-xray-sdk'
|
3
|
-
|
4
|
-
module Istox
|
5
|
-
module Xray
|
6
|
-
class GrpcClientXrayInterceptor < ::Gruf::Interceptors::ClientInterceptor
|
7
|
-
def call(request_context:)
|
8
|
-
no_segment = false
|
9
|
-
|
10
|
-
begin
|
11
|
-
no_segment = XRay.recorder.current_segment.blank?
|
12
|
-
rescue XRay::ContextMissingError
|
13
|
-
no_segment = true
|
14
|
-
end
|
15
|
-
|
16
|
-
## usually we have active segment for client interceptor
|
17
|
-
## but in some special case we might not have segment initiated
|
18
|
-
## eg. when running from rails console,
|
19
|
-
## in those case we will initiate the segment here manully
|
20
|
-
XRay.recorder.begin_segment('grpc_request') if no_segment
|
21
|
-
|
22
|
-
XRay.recorder.begin_subsegment("grpc_request.#{request_context.method}")
|
23
|
-
|
24
|
-
request_context.metadata[:xray_trace_id] = XRay.recorder.current_segment.trace_id
|
25
|
-
request_context.metadata[:xray_parent_id] = XRay.recorder.current_segment.id
|
26
|
-
|
27
|
-
result = yield
|
28
|
-
|
29
|
-
XRay.recorder.end_subsegment
|
30
|
-
|
31
|
-
XRay.recorder.end_segment if no_segment
|
32
|
-
|
33
|
-
result
|
34
|
-
rescue StandardError => e
|
35
|
-
log.error e
|
36
|
-
raise e
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
@@ -1,26 +0,0 @@
|
|
1
|
-
require 'istox/helpers/logger'
|
2
|
-
require 'aws-xray-sdk'
|
3
|
-
|
4
|
-
module Istox
|
5
|
-
module Xray
|
6
|
-
class GrpcServerXrayInterceptor < ::Gruf::Interceptors::ServerInterceptor
|
7
|
-
def call
|
8
|
-
meta = request.active_call.metadata
|
9
|
-
trace_id = (meta['xray_trace_id'] if meta.present? && meta.key?('xray_trace_id'))
|
10
|
-
parent_id = (meta['xray_parent_id'] if meta.present? && meta.key?('xray_parent_id'))
|
11
|
-
|
12
|
-
XRay.recorder.begin_segment("#{::Istox::Xray::XrayInitializer.service_name}.grpc.#{request.method_key}",
|
13
|
-
trace_id: trace_id, parent_id: parent_id)
|
14
|
-
|
15
|
-
result = yield # this returns the protobuf message
|
16
|
-
|
17
|
-
XRay.recorder.end_segment
|
18
|
-
|
19
|
-
result
|
20
|
-
rescue StandardError => e
|
21
|
-
log.error e
|
22
|
-
raise e
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
@@ -1,29 +0,0 @@
|
|
1
|
-
require 'istox/helpers/logger'
|
2
|
-
require 'aws-xray-sdk'
|
3
|
-
|
4
|
-
module Istox
|
5
|
-
module Xray
|
6
|
-
class RabbitmqConsumerInterceptor
|
7
|
-
def call(payload, _metadata, delivery_info)
|
8
|
-
payload_hash = payload.to_h
|
9
|
-
|
10
|
-
trace_id = (payload_hash[:xray_trace_id] if meta.key?(:xray_trace_id))
|
11
|
-
parent_id = (payload_hash[:xray_parent_id] if meta.key?(:xray_parent_id))
|
12
|
-
|
13
|
-
arr = [delivery_info[:exchange], delivery_info[:routing_key], payload_hash[:type]].compact
|
14
|
-
|
15
|
-
XRay.recorder.begin_segment("#{::Istox::Xray::XrayInitializer.service_name}.rabbitmq.#{arr.join('.')}",
|
16
|
-
trace_id: trace_id, parent_id: parent_id)
|
17
|
-
|
18
|
-
result = yield # this returns consumer handler message
|
19
|
-
|
20
|
-
XRay.recorder.end_segment
|
21
|
-
|
22
|
-
result
|
23
|
-
rescue StandardError => e
|
24
|
-
log.error e
|
25
|
-
raise e
|
26
|
-
end
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
@@ -1,36 +0,0 @@
|
|
1
|
-
require 'istox/helpers/logger'
|
2
|
-
require 'aws-xray-sdk'
|
3
|
-
|
4
|
-
module Istox
|
5
|
-
module Xray
|
6
|
-
class RabbitmqPublisherInterceptor
|
7
|
-
def call(message, options)
|
8
|
-
no_segment = false
|
9
|
-
|
10
|
-
begin
|
11
|
-
no_segment = XRay.recorder.current_segment.blank?
|
12
|
-
rescue XRay::ContextMissingError
|
13
|
-
no_segment = true
|
14
|
-
end
|
15
|
-
|
16
|
-
## usually we have active segment for client interceptor
|
17
|
-
## but in some special case we might not have segment initiated
|
18
|
-
## eg. when running from rails console,
|
19
|
-
## in those case we will initiate the segment here manully
|
20
|
-
XRay.recorder.begin_segment('rabbitmq_publish') if no_segment
|
21
|
-
|
22
|
-
XRay.recorder.begin_subsegment("rabbitmq_publish.#{options[:routing_key]}")
|
23
|
-
|
24
|
-
message[:xray_trace_id] = XRay.recorder.current_segment.trace_id
|
25
|
-
message[:xray_parent_id] = XRay.recorder.current_segment.id
|
26
|
-
|
27
|
-
XRay.recorder.end_subsegment
|
28
|
-
|
29
|
-
XRay.recorder.end_segment if no_segment
|
30
|
-
rescue StandardError => e
|
31
|
-
log.error e
|
32
|
-
raise e
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
@@ -1,43 +0,0 @@
|
|
1
|
-
require 'istox/helpers/logger'
|
2
|
-
require 'istox/helpers/xray/grpc_client_xray_interceptor'
|
3
|
-
require 'istox/helpers/xray/grpc_server_xray_interceptor'
|
4
|
-
require 'istox/helpers/xray/rabbitmq_publisher_interceptor'
|
5
|
-
require 'istox/helpers/xray/rabbitmq_subscriber_interceptor'
|
6
|
-
require 'istox/helpers/logger'
|
7
|
-
require 'aws-xray-sdk'
|
8
|
-
|
9
|
-
module Istox
|
10
|
-
module Xray
|
11
|
-
class XrayInitializer
|
12
|
-
class << self
|
13
|
-
attr_reader :service_name
|
14
|
-
|
15
|
-
## input gruf config to enable gruf/grpc tracing, eg. Gruf.configure do |config|
|
16
|
-
## set enable_rabbitmq_trace to enable rabbitmq tracing
|
17
|
-
def init(service_name, gruf_config: nil, enable_rabbitmq_trace: nil)
|
18
|
-
Rails.application.config.xray = {
|
19
|
-
# default segment name generated by XRay middleware
|
20
|
-
daemon_address: ENV['AWS_XRAY_DAEMON_ADDRESS'],
|
21
|
-
name: service_name,
|
22
|
-
patch: %I[net_http aws_sdk],
|
23
|
-
# record db transactions as subsegments
|
24
|
-
active_record: true,
|
25
|
-
context_missing: 'LOG_ERROR'
|
26
|
-
}
|
27
|
-
|
28
|
-
@service_name = service_name
|
29
|
-
|
30
|
-
if gruf_config.present?
|
31
|
-
gruf_config.interceptors.use(::Istox::Xray::GrpcServerXrayInterceptor)
|
32
|
-
::Istox::GrpcClient.add_interceptors(::Istox::Xray::GrpcClientXrayInterceptor.new)
|
33
|
-
end
|
34
|
-
|
35
|
-
return unless enable_rabbitmq_trace == true
|
36
|
-
|
37
|
-
::Istox::BunnyBoot.add_publisher_interceptor(::Istox::Xray::RabbitmqPublisherInterceptor.new)
|
38
|
-
::Istox::BunnyBoot.add_consumer_interceptor(::Istox::Xray::RabbitmqConsumerInterceptor.new)
|
39
|
-
end
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|