istox 0.1.153.3 → 0.1.154.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.idea/istox-gem.iml +1852 -2
- data/Gemfile.lock +15 -6
- data/bulk-update-version.py +93 -0
- data/istox.gemspec +3 -0
- data/lib/istox.rb +2 -1
- data/lib/istox/helpers/bunny_boot.rb +5 -1
- data/lib/istox/helpers/grpc_client.rb +4 -0
- data/lib/istox/helpers/logger.rb +1 -1
- data/lib/istox/helpers/publisher.rb +13 -1
- data/lib/istox/helpers/{redis.rb → redis_boot.rb} +7 -6
- data/lib/istox/helpers/redis_manager.rb +21 -0
- data/lib/istox/helpers/remote_model_cache.rb +4 -6
- data/lib/istox/helpers/subscriber.rb +45 -30
- data/lib/istox/helpers/xray/grpc_client_xray_interceptor.rb +0 -1
- data/lib/istox/helpers/xray/grpc_server_xray_interceptor.rb +0 -1
- data/lib/istox/helpers/xray/rabbitmq_consumer_interceptor.rb +0 -1
- data/lib/istox/helpers/xray/rabbitmq_publisher_interceptor.rb +0 -1
- data/lib/istox/helpers/xray/xray_initializer.rb +0 -1
- data/lib/istox/quant/bond.rb +1 -2
- data/lib/istox/version.rb +1 -1
- metadata +48 -4
data/Gemfile.lock
CHANGED
@@ -1,10 +1,12 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
istox (0.1.
|
4
|
+
istox (0.1.154)
|
5
|
+
amazing_print
|
5
6
|
awesome_print
|
6
7
|
binding_of_caller
|
7
8
|
bunny (>= 2.12.0)
|
9
|
+
exponential-backoff (= 0.0.4)
|
8
10
|
graphlient
|
9
11
|
gruf
|
10
12
|
hashie (~> 3.5.7)
|
@@ -61,6 +63,7 @@ GEM
|
|
61
63
|
i18n (>= 0.7, < 2)
|
62
64
|
minitest (~> 5.1)
|
63
65
|
tzinfo (~> 1.1)
|
66
|
+
amazing_print (1.1.0)
|
64
67
|
amq-protocol (2.3.1)
|
65
68
|
arel (9.0.0)
|
66
69
|
awesome_print (1.8.0)
|
@@ -87,6 +90,7 @@ GEM
|
|
87
90
|
uniform_notifier (~> 1.11.0)
|
88
91
|
bunny (2.15.0)
|
89
92
|
amq-protocol (~> 2.3, >= 2.3.1)
|
93
|
+
byebug (11.1.3)
|
90
94
|
coderay (1.1.2)
|
91
95
|
concurrent-ruby (1.1.6)
|
92
96
|
crass (1.0.6)
|
@@ -94,6 +98,7 @@ GEM
|
|
94
98
|
debug_inspector (0.0.3)
|
95
99
|
diff-lcs (1.3)
|
96
100
|
erubi (1.9.0)
|
101
|
+
exponential-backoff (0.0.4)
|
97
102
|
factory_bot (4.8.2)
|
98
103
|
activesupport (>= 3.0.0)
|
99
104
|
factory_bot_rails (4.8.2)
|
@@ -107,14 +112,14 @@ GEM
|
|
107
112
|
multipart-post (>= 1.2, < 3)
|
108
113
|
faraday_middleware (1.0.0)
|
109
114
|
faraday (~> 1.0)
|
110
|
-
ffi (1.
|
115
|
+
ffi (1.13.0)
|
111
116
|
globalid (0.4.2)
|
112
117
|
activesupport (>= 4.2.0)
|
113
|
-
google-protobuf (3.12.
|
118
|
+
google-protobuf (3.12.2-universal-darwin)
|
114
119
|
googleapis-common-protos-types (1.0.5)
|
115
120
|
google-protobuf (~> 3.11)
|
116
|
-
graphlient (0.
|
117
|
-
faraday
|
121
|
+
graphlient (0.4.0)
|
122
|
+
faraday (>= 1.0)
|
118
123
|
faraday_middleware
|
119
124
|
graphql-client
|
120
125
|
graphql (1.10.10)
|
@@ -169,6 +174,9 @@ GEM
|
|
169
174
|
pry (0.12.2)
|
170
175
|
coderay (~> 1.1.0)
|
171
176
|
method_source (~> 0.9.0)
|
177
|
+
pry-byebug (3.8.0)
|
178
|
+
byebug (~> 11.0)
|
179
|
+
pry (~> 0.10)
|
172
180
|
pry-rails (0.3.9)
|
173
181
|
pry (>= 0.10.4)
|
174
182
|
rack (2.2.2)
|
@@ -257,7 +265,7 @@ GEM
|
|
257
265
|
tzinfo (1.2.6)
|
258
266
|
thread_safe (~> 0.1)
|
259
267
|
uniform_notifier (1.11.0)
|
260
|
-
vault (0.
|
268
|
+
vault (0.14.0)
|
261
269
|
aws-sigv4
|
262
270
|
websocket-driver (0.7.0)
|
263
271
|
websocket-extensions (>= 0.1.0)
|
@@ -276,6 +284,7 @@ DEPENDENCIES
|
|
276
284
|
fakeredis
|
277
285
|
istox!
|
278
286
|
pry
|
287
|
+
pry-byebug
|
279
288
|
pry-rails
|
280
289
|
rails (~> 5.2.1)
|
281
290
|
rake (~> 10.0)
|
@@ -0,0 +1,93 @@
|
|
1
|
+
import sys
|
2
|
+
import re
|
3
|
+
from tempfile import mkstemp
|
4
|
+
from shutil import move, copymode
|
5
|
+
from os import fdopen, remove
|
6
|
+
import os
|
7
|
+
|
8
|
+
# HOW TO RUN: python bulk-update-version.py <version updating to, eg. 0.1.150.2>
|
9
|
+
|
10
|
+
SERVICES = ["client-api", "admin-api", "message-api", "auth", "admin-auth"]
|
11
|
+
|
12
|
+
|
13
|
+
def replace(file_path, pattern, subst):
|
14
|
+
# Create temp file
|
15
|
+
fh, abs_path = mkstemp()
|
16
|
+
with fdopen(fh, 'w') as new_file:
|
17
|
+
dirname = os.path.dirname(__file__)
|
18
|
+
file_path = os.path.join(dirname, file_path)
|
19
|
+
if os.path.exists(file_path) == False:
|
20
|
+
return
|
21
|
+
with open(file_path) as old_file:
|
22
|
+
for line in old_file:
|
23
|
+
new_file.write(re.sub(pattern, subst, line))
|
24
|
+
# Copy the file permissions from the old file to the new file
|
25
|
+
copymode(file_path, abs_path)
|
26
|
+
# Remove original file
|
27
|
+
remove(file_path)
|
28
|
+
# Move new file
|
29
|
+
move(abs_path, file_path)
|
30
|
+
|
31
|
+
|
32
|
+
def push_service(file_path, version, hotfix):
|
33
|
+
dirname = os.path.dirname(__file__)
|
34
|
+
file_path = os.path.join(dirname, file_path)
|
35
|
+
|
36
|
+
if os.path.exists(file_path) == False:
|
37
|
+
return
|
38
|
+
|
39
|
+
if hotfix == True:
|
40
|
+
branch_name = "hotfix"
|
41
|
+
else:
|
42
|
+
branch_name = "FIX/update-istox-gem-" + version
|
43
|
+
|
44
|
+
os.chdir(file_path)
|
45
|
+
os.system(
|
46
|
+
"git branch -D %s &>/dev/null" % (branch_name))
|
47
|
+
os.system("echo 'Deleted local branch'")
|
48
|
+
os.system(
|
49
|
+
"git push origin --delete %s &>/dev/null" % (branch_name))
|
50
|
+
os.system("echo 'Deleted remote branch'")
|
51
|
+
os.system("git fetch -a")
|
52
|
+
os.system("git checkout -b %s" % (branch_name))
|
53
|
+
os.system("git add Gemfile")
|
54
|
+
os.system("git commit -m 'Update istox-gem version to %s'" % (version))
|
55
|
+
os.system("git push --set-upstream origin %s" % (branch_name))
|
56
|
+
# os.system("cd " + file_path + " && git branch -D " +
|
57
|
+
# branch_name + " &>/dev/null || echo 'Deleted local branch' || git push origin --delete hotfix &>/dev/null || echo 'Deleted remote branch' || git fetch -a || git checkout -b " +
|
58
|
+
# branch_name + " && git add Gemfile && git commit -m 'Update istox-gem version to "
|
59
|
+
# + version + "' && git push --set-upstream origin " + branch_name)
|
60
|
+
|
61
|
+
|
62
|
+
arguments = sys.argv
|
63
|
+
arguments.pop(0)
|
64
|
+
|
65
|
+
new_version = arguments[0]
|
66
|
+
|
67
|
+
hotfix = False
|
68
|
+
|
69
|
+
while True:
|
70
|
+
result = raw_input("Is this a hotix? Y/N \n")
|
71
|
+
|
72
|
+
if result.lower() == 'y':
|
73
|
+
hotfix = True
|
74
|
+
break
|
75
|
+
elif result.lower() == 'n':
|
76
|
+
hotfix = False
|
77
|
+
break
|
78
|
+
else:
|
79
|
+
print("Please input only Y or N.")
|
80
|
+
|
81
|
+
|
82
|
+
print("Updating services " + str(SERVICES) +
|
83
|
+
" to istox-gem version " + new_version)
|
84
|
+
|
85
|
+
for service in SERVICES:
|
86
|
+
replace("../" + service + "/Gemfile", r"gem 'istox'.+",
|
87
|
+
"gem 'istox', '" + new_version + "'")
|
88
|
+
|
89
|
+
for service in SERVICES:
|
90
|
+
push_service("../" + service, new_version, hotfix)
|
91
|
+
|
92
|
+
print("Services " + str(SERVICES) +
|
93
|
+
" has been updated to istox-gem version " + new_version)
|
data/istox.gemspec
CHANGED
@@ -30,9 +30,11 @@ Gem::Specification.new do |spec|
|
|
30
30
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
31
31
|
spec.require_paths = ['lib']
|
32
32
|
|
33
|
+
spec.add_dependency 'amazing_print'
|
33
34
|
spec.add_dependency 'awesome_print'
|
34
35
|
spec.add_dependency 'binding_of_caller'
|
35
36
|
spec.add_dependency 'bunny', '>= 2.12.0'
|
37
|
+
spec.add_dependency 'exponential-backoff', '0.0.4'
|
36
38
|
spec.add_dependency 'graphlient'
|
37
39
|
spec.add_dependency 'gruf'
|
38
40
|
spec.add_dependency 'hashie', '~> 3.5.7'
|
@@ -52,6 +54,7 @@ Gem::Specification.new do |spec|
|
|
52
54
|
spec.add_development_dependency 'faker', '~> 1.7.3'
|
53
55
|
spec.add_development_dependency 'fakeredis'
|
54
56
|
spec.add_development_dependency 'pry'
|
57
|
+
spec.add_development_dependency 'pry-byebug'
|
55
58
|
spec.add_development_dependency 'pry-rails'
|
56
59
|
spec.add_development_dependency 'rails', '~> 5.2.1'
|
57
60
|
spec.add_development_dependency 'rake', '~> 10.0'
|
data/lib/istox.rb
CHANGED
@@ -32,7 +32,8 @@ module Istox
|
|
32
32
|
require 'istox/helpers/common_helper'
|
33
33
|
require 'istox/helpers/regex_helper'
|
34
34
|
require 'istox/helpers/result_handler'
|
35
|
-
require 'istox/helpers/
|
35
|
+
require 'istox/helpers/redis_boot'
|
36
|
+
require 'istox/helpers/redis_manager'
|
36
37
|
require 'istox/helpers/dlm'
|
37
38
|
require 'istox/helpers/remote_model_cache'
|
38
39
|
|
@@ -215,7 +215,7 @@ module Istox
|
|
215
215
|
retry_limit = queue_config_from_consumer_key!(consumer_key)['retry_limit']
|
216
216
|
|
217
217
|
if retry_limit.nil?
|
218
|
-
|
218
|
+
5
|
219
219
|
else
|
220
220
|
retry_limit.to_i
|
221
221
|
end
|
@@ -234,6 +234,10 @@ module Istox
|
|
234
234
|
end
|
235
235
|
end
|
236
236
|
|
237
|
+
def queue_retry_gap_set?(consumer_key)
|
238
|
+
queue_config_from_consumer_key!(consumer_key)['retry_gap'].present?
|
239
|
+
end
|
240
|
+
|
237
241
|
def channel_pool_size(consumer_key)
|
238
242
|
channel = channel_config(consumer_key)
|
239
243
|
return data['channel_pool_size'] || 1 if channel.nil?
|
@@ -15,6 +15,10 @@ module Istox
|
|
15
15
|
@interceptors.push(interceptor)
|
16
16
|
end
|
17
17
|
|
18
|
+
def configure_grpc_logger
|
19
|
+
Gruf.grpc_logger = log
|
20
|
+
end
|
21
|
+
|
18
22
|
def call(host_type, service, method, grpc_retries_count: 1, **keyword_args)
|
19
23
|
execute(host_type, service, method, **keyword_args)
|
20
24
|
rescue Gruf::Client::Errors::Unavailable => e
|
data/lib/istox/helpers/logger.rb
CHANGED
@@ -6,7 +6,7 @@ module Istox
|
|
6
6
|
return @logger if @logger.present?
|
7
7
|
|
8
8
|
@logger = ::Ougai::Logger.new(STDOUT)
|
9
|
-
@logger.formatter = ::Ougai::Formatters::Readable.new unless Rails.env.production?
|
9
|
+
@logger.formatter = ::Ougai::Formatters::Readable.new unless ENV.fetch('RAILS_ENV','development') == 'production'|| (defined?(Rails) && Rails.env.production?)
|
10
10
|
|
11
11
|
@logger
|
12
12
|
end
|
@@ -78,6 +78,9 @@ module Istox
|
|
78
78
|
@channel[t]['confirm-1'].close
|
79
79
|
@channel[t]['noconfirm'].close
|
80
80
|
@channel.delete t
|
81
|
+
|
82
|
+
# Remove exchange from @exchange
|
83
|
+
@exchanges.delete t
|
81
84
|
end
|
82
85
|
end
|
83
86
|
end.join
|
@@ -233,7 +236,16 @@ module Istox
|
|
233
236
|
sleep 1
|
234
237
|
do_publish(ex,options,message)
|
235
238
|
rescue => e
|
236
|
-
log.debug "Error happens: #{e}"
|
239
|
+
log.debug "Error happens: #{e.message}"
|
240
|
+
|
241
|
+
# If the error indicates that the channel is already closed
|
242
|
+
# then clear hash @channel and @exchange
|
243
|
+
if e.message.include? "cannot use a closed channel"
|
244
|
+
@channel.delete Thread.current.object_id
|
245
|
+
@exchanges.delete Thread.current.object_id
|
246
|
+
ex = exchange(eid)
|
247
|
+
do_publish(ex, options, message)
|
248
|
+
end
|
237
249
|
end
|
238
250
|
end
|
239
251
|
end
|
@@ -3,9 +3,8 @@ require 'redis'
|
|
3
3
|
module Istox
|
4
4
|
class RedisBoot
|
5
5
|
class << self
|
6
|
-
|
7
|
-
|
8
|
-
redis(db).set(k,v, options)
|
6
|
+
def sets(k, v, options = {}, db)
|
7
|
+
redis(db).set(k, v, options)
|
9
8
|
end
|
10
9
|
|
11
10
|
def scan(cursor, options = {}, db)
|
@@ -44,10 +43,12 @@ module Istox
|
|
44
43
|
|
45
44
|
def redis(db)
|
46
45
|
@redis = {} if @redis.nil?
|
47
|
-
@redis[db]
|
48
|
-
|
46
|
+
if @redis[db].nil?
|
47
|
+
@redis[db] = Redis.new(
|
48
|
+
url: ENV['REDIS_URL'] || 'redis://127.0.0.1',
|
49
49
|
db: db
|
50
|
-
|
50
|
+
)
|
51
|
+
end
|
51
52
|
@redis[db]
|
52
53
|
end
|
53
54
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Istox
|
2
|
+
class RedisManager
|
3
|
+
class << self
|
4
|
+
# for remote model cache
|
5
|
+
def remote_model_redis
|
6
|
+
@remote_model_redis ||= Redis::Namespace.new(:platform_cache, redis: Redis.new(
|
7
|
+
url: ENV['REDIS_URL'] || 'redis://127.0.0.1:16379',
|
8
|
+
db: 5
|
9
|
+
))
|
10
|
+
end
|
11
|
+
|
12
|
+
# cache request to prevent double submission
|
13
|
+
def request_redis
|
14
|
+
@request_redis ||= Redis::Namespace.new(:request_cache, redis: Redis.new(
|
15
|
+
url: ENV['REDIS_URL'] || 'redis://127.0.0.1:16379',
|
16
|
+
db: 6
|
17
|
+
))
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'istox/helpers/logger'
|
2
|
+
require 'istox/helpers/redis_manager'
|
2
3
|
|
3
4
|
module Istox
|
4
5
|
class RemoteModelCache
|
@@ -15,9 +16,9 @@ module Istox
|
|
15
16
|
log.debug "Writing data to cache, key: #{key}, fields: #{fields.inspect}"
|
16
17
|
log.debug "Cache data: #{data.inspect}"
|
17
18
|
|
18
|
-
|
19
|
-
data = data.to_json unless data.instance_of?(String)
|
19
|
+
data = data.to_json unless data.instance_of?(String)
|
20
20
|
|
21
|
+
Thread.new do
|
21
22
|
results = fields.map do |f|
|
22
23
|
redis.hset(key, f.to_s, data)
|
23
24
|
end
|
@@ -77,10 +78,7 @@ module Istox
|
|
77
78
|
end
|
78
79
|
|
79
80
|
def redis
|
80
|
-
|
81
|
-
url: ENV['REDIS_URL'] || 'redis://127.0.0.1:16379',
|
82
|
-
db: 5
|
83
|
-
))
|
81
|
+
::Istox::RedisManager.remote_model_redis
|
84
82
|
end
|
85
83
|
end
|
86
84
|
end
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'exponential_backoff'
|
1
2
|
require 'istox/helpers/logger'
|
2
3
|
|
3
4
|
module Istox
|
@@ -79,26 +80,6 @@ module Istox
|
|
79
80
|
queue.bind exchange, queue_options
|
80
81
|
end
|
81
82
|
|
82
|
-
if manual_ack
|
83
|
-
letter_exchange = if exchange.nil?
|
84
|
-
active_channel.default_exchange.name
|
85
|
-
else
|
86
|
-
exchange.name
|
87
|
-
end
|
88
|
-
retry_queue = active_channel.queue("#{queue_name}.retry", arguments: {
|
89
|
-
'x-dead-letter-exchange': letter_exchange,
|
90
|
-
'x-dead-letter-routing-key': queue_name.to_s,
|
91
|
-
'x-message-ttl': (::Istox::BunnyBoot.queue_retry_gap consumer_key)
|
92
|
-
})
|
93
|
-
if exchange_name.nil?
|
94
|
-
exchange_retry = active_channel.default_exchange
|
95
|
-
else
|
96
|
-
exchange_retry_name = "#{exchange_name}.retry"
|
97
|
-
exchange_retry = active_channel.send exchange_type, exchange_retry_name, durable: exchange_durable
|
98
|
-
retry_queue.bind exchange_retry, routing_key: "#{queue_name}.retry" if manual_ack
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
83
|
loop do
|
103
84
|
break unless ::Istox::BunnyBoot.queue_single_consumer? consumer_key
|
104
85
|
|
@@ -115,9 +96,7 @@ module Istox
|
|
115
96
|
ruby_class = ::Istox::BunnyBoot.ruby_class(consumer_key) || "#{consumer_key.to_s.underscore}_consumer"
|
116
97
|
@workers = {} if @workers.nil?
|
117
98
|
unless block || @workers[ruby_class]
|
118
|
-
klass =
|
119
|
-
'::' + ruby_class.camelize
|
120
|
-
)
|
99
|
+
klass = consumer_class(ruby_class)
|
121
100
|
param = ::Istox::BunnyBoot.queue_worker_param consumer_key
|
122
101
|
@workers[ruby_class] = if param.nil?
|
123
102
|
klass.new
|
@@ -184,15 +163,13 @@ module Istox
|
|
184
163
|
if ::Istox::BunnyBoot.queue_worker_param_format(consumer_key) == 'open_struct'
|
185
164
|
processing_payload = ::Istox::CommonHelper.to_open_struct(processing_payload)
|
186
165
|
end
|
187
|
-
|
166
|
+
|
167
|
+
retry_obj = create_retry_obj(consumer_key, active_channel, exchange, exchange_name, queue_name, retry_count - 1)
|
168
|
+
|
169
|
+
::Istox::BunnyBoot.publish(retry_obj.exchange_retry, processing_payload,
|
170
|
+
routing_key: retry_obj.routing_key, priority: retry_count, type: 'dlx')
|
188
171
|
end
|
189
172
|
end
|
190
|
-
# # For redelivered message, call 'reject' not 'nack' in order to reschedule message to tail not head of queue
|
191
|
-
# if delivery_info.redelivered
|
192
|
-
# active_channel.reject(delivery_info.delivery_tag, true)
|
193
|
-
# else
|
194
|
-
# active_channel.nack(delivery_info.delivery_tag, false, true)
|
195
|
-
# end
|
196
173
|
ensure
|
197
174
|
if manual_ack
|
198
175
|
multiple = if !multiple.nil? && !result && result == multiple
|
@@ -205,6 +182,44 @@ module Istox
|
|
205
182
|
end
|
206
183
|
end
|
207
184
|
|
185
|
+
def create_retry_obj(consumer_key, active_channel, _exchange, exchange_name, queue_name, retried_count)
|
186
|
+
minimal_interval = 1.0
|
187
|
+
maximal_elapsed_time = 60.0
|
188
|
+
|
189
|
+
# retry interval will be [1, 3, 9, 27, 60, 60....] seconds
|
190
|
+
backoff = ExponentialBackoff.new(minimal_interval, maximal_elapsed_time)
|
191
|
+
backoff.multiplier = 3
|
192
|
+
ttl = ::Istox::BunnyBoot.queue_retry_gap_set?(consumer_key) ? ::Istox::BunnyBoot.queue_retry_gap(consumer_key) : backoff.interval_at(retried_count) * 1000
|
193
|
+
ttl = ttl.to_i
|
194
|
+
|
195
|
+
routing_key = "#{queue_name}.retry.#{ttl}"
|
196
|
+
|
197
|
+
retry_queue = active_channel.queue(routing_key, arguments: {
|
198
|
+
'x-dead-letter-exchange': active_channel.default_exchange.name,
|
199
|
+
'x-dead-letter-routing-key': queue_name.to_s,
|
200
|
+
'x-message-ttl': ttl.to_i
|
201
|
+
})
|
202
|
+
if exchange_name.nil?
|
203
|
+
exchange_retry = active_channel.default_exchange
|
204
|
+
else
|
205
|
+
exchange_durable = ::Istox::BunnyBoot.exchange_durable? exchange_name
|
206
|
+
exchange_retry_name = "#{exchange_name}.retry"
|
207
|
+
exchange_retry = active_channel.direct(exchange_retry_name, durable: exchange_durable)
|
208
|
+
retry_queue.bind exchange_retry, routing_key: routing_key
|
209
|
+
end
|
210
|
+
|
211
|
+
OpenStruct.new(
|
212
|
+
exchange_retry: exchange_retry,
|
213
|
+
routing_key: routing_key
|
214
|
+
)
|
215
|
+
end
|
216
|
+
|
217
|
+
def consumer_class(ruby_class)
|
218
|
+
Object.const_get(
|
219
|
+
'::' + ruby_class.camelize
|
220
|
+
)
|
221
|
+
end
|
222
|
+
|
208
223
|
def intercept(interceptors, payload, metadata, delivery_info)
|
209
224
|
return yield if interceptors.none?
|
210
225
|
|