istox 0.1.129 → 0.1.130.pre.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.idea/istox-gem.iml +577 -562
- data/Gemfile.lock +13 -10
- data/istox.gemspec +1 -0
- data/lib/istox/helpers/bunny_boot.rb +37 -4
- data/lib/istox/helpers/dlm.rb +23 -0
- data/lib/istox/helpers/publisher.rb +1 -1
- data/lib/istox/helpers/redis.rb +6 -10
- data/lib/istox/helpers/subscriber.rb +40 -31
- data/lib/istox/version.rb +1 -1
- metadata +19 -4
data/Gemfile.lock
CHANGED
@@ -15,6 +15,7 @@ PATH
|
|
15
15
|
redis (>= 2.0.0)
|
16
16
|
redis-namespace (>= 1.0.0)
|
17
17
|
redis-rails (~> 5.0.2)
|
18
|
+
redlock (~> 1.2.0)
|
18
19
|
vault (~> 0.1)
|
19
20
|
|
20
21
|
GEM
|
@@ -90,29 +91,29 @@ GEM
|
|
90
91
|
i18n (~> 0.5)
|
91
92
|
fakeredis (0.7.0)
|
92
93
|
redis (>= 3.2, < 5.0)
|
93
|
-
faraday (0.
|
94
|
+
faraday (1.0.0)
|
94
95
|
multipart-post (>= 1.2, < 3)
|
95
|
-
faraday_middleware (0.
|
96
|
-
faraday (
|
96
|
+
faraday_middleware (1.0.0.rc1)
|
97
|
+
faraday (~> 1.0)
|
97
98
|
ffi (1.12.2)
|
98
99
|
globalid (0.4.2)
|
99
100
|
activesupport (>= 4.2.0)
|
100
|
-
google-protobuf (3.11.4
|
101
|
+
google-protobuf (3.11.4)
|
101
102
|
googleapis-common-protos-types (1.0.4)
|
102
103
|
google-protobuf (~> 3.0)
|
103
104
|
graphlient (0.3.7)
|
104
105
|
faraday
|
105
106
|
faraday_middleware
|
106
107
|
graphql-client
|
107
|
-
graphql (1.10.
|
108
|
+
graphql (1.10.5)
|
108
109
|
graphql-client (0.16.0)
|
109
110
|
activesupport (>= 3.0)
|
110
111
|
graphql (~> 1.8)
|
111
|
-
grpc (1.27.0
|
112
|
+
grpc (1.27.0)
|
112
113
|
google-protobuf (~> 3.11)
|
113
114
|
googleapis-common-protos-types (~> 1.0)
|
114
115
|
grpc-tools (1.27.0)
|
115
|
-
gruf (2.8.
|
116
|
+
gruf (2.8.1)
|
116
117
|
activesupport (> 4)
|
117
118
|
concurrent-ruby (> 1)
|
118
119
|
grpc (~> 1.10)
|
@@ -146,9 +147,9 @@ GEM
|
|
146
147
|
nio4r (2.3.1)
|
147
148
|
nokogiri (1.10.9)
|
148
149
|
mini_portile2 (~> 2.4.0)
|
149
|
-
oj (3.10.
|
150
|
-
ougai (1.8.
|
151
|
-
oj (~> 3.
|
150
|
+
oj (3.10.5)
|
151
|
+
ougai (1.8.3)
|
152
|
+
oj (~> 3.10)
|
152
153
|
paranoia (2.4.2)
|
153
154
|
activerecord (>= 4.0, < 6.1)
|
154
155
|
rack (2.2.2)
|
@@ -201,6 +202,8 @@ GEM
|
|
201
202
|
redis-store (>= 1.2, < 2)
|
202
203
|
redis-store (1.8.2)
|
203
204
|
redis (>= 4, < 5)
|
205
|
+
redlock (1.2.0)
|
206
|
+
redis (>= 3.0.0, < 5.0)
|
204
207
|
rspec (3.8.0)
|
205
208
|
rspec-core (~> 3.8.0)
|
206
209
|
rspec-expectations (~> 3.8.0)
|
data/istox.gemspec
CHANGED
@@ -44,6 +44,7 @@ Gem::Specification.new do |spec|
|
|
44
44
|
spec.add_dependency 'redis', '>= 2.0.0'
|
45
45
|
spec.add_dependency 'redis-namespace', '>= 1.0.0'
|
46
46
|
spec.add_dependency 'vault', '~> 0.1'
|
47
|
+
spec.add_dependency 'redlock', '~>1.2.0'
|
47
48
|
spec.add_development_dependency 'bullet', '~> 5.7.5'
|
48
49
|
spec.add_development_dependency 'bundler', '~> 1.16'
|
49
50
|
spec.add_development_dependency 'database_cleaner', '~> 1.6.0'
|
@@ -52,7 +52,11 @@ module Istox
|
|
52
52
|
|
53
53
|
def binding_routing_key(id)
|
54
54
|
queue = data[:binding][id][:queue]
|
55
|
-
data[:queues][queue]
|
55
|
+
if data[:queues][queue].nil?
|
56
|
+
queue
|
57
|
+
else
|
58
|
+
data[:queues][queue][:queue_name]
|
59
|
+
end
|
56
60
|
rescue
|
57
61
|
nil
|
58
62
|
end
|
@@ -106,7 +110,35 @@ module Istox
|
|
106
110
|
end
|
107
111
|
|
108
112
|
def queue_name(consumer_key)
|
109
|
-
queue_config_from_consumer_key!(consumer_key)['queue_name']
|
113
|
+
name = queue_config_from_consumer_key!(consumer_key)['queue_name']
|
114
|
+
return name if name.nil?
|
115
|
+
|
116
|
+
prefix = queue_config_from_consumer_key!(consumer_key)['queue_prefix']
|
117
|
+
suffix = queue_config_from_consumer_key!(consumer_key)['queue_suffix']
|
118
|
+
delimiter = queue_config_from_consumer_key!(consumer_key)['queue_delimiter'] || '' unless prefix.nil? && suffix.nil?
|
119
|
+
|
120
|
+
name = "#{prefix}#{delimiter}#{name}" unless prefix.nil?
|
121
|
+
name = "#{name}#{delimiter}#{suffix}" unless suffix.nil?
|
122
|
+
log.debug "name is #{name}"
|
123
|
+
name
|
124
|
+
rescue
|
125
|
+
nil
|
126
|
+
end
|
127
|
+
|
128
|
+
def queue_priority(consumer_key)
|
129
|
+
queue_config_from_consumer_key!(consumer_key)['priority']
|
130
|
+
rescue
|
131
|
+
nil
|
132
|
+
end
|
133
|
+
|
134
|
+
def queue_worker_param(consumer_key)
|
135
|
+
queue_config_from_consumer_key!(consumer_key)['worker_param']
|
136
|
+
rescue
|
137
|
+
nil
|
138
|
+
end
|
139
|
+
|
140
|
+
def queue_exclusive(consumer_key)
|
141
|
+
queue_config_from_consumer_key!(consumer_key)['exclusive'] || false
|
110
142
|
rescue
|
111
143
|
nil
|
112
144
|
end
|
@@ -129,7 +161,7 @@ module Istox
|
|
129
161
|
durable = queue_config_from_consumer_key!(consumer_key)['durable']
|
130
162
|
if durable.nil?
|
131
163
|
exchange_name = exchange_name consumer_key
|
132
|
-
durable = exchange_durable? exchange_name
|
164
|
+
durable = exchange_durable? exchange_name unless exchange_name.nil?
|
133
165
|
end
|
134
166
|
|
135
167
|
durable
|
@@ -280,7 +312,8 @@ module Istox
|
|
280
312
|
private
|
281
313
|
|
282
314
|
def data
|
283
|
-
|
315
|
+
Hashie.logger.level = 'ERROR'
|
316
|
+
@data = Hashie::Mash.new(
|
284
317
|
YAML.safe_load(
|
285
318
|
ERB.new(File.read(ENV['AMQP_CONFIG'] || 'config/amqp.yml')).result
|
286
319
|
)
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'redlock'
|
2
|
+
|
3
|
+
module Istox
|
4
|
+
class DLMBoot
|
5
|
+
class << self
|
6
|
+
private
|
7
|
+
|
8
|
+
def lock(key, timeout = 3600, db)
|
9
|
+
loop do
|
10
|
+
break if ::Istox::RedisBoot.sets(key, 1, {nx: true, px: timeout}, db)
|
11
|
+
end
|
12
|
+
|
13
|
+
begin
|
14
|
+
yield
|
15
|
+
ensure
|
16
|
+
::Istox::RedisBoot.del key, db
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
data/lib/istox/helpers/redis.rb
CHANGED
@@ -32,16 +32,12 @@ module Istox
|
|
32
32
|
redis(db).rename old, new
|
33
33
|
end
|
34
34
|
|
35
|
-
def
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
yield
|
42
|
-
ensure
|
43
|
-
del key, db
|
44
|
-
end
|
35
|
+
def exists(key, db)
|
36
|
+
redis(db).exists key
|
37
|
+
end
|
38
|
+
|
39
|
+
def llen(key, db)
|
40
|
+
redis(db).llen key
|
45
41
|
end
|
46
42
|
|
47
43
|
private
|
@@ -9,26 +9,25 @@ module Istox
|
|
9
9
|
# consumer_key must be defined in amqp.yml
|
10
10
|
# if nothing pass in it will auto subscribe to all available consumers defined in amqp.yml queues key
|
11
11
|
def start_subscribe(subscribing_consumer_keys = [])
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
@conn = ::Istox::BunnyBoot.connection
|
20
|
-
|
21
|
-
subscribing_consumer_keys = ::Istox::BunnyBoot.queues_keys_for_subscribe if subscribing_consumer_keys.empty?
|
22
|
-
subscribing_consumer_keys.each do |key|
|
23
|
-
log.debug "Do subscribe key #{key}"
|
24
|
-
if key.is_a?(String)
|
25
|
-
do_subscribe key.to_s
|
26
|
-
elsif key.is_a?(Hash)
|
27
|
-
do_subscribe key[:id], key[:suffix], key[:multiple]
|
12
|
+
unless @flag
|
13
|
+
@mutex = Mutex.new unless @mutex.present?
|
14
|
+
@mutex.synchronize do
|
15
|
+
unless @flag
|
16
|
+
# Create connection
|
17
|
+
@conn = ::Istox::BunnyBoot.connection
|
18
|
+
@flag = true
|
28
19
|
end
|
29
20
|
end
|
21
|
+
end
|
30
22
|
|
31
|
-
|
23
|
+
subscribing_consumer_keys = ::Istox::BunnyBoot.queues_keys_for_subscribe if subscribing_consumer_keys.empty?
|
24
|
+
subscribing_consumer_keys.each do |key|
|
25
|
+
log.debug "Do subscribe key #{key}"
|
26
|
+
if key.is_a?(String)
|
27
|
+
do_subscribe key.to_s
|
28
|
+
elsif key.is_a?(Hash)
|
29
|
+
do_subscribe key[:id], key[:multiple], key[:block]
|
30
|
+
end
|
32
31
|
end
|
33
32
|
end
|
34
33
|
|
@@ -38,7 +37,7 @@ module Istox
|
|
38
37
|
|
39
38
|
private
|
40
39
|
|
41
|
-
def do_subscribe(consumer_key,
|
40
|
+
def do_subscribe(consumer_key, multiple = nil, block = nil)
|
42
41
|
manual_ack = ::Istox::BunnyBoot.queue_manual_ack? consumer_key
|
43
42
|
|
44
43
|
# Create Channel with specified pool_size and prefetch
|
@@ -50,10 +49,10 @@ module Istox
|
|
50
49
|
|
51
50
|
# Declare queue and bind to exchange
|
52
51
|
queue_name = ::Istox::BunnyBoot.queue_name consumer_key
|
53
|
-
queue_name = "#{queue_name}.#{suffix}" unless suffix.nil?
|
54
52
|
queue_durable = ::Istox::BunnyBoot.queue_durable? consumer_key
|
53
|
+
queue_exclusive = ::Istox::BunnyBoot.queue_exclusive consumer_key
|
55
54
|
begin
|
56
|
-
queue = active_channel.queue(queue_name, durable: queue_durable)
|
55
|
+
queue = active_channel.queue(queue_name, durable: queue_durable, exclusive: queue_exclusive || false)
|
57
56
|
rescue Bunny::PreconditionFailed => e
|
58
57
|
# Must re-open a new channel, because now channel is already closed
|
59
58
|
active_channel = ::Istox::BunnyBoot.channel(::Istox::BunnyBoot.connection, pool_size: pool_size, prefetch: prefetch)
|
@@ -105,7 +104,7 @@ module Istox
|
|
105
104
|
break unless ::Istox::BunnyBoot.queue_single_consumer? consumer_key
|
106
105
|
|
107
106
|
flag = false
|
108
|
-
::Istox::
|
107
|
+
::Istox::DLMBoot.lock("peatio:#{consumer_key}:locking", 5) do
|
109
108
|
flag = ::Istox::BunnyBoot.queue_ok? @conn, queue.name
|
110
109
|
end
|
111
110
|
break if flag
|
@@ -115,17 +114,23 @@ module Istox
|
|
115
114
|
|
116
115
|
# Initiate klass
|
117
116
|
ruby_class = ::Istox::BunnyBoot.ruby_class consumer_key
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
117
|
+
@workers = {} if @workers.nil?
|
118
|
+
unless ruby_class.nil? || @workers[ruby_class]
|
119
|
+
klass = Object.const_get(
|
120
|
+
'::' + (ruby_class.nil? ? "#{consumer_key.to_s.underscore}_consumer" : ruby_class).camelize
|
121
|
+
)
|
122
|
+
param = ::Istox::BunnyBoot.queue_worker_param consumer_key
|
123
|
+
if param.nil?
|
124
|
+
@workers[ruby_class] = klass.new
|
125
|
+
else
|
126
|
+
@workers[ruby_class] = klass.new param
|
127
|
+
end
|
125
128
|
end
|
126
|
-
|
127
129
|
# Subscribe queue
|
128
|
-
|
130
|
+
priority = ::Istox::BunnyBoot.queue_priority consumer_key
|
131
|
+
arguments = {}
|
132
|
+
arguments['x-max-priority'] = priority unless priority.nil?
|
133
|
+
queue.subscribe manual_ack: manual_ack, arguments: arguments do |delivery_info, metadata, payload|
|
129
134
|
log.debug "Delivery_info: #{delivery_info}"
|
130
135
|
log.debug "Metadata: #{metadata}"
|
131
136
|
log.debug "Payload: #{payload}"
|
@@ -141,7 +146,11 @@ module Istox
|
|
141
146
|
|
142
147
|
log.info "Processing in consumer: #{klass}, paylod: #{payload_object.to_h.inspect}"
|
143
148
|
|
144
|
-
|
149
|
+
if @workers[ruby_class].nil?
|
150
|
+
result = block.call(payload_object, metadata, delivery_info) unless block.nil?
|
151
|
+
else
|
152
|
+
result = @workers[ruby_class].process(payload_object, metadata, delivery_info)
|
153
|
+
end
|
145
154
|
end
|
146
155
|
# active_channel.ack(delivery_info.delivery_tag) if manual_ack
|
147
156
|
rescue StandardError => e
|
data/lib/istox/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: istox
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.130.pre.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Siong Leng
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-03-
|
11
|
+
date: 2020-03-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: awesome_print
|
@@ -206,6 +206,20 @@ dependencies:
|
|
206
206
|
- - "~>"
|
207
207
|
- !ruby/object:Gem::Version
|
208
208
|
version: '0.1'
|
209
|
+
- !ruby/object:Gem::Dependency
|
210
|
+
name: redlock
|
211
|
+
requirement: !ruby/object:Gem::Requirement
|
212
|
+
requirements:
|
213
|
+
- - "~>"
|
214
|
+
- !ruby/object:Gem::Version
|
215
|
+
version: 1.2.0
|
216
|
+
type: :runtime
|
217
|
+
prerelease: false
|
218
|
+
version_requirements: !ruby/object:Gem::Requirement
|
219
|
+
requirements:
|
220
|
+
- - "~>"
|
221
|
+
- !ruby/object:Gem::Version
|
222
|
+
version: 1.2.0
|
209
223
|
- !ruby/object:Gem::Dependency
|
210
224
|
name: bullet
|
211
225
|
requirement: !ruby/object:Gem::Requirement
|
@@ -406,6 +420,7 @@ files:
|
|
406
420
|
- lib/istox/helpers/blockchain_service.rb
|
407
421
|
- lib/istox/helpers/bunny_boot.rb
|
408
422
|
- lib/istox/helpers/common_helper.rb
|
423
|
+
- lib/istox/helpers/dlm.rb
|
409
424
|
- lib/istox/helpers/f_math.rb
|
410
425
|
- lib/istox/helpers/graphql_client.rb
|
411
426
|
- lib/istox/helpers/grpc_client.rb
|
@@ -441,9 +456,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
441
456
|
version: '0'
|
442
457
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
443
458
|
requirements:
|
444
|
-
- - "
|
459
|
+
- - ">"
|
445
460
|
- !ruby/object:Gem::Version
|
446
|
-
version:
|
461
|
+
version: 1.3.1
|
447
462
|
requirements: []
|
448
463
|
rubygems_version: 3.0.8
|
449
464
|
signing_key:
|