istox 0.1.129 → 0.1.130.pre.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/Gemfile.lock CHANGED
@@ -15,6 +15,7 @@ PATH
15
15
  redis (>= 2.0.0)
16
16
  redis-namespace (>= 1.0.0)
17
17
  redis-rails (~> 5.0.2)
18
+ redlock (~> 1.2.0)
18
19
  vault (~> 0.1)
19
20
 
20
21
  GEM
@@ -90,29 +91,29 @@ GEM
90
91
  i18n (~> 0.5)
91
92
  fakeredis (0.7.0)
92
93
  redis (>= 3.2, < 5.0)
93
- faraday (0.17.3)
94
+ faraday (1.0.0)
94
95
  multipart-post (>= 1.2, < 3)
95
- faraday_middleware (0.14.0)
96
- faraday (>= 0.7.4, < 1.0)
96
+ faraday_middleware (1.0.0.rc1)
97
+ faraday (~> 1.0)
97
98
  ffi (1.12.2)
98
99
  globalid (0.4.2)
99
100
  activesupport (>= 4.2.0)
100
- google-protobuf (3.11.4-universal-darwin)
101
+ google-protobuf (3.11.4)
101
102
  googleapis-common-protos-types (1.0.4)
102
103
  google-protobuf (~> 3.0)
103
104
  graphlient (0.3.7)
104
105
  faraday
105
106
  faraday_middleware
106
107
  graphql-client
107
- graphql (1.10.3)
108
+ graphql (1.10.5)
108
109
  graphql-client (0.16.0)
109
110
  activesupport (>= 3.0)
110
111
  graphql (~> 1.8)
111
- grpc (1.27.0-universal-darwin)
112
+ grpc (1.27.0)
112
113
  google-protobuf (~> 3.11)
113
114
  googleapis-common-protos-types (~> 1.0)
114
115
  grpc-tools (1.27.0)
115
- gruf (2.8.0)
116
+ gruf (2.8.1)
116
117
  activesupport (> 4)
117
118
  concurrent-ruby (> 1)
118
119
  grpc (~> 1.10)
@@ -146,9 +147,9 @@ GEM
146
147
  nio4r (2.3.1)
147
148
  nokogiri (1.10.9)
148
149
  mini_portile2 (~> 2.4.0)
149
- oj (3.10.3)
150
- ougai (1.8.2)
151
- oj (~> 3.4)
150
+ oj (3.10.5)
151
+ ougai (1.8.3)
152
+ oj (~> 3.10)
152
153
  paranoia (2.4.2)
153
154
  activerecord (>= 4.0, < 6.1)
154
155
  rack (2.2.2)
@@ -201,6 +202,8 @@ GEM
201
202
  redis-store (>= 1.2, < 2)
202
203
  redis-store (1.8.2)
203
204
  redis (>= 4, < 5)
205
+ redlock (1.2.0)
206
+ redis (>= 3.0.0, < 5.0)
204
207
  rspec (3.8.0)
205
208
  rspec-core (~> 3.8.0)
206
209
  rspec-expectations (~> 3.8.0)
data/istox.gemspec CHANGED
@@ -44,6 +44,7 @@ Gem::Specification.new do |spec|
44
44
  spec.add_dependency 'redis', '>= 2.0.0'
45
45
  spec.add_dependency 'redis-namespace', '>= 1.0.0'
46
46
  spec.add_dependency 'vault', '~> 0.1'
47
+ spec.add_dependency 'redlock', '~>1.2.0'
47
48
  spec.add_development_dependency 'bullet', '~> 5.7.5'
48
49
  spec.add_development_dependency 'bundler', '~> 1.16'
49
50
  spec.add_development_dependency 'database_cleaner', '~> 1.6.0'
@@ -52,7 +52,11 @@ module Istox
52
52
 
53
53
  def binding_routing_key(id)
54
54
  queue = data[:binding][id][:queue]
55
- data[:queues][queue][:queue_name]
55
+ if data[:queues][queue].nil?
56
+ queue
57
+ else
58
+ data[:queues][queue][:queue_name]
59
+ end
56
60
  rescue
57
61
  nil
58
62
  end
@@ -106,7 +110,35 @@ module Istox
106
110
  end
107
111
 
108
112
  def queue_name(consumer_key)
109
- queue_config_from_consumer_key!(consumer_key)['queue_name']
113
+ name = queue_config_from_consumer_key!(consumer_key)['queue_name']
114
+ return name if name.nil?
115
+
116
+ prefix = queue_config_from_consumer_key!(consumer_key)['queue_prefix']
117
+ suffix = queue_config_from_consumer_key!(consumer_key)['queue_suffix']
118
+ delimiter = queue_config_from_consumer_key!(consumer_key)['queue_delimiter'] || '' unless prefix.nil? && suffix.nil?
119
+
120
+ name = "#{prefix}#{delimiter}#{name}" unless prefix.nil?
121
+ name = "#{name}#{delimiter}#{suffix}" unless suffix.nil?
122
+ log.debug "name is #{name}"
123
+ name
124
+ rescue
125
+ nil
126
+ end
127
+
128
+ def queue_priority(consumer_key)
129
+ queue_config_from_consumer_key!(consumer_key)['priority']
130
+ rescue
131
+ nil
132
+ end
133
+
134
+ def queue_worker_param(consumer_key)
135
+ queue_config_from_consumer_key!(consumer_key)['worker_param']
136
+ rescue
137
+ nil
138
+ end
139
+
140
+ def queue_exclusive(consumer_key)
141
+ queue_config_from_consumer_key!(consumer_key)['exclusive'] || false
110
142
  rescue
111
143
  nil
112
144
  end
@@ -129,7 +161,7 @@ module Istox
129
161
  durable = queue_config_from_consumer_key!(consumer_key)['durable']
130
162
  if durable.nil?
131
163
  exchange_name = exchange_name consumer_key
132
- durable = exchange_durable? exchange_name
164
+ durable = exchange_durable? exchange_name unless exchange_name.nil?
133
165
  end
134
166
 
135
167
  durable
@@ -280,7 +312,8 @@ module Istox
280
312
  private
281
313
 
282
314
  def data
283
- @data ||= Hashie::Mash.new(
315
+ Hashie.logger.level = 'ERROR'
316
+ @data = Hashie::Mash.new(
284
317
  YAML.safe_load(
285
318
  ERB.new(File.read(ENV['AMQP_CONFIG'] || 'config/amqp.yml')).result
286
319
  )
@@ -0,0 +1,23 @@
1
+ require 'redlock'
2
+
3
+ module Istox
4
+ class DLMBoot
5
+ class << self
6
+ private
7
+
8
+ def lock(key, timeout = 3600, db)
9
+ loop do
10
+ break if ::Istox::RedisBoot.sets(key, 1, {nx: true, px: timeout}, db)
11
+ end
12
+
13
+ begin
14
+ yield
15
+ ensure
16
+ ::Istox::RedisBoot.del key, db
17
+ end
18
+ end
19
+
20
+ end
21
+ end
22
+ end
23
+
@@ -103,7 +103,7 @@ module Istox
103
103
  end
104
104
  end
105
105
  rescue => e
106
- log.debug "wait_for_confirm error happening: #{e}"
106
+ # log.debug "wait_for_confirm error happening: #{e}"
107
107
  end
108
108
 
109
109
  sleep 2
@@ -32,16 +32,12 @@ module Istox
32
32
  redis(db).rename old, new
33
33
  end
34
34
 
35
- def lock(key, timeout = 3600, db)
36
- loop do
37
- break if sets(key, 1, {nx: true, px: timeout}, db)
38
- end
39
-
40
- begin
41
- yield
42
- ensure
43
- del key, db
44
- end
35
+ def exists(key, db)
36
+ redis(db).exists key
37
+ end
38
+
39
+ def llen(key, db)
40
+ redis(db).llen key
45
41
  end
46
42
 
47
43
  private
@@ -9,26 +9,25 @@ module Istox
9
9
  # consumer_key must be defined in amqp.yml
10
10
  # if nothing pass in it will auto subscribe to all available consumers defined in amqp.yml queues key
11
11
  def start_subscribe(subscribing_consumer_keys = [])
12
- return if @flag
13
-
14
- @mutex = Mutex.new unless @mutex.present?
15
- @mutex.synchronize do
16
- return if @flag
17
-
18
- # Create connection
19
- @conn = ::Istox::BunnyBoot.connection
20
-
21
- subscribing_consumer_keys = ::Istox::BunnyBoot.queues_keys_for_subscribe if subscribing_consumer_keys.empty?
22
- subscribing_consumer_keys.each do |key|
23
- log.debug "Do subscribe key #{key}"
24
- if key.is_a?(String)
25
- do_subscribe key.to_s
26
- elsif key.is_a?(Hash)
27
- do_subscribe key[:id], key[:suffix], key[:multiple]
12
+ unless @flag
13
+ @mutex = Mutex.new unless @mutex.present?
14
+ @mutex.synchronize do
15
+ unless @flag
16
+ # Create connection
17
+ @conn = ::Istox::BunnyBoot.connection
18
+ @flag = true
28
19
  end
29
20
  end
21
+ end
30
22
 
31
- @flag = true
23
+ subscribing_consumer_keys = ::Istox::BunnyBoot.queues_keys_for_subscribe if subscribing_consumer_keys.empty?
24
+ subscribing_consumer_keys.each do |key|
25
+ log.debug "Do subscribe key #{key}"
26
+ if key.is_a?(String)
27
+ do_subscribe key.to_s
28
+ elsif key.is_a?(Hash)
29
+ do_subscribe key[:id], key[:multiple], key[:block]
30
+ end
32
31
  end
33
32
  end
34
33
 
@@ -38,7 +37,7 @@ module Istox
38
37
 
39
38
  private
40
39
 
41
- def do_subscribe(consumer_key, suffix = nil, multiple = nil)
40
+ def do_subscribe(consumer_key, multiple = nil, block = nil)
42
41
  manual_ack = ::Istox::BunnyBoot.queue_manual_ack? consumer_key
43
42
 
44
43
  # Create Channel with specified pool_size and prefetch
@@ -50,10 +49,10 @@ module Istox
50
49
 
51
50
  # Declare queue and bind to exchange
52
51
  queue_name = ::Istox::BunnyBoot.queue_name consumer_key
53
- queue_name = "#{queue_name}.#{suffix}" unless suffix.nil?
54
52
  queue_durable = ::Istox::BunnyBoot.queue_durable? consumer_key
53
+ queue_exclusive = ::Istox::BunnyBoot.queue_exclusive consumer_key
55
54
  begin
56
- queue = active_channel.queue(queue_name, durable: queue_durable)
55
+ queue = active_channel.queue(queue_name, durable: queue_durable, exclusive: queue_exclusive || false)
57
56
  rescue Bunny::PreconditionFailed => e
58
57
  # Must re-open a new channel, because now channel is already closed
59
58
  active_channel = ::Istox::BunnyBoot.channel(::Istox::BunnyBoot.connection, pool_size: pool_size, prefetch: prefetch)
@@ -105,7 +104,7 @@ module Istox
105
104
  break unless ::Istox::BunnyBoot.queue_single_consumer? consumer_key
106
105
 
107
106
  flag = false
108
- ::Istox::RedisBoot.lock("peatio:#{consumer_key}:locking", 4) do
107
+ ::Istox::DLMBoot.lock("peatio:#{consumer_key}:locking", 5) do
109
108
  flag = ::Istox::BunnyBoot.queue_ok? @conn, queue.name
110
109
  end
111
110
  break if flag
@@ -115,17 +114,23 @@ module Istox
115
114
 
116
115
  # Initiate klass
117
116
  ruby_class = ::Istox::BunnyBoot.ruby_class consumer_key
118
- klass = Object.const_get(
119
- '::' + (ruby_class.nil? ? "#{consumer_key.to_s.underscore}_consumer" : ruby_class).camelize
120
- )
121
- if suffix.nil?
122
- worker = klass.new
123
- else
124
- worker = klass.new suffix
117
+ @workers = {} if @workers.nil?
118
+ unless ruby_class.nil? || @workers[ruby_class]
119
+ klass = Object.const_get(
120
+ '::' + (ruby_class.nil? ? "#{consumer_key.to_s.underscore}_consumer" : ruby_class).camelize
121
+ )
122
+ param = ::Istox::BunnyBoot.queue_worker_param consumer_key
123
+ if param.nil?
124
+ @workers[ruby_class] = klass.new
125
+ else
126
+ @workers[ruby_class] = klass.new param
127
+ end
125
128
  end
126
-
127
129
  # Subscribe queue
128
- queue.subscribe manual_ack: manual_ack do |delivery_info, metadata, payload|
130
+ priority = ::Istox::BunnyBoot.queue_priority consumer_key
131
+ arguments = {}
132
+ arguments['x-max-priority'] = priority unless priority.nil?
133
+ queue.subscribe manual_ack: manual_ack, arguments: arguments do |delivery_info, metadata, payload|
129
134
  log.debug "Delivery_info: #{delivery_info}"
130
135
  log.debug "Metadata: #{metadata}"
131
136
  log.debug "Payload: #{payload}"
@@ -141,7 +146,11 @@ module Istox
141
146
 
142
147
  log.info "Processing in consumer: #{klass}, paylod: #{payload_object.to_h.inspect}"
143
148
 
144
- result = worker.process(payload_object, metadata, delivery_info)
149
+ if @workers[ruby_class].nil?
150
+ result = block.call(payload_object, metadata, delivery_info) unless block.nil?
151
+ else
152
+ result = @workers[ruby_class].process(payload_object, metadata, delivery_info)
153
+ end
145
154
  end
146
155
  # active_channel.ack(delivery_info.delivery_tag) if manual_ack
147
156
  rescue StandardError => e
data/lib/istox/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Istox
2
- VERSION = '0.1.129'.freeze
2
+ VERSION = '0.1.130.pre.1'.freeze
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: istox
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.129
4
+ version: 0.1.130.pre.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Siong Leng
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-03-13 00:00:00.000000000 Z
11
+ date: 2020-03-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: awesome_print
@@ -206,6 +206,20 @@ dependencies:
206
206
  - - "~>"
207
207
  - !ruby/object:Gem::Version
208
208
  version: '0.1'
209
+ - !ruby/object:Gem::Dependency
210
+ name: redlock
211
+ requirement: !ruby/object:Gem::Requirement
212
+ requirements:
213
+ - - "~>"
214
+ - !ruby/object:Gem::Version
215
+ version: 1.2.0
216
+ type: :runtime
217
+ prerelease: false
218
+ version_requirements: !ruby/object:Gem::Requirement
219
+ requirements:
220
+ - - "~>"
221
+ - !ruby/object:Gem::Version
222
+ version: 1.2.0
209
223
  - !ruby/object:Gem::Dependency
210
224
  name: bullet
211
225
  requirement: !ruby/object:Gem::Requirement
@@ -406,6 +420,7 @@ files:
406
420
  - lib/istox/helpers/blockchain_service.rb
407
421
  - lib/istox/helpers/bunny_boot.rb
408
422
  - lib/istox/helpers/common_helper.rb
423
+ - lib/istox/helpers/dlm.rb
409
424
  - lib/istox/helpers/f_math.rb
410
425
  - lib/istox/helpers/graphql_client.rb
411
426
  - lib/istox/helpers/grpc_client.rb
@@ -441,9 +456,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
441
456
  version: '0'
442
457
  required_rubygems_version: !ruby/object:Gem::Requirement
443
458
  requirements:
444
- - - ">="
459
+ - - ">"
445
460
  - !ruby/object:Gem::Version
446
- version: '0'
461
+ version: 1.3.1
447
462
  requirements: []
448
463
  rubygems_version: 3.0.8
449
464
  signing_key: