istox 0.1.129 → 0.1.130.pre.1

data/Gemfile.lock

@@ -15,6 +15,7 @@ PATH
       redis (>= 2.0.0)
       redis-namespace (>= 1.0.0)
       redis-rails (~> 5.0.2)
+      redlock (~> 1.2.0)
       vault (~> 0.1)
 
 GEM
@@ -90,29 +91,29 @@ GEM
       i18n (~> 0.5)
     fakeredis (0.7.0)
       redis (>= 3.2, < 5.0)
-    faraday (0.17.3)
+    faraday (1.0.0)
       multipart-post (>= 1.2, < 3)
-    faraday_middleware (0.14.0)
-      faraday (>= 0.7.4, < 1.0)
+    faraday_middleware (1.0.0.rc1)
+      faraday (~> 1.0)
     ffi (1.12.2)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    google-protobuf (3.11.4-universal-darwin)
+    google-protobuf (3.11.4)
     googleapis-common-protos-types (1.0.4)
       google-protobuf (~> 3.0)
     graphlient (0.3.7)
       faraday
       faraday_middleware
       graphql-client
-    graphql (1.10.3)
+    graphql (1.10.5)
     graphql-client (0.16.0)
       activesupport (>= 3.0)
       graphql (~> 1.8)
-    grpc (1.27.0-universal-darwin)
+    grpc (1.27.0)
       google-protobuf (~> 3.11)
       googleapis-common-protos-types (~> 1.0)
     grpc-tools (1.27.0)
-    gruf (2.8.0)
+    gruf (2.8.1)
       activesupport (> 4)
       concurrent-ruby (> 1)
       grpc (~> 1.10)
@@ -146,9 +147,9 @@ GEM
     nio4r (2.3.1)
     nokogiri (1.10.9)
       mini_portile2 (~> 2.4.0)
-    oj (3.10.3)
-    ougai (1.8.2)
-      oj (~> 3.4)
+    oj (3.10.5)
+    ougai (1.8.3)
+      oj (~> 3.10)
     paranoia (2.4.2)
       activerecord (>= 4.0, < 6.1)
     rack (2.2.2)
@@ -201,6 +202,8 @@ GEM
       redis-store (>= 1.2, < 2)
     redis-store (1.8.2)
       redis (>= 4, < 5)
+    redlock (1.2.0)
+      redis (>= 3.0.0, < 5.0)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)

data/istox.gemspec

@@ -44,6 +44,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'redis', '>= 2.0.0'
   spec.add_dependency 'redis-namespace', '>= 1.0.0'
   spec.add_dependency 'vault', '~> 0.1'
+  spec.add_dependency 'redlock', '~>1.2.0'
   spec.add_development_dependency 'bullet', '~> 5.7.5'
   spec.add_development_dependency 'bundler', '~> 1.16'
   spec.add_development_dependency 'database_cleaner', '~> 1.6.0'

data/lib/istox/helpers/bunny_boot.rb

@@ -52,7 +52,11 @@ module Istox
 
       def binding_routing_key(id)
         queue = data[:binding][id][:queue]
-        data[:queues][queue][:queue_name]
+        if data[:queues][queue].nil?
+          queue
+        else
+          data[:queues][queue][:queue_name]
+        end
       rescue
         nil
       end
@@ -106,7 +110,35 @@ module Istox
       end
 
       def queue_name(consumer_key)
-        queue_config_from_consumer_key!(consumer_key)['queue_name']
+        name = queue_config_from_consumer_key!(consumer_key)['queue_name']
+        return name if name.nil?
+
+        prefix = queue_config_from_consumer_key!(consumer_key)['queue_prefix']
+        suffix = queue_config_from_consumer_key!(consumer_key)['queue_suffix']
+        delimiter = queue_config_from_consumer_key!(consumer_key)['queue_delimiter'] || '' unless prefix.nil? && suffix.nil?
+
+        name = "#{prefix}#{delimiter}#{name}" unless prefix.nil?
+        name = "#{name}#{delimiter}#{suffix}" unless suffix.nil?
+        log.debug "name is #{name}"
+        name
+      rescue
+        nil
+      end
+
+      def queue_priority(consumer_key)
+        queue_config_from_consumer_key!(consumer_key)['priority']
+      rescue
+        nil
+      end
+
+      def queue_worker_param(consumer_key)
+        queue_config_from_consumer_key!(consumer_key)['worker_param']
+      rescue
+        nil
+      end
+
+      def queue_exclusive(consumer_key)
+        queue_config_from_consumer_key!(consumer_key)['exclusive'] || false
       rescue
         nil
       end
@@ -129,7 +161,7 @@ module Istox
         durable = queue_config_from_consumer_key!(consumer_key)['durable']
         if durable.nil?
           exchange_name = exchange_name consumer_key
-          durable = exchange_durable? exchange_name
+          durable = exchange_durable? exchange_name unless exchange_name.nil?
         end
 
         durable
@@ -280,7 +312,8 @@ module Istox
       private
 
       def data
-        @data ||= Hashie::Mash.new(
+        Hashie.logger.level = 'ERROR'
+        @data = Hashie::Mash.new(
             YAML.safe_load(
                 ERB.new(File.read(ENV['AMQP_CONFIG'] || 'config/amqp.yml')).result
             )

data/lib/istox/helpers/dlm.rb

@@ -0,0 +1,23 @@
+require 'redlock'
+
+module Istox
+  class DLMBoot
+    class << self
+      private
+
+      def lock(key, timeout = 3600, db)
+        loop do
+          break if ::Istox::RedisBoot.sets(key, 1, {nx: true, px: timeout}, db)
+        end
+
+        begin
+          yield
+        ensure
+          ::Istox::RedisBoot.del key, db
+        end
+      end
+
+    end
+  end
+end
+

data/lib/istox/helpers/publisher.rb

@@ -103,7 +103,7 @@ module Istox
                 end
               end
             rescue => e
-              log.debug "wait_for_confirm error happening: #{e}"
+              # log.debug "wait_for_confirm error happening: #{e}"
             end
 
             sleep 2

data/lib/istox/helpers/redis.rb

@@ -32,16 +32,12 @@ module Istox
         redis(db).rename old, new
       end
 
-      def lock(key, timeout = 3600, db)
-        loop do
-          break if sets(key, 1, {nx: true, px: timeout}, db)
-        end
-
-        begin
-          yield
-        ensure
-          del key, db
-        end
+      def exists(key, db)
+        redis(db).exists key
+      end
+
+      def llen(key, db)
+        redis(db).llen key
       end
 
       private

data/lib/istox/helpers/subscriber.rb

@@ -9,26 +9,25 @@ module Istox
       # consumer_key must be defined in amqp.yml
       # if nothing pass in it will auto subscribe to all available consumers defined in amqp.yml queues key
       def start_subscribe(subscribing_consumer_keys = [])
-        return if @flag
-
-        @mutex = Mutex.new unless @mutex.present?
-        @mutex.synchronize do
-          return if @flag
-
-          # Create connection
-          @conn = ::Istox::BunnyBoot.connection
-
-          subscribing_consumer_keys = ::Istox::BunnyBoot.queues_keys_for_subscribe if subscribing_consumer_keys.empty?
-          subscribing_consumer_keys.each do |key|
-            log.debug "Do subscribe key #{key}"
-            if key.is_a?(String)
-              do_subscribe key.to_s
-            elsif key.is_a?(Hash)
-              do_subscribe key[:id], key[:suffix], key[:multiple]
+        unless @flag
+          @mutex = Mutex.new unless @mutex.present?
+          @mutex.synchronize do
+            unless @flag
+              # Create connection
+              @conn = ::Istox::BunnyBoot.connection
+              @flag = true
             end
           end
+        end
 
-          @flag = true
+        subscribing_consumer_keys = ::Istox::BunnyBoot.queues_keys_for_subscribe if subscribing_consumer_keys.empty?
+        subscribing_consumer_keys.each do |key|
+          log.debug "Do subscribe key #{key}"
+          if key.is_a?(String)
+            do_subscribe key.to_s
+          elsif key.is_a?(Hash)
+            do_subscribe key[:id], key[:multiple], key[:block]
+          end
         end
       end
 
@@ -38,7 +37,7 @@ module Istox
 
       private
 
-      def do_subscribe(consumer_key, suffix = nil, multiple = nil)
+      def do_subscribe(consumer_key, multiple = nil, block = nil)
         manual_ack = ::Istox::BunnyBoot.queue_manual_ack? consumer_key
 
         # Create Channel with specified pool_size and prefetch
@@ -50,10 +49,10 @@ module Istox
 
         # Declare queue and bind to exchange
         queue_name = ::Istox::BunnyBoot.queue_name consumer_key
-        queue_name = "#{queue_name}.#{suffix}" unless suffix.nil?
         queue_durable = ::Istox::BunnyBoot.queue_durable? consumer_key
+        queue_exclusive = ::Istox::BunnyBoot.queue_exclusive consumer_key
         begin
-          queue = active_channel.queue(queue_name, durable: queue_durable)
+          queue = active_channel.queue(queue_name, durable: queue_durable, exclusive: queue_exclusive || false)
         rescue Bunny::PreconditionFailed => e
           # Must re-open a new channel, because now channel is already closed
           active_channel = ::Istox::BunnyBoot.channel(::Istox::BunnyBoot.connection, pool_size: pool_size, prefetch: prefetch)
@@ -105,7 +104,7 @@ module Istox
           break unless ::Istox::BunnyBoot.queue_single_consumer? consumer_key
 
           flag = false
-          ::Istox::RedisBoot.lock("peatio:#{consumer_key}:locking", 4) do
+          ::Istox::DLMBoot.lock("peatio:#{consumer_key}:locking", 5) do
             flag = ::Istox::BunnyBoot.queue_ok? @conn, queue.name
           end
           break if flag
@@ -115,17 +114,23 @@ module Istox
 
         # Initiate klass
         ruby_class = ::Istox::BunnyBoot.ruby_class consumer_key
-        klass = Object.const_get(
-            '::' + (ruby_class.nil? ? "#{consumer_key.to_s.underscore}_consumer" : ruby_class).camelize
-        )
-        if suffix.nil?
-          worker = klass.new
-        else
-          worker = klass.new suffix
+        @workers = {} if @workers.nil?
+        unless ruby_class.nil? || @workers[ruby_class]
+          klass = Object.const_get(
+              '::' + (ruby_class.nil? ? "#{consumer_key.to_s.underscore}_consumer" : ruby_class).camelize
+          )
+          param = ::Istox::BunnyBoot.queue_worker_param consumer_key
+          if param.nil?
+            @workers[ruby_class] = klass.new
+          else
+            @workers[ruby_class] = klass.new param
+          end
         end
-
         # Subscribe queue
-        queue.subscribe manual_ack: manual_ack do |delivery_info, metadata, payload|
+        priority = ::Istox::BunnyBoot.queue_priority consumer_key
+        arguments = {}
+        arguments['x-max-priority'] = priority unless priority.nil?
+        queue.subscribe manual_ack: manual_ack, arguments: arguments do |delivery_info, metadata, payload|
           log.debug "Delivery_info: #{delivery_info}"
           log.debug "Metadata: #{metadata}"
           log.debug "Payload: #{payload}"
@@ -141,7 +146,11 @@ module Istox
 
             log.info "Processing in consumer: #{klass}, paylod: #{payload_object.to_h.inspect}"
 
-            result = worker.process(payload_object, metadata, delivery_info)
+            if @workers[ruby_class].nil?
+              result = block.call(payload_object, metadata, delivery_info) unless block.nil?
+            else
+              result = @workers[ruby_class].process(payload_object, metadata, delivery_info)
+            end
           end
           # active_channel.ack(delivery_info.delivery_tag) if manual_ack
         rescue StandardError => e

data/lib/istox/version.rb

@@ -1,3 +1,3 @@
 module Istox
-  VERSION = '0.1.129'.freeze
+  VERSION = '0.1.130.pre.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: istox
 version: !ruby/object:Gem::Version
-  version: 0.1.129
+  version: 0.1.130.pre.1
 platform: ruby
 authors:
 - Siong Leng
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-13 00:00:00.000000000 Z
+date: 2020-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -206,6 +206,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: redlock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: bullet
   requirement: !ruby/object:Gem::Requirement
@@ -406,6 +420,7 @@ files:
 - lib/istox/helpers/blockchain_service.rb
 - lib/istox/helpers/bunny_boot.rb
 - lib/istox/helpers/common_helper.rb
+- lib/istox/helpers/dlm.rb
 - lib/istox/helpers/f_math.rb
 - lib/istox/helpers/graphql_client.rb
 - lib/istox/helpers/grpc_client.rb
@@ -441,9 +456,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.8
 signing_key: