istox 0.1.120 → 0.1.121

data/Gemfile.lock

@@ -1,18 +1,20 @@
 PATH
   remote: .
   specs:
-    istox (0.1.115)
+    istox (0.1.121)
       awesome_print
       binding_of_caller
       bunny (>= 2.12.0)
       graphlient
       gruf
+      hashie (~> 3.5.7)
       istox_gruf
       listen (~> 3.0.5)
       ougai
       paranoia (~> 2.2)
       redis (>= 2.0.0)
       redis-namespace (>= 1.0.0)
+      redis-rails (~> 5.0.2)
       vault (~> 0.1)
 
 GEM
@@ -90,23 +92,23 @@ GEM
       redis (>= 3.2, < 5.0)
     faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
-    faraday_middleware (0.13.1)
+    faraday_middleware (0.14.0)
       faraday (>= 0.7.4, < 1.0)
-    ffi (1.11.3)
+    ffi (1.12.2)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    google-protobuf (3.11.2)
+    google-protobuf (3.11.3-universal-darwin)
     googleapis-common-protos-types (1.0.4)
       google-protobuf (~> 3.0)
     graphlient (0.3.7)
       faraday
       faraday_middleware
       graphql-client
-    graphql (1.9.17)
+    graphql (1.10.2)
     graphql-client (0.16.0)
       activesupport (>= 3.0)
       graphql (~> 1.8)
-    grpc (1.26.0)
+    grpc (1.26.0-universal-darwin)
       google-protobuf (~> 3.8)
       googleapis-common-protos-types (~> 1.0)
     grpc-tools (1.26.0)
@@ -116,6 +118,7 @@ GEM
       grpc (~> 1.10)
       grpc-tools (~> 1.10)
       slop (~> 4.6)
+    hashie (3.5.7)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
     istox_gruf (2.7.1)
@@ -143,7 +146,7 @@ GEM
     nio4r (2.3.1)
     nokogiri (1.10.1)
       mini_portile2 (~> 2.4.0)
-    oj (3.10.0)
+    oj (3.10.2)
     ougai (1.8.2)
       oj (~> 3.4)
     paranoia (2.4.2)
@@ -180,8 +183,24 @@ GEM
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     redis (4.1.3)
+    redis-actionpack (5.1.0)
+      actionpack (>= 4.0, < 7)
+      redis-rack (>= 1, < 3)
+      redis-store (>= 1.1.0, < 2)
+    redis-activesupport (5.2.0)
+      activesupport (>= 3, < 7)
+      redis-store (>= 1.3, < 2)
     redis-namespace (1.7.0)
       redis (>= 3.0.4)
+    redis-rack (2.0.6)
+      rack (>= 1.5, < 3)
+      redis-store (>= 1.2, < 2)
+    redis-rails (5.0.2)
+      redis-actionpack (>= 5.0, < 6)
+      redis-activesupport (>= 5.0, < 6)
+      redis-store (>= 1.2, < 2)
+    redis-store (1.8.1)
+      redis (>= 4, < 5)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -203,7 +222,7 @@ GEM
       rspec-mocks (~> 3.8.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    slop (4.7.0)
+    slop (4.8.0)
     sprockets (3.7.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)

data/istox.gemspec

@@ -36,6 +36,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'graphlient'
   spec.add_dependency 'gruf'
   spec.add_dependency 'istox_gruf'
+  spec.add_dependency 'hashie', '~> 3.5.7'
+  spec.add_dependency 'redis-rails', '~> 5.0.2'
   spec.add_dependency 'listen', '~> 3.0.5'
   spec.add_dependency 'ougai'
   spec.add_dependency 'paranoia', '~> 2.2'

data/lib/istox.rb

@@ -16,6 +16,7 @@ module Istox
   require 'istox/helpers/logger'
   require 'istox/interfaces/chainhub/transaction'
   require 'istox/helpers/publisher'
+  require 'istox/helpers/subscriber'
   require 'istox/helpers/bunny_boot'
   require 'istox/helpers/rate_limit'
   require 'istox/helpers/vault'
@@ -31,6 +32,7 @@ module Istox
   require 'istox/helpers/common_helper'
   require 'istox/helpers/regex_helper'
   require 'istox/helpers/result_handler'
+  require 'istox/helpers/redis'
   require 'istox/models/blockchain_receipt'
   require 'istox/models/concerns/blockchain_receipt_query'
   require 'istox/consumers/blockchain_status_handler'

data/lib/istox/helpers/bunny_boot.rb

@@ -1,128 +1,210 @@
 require 'bunny'
+require 'hashie/mash'
 require 'istox/helpers/logger'
 
 module Istox
   class BunnyBoot
     class << self
-      # path to amqp.yml
-      def initialize!(amqp_config_path)
-        @amqp_config_path = amqp_config_path
+      # Create physical connection to RabbitMQ
+      # During failover of RabbitMQ cluster or temporary failure, there may be error and needs retry in loop
+      def connection
+        conn = nil
+        loop do
+          conn = Bunny.new(data['connect'].symbolize_keys).tap(&:start)
+          break
+        rescue Bunny::TCPConnectionFailed, Bunny::TCPConnectionFailedForAllHosts => e
+          log.error "Fails to create connection to RabbitMQ with err: #{e}"
+          log.info 'Reconnect after 2s ...'
+          sleep 2
+        end
 
-        data
+        conn
       end
 
-      # optionally can pass in consumer_key for single subscription / consumer_keys for multiple subcriptions
-      # consumer_key must be defined in amqp.yml
-      # if nothing pass in it will auto subscribe to all available consumers defined in amqp.yml queues key
-      def start_subscribe(consumer_key: nil, consumer_keys: nil)
-        subscribing_consumer_keys = consumer_keys.present? ? consumer_keys : []
+      # Create virtual channel on established connection
+      # Configure pool_size, prefetch, confirm mode according to opts
+      def channel(conn, opts = {})
+        ch = conn.create_channel(nil, opts['pool_size'] || 1)
+        ch.prefetch(opts['prefetch']) unless opts['prefetch'].nil?
 
-        if subscribing_consumer_keys.empty? && consumer_key.nil?
-          subscribing_consumer_keys = data['queues'].keys
-        elsif subscribing_consumer_keys.empty? && consumer_key.present?
-          subscribing_consumer_keys = [consumer_key]
-        end
+        # Put channel in confirmation mode
+        ch.confirm_select if opts[:confirm]
+        log.debug "channel unconfirmed_set is #{ch.unconfirmed_set}"
 
-        subscribing_consumer_keys.each do |key|
-          subscribe_to_consumer(key.to_s)
-        end
+        ch
+      end
+
+      def exchange(eid)
+        type = data[:exchanges][eid][:type]
+        name = eid
+        settings = { durable: data[:exchanges][eid][:durable] || false }
+        confirm = data[:exchanges][eid][:confirm] || false
+        [type, name, settings, confirm]
+      rescue
+        nil
       end
 
-      #############################################################################################
-      ## INTERNAL METHODS START
-      ############################################################################################
+      def queues_keys_for_subscribe()
+        data['queues'].keys
+      end
 
-      def channel
-        return @channel if @channel.present?
+      def subscriber_queue_from_key(key)
+        data['queues'][key]
+      end
+
+      def search_exchange_of_routing_key!(routing_key)
+        data['publish'].each do |k, v|
+          v.each do |k2, _v2|
+            return k if k2 == routing_key
+          end
+        end
 
-        @channel = connection.create_channel(nil, data['channel_pool_size'] || 1)
-        @channel.prefetch(data['channel_prefetch'] || 1)
+        raise "Cannot find exchange of routing key #{routing_key}, have you forgotten to define it in exchange section in amqp.yml"
+      end
 
-        @channel
+      # Default for exchange durable attr: true
+      def exchange_durable?(exchange_name)
+        exchange_config!(exchange_name)['durable'] || true
+      rescue => e
+        raise e
       end
 
-      # calling this method will spawn another channel, please use the normal channel method if you like to use the same channel
-      def create_new_channel(channel_config)
-        channel = connection.create_channel(nil, channel_config['channel_pool_size'] || data['channel_pool_size'] || 1)
-        channel.prefetch(channel_config['channel_prefetch'] || data['channel_prefetch'] || 1)
+      # Default for channel confirm attr: false
+      def exchange_confirm?(exchange_name)
+        exchange_config!(exchange_name)['confirm'] || false
+      end
 
-        channel
+      def exchange_type(exchange_name)
+        exchange_config!(exchange_name)['type']
       end
 
-      def subscribe_to_consumer(consumer_key)
-        queue_config = data['queues'][consumer_key]
+      def exchange_name(consumer_key)
+        queue_config_from_consumer_key!(consumer_key)['exchange']
+      rescue
+        nil
+      end
 
-        raise "Unable to find consumer with key #{consumer_key}, have you forgotten to define it in amqp.yml?" if queue_config.nil?
+      def queue_name(consumer_key)
+        queue_config_from_consumer_key!(consumer_key)['queue_name']
+      rescue
+        nil
+      end
 
-        klass = Object.const_get(
-          '::' + (queue_config['ruby_class'].nil? ? "#{consumer_key.to_s.underscore}_consumer" : queue_config['ruby_class']).camelize
-        )
+      def ruby_class(consumer_key)
+        queue_config_from_consumer_key!(consumer_key)['ruby_class']
+      rescue
+        nil
+      end
 
-        exchange = nil
-        exchange_config = exchange_config!(consumer_key)
+      # Default value: belonged exchange durable attr
+      def queue_durable?(consumer_key)
+        durable = queue_config_from_consumer_key!(consumer_key)['durable']
+        if durable.nil?
+          exchange_name = exchange_name consumer_key
+          durable = exchange_durable? exchange_name
+        end
 
-        active_channel = queue_config['channel'].present? ? create_new_channel(queue_config['channel']) : channel
+        durable
+      rescue
+        nil
+      end
 
-        queue_options = {}
+      def queue_manual_ack?(consumer_key)
+        manual_ack = queue_config_from_consumer_key!(consumer_key)['manual_ack']
+        return true if manual_ack.nil?
 
-        exchange_durable = exchange_config['durable'].nil? ? true : exchange_config['durable']
+        manual_ack
+      end
 
-        case exchange_config['type']
-        when 'fanout'
-          exchange = active_channel.fanout(queue_config['exchange'], durable: exchange_durable)
-        when 'direct'
-          exchange = active_channel.direct(queue_config['exchange'], durable: exchange_durable)
-          queue_options[:routing_key] = queue_config['queue_name']
+      # By default retry-limit is 3
+      # If -1, means no limit
+      def queue_retry_limit(consumer_key)
+        retry_limit = queue_config_from_consumer_key!(consumer_key)['retry_limit']
+
+        if retry_limit.nil?
+          3
         else
-          raise "Exchange type #{exchange_config.type} is not valid/supported."
+          retry_limit.to_i
         end
+      end
+
+      # By default retry-limit is 3000
+      # If 0, means no gap and retry immediately
+      # To protect system, we can't allow unlimited retry with zero-gap
+      def queue_retry_gap(consumer_key)
+        retry_gap = queue_config_from_consumer_key!(consumer_key)['retry_gap']
 
-        queue = active_channel.queue(queue_config['queue_name'],
-                                     durable: queue_config['durable'].nil? ? exchange_durable : queue_config['durable']).bind(exchange, queue_options)
+        if retry_gap.nil? || (retry_gap == 0 && queue_retry_limit(consumer_key) == -1)
+          3000
+        else
+          retry_gap.to_i
+        end
+      end
 
-        manual_ack = queue_config['manual_ack'].nil? ? true : queue_config['manual_ack']
+      def channel_pool_size(consumer_key)
+        channel = channel_config(consumer_key)
+        return data['channel_pool_size'] || 1 if channel.nil?
 
-        queue.subscribe manual_ack: manual_ack do |delivery_info, metadata, payload|
-          log.info "Received message in consumer: #{klass}, raw payload: #{payload.inspect}"
+        channel['channel_pool_size']
+      end
 
-          processing_paylod = JSON.parse(payload)
-          payload_object = ::Istox::CommonHelper.to_open_struct(processing_paylod)
+      def channel_prefetch(consumer_key)
+        channel = channel_config(consumer_key)
+        return data['channel_prefetch'] || 1 if channel.nil?
 
-          log.info "Processing in consumer: #{klass}, paylod: #{payload_object.to_h.inspect}"
+        channel['channel_prefetch']
+      end
 
-          klass.new.process(payload_object, metadata, delivery_info)
-          active_channel.ack(delivery_info.delivery_tag) if manual_ack
-        rescue StandardError => e
-          log.error(e)
-          active_channel.nack(delivery_info.delivery_tag, false, true) if manual_ack
+      def publish(e, message, options = {})
+        persistent = false
+        mandatory = false
+        # Set Mandatory & Persistent flag for non-DLX and non-manual msg
+        unless ['dlx', 'manual'].include? options[:type]
+          persistent = data['publish'][e.name][options[:routing_key]]['persistent'] || false if options[:routing_key].present?
+          mandatory = data['publish'][e.name][options[:routing_key]]['mandatory'] || false
         end
-      end
+        options.merge!(persistent: persistent) if options[:routing_key].present?
+        options.merge!(mandatory: mandatory)
 
-      def data
-        raise 'Unable to find amqp configuration file, have you forgotten to initialise Istox::BunnyBoot?' if @amqp_config_path.empty?
+        # message.merge!(locale: I18n.locale)
+        message = JSON.dump message
 
-        @data ||= JSON.parse(YAML.safe_load(ERB.new(File.read(@amqp_config_path)).result).to_json)
+        log.debug "Publish options are: #{options}"
+        log.debug "Publish message payload #{message}"
+        e.publish(message, options)
+
+        options[:message_id]
       end
 
-      def connection
-        return @connection if @connection.present?
+      private
 
-        raise 'RabbitMQ connection configuration not found, have you forgotten to define connect key in amqp.yml?' if data['connect'].nil?
+      def data
+        @data ||= Hashie::Mash.new(
+            YAML.safe_load(
+                ERB.new(File.read(ENV['AMQP_CONFIG'] || 'config/amqp.yml')).result
+            )
+        )
+      end
 
-        @connection = Bunny.new(data['connect'].symbolize_keys)
-        @connection.start
+      def queue_config_from_consumer_key!(consumer_key)
+        queue_config = data['queues'][consumer_key]
+
+        raise "Queue for key #{consumer_key} config not found, have you forgotten to define the queue in amqp.yml?" if queue_config.nil?
 
-        @connection
+        queue_config
       end
 
-      def exchange_config!(consumer_key)
-        exchange_name = data['queues'][consumer_key]['exchange']
+      def exchange_config!(exchange_name)
         exchange_config = data['exchanges'][exchange_name]
 
         raise "Exchange #{exchange_name} config not found, have you forgotten to define the exchange in amqp.yml?" if exchange_config.nil?
 
         exchange_config
       end
+
+      def channel_config(consumer_key)
+        queue_config_from_consumer_key!(consumer_key)['channel']
+      end
     end
   end
 end

data/lib/istox/helpers/publisher.rb

@@ -1,3 +1,4 @@
+require 'securerandom'
 require 'istox/helpers/logger'
 
 module Istox
@@ -11,105 +12,195 @@ module Istox
       # extra publish options: {
       #   publish_confirm: true/false
       # }
-      def publish(exchange: nil, routing_key: nil, message:, options: {})
+      def publish(exchange: nil, routing_key: nil, message:)
         raise 'Exchange and routing key cannot be nil at the same time.' if exchange.nil? && routing_key.nil?
 
-        queue_publish_config = nil
-        if routing_key.present?
-          exchange = search_exchange_of_routing_key!(routing_key) if exchange.nil?
-          queue_publish_config = queue_publish_config!(routing_key: routing_key, exchange: exchange)
-        end
-
-        exchange_config = exchange_config!(exchange)
-        exchange_durable = exchange_config['durable'].nil? ? true : exchange_config['durable']
-
-        queue_durable = queue_publish_config.present? ? queue_publish_config['durable'] : nil
-        queue_durable = exchange_config['durable'].nil? ? true : exchange_config['durable'] if queue_durable.nil?
-
-        options[:durable] = queue_durable
-        options[:persistent] = options[:persistent].nil? ? true : options[:persistent]
+        exchange = ::Istox::BunnyBoot.search_exchange_of_routing_key!(routing_key) if routing_key.present? && exchange.nil?
+        ex = exchange(exchange)
 
-        send_message(exchange: exchange, routing_key: routing_key, message: message,
-                     exchange_type: exchange_config['type'], exchange_durable: exchange_durable, options: options)
+        do_publish ex, routing_key, message
       end
 
       # publish without defining exchange and queue in amqp.yml
       def manual_publish(exchange:, routing_key: nil, message:, exchange_type: 'direct', exchange_durable: true, options: {})
-        options[:durable] = options[:durable].nil? ? true : options[:durable]
-        options[:persistent] = options[:persistent].nil? ? true : options[:persistent]
+        ex = exchange(exchange, type: exchange_type, durable: exchange_durable, options: options)
+        # TODO: Implement mandatory & persistent flag for manual publish
+        do_publish ex, routing_key, message
+      end
+
+      def re_publish
 
-        send_message(exchange: exchange, routing_key: routing_key, message: message,
-                     exchange_type: exchange_type, exchange_durable: exchange_durable, options: options)
       end
 
       private
 
-      def send_message(exchange:, routing_key: nil, message:, exchange_type:, exchange_durable:, options: {})
-        log.info "Publishing RABBITMQ message to exchange: #{exchange}, routing key: #{routing_key}, data: #{message}, options: #{options.inspect}"
-
-        case exchange_type
-        when 'direct'
-          raise 'Routing key cannot be empty for exchange type direct' if routing_key.blank?
+      # Create physical connection for publisher
+      # Connection creation must be thread-safe
+      def connection
+        return @connection if @connection.present?
+
+        log.info 'No connection yet, create connection to RabbitMQ node ... ...'
+        @mutex = Mutex.new unless @mutex.present?
+        @mutex.synchronize do
+          if @connection.present?
+            log.info 'Connection already existed, cancel creation'
+            return @connection
+          end
 
-          options[:routing_key] = routing_key
-          channel.direct(exchange, durable: exchange_durable).publish(message.to_json, options)
-        when 'fanout'
-          channel.fanout(exchange, durable: exchange_durable).publish(message.to_json, options)
-        else
-          raise "Exchange type #{exchange_config.type} is not valid/supported."
+          @connection = ::Istox::BunnyBoot.connection
         end
+      end
 
-        # publish_confirm = options[:publish_confirm].nil? ? true : options[:publish_confirm]
+      def close_connection
+        connection.close
+        @connection = nil
+        @channel = nil
+        @exchanges = nil
+      end
 
-        # if publish_confirm
-        #   log.info 'Waiting for RABBITMQ publisher acknowledgement'
+      # For channels in publisher connection, it should be two-channel-per-thread
+      # One channel in confirm mode while other in non-confirm mode
+      # Return all channels for current thread
+      def channel
+        return @channel[Thread.current.object_id] if @channel.present? && @channel[Thread.current.object_id].present?
 
-        #   success = channel.wait_for_confirms
-        #   raise 'RABBITMQ publish acknowlegement failed' unless success
-        # end
+        log.info 'No channel yet, create 2 channels confirm-mode and non-confirm-mode ... ...'
+        @channel = Hash.new if @channel.nil?
+        @channel[Thread.current.object_id] = Hash.new
+        @channel[Thread.current.object_id]['confirm'] = ::Istox::BunnyBoot.channel(connection, confirm: true)
+        @channel[Thread.current.object_id]['noconfirm'] = ::Istox::BunnyBoot.channel(connection, confirm: false)
 
-        log.info 'Publish RABBITMQ message success'
+        @channel[Thread.current.object_id]
       end
 
-      def data
-        ::Istox::BunnyBoot.data
+      # This is version that multiple threads sharing same channel
+      # It is wrong usage, but only used for *TEST* !!!
+      def channel_fake
+        return @channel_fake if @channel_fake.present?
+
+        log.info 'No channel yet, create Channel to Existing connection ... ...'
+        @mutex_fake = Mutex.new unless @mutex_fake.present?
+        @mutex_fake.synchronize do
+          if @channel_fake.present?
+            log.info 'Channel already existed, cancel creation'
+            return @channel_fake
+          end
+
+          @channel_fake = ::Istox::BunnyBoot.channel(connection, confirm: true)
+        end
       end
 
-      def channel
-        return @channel if @channel.present?
+      # return confirm-mode chanel for current thread
+      def channel_confirm_mode
+        channel['confirm']
+        # channel_fake
+      end
 
-        @channel = ::Istox::BunnyBoot.channel
+      # return no-confirm-mode chanel for current thread
+      def channel_noconfirm_mode
+        channel['noconfirm']
+        # channel_fake
+      end
+
+      def exchanges
+        return @exchanges[Thread.current.object_id] if @exchanges.present? && @exchanges[Thread.current.object_id].present?
+
+        @exchanges = Hash.new if @exchanges.nil?
+        @exchanges[Thread.current.object_id] = { default: channel['confirm'].default_exchange }
+
+        @exchanges[Thread.current.object_id]
+        # @exchanges ||= { default: channel['confirm'].default_exchange }
+        # @exchanges ||= { default: channel_fake.default_exchange }
+      end
+
+      def exchange(id, type: nil, durable: nil, options: nil)
+        return exchanges[id] if exchanges[id].present?
+
+        log.info "Declare exchange #{id} on channel"
+        e = ::Istox::BunnyBoot.exchange(id)
+        e = [type, id,  { durable: durable || true }, options['publish_confirm'] || true] if e.nil?
+        c = e.pop ? channel_confirm_mode : channel_noconfirm_mode
+        ex = c.send *e
+
+        # For mandatory flag, listen to returned message
+        # [2020-01-13T00:34:36.111+08:00] DEBUG: Got a returned message info: {:reply_code=>312, :reply_text=>"NO_ROUTE", :exchange=>"sample.exchange.direct.durable.confirm", :routing_key=>"sample_routing_key"}
+        # [2020-01-13T00:34:36.111+08:00] DEBUG: Got a returned message properties: {:content_type=>"application/octet-stream", :delivery_mode=>2, :priority=>0, :message_id=>"68c74e57e2874d24d48bdf3d891a059c"}
+        # [2020-01-13T00:34:36.111+08:00] DEBUG: Got a returned message content: "{\"sample_key\":\"abc\"}"
+        # when an unroutable message is returned, the BasicReturn is fired first and then an ack is sent, firing the BasicAck second.
+        # So if the current status is unroutable then we need to make sure that we don't overwrite that status with Success (ack).
+        ex.on_return do |return_info, properties, content|
+          log.debug "Got a returned message info: #{return_info}"
+          log.debug "Got a returned message properties: #{properties}"
+          log.debug "Got a returned message content: #{content}"
+
+          find_trackers("message_id:#{properties[:message_id]}").each do |key|
+            log.debug("On_return, update key #{key} in redis on status and retry counter")
+            update_tracker_by_status(key, 2)
+            update_tracker_incr_retry(key)
+          end
 
-        # @channel.confirm_select
+          # publish(return_info[:exchange], return_info[:routing_key], content)
+        end
 
-        @channel
+        exchanges[id] = ex
       end
 
-      def queue_publish_config!(routing_key:, exchange:)
-        queue_publish_config = data['publish'][exchange][routing_key]
+      def channel_confirm?(ch)
+        ch.using_publisher_confirmations?
+      end
 
-        queue_publish_config
+      def channel_next_tag(ch)
+        ch.next_publish_seq_no
       end
 
-      def search_exchange_of_routing_key!(routing_key)
-        data['publish'].each do |k, v|
-          v.each do |k2, _v2|
-            return k if k2 == routing_key
+      def do_publish(ex, routing_key, message)
+        ch = ex.channel
+        if channel_confirm? ch
+          delivery_tag = channel_next_tag ch
+          message_id = SecureRandom.hex
+          # update_message_tracker(message_id, delivery_tag, 0, message)
+        end
+
+        ::Istox::BunnyBoot.publish(ex, message, routing_key: routing_key, message_id: message_id, type: 'manual')
+
+        if ex.channel.using_publisher_confirmations?
+          log.debug 'Confirm mode channel, wait for confirmation'
+          success = ex.channel.wait_for_confirms
+          if success
+            log.debug "Confirm mode result: #{success}"
+            ::Istox::RedisBoot.flushdb
+          else
+            ex.channel.nacked_set.each do |n|
+              log.debug "Confirm mode UnAcked Delivery Tag: #{n}"
+              ::Istox::RedisBoot.incr("confirm:#{n.to_s}")
+              publish(exchange: ex.name, routing_key: routing_key, message: message, )
+            end
           end
         end
+      rescue Bunny::ConnectionClosedError => e
+        log.debug "Publish fails due to #{e}"
+      rescue => e
+        log.debug "Timeout error happens #{e}"
+      end
 
-        raise "Cannot find exchange of routing key #{routing_key}, have you forgotten to define it in exchange section in amqp.yml"
+      def create_tracker(message_id, delivery_tag, status, payload)
+        log.debug "Create track: message_id #{message_id}, delivery_tag #{delivery_tag}, status #{status}"
+
+        ::Istox::RedisBoot.set("status:delivery_tag:#{delivery_tag.to_s}:message_id:#{message_id}", status)
+        ::Istox::RedisBoot.set("payload:delivery_tag:#{delivery_tag.to_s}:message_id:#{message_id}", payload.to_s)
+        ::Istox::RedisBoot.set("retry:delivery_tag:#{delivery_tag.to_s}:message_id:#{message_id}", 0)
       end
 
-      def exchange_config!(exchange_name)
-        exchange_config = data['exchanges'][exchange_name]
+      def find_trackers(key)
+        ::Istox::RedisBoot.keys("status:*#{key}*")
+      end
 
-        if exchange_config.nil?
-          raise "Exchange config of #{exchange_name} cannot be found, have you
-          forgotten to define it in exchange section in amqp.yml"
-        end
+      def update_tracker_by_status(key, status)
+        ::Istox::RedisBoot.set(key, status)
+      end
 
-        exchange_config
+      def update_tracker_incr_retry(key)
+        ::Istox::RedisBoot.incr(key)
       end
     end
   end