istox 0.1.126 → 0.1.127

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    istox (0.1.125)
+    istox (0.1.126)
       awesome_print
       binding_of_caller
       bunny (>= 2.12.0)
@@ -97,7 +97,7 @@ GEM
     ffi (1.12.2)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    google-protobuf (3.11.4)
+    google-protobuf (3.11.4-universal-darwin)
     googleapis-common-protos-types (1.0.4)
       google-protobuf (~> 3.0)
     graphlient (0.3.7)
@@ -108,7 +108,7 @@ GEM
     graphql-client (0.16.0)
       activesupport (>= 3.0)
       graphql (~> 1.8)
-    grpc (1.27.0)
+    grpc (1.27.0-universal-darwin)
       google-protobuf (~> 3.11)
       googleapis-common-protos-types (~> 1.0)
     grpc-tools (1.27.0)

data/lib/istox/helpers/bunny_boot.rb

@@ -24,8 +24,8 @@ module Istox
       # Create virtual channel on established connection
       # Configure pool_size, prefetch, confirm mode according to opts
       def channel(conn, opts = {})
-        ch = conn.create_channel(nil, opts['pool_size'] || 1)
-        ch.prefetch(opts['prefetch']) unless opts['prefetch'].nil?
+        ch = conn.create_channel(nil, opts[:pool_size] || 1)
+        ch.prefetch(opts[:prefetch]) unless opts[:prefetch].nil?
 
         # Put channel in confirmation mode
         ch.confirm_select if opts[:confirm]
@@ -218,43 +218,37 @@ module Istox
         end
       end
 
-      def create_tracker(channel_id, delivery_tag, eid, message_id, options, payload)
-        log.debug "Create track: channel_id: #{channel_id}, delivery_tag: #{delivery_tag}, message_id: #{message_id}"
-        # combination of channel_id & delivery_tag can uniquely identify a msg
+      def create_tracker(channel_id, delivery_tag, eid, options, payload)
+        log.debug "Create track: channel_id: #{channel_id}, delivery_tag: #{delivery_tag}"
+        # combination of channel_id:delivery_tag can uniquely identify a msg
+        # For each retry of msg, channel_id and delivery_tag is unchanged
+        # But each retry, there is new delivery_tag that should be updated
         id = "#{channel_id.to_s}:#{delivery_tag.to_s}"
 
-        ::Istox::RedisBoot.sets("#{id}:messageid", message_id)
-        ::Istox::RedisBoot.sets("#{id}:retry", 0)
         ::Istox::RedisBoot.sets("#{id}:payload", payload.to_s)
         ::Istox::RedisBoot.sets("#{id}:eid", eid.to_s)
         ::Istox::RedisBoot.sets("#{id}:options", options.to_s)
       end
 
-      def find_tracker_on_channel(channel_id, delivery_tag)
-        id = "#{channel_id.to_s}:#{delivery_tag.to_s}"
-        keys = ::Istox::RedisBoot.keys("#{id}:p")
-        keys.each do |k|
-          # payload =
-        end
+      def find_tracker_on_channel(channel_id, delivery_tag, key)
+        pattern = "#{channel_id.to_s}:#{delivery_tag.to_s}:#{key}"
+        keys = find_trackers pattern
+        get_tracker(keys.first)
       end
 
       def del_tracker_on_channel(channel_id)
-        trackers = find_trackers_from_channelId channel_id
+        trackers = find_trackers "#{channel_id}*"
         trackers.each do |k|
           ::Istox::RedisBoot.del k
         end
       end
 
-      def find_trackers(key)
-        a = ::Istox::RedisBoot.keys("#{key}")
-      end
-
-      def update_tracker_by_status(key, status)
-        ::Istox::RedisBoot.set(key, status)
-      end
-
-      def update_tracker_incr_retry(key)
-        ::Istox::RedisBoot.incr(key)
+      def rename_tracker(channel_id, old_delivery_tag, new_delivery_tag)
+        old_id = "#{channel_id}:#{old_delivery_tag}"
+        new_id = "#{channel_id}:#{new_delivery_tag}"
+        ::Istox::RedisBoot.rename("#{old_id}:payload", "#{new_id}:payload")
+        ::Istox::RedisBoot.rename("#{old_id}:eid", "#{new_id}:eid")
+        ::Istox::RedisBoot.rename("#{old_id}:options", "#{new_id}:options")
       end
 
       def eid(ex)
@@ -300,8 +294,12 @@ module Istox
         queue_config_from_consumer_key!(consumer_key)['channel']
       end
 
-      def find_trackers_from_channelId(channel_id)
-        ::Istox::RedisBoot.keys("#{channel_id}*")
+      def find_trackers(pattern)
+        ::Istox::RedisBoot.keys(pattern)
+      end
+
+      def get_tracker(key)
+        ::Istox::RedisBoot.get(key)
       end
     end
   end

data/lib/istox/helpers/publisher.rb

@@ -31,10 +31,6 @@ module Istox
         do_publish ex, options, message
       end
 
-      def re_publish
-        do_publish ex, options, message
-      end
-
       private
 
       # Create physical connection for publisher
@@ -78,16 +74,29 @@ module Istox
         # Start new thread on receiving ACK from 'confirm-1' channel on the specified thread
         Thread.new do
           loop do
-            success = @channel[threadId]['confirm-1'].wait_for_confirms
-            if success
-              # ::Istox::BunnyBoot.del_tracker_on_channel @channel[threadId]['confirm-1'].id
-            else
-              @channel[threadId]['confirm-1'].nacked_set.each do |n|
-                log.debug "[Async #{threadId}] Confirm mode UnAcked Delivery Tag: #{n}"
-                # ::Istox::RedisBoot.incr("confirm:#{n.to_s}")
-                # message = ::Istox::BunnyBoot.find_tracker_on_channel @channel[threadId]['confirm-1'].id, delivery_tag
-                # publish(exchange: ()::Istox::BunnyBoot.eid ex), routing_key: routing_key, message: message)
+            begin
+              success = @channel[threadId]['confirm-1'].wait_for_confirms
+              if success
+                ::Istox::BunnyBoot.del_tracker_on_channel @channel[threadId]['confirm-1'].id
+              else
+                @channel[threadId]['confirm-1'].nacked_set.each do |n|
+                  log.debug "[Async #{threadId}] Confirm mode UnAcked Delivery Tag: #{n}"
+                  eid = ::Istox::BunnyBoot.find_tracker_on_channel @channel[threadId]['confirm-1'].id, n, 'eid'
+                  payload = ::Istox::BunnyBoot.find_tracker_on_channel @channel[threadId]['confirm-1'].id, n, 'payload'
+                  payload = JSON.dump payload unless payload.nil?
+                  options = ::Istox::BunnyBoot.find_tracker_on_channel @channel[threadId]['confirm-1'].id, n, 'options'
+                  options = JSON.dump options unless options.nil?
+                  if options.nil?
+                    routing_key = nil
+                  else
+                    routing_key = options.delete :routing_key
+                    options[:delivery_tag] = n
+                  end
+                  publish(exchange: eid, routing_key: routing_key, message: payload, options: options)
+                end
               end
+            rescue => e
+              log.debug "wait_for_confirm error happening: #{e}"
             end
 
             sleep 2
@@ -156,9 +165,9 @@ module Istox
         # when an unroutable message is returned, the BasicReturn is fired first and then an ack is sent, firing the BasicAck second.
         # So if the current status is unroutable then we need to make sure that we don't overwrite that status with Success (ack).
         ex.on_return do |return_info, properties, content|
-          log.debug "Got a returned message info: #{return_info}"
-          log.debug "Got a returned message properties: #{properties}"
-          log.debug "Got a returned message content: #{content}"
+          # log.debug "Got a returned message info: #{return_info}"
+          # log.debug "Got a returned message properties: #{properties}"
+          # log.debug "Got a returned message content: #{content}"
 
           # ::Istox::BunnyBoot.find_trackers("message_id:#{properties[:message_id]}").each do |key|
           #  log.debug("On_return, update key #{key} in redis on status and retry counter")
@@ -187,31 +196,38 @@ module Istox
         confirm = channel_confirm? ch
         mode = confirm_mode eid
 
+        ::Istox::BunnyBoot.publish(ex, message, options)
         if confirm
           delivery_tag = channel_next_tag ch
           if options[:message_id].nil?
             options[:message_id] = SecureRandom.hex
-            # ::Istox::BunnyBoot.create_tracker channel_id, delivery_tag, eid, options[:message_id], options, message
+            ::Istox::BunnyBoot.create_tracker channel_id, delivery_tag, eid, options, message
+          else
+            log.debug "message_id already existed for re-publish msg #{options[:message_id]}"
+            ::Istox::BunnyBoot.rename_tracker channel_id, options[:delivery_tag], delivery_tag
           end
         end
 
-        ::Istox::BunnyBoot.publish(ex, message, options)
         if confirm && mode == 0
           success = ex.channel.wait_for_confirms
           if success
             # ::Istox::BunnyBoot.del_tracker_on_channel channel_id
             0
           else
+            ret = []
             ex.channel.nacked_set.each do |n|
               log.debug("UnAck delivery tag is #{n}")
-                #::Istox::RedisBoot.incr("confirm:#{n.to_s}")
-                # publish(exchange: eid, routing_key: routing_key, message: message, )
+              options = ::Istox::BunnyBoot.find_tracker_on_channel(channel_id, n, 'options')
+              options = JSON.dump options
+              ret << {delivery_tag: n, message_id: options[:message_id]}
             end
-            -1
+            ret
           end
         end
       rescue Bunny::ConnectionClosedError => e
         log.debug "Publish fails due to #{e}"
+        sleep 1
+        do_publish(ex,options,message)
       rescue => e
         log.debug "Timeout error happens #{e}"
       end

data/lib/istox/helpers/redis.rb

@@ -16,6 +16,10 @@ module Istox
         redis.del(k)
       end
 
+      def get(k)
+        redis.get(k)
+      end
+
       def flushdb
         redis.flushdb
       end
@@ -24,6 +28,10 @@ module Istox
         redis.incr(k)
       end
 
+      def rename(old, new)
+        redis.rename old, new
+      end
+
       def lock(key, timeout = 3600)
         loop do
           break if set(key, 1, nx: true, px: timeout)

data/lib/istox/helpers/subscriber.rb

@@ -15,6 +15,9 @@ module Istox
         @mutex.synchronize do
           return if @flag
 
+          # Create connection
+          @conn = ::Istox::BunnyBoot.connection
+
           subscribing_consumer_keys = ::Istox::BunnyBoot.queues_keys_for_subscribe if subscribing_consumer_keys.empty?
           subscribing_consumer_keys.each do |key|
             log.debug "Do subscribe key #{key}"
@@ -41,8 +44,7 @@ module Istox
         # Create Channel with specified pool_size and prefetch
         pool_size = ::Istox::BunnyBoot.channel_pool_size consumer_key
         prefetch = ::Istox::BunnyBoot.channel_prefetch consumer_key
-        conn = ::Istox::BunnyBoot.connection
-        active_channel = ::Istox::BunnyBoot.channel(conn, pool_size: pool_size, prefetch: prefetch)
+        active_channel = ::Istox::BunnyBoot.channel(@conn, pool_size: pool_size, prefetch: prefetch)
         @chs = [] if @chs.nil?
         @chs << active_channel
 
@@ -104,7 +106,7 @@ module Istox
 
           flag = false
           ::Istox::RedisBoot.lock("peatio:#{consumer_key}:locking") do
-            flag = ::Istox::BunnyBoot.queue_ok? conn, queue.name
+            flag = ::Istox::BunnyBoot.queue_ok? @conn, queue.name
           end
           break if flag
 

data/lib/istox/version.rb

@@ -1,3 +1,3 @@
 module Istox
-  VERSION = '0.1.126'.freeze
+  VERSION = '0.1.127'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: istox
 version: !ruby/object:Gem::Version
-  version: 0.1.126
+  version: 0.1.127
 platform: ruby
 authors:
 - Siong Leng
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-02 00:00:00.000000000 Z
+date: 2020-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -445,7 +445,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: istox backend shared gem