istox 0.1.153.4 → 0.1.154

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 65872c83f71e4c64acb4e856541411c2cf06afbb147fb500fe86d3ae647669d9
-  data.tar.gz: 3b6c32750ad37e9efc5412be0ee61186267f906e4e850aa5de4a39d53423f975
+  metadata.gz: 5f186bda33beb170d462308144ab748911b21da9432231a7fa4aa1b26ef38a9b
+  data.tar.gz: 584667bc13e90aba012b2433636949422336441ca2e65a6c54e3875bd6b2ba9e
 SHA512:
-  metadata.gz: 7b4169651c930a4fec4d685b4109388e791e8bf283c57c87754618b80a00ca650f4e128a21117b1a96afb6cdf54f0b232b68d1a8c5fcd71c1bfbec0a446c39c4
-  data.tar.gz: 803b1349d0ed3335dcc9994aeaeb01518c5e6a49f6523cdc904bed6083c87fdf02b69d40d0a48c8b6ed8ce3f9fece35f03ce39556ce0a169f4ab7613b6a56c34
+  metadata.gz: 425e9027421752a58f89fc771c1f76f482c706cf0604eb549173f324f440ecd6a3207602304bbb11fd1783c133d63612cd71e973dad8656a550fa1161c172efc
+  data.tar.gz: e8848fef567649c057a5ca95303992d88c5c975726bd5bc4fecc30564835a6b0b0b5119356eb68faf49fc0f6dba27cf2f9fc7f1c17d699e1be34c54656cb5ced

data/Gemfile.lock

@@ -6,6 +6,7 @@ PATH
       awesome_print
       binding_of_caller
       bunny (>= 2.12.0)
+      exponential-backoff (= 0.0.4)
       graphlient
       gruf
       hashie (~> 3.5.7)
@@ -89,6 +90,7 @@ GEM
       uniform_notifier (~> 1.11.0)
     bunny (2.15.0)
       amq-protocol (~> 2.3, >= 2.3.1)
+    byebug (11.1.3)
     coderay (1.1.2)
     concurrent-ruby (1.1.6)
     crass (1.0.6)
@@ -96,6 +98,7 @@ GEM
     debug_inspector (0.0.3)
     diff-lcs (1.3)
     erubi (1.9.0)
+    exponential-backoff (0.0.4)
     factory_bot (4.8.2)
       activesupport (>= 3.0.0)
     factory_bot_rails (4.8.2)
@@ -112,7 +115,7 @@ GEM
     ffi (1.12.2)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    google-protobuf (3.12.1-universal-darwin)
+    google-protobuf (3.12.1)
     googleapis-common-protos-types (1.0.5)
       google-protobuf (~> 3.11)
     graphlient (0.4.0)
@@ -123,7 +126,7 @@ GEM
     graphql-client (0.16.0)
       activesupport (>= 3.0)
       graphql (~> 1.8)
-    grpc (1.28.0-universal-darwin)
+    grpc (1.28.0)
       google-protobuf (~> 3.11)
       googleapis-common-protos-types (~> 1.0)
     grpc-tools (1.28.0)
@@ -171,6 +174,9 @@ GEM
     pry (0.12.2)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
+    pry-byebug (3.8.0)
+      byebug (~> 11.0)
+      pry (~> 0.10)
     pry-rails (0.3.9)
       pry (>= 0.10.4)
     rack (2.2.2)
@@ -278,6 +284,7 @@ DEPENDENCIES
   fakeredis
   istox!
   pry
+  pry-byebug
   pry-rails
   rails (~> 5.2.1)
   rake (~> 10.0)

data/bulk-update-version.py

@@ -0,0 +1,93 @@
+import sys
+import re
+from tempfile import mkstemp
+from shutil import move, copymode
+from os import fdopen, remove
+import os
+
+# HOW TO RUN: python bulk-update-version.py <version updating to, eg. 0.1.150.2>
+
+SERVICES = ["client-api", "admin-api", "message-api", "auth", "admin-auth"]
+
+
+def replace(file_path, pattern, subst):
+    # Create temp file
+    fh, abs_path = mkstemp()
+    with fdopen(fh, 'w') as new_file:
+        dirname = os.path.dirname(__file__)
+        file_path = os.path.join(dirname, file_path)
+        if os.path.exists(file_path) == False:
+            return
+        with open(file_path) as old_file:
+            for line in old_file:
+                new_file.write(re.sub(pattern, subst, line))
+    # Copy the file permissions from the old file to the new file
+    copymode(file_path, abs_path)
+    # Remove original file
+    remove(file_path)
+    # Move new file
+    move(abs_path, file_path)
+
+
+def push_service(file_path, version, hotfix):
+    dirname = os.path.dirname(__file__)
+    file_path = os.path.join(dirname, file_path)
+
+    if os.path.exists(file_path) == False:
+        return
+
+    if hotfix == True:
+        branch_name = "hotfix"
+    else:
+        branch_name = "FIX/update-istox-gem-" + version
+
+    os.chdir(file_path)
+    os.system(
+        "git branch -D %s &>/dev/null" % (branch_name))
+    os.system("echo 'Deleted local branch'")
+    os.system(
+        "git push origin --delete %s &>/dev/null" % (branch_name))
+    os.system("echo 'Deleted remote branch'")
+    os.system("git fetch -a")
+    os.system("git checkout -b %s" % (branch_name))
+    os.system("git add Gemfile")
+    os.system("git commit -m 'Update istox-gem version to %s'" % (version))
+    os.system("git push --set-upstream origin %s" % (branch_name))
+#     os.system("cd " + file_path + " && git branch -D " +
+#               branch_name + " &>/dev/null || echo 'Deleted local branch' || git push origin --delete hotfix &>/dev/null || echo 'Deleted remote branch' || git fetch -a || git checkout -b " +
+#               branch_name + " && git add Gemfile && git commit -m 'Update istox-gem version to "
+#               + version + "' && git push --set-upstream origin " + branch_name)
+
+
+arguments = sys.argv
+arguments.pop(0)
+
+new_version = arguments[0]
+
+hotfix = False
+
+while True:
+    result = raw_input("Is this a hotix? Y/N \n")
+
+    if result.lower() == 'y':
+        hotfix = True
+        break
+    elif result.lower() == 'n':
+        hotfix = False
+        break
+    else:
+        print("Please input only Y or N.")
+
+
+print("Updating services " + str(SERVICES) +
+      " to istox-gem version " + new_version)
+
+for service in SERVICES:
+    replace("../" + service + "/Gemfile", r"gem 'istox'.+",
+            "gem 'istox', '" + new_version + "'")
+
+for service in SERVICES:
+    push_service("../" + service, new_version, hotfix)
+
+print("Services " + str(SERVICES) +
+      " has been updated to istox-gem version " + new_version)

data/istox.gemspec

@@ -34,6 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'awesome_print'
   spec.add_dependency 'binding_of_caller'
   spec.add_dependency 'bunny', '>= 2.12.0'
+  spec.add_dependency 'exponential-backoff', '0.0.4'
   spec.add_dependency 'graphlient'
   spec.add_dependency 'gruf'
   spec.add_dependency 'hashie', '~> 3.5.7'
@@ -53,6 +54,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'faker', '~> 1.7.3'
   spec.add_development_dependency 'fakeredis'
   spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'pry-byebug'
   spec.add_development_dependency 'pry-rails'
   spec.add_development_dependency 'rails', '~> 5.2.1'
   spec.add_development_dependency 'rake', '~> 10.0'

data/lib/istox/helpers/bunny_boot.rb

@@ -215,7 +215,7 @@ module Istox
         retry_limit = queue_config_from_consumer_key!(consumer_key)['retry_limit']
 
         if retry_limit.nil?
-          3
+          5
         else
           retry_limit.to_i
         end
@@ -234,6 +234,10 @@ module Istox
         end
       end
 
+      def queue_retry_gap_set?(consumer_key)
+        queue_config_from_consumer_key!(consumer_key)['retry_gap'].present?
+      end
+
       def channel_pool_size(consumer_key)
         channel = channel_config(consumer_key)
         return data['channel_pool_size'] || 1 if channel.nil?

data/lib/istox/helpers/logger.rb

@@ -6,7 +6,7 @@ module Istox
       return @logger if @logger.present?
 
       @logger = ::Ougai::Logger.new(STDOUT)
-      @logger.formatter = ::Ougai::Formatters::Readable.new unless Rails.env.production?
+      @logger.formatter = ::Ougai::Formatters::Readable.new unless !defined?(Rails) || Rails.env.production?
 
       @logger
     end

data/lib/istox/helpers/publisher.rb

@@ -78,6 +78,9 @@ module Istox
                     @channel[t]['confirm-1'].close
                     @channel[t]['noconfirm'].close
                     @channel.delete t
+
+                    # Remove exchange from @exchange
+                    @exchanges.delete t
                   end
                 end
               end.join
@@ -233,7 +236,16 @@ module Istox
         sleep 1
         do_publish(ex,options,message)
       rescue => e
-        log.debug "Error happens: #{e}"
+        log.debug "Error happens: #{e.message}"
+
+        # If the error indicates that the channel is already closed
+        # then clear hash @channel and @exchange
+        if e.message.include? "cannot use a closed channel"
+          @channel.delete Thread.current.object_id
+          @exchanges.delete Thread.current.object_id
+          ex = exchange(eid)
+          do_publish(ex, options, message)
+        end
       end
     end
   end

data/lib/istox/helpers/remote_model_cache.rb

@@ -16,9 +16,9 @@ module Istox
         log.debug "Writing data to cache, key: #{key}, fields: #{fields.inspect}"
         log.debug "Cache data: #{data.inspect}"
 
-        Thread.new do
-          data = data.to_json unless data.instance_of?(String)
+        data = data.to_json unless data.instance_of?(String)
 
+        Thread.new do
           results = fields.map do |f|
             redis.hset(key, f.to_s, data)
           end

data/lib/istox/helpers/subscriber.rb

@@ -1,3 +1,4 @@
+require 'exponential_backoff'
 require 'istox/helpers/logger'
 
 module Istox
@@ -79,26 +80,6 @@ module Istox
           queue.bind exchange, queue_options
         end
 
-        if manual_ack
-          letter_exchange = if exchange.nil?
-                              active_channel.default_exchange.name
-                            else
-                              exchange.name
-                            end
-          retry_queue = active_channel.queue("#{queue_name}.retry", arguments: {
-                                               'x-dead-letter-exchange': letter_exchange,
-                                               'x-dead-letter-routing-key': queue_name.to_s,
-                                               'x-message-ttl': (::Istox::BunnyBoot.queue_retry_gap consumer_key)
-                                             })
-          if exchange_name.nil?
-            exchange_retry = active_channel.default_exchange
-          else
-            exchange_retry_name = "#{exchange_name}.retry"
-            exchange_retry = active_channel.send exchange_type, exchange_retry_name, durable: exchange_durable
-            retry_queue.bind exchange_retry, routing_key: "#{queue_name}.retry" if manual_ack
-          end
-        end
-
         loop do
           break unless ::Istox::BunnyBoot.queue_single_consumer? consumer_key
 
@@ -115,9 +96,7 @@ module Istox
         ruby_class = ::Istox::BunnyBoot.ruby_class(consumer_key) || "#{consumer_key.to_s.underscore}_consumer"
         @workers = {} if @workers.nil?
         unless block || @workers[ruby_class]
-          klass = Object.const_get(
-            '::' + ruby_class.camelize
-          )
+          klass = consumer_class(ruby_class)
           param = ::Istox::BunnyBoot.queue_worker_param consumer_key
           @workers[ruby_class] = if param.nil?
                                    klass.new
@@ -184,15 +163,13 @@ module Istox
               if ::Istox::BunnyBoot.queue_worker_param_format(consumer_key) == 'open_struct'
                 processing_payload = ::Istox::CommonHelper.to_open_struct(processing_payload)
               end
-              ::Istox::BunnyBoot.publish(exchange_retry, processing_payload, routing_key: "#{queue_name}.retry", priority: retry_count, type: 'dlx')
+
+              retry_obj = create_retry_obj(consumer_key, active_channel, exchange, exchange_name, queue_name, retry_count - 1)
+
+              ::Istox::BunnyBoot.publish(retry_obj.exchange_retry, processing_payload,
+                                         routing_key: retry_obj.routing_key, priority: retry_count, type: 'dlx')
             end
           end
-        #           # For redelivered message, call 'reject' not 'nack' in order to reschedule message to tail not head of queue
-        #           if delivery_info.redelivered
-        #             active_channel.reject(delivery_info.delivery_tag, true)
-        #           else
-        #             active_channel.nack(delivery_info.delivery_tag, false, true)
-        #           end
         ensure
           if manual_ack
             multiple = if !multiple.nil? && !result && result == multiple
@@ -205,6 +182,44 @@ module Istox
         end
       end
 
+      def create_retry_obj(consumer_key, active_channel, _exchange, exchange_name, queue_name, retried_count)
+        minimal_interval = 1.0
+        maximal_elapsed_time = 60.0
+
+        # retry interval will be [1, 3, 9, 27, 60, 60....] seconds
+        backoff = ExponentialBackoff.new(minimal_interval, maximal_elapsed_time)
+        backoff.multiplier = 3
+        ttl = ::Istox::BunnyBoot.queue_retry_gap_set?(consumer_key) ? ::Istox::BunnyBoot.queue_retry_gap(consumer_key) : backoff.interval_at(retried_count) * 1000
+        ttl = ttl.to_i
+
+        routing_key = "#{queue_name}.retry.#{ttl}"
+
+        retry_queue = active_channel.queue(routing_key, arguments: {
+                                             'x-dead-letter-exchange': active_channel.default_exchange.name,
+                                             'x-dead-letter-routing-key': queue_name.to_s,
+                                             'x-message-ttl': ttl.to_i
+                                           })
+        if exchange_name.nil?
+          exchange_retry = active_channel.default_exchange
+        else
+          exchange_durable = ::Istox::BunnyBoot.exchange_durable? exchange_name
+          exchange_retry_name = "#{exchange_name}.retry"
+          exchange_retry = active_channel.direct(exchange_retry_name, durable: exchange_durable)
+          retry_queue.bind exchange_retry, routing_key: routing_key
+        end
+
+        OpenStruct.new(
+          exchange_retry: exchange_retry,
+          routing_key: routing_key
+        )
+      end
+
+      def consumer_class(ruby_class)
+        Object.const_get(
+          '::' + ruby_class.camelize
+        )
+      end
+
       def intercept(interceptors, payload, metadata, delivery_info)
         return yield if interceptors.none?
 

data/lib/istox/quant/bond.rb

@@ -1,5 +1,4 @@
 require 'date'
-require 'rails'
 
 # https://en.wikipedia.org/wiki/Day_count_convention#30/360_Bond_Basis
 # For 360 basis: 30/360 Bond Basis
@@ -248,4 +247,4 @@ module Istox
 
     end
   end
-end
+end

data/lib/istox/version.rb

@@ -1,3 +1,3 @@
 module Istox
-  VERSION = '0.1.153.4'.freeze
+  VERSION = '0.1.154'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: istox
 version: !ruby/object:Gem::Version
-  version: 0.1.153.4
+  version: 0.1.154
 platform: ruby
 authors:
 - Siong Leng
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-28 00:00:00.000000000 Z
+date: 2020-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazing_print
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.12.0
+- !ruby/object:Gem::Dependency
+  name: exponential-backoff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.4
 - !ruby/object:Gem::Dependency
   name: graphlient
   requirement: !ruby/object:Gem::Requirement
@@ -332,6 +346,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry-rails
   requirement: !ruby/object:Gem::Requirement
@@ -455,6 +483,7 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- bulk-update-version.py
 - istox.gemspec
 - lib/istox.rb
 - lib/istox/constants/error.rb