cotcube-helpers 0.2.1.1 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1228015a21f3920ec5b1d87e3c5918e3c69e0a577746b6a18579ae51e1528ce2
-  data.tar.gz: 431798647c5d4e9d5a7a32816636bd768c4e0e9c70d789dd491c374fde75f768
+  metadata.gz: 1d08ed5fa831283b0934d2ce7f86f9da0f1be5430d0d3164ee503b0aa7bac74f
+  data.tar.gz: ac59b5558432299e6906c8a93b4ddab8b7a57a7cac4b2c11239a8f41a8c05f4e
 SHA512:
-  metadata.gz: b931c2dad6f54a1f2de64590e703c1be03ed2330691a299c2e4d4a64f5cb5d20d3be9ba44584866db66116160c2d01c87cd17f7d84f5ae82ae2668f5a8cf0bae
-  data.tar.gz: 18b50ba7237d373dec19d3144814ad55c5ec3d36274213c5749041bbf66ce008461d2be89cda0240245529e29ed383110352d945af4a82cb761d53a6d702f09a
+  metadata.gz: 8b3ce7871cbc2068523994caa155a2291512c0cce8c72398f51de275e56c6e1a75a94402327118f27ae0376ccbf70b148be3fb17d0f66326e019d21a639ae485
+  data.tar.gz: e17fe32e700a3441e5e54a4b521c0c3d20acedbfb45606cd8f3da6edb5041a7cbcab522729da6d9d0e784f739730972af1ad1210b5542e06cd879f70587d7b1e

data/CHANGELOG.md

@@ -1,6 +1,36 @@
+## 0.2.3 (December 30, 2021)
+  - merging conflict
+  - added bare josch_ and order_client s
+  - added bare js_ and order_client s
+
+## 0.2.2.5 (December 23, 2021)
+  - hash_ext: reworked keys_to_sym!
+  - minor changes
+  - deep_decode_datetime: added helper for conversion of timestrings in nested arrays and hashed
+  - cache_client: rewritten as class
+  - gemspec: raised activesupport to version 7
+
+## 0.2.2.4 (December 07, 2021)
+  - introduced cache_client as client to readcache
+
+## 0.2.2.3 (November 28, 2021)
+  - data_client: fixing troublesome output in .command
+
+## 0.2.2.2 (November 28, 2021)
+  - dataclient fixing = for false ==
+  - solving merge conflicts
+  - Bump version to 0.2.2.
+
+## 0.2.2.1 (November 28, 2021)
+  - Bump version to 0.2.2.1
+
+## 0.2.2 (November 13, 2021)
+  - some further improvements to DataClient
+  - some fixes related to ib_contracts
+  - some fixes related to DataClient
+
 ## 0.2.1.1 (November 10, 2021)
   - Bump version to 0.2.1.
-  - Bump version to 0.2.1.
 
 ## 0.2.1 (November 10, 2021)
   - added new class 'dataclient' for communication with dataproxy

data/Gemfile

@@ -6,3 +6,4 @@ source 'https://rubygems.org'
 gemspec
 gem 'parallel'
 gem 'bunny'
+gem 'httparty'

data/VERSION

@@ -1 +1 @@
-0.2.1.1
+0.2.3

data/cotcube-helpers.gemspec

@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'activesupport', '~> 6'
+  spec.add_dependency 'activesupport', '~> 7'
   spec.add_dependency 'colorize',      '~> 0.8'
 
 

data/lib/cotcube-helpers/cache_client.rb

@@ -0,0 +1,52 @@
+require 'httparty'
+require 'json'
+
+module Cotcube
+  module Helpers
+    class CacheClient
+
+      def initialize(query='keys', timezone: Cotcube::Helpers::CHICAGO, debug: false, deflate: false, update: false)
+        raise ArgumentError, "Query must not be empty." if [ nil, '' ].include? query
+        raise ArgumentError, "Query '#{query}' is garbage." if query.split('/').size > 2 or not query.match? /\A[a-zA-Z0-9?=\/]+\Z/
+        @update = update ? '?update=true' : ''
+        @request_headers = {} 
+        @request_headers['Accept-Encoding'] = 'deflate' if deflate
+        @query = query
+        @result = JSON.parse(HTTParty.get("http://100.100.0.14:8081/#{query}#{@update}").body, headers: @request_headers, symbolize_names: true) rescue { error: 1, msg: "Could not parse response for query '#{query}'." }
+        retry_once if has_errors?
+      end
+
+      def retry_once
+        sleep 2
+        raw = HTTParty.get("http://100.100.0.14:8081/#{query}#{update}")
+        @result = JSON.parse(raw.body, symbolize_names: true) rescue { error: 1, msg: "Could not parse response for query '#{query}'." }
+        if has_errors?
+          puts "ERROR in parsing response: #{raw[..300]}"
+        end
+      end
+
+      def has_errors?
+        result[:error].nil? or result[:error] > 0
+      end
+
+      def warnings
+        result[:warnings]
+      end
+
+      def payload
+        has_errors? ? false : @result[:payload]
+      end
+
+      def entity
+        query.split('/').first
+      end
+
+      def asset
+        entity, asset = query.split('/')
+        asset
+      end
+
+      attr_reader :query, :result, :update
+    end
+  end
+end

data/lib/cotcube-helpers/constants.rb

@@ -30,6 +30,8 @@ module Cotcube
 
 
     CHICAGO  = Time.find_zone('America/Chicago')
+    NEW_YORK = Time.find_zone('America/New_York')
+    BERLIN   = Time.find_zone('Europe/Berlin')
 
     DATE_FMT = '%Y-%m-%d'
 

data/lib/cotcube-helpers/data_client.rb

@@ -1,160 +1,213 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
+
 require 'bunny'
 require 'json'
 
 module Cotcube
   module Helpers
     class DataClient
+      SECRETS_DEFAULT = {
+        'dataproxy_mq_proto' => 'http',
+        'dataproxy_mq_user' => 'guest',
+        'dataproxy_mq_password' => 'guest',
+        'dataproxy_mq_host' => 'localhost',
+        'dataproxy_mq_port' => '15672',
+        'dataproxy_mq_vhost' => '%2F'
+      }.freeze
+
+      SECRETS = SECRETS_DEFAULT.merge(
+        lambda {
+          begin
+            YAML.safe_load(File.read(Cotcube::Helpers.init[:secrets_file]))
+          rescue StandardError
+            {}
+          end
+        }.call
+      )
 
       def initialize
-        @connection = Bunny.new(automatically_recover: true)
+        @connection = Bunny.new(user: SECRETS['dataproxy_mq_user'],
+                                password: SECRETS['dataproxy_mq_password'],
+                                vhost: SECRETS['dataproxy_mq_vhost'])
         @connection.start
 
-        @channel  = connection.create_channel
-        @exchange = channel.direct('dataproxy_commands', auto_delete: true)
-        @requests = {} 
+        @commands  = connection.create_channel
+        @exchange  = commands.direct('dataproxy_commands')
+        @requests  = {}
         @persistent = { depth: {}, realtimebars: {}, ticks: {} }
-        @response = nil
-
+        @debug     = false
         setup_reply_queue
       end
 
-      def help
-        puts "The following commands are available:\n\n"\
-          "\tcontracts  = client.get_contracts(symbol:)\n"\
-          "\tbars       = client.get_historical(contract:, duration:, interval:, before: nil)"\
-          "\trequest_id = client.start_realtimebars(contract: )\n"\
-          "\t             client.stop_realtimebars(request_id: )\n"\
-          "\trequest_id = client.start_ticks(contract: )\n"\
-          "\t             client.stop_ticks(request_id: )\n"
-      end
-
-      def send_command(command, timeout: 5)
+      # command acts a synchronizer: it sends the command and waits for the response
+      #     otherwise times out --- the counterpart here is the subscription within 
+      #     setup_reply_queue
+      #
+      def command(command, timeout: 10)
         command = { command: command.to_s } unless command.is_a? Hash
         command[:timestamp] ||= (Time.now.to_f * 1000).to_i
         request_id = Digest::SHA256.hexdigest(command.to_json)[..6]
-        requests[request_id] = { request: command, id: request_id }
+        requests[request_id] = {
+          request: command,
+          id: request_id,
+          lock: Mutex.new,
+          condition: ConditionVariable.new
+        }
 
         exchange.publish(command.to_json,
-                         content_type: 'application/json',
                          routing_key: 'dataproxy_commands',
                          correlation_id: request_id,
                          reply_to: reply_queue.name)
 
         # wait for the signal to continue the execution
-        lock.synchronize { 
-          condition.wait(lock, timeout) 
-        } 
+        #
+        requests[request_id][:lock].synchronize do
+          requests[request_id][:condition].wait(requests[request_id][:lock], timeout)
+        end
 
+        # if we reached timeout, we will return nil, just for explicity
+        response = requests[request_id][:response].dup
+        requests.delete(request_id)
         response
       end
 
+      alias_method :send_command, :command
+
       def stop
-        channel.close
+        %i[depth ticks realtimebars].each do |type|
+          persistent[type].each do |local_key, obj|
+            puts "Cancelling #{local_key}"
+            obj[:subscription].cancel
+          end
+        end
+        commands.close
         connection.close
       end
 
-
-      def get_contracts(symbol: )
-        send_command( { command: :get_contracts, symbol: symbol } )
+      def get_contracts(symbol:)
+        send_command({ command: :get_contracts, symbol: symbol })
       end
 
       def get_historical(contract:, interval:, duration: nil, before: nil, rth_only: false, based_on: :trades)
         # rth.true? means data outside of rth is skipped
         rth_only = rth_only ? 1 : 0
-        default_durations = {
-          sec1:  '30_M',
-          sec5:   '2_H',
-          sec15:  '6_H',
-          sec30: '12_H',
-          min1:   '1_D',
-          min2:   '2_D',
-          min5:   '5_D',
-          min15:  '1_W',
-          min30:  '1_W',
-          hour1:  '1_W',
-          day1:   '1_Y'
-        } 
-        raise "Invalid interval '#{interval}', should be in '#{default_durations.keys}'." unless default_durations.keys.include? interval
+
+        # interval most probably is given as ActiveSupport::Duration
+        if interval.is_a? ActiveSupport::Duration
+          interval = case interval
+                     when       1; :sec1 
+                     when       5; :sec5
+                     when      15; :sec15
+                     when      30; :sec30
+                     when      60; :min1
+                     when     120; :min2
+                     when     300; :min5
+                     when     900; :min15
+                     when    1800; :min30
+                     when    3600; :hour1
+                     when   86400; :day1
+                     else; interval
+                     end
+        end
+
+        default_durations = { sec1: '30_M',  sec5: '2_H', sec15: '6_H', sec30: '12_H',
+                              min1:  '1_D',  min2: '2_D',  min5: '5_D', min15:  '1_W',
+                              min30: '1_W', hour1: '1_W',  day1: '1_Y'                 }
+
+        unless default_durations.keys.include? interval
+          raise "Invalid interval '#{interval}', should be in '#{default_durations.keys}'."
+        end
+
         # TODO: Check for valid duration specification
-        duration ||= default_durations[interval] 
-        send_command( { 
-          command: :historical,
-          contract: contract,
-          interval: interval,
-          duration: duration,
-          based_on: based_on.to_s.upcase,
-          rth_only: rth_only,
-          before: nil
-        }, timeout: 20 )
+        puts 'WARNING in get_historical: param :before ignored' unless before.nil?
+        duration ||= default_durations[interval]
+        send_command({
+                       command: :historical,
+                       contract: contract,
+                       interval: interval,
+                       duration: duration,
+                       based_on: based_on.to_s.upcase,
+                       rth_only: rth_only,
+                       before: nil
+                     }, timeout: 20)
       end
 
-      def start_persistent(contract:, type: :realtimebars, &block)
-        unless %i[ depth ticks realtimebars].include? type.to_sym
+      def start_persistent(contract:, type: :realtimebars, local_id: 0, &block)
+        unless %i[depth ticks realtimebars].include? type.to_sym
           puts "ERROR: Inappropriate type in stop_realtimebars with #{type}"
           return false
         end
 
-        ib_contract = Cotcube::Helpers.get_ib_contract(contract)
-        exchange = channel.fanout( "dataproxy_#{type.to_s}_#{contract}", auto_delete: true)
-        queue  = channel.queue('', exclusive: true, auto_delete: true)
+        local_key = "#{contract}_#{local_id}"
+
+        channel = connection.create_channel
+        exchange  = channel.fanout("dataproxy_#{type}_#{contract}")
+        queue     = channel.queue('', exclusive: true, auto_delete: true)
         queue.bind(exchange)
-        block ||= ->(bar){ puts "#{bar}" }
-        queue.subscribe do  |_delivery_info, properties, payload|
+
+        ib_contract = Cotcube::Helpers.get_ib_contract(contract)
+
+        command = { command: type, contract: contract, con_id: ib_contract[:con_id],
+                    delivery: queue.name, exchange: exchange.name }
+
+        block      ||= ->(bar) { puts bar.to_s }
+
+        subscription = queue.subscribe do |_delivery_info, _properties, payload|
           block.call(JSON.parse(payload, symbolize_names: true))
         end
-        command = { command: type, contract: contract, con_id: ib_contract[:con_id], delivery: queue.name, exchange: exchange.name }
-        persistent[type][queue.name] = command
-        persistent[type][queue.name][:queue] = queue
+        persistent[type][local_key] = command.dup
+        persistent[type][local_key][:queue] = queue
+        persistent[type][local_key][:subscription] = subscription
+        persistent[type][local_key][:channel] = channel
         send_command(command)
       end
 
-      def stop_persistent(contract:, type: :realtimebars )
-        unless %i[ depth ticks realtimebars].include? type.to_sym
+      def stop_persistent(contract:, local_id: 0, type: :realtimebars)
+        unless %i[depth ticks realtimebars].include? type.to_sym
           puts "ERROR: Inappropriate type in stop_realtimebars with #{type}"
           return false
         end
-        ib_contract = Cotcube::Helpers.get_ib_contract(contract)
-        command = { command: "stop_#{type}", contract: contract, con_id: ib_contract[:con_id] }
-        send_command(command)
+        local_key = "#{contract}_#{local_id}"
+        p persistent[type][local_key][:subscription].cancel
+        p persistent[type][local_key][:channel].close
+        persistent[type].delete(local_key)
       end
 
       attr_accessor :response
       attr_reader   :lock, :condition
 
       private
-      attr_reader :call_id, :connection, :requests, :persistent,
-        :channel, :server_queue_name, :reply_queue, :exchange
 
+      attr_reader :call_id, :connection, :requests, :persistent,
+                  :commands, :server_queue_name, :reply_queue, :exchange
 
       def setup_reply_queue
-        @lock = Mutex.new
-        @condition = ConditionVariable.new
-        that = self
-        @reply_queue = channel.queue('', exclusive: true, auto_delete: true)
-        @reply_queue.bind(channel.exchange('dataproxy_replies', auto_delete: true), routing_key: @reply_queue.name)
+        @reply_queue = commands.queue('', exclusive: true, auto_delete: true)
+        @reply_queue.bind(commands.exchange('dataproxy_replies'), routing_key: @reply_queue.name)
 
         reply_queue.subscribe do |delivery_info, properties, payload|
-
-          __id__ = properties[:correlation_id] 
+          __id__ = properties[:correlation_id]
 
           if __id__.nil?
-            puts "Received without __id__: #{delivery_info.map{|k,v| "#{k}\t#{v}"}.join("\n")
-                      }\n\n#{properties   .map{|k,v| "#{k}\t#{v}"}.join("\n")
-                      }\n\n#{JSON.parse(payload).map{|k,v| "#{k}\t#{v}"}.join("\n")}"
+            puts "Received without __id__: #{delivery_info.map       { |k, v| "#{k}\t#{v}" }.join("\n")
+                                      }\n\n#{properties.map          { |k, v| "#{k}\t#{v}" }.join("\n")
+                                      }\n\n#{JSON.parse(payload).map { |k, v| "#{k}\t#{v}" }.join("\n")}" if @debug
 
           elsif requests[__id__].nil?
-            puts "Received non-matching response: \n\n#{_delivery_info}\n\n#{properties}\n\n#{payload}\n."
-          else 
-            that.response = payload
-
-            # sends the signal to continue the execution of #call
-            requests.delete(__id__)
-            that.lock.synchronize { that.condition.signal }
+            puts "Received non-matching response, maybe previously timed out: \n\n#{delivery_info}\n\n#{properties}\n\n#{payload}\n."[..620].scan(/.{1,120}/).join(' '*30 + "\n") if @debug
+          else
+            # save the payload and send the signal to continue the execution of #command
+            # need to rescue the rare case, where lock and condition are destroyed right in parallel by timeout
+            begin 
+              puts "Received result for #{__id__}" if @debug
+              requests[__id__][:response] = payload
+              requests[__id__][:lock].synchronize { requests[__id__][:condition].signal }
+            rescue nil
+            end
           end
         end
       end
-
     end
   end
 end
@@ -174,4 +227,4 @@ begin
   } 
 ensure
   client.stop
-  e,nd
+  end

data/lib/cotcube-helpers/deep_decode_datetime.rb

@@ -0,0 +1,25 @@
+module Cotcube
+  module Helpers
+    VALID_DATETIME_STRING = lambda {|str| str.is_a?(String) and [10,25,29].include?(str.length) and str.count("^0-9:TZ+-= ").zero? }
+
+    def deep_decode_datetime(data, zone: DateTime)
+      case data
+      when nil;    nil
+      when VALID_DATETIME_STRING
+        res = nil
+        begin
+          res = zone.parse(data)
+        rescue ArgumentError
+          data
+        end
+        [ DateTime, ActiveSupport::TimeWithZone ].include?(res.class) ? res : data
+      when Array; data.map!              { |d| deep_decode_datetime(d, zone: zone) }
+      when Hash;  data.transform_values! { |v| deep_decode_datetime(v, zone: zone) }
+      else;       data
+      end
+    end
+
+    module_function :deep_decode_datetime
+  end
+end
+

data/lib/cotcube-helpers/expiration.rb

@@ -0,0 +1,31 @@
+module Cotcube
+  module Helpers
+
+    class ExpirationMonth
+      attr_accessor *%i[ asset month year holidays stencil ]
+      def initialize( contract: )
+        a,b,c,d,e = contract.chars
+        @asset  = [ a, b ].join
+        if %w[ GG DL BJ GE VI ] 
+          puts "Denying to calculate expiration for #{@asset}".light_red
+          return
+        end
+        @month  = MONTHS[c] + offset
+        @month -= 1 if %w[ CL HO NG RB SB].include? @asset
+        @month += 1 if %w[ ].include? @asset
+        @month += 12 if month < 1
+        @month -= 12 if month > 12
+        @year   = [ d, e ].join.to_i
+        @year  += year > 61 ? 1900 : 2000
+        @holidays = CSV.read("/var/cotcube/bardata/holidays.csv").map{|x| DateTime.parse(x[0]).to_date}.select{|x| x.year == @year }
+        @stencil  = [ Date.new(@year, @month, 1) ] 
+        end_date = Date.new(@year, @month + 1, 1 )
+        while (next_date = @stencil.last + 1) < end_date
+          @stencil << next_date
+        end
+        @stencil.reject!{|x| [0,6].include?(x.wday) or @holidays.include? x}
+      end
+    end
+
+  end
+end

data/lib/cotcube-helpers/hash_ext.rb

@@ -2,15 +2,15 @@
 
 # Monkey patching the Ruby Core class Hash
 class Hash
-  def keys_to_sym
-    each_key do |key|
+  def keys_to_sym!
+    self.keys.each do |key|
       case self[key].class.to_s
       when 'Hash'
-        self[key].keys_to_sym
+        self[key].keys_to_sym!
       when 'Array'
-        self[key].map { |el| el.is_a?(Hash) ? el.keys_to_sym : el }
+        self[key].map { |el| el.is_a?(Hash) ? el.keys_to_sym! : el }
       end
-      self[key.to_sym] = delete(key)
+      self["#{key}".to_sym] = delete(key)
     end
     self
   end

data/lib/cotcube-helpers/init.rb

@@ -34,7 +34,7 @@ module Cotcube
       end
 
       defaults = { 
-        data_path: '/var/cotcube/' + name,
+        data_path: '/var/cotcube/' + name.split('::').last.downcase,
       }
 
       config = defaults.merge(config)

data/lib/cotcube-helpers/josch_client.rb

@@ -0,0 +1,115 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bunny'
+require 'json'
+
+module Cotcube
+  module Helpers
+    class JoSchClient
+      SECRETS_DEFAULT = {
+        'josch_mq_proto' => 'http',
+        'josch_mq_user' => 'guest',
+        'josch_mq_password' => 'guest',
+        'josch_mq_host' => 'localhost',
+        'josch_mq_port' => '15672',
+        'josch_mq_vhost' => '%2F'
+      }.freeze
+
+      SECRETS = SECRETS_DEFAULT.merge(
+        lambda {
+          begin
+            YAML.safe_load(File.read(Cotcube::Helpers.init[:secrets_file]))
+          rescue StandardError
+            {}
+          end
+        }.call
+      )
+
+      def initialize
+        @connection = Bunny.new(user:     SECRETS['josch_mq_user'],
+                                password: SECRETS['josch_mq_password'],
+                                vhost:    SECRETS['josch_mq_vhost'])
+        @connection.start
+
+        @commands  = connection.create_channel
+        @exchange  = commands.direct('josch_commands')
+        @requests  = {}
+        @debug     = false
+        setup_reply_queue
+      end
+
+      # command acts a synchronizer: it sends the command and waits for the response
+      #     otherwise times out --- the counterpart here is the subscription within 
+      #     setup_reply_queue
+      #
+      def command(command, timeout: 10)
+        command = { command: command.to_s } unless command.is_a? Hash
+        command[:timestamp] ||= (Time.now.to_f * 1000).to_i
+        request_id = Digest::SHA256.hexdigest(command.to_json)[..6]
+        requests[request_id] = {
+          request: command,
+          id: request_id,
+          lock: Mutex.new,
+          condition: ConditionVariable.new
+        }
+
+        exchange.publish(command.to_json,
+                         routing_key: 'josch_commands',
+                         correlation_id: request_id,
+                         reply_to: reply_queue.name)
+
+        # wait for the signal to continue the execution
+        #
+        requests[request_id][:lock].synchronize do
+          requests[request_id][:condition].wait(requests[request_id][:lock], timeout)
+        end
+
+        # if we reached timeout, we will return nil, just for explicity
+        response = requests[request_id][:response].dup
+        requests.delete(request_id)
+        response
+      end
+
+      alias_method :send_command, :command
+
+      attr_accessor :response
+      attr_reader   :lock, :condition
+
+      private
+
+      attr_reader :call_id, :connection, :requests, :persistent,
+                  :commands, :server_queue_name, :reply_queue, :exchange
+
+      def setup_reply_queue
+        @reply_queue = commands.queue('', exclusive: true, auto_delete: true)
+        @reply_queue.bind(commands.exchange('josch_replies'), routing_key: @reply_queue.name)
+
+        reply_queue.subscribe do |delivery_info, properties, payload|
+          __id__ = properties[:correlation_id]
+
+          if __id__.nil?
+            puts "Received without __id__: #{delivery_info.map       { |k, v| "#{k}\t#{v}" }.join("\n")
+                                      }\n\n#{properties.map          { |k, v| "#{k}\t#{v}" }.join("\n")
+                                      }\n\n#{JSON.parse(payload).map { |k, v| "#{k}\t#{v}" }.join("\n")}" if @debug
+
+          elsif requests[__id__].nil?
+            puts "Received non-matching response, maybe previously timed out: \n\n#{delivery_info}\n\n#{properties}\n\n#{payload}\n."[..620].scan(/.{1,120}/).join(' '*30 + "\n") if @debug
+          else
+            # save the payload and send the signal to continue the execution of #command
+            # need to rescue the rare case, where lock and condition are destroyed right in parallel by timeout
+            begin 
+              puts "Received result for #{__id__}" if @debug
+              requests[__id__][:response] = payload
+              requests[__id__][:lock].synchronize { requests[__id__][:condition].signal }
+            rescue nil
+            end
+          end
+        end
+      end
+    end
+    JoschClient = JoSchClient
+  end
+end
+
+__END__

data/lib/cotcube-helpers/order_client.rb

@@ -0,0 +1,131 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bunny'
+require 'json'
+
+module Cotcube
+  module Helpers
+    class OrderClient
+      SECRETS_DEFAULT = {
+        'orderproxy_mq_proto'    => 'http',
+        'orderproxy_mq_user'     => 'guest',
+        'orderproxy_mq_password' => 'guest',
+        'orderproxy_mq_host'     => 'localhost',
+        'orderproxy_mq_port'     => '15672',
+        'orderproxy_mq_vhost'    => '%2F'
+      }.freeze
+
+      SECRETS = SECRETS_DEFAULT.merge(
+        lambda {
+          begin
+            YAML.safe_load(File.read(Cotcube::Helpers.init[:secrets_file]))
+          rescue StandardError
+            {}
+          end
+        }.call
+      )
+
+      def initialize
+        @connection = Bunny.new(user:     SECRETS['orderproxy_mq_user'],
+                                password: SECRETS['orderproxy_mq_password'],
+                                vhost:    SECRETS['orderproxy_mq_vhost'])
+        @connection.start
+
+        @commands  = connection.create_channel
+        @exchange  = commands.direct('orderproxy_commands')
+        @requests  = {}
+        @persistent = { depth: {}, realtimebars: {}, ticks: {} }
+        @debug     = false
+        setup_reply_queue
+      end
+
+      # command acts a synchronizer: it sends the command and waits for the response
+      #     otherwise times out --- the counterpart here is the subscription within 
+      #     setup_reply_queue
+      #
+      def command(command, timeout: 10)
+        command = { command: command.to_s } unless command.is_a? Hash
+        command[:timestamp] ||= (Time.now.to_f * 1000).to_i
+        request_id = Digest::SHA256.hexdigest(command.to_json)[..6]
+        requests[request_id] = {
+          request: command,
+          id: request_id,
+          lock: Mutex.new,
+          condition: ConditionVariable.new
+        }
+
+        exchange.publish(command.to_json,
+                         routing_key: 'orderproxy_commands',
+                         correlation_id: request_id,
+                         reply_to: reply_queue.name)
+
+        # wait for the signal to continue the execution
+        #
+        requests[request_id][:lock].synchronize do
+          requests[request_id][:condition].wait(requests[request_id][:lock], timeout)
+        end
+
+        # if we reached timeout, we will return nil, just for explicity
+        response = requests[request_id][:response].dup
+        requests.delete(request_id)
+        response
+      end
+
+      alias_method :send_command, :command
+
+      def stop
+        commands.close
+        connection.close
+      end
+
+      def get_contracts(symbol:)
+        send_command({ command: :get_contracts, symbol: symbol })
+      end
+
+      attr_accessor :response
+      attr_reader   :lock, :condition
+
+      private
+
+      attr_reader :call_id, :connection, :requests, :persistent,
+                  :commands, :server_queue_name, :reply_queue, :exchange
+
+      def setup_reply_queue
+        @reply_queue = commands.queue('', exclusive: true, auto_delete: true)
+        @reply_queue.bind(commands.exchange('orderproxy_replies'), routing_key: @reply_queue.name)
+
+        reply_queue.subscribe do |delivery_info, properties, payload|
+          __id__ = properties[:correlation_id]
+
+          if __id__.nil?
+            puts "Received without __id__: #{delivery_info.map       { |k, v| "#{k}\t#{v}" }.join("\n")
+                                      }\n\n#{properties.map          { |k, v| "#{k}\t#{v}" }.join("\n")
+                                      }\n\n#{JSON.parse(payload).map { |k, v| "#{k}\t#{v}" }.join("\n")}" if @debug
+
+          elsif requests[__id__].nil?
+            puts "Received non-matching response, maybe previously timed out: \n\n#{delivery_info}\n\n#{properties}\n\n#{payload}\n."[..620].scan(/.{1,120}/).join(' '*30 + "\n") if @debug
+          else
+            # save the payload and send the signal to continue the execution of #command
+            # need to rescue the rare case, where lock and condition are destroyed right in parallel by timeout
+            begin 
+              puts "Received result for #{__id__}" if @debug
+              requests[__id__][:response] = payload
+              requests[__id__][:lock].synchronize { requests[__id__][:condition].signal }
+            rescue nil
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+__END__
+begin 
+  client = OrderClient.new
+  reply = client.send_command( { command: 'ping' } ) 
+  puts reply.nil? ? 'nil' : JSON.parse(reply)
+ensure
+  client.stop
+end

data/lib/cotcube-helpers/orderclient.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+module Cotcube
+  module Helpers
+    # A proxyclient is a wrapper that allows communication with cotcube-orderproxy and cotcube-dataproxy. It fulfills
+    # registration and provides the opportunity to implement the logic to respond do events.
+    # (orderproxy and dataproxy are separate gems creating a layer between tws/ib-ruby and cotcube-based
+    #  applications)
+    #
+    #  NOTE: Whats here is a provisionally version
+    #
+    class DataClient # rubocop:disable Metrics/ClassLength
+      attr_reader :power, :ticksize, :multiplier, :average, :account
+
+      # The constructor takes a lot of arguments:
+      def initialize(
+        debug: false,
+        contract: ,
+        serverport: 24001,
+        serveraddr: '127.0.0.1',
+        client:,
+        bars: true,
+        ticks: false,
+        bar_size: 5,
+        spawn_timeout: 15
+      )
+        require 'json'   unless Hash.new.respond_to? :to_json
+        require 'socket' unless defined? TCPSocket
+
+        puts 'PROXYCLIENT: Debug enabled' if @debug
+
+        @contract = contract.upcase
+        %w[debug serverport serveraddr client bars ticks bar_size].each {|var| eval("@#{var} = #{var}")}
+
+        @position     = 0
+        @account      = 0
+        @average      = 0
+
+        exit_on_startup(':client must be in range 24001..24999') if @client.nil? || (@client / 1000 != 24) || (@client == 24_000)
+
+        res = send_command({ command: 'register', contract: @contract, date: @date,
+                             ticks: @ticks, bars: @bars, bar_size: @bar_size })
+
+        # spawn_server has to be called separately after initialization. 
+        print "Waiting #{spawn_timeout} seconds on server_thread to spawn..."
+        Thread.new do 
+          begin 
+            Timeout.timeout(spawn_timeout) { sleep(0.1) while @server_thread.nil? }
+          rescue Timeout::Error
+            puts 'Could not get server_thread, has :spawn_server been called?' 
+            shutdown
+          end
+        end
+
+        unless res['error'].zero?
+          exit_on_startup("Unable to register on orderproxy, exiting")
+        end
+      end
+
+      def exit_on_startup(msg = '')
+        puts "Cannot startup client, exiting during startup: '#{msg}'"
+        shutdown
+        defined?(::IRB) ? (raise) : (exit 1)
+      end
+
+      def send_command(req)
+        req[:client_id] = @client
+        res = nil
+        puts "Connecting to #{@serveraddr}:#{@serverport} to send '#{req}'." if @debug
+
+        TCPSocket.open(@serveraddr, @serverport) do |proxy|
+          proxy.puts(req.to_json)
+          raw = proxy.gets
+          begin
+            res = JSON.parse(raw)
+          rescue StandardError
+            puts 'Error while parsing response'
+            return false
+          end
+          if @debug
+            # rubocop:disable Style/FormatStringToken, Style/FormatString
+            res.each do |k, v|
+              case v
+              when Array
+                (v.size < 2) ? puts(format '%-15s', "#{k}:") : print(format '%-15s', "#{k}:")
+                v.each_with_index { |x, i| i.zero? ? (puts x) : (puts "               #{x}") }
+              else
+                puts "#{format '%-15s', "#{k}:"}#{v}"
+              end
+            end
+            # rubocop:enable Style/FormatStringToken, Style/FormatString
+          end
+          puts "ERROR on command: #{res['msg']}" unless res['error'].nil? or res['error'].zero?
+        end
+        puts res.to_s if @debug
+        res
+      end
+
+      # #shutdown ends the @server_thread and --if :close is set-- closes the current position attached to the client
+      def shutdown(close: true)
+        return if @shutdown
+        @shutdown = true
+
+        if @position.abs.positive? && close
+          send_command({ command: 'order', action: 'create', type: 'market',
+                         side: (@position.positive? ? 'sell' : 'buy'), size: @position.abs })
+        end
+        sleep 3
+        result = send_command({ command: 'unregister' })
+        puts "FINAL ACCOUNT: #{@account}"
+        result['executions']&.each do |x|
+          x.delete('msg_type')
+          puts x.to_s
+        end
+        @server_thread.exit if @server_thread.respond_to? :exit
+      end
+
+      def spawn_server(
+        execution_proc: nil,
+        orderstate_proc: nil, 
+        tick_proc: nil,
+        depth_proc: nil, 
+        order_proc: nil, 
+        bars_proc: nil
+      )                                           # rubocop:disable Metrics/MethodLength
+
+        %w[execution_proc orderstate_proc tick_proc depth_proc order_proc bars_proc].each {|var| eval("@#{var} = #{var}") }
+
+        if @bars 
+          @bars_proc ||= lambda {|msg| puts msg.inspect }
+        end
+
+        if @ticks
+          @ticks_proc ||= lambda {|msg| puts msg.inspect }
+        end
+
+        if @shutdown
+          puts "Cannot spawn server on proxyclient that has been already shut down."
+          return
+        end
+        if @server_thread
+          puts "Cannot spawn server more than once."
+          return
+        end
+
+        @server_thread = Thread.new do                                      # rubocop:disable Metrics/BlockLength
+          puts 'Spawning RECEIVER'
+          server = TCPServer.open(@serveraddr, @client)
+          loop do                                                           # rubocop:disable Metrics/BlockLength
+            Thread.start(server.accept) do |client|                         # rubocop:disable Metrics/BlockLength
+              while (response = client.gets)
+                response = JSON.parse(response)
+
+                case response['msg_type']
+
+                when 'alert'
+                  case response['code']
+                  when 2104
+                    puts 'ALERT: data farm connection resumed   __ignored__'.light_black
+                  when 2108
+                    puts 'ALERT: data farm connection suspended __ignored__'.light_black
+                  when 2109
+                    # Order Event Warning:Attribute 'Outside Regular Trading Hours' is ignored
+                    # based on the order type and destination. PlaceOrder is now being processed.
+                    puts 'ALERT: outside_rth __ignored__'.light_black
+                  when 2100
+                    puts 'ALERT: Account_info unsubscribed __ignored__'.light_black
+                  when 202
+                    puts 'ALERT: order cancelled'
+                  else
+                    puts '-------------ALERT------------------------------'
+                    puts response.to_s
+                    puts '------------------------------------------------'
+                  end
+
+                when 'tick'
+                  @tick_proc&.call(response)
+
+                when 'depth'
+                  @depth_proc&.call(response)
+
+                when 'realtimebar'
+                  @bars_proc&.call(response)
+
+                else
+                  puts "ERROR:       #{response}"
+
+                end
+              end
+            end
+          rescue StandardError => e
+            backtrace = e.backtrace.join("\r\n")
+            puts "======= ERROR: '#{e.class}', MESSAGE: '#{e.message}'\n#{backtrace}"
+          end
+        end
+        puts '@server_thread spawned'
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity

data/lib/cotcube-helpers.rb

@@ -21,6 +21,7 @@ require_relative 'cotcube-helpers/subpattern'
 require_relative 'cotcube-helpers/parallelize'
 require_relative 'cotcube-helpers/simple_output'
 require_relative 'cotcube-helpers/simple_series_stats'
+require_relative 'cotcube-helpers/deep_decode_datetime'
 require_relative 'cotcube-helpers/constants'
 require_relative 'cotcube-helpers/input'
 require_relative 'cotcube-helpers/output'
@@ -30,7 +31,6 @@ require_relative 'cotcube-helpers/init'
 require_relative 'cotcube-helpers/get_id_set'
 require_relative 'cotcube-helpers/ib_contracts'
 require_relative 'cotcube-helpers/recognition'
-require_relative 'cotcube-helpers/data_client'
 
 module Cotcube
   module Helpers
@@ -52,3 +52,7 @@ module Cotcube
     # please not that module_functions of source provided in private files must be published there
   end
 end
+
+%w[ data cache order josch ].each do |part| 
+  require_relative "cotcube-helpers/#{part}_client"
+end

data/scripts/collect_market_depth

@@ -0,0 +1,103 @@
+#!/usr/bin/env ruby
+
+require 'bundler'
+Bundler.require
+require 'parallel'
+require_relative '../lib/cotcube-helpers'
+
+dc = Cotcube::Helpers::DataClient.new
+
+dc.start_persistent(type: :depth, contract: 'GCZ21') {|msg| p msg.size; p msg.map{|z| z[:size]}.reduce(&:+) }
+
+loop do
+ sleep 1
+end
+
+__END__
+  depthThread = Thread.new do 
+    begin
+      loop do
+        sleep 0.025 while depthQueue.empty?
+        while not depthQueue.empty?
+          msg = depthQueue.pop
+          if msg.data[:operation] == 2
+            data = [ (Time.now.to_f % 100000).round(3), :d, msg.data.values_at(:operation, :side, :position) ].flatten
+          else
+            data = [ (Time.now.to_f % 100000).round(3), :d, msg.data.values_at(:operation, :side, :position, :size, :price) ].flatten
+          end
+          puts "#{data}" if data[2]!=1 or data[4] == 0
+          next
+          writeQueue << data
+          side  = msg.data[:side]
+          price = msg.data[:price] 
+          size  = msg.data[:size]
+          pos   = msg.data[:position]
+          case msg.data[:operation]
+          when 0 # insert
+            orderbook[side].insert(pos, { price: price.to_f, size: size })
+          when 1 # update
+            orderbook[side][pos] = { price: price.to_f, size: size }
+          when 2 # remove
+            orderbook[side].delete_at(pos)
+          end
+          orderbook[1].shift while orderbook[1].size > DEPTH
+          orderbook[0].shift while orderbook[0].size > DEPTH
+          a = orderbook[0].size
+          a.times do |n|
+            s = a - 1
+            ask = orderbook[0][s-n]
+            next if ask.nil? 
+            allasks = orderbook[0][0..s-n].map{|x| x[:size]}.reduce(:+)
+            asksacc = orderbook[0][0..s-n].map{|x| x[:size] * x[:price]}.reduce(:+) / allasks        
+            puts "\t\t\t\t#{format % ask[:price]}  x #{'% 5d' % ask[:size]}\t#{'% 4d' % allasks}\t#{format % asksacc}"
+          end
+          allasks = orderbook[0].compact.map{|x| x[:size]}.reduce(:+)
+          asksacc = orderbook[0].compact.map{|x| x[:size] * x[:price]}.reduce(:+) / allasks unless orderbook[0].empty?
+          allbids = orderbook[1].compact.map{|x| x[:size]}.reduce(:+)
+          bidsacc = orderbook[1].compact.map{|x| x[:size] * x[:price]}.reduce(:+) / allbids unless orderbook[0].empty?
+          puts "#{(format % bidsacc) unless bidsacc.nil?}\t".light_red +
+            "#{('% 4d' % allbids) unless allbids.nil?}\t\t\t\t\t".light_red +
+            "#{"#{'% 5d' % allasks}" unless allasks.nil?}\t".light_red +
+            "#{"#{format % asksacc}" unless asksacc.nil?}".light_red 
+          b = orderbook[1].size
+          b.times do |n| 
+            bid = orderbook[1][n]
+            next if bid.nil?
+            allbids = orderbook[1][0..n].map{|x| x[:size]}.reduce(:+)
+            bidsacc = orderbook[1][0..n].map{|x| x[:size] * x[:price]}.reduce(:+) / allbids
+            puts "#{format % bidsacc}\t#{'% 4d' % allbids}\t#{'%5d' % bid[:size]} x #{format % bid[:price]}"
+          end
+          puts "="*50
+        end
+      end
+    rescue
+      puts "RESCUE in depthThread".light_red
+      puts "#{orderbook}"
+      raise
+    end
+  end
+
+  sleep 0.01 while Time.now.to_i % WRITE_INTERVAL != 0
+  loop do
+    t = Time.now.to_f 
+    unless writeQueue.empty?
+      data = [] 
+      data << writeQueue.pop while not writeQueue.empty?
+      CSV.open(OUTFILE, "a+") { |csv| data.each {|x| csv << x } }
+    end
+    begin
+      sleep WRITE_INTERVAL - (Time.now.to_f - t)
+    rescue 
+      sleep 3
+    end
+  end
+
+
+ensure
+  ib.send_message :CancelMarketDepth, id: ID
+  ib.send_message :CancelMarketData, id: ID
+  tickThread.kill
+  depthThread.kill
+end
+sleep 1
+puts "Done."

data/scripts/cron_ruby_wrapper.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-export rubyenv=/home/pepe/.rvm/environments/default
+export rubyenv=/home/pepe/.rvm/environments/ruby-2.7.5
 
 . $rubyenv 
 cd /home/pepe/GEMS/${1}
@@ -10,6 +10,7 @@ ruby ${2} ${3} ${4} ${5} ${6}
 
  
 exit
+
 for testing run 
 env - `cat /home/pepe/bin/cron_ruby_wrapper.sh | tail -n 6` /bin/bash
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cotcube-helpers
 version: !ruby/object:Gem::Version
-  version: 0.2.1.1
+  version: 0.2.3
 platform: ruby
 authors:
 - Benjamin L. Tischendorf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-11 00:00:00.000000000 Z
+date: 2021-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -96,16 +96,22 @@ files:
 - cotcube-helpers.gemspec
 - lib/cotcube-helpers.rb
 - lib/cotcube-helpers/array_ext.rb
+- lib/cotcube-helpers/cache_client.rb
 - lib/cotcube-helpers/constants.rb
 - lib/cotcube-helpers/data_client.rb
 - lib/cotcube-helpers/datetime_ext.rb
+- lib/cotcube-helpers/deep_decode_datetime.rb
 - lib/cotcube-helpers/enum_ext.rb
+- lib/cotcube-helpers/expiration.rb
 - lib/cotcube-helpers/get_id_set.rb
 - lib/cotcube-helpers/hash_ext.rb
 - lib/cotcube-helpers/ib_contracts.rb
 - lib/cotcube-helpers/init.rb
 - lib/cotcube-helpers/input.rb
+- lib/cotcube-helpers/josch_client.rb
 - lib/cotcube-helpers/numeric_ext.rb
+- lib/cotcube-helpers/order_client.rb
+- lib/cotcube-helpers/orderclient.rb
 - lib/cotcube-helpers/output.rb
 - lib/cotcube-helpers/parallelize.rb
 - lib/cotcube-helpers/range_ext.rb
@@ -119,6 +125,7 @@ files:
 - lib/cotcube-helpers/swig/fill_x.rb
 - lib/cotcube-helpers/swig/recognition.rb
 - lib/cotcube-helpers/symbols.rb
+- scripts/collect_market_depth
 - scripts/cron_ruby_wrapper.sh
 - scripts/symbols
 homepage: https://github.com/donkeybridge/cotcube-helpers