cotcube-helpers 0.2.0 → 0.2.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38df15256f996b15dfeb1755f7464e932a98541e62aa43efda0a264be58bafb2
-  data.tar.gz: cd336b6a91442a6d3bbace66ec8a25049ae979ab78eef03d1c8b44a548da8023
+  metadata.gz: 7496f5774fbcb9b661dac0f9315081a4b5f4d30a8c89334ef13d0639b789ba73
+  data.tar.gz: 596ab3ed0cc578c6321beb1b07b6400e109f4537d91cf0903a0c0542b9a962df
 SHA512:
-  metadata.gz: bcfc562ab118939e7c23dc37b89fee881871552b6b7b17ec7066cb4a2028fa2661e50cd5512725da28332dd1b983d6817d416d31ba05059dd15d824a0e101f7c
-  data.tar.gz: 9ca3305120eb630283e303206b764cf9d8dd533ceaf0a2aa044f702fa2f2a01055ff70a4331ec865d854580b1dcf367f365c5addbb988605537c0ee9b9b6fad9
+  metadata.gz: 7447d4f4d2297c60eb37a815b8cd0c91ddd70d1c4af0d2f8bab89ada65c2ac0c328da6e6dd2e99f46d39d75cc765ee4d724598c3d9fdbf42cd15b6a7f30a22fc
+  data.tar.gz: 348426b27122a2be123f93f4db63b7d57e032cf2451f6822a10fce94124d9d0f2cf6c9d3621b920daa651e80ec1b69a5e777d4d94a169cb03f6067f13d1d693f

data/CHANGELOG.md

@@ -1,3 +1,36 @@
+## 0.2.2.5 (December 23, 2021)
+  - hash_ext: reworked keys_to_sym!
+  - minor changes
+  - deep_decode_datetime: added helper for conversion of timestrings in nested arrays and hashed
+  - cache_client: rewritten as class
+  - gemspec: raised activesupport to version 7
+
+## 0.2.2.4 (December 07, 2021)
+  - introduced cache_client as client to readcache
+
+## 0.2.2.3 (November 28, 2021)
+  - data_client: fixing troublesome output in .command
+
+## 0.2.2.2 (November 28, 2021)
+  - dataclient fixing = for false ==
+  - solving merge conflicts
+  - Bump version to 0.2.2.
+
+## 0.2.2.1 (November 28, 2021)
+  - Bump version to 0.2.2.1
+
+## 0.2.2 (November 13, 2021)
+  - some further improvements to DataClient
+  - some fixes related to ib_contracts
+  - some fixes related to DataClient
+
+## 0.2.1.1 (November 10, 2021)
+  - Bump version to 0.2.1.
+
+## 0.2.1 (November 10, 2021)
+  - added new class 'dataclient' for communication with dataproxy
+  - added .translate_ib_contract
+
 ## 0.2.0 (November 07, 2021)
   - added module Candlestick_Recognition
   - added instance_inspect method to 'scan' objects for contents of instance variables

data/Gemfile

@@ -4,3 +4,6 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in bitangent.gemspec
 gemspec
+gem 'parallel'
+gem 'bunny'
+gem 'httparty'

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.2.2.5

data/cotcube-helpers.gemspec

@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'activesupport', '~> 6'
+  spec.add_dependency 'activesupport', '~> 7'
   spec.add_dependency 'colorize',      '~> 0.8'
 
 

data/lib/cotcube-helpers/cache_client.rb

@@ -0,0 +1,52 @@
+require 'httparty'
+require 'json'
+
+module Cotcube
+  module Helpers
+    class CacheClient
+
+      def initialize(query='keys', timezone: Cotcube::Helpers::CHICAGO, debug: false, deflate: false, update: false)
+        raise ArgumentError, "Query must not be empty." if [ nil, '' ].include? query
+        raise ArgumentError, "Query '#{query}' is garbage." if query.split('/').size > 2 or not query.match? /\A[a-zA-Z0-9?=\/]+\Z/
+        @update = update ? '?update=true' : ''
+        @request_headers = {} 
+        @request_headers['Accept-Encoding'] = 'deflate' if deflate
+        @query = query
+        @result = JSON.parse(HTTParty.get("http://100.100.0.14:8081/#{query}#{@update}").body, headers: @request_headers, symbolize_names: true) rescue { error: 1, msg: "Could not parse response for query '#{query}'." }
+        retry_once if has_errors?
+      end
+
+      def retry_once
+        sleep 2
+        raw = HTTParty.get("http://100.100.0.14:8081/#{query}#{update}")
+        @result = JSON.parse(raw.body, symbolize_names: true) rescue { error: 1, msg: "Could not parse response for query '#{query}'." }
+        if has_errors?
+          puts "ERROR in parsing response: #{raw[..300]}"
+        end
+      end
+
+      def has_errors?
+        result[:error].nil? or result[:error] > 0
+      end
+
+      def warnings
+        result[:warnings]
+      end
+
+      def payload
+        has_errors? ? false : @result[:payload]
+      end
+
+      def entity
+        query.split('/').first
+      end
+
+      def asset
+        entity, asset = query.split('/')
+        asset
+      end
+
+      attr_reader :query, :result, :update
+    end
+  end
+end

data/lib/cotcube-helpers/constants.rb

@@ -30,9 +30,17 @@ module Cotcube
 
 
     CHICAGO  = Time.find_zone('America/Chicago')
+    NEW_YORK = Time.find_zone('America/New_York')
+    BERLIN   = Time.find_zone('Europe/Berlin')
 
     DATE_FMT = '%Y-%m-%d'
 
+    # Simple mapper to get from MONTH to LETTER
+    LETTERS = { "JAN"=> "F", "FEB"=> "G", "MAR"=> "H",
+                "APR"=> "J", "MAY"=> "K", "JUN"=> "M",
+                "JUL"=> "N", "AUG"=> "Q", "SEP"=> "U",
+                "OCT"=> "V", "NOV"=> "X", "DEC"=> "Z" }
+
   end
 end
 

data/lib/cotcube-helpers/data_client.rb

@@ -0,0 +1,230 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bunny'
+require 'json'
+
+module Cotcube
+  module Helpers
+    class DataClient
+      SECRETS_DEFAULT = {
+        'dataproxy_mq_proto' => 'http',
+        'dataproxy_mq_user' => 'guest',
+        'dataproxy_mq_password' => 'guest',
+        'dataproxy_mq_host' => 'localhost',
+        'dataproxy_mq_port' => '15672',
+        'dataproxy_mq_vhost' => '%2F'
+      }.freeze
+
+      SECRETS = SECRETS_DEFAULT.merge(
+        lambda {
+          begin
+            YAML.safe_load(File.read(Cotcube::Helpers.init[:secrets_file]))
+          rescue StandardError
+            {}
+          end
+        }.call
+      )
+
+      def initialize
+        @connection = Bunny.new(user: SECRETS['dataproxy_mq_user'],
+                                password: SECRETS['dataproxy_mq_password'],
+                                vhost: SECRETS['dataproxy_mq_vhost'])
+        @connection.start
+
+        @commands  = connection.create_channel
+        @exchange  = commands.direct('dataproxy_commands')
+        @requests  = {}
+        @persistent = { depth: {}, realtimebars: {}, ticks: {} }
+        @debug     = false
+        setup_reply_queue
+      end
+
+      # command acts a synchronizer: it sends the command and waits for the response
+      #     otherwise times out --- the counterpart here is the subscription within 
+      #     setup_reply_queue
+      #
+      def command(command, timeout: 10)
+        command = { command: command.to_s } unless command.is_a? Hash
+        command[:timestamp] ||= (Time.now.to_f * 1000).to_i
+        request_id = Digest::SHA256.hexdigest(command.to_json)[..6]
+        requests[request_id] = {
+          request: command,
+          id: request_id,
+          lock: Mutex.new,
+          condition: ConditionVariable.new
+        }
+
+        exchange.publish(command.to_json,
+                         routing_key: 'dataproxy_commands',
+                         correlation_id: request_id,
+                         reply_to: reply_queue.name)
+
+        # wait for the signal to continue the execution
+        #
+        requests[request_id][:lock].synchronize do
+          requests[request_id][:condition].wait(requests[request_id][:lock], timeout)
+        end
+
+        # if we reached timeout, we will return nil, just for explicity
+        response = requests[request_id][:response].dup
+        requests.delete(request_id)
+        response
+      end
+
+      alias_method :send_command, :command
+
+      def stop
+        %i[depth ticks realtimebars].each do |type|
+          persistent[type].each do |local_key, obj|
+            puts "Cancelling #{local_key}"
+            obj[:subscription].cancel
+          end
+        end
+        commands.close
+        connection.close
+      end
+
+      def get_contracts(symbol:)
+        send_command({ command: :get_contracts, symbol: symbol })
+      end
+
+      def get_historical(contract:, interval:, duration: nil, before: nil, rth_only: false, based_on: :trades)
+        # rth.true? means data outside of rth is skipped
+        rth_only = rth_only ? 1 : 0
+
+        # interval most probably is given as ActiveSupport::Duration
+        if interval.is_a? ActiveSupport::Duration
+          interval = case interval
+                     when       1; :sec1 
+                     when       5; :sec5
+                     when      15; :sec15
+                     when      30; :sec30
+                     when      60; :min1
+                     when     120; :min2
+                     when     300; :min5
+                     when     900; :min15
+                     when    1800; :min30
+                     when    3600; :hour1
+                     when   86400; :day1
+                     else; interval
+                     end
+        end
+
+        default_durations = { sec1: '30_M',  sec5: '2_H', sec15: '6_H', sec30: '12_H',
+                              min1:  '1_D',  min2: '2_D',  min5: '5_D', min15:  '1_W',
+                              min30: '1_W', hour1: '1_W',  day1: '1_Y'                 }
+
+        unless default_durations.keys.include? interval
+          raise "Invalid interval '#{interval}', should be in '#{default_durations.keys}'."
+        end
+
+        # TODO: Check for valid duration specification
+        puts 'WARNING in get_historical: param :before ignored' unless before.nil?
+        duration ||= default_durations[interval]
+        send_command({
+                       command: :historical,
+                       contract: contract,
+                       interval: interval,
+                       duration: duration,
+                       based_on: based_on.to_s.upcase,
+                       rth_only: rth_only,
+                       before: nil
+                     }, timeout: 20)
+      end
+
+      def start_persistent(contract:, type: :realtimebars, local_id: 0, &block)
+        unless %i[depth ticks realtimebars].include? type.to_sym
+          puts "ERROR: Inappropriate type in stop_realtimebars with #{type}"
+          return false
+        end
+
+        local_key = "#{contract}_#{local_id}"
+
+        channel = connection.create_channel
+        exchange  = channel.fanout("dataproxy_#{type}_#{contract}")
+        queue     = channel.queue('', exclusive: true, auto_delete: true)
+        queue.bind(exchange)
+
+        ib_contract = Cotcube::Helpers.get_ib_contract(contract)
+
+        command = { command: type, contract: contract, con_id: ib_contract[:con_id],
+                    delivery: queue.name, exchange: exchange.name }
+
+        block      ||= ->(bar) { puts bar.to_s }
+
+        subscription = queue.subscribe do |_delivery_info, _properties, payload|
+          block.call(JSON.parse(payload, symbolize_names: true))
+        end
+        persistent[type][local_key] = command.dup
+        persistent[type][local_key][:queue] = queue
+        persistent[type][local_key][:subscription] = subscription
+        persistent[type][local_key][:channel] = channel
+        send_command(command)
+      end
+
+      def stop_persistent(contract:, local_id: 0, type: :realtimebars)
+        unless %i[depth ticks realtimebars].include? type.to_sym
+          puts "ERROR: Inappropriate type in stop_realtimebars with #{type}"
+          return false
+        end
+        local_key = "#{contract}_#{local_id}"
+        p persistent[type][local_key][:subscription].cancel
+        p persistent[type][local_key][:channel].close
+        persistent[type].delete(local_key)
+      end
+
+      attr_accessor :response
+      attr_reader   :lock, :condition
+
+      private
+
+      attr_reader :call_id, :connection, :requests, :persistent,
+                  :commands, :server_queue_name, :reply_queue, :exchange
+
+      def setup_reply_queue
+        @reply_queue = commands.queue('', exclusive: true, auto_delete: true)
+        @reply_queue.bind(commands.exchange('dataproxy_replies'), routing_key: @reply_queue.name)
+
+        reply_queue.subscribe do |delivery_info, properties, payload|
+          __id__ = properties[:correlation_id]
+
+          if __id__.nil?
+            puts "Received without __id__: #{delivery_info.map       { |k, v| "#{k}\t#{v}" }.join("\n")
+                                      }\n\n#{properties.map          { |k, v| "#{k}\t#{v}" }.join("\n")
+                                      }\n\n#{JSON.parse(payload).map { |k, v| "#{k}\t#{v}" }.join("\n")}" if @debug
+
+          elsif requests[__id__].nil?
+            puts "Received non-matching response, maybe previously timed out: \n\n#{delivery_info}\n\n#{properties}\n\n#{payload}\n."[..620].scan(/.{1,120}/).join(' '*30 + "\n") if @debug
+          else
+            # save the payload and send the signal to continue the execution of #command
+            # need to rescue the rare case, where lock and condition are destroyed right in parallel by timeout
+            begin 
+              puts "Received result for #{__id__}" if @debug
+              requests[__id__][:response] = payload
+              requests[__id__][:lock].synchronize { requests[__id__][:condition].signal }
+            rescue nil
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+__END__
+begin 
+  client = DataClient.new
+  reply = client.send_command( { command: 'ping' } ) #{ command: :hist, contract: 'A6Z21', con_id: 259130514, interval: :min15 } )
+  puts reply.nil? ? 'nil' : JSON.parse(reply)
+  reply = client.get_historical( contract: 'A6Z21',  con_id: 259130514, interval: :min15 , rth_only: false)
+  JSON.parse(reply, symbolize_names: true)[:result].map{|z| 
+    z[:datetime] = Cotcube::Helpers::CHICAGO.parse(z[:time]).strftime('%Y-%m-%d %H:%M:%S')
+    z.delete(:created_at)
+    z.delete(:time)
+    p z.slice(*%i[datetime open high low close volume]).values
+
+  } 
+ensure
+  client.stop
+  e,nd

data/lib/cotcube-helpers/deep_decode_datetime.rb

@@ -0,0 +1,25 @@
+module Cotcube
+  module Helpers
+    VALID_DATETIME_STRING = lambda {|str| str.is_a?(String) and [10,25,29].include?(str.length) and str.count("^0-9:TZ+-= ").zero? }
+
+    def deep_decode_datetime(data, zone: DateTime)
+      case data
+      when nil;    nil
+      when VALID_DATETIME_STRING
+        res = nil
+        begin
+          res = zone.parse(data)
+        rescue ArgumentError
+          data
+        end
+        [ DateTime, ActiveSupport::TimeWithZone ].include?(res.class) ? res : data
+      when Array; data.map!              { |d| deep_decode_datetime(d, zone: zone) }
+      when Hash;  data.transform_values! { |v| deep_decode_datetime(v, zone: zone) }
+      else;       data
+      end
+    end
+
+    module_function :deep_decode_datetime
+  end
+end
+

data/lib/cotcube-helpers/expiration.rb

@@ -0,0 +1,31 @@
+module Cotcube
+  module Helpers
+
+    class ExpirationMonth
+      attr_accessor *%i[ asset month year holidays stencil ]
+      def initialize( contract: )
+        a,b,c,d,e = contract.chars
+        @asset  = [ a, b ].join
+        if %w[ GG DL BJ GE VI ] 
+          puts "Denying to calculate expiration for #{@asset}".light_red
+          return
+        end
+        @month  = MONTHS[c] + offset
+        @month -= 1 if %w[ CL HO NG RB SB].include? @asset
+        @month += 1 if %w[ ].include? @asset
+        @month += 12 if month < 1
+        @month -= 12 if month > 12
+        @year   = [ d, e ].join.to_i
+        @year  += year > 61 ? 1900 : 2000
+        @holidays = CSV.read("/var/cotcube/bardata/holidays.csv").map{|x| DateTime.parse(x[0]).to_date}.select{|x| x.year == @year }
+        @stencil  = [ Date.new(@year, @month, 1) ] 
+        end_date = Date.new(@year, @month + 1, 1 )
+        while (next_date = @stencil.last + 1) < end_date
+          @stencil << next_date
+        end
+        @stencil.reject!{|x| [0,6].include?(x.wday) or @holidays.include? x}
+      end
+    end
+
+  end
+end

data/lib/cotcube-helpers/get_id_set.rb

@@ -20,10 +20,10 @@ module Cotcube
       end
 
       unless symbol.nil?
-        sym = symbols(symbol: symbol)
+        sym = symbols(symbol: symbol).presence || micros(symbol: symbol)
         if sym.nil? || sym[:id].nil?
           raise ArgumentError,
-                "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}"
+                "Could not find match in #{config[:symbols_file]} or #{config[:micros_file]} for given symbol #{symbol}"
         end
         raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if (not id.nil?) && (sym[:id] != id)
 
@@ -39,6 +39,7 @@ module Cotcube
       end
       raise ArgumentError, 'Need :id, :symbol or valid :contract '
     end
+
   end
 end
 

data/lib/cotcube-helpers/hash_ext.rb

@@ -2,15 +2,15 @@
 
 # Monkey patching the Ruby Core class Hash
 class Hash
-  def keys_to_sym
-    each_key do |key|
+  def keys_to_sym!
+    self.keys.each do |key|
       case self[key].class.to_s
       when 'Hash'
-        self[key].keys_to_sym
+        self[key].keys_to_sym!
       when 'Array'
-        self[key].map { |el| el.is_a?(Hash) ? el.keys_to_sym : el }
+        self[key].map { |el| el.is_a?(Hash) ? el.keys_to_sym! : el }
       end
-      self[key.to_sym] = delete(key)
+      self["#{key}".to_sym] = delete(key)
     end
     self
   end

data/lib/cotcube-helpers/ib_contracts.rb

@@ -0,0 +1,69 @@
+module Cotcube
+  module Helpers
+    def get_ib_contract(contract)
+      symbol = contract[..1]
+      # TODO consider file location to be found in configfile
+      filepath =  '/etc/cotcube/ibsymbols/'
+      result = YAML.load(File.read( "#{filepath}/#{symbol}.yml"))[contract].transform_keys(&:to_sym) rescue nil
+      result.nil? ? update_ib_contracts(symbol: contract[..1]) : (return result)
+      YAML.load(File.read( "#{filepath}/#{symbol}.yml"))[contract].transform_keys(&:to_sym) rescue nil
+    end
+
+    def update_ib_contracts(symbol: nil)
+      begin
+        client = DataClient.new
+        (Cotcube::Helpers.symbols + Cotcube::Helpers.micros).each do |sym|
+
+          # TODO: consider file location to be located in config
+          file = "/etc/cotcube/ibsymbols/#{sym[:symbol]}.yml"
+
+          # TODO: VI publishes weekly options which dont match, the 3 others need multiplier enabled to work
+          next if %w[ DY TM SI VI ].include? sym[:symbol]
+          next if symbol and sym[:symbol] != symbol
+          begin
+            if File.exist? file
+              next if Time.now - File.mtime(file) < 5.days
+              data = nil
+              data = YAML.load(File.read(file))
+            else
+              data = {}
+            end
+            p file
+            %w[ symbol sec_type exchange multiplier ticksize power internal ].each {|z| data.delete z}
+            raw   = client.get_contracts(symbol: sym[:symbol])
+            reply = JSON.parse(raw)['result']
+            reply.each do |set|
+              contract = translate_ib_contract set['local_symbol']
+              data[contract] ||= set
+            end
+            keys = data.keys.sort_by{|z| z[2]}.sort_by{|z| z[-2..] }.select{|z| z[..1] == sym[:symbol] }
+            data = data.slice(*keys)
+            File.open(file, 'w'){|f| f.write(data.to_yaml) }
+          rescue Exception => e
+            puts e.full_message
+            p sym
+            binding.irb
+          end
+        end
+      ensure 
+        client.stop
+        true
+      end
+    end
+
+    def translate_ib_contract(contract)
+      short = contract.split(" ").size == 1
+      sym_a = contract.split(short ? '' : ' ')
+      year  = sym_a.pop.to_i + (short ? 20 : 0)
+      if short and sym_a[-1].to_i > 0
+        year = year - 20 + sym_a.pop.to_i * 10
+      end
+      month = short ? sym_a.pop : LETTERS[sym_a.pop]
+      sym   = Cotcube::Helpers.symbols(internal: sym_a.join)[:symbol] rescue nil
+      sym ||= Cotcube::Helpers.micros(internal: sym_a.join)[:symbol] rescue nil
+      sym.nil? ? false : "#{sym}#{month}#{year}"
+    end
+
+  end
+end
+

data/lib/cotcube-helpers/init.rb

@@ -34,7 +34,7 @@ module Cotcube
       end
 
       defaults = { 
-        data_path: '/var/cotcube/' + name,
+        data_path: '/var/cotcube/' + name.split('::').last.downcase,
       }
 
       config = defaults.merge(config)

data/lib/cotcube-helpers/orderclient.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
+module Cotcube
+  module Helpers
+    # A proxyclient is a wrapper that allows communication with cotcube-orderproxy and cotcube-dataproxy. It fulfills
+    # registration and provides the opportunity to implement the logic to respond do events.
+    # (orderproxy and dataproxy are separate gems creating a layer between tws/ib-ruby and cotcube-based
+    #  applications)
+    #
+    #  NOTE: Whats here is a provisionally version
+    #
+    class DataClient # rubocop:disable Metrics/ClassLength
+      attr_reader :power, :ticksize, :multiplier, :average, :account
+
+      # The constructor takes a lot of arguments:
+      def initialize(
+        debug: false,
+        contract: ,
+        serverport: 24001,
+        serveraddr: '127.0.0.1',
+        client:,
+        bars: true,
+        ticks: false,
+        bar_size: 5,
+        spawn_timeout: 15
+      )
+        require 'json'   unless Hash.new.respond_to? :to_json
+        require 'socket' unless defined? TCPSocket
+
+        puts 'PROXYCLIENT: Debug enabled' if @debug
+
+        @contract = contract.upcase
+        %w[debug serverport serveraddr client bars ticks bar_size].each {|var| eval("@#{var} = #{var}")}
+
+        @position     = 0
+        @account      = 0
+        @average      = 0
+
+        exit_on_startup(':client must be in range 24001..24999') if @client.nil? || (@client / 1000 != 24) || (@client == 24_000)
+
+        res = send_command({ command: 'register', contract: @contract, date: @date,
+                             ticks: @ticks, bars: @bars, bar_size: @bar_size })
+
+        # spawn_server has to be called separately after initialization. 
+        print "Waiting #{spawn_timeout} seconds on server_thread to spawn..."
+        Thread.new do 
+          begin 
+            Timeout.timeout(spawn_timeout) { sleep(0.1) while @server_thread.nil? }
+          rescue Timeout::Error
+            puts 'Could not get server_thread, has :spawn_server been called?' 
+            shutdown
+          end
+        end
+
+        unless res['error'].zero?
+          exit_on_startup("Unable to register on orderproxy, exiting")
+        end
+      end
+
+      def exit_on_startup(msg = '')
+        puts "Cannot startup client, exiting during startup: '#{msg}'"
+        shutdown
+        defined?(::IRB) ? (raise) : (exit 1)
+      end
+
+      def send_command(req)
+        req[:client_id] = @client
+        res = nil
+        puts "Connecting to #{@serveraddr}:#{@serverport} to send '#{req}'." if @debug
+
+        TCPSocket.open(@serveraddr, @serverport) do |proxy|
+          proxy.puts(req.to_json)
+          raw = proxy.gets
+          begin
+            res = JSON.parse(raw)
+          rescue StandardError
+            puts 'Error while parsing response'
+            return false
+          end
+          if @debug
+            # rubocop:disable Style/FormatStringToken, Style/FormatString
+            res.each do |k, v|
+              case v
+              when Array
+                (v.size < 2) ? puts(format '%-15s', "#{k}:") : print(format '%-15s', "#{k}:")
+                v.each_with_index { |x, i| i.zero? ? (puts x) : (puts "               #{x}") }
+              else
+                puts "#{format '%-15s', "#{k}:"}#{v}"
+              end
+            end
+            # rubocop:enable Style/FormatStringToken, Style/FormatString
+          end
+          puts "ERROR on command: #{res['msg']}" unless res['error'].nil? or res['error'].zero?
+        end
+        puts res.to_s if @debug
+        res
+      end
+
+      # #shutdown ends the @server_thread and --if :close is set-- closes the current position attached to the client
+      def shutdown(close: true)
+        return if @shutdown
+        @shutdown = true
+
+        if @position.abs.positive? && close
+          send_command({ command: 'order', action: 'create', type: 'market',
+                         side: (@position.positive? ? 'sell' : 'buy'), size: @position.abs })
+        end
+        sleep 3
+        result = send_command({ command: 'unregister' })
+        puts "FINAL ACCOUNT: #{@account}"
+        result['executions']&.each do |x|
+          x.delete('msg_type')
+          puts x.to_s
+        end
+        @server_thread.exit if @server_thread.respond_to? :exit
+      end
+
+      def spawn_server(
+        execution_proc: nil,
+        orderstate_proc: nil, 
+        tick_proc: nil,
+        depth_proc: nil, 
+        order_proc: nil, 
+        bars_proc: nil
+      )                                           # rubocop:disable Metrics/MethodLength
+
+        %w[execution_proc orderstate_proc tick_proc depth_proc order_proc bars_proc].each {|var| eval("@#{var} = #{var}") }
+
+        if @bars 
+          @bars_proc ||= lambda {|msg| puts msg.inspect }
+        end
+
+        if @ticks
+          @ticks_proc ||= lambda {|msg| puts msg.inspect }
+        end
+
+        if @shutdown
+          puts "Cannot spawn server on proxyclient that has been already shut down."
+          return
+        end
+        if @server_thread
+          puts "Cannot spawn server more than once."
+          return
+        end
+
+        @server_thread = Thread.new do                                      # rubocop:disable Metrics/BlockLength
+          puts 'Spawning RECEIVER'
+          server = TCPServer.open(@serveraddr, @client)
+          loop do                                                           # rubocop:disable Metrics/BlockLength
+            Thread.start(server.accept) do |client|                         # rubocop:disable Metrics/BlockLength
+              while (response = client.gets)
+                response = JSON.parse(response)
+
+                case response['msg_type']
+
+                when 'alert'
+                  case response['code']
+                  when 2104
+                    puts 'ALERT: data farm connection resumed   __ignored__'.light_black
+                  when 2108
+                    puts 'ALERT: data farm connection suspended __ignored__'.light_black
+                  when 2109
+                    # Order Event Warning:Attribute 'Outside Regular Trading Hours' is ignored
+                    # based on the order type and destination. PlaceOrder is now being processed.
+                    puts 'ALERT: outside_rth __ignored__'.light_black
+                  when 2100
+                    puts 'ALERT: Account_info unsubscribed __ignored__'.light_black
+                  when 202
+                    puts 'ALERT: order cancelled'
+                  else
+                    puts '-------------ALERT------------------------------'
+                    puts response.to_s
+                    puts '------------------------------------------------'
+                  end
+
+                when 'tick'
+                  @tick_proc&.call(response)
+
+                when 'depth'
+                  @depth_proc&.call(response)
+
+                when 'realtimebar'
+                  @bars_proc&.call(response)
+
+                else
+                  puts "ERROR:       #{response}"
+
+                end
+              end
+            end
+          rescue StandardError => e
+            backtrace = e.backtrace.join("\r\n")
+            puts "======= ERROR: '#{e.class}', MESSAGE: '#{e.message}'\n#{backtrace}"
+          end
+        end
+        puts '@server_thread spawned'
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity

data/lib/cotcube-helpers/symbols.rb

@@ -17,7 +17,7 @@ module Cotcube
             [ :ticksize, :power, :bcf ].each {|z| row[z] = row[z].to_f}
             row[:format] = "%#{row[:format]}f"
             row[:currency] ||= 'USD'
-            row[:multiplier] = (row[:ticksize] / row[:power]).round(8)
+            row[:multiplier] = (row[:power] / row[:ticksize]).round(8)
             row
           }
           .reject{|row| row[:id].nil? }
@@ -35,7 +35,7 @@ module Cotcube
       end
     end
 
-    def micros(config: init, symbol: nil, id: nil)
+    def micros(config: init, **args)
       if config[:micros_file].nil?
         MICRO_EXAMPLES
       else
@@ -46,13 +46,14 @@ module Cotcube
             [ :ticksize, :power, :bcf ].each {|z| row[z] = row[z].to_f }
             row[:format] = "%#{row[:format]}f"
             row[:currency] ||= 'USD'
+            row[:multiplier] = (row[:power] / row[:ticksize]).round(8)
             row
           }
           .reject{|row| row[:id].nil? }
           .tap{ |all|
             args.keys.each { |header|
              unless SYMBOL_HEADERS.include? header
-                puts "WARNING in Cotcube::Helpers.symbols: '#{header}' is not a valid symbol header. Skipping..."
+                puts "WARNING in Cotcube::Helpers.micros: '#{header}' is not a valid symbol header. Skipping..."
                 next
               end
               all.select!{|x| x[header] == args[header]} unless args[header].nil?
@@ -62,6 +63,7 @@ module Cotcube
           }
       end
     end
+
   end
 
 end

data/lib/cotcube-helpers.rb

@@ -7,6 +7,8 @@ require 'active_support/core_ext/time'
 require 'active_support/core_ext/numeric'
 require 'parallel'
 require 'csv'
+require 'yaml'
+require 'json'
 
 require_relative 'cotcube-helpers/array_ext'
 require_relative 'cotcube-helpers/enum_ext'
@@ -19,6 +21,7 @@ require_relative 'cotcube-helpers/subpattern'
 require_relative 'cotcube-helpers/parallelize'
 require_relative 'cotcube-helpers/simple_output'
 require_relative 'cotcube-helpers/simple_series_stats'
+require_relative 'cotcube-helpers/deep_decode_datetime'
 require_relative 'cotcube-helpers/constants'
 require_relative 'cotcube-helpers/input'
 require_relative 'cotcube-helpers/output'
@@ -26,6 +29,7 @@ require_relative 'cotcube-helpers/reduce'
 require_relative 'cotcube-helpers/symbols'
 require_relative 'cotcube-helpers/init'
 require_relative 'cotcube-helpers/get_id_set'
+require_relative 'cotcube-helpers/ib_contracts'
 require_relative 'cotcube-helpers/recognition'
 
 module Cotcube
@@ -40,8 +44,14 @@ module Cotcube
                     :symbols,
                     :micros,
                     :get_id_set,
+                    :get_ib_contract,
+                    :update_ib_contracts,
+                    :translate_ib_contract,
                     :init
 
     # please not that module_functions of source provided in private files must be published there
   end
 end
+
+require_relative 'cotcube-helpers/data_client'
+require_relative 'cotcube-helpers/cache_client'

data/scripts/collect_market_depth

@@ -0,0 +1,103 @@
+#!/usr/bin/env ruby
+
+require 'bundler'
+Bundler.require
+require 'parallel'
+require_relative '../lib/cotcube-helpers'
+
+dc = Cotcube::Helpers::DataClient.new
+
+dc.start_persistent(type: :depth, contract: 'GCZ21') {|msg| p msg.size; p msg.map{|z| z[:size]}.reduce(&:+) }
+
+loop do
+ sleep 1
+end
+
+__END__
+  depthThread = Thread.new do 
+    begin
+      loop do
+        sleep 0.025 while depthQueue.empty?
+        while not depthQueue.empty?
+          msg = depthQueue.pop
+          if msg.data[:operation] == 2
+            data = [ (Time.now.to_f % 100000).round(3), :d, msg.data.values_at(:operation, :side, :position) ].flatten
+          else
+            data = [ (Time.now.to_f % 100000).round(3), :d, msg.data.values_at(:operation, :side, :position, :size, :price) ].flatten
+          end
+          puts "#{data}" if data[2]!=1 or data[4] == 0
+          next
+          writeQueue << data
+          side  = msg.data[:side]
+          price = msg.data[:price] 
+          size  = msg.data[:size]
+          pos   = msg.data[:position]
+          case msg.data[:operation]
+          when 0 # insert
+            orderbook[side].insert(pos, { price: price.to_f, size: size })
+          when 1 # update
+            orderbook[side][pos] = { price: price.to_f, size: size }
+          when 2 # remove
+            orderbook[side].delete_at(pos)
+          end
+          orderbook[1].shift while orderbook[1].size > DEPTH
+          orderbook[0].shift while orderbook[0].size > DEPTH
+          a = orderbook[0].size
+          a.times do |n|
+            s = a - 1
+            ask = orderbook[0][s-n]
+            next if ask.nil? 
+            allasks = orderbook[0][0..s-n].map{|x| x[:size]}.reduce(:+)
+            asksacc = orderbook[0][0..s-n].map{|x| x[:size] * x[:price]}.reduce(:+) / allasks        
+            puts "\t\t\t\t#{format % ask[:price]}  x #{'% 5d' % ask[:size]}\t#{'% 4d' % allasks}\t#{format % asksacc}"
+          end
+          allasks = orderbook[0].compact.map{|x| x[:size]}.reduce(:+)
+          asksacc = orderbook[0].compact.map{|x| x[:size] * x[:price]}.reduce(:+) / allasks unless orderbook[0].empty?
+          allbids = orderbook[1].compact.map{|x| x[:size]}.reduce(:+)
+          bidsacc = orderbook[1].compact.map{|x| x[:size] * x[:price]}.reduce(:+) / allbids unless orderbook[0].empty?
+          puts "#{(format % bidsacc) unless bidsacc.nil?}\t".light_red +
+            "#{('% 4d' % allbids) unless allbids.nil?}\t\t\t\t\t".light_red +
+            "#{"#{'% 5d' % allasks}" unless allasks.nil?}\t".light_red +
+            "#{"#{format % asksacc}" unless asksacc.nil?}".light_red 
+          b = orderbook[1].size
+          b.times do |n| 
+            bid = orderbook[1][n]
+            next if bid.nil?
+            allbids = orderbook[1][0..n].map{|x| x[:size]}.reduce(:+)
+            bidsacc = orderbook[1][0..n].map{|x| x[:size] * x[:price]}.reduce(:+) / allbids
+            puts "#{format % bidsacc}\t#{'% 4d' % allbids}\t#{'%5d' % bid[:size]} x #{format % bid[:price]}"
+          end
+          puts "="*50
+        end
+      end
+    rescue
+      puts "RESCUE in depthThread".light_red
+      puts "#{orderbook}"
+      raise
+    end
+  end
+
+  sleep 0.01 while Time.now.to_i % WRITE_INTERVAL != 0
+  loop do
+    t = Time.now.to_f 
+    unless writeQueue.empty?
+      data = [] 
+      data << writeQueue.pop while not writeQueue.empty?
+      CSV.open(OUTFILE, "a+") { |csv| data.each {|x| csv << x } }
+    end
+    begin
+      sleep WRITE_INTERVAL - (Time.now.to_f - t)
+    rescue 
+      sleep 3
+    end
+  end
+
+
+ensure
+  ib.send_message :CancelMarketDepth, id: ID
+  ib.send_message :CancelMarketData, id: ID
+  tickThread.kill
+  depthThread.kill
+end
+sleep 1
+puts "Done."

data/scripts/cron_ruby_wrapper.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-export rubyenv=/home/pepe/.rvm/environments/default
+export rubyenv=/home/pepe/.rvm/environments/ruby-2.7.5
 
 . $rubyenv 
 cd /home/pepe/GEMS/${1}
@@ -10,6 +10,7 @@ ruby ${2} ${3} ${4} ${5} ${6}
 
  
 exit
+
 for testing run 
 env - `cat /home/pepe/bin/cron_ruby_wrapper.sh | tail -n 6` /bin/bash
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cotcube-helpers
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2.5
 platform: ruby
 authors:
 - Benjamin L. Tischendorf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-07 00:00:00.000000000 Z
+date: 2021-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6'
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -96,14 +96,20 @@ files:
 - cotcube-helpers.gemspec
 - lib/cotcube-helpers.rb
 - lib/cotcube-helpers/array_ext.rb
+- lib/cotcube-helpers/cache_client.rb
 - lib/cotcube-helpers/constants.rb
+- lib/cotcube-helpers/data_client.rb
 - lib/cotcube-helpers/datetime_ext.rb
+- lib/cotcube-helpers/deep_decode_datetime.rb
 - lib/cotcube-helpers/enum_ext.rb
+- lib/cotcube-helpers/expiration.rb
 - lib/cotcube-helpers/get_id_set.rb
 - lib/cotcube-helpers/hash_ext.rb
+- lib/cotcube-helpers/ib_contracts.rb
 - lib/cotcube-helpers/init.rb
 - lib/cotcube-helpers/input.rb
 - lib/cotcube-helpers/numeric_ext.rb
+- lib/cotcube-helpers/orderclient.rb
 - lib/cotcube-helpers/output.rb
 - lib/cotcube-helpers/parallelize.rb
 - lib/cotcube-helpers/range_ext.rb
@@ -117,6 +123,7 @@ files:
 - lib/cotcube-helpers/swig/fill_x.rb
 - lib/cotcube-helpers/swig/recognition.rb
 - lib/cotcube-helpers/symbols.rb
+- scripts/collect_market_depth
 - scripts/cron_ruby_wrapper.sh
 - scripts/symbols
 homepage: https://github.com/donkeybridge/cotcube-helpers