cotcube-dataproxy 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9edcf5e59cffe3a7dbb7a03c8896da6cd98ecb288dfc648297e06c64bd0b3314
+  data.tar.gz: 3701e4dc24d822c7349015ff4d3e5bcd28dea57d126905016706c276ee088136
+SHA512:
+  metadata.gz: cf474143f34f1d81f148f1a72986d1c575f3407a19e4494c9fa6e0e08be18b90bf5057bf4f248667a6bcbd9812e32d9cb6e75a75343e05c765dd7ab9cc5c6ddf
+  data.tar.gz: 14599b72e780bcf9a942f70448ed1fdaf5ee9aad649b30c3701f056802fb3ea44275d6c47dee5e05da94eb8ed23620d393e14ec96c8ed347b33ad54c2f8db640

data/CHANGELOG.md

@@ -0,0 +1,18 @@
+## 0.1.1 (November 13, 2021)
+  - added handy test_script in bin/test_client.rb
+  - added httparty to Gem requirements
+  - added handy startup script to bin/dataproxy
+  - removed superflouus lines in commserver.rb'
+  - gc.rb: the garbage collector. containing gc, gc_start/_stop, api() (uses management api to get a list), get_mq ( for listings) and delete_mq (for API based deletion)
+  - commserver.rb, handling incoming commands from clients. containing commserver_start/_stop and subscribe_persistent
+  - subscribers.rb, handling of incoming IB messages. PLUS __int2hex__, just a tiny helper to translate IDs
+  - client_response.rb, containing client_success and client_fail
+  - init.rb: containing initialize, shutdown, recover and log
+  - 3rd_clients: class level helpers to create IB and RabbitMQ client objects
+  - central startup file, containing bundler, requires and constants
+  - added startup script
+  - initial commit. adding Gemfile and gemspec
+
+## 0.1.0 (November 10, 2021)
+
+

data/Gemfile

@@ -0,0 +1,11 @@
+source 'http://rubygems.org'
+gem 'activesupport'
+gem 'colorize'
+gem 'ib-api'
+gem 'ox'
+gem 'parallel'
+gem 'bunny'
+gem 'json'
+gem 'cotcube-helpers'
+gem 'outputhandler', '~> 0.2'
+gem 'httparty'

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/bin/dataproxy

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/cotcube-dataproxy'
+
+Signal.trap('TERM') { local_interrupt }
+Signal.trap('INT')  { local_interrupt }
+
+# TODO: Prepare reload config on SIGHUP
+Signal.trap('HUP')  { puts 'TODO: Reload config' }
+exiting = false
+
+dataproxy = Cotcube::DataProxy.new
+
+define_method :local_interrupt do
+  dataproxy.send(:log, "DATAPROXY Received termination request...")
+  exiting = true
+end
+
+
+begin
+  loop { exit if exiting; sleep 0.5 }
+ensure
+  dataproxy.shutdown
+end

data/bin/test_client.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+require 'cotcube-helpers'
+
+s = Cotcube::Helpers::DataClient.new
+
+print "Ping => "
+puts "#{s.send_command({ command: 'ping' })}"
+
+raw  = s.get_historical(contract: 'GCZ21', interval: :min15)
+
+result = JSON.parse(raw, symbolize_names: true)
+if result[:error].zero?
+  result[:result][-20..].each {|z| p z.slice(*%i[time open high low close volume]) } 
+else
+  puts "Some ERROR occured: #{result}"
+end
+
+# Please test this during business hours
+bars = [ ] 
+id = s.start_persistent(contract: 'GCZ21', type: :realtimebars) {|bar| puts "Got #{bar}"; bars << bar }
+sleep 20
+s.stop_persistent(contract: 'GCZ21', type: :realtimebars)
+
+puts "Received #{bars.count} bars. Exiting now..."

data/cotcube-dataproxy.gemspec

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |spec|
+  spec.name          = 'cotcube-dataproxy'
+  spec.version       = File.read("#{__dir__}/VERSION")
+  spec.authors       = ['Benjamin L. Tischendorf']
+  spec.email         = ['donkeybridge@jtown.eu']
+
+  spec.summary       = 'An AMQ based proxy to retrieve price and contract data from IBKR TWS.'
+  spec.description   = 'An AMQ based proxy to retrieve price and contract data from IBKR TWS. The AMQ is provided by rabbitMQ with the gem Bunny, the API to TWS/ibGateway with the gem ib-api'
+
+  spec.homepage      = 'https://github.com/donkeybridge/' + spec.name
+  spec.license       = 'BSD-3-Clause'
+  spec.required_ruby_version = Gem::Requirement.new('~> 2.7')
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['changelog_uri'] = spec.homepage + '/CHANGELOG.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport',      '~> 6'
+  spec.add_dependency 'ib-api',             '~> 972'
+  spec.add_dependency 'colorize',           '~> 0.8'
+  spec.add_dependency 'cotcube-helpers',    '~> 0.2.1'
+  spec.add_dependency 'outputhandler',      '~> 0.2'
+  spec.add_dependency 'bunny',              '~> 2'
+  spec.add_dependency 'httparty'            '~> 0.20'
+  spec.add_dependency 'yaml',               '~> 0.1'
+  spec.add_dependency 'json',               '~> 2'
+
+  spec.add_development_dependency 'rake',   '~> 13'
+  spec.add_development_dependency 'rspec', '~>3.6'
+  spec.add_development_dependency 'yard', '~>0.9'
+end

data/lib/cotcube-dataproxy/3rd_clients.rb

@@ -0,0 +1,65 @@
+module Cotcube
+  class DataProxy
+
+    # Create a connection to the locally running
+    def self.get_ib_client(host: 'localhost', port: 4002, id: 5, client_id: 5)
+      obj = {
+        id: id,
+        client_id: client_id,
+        port: port,
+        host: host
+      }
+      begin
+        obj[:ib] = IB::Connection.new(
+          id: id,
+          client_id: client_id,
+          port: port,
+          host: host
+        ) do |provider|
+          obj[:alert]            = provider.subscribe(:Alert) { true }
+          obj[:managed_accounts] = provider.subscribe(:ManagedAccounts) { true }
+        end
+        obj[:error] = 0
+      rescue Exception => e
+        obj[:error] = 1
+        obj[:message] = e.message
+        obj[:full_message] = e.full_message
+      end
+      obj
+    end
+
+
+    def self.get_mq_client(client_id: 5)
+      obj = {
+        client_id: client_id,
+      }
+      begin
+        # for more info on connection parameters see http://rubybunny.info/articles/connecting.html
+        #
+        obj[:connection]    = Bunny.new(
+          host: 'localhost',
+          port: 5672,
+          user: SECRETS['dataproxy_mq_user'],
+          password: SECRETS['dataproxy_mq_password'],
+          vhost: SECRETS['dataproxy_mq_vhost']
+        )
+        obj[:connection].start
+        obj[:commands]      = obj[:connection].create_channel
+        obj[:channel]       = obj[:connection].create_channel
+        obj[:request_queue] = obj[:commands].queue('', exclusive: true, auto_delete: true)
+        obj[:request_exch]  = obj[:commands].direct('dataproxy_commands', exclusive: true, auto_delete: true)
+        obj[:replies_exch]  = obj[:commands].direct('dataproxy_replies', auto_delete: true)
+        %w[ dataproxy_commands  ].each do |key|
+          obj[:request_queue].bind(obj[:request_exch], routing_key: key )
+        end
+        obj[:error]      = 0
+      rescue Exception => e
+        obj[:error] = 1
+        obj[:message] = e.message
+        obj[:full_message] = e.full_message
+      end
+      obj
+    end
+
+  end
+end

data/lib/cotcube-dataproxy/client_response.rb

@@ -0,0 +1,46 @@
+module Cotcube
+  class DataProxy
+
+    def client_success(request, id: nil, to: nil, exchange: :replies_exch, &block)
+      client_response( request, id: nil, to: nil, err: 0, exchange: exchange, &block)
+    end
+
+    def client_fail(request,    id: nil, to: nil, exchange: :replies_exch, &block)
+      client_response(request,  id: nil, to: nil, err: 1, exchange: exchange, &block)
+    end
+
+    private
+
+    def client_response(request, id: nil, to: nil, err:, exchange: :replies_exch)
+      __id__ = id.presence || request.delete(:__id__)
+      __to__ = to.presence || request.delete(:__to__)
+      msg = yield
+      case msg
+      when String
+        response = { error: err, msg: msg }
+      when Hash
+        response = { error: err }
+        msg.each { |k, v| response[k] = v }
+      when Array
+        response = { error: err, result: msg }
+      else
+        response = { error: 1, msg: "Processing failed for '#{msg.inspect}' after '#{request}'." }
+      end
+      if response[:error] == 1
+        log "CLIENT #{id} FAILIURE:      #{response.inspect}.".colorize(:light_red)
+      elsif response[:result].is_a?(Array)
+        log "CLIENT #{id} SUCCESS:       sent #{response[:result].size} datasets."
+      else
+        log "CLIENT #{id} SUCCESS:       #{response.to_s.scan(/.{1,120}/).join(' '*30 + "\n")}"
+      end
+      mq[exchange].publish(
+        response.to_json,
+        content_type:    'application/json',
+        priority: 7,
+        correlation_id:  __id__,
+        routing_key:     __to__,
+        reply_to:        __id__
+      )
+    end
+  end
+end

data/lib/cotcube-dataproxy/commserver.rb

@@ -0,0 +1,185 @@
+# frozen_string_literal: true
+
+module Cotcube
+  class DataProxy
+
+    def commserver_start
+      mq[:request_subscription] = mq[:request_queue].subscribe do |delivery_info, properties, payload|
+
+        ################################################################################################
+        # the request will be JSON decoded. The generic command 'failed' will be set, if decoding raises.
+        # furthermore, __id__ and __to__ are extracted and added to the request-hash
+        #
+        request    = JSON.parse(payload, symbolize_names: true) rescue { command: 'failed' }
+        request[:command] ||= 'nil'
+
+        request[:__id__] = properties[:correlation_id]
+        request[:__to__] = properties[:reply_to]
+
+        if request[:debug] 
+          log "Received \t#{delivery_info.map{|k,v| "#{k}\t#{v}"}.join("\n")
+                     }\n\n#{properties   .map{|k,v| "#{k}\t#{v}"}.join("\n")
+                     }\n\n#{request      .map{|k,v| "#{k}\t#{v}"}.join("\n")}" if request[:debug]
+        else
+          log "Received\t#{request}"
+        end
+
+        ###############################################################################################
+        # the entire set of command processing,
+        # starting with the (generic) 'failed' command, that just answers with the failure notice
+        # and      with another failure notice upon a missing command section in the request
+        # ending   with another failure notice, if an unknown command was issued
+        #
+        log "Processing #{request[:command]}:"
+        case request[:command].downcase
+        when 'failed'
+          client_fail(request) { "Failed to parse payload: '#{payload}'." }
+
+        when 'nil'
+          client_fail(request) { "missing :command in request: '#{request}'." }
+
+        ##############################################################################################
+        # ping -> pong, just for testing
+        # 
+        when 'ping'
+          client_success(request) { "pong" }
+
+        ##############################################################################################
+        # the get_contracts command tries to resolve a list of available contracts related to a
+        #     specific symbol based on a set of characteristics retrieved via Herlpers::get_id_set
+        #
+        # the reply of the message is processed asynchroniously upon reception of
+        #     the IB Message 'ContractDataEnd' in the message subscribers section
+        # 
+        when 'get_contract', 'get_contracts', Cotcube::Helpers.sub(minimum: 3) { 'contracts' }
+          if request[:symbol].nil? 
+            client_fail(request) { "Cannot requets contracts without :symbol (in '#{request}')." }
+            next
+          end
+          sym = Cotcube::Helpers.get_id_set(symbol: request[:symbol])
+          if [nil, false].include? sym
+            client_fail(request) { "Unknown symbol '#{request[:symbol]}' in '#{request}'." }
+            next
+          end
+          request[:result] = [] 
+          req_mon.synchronize { requests[request[:__id__]] = request }
+          ib_contract = IB::Contract.new symbol: sym[:ib_symbol], exchange: sym[:exchange], currency: sym[:currency], sec_type: (request[:sec_type] || 'FUT') 
+          ib.send_message :RequestContractData, contract: ib_contract, request_id: request[:__id__].to_i(16)
+
+
+        ##############################################################################################
+        # the historical command retrieves a list of bars as provided by TWS
+        # the minimum requirement is :contract parameter issued.
+        # 
+        # the reply to this message is processed asynchroniously upon reception of
+        #     the IB message 'HistoricalData' in message subscribers section
+        #
+        when Cotcube::Helpers.sub(minimum: 3) {'historical'}
+          con_id = request[:con_id] || Cotcube::Helpers.get_ib_contract(request[:contract])[:con_id] rescue nil
+          if con_id.nil? or request[:contract].nil? 
+             client_fail(request) { "Cannot get :con_id for contract:'#{request[:contract]}' in '#{request}'." } 
+             next
+          end
+          sym    = Cotcube::Helpers.get_id_set(contract: request[:contract])
+          before = Time.at(request[:before]).to_ib rescue Time.now.to_ib
+          ib_contract = IB::Contract.new(con_id: con_id, exchange: sym[:exchange])
+          req = {
+            request_id:    request[:__id__].to_i(16),
+            contract:      ib_contract,
+            end_date_time: before,
+            what_to_show:  (request[:based_on]                || :trades),
+            use_rth:       (request[:rth_only]                     || 1),
+            keep_up_to_date: 0,
+            duration:      (request[:duration].gsub('_', ' ') || '1 D'),
+            bar_size:      (request[:interval].to_sym         || :min15)
+          }
+          req_mon.synchronize { requests[request[:__id__]] = request }
+          begin
+            Timeout.timeout(2) { ib.send_message(IB::Messages::Outgoing::RequestHistoricalData.new(req)) }
+          rescue Timeout::Error, IB::Error
+            client_fail(request) { 'Could not request historical data. Is ib_client running?' }
+            req_mon.synchronize { requests.delete(request[:__id__]) } 
+            next
+          end
+
+
+        # ********************************************************************************************
+        #
+        # REQUESTS BELOW ARE BASED ON A cONTINUOUS IB SUBSCRIPTION AND MUST BE CONSIDERED
+        # GARBAGE COLLECTION ( instance.gc ) --- SUBSCRIPTION DATA IS PERSISTET IN @persistent
+        #
+        # ********************************************************************************************
+
+
+        ###############################################################################################
+        # the start_realtimebars initiates the IBKR realtime (5s) bars delivery for a specific contract
+        #     and feeds them into a fanout exchange dedicated to that contract
+        #     delivery continues as long as there are queues bound to that exchange
+        # 
+        when Cotcube::Helpers.sub(minimum:4){'realtimebars'}
+          subscribe_persistent(request, type: :realtimebars)
+          next
+
+        when 'ticks'
+          subscribe_persistent(request, type: :realtimebars)
+          next
+
+        when 'depth'
+          subscribe_persistent(request, type: :depth)
+          next
+
+        else
+          client_fail(request) { "Unknown :command '#{request[:command]}' in '#{request}'." }
+        end
+      end
+      log "Started commserver listening on #{mq[:request_queue]}"
+    end
+
+    def commserver_stop
+      mq[:request_subscription].cancel
+      log "Stopped commserver ..."
+    end
+
+    def subscribe_persistent(request, type:)
+      sym    = Cotcube::Helpers.get_id_set(contract: request[:contract])
+      con_id = request[:con_id] || Cotcube::Helpers.get_ib_contract(request[:contract])[:con_id] rescue nil
+      if sym.nil? or con_id.nil?
+        client_fail(request) { "Invalid contract '#{request[:contract]}'." }
+        return
+      end
+      if persistent[type][con_id].nil?
+        per_mon.synchronize {
+          persistent[type][con_id] = { con_id: con_id,
+                                       contract: request[:contract],
+                                       exchange: mq[:channel].fanout(request[:exchange]) }
+        }
+        if type == :depth
+          bufferthread = Thread.new do
+            sleep 5.0 - (Time.now.to_f % 5)
+            loop do
+              begin
+                @block_depth_queue = true
+                sleep 0.025
+                con = persistent[:depth][con_id]
+                con[:exchange].publish(con[:buffer].to_json)
+                con[:buffer] = []
+                @block_depth_queue = false
+              end
+              sleep 5.0 - (Time.now.to_f % 5)
+            end
+          end
+          per_mon.synchronize { persistent[:depth][con_id][:bufferthread] = bufferthread }
+        end
+        ib_contract = IB::Contract.new(con_id: con_id, exchange: sym[:exchange])
+        ib.send_message(REQUEST_TYPES[type], id: con_id, contract: ib_contract, data_type: :trades, use_rth: false)
+        client_success(request) { "Delivery of #{type} of #{request[:contract]} started." }
+      elsif persistent[type][con_id][:on_cancel]
+        client_fail(request) { { reason: :on_cancel, message: "Exchange '#{requst[:exchange]}' is marked for cancel, retry in a few seconds to recreate" } }
+      else
+        client_success(request) { "Delivery of #{type} of #{request[:contract]} attached to existing." }
+      end
+    end
+
+
+  end
+end

data/lib/cotcube-dataproxy/gc.rb

@@ -0,0 +1,157 @@
+# about garbage collection, i.e. periodically check for unused artifacts and clean them 
+#
+module Cotcube
+  class DataProxy
+
+    def gc_start
+      if gc_thread.nil?
+        @gc_thread = Thread.new do 
+          loop do
+            sleep 30 + 2 * Random.rand
+            gc
+          end
+        end
+        log 'GC_INFO: GC spawned.'
+      else
+        log 'GC_ERROR: Cannot start GC_THREAD more than once.'
+      end
+    end
+
+    def gc_stop
+      if gc_thread.nil?
+        log 'GC_ERROR: Cannot stop nonexisting GC_THREAD.'
+        false
+      else
+        gc_thread.kill
+        gc_thread = nil
+        log 'GC_INFO: GC stopped.'
+        true
+      end
+    end
+
+    def gc
+      get_mq(list: false).each do |item_type, items|
+        items.each do |key, item| 
+          case item_type
+          when :bindings
+            # NOTE if might be considerable to unbind unused queues from exchanges before removing either
+            #      as the bound value of the exchange will decrease it might be removed on next run of GC
+          when :queues
+            next if item[:consumers].nil? or item[:consumers].positive?
+            delete_mq(type: item_type, instance: item[:name]) 
+            log "GC_INFO: Deleted unsed queue: #{item}."
+            # sadly we don't know about unsed queues. basically this means that some client did not declare its queue as auto_delete.
+            # the dateproxy itself has only 1 queue -- the command queue, everything else is send out to exchanges
+          when :exchanges
+            if item[:name].empty? or item[:name] =~ /^amq\./ or item[:bound].count.positive? or 
+                %w[ dataproxy_commands dataproxy_replies ].include? item[:name]
+              next
+            end
+            #log "GC_INFO: found superfluous exchange '#{item[:name]}'"
+
+            _, subscription_type, contract = item[:name].split('_')
+            unless %w[ ticks depth realtimebars ].include? subscription_type.downcase
+              puts "GC_WARNING: Unknown subscription_type '#{subscription_type}', skipping..."
+              next
+            end
+            con_id = Cotcube::Helpers.get_ib_contract(contract)[:con_id] rescue 0
+            if con_id.zero?
+              puts "GC_WARNING: No con_id found for contract '#{contract}', skipping..."
+              next
+            end
+            if persistent[subscription_type.to_sym][con_id].nil?
+              puts "GC_WARNING: No record for subscription '#{subscription_type}_#{contract}' with #{con_id} found, deleting anyway..."
+            end
+            Thread.new do
+              per_mon.synchronize { 
+                persistent[subscription_type.to_sym][con_id][:on_cancel] = true if persistent.dig(subscription_type.to_sym, con_id)
+              } 
+              log "GC_INFO: Sending cancel for #{subscription_type}::#{contract}::#{con_id}."
+              message_type = case subscription_type;
+                             when 'ticks'; :CancelMarketData
+                             when 'depth'; :CancelMarketDepth
+                             else;         :CancelRealTimeBars
+                             end
+              if ib.send_message( message_type, id: con_id )
+                sleep 0.75 + Random.rand
+                res = delete_mq(type: item_type, instance: item[:name])
+                log "GC_SUCCESS: exchange '#{item[:name]}' with #{con_id} has been deleted ('#{res}')."
+                per_mon.synchronize { persistent[subscription_type.to_sym].delete(con_id) }
+              else
+                log "GC_FAILED: something went wrong unsubscribing '#{subscription_type}_#{contract}' with #{con_id}."
+              end
+            end
+          else
+            log "GC_ERROR: Unexpected type '#{item_type}' in GarbageCollector"
+          end
+        end
+      end
+    end
+
+    def get_mq(list: true)
+      bindings = api(type: :bindings) 
+      results  = {} 
+      %i[ queues exchanges ].each do |type|
+        results[type] = {}  
+        items = api type: type
+        items.each do |item| 
+          query_name = item[:name].empty?  ? 'amq.default' : item[:name]
+          results[type][item[:name]] = api type: type, instance: query_name
+          results[type][item[:name]][:bound] = bindings.select{|z| z[:source] == item[:name] }.map{|z| z[:destination]} if type == :exchanges
+        end
+      end
+      results.each do |type, items|
+        items.each do |key,item|
+          if item.is_a? Array
+            puts "#{key}\t\t#{item}"
+            next
+          end
+          puts "#{format '%12s', type.to_s.upcase}    #{
+          case type
+          when :queues 
+            "Key: #{format '%-30s', key}  Cons: #{item[:consumers]}"
+          when :exchanges
+            "Key: #{format '%-30s', key}  Type: #{format '%10s', item[:type]}   Bound: #{item[:bound].presence || 'None.'}"
+          else
+            "Unknown details for #{type}"
+          end
+          }" 
+        end
+      end if list
+      results[:bindings] = bindings
+      results
+    end
+
+    def delete_mq(type:, instance: )
+      allowed_types = %i[ queues exchanges ]
+      raise ArgumentError, "Type must be in '#{allowed_types}', but is  '#{type}'" unless allowed_types.include? type
+      result = HTTParty.delete("#{SECRETS['dataproxy_mq_proto']
+                            }://#{SECRETS['dataproxy_mq_user']
+                              }:#{SECRETS['dataproxy_mq_password']
+                              }@#{SECRETS['dataproxy_mq_host']
+                              }:#{SECRETS['dataproxy_mq_port']
+                          }/api/#{type.to_s
+                           }/dp/#{instance}", query: { 'if-unused' => true })
+      if result.code == 204
+        result = { error: 0}
+      else
+        result = JSON.parse(result.body, symbolize_names: true)
+        result[:error] = 1
+      end
+      result
+    end
+
+    def api(type:, instance: nil)
+      allowed_types = %i[ queues exchanges bindings ] # other types need different API sepc: channels connections definitions
+      raise ArgumentError, "Type must be in '#{allowed_types}', but is  '#{type}'" unless allowed_types.include? type
+      req = "#{type.to_s}/#{SECRETS['dataproxy_mq_vhost']}#{instance.nil? ? '' : "/#{instance}"}"
+      JSON.parse(HTTParty.get("#{SECRETS['dataproxy_mq_proto']
+                           }://#{SECRETS['dataproxy_mq_user']
+                             }:#{SECRETS['dataproxy_mq_password']
+                             }@#{SECRETS['dataproxy_mq_host']
+                             }:#{SECRETS['dataproxy_mq_port']
+                         }/api/#{req}").body, symbolize_names: true)
+    end
+
+  end
+end

data/lib/cotcube-dataproxy/init.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Cotcube
+  # top-level class documentation comment
+  class DataProxy
+
+    def initialize(
+      outputhandler: OutputHandler.new(
+        location: "/var/cotcube/log/dataproxy"
+      )
+    )
+      @output = outputhandler
+      @client = DataProxy.get_ib_client
+      @mq     = DataProxy.get_mq_client
+      @ib     = @client[:ib]
+      raise 'Could not connect to IB' unless @ib
+      raise 'Could not connect to RabbitMQ' if %i[ request_exch replies_exch request_queue ].map{|z| mq[z].nil? }.reduce(:|)
+      @requests = {}
+      @req_mon  = Monitor.new
+      @persistent = { ticks: {}, depth: {}, realtimebars: {} }
+      @per_mon  = Monitor.new
+      @gc_thread = nil
+      spawn_message_subscribers
+      commserver_start
+      recover
+      gc_start
+    end
+
+    def shutdown
+      puts "Shutting down dataproxy."
+      commserver_stop
+      gc_stop
+      mq[:commands].close
+      mq[:channel].close
+      mq[:connection].close
+      persistent.each do |type, items|
+        items.each do |con_id, item|
+          log "sending #{ CANCEL_TYPES[type.to_sym]} #{con_id} (for #{item[:contract]})"
+          ib.send_message CANCEL_TYPES[type.to_sym], id: con_id
+        end
+      end
+      sleep 1
+      gc
+      sleep 1
+      ib.close
+      puts "... done."
+    end
+
+    private 
+    attr_reader :client, :clients, :ib, :mq, :requests, :req_mon, :persistent, :per_mon, :gc_thread
+
+    def recover
+      get_mq(list: false)[:exchanges].keys.select{|z| z.split('_').size == 3 }.each do |exch|
+        src, type, contract = exch.split('_')
+        next unless src == 'dataproxy'
+        next unless %w[ ticks depth realtimebars ].include? type.downcase
+        puts "Found #{exch} to recover."
+        subscribe_persistent( { contract: contract, exchange: exch }, type: type.to_sym )
+      end
+    end
+
+    def log(msg)
+      @output.puts "#{DateTime.now.strftime('%Y%m%d-%H:%M:%S:  ')}#{msg.to_s.scan(/.{1,120}/).join("\n" + ' ' * 20)}"
+    end
+
+  end
+end
+
+__END__

data/lib/cotcube-dataproxy/subscribers.rb

@@ -0,0 +1,115 @@
+module Cotcube
+  class DataProxy
+
+    def __int2hex__(id)
+      tmp = id.to_s(16) rescue nil
+      return nil if tmp.nil?
+      tmp.prepend('0') while tmp.length < 7
+      return tmp
+    end
+    
+    def spawn_message_subscribers
+      @msg_queue = Queue.new
+      @depth_queue = Queue.new
+
+      ib.subscribe(:MarketDataType, :TickRequestParameters){|msg| log "#{msg.class}\t#{msg.data.inspect}".colorize(:yellow) }
+
+      @msg_subscriber = ib.subscribe(
+        :Alert,
+        :ContractData, :ContractDataEnd, :BondContractData,
+        :TickGeneric, :TickString, :TickPrice, :TickSize,
+        :HistoricalData,
+        :RealTimeBar
+      ) do |msg|
+
+        @msg_queue << msg
+      end
+
+      @depth_subscriber = ib.subscribe( :MarketDepth ) {|msg| @depth_queue << msg}
+
+      @msg_subscriber_thread = Thread.new do
+        loop do
+          msg = @msg_queue.pop
+
+          data             = msg.data
+          data[:time]      = msg.created_at.strftime('%H:%M:%S')
+          data[:timestamp] = (msg.created_at.to_time.to_f * 1000).to_i
+          __id__           = __int2hex__(data[:request_id])
+
+          case msg
+
+          when IB::Messages::Incoming::HistoricalData
+            client_success(requests[__id__]) { msg.results }
+            req_mon.synchronize { requests.delete(__id__) }
+
+          when IB::Messages::Incoming::Alert # Alert
+            __id__ = __int2hex__(data[:error_id])
+            case data[:code]
+            when 162
+              log("ALERT 162:".light_red + ' MISSING MARKET DATA PERMISSION')
+            when 201
+              log("ALERT 201:".light_red + ' DUPLICATE OCA_GROUP')
+            else
+              log("ALERT #{data[:code]}:".light_red + "        #{data[:message]}")
+            end
+            data[:msg_type] = 'alert'
+            client_fail(requests[__id__]) {data} unless requests[__id__].nil?
+            log data
+
+          when IB::Messages::Incoming::ContractData
+            req_mon.synchronize do
+              requests[__id__][:result] << data[:contract].slice(:local_symbol, :last_trading_day, :con_id) 
+            end
+
+          when IB::Messages::Incoming::ContractDataEnd
+            sleep 0.25
+            client_success(requests[__id__]) { requests[__id__][:result] }
+            req_mon.synchronize { requests.delete(__id__) }
+
+          when IB::Messages::Incoming::RealTimeBar
+            con_id    = data[:request_id]
+            bar       = data[:bar]
+            exchange  = persistent[:realtimebars][con_id][:exchange]
+            begin
+              exchange.publish(bar.slice(*%i[time open high low close volume trades wap]).to_json)
+            rescue Bunny::ChannelAlreadyClosed
+              ib.send_message :CancelRealTimeBars, id: con_id
+              log "Delivery for #{persistent[:realtimebars][con_id][:contract] rescue 'unknown contract' 
+                  } with con_id #{con_id} has been stopped." 
+              Thread.new{ sleep 5; per_mon.synchronize { persistent[:realtimebars].delete(con_id) } }
+            end
+
+          when IB::Messages::Incoming::TickSize,
+            IB::Messages::Incoming::TickPrice,
+            IB::Messages::Incoming::TickGeneric,
+            IB::Messages::Incoming::TickString
+            con_id    = data[:ticker_id] 
+            contract  = persistent[:ticks][con_id][:contract]
+            exchange  = persistent[:ticks][con_id][:exchange]
+            begin
+              exchange.publish(data.inspect.to_json)
+            rescue Bunny::ChannelAlreadyClosed
+              ib.send_message :CancelMarketData, id: con_id
+              log "Delivery for #{persistent[:ticks][con_id][:contract]} with con_id #{con_id} has been stopped."
+              Thread.new{ sleep 0.25; per_mon.synchronize { persistent[:ticks].delete(con_id) } }
+            end
+
+          else
+            log("WARNING".light_red + "\tUnknown messagetype: #{msg.inspect}")
+          end
+        end
+        log "SPAWN_SUBSCRIBERS\tSubscribers attached to IB" if @debug
+      end
+      @depth_subscriber_thread = Thread.new do
+        loop do
+          sleep 0.025 while @block_depth_queue
+          msg = @depth_queue.pop
+          con_id = msg.data[:request_id] 
+          msg[:contract] =  persistent[:depth][con_id][:contract]
+          persistent[:depth][con_id][:buffer] << msg.data.slice(*%i[ contract position operation side price size ])
+        end
+      end
+    end
+  end
+end
+

data/lib/cotcube-dataproxy.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'bundler'
+Bundler.require
+require_relative './cotcube-dataproxy/3rd_clients'
+require_relative './cotcube-dataproxy/init'
+require_relative './cotcube-dataproxy/client_response'
+require_relative './cotcube-dataproxy/subscribers'
+require_relative './cotcube-dataproxy/commserver'
+require_relative './cotcube-dataproxy/gc'
+
+SECRETS_DEFAULT = { 
+  'dataproxy_mq_proto'    => 'http',
+  'dataproxy_mq_user'     => 'guest', 
+  'dataproxy_mq_password' => 'guest',
+  'dataproxy_mq_host'     => 'localhost',
+  'dataproxy_mq_port'     => '15672',
+  'dataproxy_mq_vhost'    => '%2F'
+}
+
+# Load a yaml file containing actual parameter and merge those with current
+# TODO use better config file location
+SECRETS = SECRETS_DEFAULT.merge( -> {YAML.load(File.read("#{__FILE__.split('/')[..-2].join('/')}/../secrets.yml")) rescue {} }.call)
+
+CANCEL_TYPES = {
+  ticks: :CancelMarketData,
+  depth: :CancelMarketDepth,
+  realtimebars: :CancelRealTimeBars
+}
+
+REQUEST_TYPES = { 
+  ticks: :RequestMarketData,
+  depth: :RequestMarketDepth,
+  realtimebars: :RequestRealTimeBars
+}
+

metadata

@@ -0,0 +1,229 @@
+--- !ruby/object:Gem::Specification
+name: cotcube-dataproxy
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Benjamin L. Tischendorf
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-11-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6'
+- !ruby/object:Gem::Dependency
+  name: ib-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '972'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '972'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: cotcube-helpers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.1
+- !ruby/object:Gem::Dependency
+  name: outputhandler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: httparty~> 0.20
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yaml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: An AMQ based proxy to retrieve price and contract data from IBKR TWS.
+  The AMQ is provided by rabbitMQ with the gem Bunny, the API to TWS/ibGateway with
+  the gem ib-api
+email:
+- donkeybridge@jtown.eu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- VERSION
+- bin/dataproxy
+- bin/test_client.rb
+- cotcube-dataproxy.gemspec
+- lib/cotcube-dataproxy.rb
+- lib/cotcube-dataproxy/3rd_clients.rb
+- lib/cotcube-dataproxy/client_response.rb
+- lib/cotcube-dataproxy/commserver.rb
+- lib/cotcube-dataproxy/gc.rb
+- lib/cotcube-dataproxy/init.rb
+- lib/cotcube-dataproxy/subscribers.rb
+homepage: https://github.com/donkeybridge/cotcube-dataproxy
+licenses:
+- BSD-3-Clause
+metadata:
+  homepage_uri: https://github.com/donkeybridge/cotcube-dataproxy
+  source_code_uri: https://github.com/donkeybridge/cotcube-dataproxy
+  changelog_uri: https://github.com/donkeybridge/cotcube-dataproxy/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.6
+signing_key: 
+specification_version: 4
+summary: An AMQ based proxy to retrieve price and contract data from IBKR TWS.
+test_files: []