cotcube-dataproxy 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 9edcf5e59cffe3a7dbb7a03c8896da6cd98ecb288dfc648297e06c64bd0b3314
4
+ data.tar.gz: 3701e4dc24d822c7349015ff4d3e5bcd28dea57d126905016706c276ee088136
5
+ SHA512:
6
+ metadata.gz: cf474143f34f1d81f148f1a72986d1c575f3407a19e4494c9fa6e0e08be18b90bf5057bf4f248667a6bcbd9812e32d9cb6e75a75343e05c765dd7ab9cc5c6ddf
7
+ data.tar.gz: 14599b72e780bcf9a942f70448ed1fdaf5ee9aad649b30c3701f056802fb3ea44275d6c47dee5e05da94eb8ed23620d393e14ec96c8ed347b33ad54c2f8db640
data/CHANGELOG.md ADDED
@@ -0,0 +1,18 @@
1
+ ## 0.1.1 (November 13, 2021)
2
+ - added handy test_script in bin/test_client.rb
3
+ - added httparty to Gem requirements
4
+ - added handy startup script to bin/dataproxy
5
+ - removed superflouus lines in commserver.rb'
6
+ - gc.rb: the garbage collector. containing gc, gc_start/_stop, api() (uses management api to get a list), get_mq ( for listings) and delete_mq (for API based deletion)
7
+ - commserver.rb, handling incoming commands from clients. containing commserver_start/_stop and subscribe_persistent
8
+ - subscribers.rb, handling of incoming IB messages. PLUS __int2hex__, just a tiny helper to translate IDs
9
+ - client_response.rb, containing client_success and client_fail
10
+ - init.rb: containing initialize, shutdown, recover and log
11
+ - 3rd_clients: class level helpers to create IB and RabbitMQ client objects
12
+ - central startup file, containing bundler, requires and constants
13
+ - added startup script
14
+ - initial commit. adding Gemfile and gemspec
15
+
16
+ ## 0.1.0 (November 10, 2021)
17
+
18
+
data/Gemfile ADDED
@@ -0,0 +1,11 @@
1
+ source 'http://rubygems.org'
2
+ gem 'activesupport'
3
+ gem 'colorize'
4
+ gem 'ib-api'
5
+ gem 'ox'
6
+ gem 'parallel'
7
+ gem 'bunny'
8
+ gem 'json'
9
+ gem 'cotcube-helpers'
10
+ gem 'outputhandler', '~> 0.2'
11
+ gem 'httparty'
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.1
data/bin/dataproxy ADDED
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative '../lib/cotcube-dataproxy'
4
+
5
+ Signal.trap('TERM') { local_interrupt }
6
+ Signal.trap('INT') { local_interrupt }
7
+
8
+ # TODO: Prepare reload config on SIGHUP
9
+ Signal.trap('HUP') { puts 'TODO: Reload config' }
10
+ exiting = false
11
+
12
+ dataproxy = Cotcube::DataProxy.new
13
+
14
+ define_method :local_interrupt do
15
+ dataproxy.send(:log, "DATAPROXY Received termination request...")
16
+ exiting = true
17
+ end
18
+
19
+
20
+ begin
21
+ loop { exit if exiting; sleep 0.5 }
22
+ ensure
23
+ dataproxy.shutdown
24
+ end
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env ruby
2
+ require 'cotcube-helpers'
3
+
4
+ s = Cotcube::Helpers::DataClient.new
5
+
6
+ print "Ping => "
7
+ puts "#{s.send_command({ command: 'ping' })}"
8
+
9
+ raw = s.get_historical(contract: 'GCZ21', interval: :min15)
10
+
11
+ result = JSON.parse(raw, symbolize_names: true)
12
+ if result[:error].zero?
13
+ result[:result][-20..].each {|z| p z.slice(*%i[time open high low close volume]) }
14
+ else
15
+ puts "Some ERROR occured: #{result}"
16
+ end
17
+
18
+ # Please test this during business hours
19
+ bars = [ ]
20
+ id = s.start_persistent(contract: 'GCZ21', type: :realtimebars) {|bar| puts "Got #{bar}"; bars << bar }
21
+ sleep 20
22
+ s.stop_persistent(contract: 'GCZ21', type: :realtimebars)
23
+
24
+ puts "Received #{bars.count} bars. Exiting now..."
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ Gem::Specification.new do |spec|
4
+ spec.name = 'cotcube-dataproxy'
5
+ spec.version = File.read("#{__dir__}/VERSION")
6
+ spec.authors = ['Benjamin L. Tischendorf']
7
+ spec.email = ['donkeybridge@jtown.eu']
8
+
9
+ spec.summary = 'An AMQ based proxy to retrieve price and contract data from IBKR TWS.'
10
+ spec.description = 'An AMQ based proxy to retrieve price and contract data from IBKR TWS. The AMQ is provided by rabbitMQ with the gem Bunny, the API to TWS/ibGateway with the gem ib-api'
11
+
12
+ spec.homepage = 'https://github.com/donkeybridge/' + spec.name
13
+ spec.license = 'BSD-3-Clause'
14
+ spec.required_ruby_version = Gem::Requirement.new('~> 2.7')
15
+
16
+ spec.metadata['homepage_uri'] = spec.homepage
17
+ spec.metadata['source_code_uri'] = spec.homepage
18
+ spec.metadata['changelog_uri'] = spec.homepage + '/CHANGELOG.md'
19
+
20
+ # Specify which files should be added to the gem when it is released.
21
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
22
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
23
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
24
+ end
25
+ spec.bindir = 'bin'
26
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
27
+ spec.require_paths = ['lib']
28
+
29
+ spec.add_dependency 'activesupport', '~> 6'
30
+ spec.add_dependency 'ib-api', '~> 972'
31
+ spec.add_dependency 'colorize', '~> 0.8'
32
+ spec.add_dependency 'cotcube-helpers', '~> 0.2.1'
33
+ spec.add_dependency 'outputhandler', '~> 0.2'
34
+ spec.add_dependency 'bunny', '~> 2'
35
+ spec.add_dependency 'httparty' '~> 0.20'
36
+ spec.add_dependency 'yaml', '~> 0.1'
37
+ spec.add_dependency 'json', '~> 2'
38
+
39
+ spec.add_development_dependency 'rake', '~> 13'
40
+ spec.add_development_dependency 'rspec', '~>3.6'
41
+ spec.add_development_dependency 'yard', '~>0.9'
42
+ end
@@ -0,0 +1,65 @@
1
+ module Cotcube
2
+ class DataProxy
3
+
4
+ # Create a connection to the locally running
5
+ def self.get_ib_client(host: 'localhost', port: 4002, id: 5, client_id: 5)
6
+ obj = {
7
+ id: id,
8
+ client_id: client_id,
9
+ port: port,
10
+ host: host
11
+ }
12
+ begin
13
+ obj[:ib] = IB::Connection.new(
14
+ id: id,
15
+ client_id: client_id,
16
+ port: port,
17
+ host: host
18
+ ) do |provider|
19
+ obj[:alert] = provider.subscribe(:Alert) { true }
20
+ obj[:managed_accounts] = provider.subscribe(:ManagedAccounts) { true }
21
+ end
22
+ obj[:error] = 0
23
+ rescue Exception => e
24
+ obj[:error] = 1
25
+ obj[:message] = e.message
26
+ obj[:full_message] = e.full_message
27
+ end
28
+ obj
29
+ end
30
+
31
+
32
+ def self.get_mq_client(client_id: 5)
33
+ obj = {
34
+ client_id: client_id,
35
+ }
36
+ begin
37
+ # for more info on connection parameters see http://rubybunny.info/articles/connecting.html
38
+ #
39
+ obj[:connection] = Bunny.new(
40
+ host: 'localhost',
41
+ port: 5672,
42
+ user: SECRETS['dataproxy_mq_user'],
43
+ password: SECRETS['dataproxy_mq_password'],
44
+ vhost: SECRETS['dataproxy_mq_vhost']
45
+ )
46
+ obj[:connection].start
47
+ obj[:commands] = obj[:connection].create_channel
48
+ obj[:channel] = obj[:connection].create_channel
49
+ obj[:request_queue] = obj[:commands].queue('', exclusive: true, auto_delete: true)
50
+ obj[:request_exch] = obj[:commands].direct('dataproxy_commands', exclusive: true, auto_delete: true)
51
+ obj[:replies_exch] = obj[:commands].direct('dataproxy_replies', auto_delete: true)
52
+ %w[ dataproxy_commands ].each do |key|
53
+ obj[:request_queue].bind(obj[:request_exch], routing_key: key )
54
+ end
55
+ obj[:error] = 0
56
+ rescue Exception => e
57
+ obj[:error] = 1
58
+ obj[:message] = e.message
59
+ obj[:full_message] = e.full_message
60
+ end
61
+ obj
62
+ end
63
+
64
+ end
65
+ end
@@ -0,0 +1,46 @@
1
+ module Cotcube
2
+ class DataProxy
3
+
4
+ def client_success(request, id: nil, to: nil, exchange: :replies_exch, &block)
5
+ client_response( request, id: nil, to: nil, err: 0, exchange: exchange, &block)
6
+ end
7
+
8
+ def client_fail(request, id: nil, to: nil, exchange: :replies_exch, &block)
9
+ client_response(request, id: nil, to: nil, err: 1, exchange: exchange, &block)
10
+ end
11
+
12
+ private
13
+
14
+ def client_response(request, id: nil, to: nil, err:, exchange: :replies_exch)
15
+ __id__ = id.presence || request.delete(:__id__)
16
+ __to__ = to.presence || request.delete(:__to__)
17
+ msg = yield
18
+ case msg
19
+ when String
20
+ response = { error: err, msg: msg }
21
+ when Hash
22
+ response = { error: err }
23
+ msg.each { |k, v| response[k] = v }
24
+ when Array
25
+ response = { error: err, result: msg }
26
+ else
27
+ response = { error: 1, msg: "Processing failed for '#{msg.inspect}' after '#{request}'." }
28
+ end
29
+ if response[:error] == 1
30
+ log "CLIENT #{id} FAILIURE: #{response.inspect}.".colorize(:light_red)
31
+ elsif response[:result].is_a?(Array)
32
+ log "CLIENT #{id} SUCCESS: sent #{response[:result].size} datasets."
33
+ else
34
+ log "CLIENT #{id} SUCCESS: #{response.to_s.scan(/.{1,120}/).join(' '*30 + "\n")}"
35
+ end
36
+ mq[exchange].publish(
37
+ response.to_json,
38
+ content_type: 'application/json',
39
+ priority: 7,
40
+ correlation_id: __id__,
41
+ routing_key: __to__,
42
+ reply_to: __id__
43
+ )
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,185 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cotcube
4
+ class DataProxy
5
+
6
+ def commserver_start
7
+ mq[:request_subscription] = mq[:request_queue].subscribe do |delivery_info, properties, payload|
8
+
9
+ ################################################################################################
10
+ # the request will be JSON decoded. The generic command 'failed' will be set, if decoding raises.
11
+ # furthermore, __id__ and __to__ are extracted and added to the request-hash
12
+ #
13
+ request = JSON.parse(payload, symbolize_names: true) rescue { command: 'failed' }
14
+ request[:command] ||= 'nil'
15
+
16
+ request[:__id__] = properties[:correlation_id]
17
+ request[:__to__] = properties[:reply_to]
18
+
19
+ if request[:debug]
20
+ log "Received \t#{delivery_info.map{|k,v| "#{k}\t#{v}"}.join("\n")
21
+ }\n\n#{properties .map{|k,v| "#{k}\t#{v}"}.join("\n")
22
+ }\n\n#{request .map{|k,v| "#{k}\t#{v}"}.join("\n")}" if request[:debug]
23
+ else
24
+ log "Received\t#{request}"
25
+ end
26
+
27
+ ###############################################################################################
28
+ # the entire set of command processing,
29
+ # starting with the (generic) 'failed' command, that just answers with the failure notice
30
+ # and with another failure notice upon a missing command section in the request
31
+ # ending with another failure notice, if an unknown command was issued
32
+ #
33
+ log "Processing #{request[:command]}:"
34
+ case request[:command].downcase
35
+ when 'failed'
36
+ client_fail(request) { "Failed to parse payload: '#{payload}'." }
37
+
38
+ when 'nil'
39
+ client_fail(request) { "missing :command in request: '#{request}'." }
40
+
41
+ ##############################################################################################
42
+ # ping -> pong, just for testing
43
+ #
44
+ when 'ping'
45
+ client_success(request) { "pong" }
46
+
47
+ ##############################################################################################
48
+ # the get_contracts command tries to resolve a list of available contracts related to a
49
+ # specific symbol based on a set of characteristics retrieved via Herlpers::get_id_set
50
+ #
51
+ # the reply of the message is processed asynchroniously upon reception of
52
+ # the IB Message 'ContractDataEnd' in the message subscribers section
53
+ #
54
+ when 'get_contract', 'get_contracts', Cotcube::Helpers.sub(minimum: 3) { 'contracts' }
55
+ if request[:symbol].nil?
56
+ client_fail(request) { "Cannot requets contracts without :symbol (in '#{request}')." }
57
+ next
58
+ end
59
+ sym = Cotcube::Helpers.get_id_set(symbol: request[:symbol])
60
+ if [nil, false].include? sym
61
+ client_fail(request) { "Unknown symbol '#{request[:symbol]}' in '#{request}'." }
62
+ next
63
+ end
64
+ request[:result] = []
65
+ req_mon.synchronize { requests[request[:__id__]] = request }
66
+ ib_contract = IB::Contract.new symbol: sym[:ib_symbol], exchange: sym[:exchange], currency: sym[:currency], sec_type: (request[:sec_type] || 'FUT')
67
+ ib.send_message :RequestContractData, contract: ib_contract, request_id: request[:__id__].to_i(16)
68
+
69
+
70
+ ##############################################################################################
71
+ # the historical command retrieves a list of bars as provided by TWS
72
+ # the minimum requirement is :contract parameter issued.
73
+ #
74
+ # the reply to this message is processed asynchroniously upon reception of
75
+ # the IB message 'HistoricalData' in message subscribers section
76
+ #
77
+ when Cotcube::Helpers.sub(minimum: 3) {'historical'}
78
+ con_id = request[:con_id] || Cotcube::Helpers.get_ib_contract(request[:contract])[:con_id] rescue nil
79
+ if con_id.nil? or request[:contract].nil?
80
+ client_fail(request) { "Cannot get :con_id for contract:'#{request[:contract]}' in '#{request}'." }
81
+ next
82
+ end
83
+ sym = Cotcube::Helpers.get_id_set(contract: request[:contract])
84
+ before = Time.at(request[:before]).to_ib rescue Time.now.to_ib
85
+ ib_contract = IB::Contract.new(con_id: con_id, exchange: sym[:exchange])
86
+ req = {
87
+ request_id: request[:__id__].to_i(16),
88
+ contract: ib_contract,
89
+ end_date_time: before,
90
+ what_to_show: (request[:based_on] || :trades),
91
+ use_rth: (request[:rth_only] || 1),
92
+ keep_up_to_date: 0,
93
+ duration: (request[:duration].gsub('_', ' ') || '1 D'),
94
+ bar_size: (request[:interval].to_sym || :min15)
95
+ }
96
+ req_mon.synchronize { requests[request[:__id__]] = request }
97
+ begin
98
+ Timeout.timeout(2) { ib.send_message(IB::Messages::Outgoing::RequestHistoricalData.new(req)) }
99
+ rescue Timeout::Error, IB::Error
100
+ client_fail(request) { 'Could not request historical data. Is ib_client running?' }
101
+ req_mon.synchronize { requests.delete(request[:__id__]) }
102
+ next
103
+ end
104
+
105
+
106
+ # ********************************************************************************************
107
+ #
108
+ # REQUESTS BELOW ARE BASED ON A cONTINUOUS IB SUBSCRIPTION AND MUST BE CONSIDERED
109
+ # GARBAGE COLLECTION ( instance.gc ) --- SUBSCRIPTION DATA IS PERSISTET IN @persistent
110
+ #
111
+ # ********************************************************************************************
112
+
113
+
114
+ ###############################################################################################
115
+ # the start_realtimebars initiates the IBKR realtime (5s) bars delivery for a specific contract
116
+ # and feeds them into a fanout exchange dedicated to that contract
117
+ # delivery continues as long as there are queues bound to that exchange
118
+ #
119
+ when Cotcube::Helpers.sub(minimum:4){'realtimebars'}
120
+ subscribe_persistent(request, type: :realtimebars)
121
+ next
122
+
123
+ when 'ticks'
124
+ subscribe_persistent(request, type: :realtimebars)
125
+ next
126
+
127
+ when 'depth'
128
+ subscribe_persistent(request, type: :depth)
129
+ next
130
+
131
+ else
132
+ client_fail(request) { "Unknown :command '#{request[:command]}' in '#{request}'." }
133
+ end
134
+ end
135
+ log "Started commserver listening on #{mq[:request_queue]}"
136
+ end
137
+
138
+ def commserver_stop
139
+ mq[:request_subscription].cancel
140
+ log "Stopped commserver ..."
141
+ end
142
+
143
+ def subscribe_persistent(request, type:)
144
+ sym = Cotcube::Helpers.get_id_set(contract: request[:contract])
145
+ con_id = request[:con_id] || Cotcube::Helpers.get_ib_contract(request[:contract])[:con_id] rescue nil
146
+ if sym.nil? or con_id.nil?
147
+ client_fail(request) { "Invalid contract '#{request[:contract]}'." }
148
+ return
149
+ end
150
+ if persistent[type][con_id].nil?
151
+ per_mon.synchronize {
152
+ persistent[type][con_id] = { con_id: con_id,
153
+ contract: request[:contract],
154
+ exchange: mq[:channel].fanout(request[:exchange]) }
155
+ }
156
+ if type == :depth
157
+ bufferthread = Thread.new do
158
+ sleep 5.0 - (Time.now.to_f % 5)
159
+ loop do
160
+ begin
161
+ @block_depth_queue = true
162
+ sleep 0.025
163
+ con = persistent[:depth][con_id]
164
+ con[:exchange].publish(con[:buffer].to_json)
165
+ con[:buffer] = []
166
+ @block_depth_queue = false
167
+ end
168
+ sleep 5.0 - (Time.now.to_f % 5)
169
+ end
170
+ end
171
+ per_mon.synchronize { persistent[:depth][con_id][:bufferthread] = bufferthread }
172
+ end
173
+ ib_contract = IB::Contract.new(con_id: con_id, exchange: sym[:exchange])
174
+ ib.send_message(REQUEST_TYPES[type], id: con_id, contract: ib_contract, data_type: :trades, use_rth: false)
175
+ client_success(request) { "Delivery of #{type} of #{request[:contract]} started." }
176
+ elsif persistent[type][con_id][:on_cancel]
177
+ client_fail(request) { { reason: :on_cancel, message: "Exchange '#{requst[:exchange]}' is marked for cancel, retry in a few seconds to recreate" } }
178
+ else
179
+ client_success(request) { "Delivery of #{type} of #{request[:contract]} attached to existing." }
180
+ end
181
+ end
182
+
183
+
184
+ end
185
+ end
@@ -0,0 +1,157 @@
1
+ # about garbage collection, i.e. periodically check for unused artifacts and clean them
2
+ #
3
+ module Cotcube
4
+ class DataProxy
5
+
6
+ def gc_start
7
+ if gc_thread.nil?
8
+ @gc_thread = Thread.new do
9
+ loop do
10
+ sleep 30 + 2 * Random.rand
11
+ gc
12
+ end
13
+ end
14
+ log 'GC_INFO: GC spawned.'
15
+ else
16
+ log 'GC_ERROR: Cannot start GC_THREAD more than once.'
17
+ end
18
+ end
19
+
20
+ def gc_stop
21
+ if gc_thread.nil?
22
+ log 'GC_ERROR: Cannot stop nonexisting GC_THREAD.'
23
+ false
24
+ else
25
+ gc_thread.kill
26
+ gc_thread = nil
27
+ log 'GC_INFO: GC stopped.'
28
+ true
29
+ end
30
+ end
31
+
32
+ def gc
33
+ get_mq(list: false).each do |item_type, items|
34
+ items.each do |key, item|
35
+ case item_type
36
+ when :bindings
37
+ # NOTE if might be considerable to unbind unused queues from exchanges before removing either
38
+ # as the bound value of the exchange will decrease it might be removed on next run of GC
39
+ when :queues
40
+ next if item[:consumers].nil? or item[:consumers].positive?
41
+ delete_mq(type: item_type, instance: item[:name])
42
+ log "GC_INFO: Deleted unsed queue: #{item}."
43
+ # sadly we don't know about unsed queues. basically this means that some client did not declare its queue as auto_delete.
44
+ # the dateproxy itself has only 1 queue -- the command queue, everything else is send out to exchanges
45
+ when :exchanges
46
+ if item[:name].empty? or item[:name] =~ /^amq\./ or item[:bound].count.positive? or
47
+ %w[ dataproxy_commands dataproxy_replies ].include? item[:name]
48
+ next
49
+ end
50
+ #log "GC_INFO: found superfluous exchange '#{item[:name]}'"
51
+
52
+ _, subscription_type, contract = item[:name].split('_')
53
+ unless %w[ ticks depth realtimebars ].include? subscription_type.downcase
54
+ puts "GC_WARNING: Unknown subscription_type '#{subscription_type}', skipping..."
55
+ next
56
+ end
57
+ con_id = Cotcube::Helpers.get_ib_contract(contract)[:con_id] rescue 0
58
+ if con_id.zero?
59
+ puts "GC_WARNING: No con_id found for contract '#{contract}', skipping..."
60
+ next
61
+ end
62
+ if persistent[subscription_type.to_sym][con_id].nil?
63
+ puts "GC_WARNING: No record for subscription '#{subscription_type}_#{contract}' with #{con_id} found, deleting anyway..."
64
+ end
65
+ Thread.new do
66
+ per_mon.synchronize {
67
+ persistent[subscription_type.to_sym][con_id][:on_cancel] = true if persistent.dig(subscription_type.to_sym, con_id)
68
+ }
69
+ log "GC_INFO: Sending cancel for #{subscription_type}::#{contract}::#{con_id}."
70
+ message_type = case subscription_type;
71
+ when 'ticks'; :CancelMarketData
72
+ when 'depth'; :CancelMarketDepth
73
+ else; :CancelRealTimeBars
74
+ end
75
+ if ib.send_message( message_type, id: con_id )
76
+ sleep 0.75 + Random.rand
77
+ res = delete_mq(type: item_type, instance: item[:name])
78
+ log "GC_SUCCESS: exchange '#{item[:name]}' with #{con_id} has been deleted ('#{res}')."
79
+ per_mon.synchronize { persistent[subscription_type.to_sym].delete(con_id) }
80
+ else
81
+ log "GC_FAILED: something went wrong unsubscribing '#{subscription_type}_#{contract}' with #{con_id}."
82
+ end
83
+ end
84
+ else
85
+ log "GC_ERROR: Unexpected type '#{item_type}' in GarbageCollector"
86
+ end
87
+ end
88
+ end
89
+ end
90
+
91
+ def get_mq(list: true)
92
+ bindings = api(type: :bindings)
93
+ results = {}
94
+ %i[ queues exchanges ].each do |type|
95
+ results[type] = {}
96
+ items = api type: type
97
+ items.each do |item|
98
+ query_name = item[:name].empty? ? 'amq.default' : item[:name]
99
+ results[type][item[:name]] = api type: type, instance: query_name
100
+ results[type][item[:name]][:bound] = bindings.select{|z| z[:source] == item[:name] }.map{|z| z[:destination]} if type == :exchanges
101
+ end
102
+ end
103
+ results.each do |type, items|
104
+ items.each do |key,item|
105
+ if item.is_a? Array
106
+ puts "#{key}\t\t#{item}"
107
+ next
108
+ end
109
+ puts "#{format '%12s', type.to_s.upcase} #{
110
+ case type
111
+ when :queues
112
+ "Key: #{format '%-30s', key} Cons: #{item[:consumers]}"
113
+ when :exchanges
114
+ "Key: #{format '%-30s', key} Type: #{format '%10s', item[:type]} Bound: #{item[:bound].presence || 'None.'}"
115
+ else
116
+ "Unknown details for #{type}"
117
+ end
118
+ }"
119
+ end
120
+ end if list
121
+ results[:bindings] = bindings
122
+ results
123
+ end
124
+
125
+ def delete_mq(type:, instance: )
126
+ allowed_types = %i[ queues exchanges ]
127
+ raise ArgumentError, "Type must be in '#{allowed_types}', but is '#{type}'" unless allowed_types.include? type
128
+ result = HTTParty.delete("#{SECRETS['dataproxy_mq_proto']
129
+ }://#{SECRETS['dataproxy_mq_user']
130
+ }:#{SECRETS['dataproxy_mq_password']
131
+ }@#{SECRETS['dataproxy_mq_host']
132
+ }:#{SECRETS['dataproxy_mq_port']
133
+ }/api/#{type.to_s
134
+ }/dp/#{instance}", query: { 'if-unused' => true })
135
+ if result.code == 204
136
+ result = { error: 0}
137
+ else
138
+ result = JSON.parse(result.body, symbolize_names: true)
139
+ result[:error] = 1
140
+ end
141
+ result
142
+ end
143
+
144
+ def api(type:, instance: nil)
145
+ allowed_types = %i[ queues exchanges bindings ] # other types need different API sepc: channels connections definitions
146
+ raise ArgumentError, "Type must be in '#{allowed_types}', but is '#{type}'" unless allowed_types.include? type
147
+ req = "#{type.to_s}/#{SECRETS['dataproxy_mq_vhost']}#{instance.nil? ? '' : "/#{instance}"}"
148
+ JSON.parse(HTTParty.get("#{SECRETS['dataproxy_mq_proto']
149
+ }://#{SECRETS['dataproxy_mq_user']
150
+ }:#{SECRETS['dataproxy_mq_password']
151
+ }@#{SECRETS['dataproxy_mq_host']
152
+ }:#{SECRETS['dataproxy_mq_port']
153
+ }/api/#{req}").body, symbolize_names: true)
154
+ end
155
+
156
+ end
157
+ end
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cotcube
4
+ # top-level class documentation comment
5
+ class DataProxy
6
+
7
+ def initialize(
8
+ outputhandler: OutputHandler.new(
9
+ location: "/var/cotcube/log/dataproxy"
10
+ )
11
+ )
12
+ @output = outputhandler
13
+ @client = DataProxy.get_ib_client
14
+ @mq = DataProxy.get_mq_client
15
+ @ib = @client[:ib]
16
+ raise 'Could not connect to IB' unless @ib
17
+ raise 'Could not connect to RabbitMQ' if %i[ request_exch replies_exch request_queue ].map{|z| mq[z].nil? }.reduce(:|)
18
+ @requests = {}
19
+ @req_mon = Monitor.new
20
+ @persistent = { ticks: {}, depth: {}, realtimebars: {} }
21
+ @per_mon = Monitor.new
22
+ @gc_thread = nil
23
+ spawn_message_subscribers
24
+ commserver_start
25
+ recover
26
+ gc_start
27
+ end
28
+
29
+ def shutdown
30
+ puts "Shutting down dataproxy."
31
+ commserver_stop
32
+ gc_stop
33
+ mq[:commands].close
34
+ mq[:channel].close
35
+ mq[:connection].close
36
+ persistent.each do |type, items|
37
+ items.each do |con_id, item|
38
+ log "sending #{ CANCEL_TYPES[type.to_sym]} #{con_id} (for #{item[:contract]})"
39
+ ib.send_message CANCEL_TYPES[type.to_sym], id: con_id
40
+ end
41
+ end
42
+ sleep 1
43
+ gc
44
+ sleep 1
45
+ ib.close
46
+ puts "... done."
47
+ end
48
+
49
+ private
50
+ attr_reader :client, :clients, :ib, :mq, :requests, :req_mon, :persistent, :per_mon, :gc_thread
51
+
52
+ def recover
53
+ get_mq(list: false)[:exchanges].keys.select{|z| z.split('_').size == 3 }.each do |exch|
54
+ src, type, contract = exch.split('_')
55
+ next unless src == 'dataproxy'
56
+ next unless %w[ ticks depth realtimebars ].include? type.downcase
57
+ puts "Found #{exch} to recover."
58
+ subscribe_persistent( { contract: contract, exchange: exch }, type: type.to_sym )
59
+ end
60
+ end
61
+
62
+ def log(msg)
63
+ @output.puts "#{DateTime.now.strftime('%Y%m%d-%H:%M:%S: ')}#{msg.to_s.scan(/.{1,120}/).join("\n" + ' ' * 20)}"
64
+ end
65
+
66
+ end
67
+ end
68
+
69
+ __END__
@@ -0,0 +1,115 @@
1
+ module Cotcube
2
+ class DataProxy
3
+
4
+ def __int2hex__(id)
5
+ tmp = id.to_s(16) rescue nil
6
+ return nil if tmp.nil?
7
+ tmp.prepend('0') while tmp.length < 7
8
+ return tmp
9
+ end
10
+
11
+ def spawn_message_subscribers
12
+ @msg_queue = Queue.new
13
+ @depth_queue = Queue.new
14
+
15
+ ib.subscribe(:MarketDataType, :TickRequestParameters){|msg| log "#{msg.class}\t#{msg.data.inspect}".colorize(:yellow) }
16
+
17
+ @msg_subscriber = ib.subscribe(
18
+ :Alert,
19
+ :ContractData, :ContractDataEnd, :BondContractData,
20
+ :TickGeneric, :TickString, :TickPrice, :TickSize,
21
+ :HistoricalData,
22
+ :RealTimeBar
23
+ ) do |msg|
24
+
25
+ @msg_queue << msg
26
+ end
27
+
28
+ @depth_subscriber = ib.subscribe( :MarketDepth ) {|msg| @depth_queue << msg}
29
+
30
+ @msg_subscriber_thread = Thread.new do
31
+ loop do
32
+ msg = @msg_queue.pop
33
+
34
+ data = msg.data
35
+ data[:time] = msg.created_at.strftime('%H:%M:%S')
36
+ data[:timestamp] = (msg.created_at.to_time.to_f * 1000).to_i
37
+ __id__ = __int2hex__(data[:request_id])
38
+
39
+ case msg
40
+
41
+ when IB::Messages::Incoming::HistoricalData
42
+ client_success(requests[__id__]) { msg.results }
43
+ req_mon.synchronize { requests.delete(__id__) }
44
+
45
+ when IB::Messages::Incoming::Alert # Alert
46
+ __id__ = __int2hex__(data[:error_id])
47
+ case data[:code]
48
+ when 162
49
+ log("ALERT 162:".light_red + ' MISSING MARKET DATA PERMISSION')
50
+ when 201
51
+ log("ALERT 201:".light_red + ' DUPLICATE OCA_GROUP')
52
+ else
53
+ log("ALERT #{data[:code]}:".light_red + " #{data[:message]}")
54
+ end
55
+ data[:msg_type] = 'alert'
56
+ client_fail(requests[__id__]) {data} unless requests[__id__].nil?
57
+ log data
58
+
59
+ when IB::Messages::Incoming::ContractData
60
+ req_mon.synchronize do
61
+ requests[__id__][:result] << data[:contract].slice(:local_symbol, :last_trading_day, :con_id)
62
+ end
63
+
64
+ when IB::Messages::Incoming::ContractDataEnd
65
+ sleep 0.25
66
+ client_success(requests[__id__]) { requests[__id__][:result] }
67
+ req_mon.synchronize { requests.delete(__id__) }
68
+
69
+ when IB::Messages::Incoming::RealTimeBar
70
+ con_id = data[:request_id]
71
+ bar = data[:bar]
72
+ exchange = persistent[:realtimebars][con_id][:exchange]
73
+ begin
74
+ exchange.publish(bar.slice(*%i[time open high low close volume trades wap]).to_json)
75
+ rescue Bunny::ChannelAlreadyClosed
76
+ ib.send_message :CancelRealTimeBars, id: con_id
77
+ log "Delivery for #{persistent[:realtimebars][con_id][:contract] rescue 'unknown contract'
78
+ } with con_id #{con_id} has been stopped."
79
+ Thread.new{ sleep 5; per_mon.synchronize { persistent[:realtimebars].delete(con_id) } }
80
+ end
81
+
82
+ when IB::Messages::Incoming::TickSize,
83
+ IB::Messages::Incoming::TickPrice,
84
+ IB::Messages::Incoming::TickGeneric,
85
+ IB::Messages::Incoming::TickString
86
+ con_id = data[:ticker_id]
87
+ contract = persistent[:ticks][con_id][:contract]
88
+ exchange = persistent[:ticks][con_id][:exchange]
89
+ begin
90
+ exchange.publish(data.inspect.to_json)
91
+ rescue Bunny::ChannelAlreadyClosed
92
+ ib.send_message :CancelMarketData, id: con_id
93
+ log "Delivery for #{persistent[:ticks][con_id][:contract]} with con_id #{con_id} has been stopped."
94
+ Thread.new{ sleep 0.25; per_mon.synchronize { persistent[:ticks].delete(con_id) } }
95
+ end
96
+
97
+ else
98
+ log("WARNING".light_red + "\tUnknown messagetype: #{msg.inspect}")
99
+ end
100
+ end
101
+ log "SPAWN_SUBSCRIBERS\tSubscribers attached to IB" if @debug
102
+ end
103
+ @depth_subscriber_thread = Thread.new do
104
+ loop do
105
+ sleep 0.025 while @block_depth_queue
106
+ msg = @depth_queue.pop
107
+ con_id = msg.data[:request_id]
108
+ msg[:contract] = persistent[:depth][con_id][:contract]
109
+ persistent[:depth][con_id][:buffer] << msg.data.slice(*%i[ contract position operation side price size ])
110
+ end
111
+ end
112
+ end
113
+ end
114
+ end
115
+
@@ -0,0 +1,36 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler'
4
+ Bundler.require
5
+ require_relative './cotcube-dataproxy/3rd_clients'
6
+ require_relative './cotcube-dataproxy/init'
7
+ require_relative './cotcube-dataproxy/client_response'
8
+ require_relative './cotcube-dataproxy/subscribers'
9
+ require_relative './cotcube-dataproxy/commserver'
10
+ require_relative './cotcube-dataproxy/gc'
11
+
12
+ SECRETS_DEFAULT = {
13
+ 'dataproxy_mq_proto' => 'http',
14
+ 'dataproxy_mq_user' => 'guest',
15
+ 'dataproxy_mq_password' => 'guest',
16
+ 'dataproxy_mq_host' => 'localhost',
17
+ 'dataproxy_mq_port' => '15672',
18
+ 'dataproxy_mq_vhost' => '%2F'
19
+ }
20
+
21
+ # Load a yaml file containing actual parameter and merge those with current
22
+ # TODO use better config file location
23
+ SECRETS = SECRETS_DEFAULT.merge( -> {YAML.load(File.read("#{__FILE__.split('/')[..-2].join('/')}/../secrets.yml")) rescue {} }.call)
24
+
25
+ CANCEL_TYPES = {
26
+ ticks: :CancelMarketData,
27
+ depth: :CancelMarketDepth,
28
+ realtimebars: :CancelRealTimeBars
29
+ }
30
+
31
+ REQUEST_TYPES = {
32
+ ticks: :RequestMarketData,
33
+ depth: :RequestMarketDepth,
34
+ realtimebars: :RequestRealTimeBars
35
+ }
36
+
metadata ADDED
@@ -0,0 +1,229 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: cotcube-dataproxy
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.1
5
+ platform: ruby
6
+ authors:
7
+ - Benjamin L. Tischendorf
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2021-11-13 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: activesupport
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '6'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '6'
27
+ - !ruby/object:Gem::Dependency
28
+ name: ib-api
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '972'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '972'
41
+ - !ruby/object:Gem::Dependency
42
+ name: colorize
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '0.8'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '0.8'
55
+ - !ruby/object:Gem::Dependency
56
+ name: cotcube-helpers
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: 0.2.1
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: 0.2.1
69
+ - !ruby/object:Gem::Dependency
70
+ name: outputhandler
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '0.2'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '0.2'
83
+ - !ruby/object:Gem::Dependency
84
+ name: bunny
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '2'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '2'
97
+ - !ruby/object:Gem::Dependency
98
+ name: httparty~> 0.20
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '0'
104
+ type: :runtime
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: yaml
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.1'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.1'
125
+ - !ruby/object:Gem::Dependency
126
+ name: json
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: '2'
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: '2'
139
+ - !ruby/object:Gem::Dependency
140
+ name: rake
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '13'
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: '13'
153
+ - !ruby/object:Gem::Dependency
154
+ name: rspec
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: '3.6'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '3.6'
167
+ - !ruby/object:Gem::Dependency
168
+ name: yard
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '0.9'
174
+ type: :development
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '0.9'
181
+ description: An AMQ based proxy to retrieve price and contract data from IBKR TWS.
182
+ The AMQ is provided by rabbitMQ with the gem Bunny, the API to TWS/ibGateway with
183
+ the gem ib-api
184
+ email:
185
+ - donkeybridge@jtown.eu
186
+ executables: []
187
+ extensions: []
188
+ extra_rdoc_files: []
189
+ files:
190
+ - CHANGELOG.md
191
+ - Gemfile
192
+ - VERSION
193
+ - bin/dataproxy
194
+ - bin/test_client.rb
195
+ - cotcube-dataproxy.gemspec
196
+ - lib/cotcube-dataproxy.rb
197
+ - lib/cotcube-dataproxy/3rd_clients.rb
198
+ - lib/cotcube-dataproxy/client_response.rb
199
+ - lib/cotcube-dataproxy/commserver.rb
200
+ - lib/cotcube-dataproxy/gc.rb
201
+ - lib/cotcube-dataproxy/init.rb
202
+ - lib/cotcube-dataproxy/subscribers.rb
203
+ homepage: https://github.com/donkeybridge/cotcube-dataproxy
204
+ licenses:
205
+ - BSD-3-Clause
206
+ metadata:
207
+ homepage_uri: https://github.com/donkeybridge/cotcube-dataproxy
208
+ source_code_uri: https://github.com/donkeybridge/cotcube-dataproxy
209
+ changelog_uri: https://github.com/donkeybridge/cotcube-dataproxy/CHANGELOG.md
210
+ post_install_message:
211
+ rdoc_options: []
212
+ require_paths:
213
+ - lib
214
+ required_ruby_version: !ruby/object:Gem::Requirement
215
+ requirements:
216
+ - - "~>"
217
+ - !ruby/object:Gem::Version
218
+ version: '2.7'
219
+ required_rubygems_version: !ruby/object:Gem::Requirement
220
+ requirements:
221
+ - - ">="
222
+ - !ruby/object:Gem::Version
223
+ version: '0'
224
+ requirements: []
225
+ rubygems_version: 3.1.6
226
+ signing_key:
227
+ specification_version: 4
228
+ summary: An AMQ based proxy to retrieve price and contract data from IBKR TWS.
229
+ test_files: []