cotcube-helpers 0.1.9.2 → 0.2.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2bfb21e47e19a7dfd33125d42d4d049d8654fbf84e36edfe081a3878e751439e
-  data.tar.gz: 847d6eb7b98df85dc11aadc294147542071fdb4ff83d86faac3e0ef14d204d47
+  metadata.gz: 54c97a5746e0e03b971b9421be0636eb2c259beb90840ff34404f8049114019b
+  data.tar.gz: 6d565f00ed620b2458122ad686b1b6c60bb26e815f248474f746b38c10902dc6
 SHA512:
-  metadata.gz: b2a0020f5c8277b6882dbe7a00e2c885c1e86eb56b22529236b086554b7b69b398afaa1b013bb02b8f4d23f7a7444000dac77e283c7c92ecaa8e2e255e44c6f9
-  data.tar.gz: b8638126b407e5c4e74a0866fa7497d1c3f29af6f143b94fbca775987de78f350df75b239fd68dcc339aaa4858254b5957c3b9a1e4a4aff40d8c3b487a84a5b4
+  metadata.gz: f9a71ad40a61b1caf44162851d2c323c1b762501705459fbbca85d38c8a6bf66e261eb920093fa01f3ea6ddd07b48dbe7aa4f46e5010a39e0e5bb3045c9941d1
+  data.tar.gz: a862cb81820c8459a015788d273c1cd159078f11705f2c5f8db2847e355a6e5f6cf54b4532a43efef246804a1679053ca807b76c79c580881a46e3ea09fb0f01

data/CHANGELOG.md

@@ -1,3 +1,41 @@
+## 0.2.2.3 (November 28, 2021)
+  - data_client: fixing troublesome output in .command
+
+## 0.2.2.2 (November 28, 2021)
+  - dataclient fixing = for false ==
+  - solving merge conflicts
+  - Bump version to 0.2.2.
+
+## 0.2.2.1 (November 28, 2021)
+  - Bump version to 0.2.2.1
+
+## 0.2.2 (November 13, 2021)
+  - some further improvements to DataClient
+  - some fixes related to ib_contracts
+  - some fixes related to DataClient
+
+## 0.2.1.1 (November 10, 2021)
+  - Bump version to 0.2.1.
+
+## 0.2.1 (November 10, 2021)
+  - added new class 'dataclient' for communication with dataproxy
+  - added .translate_ib_contract
+
+## 0.2.0 (November 07, 2021)
+  - added module Candlestick_Recognition
+  - added instance_inspect method to 'scan' objects for contents of instance variables
+  - symbols: made selection of symbols more versatile by key
+  - added headers (:ib_symbol, :internal, :exchange, :currency) to symbol headers as well as symbol examples
+  - added scripts/symbols to list (and filter) symbols from command line (put to PATH!)
+
+## 0.1.10 (October 28, 2021)
+  - added script cron_ruby_wrapper.sh (linkable as /usr/local/bin/cruw.sh)
+  - added numeric ext .with_delimiter to support printing like 123_456_789.00121
+  - added micros to module
+  - added Helpers.micros to symbols.rb
+  - subpattern: excaping regex pattern to avoid ESC errors
+  - minor change
+
 ## 0.1.9.2 (July 24, 2021)
   - added missing module_functions
   - init: minor fix

data/Gemfile

@@ -4,3 +4,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in bitangent.gemspec
 gemspec
+gem 'parallel'
+gem 'bunny'

data/VERSION

@@ -1 +1 @@
-0.1.9.2
+0.2.2.3

data/lib/cotcube-helpers/constants.rb

@@ -3,8 +3,13 @@
 module Cotcube
   module Helpers
     SYMBOL_EXAMPLES = [
-      { id: '13874U', symbol: 'ET', ticksize: 0.25, power: 1.25, months: 'HMUZ', bcf: 1.0, reports: 'LF', format: '8.2f', name: 'S&P 500 MICRO' },
-      { id: '209747', symbol: 'NM', ticksize: 0.25, power: 0.5,  monhts: 'HMUZ', bcf: 1.0, reports: 'LF', format: '8.2f', name: 'NASDAQ 100 MICRO' }
+      { id: '13874U', symbol: 'ES', ib_symbol: 'ES', internal: 'ES', exchange: 'GLOBEX', currency: 'USD', ticksize: 0.25, power: 12.5, months: 'HMUZ', bcf: 1.0, reports: 'LF', format: '8.2f', name: 'S&P 500 MICRO' },
+      { id: '209747', symbol: 'NQ', ib_symbol: 'NQ', internal: 'NQ', exchange: 'GLOBEx', currency: 'USD', ticksize: 0.25, power: 5.0,  monhts: 'HMUZ', bcf: 1.0, reports: 'LF', format: '8.2f', name: 'NASDAQ 100 MICRO' }
+    ].freeze
+
+    MICRO_EXAMPLES = [
+      { id: '13874U', symbol: 'ET', ib_symbol: 'MES', internal: 'MES', exchange: 'GLOBEX', currency: 'USD', ticksize: 0.25, power: 1.25, months: 'HMUZ', bcf: 1.0, reports: 'LF', format: '8.2f', name: 'MICRO S&P 500 MICRO' },
+      { id: '209747', symbol: 'NM', ib_symbol: 'MNQ', internal: 'MNQ', exchange: 'GLOBEX', currency: 'USD', ticksize: 0.25, power: 0.5,  monhts: 'HMUZ', bcf: 1.0, reports: 'LF', format: '8.2f', name: 'MICRO NASDAQ 100 MICRO' }
     ].freeze
 
     COLORS         = %i[light_red light_yellow light_green red yellow green cyan magenta blue].freeze
@@ -18,9 +23,9 @@ module Cotcube
                        'J' => 4,  'K' =>  5, 'M' =>  6,
                        'N' => 7,  'Q' =>  8, 'U' =>  9,
                        'V' => 10, 'X' => 11, 'Z' => 12,
-                        1 => 'F',  2 => 'G',  3 => 'H',
-                        4 => 'J',  5 => 'K',  6 => 'M',
-                        7 => 'N',  8 => 'Q',  9 => 'U',
+                       1 => 'F',  2 => 'G',  3 => 'H',
+                       4 => 'J',  5 => 'K',  6 => 'M',
+                       7 => 'N',  8 => 'Q',  9 => 'U',
                        10 => 'V', 11 => 'X', 12 => 'Z' }.freeze
 
 
@@ -28,6 +33,12 @@ module Cotcube
 
     DATE_FMT = '%Y-%m-%d'
 
+    # Simple mapper to get from MONTH to LETTER
+    LETTERS = { "JAN"=> "F", "FEB"=> "G", "MAR"=> "H",
+                "APR"=> "J", "MAY"=> "K", "JUN"=> "M",
+                "JUL"=> "N", "AUG"=> "Q", "SEP"=> "U",
+                "OCT"=> "V", "NOV"=> "X", "DEC"=> "Z" }
+
   end
 end
 

data/lib/cotcube-helpers/data_client.rb

@@ -0,0 +1,230 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bunny'
+require 'json'
+
+module Cotcube
+  module Helpers
+    class DataClient
+      SECRETS_DEFAULT = {
+        'dataproxy_mq_proto' => 'http',
+        'dataproxy_mq_user' => 'guest',
+        'dataproxy_mq_password' => 'guest',
+        'dataproxy_mq_host' => 'localhost',
+        'dataproxy_mq_port' => '15672',
+        'dataproxy_mq_vhost' => '%2F'
+      }.freeze
+
+      SECRETS = SECRETS_DEFAULT.merge(
+        lambda {
+          begin
+            YAML.safe_load(File.read(Cotcube::Helpers.init[:secrets_file]))
+          rescue StandardError
+            {}
+          end
+        }.call
+      )
+
+      def initialize
+        @connection = Bunny.new(user: SECRETS['dataproxy_mq_user'],
+                                password: SECRETS['dataproxy_mq_password'],
+                                vhost: SECRETS['dataproxy_mq_vhost'])
+        @connection.start
+
+        @commands  = connection.create_channel
+        @exchange  = commands.direct('dataproxy_commands')
+        @requests  = {}
+        @persistent = { depth: {}, realtimebars: {}, ticks: {} }
+        @debug     = false
+        setup_reply_queue
+      end
+
+      # command acts a synchronizer: it sends the command and waits for the response
+      #     otherwise times out --- the counterpart here is the subscription within 
+      #     setup_reply_queue
+      #
+      def command(command, timeout: 5)
+        command = { command: command.to_s } unless command.is_a? Hash
+        command[:timestamp] ||= (Time.now.to_f * 1000).to_i
+        request_id = Digest::SHA256.hexdigest(command.to_json)[..6]
+        requests[request_id] = {
+          request: command,
+          id: request_id,
+          lock: Mutex.new,
+          condition: ConditionVariable.new
+        }
+
+        exchange.publish(command.to_json,
+                         routing_key: 'dataproxy_commands',
+                         correlation_id: request_id,
+                         reply_to: reply_queue.name)
+
+        # wait for the signal to continue the execution
+        #
+        requests[request_id][:lock].synchronize do
+          requests[request_id][:condition].wait(requests[request_id][:lock], timeout)
+        end
+
+        # if we reached timeout, we will return nil, just for explicity
+        response = requests[request_id][:response].dup
+        requests.delete(request_id)
+        response
+      end
+
+      alias_method :send_command, :command
+
+      def stop
+        %i[depth ticks realtimebars].each do |type|
+          persistent[type].each do |local_key, obj|
+            puts "Cancelling #{local_key}"
+            obj[:subscription].cancel
+          end
+        end
+        commands.close
+        connection.close
+      end
+
+      def get_contracts(symbol:)
+        send_command({ command: :get_contracts, symbol: symbol })
+      end
+
+      def get_historical(contract:, interval:, duration: nil, before: nil, rth_only: false, based_on: :trades)
+        # rth.true? means data outside of rth is skipped
+        rth_only = rth_only ? 1 : 0
+
+        # interval most probably is given as ActiveSupport::Duration
+        if interval.is_a? ActiveSupport::Duration
+          interval = case interval
+                     when       1; :sec1 
+                     when       5; :sec5
+                     when      15; :sec15
+                     when      30; :sec30
+                     when      60; :min1
+                     when     120; :min2
+                     when     300; :min5
+                     when     900; :min15
+                     when    1800; :min30
+                     when    3600; :hour1
+                     when   86400; :day1
+                     else; interval
+                     end
+        end
+
+        default_durations = { sec1: '30_M',  sec5: '2_H', sec15: '6_H', sec30: '12_H',
+                              min1:  '1_D',  min2: '2_D',  min5: '5_D', min15:  '1_W',
+                              min30: '1_W', hour1: '1_W',  day1: '1_Y'                 }
+
+        unless default_durations.keys.include? interval
+          raise "Invalid interval '#{interval}', should be in '#{default_durations.keys}'."
+        end
+
+        # TODO: Check for valid duration specification
+        puts 'WARNING in get_historical: param :before ignored' unless before.nil?
+        duration ||= default_durations[interval]
+        send_command({
+                       command: :historical,
+                       contract: contract,
+                       interval: interval,
+                       duration: duration,
+                       based_on: based_on.to_s.upcase,
+                       rth_only: rth_only,
+                       before: nil
+                     }, timeout: 20)
+      end
+
+      def start_persistent(contract:, type: :realtimebars, local_id: 0, &block)
+        unless %i[depth ticks realtimebars].include? type.to_sym
+          puts "ERROR: Inappropriate type in stop_realtimebars with #{type}"
+          return false
+        end
+
+        local_key = "#{contract}_#{local_id}"
+
+        channel = connection.create_channel
+        exchange  = channel.fanout("dataproxy_#{type}_#{contract}")
+        queue     = channel.queue('', exclusive: true, auto_delete: true)
+        queue.bind(exchange)
+
+        ib_contract = Cotcube::Helpers.get_ib_contract(contract)
+
+        command = { command: type, contract: contract, con_id: ib_contract[:con_id],
+                    delivery: queue.name, exchange: exchange.name }
+
+        block      ||= ->(bar) { puts bar.to_s }
+
+        subscription = queue.subscribe do |_delivery_info, _properties, payload|
+          block.call(JSON.parse(payload, symbolize_names: true))
+        end
+        persistent[type][local_key] = command.dup
+        persistent[type][local_key][:queue] = queue
+        persistent[type][local_key][:subscription] = subscription
+        persistent[type][local_key][:channel] = channel
+        send_command(command)
+      end
+
+      def stop_persistent(contract:, local_id: 0, type: :realtimebars)
+        unless %i[depth ticks realtimebars].include? type.to_sym
+          puts "ERROR: Inappropriate type in stop_realtimebars with #{type}"
+          return false
+        end
+        local_key = "#{contract}_#{local_id}"
+        p persistent[type][local_key][:subscription].cancel
+        p persistent[type][local_key][:channel].close
+        persistent[type].delete(local_key)
+      end
+
+      attr_accessor :response
+      attr_reader   :lock, :condition
+
+      private
+
+      attr_reader :call_id, :connection, :requests, :persistent,
+                  :commands, :server_queue_name, :reply_queue, :exchange
+
+      def setup_reply_queue
+        @reply_queue = commands.queue('', exclusive: true, auto_delete: true)
+        @reply_queue.bind(commands.exchange('dataproxy_replies'), routing_key: @reply_queue.name)
+
+        reply_queue.subscribe do |delivery_info, properties, payload|
+          __id__ = properties[:correlation_id]
+
+          if __id__.nil?
+            puts "Received without __id__: #{delivery_info.map       { |k, v| "#{k}\t#{v}" }.join("\n")
+                                      }\n\n#{properties.map          { |k, v| "#{k}\t#{v}" }.join("\n")
+                                      }\n\n#{JSON.parse(payload).map { |k, v| "#{k}\t#{v}" }.join("\n")}" if @debug
+
+          elsif requests[__id__].nil?
+            puts "Received non-matching response, maybe previously timed out: \n\n#{delivery_info}\n\n#{properties}\n\n#{payload}\n."[..620].scan(/.{1,120}/).join(' '*30 + "\n") if @debug
+          else
+            # save the payload and send the signal to continue the execution of #command
+            # need to rescue the rare case, where lock and condition are destroyed right in parallel by timeout
+            begin 
+              puts "Received result for #{__id__}" if @debug
+              requests[__id__][:response] = payload
+              requests[__id__][:lock].synchronize { requests[__id__][:condition].signal }
+            rescue nil
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+__END__
+begin 
+  client = DataClient.new
+  reply = client.send_command( { command: 'ping' } ) #{ command: :hist, contract: 'A6Z21', con_id: 259130514, interval: :min15 } )
+  puts reply.nil? ? 'nil' : JSON.parse(reply)
+  reply = client.get_historical( contract: 'A6Z21',  con_id: 259130514, interval: :min15 , rth_only: false)
+  JSON.parse(reply, symbolize_names: true)[:result].map{|z| 
+    z[:datetime] = Cotcube::Helpers::CHICAGO.parse(z[:time]).strftime('%Y-%m-%d %H:%M:%S')
+    z.delete(:created_at)
+    z.delete(:time)
+    p z.slice(*%i[datetime open high low close volume]).values
+
+  } 
+ensure
+  client.stop
+  e,nd

data/lib/cotcube-helpers/expiration.rb

@@ -0,0 +1,31 @@
+module Cotcube
+  module Helpers
+
+    class ExpirationMonth
+      attr_accessor *%i[ asset month year holidays stencil ]
+      def initialize( contract: )
+        a,b,c,d,e = contract.chars
+        @asset  = [ a, b ].join
+        if %w[ GG DL BJ GE VI ] 
+          puts "Denying to calculate expiration for #{@asset}".light_red
+          return
+        end
+        @month  = MONTHS[c] + offset
+        @month -= 1 if %w[ CL HO NG RB SB].include? @asset
+        @month += 1 if %w[ ].include? @asset
+        @month += 12 if month < 1
+        @month -= 12 if month > 12
+        @year   = [ d, e ].join.to_i
+        @year  += year > 61 ? 1900 : 2000
+        @holidays = CSV.read("/var/cotcube/bardata/holidays.csv").map{|x| DateTime.parse(x[0]).to_date}.select{|x| x.year == @year }
+        @stencil  = [ Date.new(@year, @month, 1) ] 
+        end_date = Date.new(@year, @month + 1, 1 )
+        while (next_date = @stencil.last + 1) < end_date
+          @stencil << next_date
+        end
+        @stencil.reject!{|x| [0,6].include?(x.wday) or @holidays.include? x}
+      end
+    end
+
+  end
+end

data/lib/cotcube-helpers/get_id_set.rb

@@ -3,12 +3,13 @@
 module Cotcube
   module Helpers
 
-    def get_id_set(symbol: nil, id: nil, contract: nil, config: init)
+    def get_id_set(symbol: nil, id: nil, contract: nil, config: init, mini: false, micro: false)
+      micro = mini || micro
       contract = contract.to_s.upcase if contract.is_a? Symbol
       id       =       id.to_s.upcase if       id.is_a? Symbol
       symbol   =   symbol.to_s.upcase if   symbol.is_a? Symbol
 
-      if contract.is_a?(String) && (contract.length == 5)
+      if contract.is_a?(String) && ([2,3,4,5].include? contract.length)
         c_symbol = contract[0..1]
         if (not symbol.nil?) && (symbol != c_symbol)
           raise ArgumentError,
@@ -19,25 +20,26 @@ module Cotcube
       end
 
       unless symbol.nil?
-        sym = symbols.select { |s| s[:symbol] == symbol.to_s.upcase }.first
+        sym = symbols(symbol: symbol).presence || micros(symbol: symbol)
         if sym.nil? || sym[:id].nil?
           raise ArgumentError,
-                "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}"
+                "Could not find match in #{config[:symbols_file]} or #{config[:micros_file]} for given symbol #{symbol}"
         end
         raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if (not id.nil?) && (sym[:id] != id)
 
-        return sym
+        return micro ? micros(id: sym[:id])  : sym
       end
       unless id.nil?
-        sym = symbols.select { |s| s[:id] == id.to_s }.first
+        sym = symbols(id: id)
         if sym.nil? || sym[:id].nil?
           raise ArgumentError,
                 "Could not find match in #{config[:symbols_file]} for given id #{id}"
         end
-        return sym
+        return micro ? micros(id: sym[:id]) : sym
       end
       raise ArgumentError, 'Need :id, :symbol or valid :contract '
     end
+
   end
 end
 

data/lib/cotcube-helpers/ib_contracts.rb

@@ -0,0 +1,69 @@
+module Cotcube
+  module Helpers
+    def get_ib_contract(contract)
+      symbol = contract[..1]
+      # TODO consider file location to be found in configfile
+      filepath =  '/etc/cotcube/ibsymbols/'
+      result = YAML.load(File.read( "#{filepath}/#{symbol}.yml"))[contract].transform_keys(&:to_sym) rescue nil
+      result.nil? ? update_ib_contracts(symbol: contract[..1]) : (return result)
+      YAML.load(File.read( "#{filepath}/#{symbol}.yml"))[contract].transform_keys(&:to_sym) rescue nil
+    end
+
+    def update_ib_contracts(symbol: nil)
+      begin
+        client = DataClient.new
+        (Cotcube::Helpers.symbols + Cotcube::Helpers.micros).each do |sym|
+
+          # TODO: consider file location to be located in config
+          file = "/etc/cotcube/ibsymbols/#{sym[:symbol]}.yml"
+
+          # TODO: VI publishes weekly options which dont match, the 3 others need multiplier enabled to work
+          next if %w[ DY TM SI VI ].include? sym[:symbol]
+          next if symbol and sym[:symbol] != symbol
+          begin
+            if File.exist? file
+              next if Time.now - File.mtime(file) < 5.days
+              data = nil
+              data = YAML.load(File.read(file))
+            else
+              data = {}
+            end
+            p file
+            %w[ symbol sec_type exchange multiplier ticksize power internal ].each {|z| data.delete z}
+            raw   = client.get_contracts(symbol: sym[:symbol])
+            reply = JSON.parse(raw)['result']
+            reply.each do |set|
+              contract = translate_ib_contract set['local_symbol']
+              data[contract] ||= set
+            end
+            keys = data.keys.sort_by{|z| z[2]}.sort_by{|z| z[-2..] }.select{|z| z[..1] == sym[:symbol] }
+            data = data.slice(*keys)
+            File.open(file, 'w'){|f| f.write(data.to_yaml) }
+          rescue Exception => e
+            puts e.full_message
+            p sym
+            binding.irb
+          end
+        end
+      ensure 
+        client.stop
+        true
+      end
+    end
+
+    def translate_ib_contract(contract)
+      short = contract.split(" ").size == 1
+      sym_a = contract.split(short ? '' : ' ')
+      year  = sym_a.pop.to_i + (short ? 20 : 0)
+      if short and sym_a[-1].to_i > 0
+        year = year - 20 + sym_a.pop.to_i * 10
+      end
+      month = short ? sym_a.pop : LETTERS[sym_a.pop]
+      sym   = Cotcube::Helpers.symbols(internal: sym_a.join)[:symbol] rescue nil
+      sym ||= Cotcube::Helpers.micros(internal: sym_a.join)[:symbol] rescue nil
+      sym.nil? ? false : "#{sym}#{month}#{year}"
+    end
+
+  end
+end
+

data/lib/cotcube-helpers/init.rb

@@ -23,10 +23,11 @@ module Cotcube
              gem_name: nil,
              debug: false)
       gem_name ||= self.ancestors.first.to_s
-      config_file_name = "#{gem_name.downcase}.yml"
+      config_file_name = "#{gem_name.downcase.split('::').last}.yml"
       config_file = config_path + "/#{config_file_name}"
 
       if File.exist?(config_file)
+        require 'yaml'
         config      = YAML.load(File.read config_file).transform_keys(&:to_sym)
       else
         config      = {} 

data/lib/cotcube-helpers/numeric_ext.rb

@@ -0,0 +1,8 @@
+class Numeric
+  def with_delimiter(deli=nil)
+    raise ArgumentError, "Param delimiter can't be nil" if deli.nil?
+    pre, post = self.to_s.split('.')
+    pre = pre.chars.to_a.reverse.each_slice(3).map(&:join).join(deli).reverse
+    post.nil? ? pre : [pre,post].join('.')
+  end
+end