cotcube-bardata 0.1.2 → 0.1.7

data/lib/cotcube-bardata/init.rb

@@ -1,80 +1,77 @@
 # frozen_string_literal: true
 
 module Cotcube
+  # Missing top level documentation comment
   module Bardata
-
-
-    def symbols(config: init)
+    def symbols(config: init, type: nil, symbol: nil)
       if config[:symbols_file].nil?
         SYMBOL_EXAMPLES
       else
         CSV
-          .read(config[:symbols_file], headers: %i{ id symbol ticksize power months type bcf reports name})
-          .map{|row| row.to_h }
-          .map{|row| [ :ticksize, :power, :bcf ].each {|z| row[z] = row[z].to_f}; row }
-          .reject{|row| row[:id].nil? }
+          .read(config[:symbols_file], headers: %i[id symbol ticksize power months type bcf reports name])
+          .map(&:to_h)
+          .map { |row| %i[ticksize power bcf].each { |z| row[z] = row[z].to_f }; row } # rubocop:disable Style/Semicolon
+          .reject { |row| row[:id].nil? }
+          .tap { |all| all.select! { |x| x[:type] == type } unless type.nil? }
+          .tap { |all| all.select! { |x| x[:symbol] == symbol } unless symbol.nil? }
       end
     end
 
     def config_prefix
-      os       = Gem::Platform.local.os
+      os = Gem::Platform.local.os
       case os
       when 'linux'
         ''
       when 'freebsd'
         '/usr/local'
       else
-        raise RuntimeError, 'unknown architecture'
+        raise 'unknown architecture'
       end
     end
 
     def config_path
-      config_prefix + '/etc/cotcube'
+      "#{config_prefix}/etc/cotcube"
     end
 
-    def init(config_file_name: 'bardata.yml', debug: false)
+    def init(config_file_name: 'bardata.yml')
       name = 'bardata'
       config_file = config_path + "/#{config_file_name}"
 
-      if File.exist?(config_file)
-        config      = YAML.load(File.read config_file).transform_keys(&:to_sym)
-      else
-        config      = {} 
-      end
+      config = if File.exist?(config_file)
+                 YAML.safe_load(File.read(config_file)).transform_keys(&:to_sym)
+               else
+                 {}
+               end
 
-      defaults = { 
-        data_path: config_prefix + '/var/cotcube/' + name,
+      defaults = {
+        data_path: "#{config_prefix}/var/cotcube/#{name}"
       }
 
-
       config = defaults.merge(config)
 
-
       # part 2 of init process: Prepare directories
 
       save_create_directory = lambda do |directory_name|
         unless Dir.exist?(directory_name)
           begin
             `mkdir -p #{directory_name}`
-            unless $?.exitstatus.zero?
+            unless $CHILD_STATUS.exitstatus.zero?
               puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
               exit 1 unless defined?(IRB)
             end
-          rescue 
+          rescue StandardError
             puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
             exit 1 unless defined?(IRB)
           end
         end
       end
-      ['',:daily,:quarters].each do |path| 
-        dir = "#{config[:data_path]}#{path == '' ? '' : '/'}#{path.to_s}"
+      ['', :daily, :quarters, :eods, :trading_hours, :cached].each do |path|
+        dir = "#{config[:data_path]}#{path == '' ? '' : '/'}#{path}"
         save_create_directory.call(dir)
       end
 
       # eventually return config
       config
     end
-
   end
 end
-

data/lib/cotcube-bardata/provide.rb

@@ -1,23 +1,48 @@
 # frozen_string_literal: true
 
 module Cotcube
+  # Missing top level documentation comment
   module Bardata
+    def provide(contract:, # rubocop:disable Metrics/ParameterLists
+                # Can be like ("2020-12-01 12:00"..."2020-12-14 11:00")
+                range: nil,
+                symbol: nil, id: nil,
+                config: init,
+                # supported types are :quarters, :hours, :days, :rth, :dailies, :weeks, :months
+                interval: :days,
+                # supported filters are :full and :rth (and custom named, if provided as file)
+                filter: :full,
+                # TODO: for future compatibility and suggestion: planning to include a function to update
+                #       with live data from broker
+                force_recent: false)
 
-    def provide(symbol: nil, id: nil, contract:, config: init, date: Date.today - 1, type: nil, fill: :none)
-      case type
-      when :eod, :eods
-        provide_eods(symbol: symbol, id: id, contract: contract, date: date) 
-      when :quarters
-        print :quarters
-      when :hours
-        print :hours
-      when :daily, :dailies
-        print :dailies
+      sym = get_id_set(symbol: symbol, id: id, contract: contract, config: config)
+
+      case interval
+      when :quarters, :hours, :quarter, :hour
+        base = provide_quarters(contract: contract, symbol: symbol, id: id, config: config)
+        base = extended_select_for_range(range: range, base: base) if range
+        requested_set = trading_hours(symbol: sym[:symbol], filter: filter)
+
+        base = base.select_within(ranges: requested_set, attr: :datetime) { |x| x.to_datetime.to_sssm }
+        return base if %i[quarters quarter].include? interval
+
+        Cotcube::Helpers.reduce(bars: base, to: :hours) do |c, b|
+          c[:day] == b[:day] and c[:datetime].hour == b[:datetime].hour
+        end
+
+      when :days, :weeks, :months
+        base = provide_cached contract: contract, symbol: symbol, id: id, config: config, filter: filter,
+                              range: range, force_recent: force_recent
+        return base if %i[day days].include? interval
+
+        # TODO: Missing implementation to reduce cached days to weeks or months
+        raise 'Missing implementation to reduce cached days to weeks or months'
+      when :dailies, :daily
+        provide_daily contract: contract, symbol: symbol, id: id, config: config, range: range
       else
-        puts "WARNING: Using provide without :type is for legacy support pointing to .provide_daily".light_yellow
-        provide_daily(symbol: symbol, id: id, contract: contract, config: config)
+        raise ArgumentError, "Unsupported or unknown interval '#{interval}' in Bardata.provide"
       end
     end
   end
-
 end

data/lib/cotcube-bardata/quarters.rb

@@ -1,80 +1,43 @@
 # frozen_string_literal: true
 
 module Cotcube
+  # Missing top level documentation
   module Bardata
+    # the following method loads the quarterly bars (15-min bars) from the directory tree
+    # also note that a former version of this method allowed to provide range or date parameters. this has been moved
+    # to #provide itself.
+    def provide_quarters(contract:, # rubocop:disable Metrics/ParameterLists
+                         symbol: nil, id: nil,
+                         timezone: Time.find_zone('America/Chicago'),
+                         config: init,
+                         keep_marker: false)
 
-    def provide_quarters(
-      symbol: nil, id: nil, 
-      contract:, 
-      as: :quarters, 
-      range: nil, date: nil, 
-      timezone: Time.find_zone('America/Chicago'),
-      config: init,
-      quiet: false
-    )
-      date = timezone.parse(date) if date.is_a? String
-      raise ArgumentError, ":range and :date are mutually exclusive" if range and date
-      raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'" unless contract.is_a? String and [3,5].include? contract.size
-      if contract.to_s.size == 5
-        symbol   = contract[0..1]
-        contract = contract[2..4] 
+      unless contract.is_a?(String) && [3, 5].include?(contract.size)
+        raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
       end
-      unless symbol.nil?
-        symbol_id = symbols.select{|s| s[:symbol] == symbol.to_s.upcase}.first[:id]
-        raise ArgumentError, "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}" if symbol_id.nil?
-        raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if not id.nil? and symbol_id != id
-        id = symbol_id
-      end
-      raise ArgumentError, ":as can only be in [:quarters, :hours, :days]" unless %i[quarters hours days].include?(as)
-      raise ArgumentError, "Could not guess :id or :symbol from 'contract: #{contract}', please clarify." if id.nil?
+
+      sym = get_id_set(symbol: symbol, id: id, contract: contract)
+
+      contract = contract[2..4] if contract.to_s.size == 5
+      id = sym[:id]
+      symbol = sym[:symbol]
+
       id_path   = "#{config[:data_path]}/quarters/#{id}"
       data_file = "#{id_path}/#{contract}.csv"
-      raise RuntimeError, "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)
-      raise RuntimeError, "No data found for requested contract #{symbol}:#{contract} in #{id_path}." unless File.exist?(data_file)
-      data      = CSV.read(data_file, headers: %i[contract datetime day open high low close volume]).map do |row|
+      raise "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)
+
+      raise "No data found for requested contract #{symbol}:#{contract} in #{id_path}." unless File.exist?(data_file)
+
+      data = CSV.read(data_file, headers: %i[contract datetime day open high low close volume]).map do |row|
         row = row.to_h
-        %i[open high low close].map{|x| row[x] = row[x].to_f}
-        %i[volume day].map{|x|         row[x] = row[x].to_i}
+        %i[open high low close].map { |x| row[x] = row[x].to_f }
+        %i[volume day].map { |x| row[x] = row[x].to_i }
         row[:datetime] = timezone.parse(row[:datetime])
+        row[:type]     = :quarter
         row
       end
-      select_specific_date = lambda do |specific_date|
-        data.select{|d| d[:day] == specific_date.day and specific_date.year == d[:datetime].year and (
-          if specific_date.day > 1
-            specific_date.month == d[:datetime].month
-          else
-            ((specific_date.month == d[:datetime].month     and d[:datetime].day == 1) or
-             (specific_date.month == d[:datetime].month + 1 and d[:datetime].day  > 25) )
-          end
-        )}
-      end
-      if range
-        starting = range.begin
-        starting = timezone.parse(starting) if starting.is_a? String
-        ending   = range.end
-        ending   = timezone.parse(  ending) if ending.is_a? String
-        if starting.hour == 0 and starting.min == 0 and ending.hour == 0 and ending.min == 0
-          puts "WARNING: When sending midnight, full trading day is assumed (starting 5 pm yesterday, ending 4 pm today)".light_yellow unless quiet
-          result = select_specific_date.call(starting)
-          result += data.select{|d| d[:datetime] > starting and d[:datetime] < ending.to_date }
-          result += select_specific_date.call(ending)
-          result.uniq!
-        else
-          result = data.select{|x| x[:datetime] >= starting and x[:datetime] < ending }
-        end
-      elsif date
-        result = select_specific_date.call(date)
-      else 
-        result = data
-      end
-      return case as
-      when :hours
-        Cotcube::Helpers.reduce(bars: result, to: 1.hour){|c,b| c[:day] == b[:day] and c[:datetime].hour == b[:datetime].hour }
-      when :days
-        Cotcube::Helpers.reduce(bars: result, to: 1.day ){|c,b| c[:day] == b[:day] }
-      else
-        result
-      end
+      data.pop if data.last[:high].zero? && (not keep_marker)
+      data
     end
   end
 end

data/lib/cotcube-bardata/range_matrix.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Cotcube
+  # Missing top level documentation
+  module Bardata
+    # this is an analysis tool to investigate actual ranges of an underlying symbol
+    # it is in particular no true range or average true range, as a 'true range' can only be applied to
+    # a steady series, what changing contracts definitely aren't
+    #
+    # The result printed / returned is a table, containing a matrix of rows:
+    #   1. size: the amount of values evaluated
+    #   2. avg:
+    #   3. lower: like median, but not at 1/2 but 1/4
+    #   4. median:
+    #   5. upper: like median, bot not at 1/2 but 3/4
+    #   6. max:
+    # and columns:
+    #   1.a) all days os the series
+    #   1.b) all days of the series, diminished by 2* :dim*100% extreme values (i.e. at both ends)
+    #   1.c) the last 200 days
+    #   2.a-c) same with days reduced to weeks (c: 52 weeks)
+    #   3.a-c) same with days reduced to months (c: 12 months)
+    def range_matrix(symbol: nil, id: nil, print: false, dim: 0.05)
+      # rubocop:disable Style/MultilineBlockChain
+      sym = get_id_set(symbol: symbol, id: id)
+      source = {}
+      target = {}
+      source[:days]   = Cotcube::Bardata.continuous_actual_ml symbol: symbol
+      source[:weeks]  = Cotcube::Helpers.reduce bars: source[:days], to: :weeks
+      source[:months] = Cotcube::Helpers.reduce bars: source[:days], to: :months
+
+      %i[days weeks months].each do |period|
+        source[period].map! do |x|
+          x[:range] = ((x[:high] - x[:low]) / sym[:ticksize]).round
+          x
+        end
+        target[period] = {}
+        target[period][:all_size]   =  source[period].size
+        target[period][:all_avg]    = (source[period].map { |x| x[:range] }.reduce(:+) / source[period].size).round
+        target[period][:all_lower]  =  source[period].sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (source[period].size * 1 / 4).round ]
+        target[period][:all_median] =  source[period].sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (source[period].size * 2 / 4).round ]
+        target[period][:all_upper]  =  source[period].sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (source[period].size * 3 / 4).round ]
+        target[period][:all_max] = source[period].map { |x| x[:range] }.max
+        target[period][:all_records] = source[period].sort_by do |x|
+                                         -x[:range]
+                                       end.map { |x| { contract: x[:contract], range: x[:range] } }.take(5)
+
+        tenth = (source[period].size * dim).round
+        custom = source[period].sort_by { |x| x[:range] }[tenth..source[period].size - tenth]
+        target[period][:dim_size]   =  custom.size
+        target[period][:dim_avg]    = (custom.map { |x| x[:range] }.reduce(:+) / custom.size).round
+        target[period][:dim_lower]  =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 1 / 4).round ]
+        target[period][:dim_median] =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 2 / 4).round ]
+        target[period][:dim_upper]  =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 3 / 4).round ]
+        target[period][:dim_max]    =  custom.map { |x| x[:range] }.max
+        target[period][:dim_records] = custom.sort_by do |x|
+                                         -x[:range]
+                                       end.map { |x| { contract: x[:contract], range: x[:range] } }.take(5)
+
+        range = case period
+                when :months
+                  -13..-2
+                when :weeks
+                  -53..-2
+                when :days
+                  -200..-1
+                else
+                  raise ArgumentError, "Unsupported period: '#{period}'"
+                end
+        custom = source[period][range]
+        target[period][:rec_size]   =  custom.size
+        target[period][:rec_avg]    = (custom.map { |x| x[:range] }.reduce(:+) / custom.size).round
+        target[period][:rec_lower]  =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 1 / 4).round ]
+        target[period][:rec_median] =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 2 / 4).round ]
+        target[period][:rec_upper]  =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 3 / 4).round ]
+        target[period][:rec_max]    =  custom.map { |x| x[:range] }.max
+        target[period][:rec_records] = custom.sort_by do |x|
+                                         -x[:range]
+                                       end.map { |x| { contract: x[:contract], range: x[:range] } }.take(5)
+      end
+
+      if print
+        %w[size avg lower median upper max].each do |a|
+          print "#{'%10s' % a} | " # rubocop:disable Style/FormatString
+          %i[days weeks months].each do |b|
+            %w[all dim rec].each do |c|
+              print ('%8d' % target[b]["#{c}_#{a}".to_sym]).to_s # rubocop:disable Style/FormatString
+            end
+            print ' | '
+          end
+          puts ''
+        end
+      end
+
+      target
+      # rubocop:enable Style/MultilineBlockChain
+    end
+  end
+end

data/lib/cotcube-bardata/trade_dates.rb

@@ -1,14 +1,27 @@
 # frozen_string_literal: true
 
 module Cotcube
+  # missing top level documentation
   module Bardata
-
-    # fetching official tradedates from CME
+    # fetching official trade dates from CME
+    # it returns the current trade date or, if today isn't a trading day, the last trade date.
     def last_trade_date
-      uri = "https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME"
-      res = nil
-      res = HTTParty.get(uri).parsed_response
-      res.map{|x| a = x["tradeDate"].chars.each_slice(2).map(&:join); "#{a[0]}#{a[1]}-#{a[2]}-#{a[3]}"}.first
+      uri = 'https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME'
+      begin
+        HTTParty.get(uri)
+                .parsed_response
+                .map do |x|
+                  a = x['tradeDate'].chars.each_slice(2).map(&:join)
+                  "#{a[0]}#{a[1]}-#{a[2]}-#{a[3]}"
+                end
+                .first
+      rescue StandardError
+        nil
+      end
+    end
+
+    def holidays(config: init)
+      CSV.read("#{config[:data_path]}/holidays.csv").map{|x| DateTime.parse(x[0])}
     end
 
   end

data/lib/cotcube-bardata/trading_hours.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Cotcube
+  # Missing top level comment
+  module Bardata
+    # returns an Array of ranges containing a week of trading hours, specified by seconds since monday morning
+    #    (as sunday is wday:0)
+    # according files are located in config[:data_path]/trading_hours and picked either
+    # by the symbol itself or by the assigned type
+    # commonly there are two filter for each symbol: :full and :rth, exceptions are e.g. meats
+    def trading_hours(symbol: nil, id: nil, # rubocop:disable Metrics/ParameterLists
+                      filter: ,
+                      force_filter: false,  # with force_filter one would avoid falling back
+                                            # to the contract_type based range set
+                      config: init, debug: false)
+      return (0...24 * 7 * 3600) if filter.to_s =~ /24x7/
+
+      prepare = lambda do |f|
+        CSV.read(f, converters: :numeric)
+           .map(&:to_a)
+           .tap { |x| x.shift unless x.first.first.is_a?(Numeric) }
+           .map { |x| (x.first...x.last) }
+      end
+
+      sym = get_id_set(symbol: symbol, id: id)
+
+      file = "#{config[:data_path]}/trading_hours/#{sym[:symbol]}_#{filter}.csv"
+      puts "Trying to use #{file} for #{symbol} + #{filter}" if debug
+      return prepare.call(file) if File.exist? file
+
+      file = "#{config[:data_path]}/trading_hours/#{sym[:symbol]}_full.csv"
+      puts "Failed. Trying to use #{file} now" if debug
+      return prepare.call(file) if File.exist?(file) && (not force_filter)
+
+      file = "#{config[:data_path]}/trading_hours/#{sym[:type]}_#{filter}.csv"
+      puts "Failed. Trying to use #{file} now." if debug
+      return prepare.call(file) if File.exist? file
+
+      file = "#{config[:data_path]}/trading_hours/#{sym[:type]}_full.csv"
+      puts "Failed. Trying to use #{file} now." if debug
+      return prepare.call(file) if File.exist?(file) && (not force_filter)
+
+      puts "Finally failed to find range filter for #{symbol} + #{filter}, returning 24x7".colorize(:light_yellow)
+      (0...24 * 7 * 3600)
+    end
+  end
+end