cotcube-bardata 0.1.1 → 0.1.6

data/lib/cotcube-bardata/daily.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module Cotcube
+  # Missing top level documentation comment
+  module Bardata
+    # just reads bardata/daily/<id>/<contract>.csv
+    def provide_daily(contract:,
+                      symbol: nil, id: nil,
+                      timezone: Time.find_zone('America/Chicago'),
+                      config: init)
+      contract = contract.to_s.upcase
+      unless contract.is_a?(String) && [3, 5].include?(contract.size)
+        raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
+      end
+
+      sym = get_id_set(symbol: symbol, id: id, contract: contract)
+      contract = contract[2..4] if contract.to_s.size == 5
+      id = sym[:id]
+      id_path   = "#{config[:data_path]}/daily/#{id}"
+      data_file = "#{id_path}/#{contract}.csv"
+      raise "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)
+
+      raise "No data found for requested contract #{symbol}:#{contract} in #{id_path}." unless File.exist?(data_file)
+
+      data = CSV.read(data_file, headers: %i[contract date open high low close volume oi]).map do |row|
+        row = row.to_h
+        row.each do |k, _|
+          row[k] = row[k].to_f if %i[open high low close].include? k
+          row[k] = row[k].to_i if %i[volume oi].include? k
+        end
+        row[:datetime] = timezone.parse(row[:date])
+        row[:type]     = :daily
+        row
+      end
+      data.pop if data.last[:high].zero?
+      data
+    end
+
+    # reads all files in  bardata/daily/<id> and aggregates by date
+    # (what is a pre-stage of a continuous based on daily bars)
+    def continuous(symbol: nil, id: nil, config: init, date: nil)
+      sym = get_id_set(symbol: symbol, id: id)
+      id  = sym[:id]
+      id_path = "#{config[:data_path]}/daily/#{id}"
+      available_contracts = Dir["#{id_path}/*.csv"].map { |x| x.split('/').last.split('.').first }
+      available_contracts.sort_by! { |x| x[-7] }.sort_by! { |x| x[-6..-5] }
+      data = []
+      available_contracts.each do |c|
+        provide_daily(id: id, config: config, contract: c).each do |x|
+          data << x
+        end
+      end
+      result = []
+      data.sort_by { |x| x[:date] }.group_by { |x| x[:date] }.map do |k, v|
+        v.map { |x| x.delete(:date) }
+        result << {
+          date: k,
+          volume: v.map { |x| x[:volume] }.reduce(:+),
+          oi: v.map { |x| x[:oi] }.reduce(:+)
+        }
+        result.last[:contracts] = v
+      end
+      date.nil? ? result : result.select { |x| x[:date] == date }.first
+    end
+
+    def continuous_ml(symbol: nil, id: nil, base: nil)
+      (base.nil? ? Cotcube::Bardata.continuous(symbol: symbol, id: id) : base).map do |x|
+        x[:ml] = x[:contracts].max_by { |z| z[:volume] }[:contract]
+        { date: x[:date], ml: x[:ml] }
+      end
+    end
+
+    # the method above delivers the most_liquid as it is found at the end of the day. D
+    # during trading, the work is done with data
+    # that is already one day old. This is is fixed here:
+    def continuous_actual_ml(symbol: nil, id: nil)
+      continuous =    Cotcube::Bardata.continuous    symbol: symbol, id: id
+      continuous_ml = Cotcube::Bardata.continuous_ml base: continuous
+      continuous_hash = continuous.to_h { |x| [x[:date], x[:contracts]] }
+      actual_ml = continuous_ml.pairwise { |a, b| { date: b[:date], ml: a[:ml] } }
+      actual_ml.map do |x|
+        r = continuous_hash[x[:date]].select { |z| x[:ml] == z[:contract] }.first
+        r = continuous_hash[x[:date]].min_by { |z| -z[:volume] } if r.nil?
+        r
+      end
+    end
+
+    # based on .continuous, this methods sorts the prepared dailies continuous for each date
+    # on either :volume (default) or :oi
+    # with this job done, it can provide the period for which a past contract was the most liquid
+    #
+    def continuous_overview(symbol: nil, id: nil, # rubocop:disable Metrics/ParameterLists
+                            config: init,
+                            selector: :volume,
+                            human: false,
+                            filter: nil)
+      raise ArgumentError, 'Selector must be either :volume or :oi' unless selector.is_a?(Symbol) &&
+                                                                           %i[volume oi].include?(selector)
+
+      sym = get_id_set(symbol: symbol, id: id)
+      id  = sym[:id]
+      # noinspection RubyNilAnalysis
+      data = continuous(id: id, config: config).map do |x|
+        {
+          date: x[:date],
+          volume: x[:contracts].sort_by { |z| - z[:volume] }[0..4].compact.reject { |z| z[:volume].zero? },
+          oi: x[:contracts].sort_by { |z| - z[:oi] }[0..4].compact.reject { |z| z[:oi].zero? }
+        }
+      end
+      data.reject! { |x| x[selector].empty? }
+      result = data.group_by { |x| x[selector].first[:contract] }
+      if human
+        result.each do |k, v|
+          next unless filter.nil? || v.first[selector].first[:contract][2..4] =~ (/#{filter}/)
+
+          # rubocop:disable Layout/ClosingParenthesisIndentation
+          puts "#{k
+             }\t#{v.first[:date]
+             }\t#{v.last[:date]
+             }\t#{format('%4d', (Date.parse(v.last[:date]) - Date.parse(v.first[:date])))
+             }\t#{result[k].map do |x|
+                    x[:volume].select do
+                      x[:contract] == k
+                    end
+                  end.size
+             }"
+          # rubocop:enable Layout/ClosingParenthesisIndentation
+        end
+      end
+      result
+    end
+  end
+end

data/lib/cotcube-bardata/eods.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Cotcube
+  # Missing top level documentation
+  module Bardata
+    def most_liquid_for(symbol: nil, id: nil, date: last_trade_date, config: init)
+      id = get_id_set(symbol: symbol, id: id, config: config)[:id]
+      provide_eods(id: id, dates: date, contracts_only: true).first
+    end
+
+    # the following method seems to be garbage. It is not used anywhere. It seems it's purpose
+    # was to retrieve a list of quarters that have not been fetched recently (--> :age)
+    def provide_most_liquids_by_eod(symbol: nil, id: nil, # rubocop:disable Metrics/ParameterLists
+                                    config: init,
+                                    date: last_trade_date,
+                                    filter: :volume_part,
+                                    age: 1.hour)
+      sym  = get_id_set(symbol: symbol, id: id) if symbol || id
+      # noinspection RubyScope
+      eods = provide_eods(id: sym.nil? ? nil : sym[:id], config: config, dates: date, filter: filter)
+      result = []
+      eods.map do |eod|
+        symbol   = eod[0..1]
+        contract = eod[2..4]
+        sym      = symbols.select { |s| s[:symbol] == symbol.to_s.upcase }.first
+        quarter  = "#{config[:data_path]}/quarters/#{sym[:id]}/#{contract}.csv"
+        if File.exist?(quarter)
+          # puts "#{quarter}: #{ Time.now } - #{File.mtime(quarter)} > #{age} : #{Time.now - File.mtime(quarter) > age}"
+          result << eod if Time.now - File.mtime(quarter) > age
+        else
+          result << eod
+        end
+      end
+      result
+    end
+
+    # provide a list of all eods for id/symbol or all symbols (default) for an
+    #   array of dates (default: [last_trade_date])
+    #
+    # filter by :threshold*100% share on entire volume(default) or oi
+    #
+    # return full data or just the contract name (default)
+    def provide_eods(symbol: nil, # rubocop:disable Metrics/ParameterLists
+                     id: nil,
+                     contract: nil,
+                     config: init,
+                     # should accept either a date or date_alike or date string OR a range of 2 dates alike
+                     # if omitted returns the eods of last trading date
+                     dates: last_trade_date,
+                     # set threshold to 0 to disable filtering at all.
+                     # otherwise only contracts with partial of >= threshold are returned
+                     threshold: 0.05,
+                     # filter can be set to volume_part and oi_part.
+                     # determines, which property is used for filtering.
+                     filter: :volume_part,
+                     # set to false to return the complete row instead
+                     # of just the contracts matching filter and threshold
+                     contracts_only: true,
+                     quiet: false)
+      unless contract.nil? || (contract.is_a?(String) && [3, 5].include?(contract.size))
+        raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
+      end
+
+      symbol = contract[0..1] if contract.to_s.size == 5
+      sym = get_id_set(symbol: symbol, id: id, config: config) if symbol || id
+      # if no id can be clarified from given arguments, return all matching contracts from all available symbols
+      # raise ArgumentError, "Could not guess :id or :symbol from 'contract: #{contract}', please clarify." if id.nil?
+      raise ArgumentError, ':filter must be in [:volume_part, :oi_part]' unless %i[volume_part oi_part].include? filter
+
+      # noinspection RubyScope
+      ids = sym.nil? ? symbols.map { |x| x[:id] } : [sym[:id]]
+      dates = [dates] unless dates.is_a?(Array) || dates.nil?
+
+      id_path_get = ->(local_id) { "#{config[:data_path]}/eods/#{local_id}" }
+
+      process_date_for_id = lambda do |d, i|
+        # l_sym        = symbols.select { |s| s[:id] == i }.first
+        # l_symbol     = l_sym[:symbol]
+        id_path    = id_path_get.call(i)
+        data_file  = "#{id_path}/#{d}.csv"
+        raise "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)
+
+        unless File.exist?(data_file)
+          unless quiet
+            puts 'WARNING: No data found for requested id/symbol'\
+            " #{id}/#{symbol} in #{id_path} for #{d}.".colorize(:light_yellow)
+          end
+          return []
+        end
+        data = CSV.read(data_file, headers: %i[contract date open high low close volume oi]).map do |row|
+          row = row.to_h
+          row.each do |k, _|
+            row[k] = row[k].to_f if %i[open high low close].include? k
+            row[k] = row[k].to_i if %i[volume oi].include? k
+          end
+          row
+        end
+        all_volume = data.map { |x| x[:volume] }.reduce(:+)
+        all_oi     = data.map { |x| x[:oi]     }.reduce(:+)
+        data.map do |x|
+          x[:volume_part] = (x[:volume] / all_volume.to_f).round(4)
+          x[:oi_part]     = (x[:oi] / all_oi.to_f).round(4)
+        end
+        data.select { |x| x[filter] >= threshold }.sort_by { |x| -x[filter] }.tap do |x|
+          if contracts_only
+            x.map! do |y|
+              y[:contract]
+            end
+          end
+        end
+      end
+      if dates
+        dates.map do |date|
+          ids.map { |local_id| process_date_for_id.call(date, local_id) }
+        end.flatten
+      else
+        raise ArgumentError,
+              'Sorry, support for unlimited dates is not implemented yet. Please send array of dates or single date'
+      end
+    end
+  end
+end

data/lib/cotcube-bardata/helpers.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Cotcube
+  # Missing top level documentation comment
+  module Bardata
+    # small helper to select a specific full trading day from quarters (or reduced)
+    #   this special handling is needed, as full trading days start '5pm CT yesterday'
+    def select_specific_date(date:, base:)
+      base.select do |d|
+        d[:day] == date.day and date.year == d[:datetime].year and (
+        if date.day > 1
+          date.month == d[:datetime].month
+        else
+          ((date.month == d[:datetime].month     and d[:datetime].day == 1) or
+           (date.month == d[:datetime].month + 1 and d[:datetime].day > 25))
+        end
+      )
+      end
+    end
+
+    # diminishes a given base of bars to fit into a given range (DO NOT CONFUSE with trading_hours)
+    # note that the last bar is simply required to _start_ within the given range, not to end withing
+    def extended_select_for_range(base:,
+                                  range: ('1900-01-01'...'2100-01-01'),
+                                  timezone: Time.find_zone('America/Chicago'),
+                                  quiet: false)
+
+      starting = range.begin
+      starting = timezone.parse(starting) if starting.is_a? String
+      ending   = range.end
+      ending   = timezone.parse(ending) if ending.is_a? String
+      puts "#{starting}\t#{ending}" unless quiet
+      if starting.hour.zero? && starting.min.zero? && ending.hour.zero? && ending.min.zero?
+        unless quiet
+          puts 'WARNING: When sending midnight, full trading day'\
+            ' is assumed (starting 5 pm CT yesterday, ending 4 pm CT today)'.colorize(:light_yellow)
+        end
+        result = select_specific_date(date: starting, base: base)
+        result += base.select { |d| d[:datetime] > starting and d[:datetime] < ending.to_date }
+        result += select_specific_date(date: ending, base: base)
+        result.uniq!
+      else
+        result = base.select { |x| x[:datetime] >= starting and x[:datetime] < ending }
+      end
+      result
+    end
+
+    def get_id_set(symbol: nil, id: nil, contract: nil, config: init)
+      if contract.is_a?(String) && (contract.length == 5)
+        c_symbol = contract[0..1]
+        if (not symbol.nil?) && (symbol != c_symbol)
+          raise ArgumentError,
+                "Mismatch between given symbol #{symbol} and contract #{contract}"
+        end
+
+        symbol = c_symbol
+      end
+
+      unless symbol.nil?
+        sym = symbols.select { |s| s[:symbol] == symbol.to_s.upcase }.first
+        if sym.nil? || sym[:id].nil?
+          raise ArgumentError,
+                "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}"
+        end
+        raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if (not id.nil?) && (sym[:id] != id)
+
+        return sym
+      end
+      unless id.nil?
+        sym = symbols.select { |s| s[:id] == id.to_s }.first
+        if sym.nil? || sym[:id].nil?
+          raise ArgumentError,
+                "Could not find match in #{config[:symbols_file]} for given id #{id}"
+        end
+        return sym
+      end
+      raise ArgumentError, 'Need :id, :symbol or valid :contract '
+    end
+
+    def compare(contract:, format: '%5.2f')
+      format = "%#{format}" unless format[0] == '%'
+      daily = provide(contract: contract, interval: :daily)
+      full  = provide(contract: contract, interval: :days, filter: :full)
+      rth   = provide(contract: contract, interval: :days, filter: :rth)
+      rth_dates = rth.map { |x| x[:datetime] }
+      daily.select! { |x| rth_dates.include? x[:datetime].to_datetime }
+      full.select! {  |x| rth_dates.include? x[:datetime].to_datetime }
+
+      printer = lambda { |z|
+           # rubocop:disable Layout/ClosingParenthesisIndentation
+           "#{z[:datetime].strftime('%m-%d') # rubocop:disable Layout/IndentationWidth
+         }\t#{format format, z[:open]
+         }\t#{format format, z[:high]
+         }\t#{format format, z[:low]
+         }\t#{format format, z[:close]
+         }\t#{format '%7d', z[:volume]}"
+        # rubocop:enable Layout/ClosingParenthesisIndentation
+      }
+      daily.each_with_index do |_x, i|
+        puts "DAILY #{printer.call daily[i]}"
+        puts "FULL  #{printer.call full[i]}"
+        puts "RTH   #{printer.call rth[i]}"
+        puts ' '
+      end
+    end
+  end
+end

data/lib/cotcube-bardata/init.rb

@@ -1,80 +1,77 @@
 # frozen_string_literal: true
 
 module Cotcube
+  # Missing top level documentation comment
   module Bardata
-
-
-    def symbols(config: init)
+    def symbols(config: init, type: nil, symbol: nil)
       if config[:symbols_file].nil?
         SYMBOL_EXAMPLES
       else
         CSV
-          .read(config[:symbols_file], headers: %i{ id symbol ticksize power months type bcf reports name})
-          .map{|row| row.to_h }
-          .map{|row| [ :ticksize, :power, :bcf ].each {|z| row[z] = row[z].to_f}; row }
-          .reject{|row| row[:id].nil? }
+          .read(config[:symbols_file], headers: %i[id symbol ticksize power months type bcf reports name])
+          .map(&:to_h)
+          .map { |row| %i[ticksize power bcf].each { |z| row[z] = row[z].to_f }; row } # rubocop:disable Style/Semicolon
+          .reject { |row| row[:id].nil? }
+          .tap { |all| all.select! { |x| x[:type] == type } unless type.nil? }
+          .tap { |all| all.select! { |x| x[:symbol] == symbol } unless symbol.nil? }
       end
     end
 
     def config_prefix
-      os       = Gem::Platform.local.os
+      os = Gem::Platform.local.os
       case os
       when 'linux'
         ''
       when 'freebsd'
         '/usr/local'
       else
-        raise RuntimeError, 'unknown architecture'
+        raise 'unknown architecture'
       end
     end
 
     def config_path
-      config_prefix + '/etc/cotcube'
+      "#{config_prefix}/etc/cotcube"
     end
 
-    def init(config_file_name: 'bardata.yml', debug: false)
+    def init(config_file_name: 'bardata.yml')
       name = 'bardata'
       config_file = config_path + "/#{config_file_name}"
 
-      if File.exist?(config_file)
-        config      = YAML.load(File.read config_file).transform_keys(&:to_sym)
-      else
-        config      = {} 
-      end
+      config = if File.exist?(config_file)
+                 YAML.safe_load(File.read(config_file)).transform_keys(&:to_sym)
+               else
+                 {}
+               end
 
-      defaults = { 
-        data_path: config_prefix + '/var/cotcube/' + name,
+      defaults = {
+        data_path: "#{config_prefix}/var/cotcube/#{name}"
       }
 
-
       config = defaults.merge(config)
 
-
       # part 2 of init process: Prepare directories
 
       save_create_directory = lambda do |directory_name|
         unless Dir.exist?(directory_name)
           begin
             `mkdir -p #{directory_name}`
-            unless $?.exitstatus.zero?
+            unless $CHILD_STATUS.exitstatus.zero?
               puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
               exit 1 unless defined?(IRB)
             end
-          rescue 
+          rescue StandardError
             puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
             exit 1 unless defined?(IRB)
           end
         end
       end
-      ['',:daily,:quarters].each do |path| 
-        dir = "#{config[:data_path]}#{path == '' ? '' : '/'}#{path.to_s}"
+      ['', :daily, :quarters, :eods, :trading_hours, :cached].each do |path|
+        dir = "#{config[:data_path]}#{path == '' ? '' : '/'}#{path}"
         save_create_directory.call(dir)
       end
 
       # eventually return config
       config
     end
-
   end
 end
-