cotcube-bardata 0.1.1 → 0.1.6

data/lib/cotcube-bardata/provide.rb

@@ -1,87 +1,51 @@
 # frozen_string_literal: true
 
 module Cotcube
+  # Missing top level documentation comment
   module Bardata
+    def provide(contract:, # rubocop:disable Metrics/ParameterLists
+                # Can be like ("2020-12-01 12:00"..."2020-12-14 11:00")
+                range: nil,
+                symbol: nil, id: nil,
+                config: init,
+                # supported types are :quarters, :hours, :days, :rth, :dailies, :weeks, :months
+                interval: :days,
+                # supported filters are :raw, :_24x7_, :full and :rth (and custom named, if provided as file)
+                filter: :full,
+                # TODO: for future compatibility and suggestion: planning to include a function to update
+                #       with live data from broker
+                force_recent: false)
 
-    def provide(symbol: nil, id: nil, contract:, config: init)
-      raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'" unless contract.is_a? String and [3,5].include? contract.size
-      if contract.to_s.size == 5
-        symbol   = contract[0..1]
-        contract = contract[2..4] 
-      end
-      unless symbol.nil?
-        symbol_id = symbols.select{|s| s[:symbol] == symbol.to_s.upcase}.first[:id]
-        raise ArgumentError, "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}" if symbol_id.nil?
-        raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if not id.nil? and symbol_id != id
-        id = symbol_id
-      end
-      raise ArgumentError, "Could not guess :id or :symbol from 'contract: #{contract}', please clarify." if id.nil?
-      id_path   = "#{config[:data_path]}/daily/#{id}"
-      data_file = "#{id_path}/#{contract}.csv"
-      raise RuntimeError, "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)
-      raise RuntimeError, "No data found for requested contract #{symbol}:#{contract} in #{id_path}." unless File.exist?(data_file)
-      data = CSV.read(data_file, headers: %i[contract date open high low close volume oi] ).map do |row|
-        row = row.to_h
-        row.each do |k, _| 
-          row[k] = row[k].to_f if [:open, :high, :low, :close].include? k
-          row[k] = row[k].to_i if [:volume, :oi].include? k
-        end
-        row
-      end
-      data
-    end
+      sym = get_id_set(symbol: symbol, id: id, contract: contract, config: config)
 
-    def continuous(symbol: nil, id: nil, config: init)
-      unless symbol.nil?
-        symbol_id = symbols.select{|s| s[:symbol] == symbol.to_s.upcase}.first[:id]
-        raise ArgumentError, "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}" if symbol_id.nil?
-        raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if not id.nil? and symbol_id != id
-        id = symbol_id
-      end
-      raise ArgumentError, "Could not guess :id or :symbol, please clarify." if id.nil?
-      id_path    = "#{config[:data_path]}/daily/#{id}"
-      available_contracts = Dir[id_path + '/*.csv'].map{|x| x.split('/').last.split('.').first}.sort_by{|x| x[-7]}.sort_by{|x| x[-6..-5]}
-      data = [] 
-      available_contracts.each do |c| 
-        provide(id: id, config: config, contract: c).each do |x| 
-          data << x
+      case interval
+      when :quarters, :hours, :quarter, :hour
+        base = provide_quarters(contract: contract, symbol: symbol, id: id, config: config)
+        base = extended_select_for_range(range: range, base: base) if range
+        requested_set = trading_hours(symbol: sym[:symbol], filter: filter)
+
+        base = base.select_within(ranges: requested_set, attr: :datetime) { |x| x.to_datetime.to_sssm }
+        return base if %i[quarters quarter].include? interval
+
+        Cotcube::Helpers.reduce(bars: base, to: :hours) do |c, b|
+          c[:day] == b[:day] and c[:datetime].hour == b[:datetime].hour
         end
-      end
-      result = [] 
-      data.sort_by{|x| x[:date]}.group_by{|x| x[:date]}.map{|k,v| 
-        v.map{|x| x.delete(:date)}
-        result << {
-          date: k,
-          volume: v.map{|x| x[:volume]}.reduce(:+),
-          oi:     v.map{|x| x[:oi    ]}.reduce(:+)
-        }
-        result.last[:contracts] = v
-      } 
-      result
-    end
 
-    def continuous_overview(symbol: nil, id: nil, config: init, selector: :volume, human: false, filter: nil)
-      raise ArgumentError, "Selector must be either :volume or :oi" unless selector.is_a? Symbol and [:volume, :oi].include? selector
-      
-      unless symbol.nil?
-        symbol_id = symbols.select{|s| s[:symbol] == symbol.to_s.upcase}.first[:id]
-        raise ArgumentError, "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}" if symbol_id.nil?
-        raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if not id.nil? and symbol_id != id
-        id = symbol_id
-      end
-      raise ArgumentError, "Could not guess :id or :symbol, please clarify." if id.nil?
-      data = continuous(id: id, config: config).map{|x| 
-        {
-          date:   x[:date],
-          volume: x[:contracts].sort_by{|x| - x[:volume]}[0..4].compact.select{|x| not x[:volume].zero?},
-          oi:     x[:contracts].sort_by{|x| - x[:oi]}[0..4].compact.select{|x| not x[:oi].zero?}
-        }
-      }.select{|x| not x[selector].empty? }
-      result = data.group_by{|x| x[selector].first[:contract]}
-      if human
-        result.each {|k,v| puts "#{k}\t#{v.first[:date]}\t#{v.last[:date]}" if filter.nil? or v.first[selector].first[:contract][2..4]=~ /#{filter}/ }
+      when :days, :weeks, :months
+        base = provide_cached contract: contract, symbol: symbol, id: id, config: config, filter: filter,
+                              force_recent: force_recent
+        base = extended_select_for_range(range: range, base: base) if range
+        return base if %i[day days].include? interval
+
+        # TODO: Missing implementation to reduce cached days to weeks or months
+        raise 'Missing implementation to reduce cached days to weeks or months'
+      when :dailies, :daily
+        base = provide_daily contract: contract, symbol: symbol, id: id, config: config
+        base = extended_select_for_range(range: range, base: base) if range
+        base
+      else
+        raise ArgumentError, "Unsupported or unknown interval '#{interval}' in Bardata.provide"
       end
-      result
     end
   end
 end

data/lib/cotcube-bardata/quarters.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Cotcube
+  # Missing top level documentation
+  module Bardata
+    # the following method loads the quarterly bars (15-min bars) from the directory tree
+    # also note that a former version of this method allowed to provide range or date parameters. this has been moved
+    # to #provide itself.
+    def provide_quarters(contract:, # rubocop:disable Metrics/ParameterLists
+                         symbol: nil, id: nil,
+                         timezone: Time.find_zone('America/Chicago'),
+                         config: init,
+                         keep_marker: false)
+
+      unless contract.is_a?(String) && [3, 5].include?(contract.size)
+        raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
+      end
+
+      sym = get_id_set(symbol: symbol, id: id, contract: contract)
+
+      contract = contract[2..4] if contract.to_s.size == 5
+      id = sym[:id]
+      symbol = sym[:symbol]
+
+      id_path   = "#{config[:data_path]}/quarters/#{id}"
+      data_file = "#{id_path}/#{contract}.csv"
+      raise "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)
+
+      raise "No data found for requested contract #{symbol}:#{contract} in #{id_path}." unless File.exist?(data_file)
+
+      data = CSV.read(data_file, headers: %i[contract datetime day open high low close volume]).map do |row|
+        row = row.to_h
+        %i[open high low close].map { |x| row[x] = row[x].to_f }
+        %i[volume day].map { |x| row[x] = row[x].to_i }
+        row[:datetime] = timezone.parse(row[:datetime])
+        row[:type]     = :quarter
+        row
+      end
+      data.pop if data.last[:high].zero? && (not keep_marker)
+      data
+    end
+  end
+end

data/lib/cotcube-bardata/range_matrix.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Cotcube
+  # Missing top level documentation
+  module Bardata
+    # this is an analysis tool to investigate actual ranges of an underlying symbol
+    # it is in particular no true range or average true range, as a 'true range' can only be applied to
+    # a steady series, what changing contracts definitely aren't
+    #
+    # The result printed / returned is a table, containing a matrix of rows:
+    #   1. size: the amount of values evaluated
+    #   2. avg:
+    #   3. lower: like median, but not at 1/2 but 1/4
+    #   4. median:
+    #   5. upper: like median, bot not at 1/2 but 3/4
+    #   6. max:
+    # and columns:
+    #   1.a) all days os the series
+    #   1.b) all days of the series, diminished by 2* :dim*100% extreme values (i.e. at both ends)
+    #   1.c) the last 200 days
+    #   2.a-c) same with days reduced to weeks (c: 52 weeks)
+    #   3.a-c) same with days reduced to months (c: 12 months)
+    def range_matrix(symbol: nil, id: nil, print: false, dim: 0.05)
+      # rubocop:disable Style/MultilineBlockChain
+      sym = get_id_set(symbol: symbol, id: id)
+      source = {}
+      target = {}
+      source[:days]   = Cotcube::Bardata.continuous_actual_ml symbol: symbol
+      source[:weeks]  = Cotcube::Helpers.reduce bars: source[:days], to: :weeks
+      source[:months] = Cotcube::Helpers.reduce bars: source[:days], to: :months
+
+      %i[days weeks months].each do |period|
+        source[period].map! do |x|
+          x[:range] = ((x[:high] - x[:low]) / sym[:ticksize]).round
+          x
+        end
+        target[period] = {}
+        target[period][:all_size]   =  source[period].size
+        target[period][:all_avg]    = (source[period].map { |x| x[:range] }.reduce(:+) / source[period].size).round
+        target[period][:all_lower]  =  source[period].sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (source[period].size * 1 / 4).round ]
+        target[period][:all_median] =  source[period].sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (source[period].size * 2 / 4).round ]
+        target[period][:all_upper]  =  source[period].sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (source[period].size * 3 / 4).round ]
+        target[period][:all_max] = source[period].map { |x| x[:range] }.max
+        target[period][:all_records] = source[period].sort_by do |x|
+                                         -x[:range]
+                                       end.map { |x| { contract: x[:contract], range: x[:range] } }.take(5)
+
+        tenth = (source[period].size * dim).round
+        custom = source[period].sort_by { |x| x[:range] }[tenth..source[period].size - tenth]
+        target[period][:dim_size]   =  custom.size
+        target[period][:dim_avg]    = (custom.map { |x| x[:range] }.reduce(:+) / custom.size).round
+        target[period][:dim_lower]  =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 1 / 4).round ]
+        target[period][:dim_median] =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 2 / 4).round ]
+        target[period][:dim_upper]  =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 3 / 4).round ]
+        target[period][:dim_max]    =  custom.map { |x| x[:range] }.max
+        target[period][:dim_records] = custom.sort_by do |x|
+                                         -x[:range]
+                                       end.map { |x| { contract: x[:contract], range: x[:range] } }.take(5)
+
+        range = case period
+                when :months
+                  -13..-2
+                when :weeks
+                  -53..-2
+                when :days
+                  -200..-1
+                else
+                  raise ArgumentError, "Unsupported period: '#{period}'"
+                end
+        custom = source[period][range]
+        target[period][:rec_size]   =  custom.size
+        target[period][:rec_avg]    = (custom.map { |x| x[:range] }.reduce(:+) / custom.size).round
+        target[period][:rec_lower]  =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 1 / 4).round ]
+        target[period][:rec_median] =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 2 / 4).round ]
+        target[period][:rec_upper]  =  custom.sort_by do |x|
+                                         x[:range]
+                                       end.map { |x| x[:range] }[ (custom.size * 3 / 4).round ]
+        target[period][:rec_max]    =  custom.map { |x| x[:range] }.max
+        target[period][:rec_records] = custom.sort_by do |x|
+                                         -x[:range]
+                                       end.map { |x| { contract: x[:contract], range: x[:range] } }.take(5)
+      end
+
+      if print
+        %w[size avg lower median upper max].each do |a|
+          print "#{'%10s' % a} | " # rubocop:disable Style/FormatString
+          %i[days weeks months].each do |b|
+            %w[all dim rec].each do |c|
+              print ('%8d' % target[b]["#{c}_#{a}".to_sym]).to_s # rubocop:disable Style/FormatString
+            end
+            print ' | '
+          end
+          puts ''
+        end
+      end
+
+      target
+      # rubocop:enable Style/MultilineBlockChain
+    end
+  end
+end

data/lib/cotcube-bardata/trade_dates.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Cotcube
+  # missing top level documentation
+  module Bardata
+    # fetching official trade dates from CME
+    # it returns the current trade date or, if today isn't a trading day, the last trade date.
+    def last_trade_date
+      uri = 'https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME'
+      begin
+        HTTParty.get(uri)
+                .parsed_response
+                .map do |x|
+                  a = x['tradeDate'].chars.each_slice(2).map(&:join)
+                  "#{a[0]}#{a[1]}-#{a[2]}-#{a[3]}"
+                end
+                .first
+      rescue StandardError
+        nil
+      end
+    end
+
+    def holidays(config: init)
+      CSV.read("#{config[:data_path]}/holidays.csv").map{|x| DateTime.parse(x[0])}
+    end
+
+  end
+end

data/lib/cotcube-bardata/trading_hours.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Cotcube
+  # Missing top level comment
+  module Bardata
+    # returns an Array of ranges containing a week of trading hours, specified by seconds since monday morning
+    #    (as sunday is wday:0)
+    # according files are located in config[:data_path]/trading_hours and picked either
+    # by the symbol itself or by the assigned type
+    # commonly there are two filter for each symbol: :full and :rth, exceptions are e.g. meats
+    def trading_hours(symbol: nil, id: nil, # rubocop:disable Metrics/ParameterLists
+                      filter: ,
+                      force_filter: false,  # with force_filter one would avoid falling back
+                                            # to the contract_type based range set
+                      config: init, debug: false)
+      return (0...24 * 7 * 3600) if filter.to_s =~ /24x7/
+
+      prepare = lambda do |f|
+        CSV.read(f, converters: :numeric)
+           .map(&:to_a)
+           .tap { |x| x.shift unless x.first.first.is_a?(Numeric) }
+           .map { |x| (x.first...x.last) }
+      end
+
+      sym = get_id_set(symbol: symbol, id: id)
+
+      file = "#{config[:data_path]}/trading_hours/#{sym[:symbol]}_#{filter}.csv"
+      puts "Trying to use #{file} for #{symbol} + #{filter}" if debug
+      return prepare.call(file) if File.exist? file
+
+      file = "#{config[:data_path]}/trading_hours/#{sym[:symbol]}_full.csv"
+      puts "Failed. Trying to use #{file} now" if debug
+      return prepare.call(file) if File.exist?(file) && (not force_filter)
+
+      file = "#{config[:data_path]}/trading_hours/#{sym[:type]}_#{filter}.csv"
+      puts "Failed. Trying to use #{file} now." if debug
+      return prepare.call(file) if File.exist? file
+
+      file = "#{config[:data_path]}/trading_hours/#{sym[:type]}_full.csv"
+      puts "Failed. Trying to use #{file} now." if debug
+      return prepare.call(file) if File.exist?(file) && (not force_filter)
+
+      puts "Finally failed to find range filter for #{symbol} + #{filter}, returning 24x7".colorize(:light_yellow)
+      (0...24 * 7 * 3600)
+    end
+  end
+end

metadata

@@ -1,113 +1,127 @@
 --- !ruby/object:Gem::Specification
 name: cotcube-bardata
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.6
 platform: ruby
 authors:
 - Benjamin L. Tischendorf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-16 00:00:00.000000000 Z
+date: 2021-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: cotcube-indicators
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6'
 - !ruby/object:Gem::Dependency
-  name: yaml
+  name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: cotcube-helpers
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: colorize
+  name: cotcube-indicators
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.18'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.18'
 - !ruby/object:Gem::Dependency
-  name: rubyzip
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: yaml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -152,9 +166,17 @@ files:
 - VERSION
 - cotcube-bardata.gemspec
 - lib/cotcube-bardata.rb
+- lib/cotcube-bardata/cached.rb
 - lib/cotcube-bardata/constants.rb
+- lib/cotcube-bardata/daily.rb
+- lib/cotcube-bardata/eods.rb
+- lib/cotcube-bardata/helpers.rb
 - lib/cotcube-bardata/init.rb
 - lib/cotcube-bardata/provide.rb
+- lib/cotcube-bardata/quarters.rb
+- lib/cotcube-bardata/range_matrix.rb
+- lib/cotcube-bardata/trade_dates.rb
+- lib/cotcube-bardata/trading_hours.rb
 homepage: https://github.com/donkeybridge/cotcube-bardata
 licenses:
 - BSD-4-Clause