cotcube-bardata 0.1.11 → 0.1.15.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc417f9b8b33204bbc1b345ec28e2c5604b2a5f59c183a6a0a1ad15a6cf223b1
-  data.tar.gz: f631c5507dff588e4f2e19209aca5d135514e450ddc9b311e407b9733f539191
+  metadata.gz: 46f85dd103fd911ba475b347f14002f1402eda630b79ded6a161969a1df487db
+  data.tar.gz: 72f66b63600041d28131df575036d0295f8ab206e132fcd69c2ea84109194505
 SHA512:
-  metadata.gz: f5276bd185a561658226f65b92ce696774f1da7442bf0c10704925e4afd295566844d02bf866921575bf60cea6944555c7dd6abc28214ed5852ca8228553d2f3
-  data.tar.gz: c73f923b1bf4399bf6a9cea56e4adf2acc3aaeab770d4b56cc1daae7d5a4f7c4617f30baff58b03f9ced45a9987a4bf8106c709f12a1cfb17f0021cfe36a6d71
+  metadata.gz: 29271f6f4b5dba379bb19e81846d52f1d2995bd010f35b7e80bad3256ae415c841eeead6a84034ce72d29cd6b3746d4d62fb182347abca1fa6d78fd0c3bfb6a7
+  data.tar.gz: 5cb42e0fbd5ae2124edb7ede368047e72c8fe68b99ed928727d8e47859a5c94470bab989a032f0689e63058dd01b190936d02f22d8be7fca3cd61f3240a40a1c

data/CHANGELOG.md

@@ -1,3 +1,33 @@
+## 0.1.15.2 (August 04, 2021)
+  - fixed leftover debug setting
+
+## 0.1.15.1 (August 04, 2021)
+  - fixed license mismatch
+
+## 0.1.15 (August 04, 2021)
+  - daily.rb: added support to add eod data on incomplete dailies
+  - adding dep: cotcube-indicators
+  - provide: new method :determine_significant_volume
+  - suggest: adding silence
+  - cached: Adding :dist
+  - daily: new method :determine_significant_volume
+  - added :dist generically to quarters
+  - trading_hours: added param to return headers only
+  - last_trade_date: Enabled caching instead of fetching each time
+
+## 0.1.14 (May 07, 2021)
+  - few changes in provide / cached / daily for a more straigtforward forcing of cache renewal
+  - suggest: new method to suggest a contract for given symbol and date
+
+## 0.1.13 (April 07, 2021)
+  - daily: fixed const_caching in continuous
+  - trade_dates: FIXING call with HTTParty must send Agent Header
+  - helpers/get_id_set: added support for params given as Symbols (:NG instead of 'NG')
+
+## 0.1.12 (March 13, 2021)
+  - range_matrix: adapted to accept block for range provision; added params :days_only and :last_n
+  - minor fix on previous patch
+
 ## 0.1.11 (March 07, 2021)
   - daily.rb: added new technique 'caching in constants' to accelerate computation, also referring to continuous
   - provide.rb: minor change, so disregarded contracts can be used in swapproximate

data/README.md

@@ -90,5 +90,5 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/donkey
 
 ## License
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+The gem is available as open source under the terms of the [BSD-3-Clause-License](https://opensource.org/licenses/BSD-3-Clause).
 

data/VERSION

@@ -1 +1 @@
-0.1.11
+0.1.15.2

data/cotcube-bardata.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.description   = 'Functions to provide bardata; and some simple time series aggregations '
 
   spec.homepage      = "https://github.com/donkeybridge/#{spec.name}"
-  spec.license       = 'BSD-4-Clause'
+  spec.license       = 'BSD-3-Clause'
   spec.required_ruby_version = Gem::Requirement.new('~> 2.7')
 
   spec.metadata['homepage_uri'] = spec.homepage

data/lib/cotcube-bardata.rb

@@ -11,6 +11,7 @@ require 'date' unless defined?(DateTime)
 require 'csv'  unless defined?(CSV)
 require 'yaml' unless defined?(YAML)
 require 'cotcube-helpers'
+require 'cotcube-indicators'
 require 'parallel'
 
 require_relative 'cotcube-bardata/constants'
@@ -22,6 +23,7 @@ require_relative 'cotcube-bardata/quarters'
 require_relative 'cotcube-bardata/eods'
 require_relative 'cotcube-bardata/cached'
 require_relative 'cotcube-bardata/provide'
+require_relative 'cotcube-bardata/suggest'
 require_relative 'cotcube-bardata/range_matrix'
 require_relative 'cotcube-bardata/trading_hours'
 
@@ -61,13 +63,17 @@ module Cotcube
                     # create an array of ranges based on specified source data
                     :trading_hours,
                     # receive id / symbol information on an uncertain set of parameters
+                    # TODO: decommision sppearance of get_id_set and symbols in cotcube::bardata
                     :get_id_set,
+                    :symbols,
+                    #
                     :select_specific_date,
                     :extended_select_for_range,
                     :provide_cached,
+                    :suggest_contract_for,
+                    # 
                     :compare,
-                    :holidays,
-                    :symbols # reads and provides the symbols file
+                    :holidays
 
     # please note that module_functions of source provided in private files must be published there
   end

data/lib/cotcube-bardata/cached.rb

@@ -12,7 +12,7 @@ module Cotcube
                        timezone: Time.find_zone('America/Chicago'),
                        filter: :full, # most probably either :full or :rth
                        force_update: false, # force reloading via provide_quarters
-                       force_recent: false) #
+                       force_recent: false) # provide base.last even if dayswitch hasn't happen yet
 
       unless range.nil? ||
              range.is_a?(Range) &&
@@ -39,17 +39,20 @@ module Cotcube
       file = "#{dir}/#{contract}.csv"
       quarters_file = "#{config[:data_path]}/quarters/#{sym[:id]}/#{contract[-3..]}.csv"
       if File.exist?(file) && (not force_update)
+        puts "Working with existing #{file}, no update was forced" if debug
+        puts "    Using quarters from #{quarters_file}" if debug
         base = CSV.read(file, headers: headers).map do |x|
           x = x.to_h
           x[:datetime] = timezone.parse(x[:datetime])
           %i[open high low close].each { |z| x[z] = x[z].to_f.round(9) }
           x[:volume] = x[:volume].to_i
+          x[:dist]     = ((x[:high] - x[:low]) / sym[:ticksize] ).to_i
           x[:type]   = "#{filter.to_s.downcase}_day".to_sym
           x
         end
         if base.last[:high].zero?
           # contract exists but is closed (has the CLOSED marker)
-          base.pop
+          base.pop 
           # rubocop:disable Metrics/BlockNesting
           result = if range.nil?
                      base
@@ -96,8 +99,9 @@ module Cotcube
       end
 
       base = Cotcube::Helpers.reduce(bars: data, to: :days)
+      puts "Reduced base ends at #{bast.last[:datetime].strftime('%Y-%m-%d')}" if debug
 
-      # remove last day of result unless marked
+      # remove last day of result if suspecting incomplete last base 
       base.pop if base.last[:datetime].to_date == timezone.now.to_date and not force_recent
 
       base.map do |x|

data/lib/cotcube-bardata/daily.rb

@@ -8,6 +8,8 @@ module Cotcube
                       symbol: nil, id: nil,
                       range: nil,
                       timezone: Time.find_zone('America/Chicago'),
+                      keep_last: false,
+                      add_eods: true,
                       config: init)
       contract = contract.to_s.upcase
       unless contract.is_a?(String) && [3, 5].include?(contract.size)
@@ -45,10 +47,29 @@ module Cotcube
           row[k] = row[k].to_i if %i[volume oi].include? k
         end
         row[:datetime] = timezone.parse(row[:date])
+        row[:dist]     = ((row[:high] - row[:low]) / sym[:ticksize] ).to_i
         row[:type]     = :daily
         row
       end
-      data.pop if data.last[:high].zero?
+      contract_expired = data.last[:high].zero?
+      data.pop if contract_expired and not keep_last
+      if not contract_expired and add_eods
+        today = Date.today
+        eods = [ ]
+        while today.strftime('%Y-%m-%d') > data.last[:date]
+          eods << provide_eods(symbol: sym[:symbol], dates: today, contracts_only: false)
+          today -= 1
+        end
+        eods.flatten!.map!{|x| x.tap {|y| %i[ volume_part oi_part ].map{|z| y.delete(z)} } }
+        eods.select{|x| x[:contract] == contract } 
+        eods.map!{|x| x.tap{|y| 
+          y[:datetime] = timezone.parse(y[:date])
+          y[:dist]     = ((y[:high] - y[:low]) / sym[:ticksize] ).to_i
+          y[:type]     = :eod
+        } }
+        data += eods.reverse
+
+      end
       if range.nil?
         data
       else
@@ -61,50 +82,128 @@ module Cotcube
 
     # reads all files in  bardata/daily/<id> and aggregates by date
     # (what is a pre-stage of a continuous based on daily bars)
-    def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false)
+    def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false, selector: nil, debug: false, add_eods: true)
       raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
       measuring = lambda {|c| puts "[continuous] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
 
       measuring.call("Starting")
       sym = get_id_set(symbol: symbol, id: id)
       id  = sym[:id]
+      symbol = sym[:symbol]
+      ticksize = sym[:ticksize] 
+      effective_selector = selector || :volume
+      raise ArgumentError, 'selector must be in %i[ nil :volume ;oi].' unless [ nil, :volume, :oi ].include? selector
       id_path = "#{config[:data_path]}/daily/#{id}"
       c_file  = "#{id_path}/continuous.csv"
+      puts "Using file #{c_file}" if debug
 
       # instead of using the provide_daily methods above, for this bulk operation a 'continuous.csv' is created
       # this boosts from 4.5sec to 0.3sec
       rewriting = force_rewrite or not(File.exist?(c_file)) or (Time.now - File.mtime(c_file) > 8.days)
       if rewriting
         puts "In daily+continuous: Rewriting #{c_file} #{force_rewrite ? "forcibly" : "due to fileage"}.".light_yellow
-        `rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v '0,0,0,0' | sort -t, -k2 | cut -d, -f1,2,7,8 > #{c_file}`
+        `rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v ',0,' | grep -v ',0$'| sort -t, -k2 | cut -d, -f1-8 | grep ',.*,' | uniq > #{c_file}`
       end
       loading = lambda do
         data = CSV.read(c_file).map do |row|
           r = { contract: row[0],
                 date:     row[1],
-                volume:   row[2].to_i,
-                oi:       row[3].to_i
+                open:     row[2],
+                high:     row[3],
+                low:      row[4],
+                close:    row[5],
+                volume:   row[6].to_i,
+                oi:       row[7].to_i
           }
         end
+        if add_eods
+          today = Date.today
+          eods = [ ]
+          while today.strftime('%Y-%m-%d') > data.last[:date]
+            eods << provide_eods(symbol: symbol, dates: today, contracts_only: false)
+            today -= 1
+          end
+          eods.flatten!.map!{|x| x.tap {|y| %i[ volume_part oi_part ].map{|z| y.delete(z)} } }
+          eods.delete_if { |elem|  elem.flatten.empty? }
+          data += eods.reverse
+
+        end
 
         measuring.call("Finished retrieving dailies.")
         result = []
+        rounding =  8 # sym[:format].split('.').last.to_i rescue 6
+        indicators ||= {
+          typical:     Cotcube::Indicators.calc(a: :high, b: :low, c: :close) {|high, low, close| (high + low + close) / 3 },
+          sma250_high: Cotcube::Indicators.sma(key: :high,    length: 250),
+          sma250_low:  Cotcube::Indicators.sma(key: :low,     length: 250),
+          sma250_typ:  Cotcube::Indicators.sma(key: :typical, length: 250), 
+          # sma60_typ:   Cotcube::Indicators.sma(key: :typical, length: short),
+          # tr:          Cotcube::Indicators.true_range,
+          # atr5:        Cotcube::Indicators.sma(key: :tr,    length: 5),
+          # dist_abs:    Cotcube::Indicators.calc(a: :sma250_high, b: :sma250_low, c: :high, d: :low) do |sma_high, sma_low, high, low| 
+          #  if high > sma_high
+          #    high - sma_high
+          #  elsif sma_low > low 
+          #    low - sma_low
+          #  else 
+          #    0
+          #  end
+          #end,
+          #dist_sma: Cotcube::Indicators.calc(a: :sma250_typ, b: :sma60_typ) do |sma250, sma60|
+          #  sma60 - sma250
+          #end,
+          #dist_index:     Cotcube::Indicators.index(key: :dist_abs, length: 60, abs: true),
+          #dev250_squared: Cotcube::Indicators.calc(a: :sma250_typ, b: :typical) {|sma, x| (sma - x) ** 2 },
+          #var250:         Cotcube::Indicators.sma(key: :dev250_squared, length: long),
+          #sig250:         Cotcube::Indicators.calc(a: :var250)                  {|var| Math.sqrt(var)},
+          #boll250_high:   Cotcube::Indicators.calc(a: :sig250, b: :sma250_typ)  {|sig, typ| (typ + sig * bollinger_factor) rescue nil},
+          #boll250_low:    Cotcube::Indicators.calc(a: :sig250, b: :sma250_typ)  {|sig, typ| (typ - sig * bollinger_factor) rescue nil},
+          #dev60_squared:  Cotcube::Indicators.calc(a: :sma60_typ, b: :typical)  {|sma, x| (sma - x) ** 2 },
+          #var60:          Cotcube::Indicators.sma(key: :dev60_squared, length: short),
+          #sig60:          Cotcube::Indicators.calc(a: :var60)                   {|var| Math.sqrt(var)},
+          #boll60_high:    Cotcube::Indicators.calc(a: :sig60, b: :sma60_typ)    {|sig, typ| (typ + sig * bollinger_factor) rescue nil},
+          #boll60_low:     Cotcube::Indicators.calc(a: :sig60, b: :sma60_typ)    {|sig, typ| (typ - sig * bollinger_factor) rescue nil}
+          dist: Cotcube::Indicators.calc(a: :high, b: :low, finalize: :to_i) {|high, low| ((high-low) / ticksize) }
+
+        }
+
+
+
         data.group_by { |x| x[:date] }.map do |k, v|
           v.map { |x| x.delete(:date) }
-          result << {
+          avg_bar = { 
             date: k,
-            volume: v.map { |x| x[:volume] }.reduce(:+),
-            oi: v.map { |x| x[:oi] }.reduce(:+)
+            contract: v.max_by{|x| x[:oi] }[:contract],
+            open: nil, high: nil, low: nil, close: nil,
+            volume:   v.map { |x| x[:volume] }.reduce(:+),
+            oi:       v.map { |x| x[:oi] }.reduce(:+),
           }
+
+          %i[ open high low close ].each do |ohlc|
+            avg_bar[ohlc] = (v.map{|x| x[ohlc].to_f * x[effective_selector] }.reduce(:+) / avg_bar[effective_selector]).round(rounding)
+          end
+          p avg_bar if debug
+          indicators.each do |k,v|
+            print format('%12s:  ', k.to_s) if debug
+            avg_bar[k] = v.call(avg_bar).round(rounding)
+            puts avg_bar[k] if debug
+          end
+          %i[tr atr5].each { |ind| 
+            avg_bar[ind]     = (avg_bar[ind] / sym[:ticksize]).round.to_i unless avg_bar[ind].nil?
+          }
+          result << avg_bar
           result.last[:contracts] = v
         end
+        result
       end
-      constname = "CONTINUOUS_#{symbol}".to_sym
+      constname = "CONTINUOUS_#{symbol}#{selector.nil? ? '' : ('_' + selector.to_s)}".to_sym
       if rewriting or not  Cotcube::Bardata.const_defined?( constname)
+        old = $VERBOSE; $VERBOSE = nil
         Cotcube::Bardata.const_set constname, loading.call
+        $VERBOSE = old
       end
       measuring.call("Finished processing")
-      date.nil? ? Cotcube::Bardata.const_get(constname) : Cotcube::Bardata.const_get(constname).find { |x| x[:date] == date }
+      date.nil? ? Cotcube::Bardata.const_get(constname).map{|z| z.dup } : Cotcube::Bardata.const_get(constname).find { |x| x[:date] == date }
     end
 
     def continuous_ml(symbol: nil, id: nil, base: nil)
@@ -157,7 +256,7 @@ module Cotcube
           oi: x[:contracts].sort_by { |z| - z[:oi] }[0..4].compact.reject { |z| z[:oi].zero? }
         }
       end
-      measuring.call("Retrieved continuous")
+      measuring.call("Retrieved continuous for #{sym[:symbol]}")
       data.reject! { |x| x[selector].empty? }
       result = data.group_by { |x| x[selector].first[:contract] }
       result.each_key do |key|
@@ -193,6 +292,8 @@ module Cotcube
                          selector: :volume,
                          filter: nil,
                          date: Date.today,
+                         short: true,
+                         silent: false,
                          measure: nil,
                          debuglevel: 1,
                          debug: false)
@@ -200,6 +301,7 @@ module Cotcube
         debuglevel = debug
         debug = debuglevel > 0 ? true : false
       end
+      silent = false if debug
 
       raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
       measuring = lambda {|c| puts "[continuous_table] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
@@ -229,69 +331,78 @@ module Cotcube
       measuring.call("Retrieved continous_overview")
       output_sent = []
       early_year=nil
+      long_output = []
       data.keys.sort.each do |month|
+        current_long = { month: month }
         puts "Processing #{sym[:symbol]}#{month}" if debuglevel > 1
         v0 = data[month]
-        ydays = v0.map { |_, v1| Date.parse(v1.last[:date]).yday }
+        ldays = v0.map { |_, v1| Date.parse(v1.last[:date]).yday }
         fdays = v0.map { |_, v1| Date.parse(v1.first[:date]).yday }.sort
         # if the last ml day nears the end of the year, we must fix
-        ydays.map! { |x| x > 350 ? x - 366 : x } if ydays.min < 50
+        ldays.map! { |x| x > 350 ? x - 366 : x } if ldays.min < 50
         fday  = fdays[fdays.size / 2]
-        yavg  = ydays.reduce(:+) / ydays.size
-        # a contract is proposed to use after fday - 1, but before ydays.min (green)
-        # it is warned to user after fday - 1 but before yavg - 1            (red)
-        # it is warned red >= yavg - 1 and <= yavg + 1
-        color = if (ytoday >= yavg - 1) && (ytoday <= yavg + 1)
+        lavg  = ldays.reduce(:+) / ldays.size
+        # a contract is proposed to use after fday - 1, but before ldays.min (green)
+        # it is warned to user after fday - 1 but before lavg - 1            (red)
+        # it is warned red >= lavg - 1 and <= lavg + 1
+        color = if (ytoday >= lavg - 1) && (ytoday <= lavg + 1)
                   :light_red
-                elsif (ytoday > ydays.min) && (ytoday < yavg - 1)
+                elsif (ytoday > ldays.min) && (ytoday < lavg - 1)
                   :light_yellow
-                elsif (ytoday >= (fday > yavg ? 0 : fday - 5)) && (ytoday <= ydays.min)
+                elsif (ytoday >= (fday > lavg ? 0 : fday - 5)) && (ytoday <= ldays.min)
                   :light_green
                 else
                   :white
                 end
         # rubocop:disable Layout/ClosingParenthesisIndentation
+        long_output << {
+          month:    month,
+          first_ml: fday,
+          last_min: ldays.min,
+          last_avg: lavg,
+          last_max: ldays.max }
         output = "#{sym[:symbol]
              }#{month
              }\t#{format '%12s', sym[:type]
-             }\t#{ytoday
-              }/#{fday
-             }\t#{format '%5d', ydays.min
-             }: #{dfm.call(ydays.min)
-             }\t#{format '%5d', yavg
-             }: #{dfm.call(yavg)
-             }\t#{format '%5d', ydays.max
-             }: #{dfm.call(ydays.max)}".colorize(color)
+             }\ttoday is #{ytoday
+             } -- median of first is #{fday
+             }\tlast ranges from #{format '%5d', ldays.min
+             }: #{dfm.call(ldays.min)
+             }\t#{format '%5d', lavg
+             }: #{dfm.call(lavg)
+             }\tto #{format '%5d', ldays.max
+             }: #{dfm.call(ldays.max)}".colorize(color)
              if debug || (color != :white)
-               puts output
+               puts output unless silent
                output_sent << "#{sym[:symbol]}#{month}" unless color == :white
              end
              early_year  ||= output
-             next unless (debug and debuglevel >= 2)
+             next if silent or not (debug and debuglevel >= 2) 
 
              v0.each do |contract, v1|
                puts "\t#{contract
                }\t#{v1.first[:date]
-               }\t#{Date.parse(v1.last[:date]).strftime('%a')
-               }, #{v1.last[:date]
-               } (#{Date.parse(v1.last[:date]).yday}"
+               } (#{format '%3d', Date.parse(v1.first[:date]).yday
+              })\t#{Date.parse(v1.last[:date]).strftime('%a, %Y-%m-%d')
+               } (#{Date.parse(v1.last[:date]).yday})" unless silent
                # rubocop:enable Layout/ClosingParenthesisIndentation
              end
       end
       case output_sent.size
       when 0
-        puts "WARNING: No output was sent for symbol '#{sym[:symbol]}'.".colorize(:light_yellow)
-        puts "         Assuming late-year-processing.".light_yellow
-        puts early_year.light_green
+        unless silent
+          puts "WARNING: No output was sent for symbol '#{sym[:symbol]}'.".colorize(:light_yellow) 
+          puts "         Assuming late-year-processing.".light_yellow
+          puts early_year.light_green
+        end
       when 1
         # all ok
         true
       else
-        puts "---- #{output_sent} for #{sym[:symbol]} ---------------"
-
+        puts "Continuous table show #{output_sent.size} active contracts ( #{output_sent} ) for #{sym[:symbol]} ---------------" unless silent
       end
       measuring.call("Finished processing")
-      output_sent
+      short ? output_sent : long_output
     end
   end
 end

data/lib/cotcube-bardata/helpers.rb

@@ -46,6 +46,10 @@ module Cotcube
     end
 
     def get_id_set(symbol: nil, id: nil, contract: nil, config: init)
+      contract = contract.to_s.upcase if contract.is_a? Symbol
+      id       =       id.to_s.upcase if       id.is_a? Symbol
+      symbol   =   symbol.to_s.upcase if   symbol.is_a? Symbol
+
       if contract.is_a?(String) && (contract.length == 5)
         c_symbol = contract[0..1]
         if (not symbol.nil?) && (symbol != c_symbol)

data/lib/cotcube-bardata/init.rb

@@ -8,9 +8,9 @@ module Cotcube
         SYMBOL_EXAMPLES
       else
         CSV
-          .read(config[:symbols_file], headers: %i[id symbol ticksize power months type bcf reports name])
+          .read(config[:symbols_file], headers: %i[id symbol ticksize power months type bcf reports format name])
           .map(&:to_h)
-          .map { |row| %i[ticksize power bcf].each { |z| row[z] = row[z].to_f }; row } # rubocop:disable Style/Semicolon
+          .map { |row| %i[ticksize power bcf].each { |z| row[z] = row[z].to_f }; row[:format] = "%#{row[:format]}f"; row } # rubocop:disable Style/Semicolon
           .reject { |row| row[:id].nil? }
           .tap { |all| all.select! { |x| x[:type] == type } unless type.nil? }
           .tap { |all| all.select! { |x| x[:symbol] == symbol } unless symbol.nil? }

data/lib/cotcube-bardata/provide.rb

@@ -14,6 +14,7 @@ module Cotcube
                 filter: :full,
                 # TODO: for future compatibility and suggestion: planning to include a function to update
                 #       with live data from broker
+                force_update: false,
                 force_recent: false)
 
       sym = get_id_set(symbol: symbol, id: id, contract: contract, config: config)
@@ -33,7 +34,7 @@ module Cotcube
 
       when :days, :weeks, :months
         base = provide_cached contract: contract, symbol: symbol, id: id, config: config, filter: filter,
-                              range: range, force_recent: force_recent
+                              range: range, force_recent: force_recent, force_update: force_update
         return base if %i[day days].include? interval
 
         # TODO: Missing implementation to reduce cached days to weeks or months
@@ -42,7 +43,7 @@ module Cotcube
         provide_daily contract: contract, symbol: symbol, id: id, config: config, range: range
       when :synth, :synthetic, :synthetic_days
         days = provide_cached contract: contract, symbol: symbol, id: id, config: config, filter: filter,
-                              range: range, force_recent: force_recent
+                              range: range, force_recent: force_recent, force_update: force_update
         dailies = provide_daily contract: contract, symbol: symbol, id: id, config: config, range: range
         if ((days.last[:datetime] > dailies.last[:datetime]) rescue false)
           dailies[..-2] + days.select { |d| d[:datetime] > dailies[-2][:datetime] }
@@ -53,5 +54,11 @@ module Cotcube
         raise ArgumentError, "Unsupported or unknown interval '#{interval}' in Bardata.provide"
       end
     end
+
+    def determine_significant_volume(base: , contract: )
+      set  = Cotcube::Bardata.trading_hours(symbol: contract[0..1], filter: :rth)
+      prod = base - base.select_within(ranges: set ,attr: :datetime) {|x| x.to_datetime.to_sssm }
+      prod.group_by{|x| x[:volume] / 500 }
+    end
   end
 end

data/lib/cotcube-bardata/quarters.rb

@@ -33,6 +33,7 @@ module Cotcube
         %i[open high low close].map { |x| row[x] = row[x].to_f }
         %i[volume day].map { |x| row[x] = row[x].to_i }
         row[:datetime] = timezone.parse(row[:datetime])
+        row[:dist]     = ((row[:high] - row[:low]) / sym[:ticksize] ).to_i
         row[:type]     = :quarter
         row
       end

data/lib/cotcube-bardata/range_matrix.rb

@@ -5,7 +5,6 @@ module Cotcube
   module Bardata
     # this is an analysis tool to investigate actual ranges of an underlying symbol
     # it is in particular no true range or average true range, as a 'true range' can only be applied to
-    # a steady series, what changing contracts definitely aren't
     #
     # The result printed / returned is a table, containing a matrix of rows:
     #   1. size: the amount of values evaluated
@@ -22,18 +21,25 @@ module Cotcube
     #   3.a-c) same with days reduced to months (c: 12 months)
     #
     # NOTE: there is now a new method Cotcube::Helpers.simple_series_stats, that should be used in favor.
-    def range_matrix(symbol: nil, id: nil, print: false, dim: 0.05)
+    def range_matrix(symbol: nil, id: nil, base: nil, print: false, dim: 0.05, days_only: false, last_n: 60, &block)
       # rubocop:disable Style/MultilineBlockChain
+      symbol ||= base.last[:contract][0..1] if id.nil?
       sym = get_id_set(symbol: symbol, id: id)
       source = {}
       target = {}
-      source[:days]   = Cotcube::Bardata.continuous_actual_ml symbol: symbol
+      if base.nil?
+        ml = (Cotcube::Bardata.continuous_actual_ml symbol: symbol)&.last[:contract]
+        source[:days]   = Cotcube::Bardata.provide contract: ml
+      else
+        source[:days]   = base
+      end
       source[:weeks]  = Cotcube::Helpers.reduce bars: source[:days], to: :weeks
       source[:months] = Cotcube::Helpers.reduce bars: source[:days], to: :months
 
       %i[days weeks months].each do |period|
+        next if days_only and %i[weeks months].include? period
         source[period].map! do |x|
-          x[:range] = ((x[:high] - x[:low]) / sym[:ticksize]).round
+          x[:range] = block_given? ? yield(x) : (((x[:high] - x[:low]) / sym[:ticksize]).round)
           x
         end
         target[period] = {}
@@ -80,14 +86,18 @@ module Cotcube
 
         range = case period
                 when :months
-                  -13..-2
+                  -(last_n/15)..-2
                 when :weeks
-                  -53..-2
+                  -(last_n/4)..-2
                 when :days
-                  -200..-1
+                  -last_n..-1
                 else
                   raise ArgumentError, "Unsupported period: '#{period}'"
                 end
+        if range.begin.abs > source[period].size
+          puts "WARNING: requested last_n = #{last_n} exceeds actual size (#{source[period].size}), adjusting...".light_yellow
+          range = (-source[period].size..range.end)
+        end
         custom = source[period][range]
         target[period][:rec_size]   =  custom.size
         target[period][:rec_avg]    = (custom.map { |x| x[:range] }.reduce(:+) / custom.size).round
@@ -110,6 +120,7 @@ module Cotcube
         %w[size avg lower median upper max].each do |a|
           print "#{'%10s' % a} | " # rubocop:disable Style/FormatString
           %i[days weeks months].each do |b|
+            next if days_only and %i[weeks months].include? b
             %w[all dim rec].each do |c|
               print ('%8d' % target[b]["#{c}_#{a}".to_sym]).to_s # rubocop:disable Style/FormatString
             end

data/lib/cotcube-bardata/suggest.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Cotcube
+  # missing top level documentation
+  module Bardata
+    # based on day(of year) and symbol, suggest best fitting contract
+    def suggest_contract_for symbol:, date: Date.today, warnings: true
+      ml = Cotcube::Bardata.continuous_table symbol: symbol, date: date, silent: true
+      if ml.size != 1 
+	puts "WARNING: No or no unique most liquid found for #{date}. please give :contract parameter".light_yellow if warnings
+	if ml.size > 1
+	  puts "\tUsing #{ml.last}. Consider breaking here, if that is not acceptable.".light_yellow if warnings
+	  sleep 1
+	else
+	  puts "\tERROR: No suggestible contract found for #{symbol} and #{date}.".light_red
+	  return
+	end
+      end
+      year = date.year % 100
+      if ml.last[2] < "K" and date.month > 9
+        "#{ml.last}#{year + 1}"
+      else
+        "#{ml.last}#{year}"
+      end
+    end
+  end
+end
+

data/lib/cotcube-bardata/trade_dates.rb

@@ -5,19 +5,45 @@ module Cotcube
   module Bardata
     # fetching official trade dates from CME
     # it returns the current trade date or, if today isn't a trading day, the last trade date.
-    def last_trade_date
-      uri = 'https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME'
-      begin
-        HTTParty.get(uri)
-                .parsed_response
-                .map do |x|
-                  a = x['tradeDate'].chars.each_slice(2).map(&:join)
-                  "#{a[0]}#{a[1]}-#{a[2]}-#{a[3]}"
-                end
-                .first
-      rescue StandardError
-        nil
+    def last_trade_date(force_update: false)
+      const_LTD = :LAST_TRADE_DATE
+      const_LTDU = :LAST_TRADE_DATE_UPDATE
+      if force_update or not Object.const_defined?(const_LTD) or Object.const_get(const_LTD).nil? or Time.now - Object.const_get(const_LTDU) > 2.hours
+        result = nil
+        uri = 'https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME'
+        headers = { "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+                  "Accept-Encoding" => "gzip, deflate, br",
+                  "Accept-Language" => "en-US,en;q=0.9",
+                  "Cache-Control" => "max-age=0",
+                  "Connection" => "keep-alive",
+                 # Cookie: ak_bmsc=602078F6DE40954BAA8C7E7D3815102CACE82AAFD237000084B5A460F4FBCA68~pluz010T49Xag3sXquUZtVJmFX701dzEgt5v6Ht1EZSLKE4HL+bgg1L9ePnL5I0mm7QWXe1qaLhUbX1IPrL/f20trRMMRlkC3UWXk27DY/EBCP4mRno8QQygLCwgs2B2AQHJyb63WwRihCko8UYUiIhb89ArPZM5OPraoKy3JU9oE9e+iERdARNZHLHqRiB1GnmbKUvQqos3sXaEe3GpoiTszzk8sHZs4ZKuoO/rvFHko=",
+                  "Host" => "www.cmegroup.com",
+                  "sec-ch-ua" => %q[" Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"],
+                  "sec-ch-ua-mobile" => "?0",
+                  "Sec-Fetch-Dest" => "document",
+                  "Sec-Fetch-Mode" => "navigate",
+                  "Sec-Fetch-Site" => "none",
+                  "Sec-Fetch-User" => "?1",
+                  "Upgrade-Insecure-Requests" => "1",
+                  "User-Agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36" }
+        begin
+#          HTTParty.get(uri, headers: { "User-Agent" => "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"})
+           result = HTTParty.get(uri, headers: headers)
+                  .parsed_response
+                  .map do |x|
+                    a = x['tradeDate'].chars.each_slice(2).map(&:join)
+                    "#{a[0]}#{a[1]}-#{a[2]}-#{a[3]}"
+                  end
+                  .first
+        rescue StandardError
+          result = nil
+        end
+        oldverbose = $VERBOSE; $VERBOSE = nil
+        Object.const_set(const_LTD, result)
+        Object.const_set(const_LTDU, Time.now) unless result.nil?
+        $VERBOSE = oldverbose
       end
+      Object.const_get(const_LTD)
     end
 
     def holidays(config: init)

data/lib/cotcube-bardata/trading_hours.rb

@@ -12,14 +12,20 @@ module Cotcube
                       filter: ,
                       force_filter: false,  # with force_filter one would avoid falling back
                                             # to the contract_type based range set
+                      headers_only: false,  # return only headers instead of ranges
                       config: init, debug: false)
       return (0...24 * 7 * 3600) if filter.to_s =~ /24x7/
 
       prepare = lambda do |f|
-        CSV.read(f, converters: :numeric)
-           .map(&:to_a)
-           .tap { |x| x.shift unless x.first.first.is_a?(Numeric) }
-           .map { |x| (x.first...x.last) }
+        if headers_only
+          CSV.read(f)
+            .first
+        else
+          CSV.read(f, converters: :numeric)
+             .map(&:to_a)
+             .tap { |x| x.shift unless x.first.first.is_a?(Numeric) }
+             .map { |x| (x.first...x.last) }
+        end
       end
 
       sym = get_id_set(symbol: symbol, id: id)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cotcube-bardata
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.1.15.2
 platform: ruby
 authors:
 - Benjamin L. Tischendorf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-03-07 00:00:00.000000000 Z
+date: 2021-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -175,11 +175,12 @@ files:
 - lib/cotcube-bardata/provide.rb
 - lib/cotcube-bardata/quarters.rb
 - lib/cotcube-bardata/range_matrix.rb
+- lib/cotcube-bardata/suggest.rb
 - lib/cotcube-bardata/trade_dates.rb
 - lib/cotcube-bardata/trading_hours.rb
 homepage: https://github.com/donkeybridge/cotcube-bardata
 licenses:
-- BSD-4-Clause
+- BSD-3-Clause
 metadata:
   homepage_uri: https://github.com/donkeybridge/cotcube-bardata
   source_code_uri: https://github.com/donkeybridge/cotcube-bardata