cotcube-bardata 0.1.14 → 0.1.15.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f517a7bf018413f424a2eba1e1c568c4db864240f88d23fe0c722526af858b4
-  data.tar.gz: 7e303f08f4be6d64f1680f5107690e78525edcdf938245b22ab97a38af1523e7
+  metadata.gz: 5798e21ced6b815404638c1f33948317338d801ea483e9e43edf21b8ad3f73ba
+  data.tar.gz: 8bfaf9b663d8de85f8c89b459b3f33e35ce836cad2d0153a02385aaafc4d6781
 SHA512:
-  metadata.gz: 5cbeae28346fca35de00cdc926724799f00c69ac8b023075c5dbd332327303ab457e1558eb289979ce95ef31b0cbc7e7cd1439ce550c4b2ddb415c79cffa21dd
-  data.tar.gz: 9fcdc6b0b14212673d1c26e9c5b6b76264a2d8baa7270d609841377ad3fd753bb1011ddad499a27412e180dcbb8bc35a831e79e79c487866634f85533440b771
+  metadata.gz: f554feb61ac91f542a9e9a24a4f9983cbfd529ace02290b531d07bb67cbe5864da1e154e023b9d00dec77b91b2243efb726d4bd005fee6cdc1af88701de583a8
+  data.tar.gz: 32a0446523adb94cc50426c0e03b3bd379746754d5314fe79af8ff02fe169fae95fc3a1ba894a313e5b8b7ac89a2da36d5605e0183b9e0ef8022550e00fc5449

data/CHANGELOG.md

@@ -1,3 +1,32 @@
+## 0.1.15.6 (November 19, 2021)
+  - daily: added filter_series
+
+## 0.1.15.5 (November 08, 2021)
+  - decommissioned .get_id_set in favor of Cotcube::Helpers.get_id_set
+
+## 0.1.15.4 (October 26, 2021)
+  - daily: including the sourcing factor (:bcf) into provide_daily and continuous
+
+## 0.1.15.3 (August 04, 2021)
+  - daily.rb: Hotfixing
+
+## 0.1.15.2 (August 04, 2021)
+  - fixed leftover debug setting
+
+## 0.1.15.1 (August 04, 2021)
+  - fixed license mismatch
+
+## 0.1.15 (August 04, 2021)
+  - daily.rb: added support to add eod data on incomplete dailies
+  - adding dep: cotcube-indicators
+  - provide: new method :determine_significant_volume
+  - suggest: adding silence
+  - cached: Adding :dist
+  - daily: new method :determine_significant_volume
+  - added :dist generically to quarters
+  - trading_hours: added param to return headers only
+  - last_trade_date: Enabled caching instead of fetching each time
+
 ## 0.1.14 (May 07, 2021)
   - few changes in provide / cached / daily for a more straigtforward forcing of cache renewal
   - suggest: new method to suggest a contract for given symbol and date

data/README.md

@@ -90,5 +90,5 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/donkey
 
 ## License
 
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+The gem is available as open source under the terms of the [BSD-3-Clause-License](https://opensource.org/licenses/BSD-3-Clause).
 

data/VERSION

@@ -1 +1 @@
-0.1.14
+0.1.15.6

data/cotcube-bardata.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.description   = 'Functions to provide bardata; and some simple time series aggregations '
 
   spec.homepage      = "https://github.com/donkeybridge/#{spec.name}"
-  spec.license       = 'BSD-4-Clause'
+  spec.license       = 'BSD-3-Clause'
   spec.required_ruby_version = Gem::Requirement.new('~> 2.7')
 
   spec.metadata['homepage_uri'] = spec.homepage
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'activesupport',      '~> 6'
   spec.add_dependency 'colorize',           '~> 0.8'
-  spec.add_dependency 'cotcube-helpers',    '~> 0.1'
+  spec.add_dependency 'cotcube-helpers',    '~> 0.2'
   spec.add_dependency 'cotcube-indicators', '~> 0.1'
   spec.add_dependency 'httparty',           '~> 0.18'
   spec.add_dependency 'parallel',           '~> 1'

data/lib/cotcube-bardata/cached.rb

@@ -30,7 +30,7 @@ module Cotcube
       end
 
       headers = %i[contract datetime open high low close volume]
-      sym      = get_id_set(symbol: symbol, id: id, contract: contract)
+      sym      = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)
       contract = contract[-3..]
       dir      = "#{config[:data_path]}/cached/#{sym[:id]}_#{filter.to_s.downcase}"
       symlink  = "#{config[:data_path]}/cached/#{sym[:symbol]}_#{filter.to_s.downcase}"
@@ -40,12 +40,13 @@ module Cotcube
       quarters_file = "#{config[:data_path]}/quarters/#{sym[:id]}/#{contract[-3..]}.csv"
       if File.exist?(file) && (not force_update)
         puts "Working with existing #{file}, no update was forced" if debug
-        puts "    Using quarters from #{quarters_file}"
+        puts "    Using quarters from #{quarters_file}" if debug
         base = CSV.read(file, headers: headers).map do |x|
           x = x.to_h
           x[:datetime] = timezone.parse(x[:datetime])
           %i[open high low close].each { |z| x[z] = x[z].to_f.round(9) }
           x[:volume] = x[:volume].to_i
+          x[:dist]     = ((x[:high] - x[:low]) / sym[:ticksize] ).to_i
           x[:type]   = "#{filter.to_s.downcase}_day".to_sym
           x
         end

data/lib/cotcube-bardata/daily.rb

@@ -9,8 +9,11 @@ module Cotcube
                       range: nil,
                       timezone: Time.find_zone('America/Chicago'),
                       keep_last: false,
+                      add_eods: true,
+                      indicators: {},
                       config: init)
       contract = contract.to_s.upcase
+      rounding = 8
       unless contract.is_a?(String) && [3, 5].include?(contract.size)
         raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
       end
@@ -30,7 +33,7 @@ module Cotcube
         range = (range_begin..range_end)
       end
 
-      sym = get_id_set(symbol: symbol, id: id, contract: contract)
+      sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)
       contract = contract[2..4] if contract.to_s.size == 5
       id = sym[:id]
       id_path   = "#{config[:data_path]}/daily/#{id}"
@@ -42,14 +45,46 @@ module Cotcube
       data = CSV.read(data_file, headers: %i[contract date open high low close volume oi]).map do |row|
         row = row.to_h
         row.each do |k, _|
-          row[k] = row[k].to_f if %i[open high low close].include? k
+          if %i[open high low close].include? k
+            row[k] = row[k].to_f
+            row[k] = (row[k] * sym[:bcf]).round(8) unless sym[:bcf] == 1.0
+          end
           row[k] = row[k].to_i if %i[volume oi].include? k
         end
         row[:datetime] = timezone.parse(row[:date])
+        row[:dist]     = ((row[:high] - row[:low]) / sym[:ticksize] ).to_i
         row[:type]     = :daily
         row
       end
-      data.pop if data.last[:high].zero? and not keep_last
+      contract_expired = data.last[:high].zero?
+      data.pop if contract_expired and not keep_last
+      if not contract_expired and add_eods
+        today = Date.today
+        eods = [ ]
+        while today.strftime('%Y-%m-%d') > data.last[:date]
+          eods << provide_eods(symbol: sym[:symbol], dates: today, contracts_only: false, quiet: true)
+          today -= 1
+        end
+        eods.flatten!.map!{|x| x.tap {|y| %i[ volume_part oi_part ].map{|z| y.delete(z)} } }
+        eods.select!{|x| x[:contract] == "#{sym[:symbol]}#{contract}" } 
+        eods.map!{|x| x.tap{|y| 
+          if sym[:bcf] != 1.0
+            %i[open high low close].map{|k|
+               y[k] = (y[k] * sym[:bcf]).round(8)
+            }
+          end
+          y[:datetime] = timezone.parse(y[:date])
+          y[:dist]     = ((y[:high] - y[:low]) / sym[:ticksize] ).to_i
+          y[:type]     = :eod
+        } }
+        data += eods.reverse
+      end
+      data.map do |bar| 
+        indicators.each do |k,v|
+          tmp = v.call(bar)
+          bar[k] = tmp.respond_to?(:round) ? tmp.round(rounding) : tmp
+        end
+      end unless indicators.empty?
       if range.nil?
         data
       else
@@ -62,54 +97,156 @@ module Cotcube
 
     # reads all files in  bardata/daily/<id> and aggregates by date
     # (what is a pre-stage of a continuous based on daily bars)
-    def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false)
+    def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false, selector: nil, debug: false, add_eods: true, indicators: nil)
       raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
       measuring = lambda {|c| puts "[continuous] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
 
       measuring.call("Starting")
-      sym = get_id_set(symbol: symbol, id: id)
+      sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
       id  = sym[:id]
       symbol = sym[:symbol]
+      ticksize = sym[:ticksize] 
+      effective_selector = selector || :volume
+      raise ArgumentError, 'selector must be in %i[ nil :volume ;oi].' unless [ nil, :volume, :oi ].include? selector
       id_path = "#{config[:data_path]}/daily/#{id}"
       c_file  = "#{id_path}/continuous.csv"
+      puts "Using file #{c_file}" if debug
 
       # instead of using the provide_daily methods above, for this bulk operation a 'continuous.csv' is created
       # this boosts from 4.5sec to 0.3sec
-      rewriting = force_rewrite or not(File.exist?(c_file)) or (Time.now - File.mtime(c_file) > 8.days)
+      rewriting = (force_rewrite or not(File.exist?(c_file)) or (Time.now - File.mtime(c_file) > 8.days))
       if rewriting
         puts "In daily+continuous: Rewriting #{c_file} #{force_rewrite ? "forcibly" : "due to fileage"}.".light_yellow
-        `rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v '0,0,0,0' | sort -t, -k2 | cut -d, -f1,2,7,8 > #{c_file}`
+        `rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v ',0,' | grep -v ',0$'| sort -t, -k2 | cut -d, -f1-8 | grep ',.*,' | uniq > #{c_file}`
       end
       loading = lambda do
         data = CSV.read(c_file).map do |row|
           r = { contract: row[0],
                 date:     row[1],
-                volume:   row[2].to_i,
-                oi:       row[3].to_i
+                open:     row[2],
+                high:     row[3],
+                low:      row[4],
+                close:    row[5],
+                volume:   row[6].to_i,
+                oi:       row[7].to_i,
+                type:     :cont
           }
         end
+        if add_eods
+          today = Date.today
+          eods = [ ]
+          while today.strftime('%Y-%m-%d') > data.last[:date]
+            eods << provide_eods(symbol: symbol, dates: today, contracts_only: false, quiet: true)
+            today -= 1
+          end
+          eods.flatten!.map!{|x| x.tap {|y| %i[ volume_part oi_part ].map{|z| y.delete(z)} } }
+          eods.delete_if { |elem|  elem.flatten.empty? }
+          data += eods.reverse
+        end
 
         measuring.call("Finished retrieving dailies.")
         result = []
+        rounding =  8 # sym[:format].split('.').last.to_i rescue 6
+        indicators ||= {
+          typical:     Cotcube::Indicators.calc(a: :high, b: :low, c: :close) {|high, low, close| (high + low + close) / 3 },
+          sma250_high: Cotcube::Indicators.sma(key: :high,    length: 250),
+          sma250_low:  Cotcube::Indicators.sma(key: :low,     length: 250),
+          sma250_typ:  Cotcube::Indicators.sma(key: :typical, length: 250), 
+          dist: Cotcube::Indicators.calc(a: :high, b: :low, finalize: :to_i) {|high, low| ((high-low) / ticksize) },
+        }
+
+
+
         data.group_by { |x| x[:date] }.map do |k, v|
           v.map { |x| x.delete(:date) }
-          result << {
+          avg_bar = { 
             date: k,
-            volume: v.map { |x| x[:volume] }.reduce(:+),
-            oi: v.map { |x| x[:oi] }.reduce(:+)
+            contract: v.max_by{|x| x[:oi] }[:contract],
+            open: nil, high: nil, low: nil, close: nil,
+            volume:   v.map { |x| x[:volume] }.reduce(:+),
+            oi:       v.map { |x| x[:oi] }.reduce(:+),
+            type:     :cont_eod
+          }
+
+          %i[ open high low close ].each do |ohlc|
+            avg_bar[ohlc] = (v.map{|x| x[ohlc].to_f * x[effective_selector] }.reduce(:+) / avg_bar[effective_selector]).round(rounding)
+            avg_bar[ohlc] = (avg_bar[ohlc] * sym[:bcf]).round(8) unless sym[:bcf] == 1.0
+
+          end
+          p avg_bar if debug
+          indicators.each do |k,v|
+            print format('%12s:  ', k.to_s) if debug
+            tmp = v.call(avg_bar)
+            avg_bar[k] = tmp.respond_to?(:round) ? tmp.round(rounding) : tmp
+            puts avg_bar[k] if debug
+          end
+          %i[tr atr5].each { |ind| 
+            avg_bar[ind]     = (avg_bar[ind] / sym[:ticksize]).round.to_i unless avg_bar[ind].nil?
           }
+          result << avg_bar
           result.last[:contracts] = v
         end
         result
       end
-      constname = "CONTINUOUS_#{symbol}".to_sym
+      constname = "CONTINUOUS_#{symbol}#{selector.nil? ? '' : ('_' + selector.to_s)}".to_sym
       if rewriting or not  Cotcube::Bardata.const_defined?( constname)
+        old = $VERBOSE; $VERBOSE = nil
         Cotcube::Bardata.const_set constname, loading.call
+        $VERBOSE = old
       end
       measuring.call("Finished processing")
       date.nil? ? Cotcube::Bardata.const_get(constname).map{|z| z.dup } : Cotcube::Bardata.const_get(constname).find { |x| x[:date] == date }
     end
 
+    # the filter series is an indicator based on the Cotcube::Bardata.continuous of the asset price.
+    #   current default filter is the ema50
+    def filter_series(ema_length: 50, symbol: , print_range: nil)
+      ema_high_n = "ema#{ema_length}_high".to_sym
+      ema_low_n  = "ema#{ema_length}_low".to_sym
+      ema_filter = "ema#{ema_length}_filter".to_sym
+      indicators = {
+        ema_high_n => Cotcube::Indicators.ema(key: :high,     length: ema_length,  smoothing: 2),
+        ema_low_n  => Cotcube::Indicators.ema(key: :low,      length: ema_length,  smoothing: 2),
+        :tr        => Cotcube::Indicators.true_range,
+        :atr5      => Cotcube::Indicators.ema(key: :tr,       length: 5,           smoothing: 2),
+        ema_filter => Cotcube::Indicators.calc(a: :high,      b: :low,             c: :close,
+                                               d: ema_high_n, e: ema_low_n,        f: :atr5,
+                                               finalize: :to_i)  do |high, low, close, ema_high, ema_low, atr5|
+
+                                                  if    close >  ema_high and (low - ema_high).abs <= atr5 / 5.0; 3 # :bullish_tipped
+                                                  elsif   low >  ema_high and (low - ema_high).abs >= atr5 * 3.0; 5 # :bullish_away
+                                                  elsif   low >  ema_high and (low - ema_high).abs <= atr5 / 1.5; 2 # :bullish_nearby
+                                                  elsif   low >  ema_high;                                        4 # :bullish
+
+                                                  elsif close <  ema_low and (high - ema_low).abs <= atr5 / 5.0; -3 # :bearish_tipped
+                                                  elsif  high <  ema_low and (high - ema_low).abs >= atr5 * 3.0; -5 # :bearish_away
+                                                  elsif  high <  ema_low and (high - ema_low).abs <= atr5 / 1.5; -2 # :bearish_nearby
+                                                  elsif  high <  ema_low;                                        -4 # :bearish
+
+                                                  elsif close >= ema_high and (close - ema_high).abs > atr5 ;     2 # :bullish_closed
+                                                  elsif close <= ema_low  and (close - ema_low ).abs > atr5 ;    -2 # :bearish_closed
+                                                  elsif close >= ema_high;                                        1 # :bullish_weak
+                                                  elsif close <= ema_low;                                        -1 # :bearish_weak
+                                                  elsif close >  ema_low and close < ema_high;                    0 # :ambigue
+                                                  else
+                                                    raise RuntimeError, "Unconsidered Indicator value with #{high}, #{low}, #{close}, #{ema_high}, #{ema_low}, #{atr5}"
+
+                                                  end
+                                                end
+      }
+      filter = Cotcube::Bardata.continuous(symbol: symbol, indicators: indicators).
+        map{ |z| z[:datetime] = DateTime.parse(z[:date]); z[:datetime] += z[:datetime].wday == 5 ? 3 : 1; z.slice(:datetime, ema_filter) }.
+        group_by{ |z| z[:datetime] }.
+        map{ |k,v| [ k, v[0][ema_filter] ] }.
+        to_h.
+        tap{ |z| z.to_a[print_range].each { |v|
+          puts "#{v[0].strftime('%Y-%m-%d')
+            } : #{format '%2d', v[1]
+            }".colorize(v[1] > 3 ? :light_green : v[1] > 1 ? :green : v[1] < -3 ? :light_red : v[1] < -1 ? :red : :white )
+        } if print_range.is_a? Range
+      }
+    end
+
     def continuous_ml(symbol: nil, id: nil, base: nil)
       (base.nil? ? Cotcube::Bardata.continuous(symbol: symbol, id: id) : base).map do |x|
         x[:ml] = x[:contracts].max_by { |z| z[:volume] }[:contract]
@@ -150,7 +287,7 @@ module Cotcube
         %i[volume oi].include?(selector)
 
       measuring.call("Starting")
-      sym = get_id_set(symbol: symbol, id: id)
+      sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
       id  = sym[:id]
       # noinspection RubyNilAnalysis
       data = continuous(id: id, config: config, measure: measure).map do |x|
@@ -214,7 +351,7 @@ module Cotcube
         %i[volume oi].include?(selector)
 
       measuring.call("Entering function")
-      sym = get_id_set(symbol: symbol, id: id)
+      sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
       if %w[R6 BJ GE].include? sym[:symbol]
         puts "Rejecting to process symbol '#{sym[:symbol]}'.".light_red
         return []
@@ -230,17 +367,18 @@ module Cotcube
 
       ytoday = date.yday
       data = continuous_overview(id: id, selector: selector, filter: filter, human: false, config: init, measure: measure)
-        .reject { |k, _| k[-2..].to_i == date.year % 2000 }
+        .reject { |k, _| k[-2..].to_i >= date.year % 2000 }
         .group_by { |k, _| k[2] }
       measuring.call("Retrieved continous_overview")
       output_sent = []
       early_year=nil
       long_output = []
       data.keys.sort.each do |month|
-        current_long = { month: month }
         puts "Processing #{sym[:symbol]}#{month}" if debuglevel > 1
         v0 = data[month]
+        # ldays is the list of 'last days'
         ldays = v0.map { |_, v1| Date.parse(v1.last[:date]).yday }
+        # fdays is the list of 'first days'
         fdays = v0.map { |_, v1| Date.parse(v1.first[:date]).yday }.sort
         # if the last ml day nears the end of the year, we must fix
         ldays.map! { |x| x > 350 ? x - 366 : x } if ldays.min < 50
@@ -277,9 +415,9 @@ module Cotcube
              }\tto #{format '%5d', ldays.max
              }: #{dfm.call(ldays.max)}".colorize(color)
              if debug || (color != :white)
-               puts output
-               output_sent << "#{sym[:symbol]}#{month}" unless color == :white
+               puts output unless silent
              end
+             output_sent << "#{sym[:symbol]}#{month}" unless color == :white
              early_year  ||= output
              next if silent or not (debug and debuglevel >= 2) 
 
@@ -288,20 +426,22 @@ module Cotcube
                }\t#{v1.first[:date]
                } (#{format '%3d', Date.parse(v1.first[:date]).yday
               })\t#{Date.parse(v1.last[:date]).strftime('%a, %Y-%m-%d')
-               } (#{Date.parse(v1.last[:date]).yday})"
+               } (#{Date.parse(v1.last[:date]).yday})" unless silent
                # rubocop:enable Layout/ClosingParenthesisIndentation
              end
-        end
+      end
       case output_sent.size
       when 0
-        puts "WARNING: No output was sent for symbol '#{sym[:symbol]}'.".colorize(:light_yellow)
-        puts "         Assuming late-year-processing.".light_yellow
-        puts early_year.light_green
+        unless silent
+          puts "WARNING: No output was sent for symbol '#{sym[:symbol]}'.".colorize(:light_yellow) 
+          puts "         Assuming late-year-processing.".light_yellow
+          puts early_year.light_green
+        end
       when 1
         # all ok
         true
       else
-        puts "Continuous table show #{output_sent.size} active contracts ( #{output_sent} ) for #{sym[:symbol]} ---------------"
+        puts "Continuous table show #{output_sent.size} active contracts ( #{output_sent} ) for #{sym[:symbol]} ---------------" unless silent
       end
       measuring.call("Finished processing")
       short ? output_sent : long_output

data/lib/cotcube-bardata/eods.rb

@@ -4,7 +4,7 @@ module Cotcube
   # Missing top level documentation
   module Bardata
     def most_liquid_for(symbol: nil, id: nil, date: last_trade_date, config: init)
-      id = get_id_set(symbol: symbol, id: id, config: config)[:id]
+      id = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, config: config)[:id]
       provide_eods(id: id, dates: date, contracts_only: true).first
     end
 
@@ -15,7 +15,7 @@ module Cotcube
                                     date: last_trade_date,
                                     filter: :volume_part,
                                     age: 1.hour)
-      sym  = get_id_set(symbol: symbol, id: id) if symbol || id
+      sym  = Cotcube::Helpers.get_id_set(symbol: symbol, id: id) if symbol || id
       # noinspection RubyScope
       eods = provide_eods(id: sym.nil? ? nil : sym[:id], config: config, dates: date, filter: filter)
       result = []
@@ -62,7 +62,7 @@ module Cotcube
       end
 
       symbol = contract[0..1] if contract.to_s.size == 5
-      sym = get_id_set(symbol: symbol, id: id, config: config) if symbol || id
+      sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, config: config) if symbol || id
       # if no id can be clarified from given arguments, return all matching contracts from all available symbols
       # raise ArgumentError, "Could not guess :id or :symbol from 'contract: #{contract}', please clarify." if id.nil?
       raise ArgumentError, ':filter must be in [:volume_part, :oi_part]' unless %i[volume_part oi_part].include? filter

data/lib/cotcube-bardata/helpers.rb

@@ -45,42 +45,6 @@ module Cotcube
       result
     end
 
-    def get_id_set(symbol: nil, id: nil, contract: nil, config: init)
-      contract = contract.to_s.upcase if contract.is_a? Symbol
-      id       =       id.to_s.upcase if       id.is_a? Symbol
-      symbol   =   symbol.to_s.upcase if   symbol.is_a? Symbol
-
-      if contract.is_a?(String) && (contract.length == 5)
-        c_symbol = contract[0..1]
-        if (not symbol.nil?) && (symbol != c_symbol)
-          raise ArgumentError,
-                "Mismatch between given symbol #{symbol} and contract #{contract}"
-        end
-
-        symbol = c_symbol
-      end
-
-      unless symbol.nil?
-        sym = symbols.select { |s| s[:symbol] == symbol.to_s.upcase }.first
-        if sym.nil? || sym[:id].nil?
-          raise ArgumentError,
-                "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}"
-        end
-        raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if (not id.nil?) && (sym[:id] != id)
-
-        return sym
-      end
-      unless id.nil?
-        sym = symbols.select { |s| s[:id] == id.to_s }.first
-        if sym.nil? || sym[:id].nil?
-          raise ArgumentError,
-                "Could not find match in #{config[:symbols_file]} for given id #{id}"
-        end
-        return sym
-      end
-      raise ArgumentError, 'Need :id, :symbol or valid :contract '
-    end
-
     def compare(contract:, format: '%5.2f')
       format = "%#{format}" unless format[0] == '%'
       daily = provide(contract: contract, interval: :daily)

data/lib/cotcube-bardata/provide.rb

@@ -17,7 +17,7 @@ module Cotcube
                 force_update: false,
                 force_recent: false)
 
-      sym = get_id_set(symbol: symbol, id: id, contract: contract, config: config)
+      sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract, config: config)
 
       case interval
       when :quarters, :hours, :quarter, :hour
@@ -54,5 +54,11 @@ module Cotcube
         raise ArgumentError, "Unsupported or unknown interval '#{interval}' in Bardata.provide"
       end
     end
+
+    def determine_significant_volume(base: , contract: )
+      set  = Cotcube::Bardata.trading_hours(symbol: contract[0..1], filter: :rth)
+      prod = base - base.select_within(ranges: set ,attr: :datetime) {|x| x.to_datetime.to_sssm }
+      prod.group_by{|x| x[:volume] / 500 }
+    end
   end
 end

data/lib/cotcube-bardata/quarters.rb

@@ -16,7 +16,7 @@ module Cotcube
         raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
       end
 
-      sym = get_id_set(symbol: symbol, id: id, contract: contract)
+      sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)
 
       contract = contract[2..4] if contract.to_s.size == 5
       id = sym[:id]
@@ -33,6 +33,7 @@ module Cotcube
         %i[open high low close].map { |x| row[x] = row[x].to_f }
         %i[volume day].map { |x| row[x] = row[x].to_i }
         row[:datetime] = timezone.parse(row[:datetime])
+        row[:dist]     = ((row[:high] - row[:low]) / sym[:ticksize] ).to_i
         row[:type]     = :quarter
         row
       end

data/lib/cotcube-bardata/range_matrix.rb

@@ -24,7 +24,7 @@ module Cotcube
     def range_matrix(symbol: nil, id: nil, base: nil, print: false, dim: 0.05, days_only: false, last_n: 60, &block)
       # rubocop:disable Style/MultilineBlockChain
       symbol ||= base.last[:contract][0..1] if id.nil?
-      sym = get_id_set(symbol: symbol, id: id)
+      sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
       source = {}
       target = {}
       if base.nil?

data/lib/cotcube-bardata/suggest.rb

@@ -5,7 +5,7 @@ module Cotcube
   module Bardata
     # based on day(of year) and symbol, suggest best fitting contract
     def suggest_contract_for symbol:, date: Date.today, warnings: true
-      ml = Cotcube::Bardata.continuous_table symbol: symbol, date: date
+      ml = Cotcube::Bardata.continuous_table symbol: symbol, date: date, silent: true
       if ml.size != 1 
 	puts "WARNING: No or no unique most liquid found for #{date}. please give :contract parameter".light_yellow if warnings
 	if ml.size > 1

data/lib/cotcube-bardata/trade_dates.rb

@@ -5,19 +5,45 @@ module Cotcube
   module Bardata
     # fetching official trade dates from CME
     # it returns the current trade date or, if today isn't a trading day, the last trade date.
-    def last_trade_date
-      uri = 'https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME'
-      begin
-        HTTParty.get(uri, headers: { "User-Agent" => "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"})
-                .parsed_response
-                .map do |x|
-                  a = x['tradeDate'].chars.each_slice(2).map(&:join)
-                  "#{a[0]}#{a[1]}-#{a[2]}-#{a[3]}"
-                end
-                .first
-      rescue StandardError
-        nil
+    def last_trade_date(force_update: false)
+      const_LTD = :LAST_TRADE_DATE
+      const_LTDU = :LAST_TRADE_DATE_UPDATE
+      if force_update or not Object.const_defined?(const_LTD) or Object.const_get(const_LTD).nil? or Time.now - Object.const_get(const_LTDU) > 2.hours
+        result = nil
+        uri = 'https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME'
+        headers = { "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
+                  "Accept-Encoding" => "gzip, deflate, br",
+                  "Accept-Language" => "en-US,en;q=0.9",
+                  "Cache-Control" => "max-age=0",
+                  "Connection" => "keep-alive",
+                 # Cookie: ak_bmsc=602078F6DE40954BAA8C7E7D3815102CACE82AAFD237000084B5A460F4FBCA68~pluz010T49Xag3sXquUZtVJmFX701dzEgt5v6Ht1EZSLKE4HL+bgg1L9ePnL5I0mm7QWXe1qaLhUbX1IPrL/f20trRMMRlkC3UWXk27DY/EBCP4mRno8QQygLCwgs2B2AQHJyb63WwRihCko8UYUiIhb89ArPZM5OPraoKy3JU9oE9e+iERdARNZHLHqRiB1GnmbKUvQqos3sXaEe3GpoiTszzk8sHZs4ZKuoO/rvFHko=",
+                  "Host" => "www.cmegroup.com",
+                  "sec-ch-ua" => %q[" Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"],
+                  "sec-ch-ua-mobile" => "?0",
+                  "Sec-Fetch-Dest" => "document",
+                  "Sec-Fetch-Mode" => "navigate",
+                  "Sec-Fetch-Site" => "none",
+                  "Sec-Fetch-User" => "?1",
+                  "Upgrade-Insecure-Requests" => "1",
+                  "User-Agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36" }
+        begin
+#          HTTParty.get(uri, headers: { "User-Agent" => "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"})
+           result = HTTParty.get(uri, headers: headers)
+                  .parsed_response
+                  .map do |x|
+                    a = x['tradeDate'].chars.each_slice(2).map(&:join)
+                    "#{a[0]}#{a[1]}-#{a[2]}-#{a[3]}"
+                  end
+                  .first
+        rescue StandardError
+          result = nil
+        end
+        oldverbose = $VERBOSE; $VERBOSE = nil
+        Object.const_set(const_LTD, result)
+        Object.const_set(const_LTDU, Time.now) unless result.nil?
+        $VERBOSE = oldverbose
       end
+      Object.const_get(const_LTD)
     end
 
     def holidays(config: init)

data/lib/cotcube-bardata/trading_hours.rb

@@ -12,17 +12,23 @@ module Cotcube
                       filter: ,
                       force_filter: false,  # with force_filter one would avoid falling back
                                             # to the contract_type based range set
+                      headers_only: false,  # return only headers instead of ranges
                       config: init, debug: false)
       return (0...24 * 7 * 3600) if filter.to_s =~ /24x7/
 
       prepare = lambda do |f|
-        CSV.read(f, converters: :numeric)
-           .map(&:to_a)
-           .tap { |x| x.shift unless x.first.first.is_a?(Numeric) }
-           .map { |x| (x.first...x.last) }
+        if headers_only
+          CSV.read(f)
+            .first
+        else
+          CSV.read(f, converters: :numeric)
+             .map(&:to_a)
+             .tap { |x| x.shift unless x.first.first.is_a?(Numeric) }
+             .map { |x| (x.first...x.last) }
+        end
       end
 
-      sym = get_id_set(symbol: symbol, id: id)
+      sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
 
       file = "#{config[:data_path]}/trading_hours/#{sym[:symbol]}_#{filter}.csv"
       puts "Trying to use #{file} for #{symbol} + #{filter}" if debug

data/lib/cotcube-bardata.rb

@@ -11,6 +11,7 @@ require 'date' unless defined?(DateTime)
 require 'csv'  unless defined?(CSV)
 require 'yaml' unless defined?(YAML)
 require 'cotcube-helpers'
+require 'cotcube-indicators'
 require 'parallel'
 
 require_relative 'cotcube-bardata/constants'
@@ -61,10 +62,6 @@ module Cotcube
                     :range_matrix,
                     # create an array of ranges based on specified source data
                     :trading_hours,
-                    # receive id / symbol information on an uncertain set of parameters
-                    # TODO: decommision sppearance of get_id_set and symbols in cotcube::bardata
-                    :get_id_set,
-                    :symbols,
                     #
                     :select_specific_date,
                     :extended_select_for_range,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cotcube-bardata
 version: !ruby/object:Gem::Version
-  version: 0.1.14
+  version: 0.1.15.6
 platform: ruby
 authors:
 - Benjamin L. Tischendorf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-07 00:00:00.000000000 Z
+date: 2021-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -44,14 +44,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: cotcube-indicators
   requirement: !ruby/object:Gem::Requirement
@@ -180,7 +180,7 @@ files:
 - lib/cotcube-bardata/trading_hours.rb
 homepage: https://github.com/donkeybridge/cotcube-bardata
 licenses:
-- BSD-4-Clause
+- BSD-3-Clause
 metadata:
   homepage_uri: https://github.com/donkeybridge/cotcube-bardata
   source_code_uri: https://github.com/donkeybridge/cotcube-bardata
@@ -200,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Functions to provide bardata; and some simple time series aggregations