cotcube-bardata 0.1.9.2 → 0.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 42d2c651e2e0cc53cbd35ea0ea824e82828930bee04d1bce9a52edf9b70c54f1
-  data.tar.gz: 388c35d25810c30a73136e66e0e204473dd2646f2ea776c23a476b8cb769afb7
+  metadata.gz: 6b1f66885f510fbe8be92a8e543f7206220db76940825862115c60be6892d5d4
+  data.tar.gz: a12e509a14c41cc17e31d95fca73d09ab09c342b029e973baf30377ebe8f1d6a
 SHA512:
-  metadata.gz: 705ded304d33270f8133a5e297a807a56288b7a710ac12f6aa2896f4971c2302f6e5a3a9b229c8c513489abb32bbe3079e0f60bdf094756935f19a6d84d7d276
-  data.tar.gz: e36f72ffc6dbf098a86b647932f8d881016477ebb74223b8db12f75f3e9c58f4962eef7da5992838c72538d1e18545f647b4df104554e70727ba45694b55de59
+  metadata.gz: dc53284daf85e21387ba1d7b24b3e2518be8dae3d49f6b2893e6aa6442e082516be12f76f23001ad3f04aee3e3ae5601ae7e747fd8451d5d9a437c7ac2b45976
+  data.tar.gz: 3b496b4e75e73feee1dbc08fd355572471ba78eb1b692c91e81dd3b1f515eab510cab0d22bc712cabab84cfe621a536e91a81e0925ce897e8d48c05770b0b015

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 0.1.13 (April 07, 2021)
+  - daily: fixed const_caching in continuous
+  - trade_dates: FIXING call with HTTParty must send Agent Header
+  - helpers/get_id_set: added support for params given as Symbols (:NG instead of 'NG')
+
+## 0.1.12 (March 13, 2021)
+  - range_matrix: adapted to accept block for range provision; added params :days_only and :last_n
+  - minor fix on previous patch
+
+## 0.1.11 (March 07, 2021)
+  - daily.rb: added new technique 'caching in constants' to accelerate computation, also referring to continuous
+  - provide.rb: minor change, so disregarded contracts can be used in swapproximate
+
+## 0.1.10 (February 11, 2021)
+  - Daily.rb: Added measure parameters to continous_suite
+  - cached.rb: Minor fix in comparison
+
+## 0.1.9.3 (February 08, 2021)
+  - cached: minor change fixing problem to get yesterday in rth subsets
+
 ## 0.1.9.2 (February 07, 2021)
   - minor changes
   - daily#continuous_table: introducing debuglevel

data/VERSION

@@ -1 +1 @@
-0.1.9.2
+0.1.13

data/lib/cotcube-bardata/cached.rb

@@ -60,7 +60,7 @@ module Cotcube
                      end
                    end
           return result
-        elsif File.mtime(file) + 1.day > File.mtime(quarters_file)
+        elsif File.mtime(file) - Time.now.beginning_of_day >= 0
           puts "CACHE #{File.mtime(file)}\t#{file}" if debug
           puts "QUART #{File.mtime(quarters_file)}\t#{quarters_file}" if debug
           result = if range.nil?
@@ -98,17 +98,17 @@ module Cotcube
       base = Cotcube::Helpers.reduce(bars: data, to: :days)
 
       # remove last day of result unless marked
-      base.pop unless contract_is_marked || force_recent
+      base.pop if base.last[:datetime].to_date == timezone.now.to_date and not force_recent
 
       base.map do |x|
-        x[:datetime] = x[:datetime].to_date
+        x[:date] = x[:datetime].to_date
         x[:type] = "#{filter}_day".to_sym
         x.delete(:day)
       end
       CSV.open(file, 'w') do |csv|
         base.each { |b| csv << b.values_at(*headers) }
         if contract_is_marked
-          marker = ["#{sym[:symbol]}#{contract}", base.last[:datetime] + 1.day, 0, 0, 0, 0, 0]
+          marker = ["#{sym[:symbol]}#{contract}", base.last[:date] + 1.day, 0, 0, 0, 0, 0]
           csv << marker
         end
       end

data/lib/cotcube-bardata/daily.rb

@@ -8,6 +8,7 @@ module Cotcube
                       symbol: nil, id: nil,
                       range: nil,
                       timezone: Time.find_zone('America/Chicago'),
+                      keep_last: false,
                       config: init)
       contract = contract.to_s.upcase
       unless contract.is_a?(String) && [3, 5].include?(contract.size)
@@ -48,7 +49,7 @@ module Cotcube
         row[:type]     = :daily
         row
       end
-      data.pop if data.last[:high].zero?
+      data.pop if data.last[:high].zero? and not keep_last
       if range.nil?
         data
       else
@@ -61,29 +62,52 @@ module Cotcube
 
     # reads all files in  bardata/daily/<id> and aggregates by date
     # (what is a pre-stage of a continuous based on daily bars)
-    def continuous(symbol: nil, id: nil, config: init, date: nil)
+    def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false)
+      raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
+      measuring = lambda {|c| puts "[continuous] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
+
+      measuring.call("Starting")
       sym = get_id_set(symbol: symbol, id: id)
       id  = sym[:id]
+      symbol = sym[:symbol]
       id_path = "#{config[:data_path]}/daily/#{id}"
-      available_contracts = Dir["#{id_path}/*.csv"].map { |x| x.split('/').last.split('.').first }
-      available_contracts.sort_by! { |x| x[-7] }.sort_by! { |x| x[-6..-5] }
-      data = []
-      available_contracts.each do |c|
-        provide_daily(id: id, config: config, contract: c).each do |x|
-          data << x
+      c_file  = "#{id_path}/continuous.csv"
+
+      # instead of using the provide_daily methods above, for this bulk operation a 'continuous.csv' is created
+      # this boosts from 4.5sec to 0.3sec
+      rewriting = force_rewrite or not(File.exist?(c_file)) or (Time.now - File.mtime(c_file) > 8.days)
+      if rewriting
+        puts "In daily+continuous: Rewriting #{c_file} #{force_rewrite ? "forcibly" : "due to fileage"}.".light_yellow
+        `rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v '0,0,0,0' | sort -t, -k2 | cut -d, -f1,2,7,8 > #{c_file}`
+      end
+      loading = lambda do
+        data = CSV.read(c_file).map do |row|
+          r = { contract: row[0],
+                date:     row[1],
+                volume:   row[2].to_i,
+                oi:       row[3].to_i
+          }
         end
+
+        measuring.call("Finished retrieving dailies.")
+        result = []
+        data.group_by { |x| x[:date] }.map do |k, v|
+          v.map { |x| x.delete(:date) }
+          result << {
+            date: k,
+            volume: v.map { |x| x[:volume] }.reduce(:+),
+            oi: v.map { |x| x[:oi] }.reduce(:+)
+          }
+          result.last[:contracts] = v
+        end
+        result
       end
-      result = []
-      data.sort_by { |x| x[:date] }.group_by { |x| x[:date] }.map do |k, v|
-        v.map { |x| x.delete(:date) }
-        result << {
-          date: k,
-          volume: v.map { |x| x[:volume] }.reduce(:+),
-          oi: v.map { |x| x[:oi] }.reduce(:+)
-        }
-        result.last[:contracts] = v
+      constname = "CONTINUOUS_#{symbol}".to_sym
+      if rewriting or not  Cotcube::Bardata.const_defined?( constname)
+        Cotcube::Bardata.const_set constname, loading.call
       end
-      date.nil? ? result : result.select { |x| x[:date] == date }.first
+      measuring.call("Finished processing")
+      date.nil? ? Cotcube::Bardata.const_get(constname) : Cotcube::Bardata.const_get(constname).find { |x| x[:date] == date }
     end
 
     def continuous_ml(symbol: nil, id: nil, base: nil)
@@ -116,20 +140,27 @@ module Cotcube
                             config: init,
                             selector: :volume,
                             human: false,
+                            measure: nil,
                             filter: nil)
+
+      raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
+      measuring = lambda {|c| puts "[continuous_overview] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
+
       raise ArgumentError, 'Selector must be either :volume or :oi' unless selector.is_a?(Symbol) &&
-                                                                           %i[volume oi].include?(selector)
+        %i[volume oi].include?(selector)
 
+      measuring.call("Starting")
       sym = get_id_set(symbol: symbol, id: id)
       id  = sym[:id]
       # noinspection RubyNilAnalysis
-      data = continuous(id: id, config: config).map do |x|
+      data = continuous(id: id, config: config, measure: measure).map do |x|
         {
           date: x[:date],
           volume: x[:contracts].sort_by { |z| - z[:volume] }[0..4].compact.reject { |z| z[:volume].zero? },
           oi: x[:contracts].sort_by { |z| - z[:oi] }[0..4].compact.reject { |z| z[:oi].zero? }
         }
       end
+      measuring.call("Retrieved continuous for #{sym[:symbol]}")
       data.reject! { |x| x[selector].empty? }
       result = data.group_by { |x| x[selector].first[:contract] }
       result.each_key do |key|
@@ -149,14 +180,15 @@ module Cotcube
              }\t#{v.last[:date]
              }\t#{format('%4d', (Date.parse(v.last[:date]) - Date.parse(v.first[:date])))
              }\t#{result[k].map do |x|
-               x[:volume].select do
-                 x[:contract] == k
-               end
-             end.size
+            x[:volume].select do
+              x[:contract] == k
+            end
+          end.size
              }"
           # rubocop:enable Layout/ClosingParenthesisIndentation
         end
       end
+      measuring.call("Finished processing")
       result
     end
 
@@ -164,19 +196,26 @@ module Cotcube
                          selector: :volume,
                          filter: nil,
                          date: Date.today,
+                         short: true,
+                         measure: nil,
                          debuglevel: 1,
                          debug: false)
       if debug.is_a?(Integer)
         debuglevel = debug
         debug = debuglevel > 0 ? true : false
       end
+
+      raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
+      measuring = lambda {|c| puts "[continuous_table] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
+
       raise ArgumentError, 'Selector must be either :volume or :oi' unless selector.is_a?(Symbol) &&
-                                                                           %i[volume oi].include?(selector)
+        %i[volume oi].include?(selector)
 
+      measuring.call("Entering function")
       sym = get_id_set(symbol: symbol, id: id)
       if %w[R6 BJ GE].include? sym[:symbol]
-        puts "Rejecting to process symbol '#{sym[:symbol]}'."
-        return
+        puts "Rejecting to process symbol '#{sym[:symbol]}'.".light_red
+        return []
       end
       id  = sym[:id]
       dfm = lambda do |x, y = date.year|
@@ -188,60 +227,69 @@ module Cotcube
       end
 
       ytoday = date.yday
-      data = continuous_overview(id: id, selector: selector, filter: filter, human: false, config: init)
+      data = continuous_overview(id: id, selector: selector, filter: filter, human: false, config: init, measure: measure)
         .reject { |k, _| k[-2..].to_i == date.year % 2000 }
-             .group_by { |k, _| k[2] }
+        .group_by { |k, _| k[2] }
+      measuring.call("Retrieved continous_overview")
       output_sent = []
       early_year=nil
+      long_output = []
       data.keys.sort.each do |month|
+        current_long = { month: month }
         puts "Processing #{sym[:symbol]}#{month}" if debuglevel > 1
         v0 = data[month]
-        ydays = v0.map { |_, v1| Date.parse(v1.last[:date]).yday }
+        ldays = v0.map { |_, v1| Date.parse(v1.last[:date]).yday }
         fdays = v0.map { |_, v1| Date.parse(v1.first[:date]).yday }.sort
         # if the last ml day nears the end of the year, we must fix
-        ydays.map! { |x| x > 350 ? x - 366 : x } if ydays.min < 50
+        ldays.map! { |x| x > 350 ? x - 366 : x } if ldays.min < 50
         fday  = fdays[fdays.size / 2]
-        yavg  = ydays.reduce(:+) / ydays.size
-        # a contract is proposed to use after fday - 1, but before ydays.min (green)
-        # it is warned to user after fday - 1 but before yavg - 1            (red)
-        # it is warned red >= yavg - 1 and <= yavg + 1
-        color = if (ytoday >= yavg - 1) && (ytoday <= yavg + 1)
+        lavg  = ldays.reduce(:+) / ldays.size
+        # a contract is proposed to use after fday - 1, but before ldays.min (green)
+        # it is warned to user after fday - 1 but before lavg - 1            (red)
+        # it is warned red >= lavg - 1 and <= lavg + 1
+        color = if (ytoday >= lavg - 1) && (ytoday <= lavg + 1)
                   :light_red
-                elsif (ytoday > ydays.min) && (ytoday < yavg - 1)
+                elsif (ytoday > ldays.min) && (ytoday < lavg - 1)
                   :light_yellow
-                elsif (ytoday >= (fday > yavg ? 0 : fday - 5)) && (ytoday <= ydays.min)
+                elsif (ytoday >= (fday > lavg ? 0 : fday - 5)) && (ytoday <= ldays.min)
                   :light_green
                 else
                   :white
                 end
         # rubocop:disable Layout/ClosingParenthesisIndentation
+        long_output << {
+          month:    month,
+          first_ml: fday,
+          last_min: ldays.min,
+          last_avg: lavg,
+          last_max: ldays.max }
         output = "#{sym[:symbol]
              }#{month
              }\t#{format '%12s', sym[:type]
-             }\t#{ytoday
-              }/#{fday
-             }\t#{format '%5d', ydays.min
-             }: #{dfm.call(ydays.min)
-             }\t#{format '%5d', yavg
-             }: #{dfm.call(yavg)
-             }\t#{format '%5d', ydays.max
-             }: #{dfm.call(ydays.max)}".colorize(color)
-        if debug || (color != :white)
-          puts output
-          output_sent << "#{sym[:symbol]}#{month}" unless color == :white
-        end
-        early_year  ||= output
-        next unless (debug and debuglevel >= 2)
+             }\ttoday is #{ytoday
+             } -- median of first is #{fday
+             }\tlast ranges from #{format '%5d', ldays.min
+             }: #{dfm.call(ldays.min)
+             }\t#{format '%5d', lavg
+             }: #{dfm.call(lavg)
+             }\tto #{format '%5d', ldays.max
+             }: #{dfm.call(ldays.max)}".colorize(color)
+             if debug || (color != :white)
+               puts output
+               output_sent << "#{sym[:symbol]}#{month}" unless color == :white
+             end
+             early_year  ||= output
+             next unless (debug and debuglevel >= 2)
 
-        v0.each do |contract, v1|
-          puts "\t#{contract
+             v0.each do |contract, v1|
+               puts "\t#{contract
                }\t#{v1.first[:date]
-               }\t#{Date.parse(v1.last[:date]).strftime('%a')
-               }, #{v1.last[:date]
-               } (#{Date.parse(v1.last[:date]).yday}"
+               } (#{format '%3d', Date.parse(v1.first[:date]).yday
+              })\t#{Date.parse(v1.last[:date]).strftime('%a, %Y-%m-%d')
+               } (#{Date.parse(v1.last[:date]).yday})"
                # rubocop:enable Layout/ClosingParenthesisIndentation
+             end
         end
-      end
       case output_sent.size
       when 0
         puts "WARNING: No output was sent for symbol '#{sym[:symbol]}'.".colorize(:light_yellow)
@@ -251,10 +299,10 @@ module Cotcube
         # all ok
         true
       else
-        puts "---- #{output_sent} for #{sym[:symbol]} ---------------"
-
+        puts "Continuous table show #{output_sent.size} active contracts ( #{output_sent} ) for #{sym[:symbol]} ---------------"
       end
-      output_sent
+      measuring.call("Finished processing")
+      short ? output_sent : long_output
     end
   end
 end

data/lib/cotcube-bardata/helpers.rb

@@ -46,6 +46,10 @@ module Cotcube
     end
 
     def get_id_set(symbol: nil, id: nil, contract: nil, config: init)
+      contract = contract.to_s.upcase if contract.is_a? Symbol
+      id       =       id.to_s.upcase if       id.is_a? Symbol
+      symbol   =   symbol.to_s.upcase if   symbol.is_a? Symbol
+
       if contract.is_a?(String) && (contract.length == 5)
         c_symbol = contract[0..1]
         if (not symbol.nil?) && (symbol != c_symbol)

data/lib/cotcube-bardata/init.rb

@@ -8,9 +8,9 @@ module Cotcube
         SYMBOL_EXAMPLES
       else
         CSV
-          .read(config[:symbols_file], headers: %i[id symbol ticksize power months type bcf reports name])
+          .read(config[:symbols_file], headers: %i[id symbol ticksize power months type bcf reports format name])
           .map(&:to_h)
-          .map { |row| %i[ticksize power bcf].each { |z| row[z] = row[z].to_f }; row } # rubocop:disable Style/Semicolon
+          .map { |row| %i[ticksize power bcf].each { |z| row[z] = row[z].to_f }; row[:format] = "%#{row[:format]}f"; row } # rubocop:disable Style/Semicolon
           .reject { |row| row[:id].nil? }
           .tap { |all| all.select! { |x| x[:type] == type } unless type.nil? }
           .tap { |all| all.select! { |x| x[:symbol] == symbol } unless symbol.nil? }

data/lib/cotcube-bardata/provide.rb

@@ -44,7 +44,7 @@ module Cotcube
         days = provide_cached contract: contract, symbol: symbol, id: id, config: config, filter: filter,
                               range: range, force_recent: force_recent
         dailies = provide_daily contract: contract, symbol: symbol, id: id, config: config, range: range
-        if days.last[:datetime] > dailies.last[:datetime]
+        if ((days.last[:datetime] > dailies.last[:datetime]) rescue false)
           dailies[..-2] + days.select { |d| d[:datetime] > dailies[-2][:datetime] }
         else
           dailies

data/lib/cotcube-bardata/range_matrix.rb

@@ -5,7 +5,6 @@ module Cotcube
   module Bardata
     # this is an analysis tool to investigate actual ranges of an underlying symbol
     # it is in particular no true range or average true range, as a 'true range' can only be applied to
-    # a steady series, what changing contracts definitely aren't
     #
     # The result printed / returned is a table, containing a matrix of rows:
     #   1. size: the amount of values evaluated
@@ -22,18 +21,25 @@ module Cotcube
     #   3.a-c) same with days reduced to months (c: 12 months)
     #
     # NOTE: there is now a new method Cotcube::Helpers.simple_series_stats, that should be used in favor.
-    def range_matrix(symbol: nil, id: nil, print: false, dim: 0.05)
+    def range_matrix(symbol: nil, id: nil, base: nil, print: false, dim: 0.05, days_only: false, last_n: 60, &block)
       # rubocop:disable Style/MultilineBlockChain
+      symbol ||= base.last[:contract][0..1] if id.nil?
       sym = get_id_set(symbol: symbol, id: id)
       source = {}
       target = {}
-      source[:days]   = Cotcube::Bardata.continuous_actual_ml symbol: symbol
+      if base.nil?
+        ml = (Cotcube::Bardata.continuous_actual_ml symbol: symbol)&.last[:contract]
+        source[:days]   = Cotcube::Bardata.provide contract: ml
+      else
+        source[:days]   = base
+      end
       source[:weeks]  = Cotcube::Helpers.reduce bars: source[:days], to: :weeks
       source[:months] = Cotcube::Helpers.reduce bars: source[:days], to: :months
 
       %i[days weeks months].each do |period|
+        next if days_only and %i[weeks months].include? period
         source[period].map! do |x|
-          x[:range] = ((x[:high] - x[:low]) / sym[:ticksize]).round
+          x[:range] = block_given? ? yield(x) : (((x[:high] - x[:low]) / sym[:ticksize]).round)
           x
         end
         target[period] = {}
@@ -80,14 +86,18 @@ module Cotcube
 
         range = case period
                 when :months
-                  -13..-2
+                  -(last_n/15)..-2
                 when :weeks
-                  -53..-2
+                  -(last_n/4)..-2
                 when :days
-                  -200..-1
+                  -last_n..-1
                 else
                   raise ArgumentError, "Unsupported period: '#{period}'"
                 end
+        if range.begin.abs > source[period].size
+          puts "WARNING: requested last_n = #{last_n} exceeds actual size (#{source[period].size}), adjusting...".light_yellow
+          range = (-source[period].size..range.end)
+        end
         custom = source[period][range]
         target[period][:rec_size]   =  custom.size
         target[period][:rec_avg]    = (custom.map { |x| x[:range] }.reduce(:+) / custom.size).round
@@ -110,6 +120,7 @@ module Cotcube
         %w[size avg lower median upper max].each do |a|
           print "#{'%10s' % a} | " # rubocop:disable Style/FormatString
           %i[days weeks months].each do |b|
+            next if days_only and %i[weeks months].include? b
             %w[all dim rec].each do |c|
               print ('%8d' % target[b]["#{c}_#{a}".to_sym]).to_s # rubocop:disable Style/FormatString
             end

data/lib/cotcube-bardata/trade_dates.rb

@@ -8,7 +8,7 @@ module Cotcube
     def last_trade_date
       uri = 'https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME'
       begin
-        HTTParty.get(uri)
+        HTTParty.get(uri, headers: { "User-Agent" => "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"})
                 .parsed_response
                 .map do |x|
                   a = x['tradeDate'].chars.each_slice(2).map(&:join)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cotcube-bardata
 version: !ruby/object:Gem::Version
-  version: 0.1.9.2
+  version: 0.1.13
 platform: ruby
 authors:
 - Benjamin L. Tischendorf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-07 00:00:00.000000000 Z
+date: 2021-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport