cotcube-bardata 0.1.8 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a701303a258aaaf6050922e35eaacf53ce09fc43fc25778492ebb0b11fb9890c
-  data.tar.gz: 389709153f639e14638832e701f19a295181e1b81e26a8aa2ba2baf5c0587ee9
+  metadata.gz: dc417f9b8b33204bbc1b345ec28e2c5604b2a5f59c183a6a0a1ad15a6cf223b1
+  data.tar.gz: f631c5507dff588e4f2e19209aca5d135514e450ddc9b311e407b9733f539191
 SHA512:
-  metadata.gz: ee32360405d9597e82c6eb9d7678895ecd1c7db9872ed2ecdd8efbd4fc0cd2ec4cea21d2546e867a45af3655ab251a34d39855d7b7d7865458ea60ff8ba65811
-  data.tar.gz: 7a4387b1834d811a61e9bd04fbe0a263f7f4a730b2a15cb928fb956929acefa5097a502fa548c26356f5ccac051d7f037344c42e320400a8a5eab55266119b81
+  metadata.gz: f5276bd185a561658226f65b92ce696774f1da7442bf0c10704925e4afd295566844d02bf866921575bf60cea6944555c7dd6abc28214ed5852ca8228553d2f3
+  data.tar.gz: c73f923b1bf4399bf6a9cea56e4adf2acc3aaeab770d4b56cc1daae7d5a4f7c4617f30baff58b03f9ced45a9987a4bf8106c709f12a1cfb17f0021cfe36a6d71

data/CHANGELOG.md

@@ -1,3 +1,23 @@
+## 0.1.11 (March 07, 2021)
+  - daily.rb: added new technique 'caching in constants' to accelerate computation, also referring to continuous
+  - provide.rb: minor change, so disregarded contracts can be used in swapproximate
+
+## 0.1.10 (February 11, 2021)
+  - Daily.rb: Added measure parameters to continous_suite
+  - cached.rb: Minor fix in comparison
+
+## 0.1.9.3 (February 08, 2021)
+  - cached: minor change fixing problem to get yesterday in rth subsets
+
+## 0.1.9.2 (February 07, 2021)
+  - minor changes
+  - daily#continuous_table: introducing debuglevel
+
+## 0.1.9.1 (January 29, 2021)
+  - provide: added new interval 'synth', mixing available dailies with synthetic days based on quarters
+  - minor change / cleanup
+  - setting gemspec to use rake 13 and changing version spec to overcome warnings
+
 ## 0.1.8 (January 27, 2021)
   - in helpers, #extended_range_for_date: fixed comparison signs
   - range_matrix: applied cops, noted appearance of Cotcube::Helpers.simple_series_stats

data/VERSION

@@ -1 +1 @@
-0.1.8
+0.1.11

data/lib/cotcube-bardata.rb

@@ -52,6 +52,8 @@ module Cotcube
                     :continuous_actual_ml,
                     # based on continuous, create list of when which contract was most liquid
                     :continuous_overview,
+                    # provider estimation of current ML usability
+                    :continuous_table,
                     # provide the list of quarters, possibly as hours or days.
                     :provide_quarters,
                     # some statistics to estimate daily volatility of specific contract

data/lib/cotcube-bardata/cached.rb

@@ -8,6 +8,7 @@ module Cotcube
                        symbol: nil, id: nil,
                        range: nil,
                        config: init,
+                       debug: false,
                        timezone: Time.find_zone('America/Chicago'),
                        filter: :full, # most probably either :full or :rth
                        force_update: false, # force reloading via provide_quarters
@@ -59,9 +60,9 @@ module Cotcube
                      end
                    end
           return result
-        elsif File.mtime(file) + 1.day > File.mtime(quarters_file)
-          puts "CACHE #{File.mtime(file)}\t#{file}"
-          puts "QUART #{File.mtime(quarters_file)}\t#{quarters_file}"
+        elsif File.mtime(file) - Time.now.beginning_of_day >= 0
+          puts "CACHE #{File.mtime(file)}\t#{file}" if debug
+          puts "QUART #{File.mtime(quarters_file)}\t#{quarters_file}" if debug
           result = if range.nil?
                      base
                    else
@@ -97,17 +98,17 @@ module Cotcube
       base = Cotcube::Helpers.reduce(bars: data, to: :days)
 
       # remove last day of result unless marked
-      base.pop unless contract_is_marked || force_recent
+      base.pop if base.last[:datetime].to_date == timezone.now.to_date and not force_recent
 
       base.map do |x|
-        x[:datetime] = x[:datetime].to_date
+        x[:date] = x[:datetime].to_date
         x[:type] = "#{filter}_day".to_sym
         x.delete(:day)
       end
       CSV.open(file, 'w') do |csv|
         base.each { |b| csv << b.values_at(*headers) }
         if contract_is_marked
-          marker = ["#{sym[:symbol]}#{contract}", base.last[:datetime] + 1.day, 0, 0, 0, 0, 0]
+          marker = ["#{sym[:symbol]}#{contract}", base.last[:date] + 1.day, 0, 0, 0, 0, 0]
           csv << marker
         end
       end

data/lib/cotcube-bardata/daily.rb

@@ -61,29 +61,50 @@ module Cotcube
 
     # reads all files in  bardata/daily/<id> and aggregates by date
     # (what is a pre-stage of a continuous based on daily bars)
-    def continuous(symbol: nil, id: nil, config: init, date: nil)
+    def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false)
+      raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
+      measuring = lambda {|c| puts "[continuous] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
+
+      measuring.call("Starting")
       sym = get_id_set(symbol: symbol, id: id)
       id  = sym[:id]
       id_path = "#{config[:data_path]}/daily/#{id}"
-      available_contracts = Dir["#{id_path}/*.csv"].map { |x| x.split('/').last.split('.').first }
-      available_contracts.sort_by! { |x| x[-7] }.sort_by! { |x| x[-6..-5] }
-      data = []
-      available_contracts.each do |c|
-        provide_daily(id: id, config: config, contract: c).each do |x|
-          data << x
+      c_file  = "#{id_path}/continuous.csv"
+
+      # instead of using the provide_daily methods above, for this bulk operation a 'continuous.csv' is created
+      # this boosts from 4.5sec to 0.3sec
+      rewriting = force_rewrite or not(File.exist?(c_file)) or (Time.now - File.mtime(c_file) > 8.days)
+      if rewriting
+        puts "In daily+continuous: Rewriting #{c_file} #{force_rewrite ? "forcibly" : "due to fileage"}.".light_yellow
+        `rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v '0,0,0,0' | sort -t, -k2 | cut -d, -f1,2,7,8 > #{c_file}`
+      end
+      loading = lambda do
+        data = CSV.read(c_file).map do |row|
+          r = { contract: row[0],
+                date:     row[1],
+                volume:   row[2].to_i,
+                oi:       row[3].to_i
+          }
+        end
+
+        measuring.call("Finished retrieving dailies.")
+        result = []
+        data.group_by { |x| x[:date] }.map do |k, v|
+          v.map { |x| x.delete(:date) }
+          result << {
+            date: k,
+            volume: v.map { |x| x[:volume] }.reduce(:+),
+            oi: v.map { |x| x[:oi] }.reduce(:+)
+          }
+          result.last[:contracts] = v
         end
       end
-      result = []
-      data.sort_by { |x| x[:date] }.group_by { |x| x[:date] }.map do |k, v|
-        v.map { |x| x.delete(:date) }
-        result << {
-          date: k,
-          volume: v.map { |x| x[:volume] }.reduce(:+),
-          oi: v.map { |x| x[:oi] }.reduce(:+)
-        }
-        result.last[:contracts] = v
+      constname = "CONTINUOUS_#{symbol}".to_sym
+      if rewriting or not  Cotcube::Bardata.const_defined?( constname)
+        Cotcube::Bardata.const_set constname, loading.call
       end
-      date.nil? ? result : result.select { |x| x[:date] == date }.first
+      measuring.call("Finished processing")
+      date.nil? ? Cotcube::Bardata.const_get(constname) : Cotcube::Bardata.const_get(constname).find { |x| x[:date] == date }
     end
 
     def continuous_ml(symbol: nil, id: nil, base: nil)
@@ -116,20 +137,27 @@ module Cotcube
                             config: init,
                             selector: :volume,
                             human: false,
+                            measure: nil,
                             filter: nil)
+
+      raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
+      measuring = lambda {|c| puts "[continuous_overview] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
+
       raise ArgumentError, 'Selector must be either :volume or :oi' unless selector.is_a?(Symbol) &&
-                                                                           %i[volume oi].include?(selector)
+        %i[volume oi].include?(selector)
 
+      measuring.call("Starting")
       sym = get_id_set(symbol: symbol, id: id)
       id  = sym[:id]
       # noinspection RubyNilAnalysis
-      data = continuous(id: id, config: config).map do |x|
+      data = continuous(id: id, config: config, measure: measure).map do |x|
         {
           date: x[:date],
           volume: x[:contracts].sort_by { |z| - z[:volume] }[0..4].compact.reject { |z| z[:volume].zero? },
           oi: x[:contracts].sort_by { |z| - z[:oi] }[0..4].compact.reject { |z| z[:oi].zero? }
         }
       end
+      measuring.call("Retrieved continuous")
       data.reject! { |x| x[selector].empty? }
       result = data.group_by { |x| x[selector].first[:contract] }
       result.each_key do |key|
@@ -149,31 +177,44 @@ module Cotcube
              }\t#{v.last[:date]
              }\t#{format('%4d', (Date.parse(v.last[:date]) - Date.parse(v.first[:date])))
              }\t#{result[k].map do |x|
-               x[:volume].select do
-                 x[:contract] == k
-               end
-             end.size
+            x[:volume].select do
+              x[:contract] == k
+            end
+          end.size
              }"
           # rubocop:enable Layout/ClosingParenthesisIndentation
         end
       end
+      measuring.call("Finished processing")
       result
     end
 
     def continuous_table(symbol: nil, id: nil,
                          selector: :volume,
                          filter: nil,
+                         date: Date.today,
+                         measure: nil,
+                         debuglevel: 1,
                          debug: false)
+      if debug.is_a?(Integer)
+        debuglevel = debug
+        debug = debuglevel > 0 ? true : false
+      end
+
+      raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
+      measuring = lambda {|c| puts "[continuous_table] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
+
       raise ArgumentError, 'Selector must be either :volume or :oi' unless selector.is_a?(Symbol) &&
-                                                                           %i[volume oi].include?(selector)
+        %i[volume oi].include?(selector)
 
+      measuring.call("Entering function")
       sym = get_id_set(symbol: symbol, id: id)
       if %w[R6 BJ GE].include? sym[:symbol]
-        puts "Rejecting to process symbol '#{sym[:symbol]}'."
-        return
+        puts "Rejecting to process symbol '#{sym[:symbol]}'.".light_red
+        return []
       end
       id  = sym[:id]
-      dfm = lambda do |x, y = Date.today.year|
+      dfm = lambda do |x, y = date.year|
         k = Date.strptime("#{y} #{x.negative? ? x + 366 : x}", '%Y %j')
         k -= 1 while [0, 6].include?(k.wday)
         k.strftime('%a, %Y-%m-%d')
@@ -181,13 +222,15 @@ module Cotcube
         puts "#{sym[:symbol]}\t#{x}\t#{y}"
       end
 
-      ytoday = Date.today.yday
-      data = continuous_overview(id: id, selector: selector, filter: filter, human: false, config: init)
-             .reject { |k, _| k[-2..].to_i == Date.today.year % 2000 }
-             .group_by { |k, _| k[2] }
-      output_sent = 0
+      ytoday = date.yday
+      data = continuous_overview(id: id, selector: selector, filter: filter, human: false, config: init, measure: measure)
+        .reject { |k, _| k[-2..].to_i == date.year % 2000 }
+        .group_by { |k, _| k[2] }
+      measuring.call("Retrieved continous_overview")
+      output_sent = []
+      early_year=nil
       data.keys.sort.each do |month|
-        puts "Processing #{sym[:symbol]}#{month}" if debug
+        puts "Processing #{sym[:symbol]}#{month}" if debuglevel > 1
         v0 = data[month]
         ydays = v0.map { |_, v1| Date.parse(v1.last[:date]).yday }
         fdays = v0.map { |_, v1| Date.parse(v1.first[:date]).yday }.sort
@@ -207,10 +250,9 @@ module Cotcube
                 else
                   :white
                 end
-        if debug || (color != :white)
-          # rubocop:disable Layout/ClosingParenthesisIndentation
-          puts "#{sym[:symbol]
-               }#{month
+        # rubocop:disable Layout/ClosingParenthesisIndentation
+        output = "#{sym[:symbol]
+             }#{month
              }\t#{format '%12s', sym[:type]
              }\t#{ytoday
               }/#{fday
@@ -220,22 +262,27 @@ module Cotcube
              }: #{dfm.call(yavg)
              }\t#{format '%5d', ydays.max
              }: #{dfm.call(ydays.max)}".colorize(color)
-          output_sent += 1
-        end
-        next unless debug
+             if debug || (color != :white)
+               puts output
+               output_sent << "#{sym[:symbol]}#{month}" unless color == :white
+             end
+             early_year  ||= output
+             next unless (debug and debuglevel >= 2)
 
-        v0.each do |contract, v1|
-          puts "\t#{contract
+             v0.each do |contract, v1|
+               puts "\t#{contract
                }\t#{v1.first[:date]
                }\t#{Date.parse(v1.last[:date]).strftime('%a')
                }, #{v1.last[:date]
                } (#{Date.parse(v1.last[:date]).yday}"
-          # rubocop:enable Layout/ClosingParenthesisIndentation
-        end
+               # rubocop:enable Layout/ClosingParenthesisIndentation
+             end
       end
-      case output_sent
+      case output_sent.size
       when 0
         puts "WARNING: No output was sent for symbol '#{sym[:symbol]}'.".colorize(:light_yellow)
+        puts "         Assuming late-year-processing.".light_yellow
+        puts early_year.light_green
       when 1
         # all ok
         true
@@ -243,7 +290,8 @@ module Cotcube
         puts "---- #{output_sent} for #{sym[:symbol]} ---------------"
 
       end
-      nil
+      measuring.call("Finished processing")
+      output_sent
     end
   end
 end

data/lib/cotcube-bardata/provide.rb

@@ -40,6 +40,15 @@ module Cotcube
         raise 'Missing implementation to reduce cached days to weeks or months'
       when :dailies, :daily
         provide_daily contract: contract, symbol: symbol, id: id, config: config, range: range
+      when :synth, :synthetic, :synthetic_days
+        days = provide_cached contract: contract, symbol: symbol, id: id, config: config, filter: filter,
+                              range: range, force_recent: force_recent
+        dailies = provide_daily contract: contract, symbol: symbol, id: id, config: config, range: range
+        if ((days.last[:datetime] > dailies.last[:datetime]) rescue false)
+          dailies[..-2] + days.select { |d| d[:datetime] > dailies[-2][:datetime] }
+        else
+          dailies
+        end
       else
         raise ArgumentError, "Unsupported or unknown interval '#{interval}' in Bardata.provide"
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cotcube-bardata
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.11
 platform: ruby
 authors:
 - Benjamin L. Tischendorf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-27 00:00:00.000000000 Z
+date: 2021-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport