cotcube-level 0.2.0 → 0.3.4

data/lib/cotcube-level/helpers.rb

@@ -2,99 +2,223 @@
 
 module Cotcube
   module Level
-      def rad2deg(deg)
-        deg * 180 / Math::PI
-      end
-
-      def deg2rad(rad)
-        rad * Math::PI / 180
-      end
 
-      def shear_to_deg(base:, deg:)
-        shear_to_rad(base: base, rad: deg2rad(deg))
-      end
+    # 3 simple, self-explaining helpers
+    def rad2deg(deg); deg * 180 / Math::PI; end
+    def deg2rad(rad); rad * Math::PI / 180; end
+    def shear_to_deg(base:, deg:); shear_to_rad(base: base, rad: deg2rad(deg)); end
 
-      def shear_to_rad(base: , rad:)
-        tan = Math.tan(rad)
-        base.map { |bar|
-          # separating lines for easier debugging
-          bar[:yy] =
-            bar[:y] +
-            (bar[:dx].nil? ? bar[:x] : bar[:dx]) * tan
-          bar
-        }
-      end
+    # the actual shearing takes place here. please not that shifting of :x takes place
+    #     by setting the new :x as :dx. so if :dx is found, it is used, otherwise :x
+    def shear_to_rad(base: , rad:)
+      tan = Math.tan(rad)
+      base.map { |member|
+        # separating lines for easier debugging
+        member[:yy] =
+          member[:y] +
+          (member[:dx].nil? ? member[:x] : member[:dx]) * tan
+        member
+      }
+    end
 
-      def member_to_human(member,side: ,format:)
-        high = side == :high
-             "#{member[:datetime].strftime("%a, %Y-%m-%d %H:%M")
-        }  x: #{format '%-4d', member[:x]
+    # human readable output
+    # please note the format must be given, that should be taken from :sym
+    def member_to_human(member,side: ,format:, daily: false)
+      high = (side == :upper)
+      "#{                         member[:datetime].strftime("%a, %Y-%m-%d#{daily ? "" :" %I:%M%p"}")
+        }  x: #{format '%-4d',    member[:x]
         } dx: #{format '%-8.3f', (member[:dx].nil? ? member[:x] : member[:dx].round(3))
             } #{high ? "high" : "low"
-           }: #{format format, member[high ? :high : :low]
-         } i: #{(format '%4d', member[:i]) unless member[:i].nil?
-            } #{member[:miss].nil? ? '' : "miss: #{member[:miss]}" }"
-      end
+           }: #{format format,    member[high ? :high : :low]
+         } i: #{(format '%4d',    member[:i]) unless member[:i].nil?
+         } d: #{format '%6.2f',   member[:dev] unless member[:dev].nil?
+            } #{member[:near].nil? ? '' : "near: #{member[:near]}"
+         }"
+    end
 
-      def puts_swaps(swaps, format: )
-        swaps = [ swaps ] unless swaps.is_a? Array
-        swaps.each do |swap|
-          puts "side: #{swap[:side] }\tlen: #{swap[:length]}  \trating: #{swap[:rating]}".colorize(swap[:color] || :white )
-          puts "diff: #{swap[:ticks]}\tdif: #{swap[:diff].round(7)}\tdepth: #{swap[:depth]}".colorize(swap[:color] || :white )
-          puts "tpi:  #{swap[:tpi]  }\tppi: #{swap[:ppi]}".colorize(swap[:color] || :white )
-          swap[:members].each {|x| puts member_to_human(x, side: swap[:side], format: format) }
-        end
+    # human readable output
+    # format: e.g. sym[:format]
+    # short:  print one line / less verbose
+    # notice: add this to output as well
+    def puts_swap(swap, format: , short: true, notice: nil, hash: 3)
+      return if swap[:empty]
+      daily =  %i[ continuous daily ].include?(swap[:interval].to_sym) rescue false
+      datetime_format = daily ? '%Y-%m-%d' : '%Y-%m-%d %I:%M %p'
+      high = swap[:side] == :high
+      ohlc = high ? :high : :low
+      if notice.nil? and swap[:exceeded]
+        notice = "exceeded #{swap[:exceeded].strftime(datetime_format)}"
       end
-
-      def get_jsonl_name(interval:, swap_type:, contract:, sym: nil)
-        raise "Interval #{interval } is not supported, please choose from #{INTERVALS}" unless INTERVALS.include? interval
-        raise "Swaptype #{swap_type} is not supported, please choose from #{SWAPTYPES}" unless SWAPTYPES.include? swap_type
-        sym ||= Cotcube::Helpers.get_id_set(contract: contract)
-        root = '/var/cotcube/level'
-        dir     = "#{root}/#{sym[:id]}"
-        symlink = "#{root}/#{sym[:symbol]}"
-        `mkdir -p #{dir}`         unless File.exist?(dir)
-        `ln -s #{dir} #{symlink}` unless File.exist?(symlink)
-        file = "#{dir}/#{contract}_#{interval.to_s}_#{swap_type.to_s}.jsonl"
-        `touch #{file}`
-        file
+      if short
+        puts (hash ? "#{swap[:digest][...hash]}    ".colorize(:light_white) : '') +
+             "S: #{swap[:side]
+            } L: #{   format '%4d', swap[:length]
+            } R: #{   format '%4d', swap[:rating]
+            } D: #{   format '%4d', swap[:depth]
+            } P: #{   format '%10s', (format '%5.2f', swap[:ppi])
+            }   F: #{ format format, swap[:members].last[ ohlc ]
+            }   S: #{ swap[:members].first[:datetime].strftime(datetime_format)
+            } - #{    swap[:members].last[:datetime].strftime(datetime_format)
+            }#{"    NOTE: #{notice}" unless notice.nil?}".colorize(swap[:color] || :white )
+      else
+        puts "side: #{swap[:side] }\tlen: #{swap[:length]}  \trating: #{swap[:rating]}".colorize(swap[:color] || :white )
+        puts "diff: #{swap[:ticks]}\tdif: #{swap[:diff].round(7)}\tdepth: #{swap[:depth]}".colorize(swap[:color] || :white )
+        puts "tpi:  #{swap[:tpi]  }\tppi: #{swap[:ppi]}".colorize(swap[:color] || :white )
+        puts "NOTE: #{notice}".colorize(:light_white) unless notice.nil?
+        swap[:members].each {|x| puts member_to_human(x, side: swap[:side], format: format, daily: daily) }
       end
+    end
 
-      def save_swaps(swaps, interval:, swap_type:, contract:, sym: nil)
-        file = get_jsonl_name(interval: interval, swap_type: swap_type, contract: contract, sym: sym)
-        swaps.each do |swap|
-          swap_json = swap.to_json
-          digest = Digest::SHA256.hexdigest swap_json
-          res = `cat #{file} | grep #{digest}`.strip
-          unless res.empty?
-            puts "Cannot save swap, it is already in #{file}:"
-            p swap
-          else
-            swap[:digest] = digest
-            File.open(file, 'a+'){|f| f.write(swap.to_json + "\n") }
-          end
+    # create a standardized name for the cache files
+    # and, on-the-fly, create these files plus their directory
+    def get_jsonl_name(interval:, swap_type:, contract:, sym: nil)
+      raise "Interval #{interval } is not supported, please choose from #{INTERVALS}" unless INTERVALS.include?(interval) || interval.is_a?(Integer)
+      raise "Swaptype #{swap_type} is not supported, please choose from #{SWAPTYPES}" unless SWAPTYPES.include? swap_type
+      sym ||= Cotcube::Helpers.get_id_set(contract: contract)
+      root = '/var/cotcube/level'
+      dir     = "#{root}/#{sym[:id]}"
+      symlink = "#{root}/#{sym[:symbol]}"
+      `mkdir -p #{dir}`         unless File.exist?(dir)
+      `ln -s #{dir} #{symlink}` unless File.exist?(symlink)
+      file = "#{dir}/#{contract}_#{interval.to_s}_#{swap_type.to_s}.jsonl"
+      `touch #{file}`
+      file
+    end
+
+    # the name says it all.
+    # just note the addition of a digest, that serves to check whether same swap has been yet saved
+    # to the cache.
+    #
+    # there are actually 3 types of information, that are saved here:
+    # 1. a swap
+    # 2. an 'empty' information, referring to an interval that has been processed but no swaps were found
+    # 3. an 'exceeded' information, referring to another swap, that has been exceeded
+    #
+    def save_swaps(swaps, interval:, swap_type:, contract:, sym: nil, quiet: false)
+      file = get_jsonl_name(interval: interval, swap_type: swap_type, contract: contract, sym: sym)
+      swaps = [ swaps ] unless swaps.is_a? Array
+      swaps.each do |swap|
+        raise "Illegal swap info: Must contain keys :datetime, :side... #{swap}" unless (%i[ datetime side ] - swap.keys).empty?
+        %i[ interval swap_type ].map {|key| swap.delete(key) }
+        sorted_keys = [ :datetime, :side ] + ( swap.keys - [ :datetime, :side ])
+        swap_json = swap.slice(*sorted_keys).to_json
+        digest = Digest::SHA256.hexdigest swap_json
+        res = `cat #{file} | grep '"digest":"#{digest}"'`.strip
+        unless res.empty?
+          puts "Cannot save swap, it is already in #{file}:".light_red unless quiet
+          p swap unless quiet
+        else
+          swap[:digest] = digest
+          sorted_keys += %i[digest]
+          File.open(file, 'a+'){|f| f.write(swap.slice(*sorted_keys).to_json + "\n") }
         end
       end
+    end
 
-      def load_swaps(interval:, swap_type:, contract:, sym: nil)
-        file = get_jsonl_name(interval: interval, swap_type: swap_type, contract: contract, sym: sym)
-        jsonl = File.read(file)
-        jsonl.
-          each_line.
-          map do |x|
-          JSON.parse(x).
-            deep_transform_keys(&:to_sym).
-            tap do |sw|
+    # loading of swaps is also straight forward
+    # it takes few more efforts to normalize the values to their expected format
+    def load_swaps(interval:, swap_type:, contract:, sym: nil, datetime: nil, recent: false, digest: nil, quiet: false, exceed: false)
+      file = get_jsonl_name(interval: interval, swap_type: swap_type, contract: contract, sym: sym)
+      jsonl = File.read(file)
+      data = jsonl.
+        each_line.
+        map do |x|
+        JSON.parse(x).
+          deep_transform_keys(&:to_sym).
+          tap do |sw|
             sw[:datetime] = DateTime.parse(sw[:datetime]) rescue nil
-            sw[:side]     = sw[:side].to_sym
-            unless sw[:empty]
+            (sw[:exceeded] = DateTime.parse(sw[:exceeded]) rescue nil) if sw[:exceeded]
+            sw[:interval] = interval
+            sw[:swap_type] = swap_type
+            sw[:contract] = contract
+            %i[ side ].each {|key| sw[key] = sw[key].to_sym rescue false }
+            unless sw[:empty] or sw[:exceeded]
               sw[:color]    = sw[:color].to_sym
               sw[:members].map{|mem| mem[:datetime] = DateTime.parse(mem[:datetime]) }
             end
+        end
+      end
+      # assign exceedance data to actual swaps
+      data.select{|swap| swap[:exceeded] }.each do |exc|
+        swap = data.find{|ref| ref[:digest] == exc[:ref]}
+        raise RuntimeError, "Inconsistent history for '#{exc}'. Origin not found." if swap.nil?
+        swap[:exceeded] = exc[:exceeded]
+      end
+      # do not return bare exceedance information
+      data.reject!{|swap| swap[:exceeded] and swap[:members].nil? }
+      # do not return swaps that are found 'later'
+      data.reject!{|swap| swap[:datetime] > datetime } unless datetime.nil?
+      # do not return exceeded swaps, that are exceeded in the past
+      recent  = 7.days  if recent.is_a? TrueClass
+      recent += 5.hours if recent
+      data.reject!{|swap| swap[:exceeded] and swap[:exceeded] < datetime - (recent ? recent : 0) } unless datetime.nil?
+      # remove exceedance information that is found 'later'
+      data.map{|swap| swap.delete(:exceeded) if swap[:exceeded] and swap[:exceeded] > datetime} unless datetime.nil?
+      unless digest.nil?
+        data.select! do |z|
+          (Cotcube::Helpers.sub(minimum: digest.length){ z[:digest] } === digest) and
+          not z[:empty]
+        end
+        case data.size
+        when 0
+          puts "No swaps found for digest '#{digest}'." unless quiet
+        when 1
+          sym ||= Cotcube::Helpers.get_id_set(contract: contract)
+          if not quiet or exceed
+            puts "Found 1 digest: "
+            data.each {|d| puts_swap( d, format: sym[:format], short: true, hash: digest.size + 2) }
+            if exceed
+              exceed = DateTime.now if exceed.is_a? TrueClass
+              mark_exceeded(swap: data.first, datetime: exceed)
+              puts "Swap marked exceeded."
+            end
+          end
+        else
+          sym ||= Cotcube::Helpers.get_id_set(contract: contract)
+          unless quiet
+            puts "Too many digests found for digest '#{digest}', please consider sending more figures: "
+            data.each {|d| puts_swap( d, format: sym[:format], short: true, hash: digest.size + 3)}
           end
         end
       end
-  end
+      data
+    end
+
+    # :swaps is an array of swaps
+    # :zero  is the current interval (ohlc)
+    # :stencil is the according current stencil (eod or intraday)
+    def check_exceedance(swaps:, zero:, stencil:, contract:, sym:, debug: false)
+      swaps.map do |swap|
+        # swaps cannot exceed the day they are found (or if they are found in the future)
+        next if  swap[:datetime] >= zero[:datetime] or swap[:empty]
+        update = stencil.use with: swap, sym: sym, zero: zero
+        if update[:exceeded]
+          to_save = {
+            datetime: zero[:datetime],
+            ref:      swap[:digest],
+            side:     swap[:side],
+            exceeded: update[:exceeded]
+          }
+          save_swaps to_save, interval: swap[:interval], swap_type: swap[:swap_type], contract: contract, sym: sym, quiet: (not debug)
+          swap[:exceeded] = update[:exceeded]
+        end
+        %i[ current_change current_value current_diff current_dist ].map{|key| swap[key] = update[key] }
+        swap
+      end.compact
+    end
 
+    def mark_exceeded(swap:, datetime:, debug: false)
+      to_save = {
+        datetime: datetime,
+        ref:      swap[:digest],
+        side:     swap[:side],
+        exceeded: datetime
+      }
+      save_swaps to_save, interval: swap[:interval], swap_type: swap[:swap_type], sym: Cotcube::Helpers.get_id_set(contract: swap[:contract]), contract: swap[:contract], quiet: (not debug)
+      swap[:exceeded] = datetime
+      swap
+    end
+
+  end
 end
 

data/lib/cotcube-level/intraday_stencil.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+
+module Cotcube
+  module Level
+
+    class Intraday_Stencil
+
+
+      # Class method that loads the (latest) shiftset for given asset
+      # These raw stencils are located in /var/cotcube/level/stencils/shiftsets.csv
+      #
+
+      def self.shiftset(asset:, sym: nil)
+        shiftset_file = '/var/cotcube/level/stencils/shiftsets.csv'
+        headers =  %i[nr tz sod pre mpre rth post mpost rth5 mpost5 symbols]
+        shiftsets = CSV.read(shiftset_file, headers: headers).
+          map{|x| x.to_h}
+        current_set = shiftsets.find{|s| s[:symbols] =~ /#{asset}/ }
+        return current_set.tap{|s| headers.map{|h| s[h] = nil if s[h] == '---------' }; s[:rth5] ||= s[:rth]; s[:mpost5] ||= s[:mpost] }  unless current_set.nil?
+        sym ||= Cotcube::Helpers.get_id_set(symbol: asset)
+        current_set = shiftsets.find{|s| s[:symbols] =~ /#{sym[:type]}/ }
+        return current_set.tap{|s| headers.map{|h| s[h] = nil if s[h] == '---------' }; s[:rth5] ||= s[:rth]; s[:mpost5] ||= s[:mpost] }  unless current_set.nil?
+        raise "Cannot get shiftset for #{sym[:type]}: #{asset}, please prepare #{shiftset_file} before!"
+      end
+
+      attr_reader :base, :shiftset, :timezone, :datetime, :zero, :index
+
+
+
+      def initialize(
+        asset:,
+        interval: 30.minutes,
+        swap_type: :full,
+        datetime: nil,
+        debug: false,
+        weeks: 6,
+        future: 2,
+        version: nil,               # when referring to a specicic version of the stencil
+        stencil: nil,               # instead of preparing, use this one if set
+        warnings: true              # be more quiet
+      )
+        @shiftset = Intraday_Stencil.shiftset(asset: asset)
+        @timezone  = Cotcube::Level::TIMEZONES[@shiftset[:tz]]
+        @debug     = debug
+        @interval  = interval
+        @swap_type = swap_type
+        @warnings = warnings
+        datetime ||= DateTime.now
+        datetime  = @timezone.at(datetime.to_i) unless datetime.is_a? ActiveSupport::TimeWithZone
+        @datetime = datetime.beginning_of_day
+        @datetime += interval while @datetime <= datetime - interval
+        @datetime -= interval
+
+        const = "RAW_INTRA_STENCIL_#{@shiftset[:nr]}_#{interval.in_minutes.to_i}".to_sym
+        if Object.const_defined? const
+          @base = (Object.const_get const).map{|z| z.dup}
+        else
+          start_time    = lambda {|x| @shiftset[x].split('-').first rescue '' }
+          start_hours   = lambda {|x| @shiftset[x].split('-').first[ 0.. 1].to_i.send(:hours)   rescue 0 }
+          start_minutes = lambda {|x| @shiftset[x].split('-').first[-2..-1].to_i.send(:minutes) rescue 0 }
+          end_time      = lambda {|x| @shiftset[x].split('-').last  rescue '' }
+          end_hours     = lambda {|x| @shiftset[x].split('-').last [ 0.. 1].to_i.send(:hours)   rescue 0 }
+          end_minutes   = lambda {|x| @shiftset[x].split('-').last [-2..-1].to_i.send(:minutes) rescue 0 }
+
+         runner = (@datetime -
+                    weeks * 7.days).beginning_of_week(:sunday)
+          tm_runner = lambda { runner.strftime('%H%M') }
+          @base = []
+          (weeks+future).times do
+            while tm_runner.call < GLOBAL_SOW[@shiftset[:tz]].split('-').last
+              # if daylight is switched, this phase will be shorter or longer
+              @base << { datetime: runner, type: :sow }
+              runner += interval
+            end
+            end_of_week = runner + 6.days + 7.hours
+
+            5.times do |i|
+              # TODO: mark holidays as such
+              [:sod, :pre, :mpre, (i<4 ? :rth : :rth5), :post, (i<4 ? :mpost : :mpost5)].each do |phase|
+                yet_rth = false
+                unless start_time.call(phase).empty?
+                  eophase = end_time.call(phase)
+                  sophase = start_time.call(phase)
+                  phase = :rth   if phase == :rth5
+                  phase = :mpost if phase == :mpost5
+                  if %i[ pre rth ].include? phase and tm_runner.call > sophase
+                    # fix previous interval
+                    @base.last[:type] = phase
+                    if phase == :rth and not yet_rth
+                      @base.last[:block] = true
+                      yet_rth = true
+                    end
+                  end
+                  while ((sophase > eophase) ? (tm_runner.call >= sophase or tm_runner.call < eophase) : (tm_runner.call < eophase))
+                    current = { datetime: runner, type: phase }
+                    if phase == :rth and not yet_rth
+                      current[:block] = true
+                      yet_rth = true
+                    end
+                    @base << current
+                    runner += interval
+                  end
+                end
+              end
+              while tm_runner.call < GLOBAL_EOD[@shiftset[:tz]].split('-').last
+                @base << { datetime: runner, type: :eod }
+                runner += interval
+              end
+            end # 5.times
+            while runner < end_of_week
+              @base << { datetime: runner, type: :eow }
+              runner += interval
+            end
+          end
+          Object.const_set(const, @base.map{|z| z.dup})
+        end
+
+        case swap_type
+        when :full
+          @base.select!{|x| %i[ pre rth post ].include?(x[:type]) }
+        when :rth
+          @base.select!{|x| x[:type] == :rth  }
+          # to.map{    |x| [:high, :low, :volume].map{|z| x[z] = nil} if x[:block] }
+        when :flow
+          @base.reject!{|x| %i[ meow postmm postmm5 ].include?(x[:type]) }
+          @base.
+            map{ |x|
+            [:high, :low, :volume].map{|z| x[z] = nil} unless x[:type] == :rth
+            # [:high, :low, :volume].map{|z| x[z] = nil} if x[:block]
+          }
+        when :run
+          @base.select!{|x| %i[ premarket rth postmarket ].include? x[:type]}
+        else
+          raise ArgumentError, "Unknown stencil/swap type '#{type}'"
+        end
+        @base.map!{|z| z.dup}
+
+        @index = @base.index{|x| x[:datetime] == @datetime }
+        @index -= 1 while %i[sow sod mpre mpost eod eow].include? @base[@index][:type]
+        @datetime = @base[@index][:datetime]
+        @zero  = @base[@index]
+        counter = 0
+        while @base[@index - counter] and @index - counter >= 0
+          @base[@index - counter][:x] = counter
+          counter += 1
+        end
+        counter = 0
+        while @base[@index + counter] and @index + counter < @base.length
+          @base[@index + counter][:x] = -counter
+          counter += 1
+        end
+      end
+
+=begin
+      def dup
+        Intraday_Stencil.new(
+          debug:      @debug,
+          interval:   @interval,
+          swap_type:  @swap_type,
+          datetime:   @datetime,
+          stencil:    @base.map{|x| x.dup}
+        )
+      end
+=end
+
+      def zero
+        @zero ||=  @base.find{|b| b[:x].zero? }
+      end
+
+      def apply(to: )
+        offset = 0
+        @base.each_index do |i|
+          begin
+            offset += 1 while to[i+offset][:datetime] < @base[i][:datetime]
+          rescue
+            # appending
+            to << @base[i]
+            next
+          end
+          if to[i+offset][:datetime] > @base[i][:datetime]
+            # skipping
+            offset -= 1
+            next
+          end
+          # merging
+          to[i+offset][:x] = @base[i][:x]
+          to[i+offset][:type] = @base[i][:type]
+        end
+        # finally remove all bars that do not belong to the stencil (i.e. holidays)
+        to.reject!{|x| x[:x].nil? }
+      end
+
+      def use(with:, sym:, zero:, grace: -2)
+        # todo: validate with (check if vslid swap
+        #                sym  (check keys)
+        #                zero (ohlc with x.zero?)
+        #                side ( upper or lower)
+        swap  = with.dup
+        high  = swap[:side] == :upper
+        ohlc  = high ? :high : :low
+        start = base.find{|x| swap[:datetime] == x[:datetime]}
+        swap[:current_change] = (swap[:tpi] * start[:x]).round(8)
+        swap[:current_value]  =  swap[:members].last[ ohlc ] + swap[:current_change] * sym[:ticksize]
+        swap[:current_diff]   = (swap[:current_value] - zero[ohlc]) * (high ? 1 : -1 )
+        swap[:current_dist]   = (swap[:current_diff] / sym[:ticksize]).to_i
+        swap[:exceeded]       =  zero[:datetime] if swap[:current_dist] < grace
+        swap
+      end
+    end
+
+  end
+
+end