sqa 0.0.14 → 0.0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41df647860d7465185b8f069d1fb51edcb540d4114cfd56c898e3373426e0bb8
-  data.tar.gz: 130b1023a4530a645ea90bf323c16dd4fb22a887222e073209b8edbfeeb6731f
+  metadata.gz: 911775606b7e0fa046261c5a9bc4d3be21ca9caf38c37011141f1393ac6e5063
+  data.tar.gz: d70ae996a39dbe7c386750286cb4cc437681d584945b31104e91b06c30b1600f
 SHA512:
-  metadata.gz: b28c7bf4231e3ced4046e83f5afd256a0dc155500493b8d2c3cee43695e8ef203534a0642dc4028830f3d068653d6a9016a785b3343611a18209cb1cfb6f4694
-  data.tar.gz: 20ee19fe13781e473953ed4fcae73ce8e687ce1808fc74fa02d834364b7b02df14c7dab4cc0390efbae0f48328d71f27aab96318fad28052588d7b623abf4fe3
+  metadata.gz: '0074688f69947d20d5aae7c81b090a91fb4f06862084b2349b248ecdf2e17376f47d5a1acffbfb7b2700865cf25f5ff5ba054514d00ed88750f46224b4c9360e'
+  data.tar.gz: 38d1ebf511e9dfa2b87084ed77093e2fc03ccf6eeb6a00e42d1060acca82e135d0b4c0d36a25d9f9bc4a2dd9adf91749a6640aab14b304e33f24fee0c3eb7e15

data/README.md

@@ -1,3 +1,7 @@
+**Replacing Daru** with Hashie::Mash
+
+This is branch hashie_df
+
 # SQA - Simple Qualitative Analysis
 
 This is a very simplistic set of tools for running technical analysis on a stock portfolio.  Simplistic means it is not reliable nor intended for any kind of financial use.  Think of it as a training tool.  I do.  Its helping me understand why I need professional help from people who know what they are doing.

data/checksums/sqa-0.0.15.gem.sha512

@@ -0,0 +1 @@
+d4f3ab1bf26de034f0f044a5cab9d86e61221fc2d160056c1c24c166586e7ce72d90095ebea01965f5cabf989c4116e7409f8fc6749cabe7d5a13e34f87f4b96

data/checksums/sqa-0.0.17.gem.sha512

@@ -0,0 +1 @@
+2ee94a54d6ac3d13685dc9b91a2bae0fe75feab6148e1aa9a9d4096961b9b7b577b7ce9d1264f0cce260640515ddd86d5fd5fd2b66f49175844c903581ff6fd9

data/docs/alpha_vantage_technical_indicators.md

@@ -0,0 +1,62 @@
+# Alpha Vantage
+## Technical Indicators
+
+The following technical indicators are available from Alpha Vantage
+using a free API key.
+
+| Acronym   | Indicator Description                                    |
+|-----------|----------------------------------------------------------|
+| AD        | Accumulation/Distribution                               |
+| ADOSC     | Accumulation/Distribution Oscillator                    |
+| ADX       | Average Directional Index                               |
+| ADXR      | Average Directional Movement Rating                     |
+| APO       | Absolute Price Oscillator                              |
+| AROON     | Aroon Indicator                                        |
+| AROONOSC  | Aroon Oscillator                                       |
+| ATR       | Average True Range                                     |
+| BBANDS    | Bollinger Bands                                        |
+| BOP       | Balance of Power                                       |
+| CCI       | Commodity Channel Index                                |
+| CMO       | Chande Momentum Oscillator                             |
+| DEMA      | Double Exponential Moving Average                      |
+| DX        | Directional Movement Index                             |
+| EMA       | Exponential Moving Average                             |
+| HT_DCPERIOD | Hilbert Transform - Dominant Cycle Period             |
+| HT_DCPHASE | Hilbert Transform - Dominant Cycle Phase              |
+| HT_PHASOR  | Hilbert Transform - Phasor Components                  |
+| HT_SINE    | Hilbert Transform - SineWave                           |
+| HT_TRENDLINE | Hilbert Transform - Instantaneous Trendline           |
+| HT_TRENDMODE | Hilbert Transform - Trend vs Cycle Mode               |
+| KAMA       | Kaufman Adaptive Moving Average                        |
+| MACD       | Moving Average Convergence Divergence                  |
+| MACDEXT    | MACD with controllable MA type                         |
+| MAMA       | MESA Adaptive Moving Average                           |
+| MFI        | Money Flow Index                                      |
+| MIDPOINT   | MidPoint over period                                   |
+| MIDPRICE   | Midpoint Price over period                             |
+| MINUS_DI   | Minus Directional Indicator                            |
+| MINUS_DM   | Minus Directional Movement                             |
+| MOM        | Momentum                                              |
+| NATR       | Normalized Average True Range                          |
+| OBV        | On Balance Volume                                     |
+| PLUS_DI    | Plus Directional Indicator                             |
+| PLUS_DM    | Plus Directional Movement                              |
+| PPO        | Percentage Price Oscillator                            |
+| ROC        | Rate of Change                                        |
+| ROCR       | Rate of Change Ratio                                  |
+| RSI        | Relative Strength Index                               |
+| SAR        | Parabolic SAR                                         |
+| SMA        | Simple Moving Average                                  |
+| STOCH      | Stochastic Oscillator                                 |
+| STOCHF     | Stochastic Fast                                       |
+| STOCHRSI   | Stochastic Relative Strength Index                     |
+| T3         | Triple Exponential Moving Average (T3)                |
+| TEMA       | Triple Exponential Moving Average                      |
+| TRANGE     | True Range                                            |
+| TRIMA      | Triangular Moving Average                              |
+| TRIX       | 1-day Rate of Change of a Triple Smooth EMA            |
+| ULTOSC     | Ultimate Oscillator                                   |
+| VWAP       | Volume Weighted Average Price                          |
+| WILLR      | Williams' %R                                          |
+| WMA        | Weighted Moving Average                                |
+

data/lib/sqa/cli.rb

@@ -1,6 +1,5 @@
 # lib/sqa/cli.rb
 
-require 'tty-option'
 
 require_relative '../sqa'
 

data/lib/sqa/config.rb

@@ -6,17 +6,18 @@
 #   config file ..... overrides envar
 #   command line parameters ...... overrides config file
 
-require 'hashie'
-require 'yaml'
-require 'json'
-require 'toml-rb'
-
 
 module SQA
 	class Config < Hashie::Dash
     include Hashie::Extensions::Dash::PropertyTranslation
     include Hashie::Extensions::Coercion
-    include Hashie::Extensions::Dash::PredefinedValues
+
+    # FIXME:  Getting undefined error PredefinedValues
+    #         I'm thinking that Ruby is dropping it from the ObjectSpace
+    #         Looks like it is only used for the log level.  Should
+    #         able to work around that.
+    #
+    # include Hashie::Extensions::Dash::PredefinedValues
 
     property :config_file   #,a String filepath for the current config overriden by cli options
     property :dump_config   # a String filepath into which to dump the current config

data/lib/sqa/data_frame/alpha_vantage.rb

@@ -4,12 +4,9 @@
 # Using the Alpha Vantage JSON interface
 #
 
-require 'faraday'
-require 'json'
 
-class SQA::DataFrame < Daru::DataFrame
+class SQA::DataFrame
   class AlphaVantage
-    API_KEY     = Nenv.av_api_key
     CONNECTION  = Faraday.new(url: 'https://www.alphavantage.co')
     HEADERS     = YahooFinance::HEADERS
 
@@ -26,47 +23,16 @@ class SQA::DataFrame < Daru::DataFrame
       "volume"          => HEADERS[6]
     }
 
+    TRANSFORMERS  = {
+      HEADERS[1] => -> (v) { v.to_f.round(3) },
+      HEADERS[2] => -> (v) { v.to_f.round(3) },
+      HEADERS[3] => -> (v) { v.to_f.round(3) },
+      HEADERS[4] => -> (v) { v.to_f.round(3) },
+      HEADERS[5] => -> (v) { v.to_f.round(3) },
+      HEADERS[6] => -> (v) { v.to_i }
+    }
 
     ################################################################
-    # Load a Dataframe from a csv file
-    def self.load(ticker, type="csv")
-      filepath = SQA.data_dir + "#{ticker}.#{type}"
-
-      if filepath.exist?
-        df = normalize_vector_names SQA::DataFrame.load(ticker, type)
-      else
-        df = recent(ticker, full: true)
-        df.send("to_#{type}",filepath)
-      end
-
-      df
-    end
-
-
-    # Normalize the vector (aka column) names as
-    # symbols using the standard names set by
-    # Yahoo Finance ... since it was the first one
-    # not because its anything special.
-    #
-    def self.normalize_vector_names(df)
-      headers = df.vectors.to_a
-
-      # convert vector names to symbols
-      # when they are strings.  They become stings
-      # when the data frame is saved to a CSV file
-      # and then loaded back in.
-
-      if headers.first == HEADERS.first.to_s
-        a_hash = {}
-        HEADERS.each {|k| a_hash[k.to_s] = k}
-        df.rename_vectors(a_hash) # renames from String to Symbol
-      else
-        df.rename_vectors(HEADER_MAPPING)
-      end
-
-      df
-    end
-
 
     # Get recent data from JSON API
     #
@@ -84,7 +50,8 @@ class SQA::DataFrame < Daru::DataFrame
     #       and adding that to the data frame as if it were
     #       adjusted.
     #
-    def self.recent(ticker, full: false)
+    def self.recent(ticker, full: false, from_date: nil)
+
       # NOTE: Using the CSV format because the JSON format has
       #       really silly key values.  The column names for the
       #       CSV format are much better.
@@ -92,7 +59,7 @@ class SQA::DataFrame < Daru::DataFrame
         "/query?" +
         "function=TIME_SERIES_DAILY&" +
         "symbol=#{ticker.upcase}&" +
-        "apikey=#{API_KEY}&" +
+        "apikey=#{SQA.av.key}&" +
         "datatype=csv&" +
         "outputsize=#{full ? 'full' : 'compact'}"
       ).to_hash
@@ -102,18 +69,19 @@ class SQA::DataFrame < Daru::DataFrame
       end
 
       raw           = response[:body].split
-
       headers       = raw.shift.split(',')
+
       headers[0]    = 'date'  # website returns "timestamp" but that
                               # has an unintended side-effect when
                               # the names are normalized.
+                              # SMELL: IS THIS STILL TRUE?
 
       close_inx     = headers.size - 2
       adj_close_inx = close_inx + 1
 
       headers.insert(adj_close_inx, 'adjusted_close')
 
-      data    = raw.map do |e|
+      aofh    = raw.map do |e|
                   e2 = e.split(',')
                   e2[1..-2] = e2[1..-2].map(&:to_f) # converting open, high, low, close
                   e2[-1]    = e2[-1].to_i           # converting volumn
@@ -121,35 +89,20 @@ class SQA::DataFrame < Daru::DataFrame
                   headers.zip(e2).to_h
                 end
 
-      # What oldest data first in the data frame
-      normalize_vector_names Daru::DataFrame.new(data.reverse)
-    end
-
+      if from_date
+        aofh.reject!{|e| Date.parse(e['date']) < from_date}
+      end
 
-    # Append update_df rows to the base_df
-    #
-    # base_df is ascending on timestamp
-    # update_df is descending on timestamp
-    #
-    # base_df content came from CSV file downloaded
-    # from Yahoo Finance.
-    #
-    # update_df came from scraping the webpage
-    # at Yahoo Finance for the recent history.
-    #
-    # Returns a combined DataFrame.
-    #
-    def self.append(base_df, updates_df)
-      last_timestamp  = Date.parse base_df.timestamp.last
-      filtered_df     = updates_df.filter_rows { |row| Date.parse(row[:timestamp]) > last_timestamp }
+      return nil if aofh.empty?
 
-      last_inx = filtered_df.size - 1
+      # ensure tha the data frame is
+      # always sorted oldest to newest.
 
-      (0..last_inx).each do |x|
-        base_df.add_row filtered_df.row[last_inx-x]
+      if aofh.first['date'] > aofh.last['date']
+        aofh.reverse!
       end
 
-      base_df
+      SQA::DataFrame.from_aofh(aofh, mapping: HEADER_MAPPING, transformers: TRANSFORMERS)
     end
   end
 end

data/lib/sqa/data_frame/yahoo_finance.rb

@@ -1,10 +1,8 @@
 # lib/sqa/data_frame/yahoo_finance.rb
 # frozen_string_literal: true
 
-require 'faraday'
-require 'nokogiri'
 
-class SQA::DataFrame < Daru::DataFrame
+class SQA::DataFrame
   class YahooFinance
     CONNECTION  = Faraday.new(url: 'https://finance.yahoo.com')
     HEADERS     = [
@@ -32,21 +30,6 @@ class SQA::DataFrame < Daru::DataFrame
       }
 
     ################################################################
-    def self.load(filename, options={}, &block)
-      df = SQA::DataFrame.load(filename, options={}, &block)
-
-      headers = df.vectors
-
-      if headers.first == HEADERS.first.to_s
-        a_hash = {}
-        HEADERS.each {|k| a_hash[k.to_s] = k}
-        df.rename_vectors(a_hash)
-      else
-        df.rename_vectors(HEADER_MAPPING)
-      end
-
-      df
-    end
 
 
     # Scrape the Yahoo Finance website to get recent
@@ -64,7 +47,7 @@ class SQA::DataFrame < Daru::DataFrame
 
       rows      = table.css('tbody tr')
 
-      data = []
+      aofh = []
 
       rows.each do |row|
         cols = row.css('td').map{|c| c.children[0].text}
@@ -82,37 +65,10 @@ class SQA::DataFrame < Daru::DataFrame
         cols[0] = Date.parse(cols[0]).to_s
         cols[6] = cols[6].tr(',','').to_i
         (1..5).each {|x| cols[x] = cols[x].to_f}
-        data << HEADERS.zip(cols).to_h
-      end
-
-      Daru::DataFrame.new(data)
-    end
-
-
-    # Append update_df rows to the base_df
-    #
-    # base_df is ascending on timestamp
-    # update_df is descending on timestamp
-    #
-    # base_df content came from CSV file downloaded
-    # from Yahoo Finance.
-    #
-    # update_df came from scraping the webpage
-    # at Yahoo Finance for the recent history.
-    #
-    # Returns a combined DataFrame.
-    #
-    def self.append(base_df, updates_df)
-      last_timestamp  = Date.parse base_df.timestamp.last
-      filtered_df     = updates_df.filter_rows { |row| Date.parse(row[:timestamp]) > last_timestamp }
-
-      last_inx = filtered_df.size - 1
-
-      (0..last_inx).each do |x|
-        base_df.add_row filtered_df.row[last_inx-x]
+        aofh << HEADERS.zip(cols).to_h
       end
 
-      base_df
+      aofh
     end
   end
 end

data/lib/sqa/data_frame.rb

@@ -1,52 +1,302 @@
 # lib/sqa/data_frame.rb
 # frozen_string_literal: true
 
-require_relative  'data_frame/yahoo_finance'
-require_relative  'data_frame/alpha_vantage'
+require 'forwardable'
 
-class Daru::DataFrame
+require_relative 'data_frame/yahoo_finance'
+require_relative 'data_frame/alpha_vantage'
 
-  def to_csv(path_to_file, opts={})
-    options = {
-      headers:    true,
-      converters: :numeric
-    }.merge(opts)
+class SQA::DataFrame
+  class Data < Hashie::Mash
+    # SNELL: Are all of these needed?
+    include Hashie::Extensions::Mash::KeepOriginalKeys
+    # include Hashie::Extensions::Mash::PermissiveRespondTo
+    include Hashie::Extensions::Mash::SafeAssignment
+    include Hashie::Extensions::Mash::SymbolizeKeys
+    # include Hashie::Extensions::Mash::DefineAccessors
+  end
+
+  extend Forwardable
 
-    writer = ::CSV.open(path_to_file, 'wb')
+  # @data is of class Data
+  attr_accessor :data
+
+  # Expects a Hash of Arrays (hofa)
+  def initialize(a_hash={})
+    @data = Data.new(a_hash)
+  end
 
-    writer << vectors.to_a if options[:headers]
 
-    each_row do |row|
-      writer << if options[:convert_comma]
-                  row.map { |v| v.to_s.tr('.', ',') }
-                else
-                  row.to_a
-                end
+  def to_csv(path_to_file)
+    CSV.open(path_to_file, 'w') do |csv|
+      csv << keys
+      size.times do |x|
+        csv << row(x)
+      end
     end
+  end
+
 
-    writer.close
+  def to_json(path_to_file)
+    NotImplemented.raise
   end
-end
 
 
+  def to_aofh
+    NotImplemented.raise
+  end
 
 
-class SQA::DataFrame < Daru::DataFrame
+  def_delegator :@data, :to_h, :to_hofa
+  alias_method :to_h, :to_hofa
+
+
+  # The number of data rows
+  def size
+    data[@data.keys[0]].size
+  end
+  alias_method :nrows,  :size
+  alias_method :length, :size
+
+
+  def_delegator :@data,  :keys
+  alias_method :vectors, :keys
+  alias_method :columns, :keys
+
+
+  def ncols
+    keys.size
+  end
+
+
+  def_delegator :@data, :values, :values
+  def_delegator :@data, :[], :[]
+  def_delegator :@data, :[]=, :[]=
+
+
+  def rows
+    result = []
+    (0..size - 1).each do |x|
+      entry = row(x)
+      result << entry
+    end
+    result
+  end
+  alias_method :to_a, :rows
+
+
+  def row(x)
+    if x.is_a?(Integer)
+      raise BadParameterError if x < 0 || x >= size
+
+    elsif x.is_a?(Hash)
+      raise BadParameterError, "x is #{x}" if x.size > 1
+      key = x.keys[0]
+      x   = @data[key].index(x[key])
+      raise BadParameterError, 'Not Found #{x}' if x.nil?
+      return keys.zip(row(x)).to_h
 
-  #################################################
-  def self.load(ticker, type="csv", options={}, &block)
-    source  = SQA.data_dir + "#{ticker}.#{type}"
-
-    if :csv == type
-     from_csv(source, options={}, &block)
-    elsif :json == type
-      from_json(source, options={}, &block)
-    elsif %i[txt dat].include?(type)
-      from_plaintext(source, options={}, &block)
-    elsif :xls == type
-      from_excel(source, options={}, &block)
     else
-      raise SQA::BadParameterError, "un-supported file type: #{type}"
+      raise BadParameterError, "Unknown x.class: #{x.class}"
+    end
+
+    entry = []
+
+    keys.each do |key|
+      entry << @data[key][x]
+    end
+
+    entry
+  end
+
+
+  def append(new_df)
+    raise(BadParameterError, "Key mismatch") if keys != new_df.keys
+
+    keys.each do |key|
+      @data[key] += new_df[key]
+    end
+  end
+  alias_method :concat, :append
+
+
+  # Creates a new instance with new keys
+  # based on the mapping hash where
+  #   { old_key => new_key }
+  #
+  def rename(mapping)
+    SQA::DataFrame.new(
+      self.class.rename(
+        mapping,
+        @data.to_h
+      )
+    )
+  end
+  alias_method :rename_vectors, :rename
+
+
+  # Map the values of the vectors into different objects
+  # types is a Hash where the key is the vector name and
+  #   the value is a proc
+  #
+  # For Example:
+  # {
+  #   price: -> (v) {v.to_f.round(3)}
+  # }
+  #
+  def coerce_vectors(transformers)
+    transformers.each_pair do |key, transformer|
+      @data[key].map!{|v| transformer.call(v)}
+    end
+  end
+
+
+  def method_missing(method_name, *args, &block)
+    if @data.respond_to?(method_name)
+      self.class.send(:define_method, method_name) do |*method_args, &method_block|
+        @data.send(method_name, *method_args, &method_block)
+      end
+      send(method_name, *args, &block)
+    else
+      super
+    end
+  end
+
+
+  def respond_to_missing?(method_name, include_private = false)
+    @data.respond_to?(method_name) || super
+  end
+
+  #################################################
+  class << self
+
+    def append(base_df, other_df)
+      base_df.append(other_df)
+    end
+
+
+    # TODO: The Data class has its own load which also supports
+    #       YAML by default.  Maybe this method should
+    #       make use of @data = Data.load(source)
+    #
+    def load(source:, mapping: {}, transformers:{})
+      file_type = source.extname[1..].downcase.to_sym
+
+      df  = if :csv == file_type
+              from_csv_file(source, mapping: mapping, transformers: transformers)
+            elsif :json == file_type
+              from_json_file(source, mapping: mapping, transformers: transformers)
+            else
+              raise BadParameterError, "unsupported file type: #{file_type}"
+            end
+
+      unless transformers.empty?
+        df.coerce_vectors(transformers)
+      end
+
+      df
+    end
+
+
+    def from_aofh(aofh, mapping: {}, transformers: {})
+      new(
+        aofh_to_hofa(
+          aofh,
+          mapping: mapping,
+          transformers: transformers
+        )
+      )
+    end
+
+
+    def from_csv_file(source, mapping: {}, transformers: {})
+      aofh = []
+
+      CSV.foreach(source, headers: true) do |row|
+        aofh << row.to_h
+      end
+
+      from_aofh(aofh, mapping: mapping, transformers: transformers)
+    end
+
+
+    def from_json_file(source, mapping: {}, transformers: {})
+      aofh = JSON.parse(source.read)
+
+      from_aofh(aofh, mapping: mapping, transformers: transformers)
+    end
+
+
+    # aofh -- Array of Hashes
+    # hofa -- Hash of Arrays
+    def aofh_to_hofa(aofh, mapping: {}, transformers: {})
+      hofa = {}
+      keys = aofh.first.keys
+
+      keys.each do |key|
+        hofa[key] = []
+      end
+
+      aofh.each do |entry|
+        keys.each do |key|
+          hofa[key] << entry[key]
+        end
+      end
+
+      # SMELL: This might be necessary
+      normalize_keys(hofa, adapter_mapping: mapping)
+    end
+
+
+    def normalize_keys(hofa, adapter_mapping: {})
+      hofa    = rename(adapter_mapping, hofa)
+      mapping = generate_mapping(hofa.keys)
+      rename(mapping, hofa)
+    end
+
+
+    def rename(mapping, hofa)
+      mapping.each_pair do |old_key, new_key|
+        hofa[new_key] = hofa.delete(old_key)
+      end
+
+      hofa
+    end
+
+
+    def generate_mapping(keys)
+      mapping = {}
+
+      keys.each do |key|
+        mapping[key] = underscore_key(sanitize_key(key)) unless key.is_a?(Symbol)
+      end
+
+      mapping
+    end
+
+
+    # returns a snake_case Symbol
+    def underscore_key(key)
+      key.to_s.gsub(/::/, '/').
+      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z\d])([A-Z])/,'\1_\2').
+      tr("-", "_").
+      downcase.to_sym
+    end
+
+
+    # removes punctuation and specal characters,
+    # replaces space with underscore.
+    def sanitize_key(key)
+      key.tr('.():/','').gsub(/^\d+.?\s/, "").tr(' ','_')
+    end
+
+
+    # returns true if key is in a date format
+    # like 2023-06-03
+    def is_date?(key)
+      !/(\d{4}-\d{2}-\d{2})/.match(key.to_s).nil?
     end
   end
 end
+