sqa 0.0.15 → 0.0.18

data/lib/sqa/data_frame.rb

@@ -1,52 +1,302 @@
 # lib/sqa/data_frame.rb
 # frozen_string_literal: true
 
-require_relative  'data_frame/yahoo_finance'
-require_relative  'data_frame/alpha_vantage'
+require 'forwardable'
 
-class Daru::DataFrame
+require_relative 'data_frame/yahoo_finance'
+require_relative 'data_frame/alpha_vantage'
 
-  def to_csv(path_to_file, opts={})
-    options = {
-      headers:    true,
-      converters: :numeric
-    }.merge(opts)
+class SQA::DataFrame
+  class Data < Hashie::Mash
+    # SNELL: Are all of these needed?
+    include Hashie::Extensions::Mash::KeepOriginalKeys
+    # include Hashie::Extensions::Mash::PermissiveRespondTo
+    include Hashie::Extensions::Mash::SafeAssignment
+    include Hashie::Extensions::Mash::SymbolizeKeys
+    # include Hashie::Extensions::Mash::DefineAccessors
+  end
+
+  extend Forwardable
 
-    writer = ::CSV.open(path_to_file, 'wb')
+  # @data is of class Data
+  attr_accessor :data
+
+  # Expects a Hash of Arrays (hofa)
+  def initialize(a_hash={})
+    @data = Data.new(a_hash)
+  end
 
-    writer << vectors.to_a if options[:headers]
 
-    each_row do |row|
-      writer << if options[:convert_comma]
-                  row.map { |v| v.to_s.tr('.', ',') }
-                else
-                  row.to_a
-                end
+  def to_csv(path_to_file)
+    CSV.open(path_to_file, 'w') do |csv|
+      csv << keys
+      size.times do |x|
+        csv << row(x)
+      end
     end
+  end
+
 
-    writer.close
+  def to_json(path_to_file)
+    NotImplemented.raise
   end
-end
 
 
+  def to_aofh
+    NotImplemented.raise
+  end
 
 
-class SQA::DataFrame < Daru::DataFrame
+  def_delegator :@data, :to_h, :to_hofa
+  alias_method :to_h, :to_hofa
+
+
+  # The number of data rows
+  def size
+    data[@data.keys[0]].size
+  end
+  alias_method :nrows,  :size
+  alias_method :length, :size
+
+
+  def_delegator :@data,  :keys
+  alias_method :vectors, :keys
+  alias_method :columns, :keys
+
+
+  def ncols
+    keys.size
+  end
+
+
+  def_delegator :@data, :values, :values
+  def_delegator :@data, :[], :[]
+  def_delegator :@data, :[]=, :[]=
+
+
+  def rows
+    result = []
+    (0..size - 1).each do |x|
+      entry = row(x)
+      result << entry
+    end
+    result
+  end
+  alias_method :to_a, :rows
+
+
+  def row(x)
+    if x.is_a?(Integer)
+      raise BadParameterError if x < 0 || x >= size
+
+    elsif x.is_a?(Hash)
+      raise BadParameterError, "x is #{x}" if x.size > 1
+      key = x.keys[0]
+      x   = @data[key].index(x[key])
+      raise BadParameterError, 'Not Found #{x}' if x.nil?
+      return keys.zip(row(x)).to_h
 
-  #################################################
-  def self.load(ticker, type="csv", options={}, &block)
-    source  = SQA.data_dir + "#{ticker}.#{type}"
-
-    if :csv == type
-     from_csv(source, options={}, &block)
-    elsif :json == type
-      from_json(source, options={}, &block)
-    elsif %i[txt dat].include?(type)
-      from_plaintext(source, options={}, &block)
-    elsif :xls == type
-      from_excel(source, options={}, &block)
     else
-      raise SQA::BadParameterError, "un-supported file type: #{type}"
+      raise BadParameterError, "Unknown x.class: #{x.class}"
+    end
+
+    entry = []
+
+    keys.each do |key|
+      entry << @data[key][x]
+    end
+
+    entry
+  end
+
+
+  def append(new_df)
+    raise(BadParameterError, "Key mismatch") if keys != new_df.keys
+
+    keys.each do |key|
+      @data[key] += new_df[key]
+    end
+  end
+  alias_method :concat, :append
+
+
+  # Creates a new instance with new keys
+  # based on the mapping hash where
+  #   { old_key => new_key }
+  #
+  def rename(mapping)
+    SQA::DataFrame.new(
+      self.class.rename(
+        mapping,
+        @data.to_h
+      )
+    )
+  end
+  alias_method :rename_vectors, :rename
+
+
+  # Map the values of the vectors into different objects
+  # types is a Hash where the key is the vector name and
+  #   the value is a proc
+  #
+  # For Example:
+  # {
+  #   price: -> (v) {v.to_f.round(3)}
+  # }
+  #
+  def coerce_vectors(transformers)
+    transformers.each_pair do |key, transformer|
+      @data[key].map!{|v| transformer.call(v)}
+    end
+  end
+
+
+  def method_missing(method_name, *args, &block)
+    if @data.respond_to?(method_name)
+      self.class.send(:define_method, method_name) do |*method_args, &method_block|
+        @data.send(method_name, *method_args, &method_block)
+      end
+      send(method_name, *args, &block)
+    else
+      super
+    end
+  end
+
+
+  def respond_to_missing?(method_name, include_private = false)
+    @data.respond_to?(method_name) || super
+  end
+
+  #################################################
+  class << self
+
+    def append(base_df, other_df)
+      base_df.append(other_df)
+    end
+
+
+    # TODO: The Data class has its own load which also supports
+    #       YAML by default.  Maybe this method should
+    #       make use of @data = Data.load(source)
+    #
+    def load(source:, mapping: {}, transformers:{})
+      file_type = source.extname[1..].downcase.to_sym
+
+      df  = if :csv == file_type
+              from_csv_file(source, mapping: mapping, transformers: transformers)
+            elsif :json == file_type
+              from_json_file(source, mapping: mapping, transformers: transformers)
+            else
+              raise BadParameterError, "unsupported file type: #{file_type}"
+            end
+
+      unless transformers.empty?
+        df.coerce_vectors(transformers)
+      end
+
+      df
+    end
+
+
+    def from_aofh(aofh, mapping: {}, transformers: {})
+      new(
+        aofh_to_hofa(
+          aofh,
+          mapping: mapping,
+          transformers: transformers
+        )
+      )
+    end
+
+
+    def from_csv_file(source, mapping: {}, transformers: {})
+      aofh = []
+
+      CSV.foreach(source, headers: true) do |row|
+        aofh << row.to_h
+      end
+
+      from_aofh(aofh, mapping: mapping, transformers: transformers)
+    end
+
+
+    def from_json_file(source, mapping: {}, transformers: {})
+      aofh = JSON.parse(source.read)
+
+      from_aofh(aofh, mapping: mapping, transformers: transformers)
+    end
+
+
+    # aofh -- Array of Hashes
+    # hofa -- Hash of Arrays
+    def aofh_to_hofa(aofh, mapping: {}, transformers: {})
+      hofa = {}
+      keys = aofh.first.keys
+
+      keys.each do |key|
+        hofa[key] = []
+      end
+
+      aofh.each do |entry|
+        keys.each do |key|
+          hofa[key] << entry[key]
+        end
+      end
+
+      # SMELL: This might be necessary
+      normalize_keys(hofa, adapter_mapping: mapping)
+    end
+
+
+    def normalize_keys(hofa, adapter_mapping: {})
+      hofa    = rename(adapter_mapping, hofa)
+      mapping = generate_mapping(hofa.keys)
+      rename(mapping, hofa)
+    end
+
+
+    def rename(mapping, hofa)
+      mapping.each_pair do |old_key, new_key|
+        hofa[new_key] = hofa.delete(old_key)
+      end
+
+      hofa
+    end
+
+
+    def generate_mapping(keys)
+      mapping = {}
+
+      keys.each do |key|
+        mapping[key] = underscore_key(sanitize_key(key)) unless key.is_a?(Symbol)
+      end
+
+      mapping
+    end
+
+
+    # returns a snake_case Symbol
+    def underscore_key(key)
+      key.to_s.gsub(/::/, '/').
+      gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+      gsub(/([a-z\d])([A-Z])/,'\1_\2').
+      tr("-", "_").
+      downcase.to_sym
+    end
+
+
+    # removes punctuation and specal characters,
+    # replaces space with underscore.
+    def sanitize_key(key)
+      key.tr('.():/','').gsub(/^\d+.?\s/, "").tr(' ','_')
+    end
+
+
+    # returns true if key is in a date format
+    # like 2023-06-03
+    def is_date?(key)
+      !/(\d{4}-\d{2}-\d{2})/.match(key.to_s).nil?
     end
   end
 end
+

data/lib/sqa/errors.rb

@@ -1,6 +1,30 @@
 # lib/sqa/errors.rb
 
-module SQA
-	# raised when an API contract is broken
-	class BadParameterError < ArgumentError; end
+# raised when a method is still in TODO state
+class ApiError < RuntimeError
+  def self.raise(why)
+    puts "="*64
+    puts "== API Error"
+    puts why
+    puts
+    puts "Callback trace:"
+    puts caller
+    puts "="*64
+    super
+  end
 end
+
+# raised when a method is still in TODO state
+class NotImplemented < RuntimeError
+  def self.raise
+    puts "="*64
+    puts "== Not Yet Implemented"
+    puts "Callback trace:"
+    puts caller
+    puts "="*64
+    super
+  end
+end
+
+# raised when an API contract is broken
+class BadParameterError < ArgumentError; end

data/lib/sqa/init.rb

@@ -2,7 +2,13 @@
 
 module SQA
 	class << self
-		@@config = nil
+		@@config 	= nil
+		@@av 			= ApiKeyManager::RateLimited.new(
+									api_keys: 		ENV['AV_API_KEYS'],
+									delay: 				true,
+									rate_count: 	ENV['AV_RATE_CNT'] ||  5,
+									rate_period: 	ENV['AV_RATE_PER'] || 60
+								)
 
 		# Initializes the SQA modules
 		# returns the configuration
@@ -26,12 +32,11 @@ module SQA
 
 			config.data_dir = homify(config.data_dir)
 
-			Daru.lazy_update 			= config.lazy_update
-			Daru.plotting_library = config.plotting_library
-
 			config
 		end
 
+		def av() 								= @@av
+
 		def debug?() 						= @@config.debug?
 		def verbose?() 					= @@config.verbose?
 

data/lib/sqa/stock.rb

@@ -6,83 +6,143 @@
 #         separate class and injected by the requiring program?
 
 class SQA::Stock
+  extend Forwardable
+
   CONNECTION = Faraday.new(url: "https://www.alphavantage.co")
 
-  attr_accessor :company_name
-  attr_accessor :df             # The DataFrane
-  attr_accessor :ticker
-  attr_accessor :type           # type of data store (default is CSV)
-  attr_accessor :indicators
+  attr_accessor :data # General Info      -- SQA::DataFrame::Data
+  attr_accessor :df   # Historical Prices -- SQA::DataFrame::Data
+
+  attr_accessor :klass        # class of historical and current prices
+  attr_accessor :transformers # procs for changing column values from String to Numeric
 
   def initialize(
         ticker:,
-        source: :alpha_vantage,
-        type:   :csv
+        source: :alpha_vantage
       )
+
+    @ticker     = ticker.downcase
+    @source     = source
+
     raise "Invalid Ticker #{ticker}" unless SQA::Ticker.valid?(ticker)
 
-    # TODO: Change API on lookup to return array instead of hash
-    #       Could this also incorporate the validation process to
-    #       save an additiona hash lookup?
+    @data_path  = SQA.data_dir + "#{@ticker}.json"
+    @df_path    = SQA.data_dir + "#{@ticker}.csv"
 
-    entry         = SQA::Ticker.lookup(ticker)
+    @klass         = "SQA::DataFrame::#{@source.to_s.camelize}".constantize
+    @transformers  = "SQA::DataFrame::#{@source.to_s.camelize}::TRANSFORMERS".constantize
 
-    @ticker       = ticker.downcase
-    @company_name = entry[:name]
-    @exchange     = entry[:exchange]
-    @klass        = "SQA::DataFrame::#{source.to_s.camelize}".constantize
-    @type         = type
-    @indicators   = OpenStruct.new
+    if @data_path.exist?
+      load
+    else
+      create
+      update
+      save
+    end
 
     update_the_dataframe
   end
 
 
-  def update_the_dataframe
-    df1 = @klass.load(@ticker, type)
-    df2 = @klass.recent(@ticker)
+  def load
+    @data = SQA::DataFrame::Data.new(
+              JSON.parse(@data_path.read)
+            )
+  end
 
-    df1_nrows = df1.nrows
-    @df       = @klass.append(df1, df2)
 
-    if @df.nrows > df1_nrows
-      @df.send("to_#{@type}", SQA.data_dir + "#{ticker}.csv")
+  def create
+    @data =
+      SQA::DataFrame::Data.new(
+        {
+          ticker:       @ticker,
+          source:       @source,
+          indicators:   { xyzzy: "Magic" },
+        }
+      )
+  end
+
+
+  def update
+    merge_overview
+  end
+
+
+  def save
+    @data_path.write @data.to_json
+  end
+
+
+  def_delegator :@data, :ticker,      :ticker
+  def_delegator :@data, :name,        :name
+  def_delegator :@data, :exchange,    :exchange
+  def_delegator :@data, :source,      :source
+  def_delegator :@data, :indicators,  :indicators
+  def_delegator :@data, :indicators=, :indicators=
+  def_delegator :@data, :overview,    :overview
+
+
+
+  def update_the_dataframe
+    if @df_path.exist?
+      @df     = SQA::DataFrame.load(
+        source:       @df_path,
+        transformers: @transformers
+      )
+    else
+      @df     = klass.recent(@ticker, full: true)
+      @df.to_csv(@df_path)
+      return
     end
 
-    # Adding a ticker vector in case I want to do
-    # some multi-stock analysis in the same data frame.
-    # For example to see how one stock coorelates with another.
-    @df[:ticker]  = @ticker
+    from_date = Date.parse(@df.timestamp.last) + 1
+    df2       = klass.recent(@ticker, from_date: from_date)
+
+    return if df2.nil?  # CSV file is up to date.
+
+    df_nrows  = @df.nrows
+    @df.append(df2)
+
+    if @df.nrows > df_nrows
+      @df.to_csv(@df_path)
+    end
   end
 
+
   def to_s
     "#{ticker} with #{@df.size} data points from #{@df.timestamp.first} to #{@df.timestamp.last}"
   end
+  alias_method :inspect, :to_s
 
-  # TODO: Turn this into a class Stock::Overview
-  #       which is a sub-class of Hashie::Dash
-  def overview
-    return @overview unless @overview.nil?
 
+  def merge_overview
     temp = JSON.parse(
-      CONNECTION.get("/query?function=OVERVIEW&symbol=#{@ticker.upcase}&apikey=#{Nenv.av_api_key}")
+      CONNECTION.get("/query?function=OVERVIEW&symbol=#{ticker.upcase}&apikey=#{SQA.av.key}")
         .to_hash[:body]
     )
 
+    if temp.has_key? "Information"
+      ApiError.raise(temp["Information"])
+    end
+
     # TODO: CamelCase hash keys look common in Alpha Vantage
     #       JSON; look at making a special Hashie-based class
     #       to convert the keys to normal Ruby standards.
 
     temp2 = {}
 
-    string_values = %w[ address asset_type cik country currency description dividend_date ex_dividend_date exchange fiscal_year_end industry latest_quarter name sector symbol ]
+    string_values = %w[ address asset_type cik country currency
+                        description dividend_date ex_dividend_date
+                        exchange fiscal_year_end industry latest_quarter
+                        name sector symbol
+                      ]
 
     temp.keys.each do |k|
       new_k         = k.underscore
       temp2[new_k]  = string_values.include?(new_k) ? temp[k] : temp[k].to_f
     end
 
-    @overview = Hashie::Mash.new temp2
+    @data.overview = temp2
   end
 
 
@@ -100,7 +160,7 @@ class SQA::Stock
 
       a_hash  = JSON.parse(
                   CONNECTION.get(
-                    "/query?function=TOP_GAINERS_LOSERS&apikey=#{Nenv.av_api_key}"
+                    "/query?function=TOP_GAINERS_LOSERS&apikey=#{SQA.av.key}"
                   ).to_hash[:body]
                 )
 

data/lib/sqa/strategy.rb

@@ -8,7 +8,7 @@ class SQA::Strategy
   end
 
   def add(a_strategy)
-    raise SQA::BadParameterError unless [Class, Method].include? a_strategy.class
+    raise BadParameterError unless [Class, Method].include? a_strategy.class
 
     a_proc  = if Class == a_strategy.class
                 a_strategy.method(:trade)

data/lib/sqa/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SQA
-  VERSION = "0.0.15"
+  VERSION = "0.0.18"
 
   class << self
     def version

data/lib/sqa.rb

@@ -17,12 +17,13 @@ end
 ## Additional Libraries
 
 require 'active_support/core_ext/string'
-require 'alphavantage' 	# TODO: add rate limiter to it
+require 'alphavantage' 	# TODO: add rate limiter to it; ** PR submitted! **
+require 'api_key_manager'
 require 'amazing_print'
-require 'daru' 					# TODO: Replace this gem with something better
-require 'descriptive_statistics'
 require 'faraday'
 require 'hashie'
+require 'lite/statistics'
+require 'lite/statistics/monkey_patches' # patch to Enumerable
 require 'nenv'
 require 'sem_version'
 require 'sem_version/core_ext'
@@ -38,7 +39,6 @@ require_relative "sqa/errors"
 
 require_relative 'sqa/init.rb'
 
-# require_relative "patches/daru" # TODO: extract Daru::DataFrame in new gem sqa-data_frame
 
 # TODO: Some of these components make direct calls to the
 # 			Alpha Vantage API.  Convert them to use the