sqa 0.0.12 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 39185fb1deccd8cc248d4ad776b724ea5ebe1a7e93fdc1d6ada4f53661a55bba
-  data.tar.gz: 3b394f997cdcdd67ec9eced2cd7bc08c1af2b407b8945e1a76e0383591ca3495
+  metadata.gz: 8dd371b757bcc5da8e913681a06666715cbe88ca11f582df64d68ef305391d28
+  data.tar.gz: 8b1851af5e7875266acbd3fb35a7ba108fdb267d7490eb19984819bba5f15538
 SHA512:
-  metadata.gz: 3d805f6db11375738db0cdad8385f7a74c26c3fba39f0a7aa88593532e1127af7c4c690844eae56f6203696743e1b655bb9d996f2e71745cefb4f04227bbe622
-  data.tar.gz: c52327bfbf15f25159a14af31be0881112e169dc73dbdab7d7c7ac7519208e366314e4ab6bd31d44134fbda5238dc7b56e553dcc8c7d231938b78af3ec17d137
+  metadata.gz: ab803635e2f85e71bebacab6be10cc345f6050a890c730a658741d4a3ae4530490be33fc692a3ced0e843e473cc5613644dc5a0c0d0bfd3a43d5c0d2b4d41510
+  data.tar.gz: 7ee788d760d4020efb41baa3f5bc894cc4824d0308144b68f8c352bed03a01ce2049547bace95636264fab1c7b58ab12f97c99209433e2b32719db575017d25c

data/lib/sqa/cli.rb

@@ -140,6 +140,7 @@ module SQA
 
         elsif params[:config_file]
           # Override the defaults <- envars <- config file content
+          params[:config_file]   = SQA.homify params[:config_file]
           SQA.config.config_file = params[:config_file]
           SQA.config.from_file
         end

data/lib/sqa/data_frame/alpha_vantage.rb

@@ -0,0 +1,155 @@
+# lib/sqa/data_frame/alpha_vantage.rb
+# frozen_string_literal: true
+#
+# Using the Alpha Vantage JSON interface
+#
+
+require 'faraday'
+require 'json'
+
+class SQA::DataFrame < Daru::DataFrame
+  class AlphaVantage
+    API_KEY     = Nenv.av_api_key
+    CONNECTION  = Faraday.new(url: 'https://www.alphavantage.co')
+    HEADERS     = YahooFinance::HEADERS
+
+    # The Alpha Vantage headers are being remapped so that
+    # they match those of the Yahoo Finance CSV file.
+    #
+    HEADER_MAPPING = {
+      "date"            => HEADERS[0],
+      "open"            => HEADERS[1],
+      "high"            => HEADERS[2],
+      "low"             => HEADERS[3],
+      "close"           => HEADERS[4],
+      "adjusted_close"  => HEADERS[5],
+      "volume"          => HEADERS[6]
+    }
+
+
+    ################################################################
+    # Load a Dataframe from a csv file
+    def self.load(ticker, type="csv")
+      filepath = SQA.data_dir + "#{ticker}.#{type}"
+
+      if filepath.exist?
+        df = normalize_vector_names SQA::DataFrame.load(ticker, type)
+      else
+        df = recent(ticker, full: true)
+        df.send("to_#{type}",filepath)
+      end
+
+      df
+    end
+
+
+    # Normalize the vector (aka column) names as
+    # symbols using the standard names set by
+    # Yahoo Finance ... since it was the first one
+    # not because its anything special.
+    #
+    def self.normalize_vector_names(df)
+      headers = df.vectors.to_a
+
+      # convert vector names to symbols
+      # when they are strings.  They become stings
+      # when the data frame is saved to a CSV file
+      # and then loaded back in.
+
+      if headers.first == HEADERS.first.to_s
+        a_hash = {}
+        HEADERS.each {|k| a_hash[k.to_s] = k}
+        df.rename_vectors(a_hash) # renames from String to Symbol
+      else
+        df.rename_vectors(HEADER_MAPPING)
+      end
+
+      df
+    end
+
+
+    # Get recent data from JSON API
+    #
+    # ticker String the security to retrieve
+    # returns a DataFrame
+    #
+    # NOTE: The function=TIME_SERIES_DAILY_ADJUSTED
+    #       is not a free API endpoint from Alpha Vantange.
+    #       So we are just using the free API endpoint
+    #       function=TIME_SERIES_DAILY
+    #       This means that we are not getting the
+    #       real adjusted closing price.  To sync
+    #       the columns with those from Yahoo Finance
+    #       we are duplicating the unadjusted clossing price
+    #       and adding that to the data frame as if it were
+    #       adjusted.
+    #
+    def self.recent(ticker, full: false)
+      # NOTE: Using the CSV format because the JSON format has
+      #       really silly key values.  The column names for the
+      #       CSV format are much better.
+      response  = CONNECTION.get(
+        "/query?" +
+        "function=TIME_SERIES_DAILY&" +
+        "symbol=#{ticker.upcase}&" +
+        "apikey=#{API_KEY}&" +
+        "datatype=csv&" +
+        "outputsize=#{full ? 'full' : 'compact'}"
+      ).to_hash
+
+      unless 200 == response[:status]
+        raise "Bad Response: #{response[:status]}"
+      end
+
+      raw           = response[:body].split
+
+      headers       = raw.shift.split(',')
+      headers[0]    = 'date'  # website returns "timestamp" but that
+                              # has an unintended side-effect when
+                              # the names are normalized.
+
+      close_inx     = headers.size - 2
+      adj_close_inx = close_inx + 1
+
+      headers.insert(adj_close_inx, 'adjusted_close')
+
+      data    = raw.map do |e|
+                  e2 = e.split(',')
+                  e2[1..-2] = e2[1..-2].map(&:to_f) # converting open, high, low, close
+                  e2[-1]    = e2[-1].to_i           # converting volumn
+                  e2.insert(adj_close_inx, e2[close_inx]) # duplicate the close price as a fake adj close price
+                  headers.zip(e2).to_h
+                end
+
+      # What oldest data first in the data frame
+      normalize_vector_names Daru::DataFrame.new(data.reverse)
+    end
+
+
+    # Append update_df rows to the base_df
+    #
+    # base_df is ascending on timestamp
+    # update_df is descending on timestamp
+    #
+    # base_df content came from CSV file downloaded
+    # from Yahoo Finance.
+    #
+    # update_df came from scraping the webpage
+    # at Yahoo Finance for the recent history.
+    #
+    # Returns a combined DataFrame.
+    #
+    def self.append(base_df, updates_df)
+      last_timestamp  = Date.parse base_df.timestamp.last
+      filtered_df     = updates_df.filter_rows { |row| Date.parse(row[:timestamp]) > last_timestamp }
+
+      last_inx = filtered_df.size - 1
+
+      (0..last_inx).each do |x|
+        base_df.add_row filtered_df.row[last_inx-x]
+      end
+
+      base_df
+    end
+  end
+end

data/lib/sqa/data_frame/yahoo_finance.rb

@@ -59,6 +59,9 @@ class SQA::DataFrame < Daru::DataFrame
       response  = CONNECTION.get("/quote/#{ticker.upcase}/history")
       doc       = Nokogiri::HTML(response.body)
       table     = doc.css('table').first
+
+      raise "NoDataError" if table.nil?
+
       rows      = table.css('tbody tr')
 
       data = []

data/lib/sqa/data_frame.rb

@@ -2,17 +2,19 @@
 # frozen_string_literal: true
 
 require_relative  'data_frame/yahoo_finance'
+require_relative  'data_frame/alpha_vantage'
 
 class Daru::DataFrame
 
   def to_csv(path_to_file, opts={})
     options = {
+      headers:    true,
       converters: :numeric
     }.merge(opts)
 
     writer = ::CSV.open(path_to_file, 'wb')
 
-    writer << vectors.to_a unless options[:headers] == false
+    writer << vectors.to_a if options[:headers]
 
     each_row do |row|
       writer << if options[:convert_comma]
@@ -31,27 +33,20 @@ end
 
 class SQA::DataFrame < Daru::DataFrame
 
-
   #################################################
+  def self.load(ticker, type="csv", options={}, &block)
+    source  = SQA.data_dir + "#{ticker}.#{type}"
 
-  def self.path(filename)
-    Pathname.new(SQA.config.data_dir) + filename
-  end
-
-  def self.load(filename, options={}, &block)
-    source  = path(filename)
-    type    = source.extname.downcase
-
-    if ".csv" == type
+    if :csv == type
      from_csv(source, options={}, &block)
-    elsif ".json" == type
+    elsif :json == type
       from_json(source, options={}, &block)
-    elsif %w[.txt .dat].include?(type)
+    elsif %i[txt dat].include?(type)
       from_plaintext(source, options={}, &block)
-    elsif ".xls" == type
+    elsif :xls == type
       from_excel(source, options={}, &block)
     else
-      raise SQA::BadParamenterError, "un-suppod file type: #{type}"
+      raise SQA::BadParameterError, "un-supported file type: #{type}"
     end
   end
 end

data/lib/sqa/stock.rb

@@ -4,14 +4,27 @@ class SQA::Stock
   attr_accessor :company_name
   attr_accessor :df             # The DataFrane
   attr_accessor :ticker
+  attr_accessor :type           # type of data store (default is CSV)
   attr_accessor :indicators
 
-  def initialize(ticker:, source: :yahoo_finance, type: :csv)
+  def initialize(
+        ticker:,
+        source: :alpha_vantage,
+        type:   :csv
+      )
+    raise "Invalid Ticker #{ticker}" unless SQA::Ticker.valid?(ticker)
+
+    # TODO: Change API on lookup to return array instead of hash
+    #       Could this also incorporate the validation process to
+    #       save an additiona hash lookup?
+
+    entry         = SQA::Ticker.lookup(ticker)
+
     @ticker       = ticker.downcase
-    @company_name = "Company Name"
+    @company_name = entry[:name]
+    @exchange     = entry[:exchange]
     @klass        = "SQA::DataFrame::#{source.to_s.camelize}".constantize
     @type         = type
-    @filename     = "#{@ticker}.#{type}"
     @indicators   = OpenStruct.new
 
     update_the_dataframe
@@ -19,16 +32,19 @@ class SQA::Stock
 
 
   def update_the_dataframe
-    df1 = @klass.load(@filename)
+    df1 = @klass.load(@ticker, type)
     df2 = @klass.recent(@ticker)
-    @df = @klass.append(df1, df2)
 
-    if @df.nrows > df1.nrows
-      @df.send("to_#{@type}", SQA::DataFrame.path(@filename))
+    df1_nrows = df1.nrows
+    @df       = @klass.append(df1, df2)
+
+    if @df.nrows > df1_nrows
+      @df.send("to_#{@type}", SQA.data_dir + "#{ticker}.csv")
     end
 
     # Adding a ticker vector in case I want to do
     # some multi-stock analysis in the same data frame.
+    # For example to see how one stock coorelates with another.
     @df[:ticker]  = @ticker
   end
 
@@ -36,7 +52,3 @@ class SQA::Stock
     "#{ticker} with #{@df.size} data points from #{@df.timestamp.first} to #{@df.timestamp.last}"
   end
 end
-
-__END__
-
-aapl = Stock.new('aapl', SQA::Datastore::CSV)

data/lib/sqa/ticker.rb

@@ -0,0 +1,68 @@
+# sqa/lib/sqa/ticker.rb
+#
+# Uses the https://dumbstockapi.com/ website to download a CSV file
+#
+# The CSV files have names like this:
+#   "dumbstockapi-2023-09-21T16 39 55.165Z.csv"
+#
+# which has this header:
+#   ticker,name,is_etf,exchange
+#
+# Not using the is_etf columns
+#
+class SQA::Ticker
+  FILENAME_PREFIX = "dumbstockapi"
+  CONNECTION      = Faraday.new(url: "https://dumbstockapi.com")
+  @@data          = {}
+
+
+  def self.download(country="US")
+    response = CONNECTION.get("/stock?format=csv&countries=#{country.upcase}").to_hash
+
+    if 200 == response[:status]
+      filename = response[:response_headers]["content-disposition"].split('=').last.gsub('"','')
+      out_path = Pathname.new(SQA.config.data_dir) + filename
+      out_path.write response[:body]
+    end
+
+    response[:status]
+  end
+
+
+  def self.load
+    tries = 0
+    found = false
+
+     until(found || tries >= 3) do
+      files     = Pathname.new(SQA.config.data_dir).children.select{|c| c.basename.to_s.start_with?(FILENAME_PREFIX)}.sort
+      if files.empty?
+        download
+        tries += 1
+      else
+        found = true
+      end
+    end
+
+    raise "NoDataError" if files.empty?
+
+    load_from_csv files.last
+  end
+
+
+  def self.load_from_csv(csv_path)
+    CSV.foreach(csv_path, headers: true) do |row|
+      @@data[row["ticker"]] = {
+        name:     row["name"],
+        exchange: row["exchange"]
+      }
+    end
+
+    @@data
+  end
+
+
+
+  def self.data           = @@data.empty? ? load : @@data
+  def self.lookup(ticker) = data[ticker.upcase]
+  def self.valid?(ticker) = data.has_key?(ticker.upcase)
+end

data/lib/sqa/version.rb

@@ -4,7 +4,7 @@ require 'sem_version'
 require 'sem_version/core_ext'
 
 module SQA
-  VERSION = "0.0.12"
+  VERSION = "0.0.13"
 
   class << self
     def version

data/lib/sqa.rb

@@ -10,6 +10,7 @@ require 'nenv'
 require 'pathname'
 
 require_relative "sqa/version"
+require_relative "sqa/errors"
 
 
 unless defined?(HOME)
@@ -34,10 +35,12 @@ module SQA
 				CLI.run(argv)
 			else
 				# There are no real command line parameters
-				# because the sqa gem is be required within
+				# because the sqa gem is being required within
 				# the context of a larger program.
 			end
 
+			config.data_dir = homify(config.data_dir)
+
 			Daru.lazy_update 			= config.lazy_update
 			Daru.plotting_library = config.plotting_library
 
@@ -50,25 +53,16 @@ module SQA
 			nil
 		end
 
-		def homify(filepath)
-			filepath.gsub(/^~/, Nenv.home)
-		end
+		def debug?() 						= @@config.debug?
+		def verbose?() 					= @@config.verbose?
 
-		def config
-			@@config
-		end
+		def homify(filepath) 		= filepath.gsub(/^~/, Nenv.home)
+		def data_dir() 					= Pathname.new(config.data_dir)
+		def config()            = @@config
 
 		def config=(an_object)
 			@@config = an_object
 		end
-
-		def debug?
-			@@config.debug?
-		end
-
-		def verbose?
-			@@config.verbose?
-		end
 	end
 end
 
@@ -77,9 +71,9 @@ end
 require_relative "sqa/config"
 require_relative "sqa/constants"
 require_relative "sqa/data_frame"
-require_relative "sqa/errors"
 require_relative "sqa/indicator"
 require_relative "sqa/portfolio"
 require_relative "sqa/strategy"
 require_relative "sqa/stock"
+require_relative "sqa/ticker"
 require_relative "sqa/trade"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqa
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
 platform: ruby
 authors:
 - Dewayne VanHoozer
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-09-20 00:00:00.000000000 Z
+date: 2023-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -329,6 +329,7 @@ files:
 - lib/sqa/config.rb
 - lib/sqa/constants.rb
 - lib/sqa/data_frame.rb
+- lib/sqa/data_frame/alpha_vantage.rb
 - lib/sqa/data_frame/yahoo_finance.rb
 - lib/sqa/errors.rb
 - lib/sqa/indicator.rb
@@ -366,6 +367,7 @@ files:
 - lib/sqa/strategy/random.rb
 - lib/sqa/strategy/rsi.rb
 - lib/sqa/strategy/sma.rb
+- lib/sqa/ticker.rb
 - lib/sqa/trade.rb
 - lib/sqa/version.rb
 - lib/sqa/web.rb