stockcruncher 1.1.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51887b6440cf8d3958f52c05d5daa9e77c0bada0e385445ec6bcb6553fe9c16d
-  data.tar.gz: 66ade1c3623130c5e5588ceca005bddb201b176d1d81ffbb0cabd41568b1692c
+  metadata.gz: 6c5f9d2cc2dba8ad0fbd9882cebcda893500e0f029610107004481b0b236d6b9
+  data.tar.gz: 65815caea89d856ef4e4c76365df166bdb971302a95f6665d657c0ba6420e671
 SHA512:
-  metadata.gz: 0bbdd83e653eadf37c57b6cf5e2bf3234c94ccf74d59ffcd21f995d74260a7673da8a79564f37b4b69715fb5f295d0e1bcf2465fee119a84965cbcc333d0f0bc
-  data.tar.gz: 979223a1ee0cba78f9d482d22f3025c17fb0506c487d237b94b5855a7843ef3eca610fd89fcbd0a41c66325a1f9e86786e0b56703d666fbca8d5417dd3906321
+  metadata.gz: 19b7f5eafd15cbcc5713a6729e17c3a2ae5dd4c471f0d77c7becb4ba10be08c6c492f4469b7ab4a93858215a406d95fced52cde98cffd0644c081ce331b43e9a
+  data.tar.gz: c73d7c0caa396f0cf1b06f88f74e2e1cdbd0c7b7fa1739e4b72df4e5852b16e847a46d09071699db51bc6ed07a85e51766c8259221214d62e845ed15331bc980

data/.gitlab-ci.yml

@@ -1,9 +1,11 @@
+
 image: ruby:2.4
 
+before_script:
+  - bundle install
+
 rubocop:
   stage: test
-  before_script:
-    - gem install --silent rubocop
   script:
     - rubocop
 
@@ -25,7 +27,6 @@ git_history:
 rspec:
   stage: test
   script:
-    - bundle install
     - rspec
   artifacts:
     paths:

data/.rubocop.yml

@@ -4,3 +4,4 @@ AllCops:
   DisplayStyleGuide: true
   DisplayCopNames: false
   TargetRubyVersion: 2.4
+  NewCops: disable

data/CHANGELOG

@@ -11,3 +11,20 @@ Changelog
 
 - Add InfluxDB as database.
 - WIP for daily time serie.
+
+1.1.1
+-----
+
+- Rescue error on no data
+
+1.2.0
+-----
+
+- daily subcommand now recalculates missing data
+- daily subcommand also writes data to database
+
+1.2.1
+-----
+
+- Refactor to preprocess data in source class
+- Refactor to print pretty json instead of comma separated striing

data/lib/stockcruncher.rb

@@ -10,6 +10,9 @@ module StockCruncher
 
     def start(args = ARGV)
       StockCruncher::CLI.start(args)
+    rescue StandardError => e
+      warn e.message
+      exit 1
     end
   end
 end

data/lib/stockcruncher/alphavantage.rb

@@ -8,12 +8,40 @@ module StockCruncher
   class AlphaVantage < Cruncher
     API_URL = 'https://www.alphavantage.co/query?'
 
+    # Method to calculate missing data (previousClose, change, changePercent)
+    def calculate_missing_data(hash)
+      keys = hash.keys
+      hash.each_with_index do |(date, v), index|
+        prevday = keys[index + 1]
+        next if prevday.nil?
+
+        prevclose = hash[prevday]['close']
+        hash[date] = v.merge(generate_missing_data(v['close'], prevclose))
+      end
+      hash
+    end
+
+    # Method to calculate change difference
+    def change(value, base)
+      (value - base).round(4).to_s
+    end
+
+    # Method to calculate percentage of change
+    def change_percent(value, base)
+      ((value / base - 1) * 100).round(4).to_s
+    end
+
+    # Method to create a new hash from two arrays of keys and values
+    def create_hash(descriptions, values)
+      descriptions.split(',').zip(values.split(',')).to_h
+    end
+
     # Main method to crunch data.
     def crunch_daily(symbol, fullsize)
       url = API_URL + parameters(symbol, 'TIME_SERIES_DAILY')
-      url += '&datatype=csv&outputsize=' + (fullsize ? 'full' : 'compact')
+      url += "&datatype=csv&outputsize=#{fullsize ? 'full' : 'compact'}"
       res = request(url)
-      res.body
+      transform_daily(res.body)
     end
 
     # Main method to crunch data.
@@ -21,14 +49,55 @@ module StockCruncher
       url = API_URL + parameters(symbol, 'GLOBAL_QUOTE')
       url += '&datatype=csv'
       res = request(url)
-      res.body
+      transform_quote(res.body)
     end
 
+    # Method to generate missing data
+    def generate_missing_data(current, previous)
+      {
+        'previousClose' => previous,
+        'change' => change(current.to_f, previous.to_f),
+        'changePercent' => change_percent(current.to_f, previous.to_f)
+      }
+    end
+
+    # Set parameters of api call
     def parameters(symbol, serie)
-      p = 'function=' + serie
-      p += '&symbol=' + symbol
-      p += '&apikey=' + @config[self.class.name.split('::').last]['apikey']
+      p = "function=#{serie}"
+      p += "&symbol=#{symbol}"
+      p += "&apikey=#{@config[self.class.name.split('::').last]['apikey']}"
       p
     end
+
+    # Method to transform raw data to constructed hash
+    def prepare_daily_timeserie(data)
+      lines = data.split("\r\n")
+      desc = lines.shift.split(',').drop(1)
+      hash = {}
+      lines.each do |line|
+        values = line.split(',')
+        date = values.shift
+        hash[date] = desc.zip(values).to_h
+      end
+      hash
+    end
+
+    # Method to transform daily result to nested hash
+    def transform_daily(rawdata)
+      raise StandardError, 'No data' if rawdata.match?(/Error Message/)
+
+      values = prepare_daily_timeserie(rawdata)
+      calculate_missing_data(values)
+    end
+
+    # Method to transform quote result to hash
+    def transform_quote(rawdata)
+      raise StandardError, 'No data' if rawdata.match?(/{}/)
+
+      values = create_hash(*rawdata.split("\r\n"))
+      values['close'] = values.delete('price')
+      values['changePercent'] = values['changePercent'].delete('%')
+      values
+    end
   end
 end

data/lib/stockcruncher/cli.rb

@@ -1,6 +1,7 @@
 #!/usr/bin/ruby
 # frozen_string_literal: true
 
+require 'json'
 require 'thor'
 require 'yaml'
 
@@ -48,20 +49,35 @@ module StockCruncher
       opts = options.dup
       config = YAML.load_file(opts['config'])
       cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
-      raw_data = cruncher.crunch_daily(symbol, opts['full'])
-      # TODO: export to DB here
-      puts raw_data unless opts['quiet']
+      data = cruncher.crunch_daily(symbol, opts['full'])
+      StockCruncher::InfluxDB.new(config).export_history(symbol, data)
+      puts JSON.pretty_generate(data) unless opts['quiet']
+    end
+
+    desc('movingaverages SYMBOL [options]',
+         'Calculate and export moving averages for requested symbol.')
+    option(
+      :all,
+      aliases: ['-a'],
+      type: :boolean,
+      default: false,
+      desc: 'Recalculate all MA historical values.'
+    )
+    def movingaverages(symbol)
+      opts = options.dup
+      config = YAML.load_file(opts['config'])
+      StockCruncher::InfluxDB.new(config).moving_averages(symbol, opts['all'])
     end
 
     desc('quote SYMBOL [options]',
          'Crunch SYMBOL stock market data for last day quote.')
     def quote(symbol)
       opts = options.dup
-      config = YAML.load_file(opts['config'])
+      config = StockCruncher::Config.load(opts['config'])
       cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
-      raw_data = cruncher.crunch_quote(symbol)
-      puts raw_data unless opts['quiet']
-      StockCruncher::InfluxDB.new(config).export_last_day(raw_data)
+      data = cruncher.crunch_quote(symbol)
+      StockCruncher::InfluxDB.new(config).export_last_day(data)
+      puts JSON.pretty_generate(data) unless opts['quiet']
     end
   end
 end

data/lib/stockcruncher/config.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/ruby
+# frozen_string_literal: true
+
+require 'yaml'
+
+module StockCruncher
+  # this is a module to load configuration file and environment variable
+  module Config
+    module_function
+
+    # Load config file and override with env variables
+    def load(file)
+      config = YAML.load_file(file)
+      overload_alphavantage(config)
+      overload_influxdb(config)
+    end
+
+    def overload(config, prefix, component, items)
+      items.each do |key|
+        var = "#{prefix}#{key.upcase}"
+        config[component][key] = ENV[var] unless ENV[var].nil?
+      end
+      config
+    end
+
+    def overload_alphavantage(config)
+      prefix = 'SCR_AV_'
+      component = 'AlphaVantage'
+      items = %w[apikey]
+      overload(config, prefix, component, items)
+    end
+
+    def overload_influxdb(config)
+      prefix = 'SCR_IDB_'
+      component = 'InfluxDB'
+      items = %w[scheme host port user password dbname]
+      overload(config, prefix, component, items)
+    end
+  end
+end

data/lib/stockcruncher/influxdb.rb

@@ -1,6 +1,8 @@
 #!/usr/bin/ruby
 # frozen_string_literal: true
 
+require 'date'
+require 'json'
 require 'net/http'
 
 module StockCruncher
@@ -12,30 +14,69 @@ module StockCruncher
       @insecure = insecure
     end
 
-    # Method to export latest data to database
-    def export_last_day(raw)
-      (desc, line) = raw.split("\r\n")
-      values = desc.split(',').zip(line.split(',')).to_h
-      values['close'] = values.delete('price')
-      values['changePercent'] = values['changePercent'].delete('%')
-      tags = { 'symbol' => values.delete('symbol') }
-      date = values.delete('latestDay')
-      write('daily', tags, values, date)
+    def get_daily_values(symbol, fullsize)
+      values = %w[close change changePercent volume]
+      data = query('daily', symbol, values, fullsize)
+      data['columns'].zip(data['values'].transpose).to_h
+    end
+
+    # Method to calculate moving averages based on last day values
+    def moving_averages(symbol, fullsize)
+      series = get_daily_values(symbol, fullsize)
+      tags = create_tags(symbol)
+      series['close'].each_index do |i|
+        date = series['time'][i]
+        serie = series['close'][i, 201]
+        weights = series['volume'][i, 201]
+        write_moving_averages(tags, serie, weights, date)
+        break unless fullsize
+      end
+    end
+
+    # Method to create tags hash containing only symbol
+    def create_tags(symbol)
+      { 'symbol' => symbol }
     end
 
     # Method to export historical data to database
-    def export_history(raw)
-      # TODO: export whole history and calculated missing values
+    def export_history(symbol, timeseries)
+      tags = create_tags(symbol)
+      timeseries.each_pair do |date, values|
+        write('daily', tags, values, date)
+      end
+    end
+
+    # Method to export latest data to database
+    def export_last_day(values)
+      tags = create_tags(values.delete('symbol'))
+      date = values.delete('latestDay')
+      write('daily', tags, values, date)
     end
 
     # Method to format and array of values into comma separated string
     def format_values(values)
-      string = ''
-      values.each_pair do |k, v|
-        string += "#{k}=#{v}"
-        string += ',' unless k == values.keys.last
-      end
-      string
+      values.map { |k, v| "#{k}=#{v}" }.join(',')
+    end
+
+    # Method to calculate all statistics
+    def write_moving_averages(tags, serie, weights, date)
+      write('ema', tags, StockCruncher::Stats.list_ema(serie), date)
+      write('lwma', tags, StockCruncher::Stats.list_lwma(serie), date)
+      write('sma', tags, StockCruncher::Stats.list_sma(serie), date)
+      write('vwma', tags, StockCruncher::Stats.list_vwma(serie, weights), date)
+    end
+
+    # Method to query data in bucket
+    def query(name, symbol, values, full)
+      url = "#{@cfg['scheme']}://#{@cfg['host']}:#{@cfg['port']}/query?" \
+            "db=#{@cfg['dbname']}"
+      size = full ? '' : 'LIMIT 201'
+      body = "q=SELECT #{values.join(',')} FROM #{name} " \
+             "WHERE symbol = '#{symbol}' ORDER BY time DESC #{size}"
+      data = JSON.parse(request(url, body).body)['results'][0]['series']
+      raise StandardError, 'No data' if data.nil?
+
+      data[0]
     end
 
     # Method to send http post request
@@ -54,7 +95,7 @@ module StockCruncher
     def write(name, tags, values, date)
       url = "#{@cfg['scheme']}://#{@cfg['host']}:#{@cfg['port']}/write?" \
             "db=#{@cfg['dbname']}"
-      timestamp = DateTime.parse(date + 'T18:00:00').strftime('%s%N')
+      timestamp = DateTime.parse("#{date}T18:00:00").strftime('%s%N')
       body = "#{name},#{format_values(tags)} #{format_values(values)} " \
              "#{timestamp}"
       request(url, body)

data/lib/stockcruncher/stats.rb

@@ -0,0 +1,74 @@
+#!/usr/bin/ruby
+# frozen_string_literal: true
+
+module StockCruncher
+  # this is a module with various statistic calculation methods
+  module Stats
+    extend self
+
+    RANGES = [5, 10, 20, 30, 50, 100, 200].freeze
+
+    # Calculate multiple range of exponential moving average
+    def list_ema(values)
+      h = {}
+      RANGES.each do |n|
+        next if values.size < n + 1
+
+        h["ema#{n}"] = ema(values[0, n + 1])
+      end
+      h
+    end
+
+    # Calculate multiple range of linearly weighted moving average
+    def list_lwma(values)
+      h = {}
+      RANGES.each do |n|
+        next if values.size < n
+
+        weights = (1..n).to_a.reverse
+        h["lwma#{n}"] = sma(values[0, n], weights)
+      end
+      h
+    end
+
+    # Calculate multiple range of simple moving average
+    def list_sma(values)
+      h = {}
+      RANGES.each do |n|
+        next if values.size < n
+
+        h["sma#{n}"] = sma(values[0, n])
+      end
+      h
+    end
+
+    # Calculate multiple range of volume weighted moving average
+    def list_vwma(values, volumes)
+      h = {}
+      RANGES.each do |n|
+        next if values.size < n
+
+        h["lwma#{n}"] = sma(values[0, n], volumes[0, n])
+      end
+      h
+    end
+
+    private
+
+    # Calculate exponential moving average
+    def ema(array, factor = 2, weights = nil)
+      f = factor.to_f / array.size
+      n = array.size - 1
+      tsma = sma(array[0, n], weights)
+      ysma = sma(array[1, n], weights)
+      (tsma * f + ysma * (1 - f)).round(4)
+    end
+
+    # Calculate simple moving average
+    def sma(array, weights = nil)
+      factor = weights.nil? ? Array.new(array.size, 1) : weights
+      dividend = array.each_with_index.map { |v, i| v * factor[i] }
+      (dividend.sum.to_f / factor.sum).round(4)
+    end
+  end
+end

data/lib/stockcruncher/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module StockCruncher
-  VERSION = '1.1.0'
+  VERSION = '1.3.1'
 end

data/spec/files/SYM.daily

@@ -0,0 +1,49 @@
+{
+  "2020-08-03": {
+    "open": "23.8500",
+    "high": "24.6500",
+    "low": "23.7400",
+    "close": "24.5500",
+    "volume": "3112972",
+    "previousClose": "23.8100",
+    "change": "0.74",
+    "changePercent": "3.1079"
+  },
+  "2020-07-31": {
+    "open": "24.0100",
+    "high": "24.5100",
+    "low": "23.5900",
+    "close": "23.8100",
+    "volume": "5482485",
+    "previousClose": "23.6600",
+    "change": "0.15",
+    "changePercent": "0.634"
+  },
+  "2020-07-30": {
+    "open": "24.4700",
+    "high": "24.6600",
+    "low": "23.5200",
+    "close": "23.6600",
+    "volume": "6466738",
+    "previousClose": "24.5600",
+    "change": "-0.9",
+    "changePercent": "-3.6645"
+  },
+  "2020-07-29": {
+    "open": "24.7500",
+    "high": "25.0300",
+    "low": "24.0400",
+    "close": "24.5600",
+    "volume": "4605804",
+    "previousClose": "24.7500",
+    "change": "-0.19",
+    "changePercent": "-0.7677"
+  },
+  "2020-07-28": {
+    "open": "25.9900",
+    "high": "26.0700",
+    "low": "24.5100",
+    "close": "24.7500",
+    "volume": "6904261"
+  }
+}

data/spec/files/SYM.quote

@@ -0,0 +1,10 @@
+{
+  "open": "100.0000",
+  "high": "100.1000",
+  "low": "99.9000",
+  "volume": "4",
+  "previousClose": "100.0000",
+  "change": "0.0000",
+  "changePercent": "0.0000",
+  "close": "100.0000"
+}

data/spec/stockcruncher/cli_spec.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
+require 'json'
 require 'spec_helper'
 
-quote = 'symbol,open,high,low,price,volume,latestDay,previousClose,change,ch' \
-        "angePercent\r\nSYM,100.0000,100.1000,99.9000,100.0000,4,2020-07-30," \
-        "100.0000,0.0000,0.0000%\r\n"
+daily = File.read('spec/files/SYM.daily')
+quote = File.read('spec/files/SYM.quote')
 
-describe StockCruncher::CLI do
+describe StockCruncher::CLI do # rubocop:disable Metrics/BlockLength
   context 'version' do
     it 'prints the version.' do
       out = "StockCruncher version #{StockCruncher::VERSION}\n"
@@ -14,9 +14,27 @@ describe StockCruncher::CLI do
     end
   end
 
+  context 'daily NODATA -c spec/files/stockcruncher.yml' do
+    it 'Should not get any data and should fail.' do
+      expect { start(self) }.to raise_error(SystemExit)
+    end
+  end
+
   context 'daily SYM -c spec/files/stockcruncher.yml' do
     it 'Get the daily time serie for SYM.' do
-      expect { start(self) }.to output("{}\n").to_stdout
+      expect { start(self) }.to output(daily).to_stdout
+    end
+  end
+
+  context 'movingaverages SYM -c spec/files/stockcruncher.yml' do
+    it 'Get the daily time serie for SYM.' do
+      expect { start(self) }.to output('').to_stdout
+    end
+  end
+
+  context 'quote NODATA -c spec/files/stockcruncher.yml' do
+    it 'Should not get any data and should fail.' do
+      expect { start(self) }.to raise_error(SystemExit)
     end
   end
 

data/spec/stockcruncher/stubs/servers_stubs.rb

@@ -5,17 +5,38 @@ require 'spec_helper'
 quote = 'symbol,open,high,low,price,volume,latestDay,previousClose,change,ch' \
         "angePercent\r\nSYM,100.0000,100.1000,99.9000,100.0000,4,2020-07-30," \
         "100.0000,0.0000,0.0000%\r\n"
+q_err = '{}'
+daily = "timestamp,open,high,low,close,volume\r\n2020-08-03,23.8500,24.6500," \
+        "23.7400,24.5500,3112972\r\n2020-07-31,24.0100,24.5100,23.5900,23.81" \
+        "00,5482485\r\n2020-07-30,24.4700,24.6600,23.5200,23.6600,6466738\r" \
+        "\n2020-07-29,24.7500,25.0300,24.0400,24.5600,4605804\r\n2020-07-28," \
+        "25.9900,26.0700,24.5100,24.7500,6904261\r\n"
+d_err = "{\n    \"Error Message\": \"Invalid API call.\"\n}"
+mvavg = '{"results":[{"statement_id":0,"series":[{"name":"daily","columns":[' \
+        '"time","close","change","changePercent","volume"],"values":[["2017-' \
+        '03-01T18:00:00Z",1,0,0,1],["2017-03-02T18:00:00Z",1,0,0,1],["2017-0' \
+        '3-03T18:00:00Z",1,0,0,1],["2017-03-04T18:00:00Z",1,0,0,1],["2017-03' \
+        '-05T18:00:00Z",1,0,0,1],["2017-03-06T18:00:00Z",1,0,0,1]]}]}]}'
 
 RSpec.configure do |config|
   config.before(:each) do
     # requests an API without extra arguments
+    stub_request(:get, 'https://www.alphavantage.co/query?' \
+      'function=GLOBAL_QUOTE&symbol=NODATA&apikey=demo&datatype=csv')
+      .to_return('status' => 200, 'body' => q_err, 'headers' => {})
     stub_request(:get, 'https://www.alphavantage.co/query?' \
       'function=GLOBAL_QUOTE&symbol=SYM&apikey=demo&datatype=csv')
       .to_return('status' => 200, 'body' => quote, 'headers' => {})
+    stub_request(:get, 'https://www.alphavantage.co/query?' \
+      'function=TIME_SERIES_DAILY&symbol=NODATA&apikey=demo' \
+      '&datatype=csv&outputsize=compact')
+      .to_return('status' => 200, 'body' => d_err, 'headers' => {})
     stub_request(:get, 'https://www.alphavantage.co/query?' \
       'function=TIME_SERIES_DAILY&symbol=SYM&apikey=demo' \
       '&datatype=csv&outputsize=compact')
-      .to_return('status' => 200, 'body' => '{}', 'headers' => {})
+      .to_return('status' => 200, 'body' => daily, 'headers' => {})
+    stub_request(:post, 'http://localhost:8086/query?db=test')
+      .to_return('status' => 204, 'body' => mvavg, 'headers' => {})
     stub_request(:post, 'http://localhost:8086/write?db=test')
       .to_return('status' => 204, 'body' => '', 'headers' => {})
   end

data/stockcruncher.gemspec

@@ -8,7 +8,7 @@ dev_deps = {
   'bundler' => '~> 1.17',
   'rspec' => '~> 3.9',
   'rake' => '~> 11.3',
-  'rubocop' => '0.88',
+  'rubocop' => '0.90',
   'webmock' => '~> 2.0.3',
   'simplecov' => '~> 0.18'
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stockcruncher
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Richard Delaplace
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-31 00:00:00.000000000 Z
+date: 2021-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: '0.90'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: '0.90'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -129,9 +129,13 @@ files:
 - lib/stockcruncher.rb
 - lib/stockcruncher/alphavantage.rb
 - lib/stockcruncher/cli.rb
+- lib/stockcruncher/config.rb
 - lib/stockcruncher/cruncher.rb
 - lib/stockcruncher/influxdb.rb
+- lib/stockcruncher/stats.rb
 - lib/stockcruncher/version.rb
+- spec/files/SYM.daily
+- spec/files/SYM.quote
 - spec/files/stockcruncher.yml
 - spec/spec_helper.rb
 - spec/stockcruncher/cli_spec.rb