stockcruncher 1.2.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e4729948dfb9635323240f73d947c2257b1ca4b8a6c2a065a812a8726f502599
-  data.tar.gz: 3686ad0bbb61ef2a3801a2a2c1b3258b543da3015c49b2c959570c9d7b0e8a3a
+  metadata.gz: d575ec081ba8f154b940a03a7eda7d79606bc78ff9c01132f2da78170b641367
+  data.tar.gz: 5949b3b5108d9d0d67907e49711ae9ecf8dbdfebfba669b9ed952f8592cfa95a
 SHA512:
-  metadata.gz: 24120c82f3e7cd9d622bcf1636ee528ea69d170fe0f60951b33b34754870a1b92837d5f2cbcd3f59ab4be85264195e7c2058036a557e8b486c8fe55cf81e5c18
-  data.tar.gz: e849aea61ae75175742d97611cffdc889aa3c95394aec2cc8ccbe93ed35bafc6958984fb0fc92b08190a41003c183b753ad287a51676990eee855d32f43bfceb
+  metadata.gz: 6f3dbd7ade6b0ad21e39842111ba21aa21ddf38ac0d984a24cf786a9bcd503c5427b85c5e5c500a0acc278fda9e83a08472144117148a8f624c49c724269884c
+  data.tar.gz: 98690a9871f4d68c2a199cdbcce134690a12b594aba5c4842f4c36346dcd143b8ea15079ff69e7996af19958a1feec5859340e0ea6657d3c649c24cc66007832

data/.gitlab-ci.yml

@@ -1,9 +1,11 @@
+
 image: ruby:2.4
 
+before_script:
+  - bundle install
+
 rubocop:
   stage: test
-  before_script:
-    - gem install --silent rubocop
   script:
     - rubocop
 
@@ -25,7 +27,6 @@ git_history:
 rspec:
   stage: test
   script:
-    - bundle install
     - rspec
   artifacts:
     paths:

data/.rubocop.yml

@@ -4,3 +4,4 @@ AllCops:
   DisplayStyleGuide: true
   DisplayCopNames: false
   TargetRubyVersion: 2.4
+  NewCops: disable

data/README.md

@@ -5,12 +5,14 @@
 [![Gem Version][gem-img]][gem]
 
 1. [Overview](#overview)
-2. [Description](#role-description)
-3. [Setup](#setup)
-4. [Usage](#usage)
-5. [Limitations](#limitations)
-6. [Development](#development)
-7. [Miscellaneous](#miscellaneous)
+1. [Description](#description)
+1. [Setup](#setup)
+1. [Usage](#usage)
+1. [Environment variables](#environment-variables)
+1. [Examples](#examples)
+1. [Limitations](#limitations)
+1. [Development](#development)
+1. [Miscellaneous](#miscellaneous)
 
 ## Overview
 
@@ -41,6 +43,17 @@ An interactive help is available with:
 
     $ stockcruncher help [subcommand]
 
+## Environment variables
+
+Parameters in /etc/stockcruncher/stockcrunucher.yml can be overloaded with
+environment variables.
+Environment variables should match SCR_<COMPONENTID>_<ITEM>.
+Component ID are as follow.
+- AlphaVantage: AV
+- InfluxDB: IDB
+Items are upcase keys. See template /etc/stockcruncher/stockcrunucher.yml for
+reference.
+
 ## Examples
 
 To get daily time serie data of a symbol:
@@ -51,6 +64,10 @@ To get last day endpoint data of a symbol:
 
     $ stockcruncher quote AAPL
 
+To override a parameter with environment variable:
+
+    $ SCR_IDB_HOST=192.168.0.80; stockcruncher quote AAPL
+
 ## Limitations
 
 Data are currently scraped from AlphaVantage API.

data/lib/stockcruncher/alphavantage.rb

@@ -39,7 +39,7 @@ module StockCruncher
     # Main method to crunch data.
     def crunch_daily(symbol, fullsize)
       url = API_URL + parameters(symbol, 'TIME_SERIES_DAILY')
-      url += '&datatype=csv&outputsize=' + (fullsize ? 'full' : 'compact')
+      url += "&datatype=csv&outputsize=#{fullsize ? 'full' : 'compact'}"
       res = request(url)
       transform_daily(res.body)
     end
@@ -63,9 +63,9 @@ module StockCruncher
 
     # Set parameters of api call
     def parameters(symbol, serie)
-      p = 'function=' + serie
-      p += '&symbol=' + symbol
-      p += '&apikey=' + @config[self.class.name.split('::').last]['apikey']
+      p = "function=#{serie}"
+      p += "&symbol=#{symbol}"
+      p += "&apikey=#{@config[self.class.name.split('::').last]['apikey']}"
       p
     end
 
@@ -87,8 +87,7 @@ module StockCruncher
       raise StandardError, 'No data' if rawdata.match?(/Error Message/)
 
       values = prepare_daily_timeserie(rawdata)
-      values = calculate_missing_data(values)
-      values
+      calculate_missing_data(values)
     end
 
     # Method to transform quote result to hash

data/lib/stockcruncher/cli.rb

@@ -45,23 +45,55 @@ module StockCruncher
       default: false,
       desc: 'Full data size.'
     )
+    option(
+      :catchup,
+      aliases: ['-u'],
+      type: :boolean,
+      default: false,
+      desc: 'Catch up the missing data only.'
+    )
     def daily(symbol)
       opts = options.dup
       config = YAML.load_file(opts['config'])
       cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
       data = cruncher.crunch_daily(symbol, opts['full'])
-      StockCruncher::InfluxDB.new(config).export_history(symbol, data)
+      influx = StockCruncher::InfluxDB.new(config)
+      influx.export_history(symbol, data, opts['catchup'])
       puts JSON.pretty_generate(data) unless opts['quiet']
     end
 
+    desc('movingaverages SYMBOL [options]',
+         'Calculate and export moving averages for requested symbol.')
+    option(
+      :all,
+      aliases: ['-a'],
+      type: :boolean,
+      default: false,
+      desc: 'Recalculate all MA historical values.'
+    )
+    option(
+      :catchup,
+      aliases: ['-u'],
+      type: :boolean,
+      default: false,
+      desc: 'Catch up the missing data only.'
+    )
+    def movingaverages(symbol)
+      opts = options.dup
+      config = YAML.load_file(opts['config'])
+      influx = StockCruncher::InfluxDB.new(config)
+      influx.moving_averages(symbol, opts['all'], opts['catchup'])
+    end
+
     desc('quote SYMBOL [options]',
          'Crunch SYMBOL stock market data for last day quote.')
     def quote(symbol)
       opts = options.dup
-      config = YAML.load_file(opts['config'])
+      config = StockCruncher::Config.load(opts['config'])
       cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
       data = cruncher.crunch_quote(symbol)
-      StockCruncher::InfluxDB.new(config).export_last_day(data)
+      influx = StockCruncher::InfluxDB.new(config)
+      influx.export_last_day(data)
       puts JSON.pretty_generate(data) unless opts['quiet']
     end
   end

data/lib/stockcruncher/config.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/ruby
+# frozen_string_literal: true
+
+require 'yaml'
+
+module StockCruncher
+  # this is a module to load configuration file and environment variable
+  module Config
+    module_function
+
+    # Load config file and override with env variables
+    def load(file)
+      config = YAML.load_file(file)
+      overload_alphavantage(config)
+      overload_influxdb(config)
+    end
+
+    def overload(config, prefix, component, items)
+      items.each do |key|
+        var = "#{prefix}#{key.upcase}"
+        config[component][key] = ENV[var] unless ENV[var].nil?
+      end
+      config
+    end
+
+    def overload_alphavantage(config)
+      prefix = 'SCR_AV_'
+      component = 'AlphaVantage'
+      items = %w[apikey]
+      overload(config, prefix, component, items)
+    end
+
+    def overload_influxdb(config)
+      prefix = 'SCR_IDB_'
+      component = 'InfluxDB'
+      items = %w[scheme host port user password dbname]
+      overload(config, prefix, component, items)
+    end
+  end
+end

data/lib/stockcruncher/influxdb.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require 'date'
+require 'json'
 require 'net/http'
 
 module StockCruncher
@@ -13,29 +14,80 @@ module StockCruncher
       @insecure = insecure
     end
 
-    # Method to export latest data to database
-    def export_last_day(values)
-      tags = { 'symbol' => values.delete('symbol') }
-      date = values.delete('latestDay')
-      write('daily', tags, values, date)
+    def get_daily_values(symbol, fullsize)
+      values = %w[close change changePercent volume]
+      data = query('daily', symbol, values, fullsize)
+      data['columns'].zip(data['values'].transpose).to_h
+    end
+
+    def get_ma_values(symbol, fullsize)
+      values = %w[ema200]
+      data = query('ema', symbol, values, fullsize)
+      data['columns'].zip(data['values'].transpose).to_h
+    end
+
+    # Method to calculate moving averages based on last day values
+    def moving_averages(symbol, fullsize, catchup)
+      series = get_daily_values(symbol, fullsize)
+      mas = catchup ? get_ma_values(symbol, fullsize) : {}
+      tags = create_tags(symbol)
+      series['close'].each_index do |i|
+        date = series['time'][i]
+        serie = series['close'][i, 201]
+        weights = series['volume'][i, 201]
+        write_moving_averages(tags, serie, weights, date) unless mas.key? date
+        break unless fullsize
+      end
+    end
+
+    # Method to create tags hash containing only symbol
+    def create_tags(symbol)
+      { 'symbol' => symbol }
     end
 
     # Method to export historical data to database
-    def export_history(symbol, timeseries)
-      tags = { 'symbol' => symbol }
+    def export_history(symbol, timeseries, catchup)
+      tags = create_tags(symbol)
+      if catchup
+        series = get_daily_values(symbol, fullsize)
+        series['time'].each { |date| timeseries.delete(date) }
+      end
       timeseries.each_pair do |date, values|
         write('daily', tags, values, date)
       end
     end
 
+    # Method to export latest data to database
+    def export_last_day(values)
+      tags = create_tags(values.delete('symbol'))
+      date = values.delete('latestDay')
+      write('daily', tags, values, date)
+    end
+
     # Method to format and array of values into comma separated string
     def format_values(values)
-      string = ''
-      values.each_pair do |k, v|
-        string += "#{k}=#{v}"
-        string += ',' unless k == values.keys.last
-      end
-      string
+      values.map { |k, v| "#{k}=#{v}" }.join(',')
+    end
+
+    # Method to calculate all statistics
+    def write_moving_averages(tags, serie, weights, date)
+      write('ema', tags, StockCruncher::Stats.list_ema(serie), date)
+      write('lwma', tags, StockCruncher::Stats.list_lwma(serie), date)
+      write('sma', tags, StockCruncher::Stats.list_sma(serie), date)
+      write('vwma', tags, StockCruncher::Stats.list_vwma(serie, weights), date)
+    end
+
+    # Method to query data in bucket
+    def query(name, symbol, values, full)
+      url = "#{@cfg['scheme']}://#{@cfg['host']}:#{@cfg['port']}/query?" \
+            "db=#{@cfg['dbname']}"
+      size = full ? '' : 'LIMIT 201'
+      body = "q=SELECT #{values.join(',')} FROM #{name} " \
+             "WHERE symbol = '#{symbol}' ORDER BY time DESC #{size}"
+      data = JSON.parse(request(url, body).body)['results'][0]['series']
+      raise StandardError, 'No data' if data.nil?
+
+      data[0]
     end
 
     # Method to send http post request
@@ -54,7 +106,7 @@ module StockCruncher
     def write(name, tags, values, date)
       url = "#{@cfg['scheme']}://#{@cfg['host']}:#{@cfg['port']}/write?" \
             "db=#{@cfg['dbname']}"
-      timestamp = DateTime.parse(date + 'T18:00:00').strftime('%s%N')
+      timestamp = DateTime.parse("#{date}T18:00:00").strftime('%s%N')
       body = "#{name},#{format_values(tags)} #{format_values(values)} " \
              "#{timestamp}"
       request(url, body)

data/lib/stockcruncher/stats.rb

@@ -0,0 +1,74 @@
+#!/usr/bin/ruby
+# frozen_string_literal: true
+
+module StockCruncher
+  # this is a module with various statistic calculation methods
+  module Stats
+    extend self
+
+    RANGES = [5, 10, 20, 30, 50, 100, 200].freeze
+
+    # Calculate multiple range of exponential moving average
+    def list_ema(values)
+      h = {}
+      RANGES.each do |n|
+        next if values.size < n + 1
+
+        h["ema#{n}"] = ema(values[0, n + 1])
+      end
+      h
+    end
+
+    # Calculate multiple range of linearly weighted moving average
+    def list_lwma(values)
+      h = {}
+      RANGES.each do |n|
+        next if values.size < n
+
+        weights = (1..n).to_a.reverse
+        h["lwma#{n}"] = sma(values[0, n], weights)
+      end
+      h
+    end
+
+    # Calculate multiple range of simple moving average
+    def list_sma(values)
+      h = {}
+      RANGES.each do |n|
+        next if values.size < n
+
+        h["sma#{n}"] = sma(values[0, n])
+      end
+      h
+    end
+
+    # Calculate multiple range of volume weighted moving average
+    def list_vwma(values, volumes)
+      h = {}
+      RANGES.each do |n|
+        next if values.size < n
+
+        h["vwma#{n}"] = sma(values[0, n], volumes[0, n])
+      end
+      h
+    end
+
+    private
+
+    # Calculate exponential moving average
+    def ema(array, factor = 2, weights = nil)
+      f = factor.to_f / array.size
+      n = array.size - 1
+      tsma = sma(array[0, n], weights)
+      ysma = sma(array[1, n], weights)
+      (tsma * f + ysma * (1 - f)).round(4)
+    end
+
+    # Calculate simple moving average
+    def sma(array, weights = nil)
+      factor = weights.nil? ? Array.new(array.size, 1) : weights
+      dividend = array.each_with_index.map { |v, i| v * factor[i] }
+      (dividend.sum.to_f / factor.sum).round(4)
+    end
+  end
+end

data/lib/stockcruncher/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module StockCruncher
-  VERSION = '1.2.1'
+  VERSION = '1.4.0'
 end

data/spec/stockcruncher/cli_spec.rb

@@ -26,6 +26,12 @@ describe StockCruncher::CLI do # rubocop:disable Metrics/BlockLength
     end
   end
 
+  context 'movingaverages SYM -c spec/files/stockcruncher.yml' do
+    it 'Get the daily time serie for SYM.' do
+      expect { start(self) }.to output('').to_stdout
+    end
+  end
+
   context 'quote NODATA -c spec/files/stockcruncher.yml' do
     it 'Should not get any data and should fail.' do
       expect { start(self) }.to raise_error(SystemExit)

data/spec/stockcruncher/stubs/servers_stubs.rb

@@ -12,6 +12,11 @@ daily = "timestamp,open,high,low,close,volume\r\n2020-08-03,23.8500,24.6500," \
         "\n2020-07-29,24.7500,25.0300,24.0400,24.5600,4605804\r\n2020-07-28," \
         "25.9900,26.0700,24.5100,24.7500,6904261\r\n"
 d_err = "{\n    \"Error Message\": \"Invalid API call.\"\n}"
+mvavg = '{"results":[{"statement_id":0,"series":[{"name":"daily","columns":[' \
+        '"time","close","change","changePercent","volume"],"values":[["2017-' \
+        '03-01T18:00:00Z",1,0,0,1],["2017-03-02T18:00:00Z",1,0,0,1],["2017-0' \
+        '3-03T18:00:00Z",1,0,0,1],["2017-03-04T18:00:00Z",1,0,0,1],["2017-03' \
+        '-05T18:00:00Z",1,0,0,1],["2017-03-06T18:00:00Z",1,0,0,1]]}]}]}'
 
 RSpec.configure do |config|
   config.before(:each) do
@@ -30,6 +35,8 @@ RSpec.configure do |config|
       'function=TIME_SERIES_DAILY&symbol=SYM&apikey=demo' \
       '&datatype=csv&outputsize=compact')
       .to_return('status' => 200, 'body' => daily, 'headers' => {})
+    stub_request(:post, 'http://localhost:8086/query?db=test')
+      .to_return('status' => 204, 'body' => mvavg, 'headers' => {})
     stub_request(:post, 'http://localhost:8086/write?db=test')
       .to_return('status' => 204, 'body' => '', 'headers' => {})
   end

data/stockcruncher.gemspec

@@ -8,7 +8,7 @@ dev_deps = {
   'bundler' => '~> 1.17',
   'rspec' => '~> 3.9',
   'rake' => '~> 11.3',
-  'rubocop' => '0.88',
+  'rubocop' => '0.90',
   'webmock' => '~> 2.0.3',
   'simplecov' => '~> 0.18'
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stockcruncher
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Richard Delaplace
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-12 00:00:00.000000000 Z
+date: 2021-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: '0.90'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: '0.88'
+        version: '0.90'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -129,8 +129,10 @@ files:
 - lib/stockcruncher.rb
 - lib/stockcruncher/alphavantage.rb
 - lib/stockcruncher/cli.rb
+- lib/stockcruncher/config.rb
 - lib/stockcruncher/cruncher.rb
 - lib/stockcruncher/influxdb.rb
+- lib/stockcruncher/stats.rb
 - lib/stockcruncher/version.rb
 - spec/files/SYM.daily
 - spec/files/SYM.quote