stockcruncher 1.0.1 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82cc9e3595d2f91bded92cbbf001fc8fb76ff3bd0d4604361c24f78c3ef6da01
-  data.tar.gz: 510435224f9e7362f51f692c3b6634d7372adae0db724d9d9545f3d869b53bfa
+  metadata.gz: e4729948dfb9635323240f73d947c2257b1ca4b8a6c2a065a812a8726f502599
+  data.tar.gz: 3686ad0bbb61ef2a3801a2a2c1b3258b543da3015c49b2c959570c9d7b0e8a3a
 SHA512:
-  metadata.gz: 4a4e42a9068eabbbd9b5da44212d39c98aebce85c2b36add25f5da51853bbb00117cd9d6783d5f3cb2b35789eba7c917fe9020198de54e9bd9c45eff49221a0c
-  data.tar.gz: 191d02a244a4fe2b575a7252824c2c510140072b56f1ea9791ddd62b94da5fda784046f246a812c4feb141e66196c711bfdfa71c951f0c8cac33409c4dacea31
+  metadata.gz: 24120c82f3e7cd9d622bcf1636ee528ea69d170fe0f60951b33b34754870a1b92837d5f2cbcd3f59ab4be85264195e7c2058036a557e8b486c8fe55cf81e5c18
+  data.tar.gz: e849aea61ae75175742d97611cffdc889aa3c95394aec2cc8ccbe93ed35bafc6958984fb0fc92b08190a41003c183b753ad287a51676990eee855d32f43bfceb

data/CHANGELOG

@@ -5,3 +5,26 @@ Changelog
 -----
 
 - Initial version
+
+1.1.0
+-----
+
+- Add InfluxDB as database.
+- WIP for daily time serie.
+
+1.1.1
+-----
+
+- Rescue error on no data
+
+1.2.0
+-----
+
+- daily subcommand now recalculates missing data
+- daily subcommand also writes data to database
+
+1.2.1
+-----
+
+- Refactor to preprocess data in source class
+- Refactor to print pretty json instead of comma separated striing

data/README.md

@@ -25,29 +25,38 @@ to write the data in a database for later calculation and use.
 
     $ gem install stockcruncher
     $ mkdir /etc/stockcruncher && cd /etc/stockcruncher
-    $ echo $'AlphaVantage:\n  apikey: CHANGEME' > stockcruncher.yml
+    $ echo 'AlphaVantage:' > stockcruncher.yml
+    $ echo '  apikey: CHANGEME' >> stockcruncher.yml
+    $ echo 'InfluxDB:' >> stockcruncher.yml
+    $ echo '  scheme: http' >> stockcruncher.yml
+    $ echo '  host: localhost' >> stockcruncher.yml
+    $ echo '  port: 8086' >> stockcruncher.yml
+    $ echo '  user: CHANGEMEAGAIN' >> stockcruncher.yml
+    $ echo '  password: CHANGEMETOO' >> stockcruncher.yml
+    $ echo '  dbname: stock' >> stockcruncher.yml
 
 ## Usage
 
 An interactive help is available with:
 
-    $ stockcruncher help
+    $ stockcruncher help [subcommand]
 
 ## Examples
 
 To get daily time serie data of a symbol:
 
-    $ stockcruncher crunch AAPL daily
+    $ stockcruncher daily AAPL
 
 To get last day endpoint data of a symbol:
 
-    $ stockcruncher crunch AAPL quote_endpoint
+    $ stockcruncher quote AAPL
 
 ## Limitations
 
 Data are currently scraped from AlphaVantage API.
 More source could be great especially because AlphaVantage doesn't provide EU
 intraday data.
+InfluxDB is used as database to keep time series data.
 
 ## Development
 

data/lib/stockcruncher.rb

@@ -10,9 +10,9 @@ module StockCruncher
 
     def start(args = ARGV)
       StockCruncher::CLI.start(args)
-    rescue NotImplementedError => e
+    rescue StandardError => e
       warn e.message
-      exit(3)
+      exit 1
     end
   end
 end

data/lib/stockcruncher/alphavantage.rb

@@ -7,35 +7,98 @@ module StockCruncher
   # This is an data cruncher class for AlphaVantage API.
   class AlphaVantage < Cruncher
     API_URL = 'https://www.alphavantage.co/query?'
-    SERIE = {
-      'daily' => 'TIME_SERIES_DAILY',
-      'quote_endpoint' => 'GLOBAL_QUOTE'
-    }.freeze
+
+    # Method to calculate missing data (previousClose, change, changePercent)
+    def calculate_missing_data(hash)
+      keys = hash.keys
+      hash.each_with_index do |(date, v), index|
+        prevday = keys[index + 1]
+        next if prevday.nil?
+
+        prevclose = hash[prevday]['close']
+        hash[date] = v.merge(generate_missing_data(v['close'], prevclose))
+      end
+      hash
+    end
+
+    # Method to calculate change difference
+    def change(value, base)
+      (value - base).round(4).to_s
+    end
+
+    # Method to calculate percentage of change
+    def change_percent(value, base)
+      ((value / base - 1) * 100).round(4).to_s
+    end
+
+    # Method to create a new hash from two arrays of keys and values
+    def create_hash(descriptions, values)
+      descriptions.split(',').zip(values.split(',')).to_h
+    end
 
     # Main method to crunch data.
-    def crunch(symbol, serie, opts)
-      err_msg = "#{serie} does not exist."
-      raise NotImplementedError, err_msg unless SERIE.key?(serie)
+    def crunch_daily(symbol, fullsize)
+      url = API_URL + parameters(symbol, 'TIME_SERIES_DAILY')
+      url += '&datatype=csv&outputsize=' + (fullsize ? 'full' : 'compact')
+      res = request(url)
+      transform_daily(res.body)
+    end
 
-      url = API_URL + parameters(symbol, serie) + options(serie, opts)
+    # Main method to crunch data.
+    def crunch_quote(symbol)
+      url = API_URL + parameters(symbol, 'GLOBAL_QUOTE')
+      url += '&datatype=csv'
       res = request(url)
-      puts res.body
+      transform_quote(res.body)
     end
 
-    def options(serie, opts)
-      o = '&datatype=' + (opts['json'] ? 'json' : 'csv')
-      if serie == 'daily'
-        o += '&outputsize='
-        o += opts['full'] ? 'full' : 'compact'
-      end
-      o
+    # Method to generate missing data
+    def generate_missing_data(current, previous)
+      {
+        'previousClose' => previous,
+        'change' => change(current.to_f, previous.to_f),
+        'changePercent' => change_percent(current.to_f, previous.to_f)
+      }
     end
 
+    # Set parameters of api call
     def parameters(symbol, serie)
-      p = 'function=' + SERIE[serie]
+      p = 'function=' + serie
       p += '&symbol=' + symbol
       p += '&apikey=' + @config[self.class.name.split('::').last]['apikey']
       p
     end
+
+    # Method to transform raw data to constructed hash
+    def prepare_daily_timeserie(data)
+      lines = data.split("\r\n")
+      desc = lines.shift.split(',').drop(1)
+      hash = {}
+      lines.each do |line|
+        values = line.split(',')
+        date = values.shift
+        hash[date] = desc.zip(values).to_h
+      end
+      hash
+    end
+
+    # Method to transform daily result to nested hash
+    def transform_daily(rawdata)
+      raise StandardError, 'No data' if rawdata.match?(/Error Message/)
+
+      values = prepare_daily_timeserie(rawdata)
+      values = calculate_missing_data(values)
+      values
+    end
+
+    # Method to transform quote result to hash
+    def transform_quote(rawdata)
+      raise StandardError, 'No data' if rawdata.match?(/{}/)
+
+      values = create_hash(*rawdata.split("\r\n"))
+      values['close'] = values.delete('price')
+      values['changePercent'] = values['changePercent'].delete('%')
+      values
+    end
   end
 end

data/lib/stockcruncher/cli.rb

@@ -1,20 +1,14 @@
 #!/usr/bin/ruby
 # frozen_string_literal: true
 
+require 'json'
 require 'thor'
+require 'yaml'
 
 module StockCruncher
   # Simple CLI for StockCruncher
   class CLI < Thor
-    desc 'version', 'Print stockcruncher current version'
-    def version
-      puts "StockCruncher version #{StockCruncher::VERSION}"
-    end
-
-    desc('crunch SYMBOL TIMESERIES [options]',
-         'Crunch SYMBOL stock market data for time series.' \
-         'Possible timeseries: daily, quote_endpoint.')
-    option(
+    class_option(
       :config,
       aliases: ['-c'],
       type: :string,
@@ -22,13 +16,28 @@ module StockCruncher
       desc: 'Yaml formatted config file to load ' \
             '(default to /etc/stockcruncher/stockcruncher.yml).'
     )
-    option(
-      :json,
-      aliases: ['-j'],
+    class_option(
+      :insecure,
+      aliases: ['-k'],
+      type: :boolean,
+      default: false,
+      desc: 'Ignore SSL certificate (default to false).'
+    )
+    class_option(
+      :quiet,
+      aliases: ['-q'],
       type: :boolean,
       default: false,
-      desc: 'Json format data (default to csv).'
+      desc: 'Run silently (default to false).'
     )
+
+    desc 'version', 'Print stockcruncher current version'
+    def version
+      puts "StockCruncher version #{StockCruncher::VERSION}"
+    end
+
+    desc('daily SYMBOL [options]',
+         'Crunch SYMBOL stock market data for daily time series.')
     option(
       :full,
       aliases: ['-f'],
@@ -36,10 +45,24 @@ module StockCruncher
       default: false,
       desc: 'Full data size.'
     )
-    def crunch(symbol, timeserie)
+    def daily(symbol)
+      opts = options.dup
+      config = YAML.load_file(opts['config'])
+      cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
+      data = cruncher.crunch_daily(symbol, opts['full'])
+      StockCruncher::InfluxDB.new(config).export_history(symbol, data)
+      puts JSON.pretty_generate(data) unless opts['quiet']
+    end
+
+    desc('quote SYMBOL [options]',
+         'Crunch SYMBOL stock market data for last day quote.')
+    def quote(symbol)
       opts = options.dup
-      cruncher = StockCruncher::AlphaVantage.new(opts['config'])
-      cruncher.crunch(symbol, timeserie, opts)
+      config = YAML.load_file(opts['config'])
+      cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
+      data = cruncher.crunch_quote(symbol)
+      StockCruncher::InfluxDB.new(config).export_last_day(data)
+      puts JSON.pretty_generate(data) unless opts['quiet']
     end
   end
 end

data/lib/stockcruncher/cruncher.rb

@@ -2,27 +2,22 @@
 # frozen_string_literal: true
 
 require 'net/http'
-require 'yaml'
 
 module StockCruncher
   # This is an data cruncher abstract class.
   class Cruncher
-    # Class abstract constructor method
-    def initialize(file)
-      @config = load_conf(file)
-    end
-
-    # Method to load configurations described in config_file.
-    def load_conf(file)
-      YAML.load_file(file)
+    # Class constructor method
+    def initialize(config, insecure = false)
+      @config = config
+      @insecure = insecure
     end
 
     # Method to send http get request
-    def request(url, insecure = false)
+    def request(url)
       uri = URI.parse(url)
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = uri.scheme.eql?('https')
-      http.verify_mode = OpenSSL::SSL::VERIFY_NONE if insecure
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE if @insecure
       req = Net::HTTP::Get.new(uri.request_uri)
       http.request(req)
     end

data/lib/stockcruncher/influxdb.rb

@@ -0,0 +1,63 @@
+#!/usr/bin/ruby
+# frozen_string_literal: true
+
+require 'date'
+require 'net/http'
+
+module StockCruncher
+  # this is a class to write time series to database
+  class InfluxDB
+    # Class constructor method
+    def initialize(config, insecure = false)
+      @cfg = config[self.class.name.split('::').last]
+      @insecure = insecure
+    end
+
+    # Method to export latest data to database
+    def export_last_day(values)
+      tags = { 'symbol' => values.delete('symbol') }
+      date = values.delete('latestDay')
+      write('daily', tags, values, date)
+    end
+
+    # Method to export historical data to database
+    def export_history(symbol, timeseries)
+      tags = { 'symbol' => symbol }
+      timeseries.each_pair do |date, values|
+        write('daily', tags, values, date)
+      end
+    end
+
+    # Method to format and array of values into comma separated string
+    def format_values(values)
+      string = ''
+      values.each_pair do |k, v|
+        string += "#{k}=#{v}"
+        string += ',' unless k == values.keys.last
+      end
+      string
+    end
+
+    # Method to send http post request
+    def request(url, body)
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme.eql?('https')
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE if @insecure
+      req = Net::HTTP::Post.new(uri.request_uri)
+      req.basic_auth(@cfg['user'], @cfg['password'])
+      req.body = body
+      http.request(req)
+    end
+
+    # Method to write data in bucket
+    def write(name, tags, values, date)
+      url = "#{@cfg['scheme']}://#{@cfg['host']}:#{@cfg['port']}/write?" \
+            "db=#{@cfg['dbname']}"
+      timestamp = DateTime.parse(date + 'T18:00:00').strftime('%s%N')
+      body = "#{name},#{format_values(tags)} #{format_values(values)} " \
+             "#{timestamp}"
+      request(url, body)
+    end
+  end
+end

data/lib/stockcruncher/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module StockCruncher
-  VERSION = '1.0.1'
+  VERSION = '1.2.1'
 end

data/spec/files/SYM.daily

@@ -0,0 +1,49 @@
+{
+  "2020-08-03": {
+    "open": "23.8500",
+    "high": "24.6500",
+    "low": "23.7400",
+    "close": "24.5500",
+    "volume": "3112972",
+    "previousClose": "23.8100",
+    "change": "0.74",
+    "changePercent": "3.1079"
+  },
+  "2020-07-31": {
+    "open": "24.0100",
+    "high": "24.5100",
+    "low": "23.5900",
+    "close": "23.8100",
+    "volume": "5482485",
+    "previousClose": "23.6600",
+    "change": "0.15",
+    "changePercent": "0.634"
+  },
+  "2020-07-30": {
+    "open": "24.4700",
+    "high": "24.6600",
+    "low": "23.5200",
+    "close": "23.6600",
+    "volume": "6466738",
+    "previousClose": "24.5600",
+    "change": "-0.9",
+    "changePercent": "-3.6645"
+  },
+  "2020-07-29": {
+    "open": "24.7500",
+    "high": "25.0300",
+    "low": "24.0400",
+    "close": "24.5600",
+    "volume": "4605804",
+    "previousClose": "24.7500",
+    "change": "-0.19",
+    "changePercent": "-0.7677"
+  },
+  "2020-07-28": {
+    "open": "25.9900",
+    "high": "26.0700",
+    "low": "24.5100",
+    "close": "24.7500",
+    "volume": "6904261"
+  }
+}

data/spec/files/SYM.quote

@@ -0,0 +1,10 @@
+{
+  "open": "100.0000",
+  "high": "100.1000",
+  "low": "99.9000",
+  "volume": "4",
+  "previousClose": "100.0000",
+  "change": "0.0000",
+  "changePercent": "0.0000",
+  "close": "100.0000"
+}

data/spec/files/stockcruncher.yml

@@ -1,2 +1,9 @@
 AlphaVantage:
   apikey: demo
+InfluxDB:
+  scheme: http
+  host: localhost
+  port: 8086
+  user: testuser
+  password: testpassword
+  dbname: test

data/spec/stockcruncher/cli_spec.rb

@@ -1,8 +1,12 @@
 # frozen_string_literal: true
 
+require 'json'
 require 'spec_helper'
 
-describe StockCruncher::CLI do
+daily = File.read('spec/files/SYM.daily')
+quote = File.read('spec/files/SYM.quote')
+
+describe StockCruncher::CLI do # rubocop:disable Metrics/BlockLength
   context 'version' do
     it 'prints the version.' do
       out = "StockCruncher version #{StockCruncher::VERSION}\n"
@@ -10,9 +14,27 @@ describe StockCruncher::CLI do
     end
   end
 
-  context 'crunch SYM quote_endpoint -c spec/files/stockcruncher.yml' do
+  context 'daily NODATA -c spec/files/stockcruncher.yml' do
+    it 'Should not get any data and should fail.' do
+      expect { start(self) }.to raise_error(SystemExit)
+    end
+  end
+
+  context 'daily SYM -c spec/files/stockcruncher.yml' do
+    it 'Get the daily time serie for SYM.' do
+      expect { start(self) }.to output(daily).to_stdout
+    end
+  end
+
+  context 'quote NODATA -c spec/files/stockcruncher.yml' do
+    it 'Should not get any data and should fail.' do
+      expect { start(self) }.to raise_error(SystemExit)
+    end
+  end
+
+  context 'quote SYM -c spec/files/stockcruncher.yml' do
     it 'Get the quote for SYM.' do
-      expect { start(self) }.to output("{}\n").to_stdout
+      expect { start(self) }.to output(quote).to_stdout
     end
   end
 

data/spec/stockcruncher/stubs/servers_stubs.rb

@@ -2,15 +2,35 @@
 
 require 'spec_helper'
 
+quote = 'symbol,open,high,low,price,volume,latestDay,previousClose,change,ch' \
+        "angePercent\r\nSYM,100.0000,100.1000,99.9000,100.0000,4,2020-07-30," \
+        "100.0000,0.0000,0.0000%\r\n"
+q_err = '{}'
+daily = "timestamp,open,high,low,close,volume\r\n2020-08-03,23.8500,24.6500," \
+        "23.7400,24.5500,3112972\r\n2020-07-31,24.0100,24.5100,23.5900,23.81" \
+        "00,5482485\r\n2020-07-30,24.4700,24.6600,23.5200,23.6600,6466738\r" \
+        "\n2020-07-29,24.7500,25.0300,24.0400,24.5600,4605804\r\n2020-07-28," \
+        "25.9900,26.0700,24.5100,24.7500,6904261\r\n"
+d_err = "{\n    \"Error Message\": \"Invalid API call.\"\n}"
+
 RSpec.configure do |config|
   config.before(:each) do
     # requests an API without extra arguments
+    stub_request(:get, 'https://www.alphavantage.co/query?' \
+      'function=GLOBAL_QUOTE&symbol=NODATA&apikey=demo&datatype=csv')
+      .to_return('status' => 200, 'body' => q_err, 'headers' => {})
     stub_request(:get, 'https://www.alphavantage.co/query?' \
       'function=GLOBAL_QUOTE&symbol=SYM&apikey=demo&datatype=csv')
-      .to_return('status' => 200, 'body' => '{}', 'headers' => {})
+      .to_return('status' => 200, 'body' => quote, 'headers' => {})
+    stub_request(:get, 'https://www.alphavantage.co/query?' \
+      'function=TIME_SERIES_DAILY&symbol=NODATA&apikey=demo' \
+      '&datatype=csv&outputsize=compact')
+      .to_return('status' => 200, 'body' => d_err, 'headers' => {})
     stub_request(:get, 'https://www.alphavantage.co/query?' \
       'function=TIME_SERIES_DAILY&symbol=SYM&apikey=demo' \
       '&datatype=csv&outputsize=compact')
-      .to_return('status' => 200, 'body' => '{}', 'headers' => {})
+      .to_return('status' => 200, 'body' => daily, 'headers' => {})
+    stub_request(:post, 'http://localhost:8086/write?db=test')
+      .to_return('status' => 204, 'body' => '', 'headers' => {})
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stockcruncher
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.2.1
 platform: ruby
 authors:
 - Richard Delaplace
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-27 00:00:00.000000000 Z
+date: 2020-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -130,10 +130,12 @@ files:
 - lib/stockcruncher/alphavantage.rb
 - lib/stockcruncher/cli.rb
 - lib/stockcruncher/cruncher.rb
+- lib/stockcruncher/influxdb.rb
 - lib/stockcruncher/version.rb
+- spec/files/SYM.daily
+- spec/files/SYM.quote
 - spec/files/stockcruncher.yml
 - spec/spec_helper.rb
-- spec/stockcruncher/alphavantage_spec.rb
 - spec/stockcruncher/cli_spec.rb
 - spec/stockcruncher/stubs/servers_stubs.rb
 - spec/stockcruncher_spec.rb

data/spec/stockcruncher/alphavantage_spec.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-describe StockCruncher::AlphaVantage do
-  context 'crunch SYM daily -c spec/files/stockcruncher.yml' do
-    it 'requests a source and get the result.' do
-      expect { start(self) }.to output("{}\n").to_stdout
-    end
-  end
-  context 'crunch SYM weekly -c spec/files/stockcruncher.yml' do
-    it 'requests a not implemented time serie.' do
-      expect { start(self) }.to raise_error(SystemExit)
-    end
-  end
-end