stockcruncher 1.0.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 58f00f855043839b976fb6bd52a7d6044187fa24dd97c5d50cfb90b637bfd26a
-  data.tar.gz: d2f59a0979b3f1da3c1ba8b491efe6b6588b5af26d1b17f69a3ab6455f3aebbb
+  metadata.gz: 7bd12e5a6e5db6f83f239c1b6da9aca85931048dd427778ffb96f043e7334596
+  data.tar.gz: 48fe8efa593f30fe39c294725d7090c89a4a14413eaa59a1d6741c817fc02d64
 SHA512:
-  metadata.gz: 3eaeeb37d7f845ae3c163053d455650f519c29549c5750c70a443efe3e993d4ebc002595f7d3e5b5b093aee1d790ac026a6037c86717319d5170cf60bcf42e6a
-  data.tar.gz: d0cd2ee27442ccb1f814958759e7c826eef5bbef468b060ba39674d5d8560d010f82b843e75b79f4401f5e45cea66f72b09a8a0dd446c0fca49f5b73621c6e5b
+  metadata.gz: ad4ed352a8ed30d41edbe9c4bac03ba721c9af42837f9e94f9c14df2175013d4ae7a09e29c7076e034cf375abaf222ce04c251a4b8fde31ce10ae29936c18894
+  data.tar.gz: 06206e1a91e23706c6a83c02357b31064124a21e522f3144d4ec93d4d4e7863e82e38cf4b561eedb662e59ea801120ee05f8cd3fff1b414da5ea5a069dd337ff

data/CHANGELOG

@@ -5,3 +5,20 @@ Changelog
 -----
 
 - Initial version
+
+1.1.0
+-----
+
+- Add InfluxDB as database.
+- WIP for daily time serie.
+
+1.1.1
+-----
+
+- Rescue error on no data
+
+1.2.0
+-----
+
+- daily subcommand now recalculates missing data
+- daily subcommand also writes data to database

data/README.md

@@ -25,29 +25,38 @@ to write the data in a database for later calculation and use.
 
     $ gem install stockcruncher
     $ mkdir /etc/stockcruncher && cd /etc/stockcruncher
-    $ echo $'AlphaVantage:\n  apikey: CHANGEME' > stockcruncher.yml
+    $ echo 'AlphaVantage:' > stockcruncher.yml
+    $ echo '  apikey: CHANGEME' >> stockcruncher.yml
+    $ echo 'InfluxDB:' >> stockcruncher.yml
+    $ echo '  scheme: http' >> stockcruncher.yml
+    $ echo '  host: localhost' >> stockcruncher.yml
+    $ echo '  port: 8086' >> stockcruncher.yml
+    $ echo '  user: CHANGEMEAGAIN' >> stockcruncher.yml
+    $ echo '  password: CHANGEMETOO' >> stockcruncher.yml
+    $ echo '  dbname: stock' >> stockcruncher.yml
 
 ## Usage
 
 An interactive help is available with:
 
-    $ stockcruncher help
+    $ stockcruncher help [subcommand]
 
 ## Examples
 
 To get daily time serie data of a symbol:
 
-    $ stockcruncher crunch AAPL daily
+    $ stockcruncher daily AAPL
 
 To get last day endpoint data of a symbol:
 
-    $ stockcruncher crunch AAPL quote_endpoint
+    $ stockcruncher quote AAPL
 
 ## Limitations
 
 Data are currently scraped from AlphaVantage API.
 More source could be great especially because AlphaVantage doesn't provide EU
 intraday data.
+InfluxDB is used as database to keep time series data.
 
 ## Development
 

data/lib/stockcruncher.rb

@@ -10,9 +10,9 @@ module StockCruncher
 
     def start(args = ARGV)
       StockCruncher::CLI.start(args)
-    rescue NotImplementedError => e
+    rescue StandardError => e
       warn e.message
-      exit(3)
+      exit 1
     end
   end
 end

data/lib/stockcruncher/alphavantage.rb

@@ -1,38 +1,31 @@
 #!/usr/bin/ruby
 # frozen_string_literal: true
 
-require './lib/stockcruncher/cruncher.rb'
+require File.join(File.dirname(__FILE__), 'cruncher.rb')
 
 module StockCruncher
   # This is an data cruncher class for AlphaVantage API.
   class AlphaVantage < Cruncher
     API_URL = 'https://www.alphavantage.co/query?'
-    SERIE = {
-      'daily' => 'TIME_SERIES_DAILY',
-      'quote_endpoint' => 'GLOBAL_QUOTE'
-    }.freeze
 
     # Main method to crunch data.
-    def crunch(symbol, serie, opts)
-      err_msg = "#{serie} does not exist."
-      raise NotImplementedError, err_msg unless SERIE.key?(serie)
-
-      url = API_URL + parameters(symbol, serie) + options(serie, opts)
+    def crunch_daily(symbol, fullsize)
+      url = API_URL + parameters(symbol, 'TIME_SERIES_DAILY')
+      url += '&datatype=csv&outputsize=' + (fullsize ? 'full' : 'compact')
       res = request(url)
-      puts res.body
+      res.body
     end
 
-    def options(serie, opts)
-      o = '&datatype=' + (opts['json'] ? 'json' : 'csv')
-      if serie == 'daily'
-        o += '&outputsize='
-        o += opts['full'] ? 'full' : 'compact'
-      end
-      o
+    # Main method to crunch data.
+    def crunch_quote(symbol)
+      url = API_URL + parameters(symbol, 'GLOBAL_QUOTE')
+      url += '&datatype=csv'
+      res = request(url)
+      res.body
     end
 
     def parameters(symbol, serie)
-      p = 'function=' + SERIE[serie]
+      p = 'function=' + serie
       p += '&symbol=' + symbol
       p += '&apikey=' + @config[self.class.name.split('::').last]['apikey']
       p

data/lib/stockcruncher/cli.rb

@@ -2,19 +2,12 @@
 # frozen_string_literal: true
 
 require 'thor'
+require 'yaml'
 
 module StockCruncher
   # Simple CLI for StockCruncher
   class CLI < Thor
-    desc 'version', 'Print stockcruncher current version'
-    def version
-      puts "StockCruncher version #{StockCruncher::VERSION}"
-    end
-
-    desc('crunch SYMBOL TIMESERIES [options]',
-         'Crunch SYMBOL stock market data for time series.' \
-         'Possible timeseries: daily, quote_endpoint.')
-    option(
+    class_option(
       :config,
       aliases: ['-c'],
       type: :string,
@@ -22,13 +15,28 @@ module StockCruncher
       desc: 'Yaml formatted config file to load ' \
             '(default to /etc/stockcruncher/stockcruncher.yml).'
     )
-    option(
-      :json,
-      aliases: ['-j'],
+    class_option(
+      :insecure,
+      aliases: ['-k'],
+      type: :boolean,
+      default: false,
+      desc: 'Ignore SSL certificate (default to false).'
+    )
+    class_option(
+      :quiet,
+      aliases: ['-q'],
       type: :boolean,
       default: false,
-      desc: 'Json format data (default to csv).'
+      desc: 'Run silently (default to false).'
     )
+
+    desc 'version', 'Print stockcruncher current version'
+    def version
+      puts "StockCruncher version #{StockCruncher::VERSION}"
+    end
+
+    desc('daily SYMBOL [options]',
+         'Crunch SYMBOL stock market data for daily time series.')
     option(
       :full,
       aliases: ['-f'],
@@ -36,10 +44,24 @@ module StockCruncher
       default: false,
       desc: 'Full data size.'
     )
-    def crunch(symbol, timeserie)
+    def daily(symbol)
+      opts = options.dup
+      config = YAML.load_file(opts['config'])
+      cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
+      raw_data = cruncher.crunch_daily(symbol, opts['full'])
+      StockCruncher::InfluxDB.new(config).export_history(symbol, raw_data)
+      puts raw_data unless opts['quiet']
+    end
+
+    desc('quote SYMBOL [options]',
+         'Crunch SYMBOL stock market data for last day quote.')
+    def quote(symbol)
       opts = options.dup
-      cruncher = StockCruncher::AlphaVantage.new(opts['config'])
-      cruncher.crunch(symbol, timeserie, opts)
+      config = YAML.load_file(opts['config'])
+      cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
+      raw_data = cruncher.crunch_quote(symbol)
+      StockCruncher::InfluxDB.new(config).export_last_day(raw_data)
+      puts raw_data unless opts['quiet']
     end
   end
 end

data/lib/stockcruncher/cruncher.rb

@@ -2,27 +2,22 @@
 # frozen_string_literal: true
 
 require 'net/http'
-require 'yaml'
 
 module StockCruncher
   # This is an data cruncher abstract class.
   class Cruncher
-    # Class abstract constructor method
-    def initialize(file)
-      @config = load_conf(file)
-    end
-
-    # Method to load configurations described in config_file.
-    def load_conf(file)
-      YAML.load_file(file)
+    # Class constructor method
+    def initialize(config, insecure = false)
+      @config = config
+      @insecure = insecure
     end
 
     # Method to send http get request
-    def request(url, insecure = false)
+    def request(url)
       uri = URI.parse(url)
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = uri.scheme.eql?('https')
-      http.verify_mode = OpenSSL::SSL::VERIFY_NONE if insecure
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE if @insecure
       req = Net::HTTP::Get.new(uri.request_uri)
       http.request(req)
     end

data/lib/stockcruncher/influxdb.rb

@@ -0,0 +1,122 @@
+#!/usr/bin/ruby
+# frozen_string_literal: true
+
+require 'date'
+require 'net/http'
+
+module StockCruncher
+  # this is a class to write time series to database
+  class InfluxDB
+    # Class constructor method
+    def initialize(config, insecure = false)
+      @cfg = config[self.class.name.split('::').last]
+      @insecure = insecure
+    end
+
+    # Method to calculate missing data (previousClose, change, changePercent)
+    def calculate_missing_data(hash)
+      keys = hash.keys
+      hash.each_with_index do |(date, v), index|
+        prevday = keys[index + 1]
+        next if prevday.nil?
+
+        prevclose = hash[prevday]['close']
+        hash[date] = v.merge(generate_missing_data(v['close'], prevclose))
+      end
+      hash
+    end
+
+    # Method to generate missing data
+    def generate_missing_data(current, previous)
+      {
+        'previousClose' => previous,
+        'change' => change(current.to_f, previous.to_f),
+        'changePercent' => change_percent(current.to_f, previous.to_f)
+      }
+    end
+
+    # Method to calculate change difference
+    def change(value, base)
+      (value - base).round(4).to_s
+    end
+
+    # Method to calculate percentage of change
+    def change_percent(value, base)
+      ((value / base - 1) * 100).round(4).to_s
+    end
+
+    # Method to create a new hash from two arrays of keys and values
+    def create_hash(descriptions, values)
+      descriptions.split(',').zip(values.split(',')).to_h
+    end
+
+    # Method to export latest data to database
+    def export_last_day(raw)
+      raise StandardError, 'No data to export' if raw.match?(/{}/)
+
+      values = create_hash(*raw.split("\r\n"))
+      values['close'] = values.delete('price')
+      values['changePercent'] = values['changePercent'].delete('%')
+      tags = { 'symbol' => values.delete('symbol') }
+      date = values.delete('latestDay')
+      write('daily', tags, values, date)
+    end
+
+    # Method to export historical data to database
+    def export_history(symbol, raw)
+      raise StandardError, 'No data to export' if raw.match?(/Error Message/)
+
+      tags = { 'symbol' => symbol }
+      timeseries = prepare_daily_timeserie(raw)
+      timeseries = calculate_missing_data(timeseries)
+      timeseries.each_pair do |date, values|
+        write('daily', tags, values, date)
+      end
+    end
+
+    # Method to format and array of values into comma separated string
+    def format_values(values)
+      string = ''
+      values.each_pair do |k, v|
+        string += "#{k}=#{v}"
+        string += ',' unless k == values.keys.last
+      end
+      string
+    end
+
+    # Method to transform raw data to constructed hash
+    def prepare_daily_timeserie(data)
+      lines = data.split("\r\n")
+      desc = lines.shift.split(',').drop(1)
+      hash = {}
+      lines.each do |line|
+        values = line.split(',')
+        date = values.shift
+        hash[date] = desc.zip(values).to_h
+      end
+      hash
+    end
+
+    # Method to send http post request
+    def request(url, body)
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme.eql?('https')
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE if @insecure
+      req = Net::HTTP::Post.new(uri.request_uri)
+      req.basic_auth(@cfg['user'], @cfg['password'])
+      req.body = body
+      http.request(req)
+    end
+
+    # Method to write data in bucket
+    def write(name, tags, values, date)
+      url = "#{@cfg['scheme']}://#{@cfg['host']}:#{@cfg['port']}/write?" \
+            "db=#{@cfg['dbname']}"
+      timestamp = DateTime.parse(date + 'T18:00:00').strftime('%s%N')
+      body = "#{name},#{format_values(tags)} #{format_values(values)} " \
+             "#{timestamp}"
+      request(url, body)
+    end
+  end
+end

data/lib/stockcruncher/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module StockCruncher
-  VERSION = '1.0.0'
+  VERSION = '1.2.0'
 end

data/spec/files/stockcruncher.yml

@@ -1,2 +1,9 @@
 AlphaVantage:
   apikey: demo
+InfluxDB:
+  scheme: http
+  host: localhost
+  port: 8086
+  user: testuser
+  password: testpassword
+  dbname: test

data/spec/stockcruncher/cli_spec.rb

@@ -2,7 +2,16 @@
 
 require 'spec_helper'
 
-describe StockCruncher::CLI do
+quote = 'symbol,open,high,low,price,volume,latestDay,previousClose,change,ch' \
+        "angePercent\r\nSYM,100.0000,100.1000,99.9000,100.0000,4,2020-07-30," \
+        "100.0000,0.0000,0.0000%\r\n"
+daily = "timestamp,open,high,low,close,volume\r\n2020-08-03,23.8500,24.6500," \
+        "23.7400,24.5500,3112972\r\n2020-07-31,24.0100,24.5100,23.5900,23.81" \
+        "00,5482485\r\n2020-07-30,24.4700,24.6600,23.5200,23.6600,6466738\r" \
+        "\n2020-07-29,24.7500,25.0300,24.0400,24.5600,4605804\r\n2020-07-28," \
+        "25.9900,26.0700,24.5100,24.7500,6904261\r\n"
+
+describe StockCruncher::CLI do # rubocop:disable Metrics/BlockLength
   context 'version' do
     it 'prints the version.' do
       out = "StockCruncher version #{StockCruncher::VERSION}\n"
@@ -10,9 +19,27 @@ describe StockCruncher::CLI do
     end
   end
 
-  context 'crunch SYM quote_endpoint -c spec/files/stockcruncher.yml' do
+  context 'daily NODATA -c spec/files/stockcruncher.yml' do
+    it 'Should not get any data and should fail.' do
+      expect { start(self) }.to raise_error(SystemExit)
+    end
+  end
+
+  context 'daily SYM -c spec/files/stockcruncher.yml' do
+    it 'Get the daily time serie for SYM.' do
+      expect { start(self) }.to output(daily).to_stdout
+    end
+  end
+
+  context 'quote NODATA -c spec/files/stockcruncher.yml' do
+    it 'Should not get any data and should fail.' do
+      expect { start(self) }.to raise_error(SystemExit)
+    end
+  end
+
+  context 'quote SYM -c spec/files/stockcruncher.yml' do
     it 'Get the quote for SYM.' do
-      expect { start(self) }.to output("{}\n").to_stdout
+      expect { start(self) }.to output(quote).to_stdout
     end
   end
 

data/spec/stockcruncher/stubs/servers_stubs.rb

@@ -2,15 +2,35 @@
 
 require 'spec_helper'
 
+quote = 'symbol,open,high,low,price,volume,latestDay,previousClose,change,ch' \
+        "angePercent\r\nSYM,100.0000,100.1000,99.9000,100.0000,4,2020-07-30," \
+        "100.0000,0.0000,0.0000%\r\n"
+q_err = '{}'
+daily = "timestamp,open,high,low,close,volume\r\n2020-08-03,23.8500,24.6500," \
+        "23.7400,24.5500,3112972\r\n2020-07-31,24.0100,24.5100,23.5900,23.81" \
+        "00,5482485\r\n2020-07-30,24.4700,24.6600,23.5200,23.6600,6466738\r" \
+        "\n2020-07-29,24.7500,25.0300,24.0400,24.5600,4605804\r\n2020-07-28," \
+        "25.9900,26.0700,24.5100,24.7500,6904261\r\n"
+d_err = "{\n    \"Error Message\": \"Invalid API call.\"\n}"
+
 RSpec.configure do |config|
   config.before(:each) do
     # requests an API without extra arguments
+    stub_request(:get, 'https://www.alphavantage.co/query?' \
+      'function=GLOBAL_QUOTE&symbol=NODATA&apikey=demo&datatype=csv')
+      .to_return('status' => 200, 'body' => q_err, 'headers' => {})
     stub_request(:get, 'https://www.alphavantage.co/query?' \
       'function=GLOBAL_QUOTE&symbol=SYM&apikey=demo&datatype=csv')
-      .to_return('status' => 200, 'body' => '{}', 'headers' => {})
+      .to_return('status' => 200, 'body' => quote, 'headers' => {})
+    stub_request(:get, 'https://www.alphavantage.co/query?' \
+      'function=TIME_SERIES_DAILY&symbol=NODATA&apikey=demo' \
+      '&datatype=csv&outputsize=compact')
+      .to_return('status' => 200, 'body' => d_err, 'headers' => {})
     stub_request(:get, 'https://www.alphavantage.co/query?' \
       'function=TIME_SERIES_DAILY&symbol=SYM&apikey=demo' \
       '&datatype=csv&outputsize=compact')
-      .to_return('status' => 200, 'body' => '{}', 'headers' => {})
+      .to_return('status' => 200, 'body' => daily, 'headers' => {})
+    stub_request(:post, 'http://localhost:8086/write?db=test')
+      .to_return('status' => 204, 'body' => '', 'headers' => {})
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stockcruncher
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Richard Delaplace
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-24 00:00:00.000000000 Z
+date: 2020-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -130,10 +130,10 @@ files:
 - lib/stockcruncher/alphavantage.rb
 - lib/stockcruncher/cli.rb
 - lib/stockcruncher/cruncher.rb
+- lib/stockcruncher/influxdb.rb
 - lib/stockcruncher/version.rb
 - spec/files/stockcruncher.yml
 - spec/spec_helper.rb
-- spec/stockcruncher/alphavantage_spec.rb
 - spec/stockcruncher/cli_spec.rb
 - spec/stockcruncher/stubs/servers_stubs.rb
 - spec/stockcruncher_spec.rb

data/spec/stockcruncher/alphavantage_spec.rb

@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-describe StockCruncher::AlphaVantage do
-  context 'crunch SYM daily -c spec/files/stockcruncher.yml' do
-    it 'requests a source and get the result.' do
-      expect { start(self) }.to output("{}\n").to_stdout
-    end
-  end
-  context 'crunch SYM weekly -c spec/files/stockcruncher.yml' do
-    it 'requests a not implemented time serie.' do
-      expect { start(self) }.to raise_error(SystemExit)
-    end
-  end
-end