stockcruncher 1.0.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 58f00f855043839b976fb6bd52a7d6044187fa24dd97c5d50cfb90b637bfd26a
4
- data.tar.gz: d2f59a0979b3f1da3c1ba8b491efe6b6588b5af26d1b17f69a3ab6455f3aebbb
3
+ metadata.gz: 7bd12e5a6e5db6f83f239c1b6da9aca85931048dd427778ffb96f043e7334596
4
+ data.tar.gz: 48fe8efa593f30fe39c294725d7090c89a4a14413eaa59a1d6741c817fc02d64
5
5
  SHA512:
6
- metadata.gz: 3eaeeb37d7f845ae3c163053d455650f519c29549c5750c70a443efe3e993d4ebc002595f7d3e5b5b093aee1d790ac026a6037c86717319d5170cf60bcf42e6a
7
- data.tar.gz: d0cd2ee27442ccb1f814958759e7c826eef5bbef468b060ba39674d5d8560d010f82b843e75b79f4401f5e45cea66f72b09a8a0dd446c0fca49f5b73621c6e5b
6
+ metadata.gz: ad4ed352a8ed30d41edbe9c4bac03ba721c9af42837f9e94f9c14df2175013d4ae7a09e29c7076e034cf375abaf222ce04c251a4b8fde31ce10ae29936c18894
7
+ data.tar.gz: 06206e1a91e23706c6a83c02357b31064124a21e522f3144d4ec93d4d4e7863e82e38cf4b561eedb662e59ea801120ee05f8cd3fff1b414da5ea5a069dd337ff
data/CHANGELOG CHANGED
@@ -5,3 +5,20 @@ Changelog
5
5
  -----
6
6
 
7
7
  - Initial version
8
+
9
+ 1.1.0
10
+ -----
11
+
12
+ - Add InfluxDB as database.
13
+ - WIP for daily time serie.
14
+
15
+ 1.1.1
16
+ -----
17
+
18
+ - Rescue error on no data
19
+
20
+ 1.2.0
21
+ -----
22
+
23
+ - daily subcommand now recalculates missing data
24
+ - daily subcommand also writes data to database
data/README.md CHANGED
@@ -25,29 +25,38 @@ to write the data in a database for later calculation and use.
25
25
 
26
26
  $ gem install stockcruncher
27
27
  $ mkdir /etc/stockcruncher && cd /etc/stockcruncher
28
- $ echo $'AlphaVantage:\n apikey: CHANGEME' > stockcruncher.yml
28
+ $ echo 'AlphaVantage:' > stockcruncher.yml
29
+ $ echo ' apikey: CHANGEME' >> stockcruncher.yml
30
+ $ echo 'InfluxDB:' >> stockcruncher.yml
31
+ $ echo ' scheme: http' >> stockcruncher.yml
32
+ $ echo ' host: localhost' >> stockcruncher.yml
33
+ $ echo ' port: 8086' >> stockcruncher.yml
34
+ $ echo ' user: CHANGEMEAGAIN' >> stockcruncher.yml
35
+ $ echo ' password: CHANGEMETOO' >> stockcruncher.yml
36
+ $ echo ' dbname: stock' >> stockcruncher.yml
29
37
 
30
38
  ## Usage
31
39
 
32
40
  An interactive help is available with:
33
41
 
34
- $ stockcruncher help
42
+ $ stockcruncher help [subcommand]
35
43
 
36
44
  ## Examples
37
45
 
38
46
  To get daily time serie data of a symbol:
39
47
 
40
- $ stockcruncher crunch AAPL daily
48
+ $ stockcruncher daily AAPL
41
49
 
42
50
  To get last day endpoint data of a symbol:
43
51
 
44
- $ stockcruncher crunch AAPL quote_endpoint
52
+ $ stockcruncher quote AAPL
45
53
 
46
54
  ## Limitations
47
55
 
48
56
  Data are currently scraped from AlphaVantage API.
49
57
  More source could be great especially because AlphaVantage doesn't provide EU
50
58
  intraday data.
59
+ InfluxDB is used as database to keep time series data.
51
60
 
52
61
  ## Development
53
62
 
@@ -10,9 +10,9 @@ module StockCruncher
10
10
 
11
11
  def start(args = ARGV)
12
12
  StockCruncher::CLI.start(args)
13
- rescue NotImplementedError => e
13
+ rescue StandardError => e
14
14
  warn e.message
15
- exit(3)
15
+ exit 1
16
16
  end
17
17
  end
18
18
  end
@@ -1,38 +1,31 @@
1
1
  #!/usr/bin/ruby
2
2
  # frozen_string_literal: true
3
3
 
4
- require './lib/stockcruncher/cruncher.rb'
4
+ require File.join(File.dirname(__FILE__), 'cruncher.rb')
5
5
 
6
6
  module StockCruncher
7
7
  # This is an data cruncher class for AlphaVantage API.
8
8
  class AlphaVantage < Cruncher
9
9
  API_URL = 'https://www.alphavantage.co/query?'
10
- SERIE = {
11
- 'daily' => 'TIME_SERIES_DAILY',
12
- 'quote_endpoint' => 'GLOBAL_QUOTE'
13
- }.freeze
14
10
 
15
11
  # Main method to crunch data.
16
- def crunch(symbol, serie, opts)
17
- err_msg = "#{serie} does not exist."
18
- raise NotImplementedError, err_msg unless SERIE.key?(serie)
19
-
20
- url = API_URL + parameters(symbol, serie) + options(serie, opts)
12
+ def crunch_daily(symbol, fullsize)
13
+ url = API_URL + parameters(symbol, 'TIME_SERIES_DAILY')
14
+ url += '&datatype=csv&outputsize=' + (fullsize ? 'full' : 'compact')
21
15
  res = request(url)
22
- puts res.body
16
+ res.body
23
17
  end
24
18
 
25
- def options(serie, opts)
26
- o = '&datatype=' + (opts['json'] ? 'json' : 'csv')
27
- if serie == 'daily'
28
- o += '&outputsize='
29
- o += opts['full'] ? 'full' : 'compact'
30
- end
31
- o
19
+ # Main method to crunch data.
20
+ def crunch_quote(symbol)
21
+ url = API_URL + parameters(symbol, 'GLOBAL_QUOTE')
22
+ url += '&datatype=csv'
23
+ res = request(url)
24
+ res.body
32
25
  end
33
26
 
34
27
  def parameters(symbol, serie)
35
- p = 'function=' + SERIE[serie]
28
+ p = 'function=' + serie
36
29
  p += '&symbol=' + symbol
37
30
  p += '&apikey=' + @config[self.class.name.split('::').last]['apikey']
38
31
  p
@@ -2,19 +2,12 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require 'thor'
5
+ require 'yaml'
5
6
 
6
7
  module StockCruncher
7
8
  # Simple CLI for StockCruncher
8
9
  class CLI < Thor
9
- desc 'version', 'Print stockcruncher current version'
10
- def version
11
- puts "StockCruncher version #{StockCruncher::VERSION}"
12
- end
13
-
14
- desc('crunch SYMBOL TIMESERIES [options]',
15
- 'Crunch SYMBOL stock market data for time series.' \
16
- 'Possible timeseries: daily, quote_endpoint.')
17
- option(
10
+ class_option(
18
11
  :config,
19
12
  aliases: ['-c'],
20
13
  type: :string,
@@ -22,13 +15,28 @@ module StockCruncher
22
15
  desc: 'Yaml formatted config file to load ' \
23
16
  '(default to /etc/stockcruncher/stockcruncher.yml).'
24
17
  )
25
- option(
26
- :json,
27
- aliases: ['-j'],
18
+ class_option(
19
+ :insecure,
20
+ aliases: ['-k'],
21
+ type: :boolean,
22
+ default: false,
23
+ desc: 'Ignore SSL certificate (default to false).'
24
+ )
25
+ class_option(
26
+ :quiet,
27
+ aliases: ['-q'],
28
28
  type: :boolean,
29
29
  default: false,
30
- desc: 'Json format data (default to csv).'
30
+ desc: 'Run silently (default to false).'
31
31
  )
32
+
33
+ desc 'version', 'Print stockcruncher current version'
34
+ def version
35
+ puts "StockCruncher version #{StockCruncher::VERSION}"
36
+ end
37
+
38
+ desc('daily SYMBOL [options]',
39
+ 'Crunch SYMBOL stock market data for daily time series.')
32
40
  option(
33
41
  :full,
34
42
  aliases: ['-f'],
@@ -36,10 +44,24 @@ module StockCruncher
36
44
  default: false,
37
45
  desc: 'Full data size.'
38
46
  )
39
- def crunch(symbol, timeserie)
47
+ def daily(symbol)
48
+ opts = options.dup
49
+ config = YAML.load_file(opts['config'])
50
+ cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
51
+ raw_data = cruncher.crunch_daily(symbol, opts['full'])
52
+ StockCruncher::InfluxDB.new(config).export_history(symbol, raw_data)
53
+ puts raw_data unless opts['quiet']
54
+ end
55
+
56
+ desc('quote SYMBOL [options]',
57
+ 'Crunch SYMBOL stock market data for last day quote.')
58
+ def quote(symbol)
40
59
  opts = options.dup
41
- cruncher = StockCruncher::AlphaVantage.new(opts['config'])
42
- cruncher.crunch(symbol, timeserie, opts)
60
+ config = YAML.load_file(opts['config'])
61
+ cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
62
+ raw_data = cruncher.crunch_quote(symbol)
63
+ StockCruncher::InfluxDB.new(config).export_last_day(raw_data)
64
+ puts raw_data unless opts['quiet']
43
65
  end
44
66
  end
45
67
  end
@@ -2,27 +2,22 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require 'net/http'
5
- require 'yaml'
6
5
 
7
6
  module StockCruncher
8
7
  # This is an data cruncher abstract class.
9
8
  class Cruncher
10
- # Class abstract constructor method
11
- def initialize(file)
12
- @config = load_conf(file)
13
- end
14
-
15
- # Method to load configurations described in config_file.
16
- def load_conf(file)
17
- YAML.load_file(file)
9
+ # Class constructor method
10
+ def initialize(config, insecure = false)
11
+ @config = config
12
+ @insecure = insecure
18
13
  end
19
14
 
20
15
  # Method to send http get request
21
- def request(url, insecure = false)
16
+ def request(url)
22
17
  uri = URI.parse(url)
23
18
  http = Net::HTTP.new(uri.host, uri.port)
24
19
  http.use_ssl = uri.scheme.eql?('https')
25
- http.verify_mode = OpenSSL::SSL::VERIFY_NONE if insecure
20
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE if @insecure
26
21
  req = Net::HTTP::Get.new(uri.request_uri)
27
22
  http.request(req)
28
23
  end
@@ -0,0 +1,122 @@
1
+ #!/usr/bin/ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'date'
5
+ require 'net/http'
6
+
7
+ module StockCruncher
8
+ # this is a class to write time series to database
9
+ class InfluxDB
10
+ # Class constructor method
11
+ def initialize(config, insecure = false)
12
+ @cfg = config[self.class.name.split('::').last]
13
+ @insecure = insecure
14
+ end
15
+
16
+ # Method to calculate missing data (previousClose, change, changePercent)
17
+ def calculate_missing_data(hash)
18
+ keys = hash.keys
19
+ hash.each_with_index do |(date, v), index|
20
+ prevday = keys[index + 1]
21
+ next if prevday.nil?
22
+
23
+ prevclose = hash[prevday]['close']
24
+ hash[date] = v.merge(generate_missing_data(v['close'], prevclose))
25
+ end
26
+ hash
27
+ end
28
+
29
+ # Method to generate missing data
30
+ def generate_missing_data(current, previous)
31
+ {
32
+ 'previousClose' => previous,
33
+ 'change' => change(current.to_f, previous.to_f),
34
+ 'changePercent' => change_percent(current.to_f, previous.to_f)
35
+ }
36
+ end
37
+
38
+ # Method to calculate change difference
39
+ def change(value, base)
40
+ (value - base).round(4).to_s
41
+ end
42
+
43
+ # Method to calculate percentage of change
44
+ def change_percent(value, base)
45
+ ((value / base - 1) * 100).round(4).to_s
46
+ end
47
+
48
+ # Method to create a new hash from two arrays of keys and values
49
+ def create_hash(descriptions, values)
50
+ descriptions.split(',').zip(values.split(',')).to_h
51
+ end
52
+
53
+ # Method to export latest data to database
54
+ def export_last_day(raw)
55
+ raise StandardError, 'No data to export' if raw.match?(/{}/)
56
+
57
+ values = create_hash(*raw.split("\r\n"))
58
+ values['close'] = values.delete('price')
59
+ values['changePercent'] = values['changePercent'].delete('%')
60
+ tags = { 'symbol' => values.delete('symbol') }
61
+ date = values.delete('latestDay')
62
+ write('daily', tags, values, date)
63
+ end
64
+
65
+ # Method to export historical data to database
66
+ def export_history(symbol, raw)
67
+ raise StandardError, 'No data to export' if raw.match?(/Error Message/)
68
+
69
+ tags = { 'symbol' => symbol }
70
+ timeseries = prepare_daily_timeserie(raw)
71
+ timeseries = calculate_missing_data(timeseries)
72
+ timeseries.each_pair do |date, values|
73
+ write('daily', tags, values, date)
74
+ end
75
+ end
76
+
77
+ # Method to format and array of values into comma separated string
78
+ def format_values(values)
79
+ string = ''
80
+ values.each_pair do |k, v|
81
+ string += "#{k}=#{v}"
82
+ string += ',' unless k == values.keys.last
83
+ end
84
+ string
85
+ end
86
+
87
+ # Method to transform raw data to constructed hash
88
+ def prepare_daily_timeserie(data)
89
+ lines = data.split("\r\n")
90
+ desc = lines.shift.split(',').drop(1)
91
+ hash = {}
92
+ lines.each do |line|
93
+ values = line.split(',')
94
+ date = values.shift
95
+ hash[date] = desc.zip(values).to_h
96
+ end
97
+ hash
98
+ end
99
+
100
+ # Method to send http post request
101
+ def request(url, body)
102
+ uri = URI.parse(url)
103
+ http = Net::HTTP.new(uri.host, uri.port)
104
+ http.use_ssl = uri.scheme.eql?('https')
105
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE if @insecure
106
+ req = Net::HTTP::Post.new(uri.request_uri)
107
+ req.basic_auth(@cfg['user'], @cfg['password'])
108
+ req.body = body
109
+ http.request(req)
110
+ end
111
+
112
+ # Method to write data in bucket
113
+ def write(name, tags, values, date)
114
+ url = "#{@cfg['scheme']}://#{@cfg['host']}:#{@cfg['port']}/write?" \
115
+ "db=#{@cfg['dbname']}"
116
+ timestamp = DateTime.parse(date + 'T18:00:00').strftime('%s%N')
117
+ body = "#{name},#{format_values(tags)} #{format_values(values)} " \
118
+ "#{timestamp}"
119
+ request(url, body)
120
+ end
121
+ end
122
+ end
@@ -2,5 +2,5 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  module StockCruncher
5
- VERSION = '1.0.0'
5
+ VERSION = '1.2.0'
6
6
  end
@@ -1,2 +1,9 @@
1
1
  AlphaVantage:
2
2
  apikey: demo
3
+ InfluxDB:
4
+ scheme: http
5
+ host: localhost
6
+ port: 8086
7
+ user: testuser
8
+ password: testpassword
9
+ dbname: test
@@ -2,7 +2,16 @@
2
2
 
3
3
  require 'spec_helper'
4
4
 
5
- describe StockCruncher::CLI do
5
+ quote = 'symbol,open,high,low,price,volume,latestDay,previousClose,change,ch' \
6
+ "angePercent\r\nSYM,100.0000,100.1000,99.9000,100.0000,4,2020-07-30," \
7
+ "100.0000,0.0000,0.0000%\r\n"
8
+ daily = "timestamp,open,high,low,close,volume\r\n2020-08-03,23.8500,24.6500," \
9
+ "23.7400,24.5500,3112972\r\n2020-07-31,24.0100,24.5100,23.5900,23.81" \
10
+ "00,5482485\r\n2020-07-30,24.4700,24.6600,23.5200,23.6600,6466738\r" \
11
+ "\n2020-07-29,24.7500,25.0300,24.0400,24.5600,4605804\r\n2020-07-28," \
12
+ "25.9900,26.0700,24.5100,24.7500,6904261\r\n"
13
+
14
+ describe StockCruncher::CLI do # rubocop:disable Metrics/BlockLength
6
15
  context 'version' do
7
16
  it 'prints the version.' do
8
17
  out = "StockCruncher version #{StockCruncher::VERSION}\n"
@@ -10,9 +19,27 @@ describe StockCruncher::CLI do
10
19
  end
11
20
  end
12
21
 
13
- context 'crunch SYM quote_endpoint -c spec/files/stockcruncher.yml' do
22
+ context 'daily NODATA -c spec/files/stockcruncher.yml' do
23
+ it 'Should not get any data and should fail.' do
24
+ expect { start(self) }.to raise_error(SystemExit)
25
+ end
26
+ end
27
+
28
+ context 'daily SYM -c spec/files/stockcruncher.yml' do
29
+ it 'Get the daily time serie for SYM.' do
30
+ expect { start(self) }.to output(daily).to_stdout
31
+ end
32
+ end
33
+
34
+ context 'quote NODATA -c spec/files/stockcruncher.yml' do
35
+ it 'Should not get any data and should fail.' do
36
+ expect { start(self) }.to raise_error(SystemExit)
37
+ end
38
+ end
39
+
40
+ context 'quote SYM -c spec/files/stockcruncher.yml' do
14
41
  it 'Get the quote for SYM.' do
15
- expect { start(self) }.to output("{}\n").to_stdout
42
+ expect { start(self) }.to output(quote).to_stdout
16
43
  end
17
44
  end
18
45
 
@@ -2,15 +2,35 @@
2
2
 
3
3
  require 'spec_helper'
4
4
 
5
+ quote = 'symbol,open,high,low,price,volume,latestDay,previousClose,change,ch' \
6
+ "angePercent\r\nSYM,100.0000,100.1000,99.9000,100.0000,4,2020-07-30," \
7
+ "100.0000,0.0000,0.0000%\r\n"
8
+ q_err = '{}'
9
+ daily = "timestamp,open,high,low,close,volume\r\n2020-08-03,23.8500,24.6500," \
10
+ "23.7400,24.5500,3112972\r\n2020-07-31,24.0100,24.5100,23.5900,23.81" \
11
+ "00,5482485\r\n2020-07-30,24.4700,24.6600,23.5200,23.6600,6466738\r" \
12
+ "\n2020-07-29,24.7500,25.0300,24.0400,24.5600,4605804\r\n2020-07-28," \
13
+ "25.9900,26.0700,24.5100,24.7500,6904261\r\n"
14
+ d_err = "{\n \"Error Message\": \"Invalid API call.\"\n}"
15
+
5
16
  RSpec.configure do |config|
6
17
  config.before(:each) do
7
18
  # requests an API without extra arguments
19
+ stub_request(:get, 'https://www.alphavantage.co/query?' \
20
+ 'function=GLOBAL_QUOTE&symbol=NODATA&apikey=demo&datatype=csv')
21
+ .to_return('status' => 200, 'body' => q_err, 'headers' => {})
8
22
  stub_request(:get, 'https://www.alphavantage.co/query?' \
9
23
  'function=GLOBAL_QUOTE&symbol=SYM&apikey=demo&datatype=csv')
10
- .to_return('status' => 200, 'body' => '{}', 'headers' => {})
24
+ .to_return('status' => 200, 'body' => quote, 'headers' => {})
25
+ stub_request(:get, 'https://www.alphavantage.co/query?' \
26
+ 'function=TIME_SERIES_DAILY&symbol=NODATA&apikey=demo' \
27
+ '&datatype=csv&outputsize=compact')
28
+ .to_return('status' => 200, 'body' => d_err, 'headers' => {})
11
29
  stub_request(:get, 'https://www.alphavantage.co/query?' \
12
30
  'function=TIME_SERIES_DAILY&symbol=SYM&apikey=demo' \
13
31
  '&datatype=csv&outputsize=compact')
14
- .to_return('status' => 200, 'body' => '{}', 'headers' => {})
32
+ .to_return('status' => 200, 'body' => daily, 'headers' => {})
33
+ stub_request(:post, 'http://localhost:8086/write?db=test')
34
+ .to_return('status' => 204, 'body' => '', 'headers' => {})
15
35
  end
16
36
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: stockcruncher
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Richard Delaplace
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-07-24 00:00:00.000000000 Z
11
+ date: 2020-08-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -130,10 +130,10 @@ files:
130
130
  - lib/stockcruncher/alphavantage.rb
131
131
  - lib/stockcruncher/cli.rb
132
132
  - lib/stockcruncher/cruncher.rb
133
+ - lib/stockcruncher/influxdb.rb
133
134
  - lib/stockcruncher/version.rb
134
135
  - spec/files/stockcruncher.yml
135
136
  - spec/spec_helper.rb
136
- - spec/stockcruncher/alphavantage_spec.rb
137
137
  - spec/stockcruncher/cli_spec.rb
138
138
  - spec/stockcruncher/stubs/servers_stubs.rb
139
139
  - spec/stockcruncher_spec.rb
@@ -1,16 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'spec_helper'
4
-
5
- describe StockCruncher::AlphaVantage do
6
- context 'crunch SYM daily -c spec/files/stockcruncher.yml' do
7
- it 'requests a source and get the result.' do
8
- expect { start(self) }.to output("{}\n").to_stdout
9
- end
10
- end
11
- context 'crunch SYM weekly -c spec/files/stockcruncher.yml' do
12
- it 'requests a not implemented time serie.' do
13
- expect { start(self) }.to raise_error(SystemExit)
14
- end
15
- end
16
- end