stockcruncher 1.2.1 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitlab-ci.yml +4 -3
- data/.rubocop.yml +1 -0
- data/README.md +23 -6
- data/lib/stockcruncher/alphavantage.rb +5 -6
- data/lib/stockcruncher/cli.rb +35 -3
- data/lib/stockcruncher/config.rb +40 -0
- data/lib/stockcruncher/influxdb.rb +66 -14
- data/lib/stockcruncher/stats.rb +74 -0
- data/lib/stockcruncher/version.rb +1 -1
- data/spec/stockcruncher/cli_spec.rb +6 -0
- data/spec/stockcruncher/stubs/servers_stubs.rb +7 -0
- data/stockcruncher.gemspec +1 -1
- metadata +6 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d575ec081ba8f154b940a03a7eda7d79606bc78ff9c01132f2da78170b641367
|
4
|
+
data.tar.gz: 5949b3b5108d9d0d67907e49711ae9ecf8dbdfebfba669b9ed952f8592cfa95a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6f3dbd7ade6b0ad21e39842111ba21aa21ddf38ac0d984a24cf786a9bcd503c5427b85c5e5c500a0acc278fda9e83a08472144117148a8f624c49c724269884c
|
7
|
+
data.tar.gz: 98690a9871f4d68c2a199cdbcce134690a12b594aba5c4842f4c36346dcd143b8ea15079ff69e7996af19958a1feec5859340e0ea6657d3c649c24cc66007832
|
data/.gitlab-ci.yml
CHANGED
@@ -1,9 +1,11 @@
|
|
1
|
+
|
1
2
|
image: ruby:2.4
|
2
3
|
|
4
|
+
before_script:
|
5
|
+
- bundle install
|
6
|
+
|
3
7
|
rubocop:
|
4
8
|
stage: test
|
5
|
-
before_script:
|
6
|
-
- gem install --silent rubocop
|
7
9
|
script:
|
8
10
|
- rubocop
|
9
11
|
|
@@ -25,7 +27,6 @@ git_history:
|
|
25
27
|
rspec:
|
26
28
|
stage: test
|
27
29
|
script:
|
28
|
-
- bundle install
|
29
30
|
- rspec
|
30
31
|
artifacts:
|
31
32
|
paths:
|
data/.rubocop.yml
CHANGED
data/README.md
CHANGED
@@ -5,12 +5,14 @@
|
|
5
5
|
[![Gem Version][gem-img]][gem]
|
6
6
|
|
7
7
|
1. [Overview](#overview)
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
8
|
+
1. [Description](#description)
|
9
|
+
1. [Setup](#setup)
|
10
|
+
1. [Usage](#usage)
|
11
|
+
1. [Environment variables](#environment-variables)
|
12
|
+
1. [Examples](#examples)
|
13
|
+
1. [Limitations](#limitations)
|
14
|
+
1. [Development](#development)
|
15
|
+
1. [Miscellaneous](#miscellaneous)
|
14
16
|
|
15
17
|
## Overview
|
16
18
|
|
@@ -41,6 +43,17 @@ An interactive help is available with:
|
|
41
43
|
|
42
44
|
$ stockcruncher help [subcommand]
|
43
45
|
|
46
|
+
## Environment variables
|
47
|
+
|
48
|
+
Parameters in /etc/stockcruncher/stockcrunucher.yml can be overloaded with
|
49
|
+
environment variables.
|
50
|
+
Environment variables should match SCR_<COMPONENTID>_<ITEM>.
|
51
|
+
Component ID are as follow.
|
52
|
+
- AlphaVantage: AV
|
53
|
+
- InfluxDB: IDB
|
54
|
+
Items are upcase keys. See template /etc/stockcruncher/stockcrunucher.yml for
|
55
|
+
reference.
|
56
|
+
|
44
57
|
## Examples
|
45
58
|
|
46
59
|
To get daily time serie data of a symbol:
|
@@ -51,6 +64,10 @@ To get last day endpoint data of a symbol:
|
|
51
64
|
|
52
65
|
$ stockcruncher quote AAPL
|
53
66
|
|
67
|
+
To override a parameter with environment variable:
|
68
|
+
|
69
|
+
$ SCR_IDB_HOST=192.168.0.80; stockcruncher quote AAPL
|
70
|
+
|
54
71
|
## Limitations
|
55
72
|
|
56
73
|
Data are currently scraped from AlphaVantage API.
|
@@ -39,7 +39,7 @@ module StockCruncher
|
|
39
39
|
# Main method to crunch data.
|
40
40
|
def crunch_daily(symbol, fullsize)
|
41
41
|
url = API_URL + parameters(symbol, 'TIME_SERIES_DAILY')
|
42
|
-
url +=
|
42
|
+
url += "&datatype=csv&outputsize=#{fullsize ? 'full' : 'compact'}"
|
43
43
|
res = request(url)
|
44
44
|
transform_daily(res.body)
|
45
45
|
end
|
@@ -63,9 +63,9 @@ module StockCruncher
|
|
63
63
|
|
64
64
|
# Set parameters of api call
|
65
65
|
def parameters(symbol, serie)
|
66
|
-
p =
|
67
|
-
p +=
|
68
|
-
p +=
|
66
|
+
p = "function=#{serie}"
|
67
|
+
p += "&symbol=#{symbol}"
|
68
|
+
p += "&apikey=#{@config[self.class.name.split('::').last]['apikey']}"
|
69
69
|
p
|
70
70
|
end
|
71
71
|
|
@@ -87,8 +87,7 @@ module StockCruncher
|
|
87
87
|
raise StandardError, 'No data' if rawdata.match?(/Error Message/)
|
88
88
|
|
89
89
|
values = prepare_daily_timeserie(rawdata)
|
90
|
-
|
91
|
-
values
|
90
|
+
calculate_missing_data(values)
|
92
91
|
end
|
93
92
|
|
94
93
|
# Method to transform quote result to hash
|
data/lib/stockcruncher/cli.rb
CHANGED
@@ -45,23 +45,55 @@ module StockCruncher
|
|
45
45
|
default: false,
|
46
46
|
desc: 'Full data size.'
|
47
47
|
)
|
48
|
+
option(
|
49
|
+
:catchup,
|
50
|
+
aliases: ['-u'],
|
51
|
+
type: :boolean,
|
52
|
+
default: false,
|
53
|
+
desc: 'Catch up the missing data only.'
|
54
|
+
)
|
48
55
|
def daily(symbol)
|
49
56
|
opts = options.dup
|
50
57
|
config = YAML.load_file(opts['config'])
|
51
58
|
cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
|
52
59
|
data = cruncher.crunch_daily(symbol, opts['full'])
|
53
|
-
StockCruncher::InfluxDB.new(config)
|
60
|
+
influx = StockCruncher::InfluxDB.new(config)
|
61
|
+
influx.export_history(symbol, data, opts['catchup'])
|
54
62
|
puts JSON.pretty_generate(data) unless opts['quiet']
|
55
63
|
end
|
56
64
|
|
65
|
+
desc('movingaverages SYMBOL [options]',
|
66
|
+
'Calculate and export moving averages for requested symbol.')
|
67
|
+
option(
|
68
|
+
:all,
|
69
|
+
aliases: ['-a'],
|
70
|
+
type: :boolean,
|
71
|
+
default: false,
|
72
|
+
desc: 'Recalculate all MA historical values.'
|
73
|
+
)
|
74
|
+
option(
|
75
|
+
:catchup,
|
76
|
+
aliases: ['-u'],
|
77
|
+
type: :boolean,
|
78
|
+
default: false,
|
79
|
+
desc: 'Catch up the missing data only.'
|
80
|
+
)
|
81
|
+
def movingaverages(symbol)
|
82
|
+
opts = options.dup
|
83
|
+
config = YAML.load_file(opts['config'])
|
84
|
+
influx = StockCruncher::InfluxDB.new(config)
|
85
|
+
influx.moving_averages(symbol, opts['all'], opts['catchup'])
|
86
|
+
end
|
87
|
+
|
57
88
|
desc('quote SYMBOL [options]',
|
58
89
|
'Crunch SYMBOL stock market data for last day quote.')
|
59
90
|
def quote(symbol)
|
60
91
|
opts = options.dup
|
61
|
-
config =
|
92
|
+
config = StockCruncher::Config.load(opts['config'])
|
62
93
|
cruncher = StockCruncher::AlphaVantage.new(config, opts['insecure'])
|
63
94
|
data = cruncher.crunch_quote(symbol)
|
64
|
-
StockCruncher::InfluxDB.new(config)
|
95
|
+
influx = StockCruncher::InfluxDB.new(config)
|
96
|
+
influx.export_last_day(data)
|
65
97
|
puts JSON.pretty_generate(data) unless opts['quiet']
|
66
98
|
end
|
67
99
|
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
module StockCruncher
|
7
|
+
# this is a module to load configuration file and environment variable
|
8
|
+
module Config
|
9
|
+
module_function
|
10
|
+
|
11
|
+
# Load config file and override with env variables
|
12
|
+
def load(file)
|
13
|
+
config = YAML.load_file(file)
|
14
|
+
overload_alphavantage(config)
|
15
|
+
overload_influxdb(config)
|
16
|
+
end
|
17
|
+
|
18
|
+
def overload(config, prefix, component, items)
|
19
|
+
items.each do |key|
|
20
|
+
var = "#{prefix}#{key.upcase}"
|
21
|
+
config[component][key] = ENV[var] unless ENV[var].nil?
|
22
|
+
end
|
23
|
+
config
|
24
|
+
end
|
25
|
+
|
26
|
+
def overload_alphavantage(config)
|
27
|
+
prefix = 'SCR_AV_'
|
28
|
+
component = 'AlphaVantage'
|
29
|
+
items = %w[apikey]
|
30
|
+
overload(config, prefix, component, items)
|
31
|
+
end
|
32
|
+
|
33
|
+
def overload_influxdb(config)
|
34
|
+
prefix = 'SCR_IDB_'
|
35
|
+
component = 'InfluxDB'
|
36
|
+
items = %w[scheme host port user password dbname]
|
37
|
+
overload(config, prefix, component, items)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -2,6 +2,7 @@
|
|
2
2
|
# frozen_string_literal: true
|
3
3
|
|
4
4
|
require 'date'
|
5
|
+
require 'json'
|
5
6
|
require 'net/http'
|
6
7
|
|
7
8
|
module StockCruncher
|
@@ -13,29 +14,80 @@ module StockCruncher
|
|
13
14
|
@insecure = insecure
|
14
15
|
end
|
15
16
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
17
|
+
def get_daily_values(symbol, fullsize)
|
18
|
+
values = %w[close change changePercent volume]
|
19
|
+
data = query('daily', symbol, values, fullsize)
|
20
|
+
data['columns'].zip(data['values'].transpose).to_h
|
21
|
+
end
|
22
|
+
|
23
|
+
def get_ma_values(symbol, fullsize)
|
24
|
+
values = %w[ema200]
|
25
|
+
data = query('ema', symbol, values, fullsize)
|
26
|
+
data['columns'].zip(data['values'].transpose).to_h
|
27
|
+
end
|
28
|
+
|
29
|
+
# Method to calculate moving averages based on last day values
|
30
|
+
def moving_averages(symbol, fullsize, catchup)
|
31
|
+
series = get_daily_values(symbol, fullsize)
|
32
|
+
mas = catchup ? get_ma_values(symbol, fullsize) : {}
|
33
|
+
tags = create_tags(symbol)
|
34
|
+
series['close'].each_index do |i|
|
35
|
+
date = series['time'][i]
|
36
|
+
serie = series['close'][i, 201]
|
37
|
+
weights = series['volume'][i, 201]
|
38
|
+
write_moving_averages(tags, serie, weights, date) unless mas.key? date
|
39
|
+
break unless fullsize
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
# Method to create tags hash containing only symbol
|
44
|
+
def create_tags(symbol)
|
45
|
+
{ 'symbol' => symbol }
|
21
46
|
end
|
22
47
|
|
23
48
|
# Method to export historical data to database
|
24
|
-
def export_history(symbol, timeseries)
|
25
|
-
tags =
|
49
|
+
def export_history(symbol, timeseries, catchup)
|
50
|
+
tags = create_tags(symbol)
|
51
|
+
if catchup
|
52
|
+
series = get_daily_values(symbol, fullsize)
|
53
|
+
series['time'].each { |date| timeseries.delete(date) }
|
54
|
+
end
|
26
55
|
timeseries.each_pair do |date, values|
|
27
56
|
write('daily', tags, values, date)
|
28
57
|
end
|
29
58
|
end
|
30
59
|
|
60
|
+
# Method to export latest data to database
|
61
|
+
def export_last_day(values)
|
62
|
+
tags = create_tags(values.delete('symbol'))
|
63
|
+
date = values.delete('latestDay')
|
64
|
+
write('daily', tags, values, date)
|
65
|
+
end
|
66
|
+
|
31
67
|
# Method to format and array of values into comma separated string
|
32
68
|
def format_values(values)
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
69
|
+
values.map { |k, v| "#{k}=#{v}" }.join(',')
|
70
|
+
end
|
71
|
+
|
72
|
+
# Method to calculate all statistics
|
73
|
+
def write_moving_averages(tags, serie, weights, date)
|
74
|
+
write('ema', tags, StockCruncher::Stats.list_ema(serie), date)
|
75
|
+
write('lwma', tags, StockCruncher::Stats.list_lwma(serie), date)
|
76
|
+
write('sma', tags, StockCruncher::Stats.list_sma(serie), date)
|
77
|
+
write('vwma', tags, StockCruncher::Stats.list_vwma(serie, weights), date)
|
78
|
+
end
|
79
|
+
|
80
|
+
# Method to query data in bucket
|
81
|
+
def query(name, symbol, values, full)
|
82
|
+
url = "#{@cfg['scheme']}://#{@cfg['host']}:#{@cfg['port']}/query?" \
|
83
|
+
"db=#{@cfg['dbname']}"
|
84
|
+
size = full ? '' : 'LIMIT 201'
|
85
|
+
body = "q=SELECT #{values.join(',')} FROM #{name} " \
|
86
|
+
"WHERE symbol = '#{symbol}' ORDER BY time DESC #{size}"
|
87
|
+
data = JSON.parse(request(url, body).body)['results'][0]['series']
|
88
|
+
raise StandardError, 'No data' if data.nil?
|
89
|
+
|
90
|
+
data[0]
|
39
91
|
end
|
40
92
|
|
41
93
|
# Method to send http post request
|
@@ -54,7 +106,7 @@ module StockCruncher
|
|
54
106
|
def write(name, tags, values, date)
|
55
107
|
url = "#{@cfg['scheme']}://#{@cfg['host']}:#{@cfg['port']}/write?" \
|
56
108
|
"db=#{@cfg['dbname']}"
|
57
|
-
timestamp = DateTime.parse(date
|
109
|
+
timestamp = DateTime.parse("#{date}T18:00:00").strftime('%s%N')
|
58
110
|
body = "#{name},#{format_values(tags)} #{format_values(values)} " \
|
59
111
|
"#{timestamp}"
|
60
112
|
request(url, body)
|
@@ -0,0 +1,74 @@
|
|
1
|
+
#!/usr/bin/ruby
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
module StockCruncher
|
5
|
+
# this is a module with various statistic calculation methods
|
6
|
+
module Stats
|
7
|
+
extend self
|
8
|
+
|
9
|
+
RANGES = [5, 10, 20, 30, 50, 100, 200].freeze
|
10
|
+
|
11
|
+
# Calculate multiple range of exponential moving average
|
12
|
+
def list_ema(values)
|
13
|
+
h = {}
|
14
|
+
RANGES.each do |n|
|
15
|
+
next if values.size < n + 1
|
16
|
+
|
17
|
+
h["ema#{n}"] = ema(values[0, n + 1])
|
18
|
+
end
|
19
|
+
h
|
20
|
+
end
|
21
|
+
|
22
|
+
# Calculate multiple range of linearly weighted moving average
|
23
|
+
def list_lwma(values)
|
24
|
+
h = {}
|
25
|
+
RANGES.each do |n|
|
26
|
+
next if values.size < n
|
27
|
+
|
28
|
+
weights = (1..n).to_a.reverse
|
29
|
+
h["lwma#{n}"] = sma(values[0, n], weights)
|
30
|
+
end
|
31
|
+
h
|
32
|
+
end
|
33
|
+
|
34
|
+
# Calculate multiple range of simple moving average
|
35
|
+
def list_sma(values)
|
36
|
+
h = {}
|
37
|
+
RANGES.each do |n|
|
38
|
+
next if values.size < n
|
39
|
+
|
40
|
+
h["sma#{n}"] = sma(values[0, n])
|
41
|
+
end
|
42
|
+
h
|
43
|
+
end
|
44
|
+
|
45
|
+
# Calculate multiple range of volume weighted moving average
|
46
|
+
def list_vwma(values, volumes)
|
47
|
+
h = {}
|
48
|
+
RANGES.each do |n|
|
49
|
+
next if values.size < n
|
50
|
+
|
51
|
+
h["vwma#{n}"] = sma(values[0, n], volumes[0, n])
|
52
|
+
end
|
53
|
+
h
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
# Calculate exponential moving average
|
59
|
+
def ema(array, factor = 2, weights = nil)
|
60
|
+
f = factor.to_f / array.size
|
61
|
+
n = array.size - 1
|
62
|
+
tsma = sma(array[0, n], weights)
|
63
|
+
ysma = sma(array[1, n], weights)
|
64
|
+
(tsma * f + ysma * (1 - f)).round(4)
|
65
|
+
end
|
66
|
+
|
67
|
+
# Calculate simple moving average
|
68
|
+
def sma(array, weights = nil)
|
69
|
+
factor = weights.nil? ? Array.new(array.size, 1) : weights
|
70
|
+
dividend = array.each_with_index.map { |v, i| v * factor[i] }
|
71
|
+
(dividend.sum.to_f / factor.sum).round(4)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -26,6 +26,12 @@ describe StockCruncher::CLI do # rubocop:disable Metrics/BlockLength
|
|
26
26
|
end
|
27
27
|
end
|
28
28
|
|
29
|
+
context 'movingaverages SYM -c spec/files/stockcruncher.yml' do
|
30
|
+
it 'Get the daily time serie for SYM.' do
|
31
|
+
expect { start(self) }.to output('').to_stdout
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
29
35
|
context 'quote NODATA -c spec/files/stockcruncher.yml' do
|
30
36
|
it 'Should not get any data and should fail.' do
|
31
37
|
expect { start(self) }.to raise_error(SystemExit)
|
@@ -12,6 +12,11 @@ daily = "timestamp,open,high,low,close,volume\r\n2020-08-03,23.8500,24.6500," \
|
|
12
12
|
"\n2020-07-29,24.7500,25.0300,24.0400,24.5600,4605804\r\n2020-07-28," \
|
13
13
|
"25.9900,26.0700,24.5100,24.7500,6904261\r\n"
|
14
14
|
d_err = "{\n \"Error Message\": \"Invalid API call.\"\n}"
|
15
|
+
mvavg = '{"results":[{"statement_id":0,"series":[{"name":"daily","columns":[' \
|
16
|
+
'"time","close","change","changePercent","volume"],"values":[["2017-' \
|
17
|
+
'03-01T18:00:00Z",1,0,0,1],["2017-03-02T18:00:00Z",1,0,0,1],["2017-0' \
|
18
|
+
'3-03T18:00:00Z",1,0,0,1],["2017-03-04T18:00:00Z",1,0,0,1],["2017-03' \
|
19
|
+
'-05T18:00:00Z",1,0,0,1],["2017-03-06T18:00:00Z",1,0,0,1]]}]}]}'
|
15
20
|
|
16
21
|
RSpec.configure do |config|
|
17
22
|
config.before(:each) do
|
@@ -30,6 +35,8 @@ RSpec.configure do |config|
|
|
30
35
|
'function=TIME_SERIES_DAILY&symbol=SYM&apikey=demo' \
|
31
36
|
'&datatype=csv&outputsize=compact')
|
32
37
|
.to_return('status' => 200, 'body' => daily, 'headers' => {})
|
38
|
+
stub_request(:post, 'http://localhost:8086/query?db=test')
|
39
|
+
.to_return('status' => 204, 'body' => mvavg, 'headers' => {})
|
33
40
|
stub_request(:post, 'http://localhost:8086/write?db=test')
|
34
41
|
.to_return('status' => 204, 'body' => '', 'headers' => {})
|
35
42
|
end
|
data/stockcruncher.gemspec
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: stockcruncher
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Richard Delaplace
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-08-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -58,14 +58,14 @@ dependencies:
|
|
58
58
|
requirements:
|
59
59
|
- - '='
|
60
60
|
- !ruby/object:Gem::Version
|
61
|
-
version: '0.
|
61
|
+
version: '0.90'
|
62
62
|
type: :development
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
66
|
- - '='
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '0.
|
68
|
+
version: '0.90'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: webmock
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -129,8 +129,10 @@ files:
|
|
129
129
|
- lib/stockcruncher.rb
|
130
130
|
- lib/stockcruncher/alphavantage.rb
|
131
131
|
- lib/stockcruncher/cli.rb
|
132
|
+
- lib/stockcruncher/config.rb
|
132
133
|
- lib/stockcruncher/cruncher.rb
|
133
134
|
- lib/stockcruncher/influxdb.rb
|
135
|
+
- lib/stockcruncher/stats.rb
|
134
136
|
- lib/stockcruncher/version.rb
|
135
137
|
- spec/files/SYM.daily
|
136
138
|
- spec/files/SYM.quote
|