cotcube-bardata 0.1.14 → 0.1.15.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +29 -0
- data/README.md +1 -1
- data/VERSION +1 -1
- data/cotcube-bardata.gemspec +2 -2
- data/lib/cotcube-bardata/cached.rb +3 -2
- data/lib/cotcube-bardata/daily.rb +165 -25
- data/lib/cotcube-bardata/eods.rb +3 -3
- data/lib/cotcube-bardata/helpers.rb +0 -36
- data/lib/cotcube-bardata/provide.rb +7 -1
- data/lib/cotcube-bardata/quarters.rb +2 -1
- data/lib/cotcube-bardata/range_matrix.rb +1 -1
- data/lib/cotcube-bardata/suggest.rb +1 -1
- data/lib/cotcube-bardata/trade_dates.rb +38 -12
- data/lib/cotcube-bardata/trading_hours.rb +11 -5
- data/lib/cotcube-bardata.rb +1 -4
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5798e21ced6b815404638c1f33948317338d801ea483e9e43edf21b8ad3f73ba
|
4
|
+
data.tar.gz: 8bfaf9b663d8de85f8c89b459b3f33e35ce836cad2d0153a02385aaafc4d6781
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f554feb61ac91f542a9e9a24a4f9983cbfd529ace02290b531d07bb67cbe5864da1e154e023b9d00dec77b91b2243efb726d4bd005fee6cdc1af88701de583a8
|
7
|
+
data.tar.gz: 32a0446523adb94cc50426c0e03b3bd379746754d5314fe79af8ff02fe169fae95fc3a1ba894a313e5b8b7ac89a2da36d5605e0183b9e0ef8022550e00fc5449
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,32 @@
|
|
1
|
+
## 0.1.15.6 (November 19, 2021)
|
2
|
+
- daily: added filter_series
|
3
|
+
|
4
|
+
## 0.1.15.5 (November 08, 2021)
|
5
|
+
- decommissioned .get_id_set in favor of Cotcube::Helpers.get_id_set
|
6
|
+
|
7
|
+
## 0.1.15.4 (October 26, 2021)
|
8
|
+
- daily: including the sourcing factor (:bcf) into provide_daily and continuous
|
9
|
+
|
10
|
+
## 0.1.15.3 (August 04, 2021)
|
11
|
+
- daily.rb: Hotfixing
|
12
|
+
|
13
|
+
## 0.1.15.2 (August 04, 2021)
|
14
|
+
- fixed leftover debug setting
|
15
|
+
|
16
|
+
## 0.1.15.1 (August 04, 2021)
|
17
|
+
- fixed license mismatch
|
18
|
+
|
19
|
+
## 0.1.15 (August 04, 2021)
|
20
|
+
- daily.rb: added support to add eod data on incomplete dailies
|
21
|
+
- adding dep: cotcube-indicators
|
22
|
+
- provide: new method :determine_significant_volume
|
23
|
+
- suggest: adding silence
|
24
|
+
- cached: Adding :dist
|
25
|
+
- daily: new method :determine_significant_volume
|
26
|
+
- added :dist generically to quarters
|
27
|
+
- trading_hours: added param to return headers only
|
28
|
+
- last_trade_date: Enabled caching instead of fetching each time
|
29
|
+
|
1
30
|
## 0.1.14 (May 07, 2021)
|
2
31
|
- few changes in provide / cached / daily for a more straigtforward forcing of cache renewal
|
3
32
|
- suggest: new method to suggest a contract for given symbol and date
|
data/README.md
CHANGED
@@ -90,5 +90,5 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/donkey
|
|
90
90
|
|
91
91
|
## License
|
92
92
|
|
93
|
-
The gem is available as open source under the terms of the [
|
93
|
+
The gem is available as open source under the terms of the [BSD-3-Clause-License](https://opensource.org/licenses/BSD-3-Clause).
|
94
94
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.1.
|
1
|
+
0.1.15.6
|
data/cotcube-bardata.gemspec
CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.description = 'Functions to provide bardata; and some simple time series aggregations '
|
11
11
|
|
12
12
|
spec.homepage = "https://github.com/donkeybridge/#{spec.name}"
|
13
|
-
spec.license = 'BSD-
|
13
|
+
spec.license = 'BSD-3-Clause'
|
14
14
|
spec.required_ruby_version = Gem::Requirement.new('~> 2.7')
|
15
15
|
|
16
16
|
spec.metadata['homepage_uri'] = spec.homepage
|
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
|
|
28
28
|
|
29
29
|
spec.add_dependency 'activesupport', '~> 6'
|
30
30
|
spec.add_dependency 'colorize', '~> 0.8'
|
31
|
-
spec.add_dependency 'cotcube-helpers', '~> 0.
|
31
|
+
spec.add_dependency 'cotcube-helpers', '~> 0.2'
|
32
32
|
spec.add_dependency 'cotcube-indicators', '~> 0.1'
|
33
33
|
spec.add_dependency 'httparty', '~> 0.18'
|
34
34
|
spec.add_dependency 'parallel', '~> 1'
|
@@ -30,7 +30,7 @@ module Cotcube
|
|
30
30
|
end
|
31
31
|
|
32
32
|
headers = %i[contract datetime open high low close volume]
|
33
|
-
sym = get_id_set(symbol: symbol, id: id, contract: contract)
|
33
|
+
sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)
|
34
34
|
contract = contract[-3..]
|
35
35
|
dir = "#{config[:data_path]}/cached/#{sym[:id]}_#{filter.to_s.downcase}"
|
36
36
|
symlink = "#{config[:data_path]}/cached/#{sym[:symbol]}_#{filter.to_s.downcase}"
|
@@ -40,12 +40,13 @@ module Cotcube
|
|
40
40
|
quarters_file = "#{config[:data_path]}/quarters/#{sym[:id]}/#{contract[-3..]}.csv"
|
41
41
|
if File.exist?(file) && (not force_update)
|
42
42
|
puts "Working with existing #{file}, no update was forced" if debug
|
43
|
-
puts " Using quarters from #{quarters_file}"
|
43
|
+
puts " Using quarters from #{quarters_file}" if debug
|
44
44
|
base = CSV.read(file, headers: headers).map do |x|
|
45
45
|
x = x.to_h
|
46
46
|
x[:datetime] = timezone.parse(x[:datetime])
|
47
47
|
%i[open high low close].each { |z| x[z] = x[z].to_f.round(9) }
|
48
48
|
x[:volume] = x[:volume].to_i
|
49
|
+
x[:dist] = ((x[:high] - x[:low]) / sym[:ticksize] ).to_i
|
49
50
|
x[:type] = "#{filter.to_s.downcase}_day".to_sym
|
50
51
|
x
|
51
52
|
end
|
@@ -9,8 +9,11 @@ module Cotcube
|
|
9
9
|
range: nil,
|
10
10
|
timezone: Time.find_zone('America/Chicago'),
|
11
11
|
keep_last: false,
|
12
|
+
add_eods: true,
|
13
|
+
indicators: {},
|
12
14
|
config: init)
|
13
15
|
contract = contract.to_s.upcase
|
16
|
+
rounding = 8
|
14
17
|
unless contract.is_a?(String) && [3, 5].include?(contract.size)
|
15
18
|
raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
|
16
19
|
end
|
@@ -30,7 +33,7 @@ module Cotcube
|
|
30
33
|
range = (range_begin..range_end)
|
31
34
|
end
|
32
35
|
|
33
|
-
sym = get_id_set(symbol: symbol, id: id, contract: contract)
|
36
|
+
sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)
|
34
37
|
contract = contract[2..4] if contract.to_s.size == 5
|
35
38
|
id = sym[:id]
|
36
39
|
id_path = "#{config[:data_path]}/daily/#{id}"
|
@@ -42,14 +45,46 @@ module Cotcube
|
|
42
45
|
data = CSV.read(data_file, headers: %i[contract date open high low close volume oi]).map do |row|
|
43
46
|
row = row.to_h
|
44
47
|
row.each do |k, _|
|
45
|
-
|
48
|
+
if %i[open high low close].include? k
|
49
|
+
row[k] = row[k].to_f
|
50
|
+
row[k] = (row[k] * sym[:bcf]).round(8) unless sym[:bcf] == 1.0
|
51
|
+
end
|
46
52
|
row[k] = row[k].to_i if %i[volume oi].include? k
|
47
53
|
end
|
48
54
|
row[:datetime] = timezone.parse(row[:date])
|
55
|
+
row[:dist] = ((row[:high] - row[:low]) / sym[:ticksize] ).to_i
|
49
56
|
row[:type] = :daily
|
50
57
|
row
|
51
58
|
end
|
52
|
-
|
59
|
+
contract_expired = data.last[:high].zero?
|
60
|
+
data.pop if contract_expired and not keep_last
|
61
|
+
if not contract_expired and add_eods
|
62
|
+
today = Date.today
|
63
|
+
eods = [ ]
|
64
|
+
while today.strftime('%Y-%m-%d') > data.last[:date]
|
65
|
+
eods << provide_eods(symbol: sym[:symbol], dates: today, contracts_only: false, quiet: true)
|
66
|
+
today -= 1
|
67
|
+
end
|
68
|
+
eods.flatten!.map!{|x| x.tap {|y| %i[ volume_part oi_part ].map{|z| y.delete(z)} } }
|
69
|
+
eods.select!{|x| x[:contract] == "#{sym[:symbol]}#{contract}" }
|
70
|
+
eods.map!{|x| x.tap{|y|
|
71
|
+
if sym[:bcf] != 1.0
|
72
|
+
%i[open high low close].map{|k|
|
73
|
+
y[k] = (y[k] * sym[:bcf]).round(8)
|
74
|
+
}
|
75
|
+
end
|
76
|
+
y[:datetime] = timezone.parse(y[:date])
|
77
|
+
y[:dist] = ((y[:high] - y[:low]) / sym[:ticksize] ).to_i
|
78
|
+
y[:type] = :eod
|
79
|
+
} }
|
80
|
+
data += eods.reverse
|
81
|
+
end
|
82
|
+
data.map do |bar|
|
83
|
+
indicators.each do |k,v|
|
84
|
+
tmp = v.call(bar)
|
85
|
+
bar[k] = tmp.respond_to?(:round) ? tmp.round(rounding) : tmp
|
86
|
+
end
|
87
|
+
end unless indicators.empty?
|
53
88
|
if range.nil?
|
54
89
|
data
|
55
90
|
else
|
@@ -62,54 +97,156 @@ module Cotcube
|
|
62
97
|
|
63
98
|
# reads all files in bardata/daily/<id> and aggregates by date
|
64
99
|
# (what is a pre-stage of a continuous based on daily bars)
|
65
|
-
def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false)
|
100
|
+
def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false, selector: nil, debug: false, add_eods: true, indicators: nil)
|
66
101
|
raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
|
67
102
|
measuring = lambda {|c| puts "[continuous] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
|
68
103
|
|
69
104
|
measuring.call("Starting")
|
70
|
-
sym = get_id_set(symbol: symbol, id: id)
|
105
|
+
sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
|
71
106
|
id = sym[:id]
|
72
107
|
symbol = sym[:symbol]
|
108
|
+
ticksize = sym[:ticksize]
|
109
|
+
effective_selector = selector || :volume
|
110
|
+
raise ArgumentError, 'selector must be in %i[ nil :volume ;oi].' unless [ nil, :volume, :oi ].include? selector
|
73
111
|
id_path = "#{config[:data_path]}/daily/#{id}"
|
74
112
|
c_file = "#{id_path}/continuous.csv"
|
113
|
+
puts "Using file #{c_file}" if debug
|
75
114
|
|
76
115
|
# instead of using the provide_daily methods above, for this bulk operation a 'continuous.csv' is created
|
77
116
|
# this boosts from 4.5sec to 0.3sec
|
78
|
-
rewriting = force_rewrite or not(File.exist?(c_file)) or (Time.now - File.mtime(c_file) > 8.days)
|
117
|
+
rewriting = (force_rewrite or not(File.exist?(c_file)) or (Time.now - File.mtime(c_file) > 8.days))
|
79
118
|
if rewriting
|
80
119
|
puts "In daily+continuous: Rewriting #{c_file} #{force_rewrite ? "forcibly" : "due to fileage"}.".light_yellow
|
81
|
-
`rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v '
|
120
|
+
`rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v ',0,' | grep -v ',0$'| sort -t, -k2 | cut -d, -f1-8 | grep ',.*,' | uniq > #{c_file}`
|
82
121
|
end
|
83
122
|
loading = lambda do
|
84
123
|
data = CSV.read(c_file).map do |row|
|
85
124
|
r = { contract: row[0],
|
86
125
|
date: row[1],
|
87
|
-
|
88
|
-
|
126
|
+
open: row[2],
|
127
|
+
high: row[3],
|
128
|
+
low: row[4],
|
129
|
+
close: row[5],
|
130
|
+
volume: row[6].to_i,
|
131
|
+
oi: row[7].to_i,
|
132
|
+
type: :cont
|
89
133
|
}
|
90
134
|
end
|
135
|
+
if add_eods
|
136
|
+
today = Date.today
|
137
|
+
eods = [ ]
|
138
|
+
while today.strftime('%Y-%m-%d') > data.last[:date]
|
139
|
+
eods << provide_eods(symbol: symbol, dates: today, contracts_only: false, quiet: true)
|
140
|
+
today -= 1
|
141
|
+
end
|
142
|
+
eods.flatten!.map!{|x| x.tap {|y| %i[ volume_part oi_part ].map{|z| y.delete(z)} } }
|
143
|
+
eods.delete_if { |elem| elem.flatten.empty? }
|
144
|
+
data += eods.reverse
|
145
|
+
end
|
91
146
|
|
92
147
|
measuring.call("Finished retrieving dailies.")
|
93
148
|
result = []
|
149
|
+
rounding = 8 # sym[:format].split('.').last.to_i rescue 6
|
150
|
+
indicators ||= {
|
151
|
+
typical: Cotcube::Indicators.calc(a: :high, b: :low, c: :close) {|high, low, close| (high + low + close) / 3 },
|
152
|
+
sma250_high: Cotcube::Indicators.sma(key: :high, length: 250),
|
153
|
+
sma250_low: Cotcube::Indicators.sma(key: :low, length: 250),
|
154
|
+
sma250_typ: Cotcube::Indicators.sma(key: :typical, length: 250),
|
155
|
+
dist: Cotcube::Indicators.calc(a: :high, b: :low, finalize: :to_i) {|high, low| ((high-low) / ticksize) },
|
156
|
+
}
|
157
|
+
|
158
|
+
|
159
|
+
|
94
160
|
data.group_by { |x| x[:date] }.map do |k, v|
|
95
161
|
v.map { |x| x.delete(:date) }
|
96
|
-
|
162
|
+
avg_bar = {
|
97
163
|
date: k,
|
98
|
-
|
99
|
-
|
164
|
+
contract: v.max_by{|x| x[:oi] }[:contract],
|
165
|
+
open: nil, high: nil, low: nil, close: nil,
|
166
|
+
volume: v.map { |x| x[:volume] }.reduce(:+),
|
167
|
+
oi: v.map { |x| x[:oi] }.reduce(:+),
|
168
|
+
type: :cont_eod
|
169
|
+
}
|
170
|
+
|
171
|
+
%i[ open high low close ].each do |ohlc|
|
172
|
+
avg_bar[ohlc] = (v.map{|x| x[ohlc].to_f * x[effective_selector] }.reduce(:+) / avg_bar[effective_selector]).round(rounding)
|
173
|
+
avg_bar[ohlc] = (avg_bar[ohlc] * sym[:bcf]).round(8) unless sym[:bcf] == 1.0
|
174
|
+
|
175
|
+
end
|
176
|
+
p avg_bar if debug
|
177
|
+
indicators.each do |k,v|
|
178
|
+
print format('%12s: ', k.to_s) if debug
|
179
|
+
tmp = v.call(avg_bar)
|
180
|
+
avg_bar[k] = tmp.respond_to?(:round) ? tmp.round(rounding) : tmp
|
181
|
+
puts avg_bar[k] if debug
|
182
|
+
end
|
183
|
+
%i[tr atr5].each { |ind|
|
184
|
+
avg_bar[ind] = (avg_bar[ind] / sym[:ticksize]).round.to_i unless avg_bar[ind].nil?
|
100
185
|
}
|
186
|
+
result << avg_bar
|
101
187
|
result.last[:contracts] = v
|
102
188
|
end
|
103
189
|
result
|
104
190
|
end
|
105
|
-
constname = "CONTINUOUS_#{symbol}".to_sym
|
191
|
+
constname = "CONTINUOUS_#{symbol}#{selector.nil? ? '' : ('_' + selector.to_s)}".to_sym
|
106
192
|
if rewriting or not Cotcube::Bardata.const_defined?( constname)
|
193
|
+
old = $VERBOSE; $VERBOSE = nil
|
107
194
|
Cotcube::Bardata.const_set constname, loading.call
|
195
|
+
$VERBOSE = old
|
108
196
|
end
|
109
197
|
measuring.call("Finished processing")
|
110
198
|
date.nil? ? Cotcube::Bardata.const_get(constname).map{|z| z.dup } : Cotcube::Bardata.const_get(constname).find { |x| x[:date] == date }
|
111
199
|
end
|
112
200
|
|
201
|
+
# the filter series is an indicator based on the Cotcube::Bardata.continuous of the asset price.
|
202
|
+
# current default filter is the ema50
|
203
|
+
def filter_series(ema_length: 50, symbol: , print_range: nil)
|
204
|
+
ema_high_n = "ema#{ema_length}_high".to_sym
|
205
|
+
ema_low_n = "ema#{ema_length}_low".to_sym
|
206
|
+
ema_filter = "ema#{ema_length}_filter".to_sym
|
207
|
+
indicators = {
|
208
|
+
ema_high_n => Cotcube::Indicators.ema(key: :high, length: ema_length, smoothing: 2),
|
209
|
+
ema_low_n => Cotcube::Indicators.ema(key: :low, length: ema_length, smoothing: 2),
|
210
|
+
:tr => Cotcube::Indicators.true_range,
|
211
|
+
:atr5 => Cotcube::Indicators.ema(key: :tr, length: 5, smoothing: 2),
|
212
|
+
ema_filter => Cotcube::Indicators.calc(a: :high, b: :low, c: :close,
|
213
|
+
d: ema_high_n, e: ema_low_n, f: :atr5,
|
214
|
+
finalize: :to_i) do |high, low, close, ema_high, ema_low, atr5|
|
215
|
+
|
216
|
+
if close > ema_high and (low - ema_high).abs <= atr5 / 5.0; 3 # :bullish_tipped
|
217
|
+
elsif low > ema_high and (low - ema_high).abs >= atr5 * 3.0; 5 # :bullish_away
|
218
|
+
elsif low > ema_high and (low - ema_high).abs <= atr5 / 1.5; 2 # :bullish_nearby
|
219
|
+
elsif low > ema_high; 4 # :bullish
|
220
|
+
|
221
|
+
elsif close < ema_low and (high - ema_low).abs <= atr5 / 5.0; -3 # :bearish_tipped
|
222
|
+
elsif high < ema_low and (high - ema_low).abs >= atr5 * 3.0; -5 # :bearish_away
|
223
|
+
elsif high < ema_low and (high - ema_low).abs <= atr5 / 1.5; -2 # :bearish_nearby
|
224
|
+
elsif high < ema_low; -4 # :bearish
|
225
|
+
|
226
|
+
elsif close >= ema_high and (close - ema_high).abs > atr5 ; 2 # :bullish_closed
|
227
|
+
elsif close <= ema_low and (close - ema_low ).abs > atr5 ; -2 # :bearish_closed
|
228
|
+
elsif close >= ema_high; 1 # :bullish_weak
|
229
|
+
elsif close <= ema_low; -1 # :bearish_weak
|
230
|
+
elsif close > ema_low and close < ema_high; 0 # :ambigue
|
231
|
+
else
|
232
|
+
raise RuntimeError, "Unconsidered Indicator value with #{high}, #{low}, #{close}, #{ema_high}, #{ema_low}, #{atr5}"
|
233
|
+
|
234
|
+
end
|
235
|
+
end
|
236
|
+
}
|
237
|
+
filter = Cotcube::Bardata.continuous(symbol: symbol, indicators: indicators).
|
238
|
+
map{ |z| z[:datetime] = DateTime.parse(z[:date]); z[:datetime] += z[:datetime].wday == 5 ? 3 : 1; z.slice(:datetime, ema_filter) }.
|
239
|
+
group_by{ |z| z[:datetime] }.
|
240
|
+
map{ |k,v| [ k, v[0][ema_filter] ] }.
|
241
|
+
to_h.
|
242
|
+
tap{ |z| z.to_a[print_range].each { |v|
|
243
|
+
puts "#{v[0].strftime('%Y-%m-%d')
|
244
|
+
} : #{format '%2d', v[1]
|
245
|
+
}".colorize(v[1] > 3 ? :light_green : v[1] > 1 ? :green : v[1] < -3 ? :light_red : v[1] < -1 ? :red : :white )
|
246
|
+
} if print_range.is_a? Range
|
247
|
+
}
|
248
|
+
end
|
249
|
+
|
113
250
|
def continuous_ml(symbol: nil, id: nil, base: nil)
|
114
251
|
(base.nil? ? Cotcube::Bardata.continuous(symbol: symbol, id: id) : base).map do |x|
|
115
252
|
x[:ml] = x[:contracts].max_by { |z| z[:volume] }[:contract]
|
@@ -150,7 +287,7 @@ module Cotcube
|
|
150
287
|
%i[volume oi].include?(selector)
|
151
288
|
|
152
289
|
measuring.call("Starting")
|
153
|
-
sym = get_id_set(symbol: symbol, id: id)
|
290
|
+
sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
|
154
291
|
id = sym[:id]
|
155
292
|
# noinspection RubyNilAnalysis
|
156
293
|
data = continuous(id: id, config: config, measure: measure).map do |x|
|
@@ -214,7 +351,7 @@ module Cotcube
|
|
214
351
|
%i[volume oi].include?(selector)
|
215
352
|
|
216
353
|
measuring.call("Entering function")
|
217
|
-
sym = get_id_set(symbol: symbol, id: id)
|
354
|
+
sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
|
218
355
|
if %w[R6 BJ GE].include? sym[:symbol]
|
219
356
|
puts "Rejecting to process symbol '#{sym[:symbol]}'.".light_red
|
220
357
|
return []
|
@@ -230,17 +367,18 @@ module Cotcube
|
|
230
367
|
|
231
368
|
ytoday = date.yday
|
232
369
|
data = continuous_overview(id: id, selector: selector, filter: filter, human: false, config: init, measure: measure)
|
233
|
-
.reject { |k, _| k[-2..].to_i
|
370
|
+
.reject { |k, _| k[-2..].to_i >= date.year % 2000 }
|
234
371
|
.group_by { |k, _| k[2] }
|
235
372
|
measuring.call("Retrieved continous_overview")
|
236
373
|
output_sent = []
|
237
374
|
early_year=nil
|
238
375
|
long_output = []
|
239
376
|
data.keys.sort.each do |month|
|
240
|
-
current_long = { month: month }
|
241
377
|
puts "Processing #{sym[:symbol]}#{month}" if debuglevel > 1
|
242
378
|
v0 = data[month]
|
379
|
+
# ldays is the list of 'last days'
|
243
380
|
ldays = v0.map { |_, v1| Date.parse(v1.last[:date]).yday }
|
381
|
+
# fdays is the list of 'first days'
|
244
382
|
fdays = v0.map { |_, v1| Date.parse(v1.first[:date]).yday }.sort
|
245
383
|
# if the last ml day nears the end of the year, we must fix
|
246
384
|
ldays.map! { |x| x > 350 ? x - 366 : x } if ldays.min < 50
|
@@ -277,9 +415,9 @@ module Cotcube
|
|
277
415
|
}\tto #{format '%5d', ldays.max
|
278
416
|
}: #{dfm.call(ldays.max)}".colorize(color)
|
279
417
|
if debug || (color != :white)
|
280
|
-
puts output
|
281
|
-
output_sent << "#{sym[:symbol]}#{month}" unless color == :white
|
418
|
+
puts output unless silent
|
282
419
|
end
|
420
|
+
output_sent << "#{sym[:symbol]}#{month}" unless color == :white
|
283
421
|
early_year ||= output
|
284
422
|
next if silent or not (debug and debuglevel >= 2)
|
285
423
|
|
@@ -288,20 +426,22 @@ module Cotcube
|
|
288
426
|
}\t#{v1.first[:date]
|
289
427
|
} (#{format '%3d', Date.parse(v1.first[:date]).yday
|
290
428
|
})\t#{Date.parse(v1.last[:date]).strftime('%a, %Y-%m-%d')
|
291
|
-
} (#{Date.parse(v1.last[:date]).yday})"
|
429
|
+
} (#{Date.parse(v1.last[:date]).yday})" unless silent
|
292
430
|
# rubocop:enable Layout/ClosingParenthesisIndentation
|
293
431
|
end
|
294
|
-
|
432
|
+
end
|
295
433
|
case output_sent.size
|
296
434
|
when 0
|
297
|
-
|
298
|
-
|
299
|
-
|
435
|
+
unless silent
|
436
|
+
puts "WARNING: No output was sent for symbol '#{sym[:symbol]}'.".colorize(:light_yellow)
|
437
|
+
puts " Assuming late-year-processing.".light_yellow
|
438
|
+
puts early_year.light_green
|
439
|
+
end
|
300
440
|
when 1
|
301
441
|
# all ok
|
302
442
|
true
|
303
443
|
else
|
304
|
-
puts "Continuous table show #{output_sent.size} active contracts ( #{output_sent} ) for #{sym[:symbol]} ---------------"
|
444
|
+
puts "Continuous table show #{output_sent.size} active contracts ( #{output_sent} ) for #{sym[:symbol]} ---------------" unless silent
|
305
445
|
end
|
306
446
|
measuring.call("Finished processing")
|
307
447
|
short ? output_sent : long_output
|
data/lib/cotcube-bardata/eods.rb
CHANGED
@@ -4,7 +4,7 @@ module Cotcube
|
|
4
4
|
# Missing top level documentation
|
5
5
|
module Bardata
|
6
6
|
def most_liquid_for(symbol: nil, id: nil, date: last_trade_date, config: init)
|
7
|
-
id = get_id_set(symbol: symbol, id: id, config: config)[:id]
|
7
|
+
id = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, config: config)[:id]
|
8
8
|
provide_eods(id: id, dates: date, contracts_only: true).first
|
9
9
|
end
|
10
10
|
|
@@ -15,7 +15,7 @@ module Cotcube
|
|
15
15
|
date: last_trade_date,
|
16
16
|
filter: :volume_part,
|
17
17
|
age: 1.hour)
|
18
|
-
sym = get_id_set(symbol: symbol, id: id) if symbol || id
|
18
|
+
sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id) if symbol || id
|
19
19
|
# noinspection RubyScope
|
20
20
|
eods = provide_eods(id: sym.nil? ? nil : sym[:id], config: config, dates: date, filter: filter)
|
21
21
|
result = []
|
@@ -62,7 +62,7 @@ module Cotcube
|
|
62
62
|
end
|
63
63
|
|
64
64
|
symbol = contract[0..1] if contract.to_s.size == 5
|
65
|
-
sym = get_id_set(symbol: symbol, id: id, config: config) if symbol || id
|
65
|
+
sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, config: config) if symbol || id
|
66
66
|
# if no id can be clarified from given arguments, return all matching contracts from all available symbols
|
67
67
|
# raise ArgumentError, "Could not guess :id or :symbol from 'contract: #{contract}', please clarify." if id.nil?
|
68
68
|
raise ArgumentError, ':filter must be in [:volume_part, :oi_part]' unless %i[volume_part oi_part].include? filter
|
@@ -45,42 +45,6 @@ module Cotcube
|
|
45
45
|
result
|
46
46
|
end
|
47
47
|
|
48
|
-
def get_id_set(symbol: nil, id: nil, contract: nil, config: init)
|
49
|
-
contract = contract.to_s.upcase if contract.is_a? Symbol
|
50
|
-
id = id.to_s.upcase if id.is_a? Symbol
|
51
|
-
symbol = symbol.to_s.upcase if symbol.is_a? Symbol
|
52
|
-
|
53
|
-
if contract.is_a?(String) && (contract.length == 5)
|
54
|
-
c_symbol = contract[0..1]
|
55
|
-
if (not symbol.nil?) && (symbol != c_symbol)
|
56
|
-
raise ArgumentError,
|
57
|
-
"Mismatch between given symbol #{symbol} and contract #{contract}"
|
58
|
-
end
|
59
|
-
|
60
|
-
symbol = c_symbol
|
61
|
-
end
|
62
|
-
|
63
|
-
unless symbol.nil?
|
64
|
-
sym = symbols.select { |s| s[:symbol] == symbol.to_s.upcase }.first
|
65
|
-
if sym.nil? || sym[:id].nil?
|
66
|
-
raise ArgumentError,
|
67
|
-
"Could not find match in #{config[:symbols_file]} for given symbol #{symbol}"
|
68
|
-
end
|
69
|
-
raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if (not id.nil?) && (sym[:id] != id)
|
70
|
-
|
71
|
-
return sym
|
72
|
-
end
|
73
|
-
unless id.nil?
|
74
|
-
sym = symbols.select { |s| s[:id] == id.to_s }.first
|
75
|
-
if sym.nil? || sym[:id].nil?
|
76
|
-
raise ArgumentError,
|
77
|
-
"Could not find match in #{config[:symbols_file]} for given id #{id}"
|
78
|
-
end
|
79
|
-
return sym
|
80
|
-
end
|
81
|
-
raise ArgumentError, 'Need :id, :symbol or valid :contract '
|
82
|
-
end
|
83
|
-
|
84
48
|
def compare(contract:, format: '%5.2f')
|
85
49
|
format = "%#{format}" unless format[0] == '%'
|
86
50
|
daily = provide(contract: contract, interval: :daily)
|
@@ -17,7 +17,7 @@ module Cotcube
|
|
17
17
|
force_update: false,
|
18
18
|
force_recent: false)
|
19
19
|
|
20
|
-
sym = get_id_set(symbol: symbol, id: id, contract: contract, config: config)
|
20
|
+
sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract, config: config)
|
21
21
|
|
22
22
|
case interval
|
23
23
|
when :quarters, :hours, :quarter, :hour
|
@@ -54,5 +54,11 @@ module Cotcube
|
|
54
54
|
raise ArgumentError, "Unsupported or unknown interval '#{interval}' in Bardata.provide"
|
55
55
|
end
|
56
56
|
end
|
57
|
+
|
58
|
+
def determine_significant_volume(base: , contract: )
|
59
|
+
set = Cotcube::Bardata.trading_hours(symbol: contract[0..1], filter: :rth)
|
60
|
+
prod = base - base.select_within(ranges: set ,attr: :datetime) {|x| x.to_datetime.to_sssm }
|
61
|
+
prod.group_by{|x| x[:volume] / 500 }
|
62
|
+
end
|
57
63
|
end
|
58
64
|
end
|
@@ -16,7 +16,7 @@ module Cotcube
|
|
16
16
|
raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
|
17
17
|
end
|
18
18
|
|
19
|
-
sym = get_id_set(symbol: symbol, id: id, contract: contract)
|
19
|
+
sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)
|
20
20
|
|
21
21
|
contract = contract[2..4] if contract.to_s.size == 5
|
22
22
|
id = sym[:id]
|
@@ -33,6 +33,7 @@ module Cotcube
|
|
33
33
|
%i[open high low close].map { |x| row[x] = row[x].to_f }
|
34
34
|
%i[volume day].map { |x| row[x] = row[x].to_i }
|
35
35
|
row[:datetime] = timezone.parse(row[:datetime])
|
36
|
+
row[:dist] = ((row[:high] - row[:low]) / sym[:ticksize] ).to_i
|
36
37
|
row[:type] = :quarter
|
37
38
|
row
|
38
39
|
end
|
@@ -24,7 +24,7 @@ module Cotcube
|
|
24
24
|
def range_matrix(symbol: nil, id: nil, base: nil, print: false, dim: 0.05, days_only: false, last_n: 60, &block)
|
25
25
|
# rubocop:disable Style/MultilineBlockChain
|
26
26
|
symbol ||= base.last[:contract][0..1] if id.nil?
|
27
|
-
sym = get_id_set(symbol: symbol, id: id)
|
27
|
+
sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
|
28
28
|
source = {}
|
29
29
|
target = {}
|
30
30
|
if base.nil?
|
@@ -5,7 +5,7 @@ module Cotcube
|
|
5
5
|
module Bardata
|
6
6
|
# based on day(of year) and symbol, suggest best fitting contract
|
7
7
|
def suggest_contract_for symbol:, date: Date.today, warnings: true
|
8
|
-
ml = Cotcube::Bardata.continuous_table symbol: symbol, date: date
|
8
|
+
ml = Cotcube::Bardata.continuous_table symbol: symbol, date: date, silent: true
|
9
9
|
if ml.size != 1
|
10
10
|
puts "WARNING: No or no unique most liquid found for #{date}. please give :contract parameter".light_yellow if warnings
|
11
11
|
if ml.size > 1
|
@@ -5,19 +5,45 @@ module Cotcube
|
|
5
5
|
module Bardata
|
6
6
|
# fetching official trade dates from CME
|
7
7
|
# it returns the current trade date or, if today isn't a trading day, the last trade date.
|
8
|
-
def last_trade_date
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
"
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
8
|
+
def last_trade_date(force_update: false)
|
9
|
+
const_LTD = :LAST_TRADE_DATE
|
10
|
+
const_LTDU = :LAST_TRADE_DATE_UPDATE
|
11
|
+
if force_update or not Object.const_defined?(const_LTD) or Object.const_get(const_LTD).nil? or Time.now - Object.const_get(const_LTDU) > 2.hours
|
12
|
+
result = nil
|
13
|
+
uri = 'https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME'
|
14
|
+
headers = { "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
|
15
|
+
"Accept-Encoding" => "gzip, deflate, br",
|
16
|
+
"Accept-Language" => "en-US,en;q=0.9",
|
17
|
+
"Cache-Control" => "max-age=0",
|
18
|
+
"Connection" => "keep-alive",
|
19
|
+
# Cookie: ak_bmsc=602078F6DE40954BAA8C7E7D3815102CACE82AAFD237000084B5A460F4FBCA68~pluz010T49Xag3sXquUZtVJmFX701dzEgt5v6Ht1EZSLKE4HL+bgg1L9ePnL5I0mm7QWXe1qaLhUbX1IPrL/f20trRMMRlkC3UWXk27DY/EBCP4mRno8QQygLCwgs2B2AQHJyb63WwRihCko8UYUiIhb89ArPZM5OPraoKy3JU9oE9e+iERdARNZHLHqRiB1GnmbKUvQqos3sXaEe3GpoiTszzk8sHZs4ZKuoO/rvFHko=",
|
20
|
+
"Host" => "www.cmegroup.com",
|
21
|
+
"sec-ch-ua" => %q[" Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"],
|
22
|
+
"sec-ch-ua-mobile" => "?0",
|
23
|
+
"Sec-Fetch-Dest" => "document",
|
24
|
+
"Sec-Fetch-Mode" => "navigate",
|
25
|
+
"Sec-Fetch-Site" => "none",
|
26
|
+
"Sec-Fetch-User" => "?1",
|
27
|
+
"Upgrade-Insecure-Requests" => "1",
|
28
|
+
"User-Agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36" }
|
29
|
+
begin
|
30
|
+
# HTTParty.get(uri, headers: { "User-Agent" => "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"})
|
31
|
+
result = HTTParty.get(uri, headers: headers)
|
32
|
+
.parsed_response
|
33
|
+
.map do |x|
|
34
|
+
a = x['tradeDate'].chars.each_slice(2).map(&:join)
|
35
|
+
"#{a[0]}#{a[1]}-#{a[2]}-#{a[3]}"
|
36
|
+
end
|
37
|
+
.first
|
38
|
+
rescue StandardError
|
39
|
+
result = nil
|
40
|
+
end
|
41
|
+
oldverbose = $VERBOSE; $VERBOSE = nil
|
42
|
+
Object.const_set(const_LTD, result)
|
43
|
+
Object.const_set(const_LTDU, Time.now) unless result.nil?
|
44
|
+
$VERBOSE = oldverbose
|
20
45
|
end
|
46
|
+
Object.const_get(const_LTD)
|
21
47
|
end
|
22
48
|
|
23
49
|
def holidays(config: init)
|
@@ -12,17 +12,23 @@ module Cotcube
|
|
12
12
|
filter: ,
|
13
13
|
force_filter: false, # with force_filter one would avoid falling back
|
14
14
|
# to the contract_type based range set
|
15
|
+
headers_only: false, # return only headers instead of ranges
|
15
16
|
config: init, debug: false)
|
16
17
|
return (0...24 * 7 * 3600) if filter.to_s =~ /24x7/
|
17
18
|
|
18
19
|
prepare = lambda do |f|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
20
|
+
if headers_only
|
21
|
+
CSV.read(f)
|
22
|
+
.first
|
23
|
+
else
|
24
|
+
CSV.read(f, converters: :numeric)
|
25
|
+
.map(&:to_a)
|
26
|
+
.tap { |x| x.shift unless x.first.first.is_a?(Numeric) }
|
27
|
+
.map { |x| (x.first...x.last) }
|
28
|
+
end
|
23
29
|
end
|
24
30
|
|
25
|
-
sym = get_id_set(symbol: symbol, id: id)
|
31
|
+
sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
|
26
32
|
|
27
33
|
file = "#{config[:data_path]}/trading_hours/#{sym[:symbol]}_#{filter}.csv"
|
28
34
|
puts "Trying to use #{file} for #{symbol} + #{filter}" if debug
|
data/lib/cotcube-bardata.rb
CHANGED
@@ -11,6 +11,7 @@ require 'date' unless defined?(DateTime)
|
|
11
11
|
require 'csv' unless defined?(CSV)
|
12
12
|
require 'yaml' unless defined?(YAML)
|
13
13
|
require 'cotcube-helpers'
|
14
|
+
require 'cotcube-indicators'
|
14
15
|
require 'parallel'
|
15
16
|
|
16
17
|
require_relative 'cotcube-bardata/constants'
|
@@ -61,10 +62,6 @@ module Cotcube
|
|
61
62
|
:range_matrix,
|
62
63
|
# create an array of ranges based on specified source data
|
63
64
|
:trading_hours,
|
64
|
-
# receive id / symbol information on an uncertain set of parameters
|
65
|
-
# TODO: decommision sppearance of get_id_set and symbols in cotcube::bardata
|
66
|
-
:get_id_set,
|
67
|
-
:symbols,
|
68
65
|
#
|
69
66
|
:select_specific_date,
|
70
67
|
:extended_select_for_range,
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cotcube-bardata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.15.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Benjamin L. Tischendorf
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-11-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -44,14 +44,14 @@ dependencies:
|
|
44
44
|
requirements:
|
45
45
|
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '0.
|
47
|
+
version: '0.2'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
|
-
version: '0.
|
54
|
+
version: '0.2'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: cotcube-indicators
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -180,7 +180,7 @@ files:
|
|
180
180
|
- lib/cotcube-bardata/trading_hours.rb
|
181
181
|
homepage: https://github.com/donkeybridge/cotcube-bardata
|
182
182
|
licenses:
|
183
|
-
- BSD-
|
183
|
+
- BSD-3-Clause
|
184
184
|
metadata:
|
185
185
|
homepage_uri: https://github.com/donkeybridge/cotcube-bardata
|
186
186
|
source_code_uri: https://github.com/donkeybridge/cotcube-bardata
|
@@ -200,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
200
200
|
- !ruby/object:Gem::Version
|
201
201
|
version: '0'
|
202
202
|
requirements: []
|
203
|
-
rubygems_version: 3.1.
|
203
|
+
rubygems_version: 3.1.6
|
204
204
|
signing_key:
|
205
205
|
specification_version: 4
|
206
206
|
summary: Functions to provide bardata; and some simple time series aggregations
|