cotcube-bardata 0.1.14 → 0.1.15.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6f517a7bf018413f424a2eba1e1c568c4db864240f88d23fe0c722526af858b4
4
- data.tar.gz: 7e303f08f4be6d64f1680f5107690e78525edcdf938245b22ab97a38af1523e7
3
+ metadata.gz: 5798e21ced6b815404638c1f33948317338d801ea483e9e43edf21b8ad3f73ba
4
+ data.tar.gz: 8bfaf9b663d8de85f8c89b459b3f33e35ce836cad2d0153a02385aaafc4d6781
5
5
  SHA512:
6
- metadata.gz: 5cbeae28346fca35de00cdc926724799f00c69ac8b023075c5dbd332327303ab457e1558eb289979ce95ef31b0cbc7e7cd1439ce550c4b2ddb415c79cffa21dd
7
- data.tar.gz: 9fcdc6b0b14212673d1c26e9c5b6b76264a2d8baa7270d609841377ad3fd753bb1011ddad499a27412e180dcbb8bc35a831e79e79c487866634f85533440b771
6
+ metadata.gz: f554feb61ac91f542a9e9a24a4f9983cbfd529ace02290b531d07bb67cbe5864da1e154e023b9d00dec77b91b2243efb726d4bd005fee6cdc1af88701de583a8
7
+ data.tar.gz: 32a0446523adb94cc50426c0e03b3bd379746754d5314fe79af8ff02fe169fae95fc3a1ba894a313e5b8b7ac89a2da36d5605e0183b9e0ef8022550e00fc5449
data/CHANGELOG.md CHANGED
@@ -1,3 +1,32 @@
1
+ ## 0.1.15.6 (November 19, 2021)
2
+ - daily: added filter_series
3
+
4
+ ## 0.1.15.5 (November 08, 2021)
5
+ - decommissioned .get_id_set in favor of Cotcube::Helpers.get_id_set
6
+
7
+ ## 0.1.15.4 (October 26, 2021)
8
+ - daily: including the sourcing factor (:bcf) into provide_daily and continuous
9
+
10
+ ## 0.1.15.3 (August 04, 2021)
11
+ - daily.rb: Hotfixing
12
+
13
+ ## 0.1.15.2 (August 04, 2021)
14
+ - fixed leftover debug setting
15
+
16
+ ## 0.1.15.1 (August 04, 2021)
17
+ - fixed license mismatch
18
+
19
+ ## 0.1.15 (August 04, 2021)
20
+ - daily.rb: added support to add eod data on incomplete dailies
21
+ - adding dep: cotcube-indicators
22
+ - provide: new method :determine_significant_volume
23
+ - suggest: adding silence
24
+ - cached: Adding :dist
25
+ - daily: new method :determine_significant_volume
26
+ - added :dist generically to quarters
27
+ - trading_hours: added param to return headers only
28
+ - last_trade_date: Enabled caching instead of fetching each time
29
+
1
30
  ## 0.1.14 (May 07, 2021)
2
31
  - few changes in provide / cached / daily for a more straigtforward forcing of cache renewal
3
32
  - suggest: new method to suggest a contract for given symbol and date
data/README.md CHANGED
@@ -90,5 +90,5 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/donkey
90
90
 
91
91
  ## License
92
92
 
93
- The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
93
+ The gem is available as open source under the terms of the [BSD-3-Clause-License](https://opensource.org/licenses/BSD-3-Clause).
94
94
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.14
1
+ 0.1.15.6
@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
10
10
  spec.description = 'Functions to provide bardata; and some simple time series aggregations '
11
11
 
12
12
  spec.homepage = "https://github.com/donkeybridge/#{spec.name}"
13
- spec.license = 'BSD-4-Clause'
13
+ spec.license = 'BSD-3-Clause'
14
14
  spec.required_ruby_version = Gem::Requirement.new('~> 2.7')
15
15
 
16
16
  spec.metadata['homepage_uri'] = spec.homepage
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
28
28
 
29
29
  spec.add_dependency 'activesupport', '~> 6'
30
30
  spec.add_dependency 'colorize', '~> 0.8'
31
- spec.add_dependency 'cotcube-helpers', '~> 0.1'
31
+ spec.add_dependency 'cotcube-helpers', '~> 0.2'
32
32
  spec.add_dependency 'cotcube-indicators', '~> 0.1'
33
33
  spec.add_dependency 'httparty', '~> 0.18'
34
34
  spec.add_dependency 'parallel', '~> 1'
@@ -30,7 +30,7 @@ module Cotcube
30
30
  end
31
31
 
32
32
  headers = %i[contract datetime open high low close volume]
33
- sym = get_id_set(symbol: symbol, id: id, contract: contract)
33
+ sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)
34
34
  contract = contract[-3..]
35
35
  dir = "#{config[:data_path]}/cached/#{sym[:id]}_#{filter.to_s.downcase}"
36
36
  symlink = "#{config[:data_path]}/cached/#{sym[:symbol]}_#{filter.to_s.downcase}"
@@ -40,12 +40,13 @@ module Cotcube
40
40
  quarters_file = "#{config[:data_path]}/quarters/#{sym[:id]}/#{contract[-3..]}.csv"
41
41
  if File.exist?(file) && (not force_update)
42
42
  puts "Working with existing #{file}, no update was forced" if debug
43
- puts " Using quarters from #{quarters_file}"
43
+ puts " Using quarters from #{quarters_file}" if debug
44
44
  base = CSV.read(file, headers: headers).map do |x|
45
45
  x = x.to_h
46
46
  x[:datetime] = timezone.parse(x[:datetime])
47
47
  %i[open high low close].each { |z| x[z] = x[z].to_f.round(9) }
48
48
  x[:volume] = x[:volume].to_i
49
+ x[:dist] = ((x[:high] - x[:low]) / sym[:ticksize] ).to_i
49
50
  x[:type] = "#{filter.to_s.downcase}_day".to_sym
50
51
  x
51
52
  end
@@ -9,8 +9,11 @@ module Cotcube
9
9
  range: nil,
10
10
  timezone: Time.find_zone('America/Chicago'),
11
11
  keep_last: false,
12
+ add_eods: true,
13
+ indicators: {},
12
14
  config: init)
13
15
  contract = contract.to_s.upcase
16
+ rounding = 8
14
17
  unless contract.is_a?(String) && [3, 5].include?(contract.size)
15
18
  raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
16
19
  end
@@ -30,7 +33,7 @@ module Cotcube
30
33
  range = (range_begin..range_end)
31
34
  end
32
35
 
33
- sym = get_id_set(symbol: symbol, id: id, contract: contract)
36
+ sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)
34
37
  contract = contract[2..4] if contract.to_s.size == 5
35
38
  id = sym[:id]
36
39
  id_path = "#{config[:data_path]}/daily/#{id}"
@@ -42,14 +45,46 @@ module Cotcube
42
45
  data = CSV.read(data_file, headers: %i[contract date open high low close volume oi]).map do |row|
43
46
  row = row.to_h
44
47
  row.each do |k, _|
45
- row[k] = row[k].to_f if %i[open high low close].include? k
48
+ if %i[open high low close].include? k
49
+ row[k] = row[k].to_f
50
+ row[k] = (row[k] * sym[:bcf]).round(8) unless sym[:bcf] == 1.0
51
+ end
46
52
  row[k] = row[k].to_i if %i[volume oi].include? k
47
53
  end
48
54
  row[:datetime] = timezone.parse(row[:date])
55
+ row[:dist] = ((row[:high] - row[:low]) / sym[:ticksize] ).to_i
49
56
  row[:type] = :daily
50
57
  row
51
58
  end
52
- data.pop if data.last[:high].zero? and not keep_last
59
+ contract_expired = data.last[:high].zero?
60
+ data.pop if contract_expired and not keep_last
61
+ if not contract_expired and add_eods
62
+ today = Date.today
63
+ eods = [ ]
64
+ while today.strftime('%Y-%m-%d') > data.last[:date]
65
+ eods << provide_eods(symbol: sym[:symbol], dates: today, contracts_only: false, quiet: true)
66
+ today -= 1
67
+ end
68
+ eods.flatten!.map!{|x| x.tap {|y| %i[ volume_part oi_part ].map{|z| y.delete(z)} } }
69
+ eods.select!{|x| x[:contract] == "#{sym[:symbol]}#{contract}" }
70
+ eods.map!{|x| x.tap{|y|
71
+ if sym[:bcf] != 1.0
72
+ %i[open high low close].map{|k|
73
+ y[k] = (y[k] * sym[:bcf]).round(8)
74
+ }
75
+ end
76
+ y[:datetime] = timezone.parse(y[:date])
77
+ y[:dist] = ((y[:high] - y[:low]) / sym[:ticksize] ).to_i
78
+ y[:type] = :eod
79
+ } }
80
+ data += eods.reverse
81
+ end
82
+ data.map do |bar|
83
+ indicators.each do |k,v|
84
+ tmp = v.call(bar)
85
+ bar[k] = tmp.respond_to?(:round) ? tmp.round(rounding) : tmp
86
+ end
87
+ end unless indicators.empty?
53
88
  if range.nil?
54
89
  data
55
90
  else
@@ -62,54 +97,156 @@ module Cotcube
62
97
 
63
98
  # reads all files in bardata/daily/<id> and aggregates by date
64
99
  # (what is a pre-stage of a continuous based on daily bars)
65
- def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false)
100
+ def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false, selector: nil, debug: false, add_eods: true, indicators: nil)
66
101
  raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
67
102
  measuring = lambda {|c| puts "[continuous] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
68
103
 
69
104
  measuring.call("Starting")
70
- sym = get_id_set(symbol: symbol, id: id)
105
+ sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
71
106
  id = sym[:id]
72
107
  symbol = sym[:symbol]
108
+ ticksize = sym[:ticksize]
109
+ effective_selector = selector || :volume
110
+ raise ArgumentError, 'selector must be in %i[ nil :volume ;oi].' unless [ nil, :volume, :oi ].include? selector
73
111
  id_path = "#{config[:data_path]}/daily/#{id}"
74
112
  c_file = "#{id_path}/continuous.csv"
113
+ puts "Using file #{c_file}" if debug
75
114
 
76
115
  # instead of using the provide_daily methods above, for this bulk operation a 'continuous.csv' is created
77
116
  # this boosts from 4.5sec to 0.3sec
78
- rewriting = force_rewrite or not(File.exist?(c_file)) or (Time.now - File.mtime(c_file) > 8.days)
117
+ rewriting = (force_rewrite or not(File.exist?(c_file)) or (Time.now - File.mtime(c_file) > 8.days))
79
118
  if rewriting
80
119
  puts "In daily+continuous: Rewriting #{c_file} #{force_rewrite ? "forcibly" : "due to fileage"}.".light_yellow
81
- `rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v '0,0,0,0' | sort -t, -k2 | cut -d, -f1,2,7,8 > #{c_file}`
120
+ `rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v ',0,' | grep -v ',0$'| sort -t, -k2 | cut -d, -f1-8 | grep ',.*,' | uniq > #{c_file}`
82
121
  end
83
122
  loading = lambda do
84
123
  data = CSV.read(c_file).map do |row|
85
124
  r = { contract: row[0],
86
125
  date: row[1],
87
- volume: row[2].to_i,
88
- oi: row[3].to_i
126
+ open: row[2],
127
+ high: row[3],
128
+ low: row[4],
129
+ close: row[5],
130
+ volume: row[6].to_i,
131
+ oi: row[7].to_i,
132
+ type: :cont
89
133
  }
90
134
  end
135
+ if add_eods
136
+ today = Date.today
137
+ eods = [ ]
138
+ while today.strftime('%Y-%m-%d') > data.last[:date]
139
+ eods << provide_eods(symbol: symbol, dates: today, contracts_only: false, quiet: true)
140
+ today -= 1
141
+ end
142
+ eods.flatten!.map!{|x| x.tap {|y| %i[ volume_part oi_part ].map{|z| y.delete(z)} } }
143
+ eods.delete_if { |elem| elem.flatten.empty? }
144
+ data += eods.reverse
145
+ end
91
146
 
92
147
  measuring.call("Finished retrieving dailies.")
93
148
  result = []
149
+ rounding = 8 # sym[:format].split('.').last.to_i rescue 6
150
+ indicators ||= {
151
+ typical: Cotcube::Indicators.calc(a: :high, b: :low, c: :close) {|high, low, close| (high + low + close) / 3 },
152
+ sma250_high: Cotcube::Indicators.sma(key: :high, length: 250),
153
+ sma250_low: Cotcube::Indicators.sma(key: :low, length: 250),
154
+ sma250_typ: Cotcube::Indicators.sma(key: :typical, length: 250),
155
+ dist: Cotcube::Indicators.calc(a: :high, b: :low, finalize: :to_i) {|high, low| ((high-low) / ticksize) },
156
+ }
157
+
158
+
159
+
94
160
  data.group_by { |x| x[:date] }.map do |k, v|
95
161
  v.map { |x| x.delete(:date) }
96
- result << {
162
+ avg_bar = {
97
163
  date: k,
98
- volume: v.map { |x| x[:volume] }.reduce(:+),
99
- oi: v.map { |x| x[:oi] }.reduce(:+)
164
+ contract: v.max_by{|x| x[:oi] }[:contract],
165
+ open: nil, high: nil, low: nil, close: nil,
166
+ volume: v.map { |x| x[:volume] }.reduce(:+),
167
+ oi: v.map { |x| x[:oi] }.reduce(:+),
168
+ type: :cont_eod
169
+ }
170
+
171
+ %i[ open high low close ].each do |ohlc|
172
+ avg_bar[ohlc] = (v.map{|x| x[ohlc].to_f * x[effective_selector] }.reduce(:+) / avg_bar[effective_selector]).round(rounding)
173
+ avg_bar[ohlc] = (avg_bar[ohlc] * sym[:bcf]).round(8) unless sym[:bcf] == 1.0
174
+
175
+ end
176
+ p avg_bar if debug
177
+ indicators.each do |k,v|
178
+ print format('%12s: ', k.to_s) if debug
179
+ tmp = v.call(avg_bar)
180
+ avg_bar[k] = tmp.respond_to?(:round) ? tmp.round(rounding) : tmp
181
+ puts avg_bar[k] if debug
182
+ end
183
+ %i[tr atr5].each { |ind|
184
+ avg_bar[ind] = (avg_bar[ind] / sym[:ticksize]).round.to_i unless avg_bar[ind].nil?
100
185
  }
186
+ result << avg_bar
101
187
  result.last[:contracts] = v
102
188
  end
103
189
  result
104
190
  end
105
- constname = "CONTINUOUS_#{symbol}".to_sym
191
+ constname = "CONTINUOUS_#{symbol}#{selector.nil? ? '' : ('_' + selector.to_s)}".to_sym
106
192
  if rewriting or not Cotcube::Bardata.const_defined?( constname)
193
+ old = $VERBOSE; $VERBOSE = nil
107
194
  Cotcube::Bardata.const_set constname, loading.call
195
+ $VERBOSE = old
108
196
  end
109
197
  measuring.call("Finished processing")
110
198
  date.nil? ? Cotcube::Bardata.const_get(constname).map{|z| z.dup } : Cotcube::Bardata.const_get(constname).find { |x| x[:date] == date }
111
199
  end
112
200
 
201
+ # the filter series is an indicator based on the Cotcube::Bardata.continuous of the asset price.
202
+ # current default filter is the ema50
203
+ def filter_series(ema_length: 50, symbol: , print_range: nil)
204
+ ema_high_n = "ema#{ema_length}_high".to_sym
205
+ ema_low_n = "ema#{ema_length}_low".to_sym
206
+ ema_filter = "ema#{ema_length}_filter".to_sym
207
+ indicators = {
208
+ ema_high_n => Cotcube::Indicators.ema(key: :high, length: ema_length, smoothing: 2),
209
+ ema_low_n => Cotcube::Indicators.ema(key: :low, length: ema_length, smoothing: 2),
210
+ :tr => Cotcube::Indicators.true_range,
211
+ :atr5 => Cotcube::Indicators.ema(key: :tr, length: 5, smoothing: 2),
212
+ ema_filter => Cotcube::Indicators.calc(a: :high, b: :low, c: :close,
213
+ d: ema_high_n, e: ema_low_n, f: :atr5,
214
+ finalize: :to_i) do |high, low, close, ema_high, ema_low, atr5|
215
+
216
+ if close > ema_high and (low - ema_high).abs <= atr5 / 5.0; 3 # :bullish_tipped
217
+ elsif low > ema_high and (low - ema_high).abs >= atr5 * 3.0; 5 # :bullish_away
218
+ elsif low > ema_high and (low - ema_high).abs <= atr5 / 1.5; 2 # :bullish_nearby
219
+ elsif low > ema_high; 4 # :bullish
220
+
221
+ elsif close < ema_low and (high - ema_low).abs <= atr5 / 5.0; -3 # :bearish_tipped
222
+ elsif high < ema_low and (high - ema_low).abs >= atr5 * 3.0; -5 # :bearish_away
223
+ elsif high < ema_low and (high - ema_low).abs <= atr5 / 1.5; -2 # :bearish_nearby
224
+ elsif high < ema_low; -4 # :bearish
225
+
226
+ elsif close >= ema_high and (close - ema_high).abs > atr5 ; 2 # :bullish_closed
227
+ elsif close <= ema_low and (close - ema_low ).abs > atr5 ; -2 # :bearish_closed
228
+ elsif close >= ema_high; 1 # :bullish_weak
229
+ elsif close <= ema_low; -1 # :bearish_weak
230
+ elsif close > ema_low and close < ema_high; 0 # :ambigue
231
+ else
232
+ raise RuntimeError, "Unconsidered Indicator value with #{high}, #{low}, #{close}, #{ema_high}, #{ema_low}, #{atr5}"
233
+
234
+ end
235
+ end
236
+ }
237
+ filter = Cotcube::Bardata.continuous(symbol: symbol, indicators: indicators).
238
+ map{ |z| z[:datetime] = DateTime.parse(z[:date]); z[:datetime] += z[:datetime].wday == 5 ? 3 : 1; z.slice(:datetime, ema_filter) }.
239
+ group_by{ |z| z[:datetime] }.
240
+ map{ |k,v| [ k, v[0][ema_filter] ] }.
241
+ to_h.
242
+ tap{ |z| z.to_a[print_range].each { |v|
243
+ puts "#{v[0].strftime('%Y-%m-%d')
244
+ } : #{format '%2d', v[1]
245
+ }".colorize(v[1] > 3 ? :light_green : v[1] > 1 ? :green : v[1] < -3 ? :light_red : v[1] < -1 ? :red : :white )
246
+ } if print_range.is_a? Range
247
+ }
248
+ end
249
+
113
250
  def continuous_ml(symbol: nil, id: nil, base: nil)
114
251
  (base.nil? ? Cotcube::Bardata.continuous(symbol: symbol, id: id) : base).map do |x|
115
252
  x[:ml] = x[:contracts].max_by { |z| z[:volume] }[:contract]
@@ -150,7 +287,7 @@ module Cotcube
150
287
  %i[volume oi].include?(selector)
151
288
 
152
289
  measuring.call("Starting")
153
- sym = get_id_set(symbol: symbol, id: id)
290
+ sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
154
291
  id = sym[:id]
155
292
  # noinspection RubyNilAnalysis
156
293
  data = continuous(id: id, config: config, measure: measure).map do |x|
@@ -214,7 +351,7 @@ module Cotcube
214
351
  %i[volume oi].include?(selector)
215
352
 
216
353
  measuring.call("Entering function")
217
- sym = get_id_set(symbol: symbol, id: id)
354
+ sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
218
355
  if %w[R6 BJ GE].include? sym[:symbol]
219
356
  puts "Rejecting to process symbol '#{sym[:symbol]}'.".light_red
220
357
  return []
@@ -230,17 +367,18 @@ module Cotcube
230
367
 
231
368
  ytoday = date.yday
232
369
  data = continuous_overview(id: id, selector: selector, filter: filter, human: false, config: init, measure: measure)
233
- .reject { |k, _| k[-2..].to_i == date.year % 2000 }
370
+ .reject { |k, _| k[-2..].to_i >= date.year % 2000 }
234
371
  .group_by { |k, _| k[2] }
235
372
  measuring.call("Retrieved continous_overview")
236
373
  output_sent = []
237
374
  early_year=nil
238
375
  long_output = []
239
376
  data.keys.sort.each do |month|
240
- current_long = { month: month }
241
377
  puts "Processing #{sym[:symbol]}#{month}" if debuglevel > 1
242
378
  v0 = data[month]
379
+ # ldays is the list of 'last days'
243
380
  ldays = v0.map { |_, v1| Date.parse(v1.last[:date]).yday }
381
+ # fdays is the list of 'first days'
244
382
  fdays = v0.map { |_, v1| Date.parse(v1.first[:date]).yday }.sort
245
383
  # if the last ml day nears the end of the year, we must fix
246
384
  ldays.map! { |x| x > 350 ? x - 366 : x } if ldays.min < 50
@@ -277,9 +415,9 @@ module Cotcube
277
415
  }\tto #{format '%5d', ldays.max
278
416
  }: #{dfm.call(ldays.max)}".colorize(color)
279
417
  if debug || (color != :white)
280
- puts output
281
- output_sent << "#{sym[:symbol]}#{month}" unless color == :white
418
+ puts output unless silent
282
419
  end
420
+ output_sent << "#{sym[:symbol]}#{month}" unless color == :white
283
421
  early_year ||= output
284
422
  next if silent or not (debug and debuglevel >= 2)
285
423
 
@@ -288,20 +426,22 @@ module Cotcube
288
426
  }\t#{v1.first[:date]
289
427
  } (#{format '%3d', Date.parse(v1.first[:date]).yday
290
428
  })\t#{Date.parse(v1.last[:date]).strftime('%a, %Y-%m-%d')
291
- } (#{Date.parse(v1.last[:date]).yday})"
429
+ } (#{Date.parse(v1.last[:date]).yday})" unless silent
292
430
  # rubocop:enable Layout/ClosingParenthesisIndentation
293
431
  end
294
- end
432
+ end
295
433
  case output_sent.size
296
434
  when 0
297
- puts "WARNING: No output was sent for symbol '#{sym[:symbol]}'.".colorize(:light_yellow)
298
- puts " Assuming late-year-processing.".light_yellow
299
- puts early_year.light_green
435
+ unless silent
436
+ puts "WARNING: No output was sent for symbol '#{sym[:symbol]}'.".colorize(:light_yellow)
437
+ puts " Assuming late-year-processing.".light_yellow
438
+ puts early_year.light_green
439
+ end
300
440
  when 1
301
441
  # all ok
302
442
  true
303
443
  else
304
- puts "Continuous table show #{output_sent.size} active contracts ( #{output_sent} ) for #{sym[:symbol]} ---------------"
444
+ puts "Continuous table show #{output_sent.size} active contracts ( #{output_sent} ) for #{sym[:symbol]} ---------------" unless silent
305
445
  end
306
446
  measuring.call("Finished processing")
307
447
  short ? output_sent : long_output
@@ -4,7 +4,7 @@ module Cotcube
4
4
  # Missing top level documentation
5
5
  module Bardata
6
6
  def most_liquid_for(symbol: nil, id: nil, date: last_trade_date, config: init)
7
- id = get_id_set(symbol: symbol, id: id, config: config)[:id]
7
+ id = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, config: config)[:id]
8
8
  provide_eods(id: id, dates: date, contracts_only: true).first
9
9
  end
10
10
 
@@ -15,7 +15,7 @@ module Cotcube
15
15
  date: last_trade_date,
16
16
  filter: :volume_part,
17
17
  age: 1.hour)
18
- sym = get_id_set(symbol: symbol, id: id) if symbol || id
18
+ sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id) if symbol || id
19
19
  # noinspection RubyScope
20
20
  eods = provide_eods(id: sym.nil? ? nil : sym[:id], config: config, dates: date, filter: filter)
21
21
  result = []
@@ -62,7 +62,7 @@ module Cotcube
62
62
  end
63
63
 
64
64
  symbol = contract[0..1] if contract.to_s.size == 5
65
- sym = get_id_set(symbol: symbol, id: id, config: config) if symbol || id
65
+ sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, config: config) if symbol || id
66
66
  # if no id can be clarified from given arguments, return all matching contracts from all available symbols
67
67
  # raise ArgumentError, "Could not guess :id or :symbol from 'contract: #{contract}', please clarify." if id.nil?
68
68
  raise ArgumentError, ':filter must be in [:volume_part, :oi_part]' unless %i[volume_part oi_part].include? filter
@@ -45,42 +45,6 @@ module Cotcube
45
45
  result
46
46
  end
47
47
 
48
- def get_id_set(symbol: nil, id: nil, contract: nil, config: init)
49
- contract = contract.to_s.upcase if contract.is_a? Symbol
50
- id = id.to_s.upcase if id.is_a? Symbol
51
- symbol = symbol.to_s.upcase if symbol.is_a? Symbol
52
-
53
- if contract.is_a?(String) && (contract.length == 5)
54
- c_symbol = contract[0..1]
55
- if (not symbol.nil?) && (symbol != c_symbol)
56
- raise ArgumentError,
57
- "Mismatch between given symbol #{symbol} and contract #{contract}"
58
- end
59
-
60
- symbol = c_symbol
61
- end
62
-
63
- unless symbol.nil?
64
- sym = symbols.select { |s| s[:symbol] == symbol.to_s.upcase }.first
65
- if sym.nil? || sym[:id].nil?
66
- raise ArgumentError,
67
- "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}"
68
- end
69
- raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if (not id.nil?) && (sym[:id] != id)
70
-
71
- return sym
72
- end
73
- unless id.nil?
74
- sym = symbols.select { |s| s[:id] == id.to_s }.first
75
- if sym.nil? || sym[:id].nil?
76
- raise ArgumentError,
77
- "Could not find match in #{config[:symbols_file]} for given id #{id}"
78
- end
79
- return sym
80
- end
81
- raise ArgumentError, 'Need :id, :symbol or valid :contract '
82
- end
83
-
84
48
  def compare(contract:, format: '%5.2f')
85
49
  format = "%#{format}" unless format[0] == '%'
86
50
  daily = provide(contract: contract, interval: :daily)
@@ -17,7 +17,7 @@ module Cotcube
17
17
  force_update: false,
18
18
  force_recent: false)
19
19
 
20
- sym = get_id_set(symbol: symbol, id: id, contract: contract, config: config)
20
+ sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract, config: config)
21
21
 
22
22
  case interval
23
23
  when :quarters, :hours, :quarter, :hour
@@ -54,5 +54,11 @@ module Cotcube
54
54
  raise ArgumentError, "Unsupported or unknown interval '#{interval}' in Bardata.provide"
55
55
  end
56
56
  end
57
+
58
+ def determine_significant_volume(base: , contract: )
59
+ set = Cotcube::Bardata.trading_hours(symbol: contract[0..1], filter: :rth)
60
+ prod = base - base.select_within(ranges: set ,attr: :datetime) {|x| x.to_datetime.to_sssm }
61
+ prod.group_by{|x| x[:volume] / 500 }
62
+ end
57
63
  end
58
64
  end
@@ -16,7 +16,7 @@ module Cotcube
16
16
  raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
17
17
  end
18
18
 
19
- sym = get_id_set(symbol: symbol, id: id, contract: contract)
19
+ sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id, contract: contract)
20
20
 
21
21
  contract = contract[2..4] if contract.to_s.size == 5
22
22
  id = sym[:id]
@@ -33,6 +33,7 @@ module Cotcube
33
33
  %i[open high low close].map { |x| row[x] = row[x].to_f }
34
34
  %i[volume day].map { |x| row[x] = row[x].to_i }
35
35
  row[:datetime] = timezone.parse(row[:datetime])
36
+ row[:dist] = ((row[:high] - row[:low]) / sym[:ticksize] ).to_i
36
37
  row[:type] = :quarter
37
38
  row
38
39
  end
@@ -24,7 +24,7 @@ module Cotcube
24
24
  def range_matrix(symbol: nil, id: nil, base: nil, print: false, dim: 0.05, days_only: false, last_n: 60, &block)
25
25
  # rubocop:disable Style/MultilineBlockChain
26
26
  symbol ||= base.last[:contract][0..1] if id.nil?
27
- sym = get_id_set(symbol: symbol, id: id)
27
+ sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
28
28
  source = {}
29
29
  target = {}
30
30
  if base.nil?
@@ -5,7 +5,7 @@ module Cotcube
5
5
  module Bardata
6
6
  # based on day(of year) and symbol, suggest best fitting contract
7
7
  def suggest_contract_for symbol:, date: Date.today, warnings: true
8
- ml = Cotcube::Bardata.continuous_table symbol: symbol, date: date
8
+ ml = Cotcube::Bardata.continuous_table symbol: symbol, date: date, silent: true
9
9
  if ml.size != 1
10
10
  puts "WARNING: No or no unique most liquid found for #{date}. please give :contract parameter".light_yellow if warnings
11
11
  if ml.size > 1
@@ -5,19 +5,45 @@ module Cotcube
5
5
  module Bardata
6
6
  # fetching official trade dates from CME
7
7
  # it returns the current trade date or, if today isn't a trading day, the last trade date.
8
- def last_trade_date
9
- uri = 'https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME'
10
- begin
11
- HTTParty.get(uri, headers: { "User-Agent" => "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"})
12
- .parsed_response
13
- .map do |x|
14
- a = x['tradeDate'].chars.each_slice(2).map(&:join)
15
- "#{a[0]}#{a[1]}-#{a[2]}-#{a[3]}"
16
- end
17
- .first
18
- rescue StandardError
19
- nil
8
+ def last_trade_date(force_update: false)
9
+ const_LTD = :LAST_TRADE_DATE
10
+ const_LTDU = :LAST_TRADE_DATE_UPDATE
11
+ if force_update or not Object.const_defined?(const_LTD) or Object.const_get(const_LTD).nil? or Time.now - Object.const_get(const_LTDU) > 2.hours
12
+ result = nil
13
+ uri = 'https://www.cmegroup.com/CmeWS/mvc/Volume/TradeDates?exchange=CME'
14
+ headers = { "Accept" => "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
15
+ "Accept-Encoding" => "gzip, deflate, br",
16
+ "Accept-Language" => "en-US,en;q=0.9",
17
+ "Cache-Control" => "max-age=0",
18
+ "Connection" => "keep-alive",
19
+ # Cookie: ak_bmsc=602078F6DE40954BAA8C7E7D3815102CACE82AAFD237000084B5A460F4FBCA68~pluz010T49Xag3sXquUZtVJmFX701dzEgt5v6Ht1EZSLKE4HL+bgg1L9ePnL5I0mm7QWXe1qaLhUbX1IPrL/f20trRMMRlkC3UWXk27DY/EBCP4mRno8QQygLCwgs2B2AQHJyb63WwRihCko8UYUiIhb89ArPZM5OPraoKy3JU9oE9e+iERdARNZHLHqRiB1GnmbKUvQqos3sXaEe3GpoiTszzk8sHZs4ZKuoO/rvFHko=",
20
+ "Host" => "www.cmegroup.com",
21
+ "sec-ch-ua" => %q[" Not A;Brand";v="99", "Chromium";v="90", "Google Chrome";v="90"],
22
+ "sec-ch-ua-mobile" => "?0",
23
+ "Sec-Fetch-Dest" => "document",
24
+ "Sec-Fetch-Mode" => "navigate",
25
+ "Sec-Fetch-Site" => "none",
26
+ "Sec-Fetch-User" => "?1",
27
+ "Upgrade-Insecure-Requests" => "1",
28
+ "User-Agent" => "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36" }
29
+ begin
30
+ # HTTParty.get(uri, headers: { "User-Agent" => "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/81.0"})
31
+ result = HTTParty.get(uri, headers: headers)
32
+ .parsed_response
33
+ .map do |x|
34
+ a = x['tradeDate'].chars.each_slice(2).map(&:join)
35
+ "#{a[0]}#{a[1]}-#{a[2]}-#{a[3]}"
36
+ end
37
+ .first
38
+ rescue StandardError
39
+ result = nil
40
+ end
41
+ oldverbose = $VERBOSE; $VERBOSE = nil
42
+ Object.const_set(const_LTD, result)
43
+ Object.const_set(const_LTDU, Time.now) unless result.nil?
44
+ $VERBOSE = oldverbose
20
45
  end
46
+ Object.const_get(const_LTD)
21
47
  end
22
48
 
23
49
  def holidays(config: init)
@@ -12,17 +12,23 @@ module Cotcube
12
12
  filter: ,
13
13
  force_filter: false, # with force_filter one would avoid falling back
14
14
  # to the contract_type based range set
15
+ headers_only: false, # return only headers instead of ranges
15
16
  config: init, debug: false)
16
17
  return (0...24 * 7 * 3600) if filter.to_s =~ /24x7/
17
18
 
18
19
  prepare = lambda do |f|
19
- CSV.read(f, converters: :numeric)
20
- .map(&:to_a)
21
- .tap { |x| x.shift unless x.first.first.is_a?(Numeric) }
22
- .map { |x| (x.first...x.last) }
20
+ if headers_only
21
+ CSV.read(f)
22
+ .first
23
+ else
24
+ CSV.read(f, converters: :numeric)
25
+ .map(&:to_a)
26
+ .tap { |x| x.shift unless x.first.first.is_a?(Numeric) }
27
+ .map { |x| (x.first...x.last) }
28
+ end
23
29
  end
24
30
 
25
- sym = get_id_set(symbol: symbol, id: id)
31
+ sym = Cotcube::Helpers.get_id_set(symbol: symbol, id: id)
26
32
 
27
33
  file = "#{config[:data_path]}/trading_hours/#{sym[:symbol]}_#{filter}.csv"
28
34
  puts "Trying to use #{file} for #{symbol} + #{filter}" if debug
@@ -11,6 +11,7 @@ require 'date' unless defined?(DateTime)
11
11
  require 'csv' unless defined?(CSV)
12
12
  require 'yaml' unless defined?(YAML)
13
13
  require 'cotcube-helpers'
14
+ require 'cotcube-indicators'
14
15
  require 'parallel'
15
16
 
16
17
  require_relative 'cotcube-bardata/constants'
@@ -61,10 +62,6 @@ module Cotcube
61
62
  :range_matrix,
62
63
  # create an array of ranges based on specified source data
63
64
  :trading_hours,
64
- # receive id / symbol information on an uncertain set of parameters
65
- # TODO: decommision sppearance of get_id_set and symbols in cotcube::bardata
66
- :get_id_set,
67
- :symbols,
68
65
  #
69
66
  :select_specific_date,
70
67
  :extended_select_for_range,
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cotcube-bardata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.14
4
+ version: 0.1.15.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Benjamin L. Tischendorf
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-05-07 00:00:00.000000000 Z
11
+ date: 2021-11-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -44,14 +44,14 @@ dependencies:
44
44
  requirements:
45
45
  - - "~>"
46
46
  - !ruby/object:Gem::Version
47
- version: '0.1'
47
+ version: '0.2'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
52
  - - "~>"
53
53
  - !ruby/object:Gem::Version
54
- version: '0.1'
54
+ version: '0.2'
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: cotcube-indicators
57
57
  requirement: !ruby/object:Gem::Requirement
@@ -180,7 +180,7 @@ files:
180
180
  - lib/cotcube-bardata/trading_hours.rb
181
181
  homepage: https://github.com/donkeybridge/cotcube-bardata
182
182
  licenses:
183
- - BSD-4-Clause
183
+ - BSD-3-Clause
184
184
  metadata:
185
185
  homepage_uri: https://github.com/donkeybridge/cotcube-bardata
186
186
  source_code_uri: https://github.com/donkeybridge/cotcube-bardata
@@ -200,7 +200,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
200
200
  - !ruby/object:Gem::Version
201
201
  version: '0'
202
202
  requirements: []
203
- rubygems_version: 3.1.2
203
+ rubygems_version: 3.1.6
204
204
  signing_key:
205
205
  specification_version: 4
206
206
  summary: Functions to provide bardata; and some simple time series aggregations