cotcube-bardata 0.1.9.1 → 0.1.12

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8a288595522cb8414d8db207702ccfa4f2982aee92aea7fbd9bbb69dc9bff951
4
- data.tar.gz: 05707b28f388968d3fad8cf4d5145fb0b85c5c8cd30f3cd68b722bac5d4874d1
3
+ metadata.gz: 5607a73308b8d9d7aff10caccdd3785189072a7d5b2a0b37ea0b6d2bd1ebf4f9
4
+ data.tar.gz: 18bf8b90e2aad0d32a6608037de861d5383df1bd6d8c6b11e903724c91f172df
5
5
  SHA512:
6
- metadata.gz: d1a8385d8dc58b8955aa443e3dda7e66bcb2ed172518d8bf6070e6506e33bdab805a4b9d821c1aa7fc308d69bba5bfa5a80ed5fb3fca8afffeec01ff9a9e10e1
7
- data.tar.gz: 12013f03b33649879ab5f93778a9a4cf8a9586b2080b7113dac60171681dd5a158fea1d5c9695edc64f6e1ea738c33d0b82c776f0b881b38112cb2b99a76df8a
6
+ metadata.gz: 3589e6b49ba104aed0177032cd46299c2c0e1db37330e2b869ab6a97f8b81400917bf29ec631cb37b2f35245ece8dcdb147dd56194487d441773d8c79628d38e
7
+ data.tar.gz: 3be51d9604d9fb1092939b30fa099801c989babae6ec528479d8af5c8b8750e087dc5965f8b05a3cc5d68281a58de93295cba1b2249030a106220b0eea9594bd
data/CHANGELOG.md CHANGED
@@ -1,3 +1,22 @@
1
+ ## 0.1.12 (March 13, 2021)
2
+ - range_matrix: adapted to accept block for range provision; added params :days_only and :last_n
3
+ - minor fix on previous patch
4
+
5
+ ## 0.1.11 (March 07, 2021)
6
+ - daily.rb: added new technique 'caching in constants' to accelerate computation, also referring to continuous
7
+ - provide.rb: minor change, so disregarded contracts can be used in swapproximate
8
+
9
+ ## 0.1.10 (February 11, 2021)
10
+ - Daily.rb: Added measure parameters to continous_suite
11
+ - cached.rb: Minor fix in comparison
12
+
13
+ ## 0.1.9.3 (February 08, 2021)
14
+ - cached: minor change fixing problem to get yesterday in rth subsets
15
+
16
+ ## 0.1.9.2 (February 07, 2021)
17
+ - minor changes
18
+ - daily#continuous_table: introducing debuglevel
19
+
1
20
  ## 0.1.9.1 (January 29, 2021)
2
21
  - provide: added new interval 'synth', mixing available dailies with synthetic days based on quarters
3
22
  - minor change / cleanup
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.1.9.1
1
+ 0.1.12
@@ -52,6 +52,8 @@ module Cotcube
52
52
  :continuous_actual_ml,
53
53
  # based on continuous, create list of when which contract was most liquid
54
54
  :continuous_overview,
55
+ # provider estimation of current ML usability
56
+ :continuous_table,
55
57
  # provide the list of quarters, possibly as hours or days.
56
58
  :provide_quarters,
57
59
  # some statistics to estimate daily volatility of specific contract
@@ -60,7 +60,7 @@ module Cotcube
60
60
  end
61
61
  end
62
62
  return result
63
- elsif File.mtime(file) + 1.day > File.mtime(quarters_file)
63
+ elsif File.mtime(file) - Time.now.beginning_of_day >= 0
64
64
  puts "CACHE #{File.mtime(file)}\t#{file}" if debug
65
65
  puts "QUART #{File.mtime(quarters_file)}\t#{quarters_file}" if debug
66
66
  result = if range.nil?
@@ -98,17 +98,17 @@ module Cotcube
98
98
  base = Cotcube::Helpers.reduce(bars: data, to: :days)
99
99
 
100
100
  # remove last day of result unless marked
101
- base.pop unless contract_is_marked || force_recent
101
+ base.pop if base.last[:datetime].to_date == timezone.now.to_date and not force_recent
102
102
 
103
103
  base.map do |x|
104
- x[:datetime] = x[:datetime].to_date
104
+ x[:date] = x[:datetime].to_date
105
105
  x[:type] = "#{filter}_day".to_sym
106
106
  x.delete(:day)
107
107
  end
108
108
  CSV.open(file, 'w') do |csv|
109
109
  base.each { |b| csv << b.values_at(*headers) }
110
110
  if contract_is_marked
111
- marker = ["#{sym[:symbol]}#{contract}", base.last[:datetime] + 1.day, 0, 0, 0, 0, 0]
111
+ marker = ["#{sym[:symbol]}#{contract}", base.last[:date] + 1.day, 0, 0, 0, 0, 0]
112
112
  csv << marker
113
113
  end
114
114
  end
@@ -61,29 +61,51 @@ module Cotcube
61
61
 
62
62
  # reads all files in bardata/daily/<id> and aggregates by date
63
63
  # (what is a pre-stage of a continuous based on daily bars)
64
- def continuous(symbol: nil, id: nil, config: init, date: nil)
64
+ def continuous(symbol: nil, id: nil, config: init, date: nil, measure: nil, force_rewrite: false)
65
+ raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
66
+ measuring = lambda {|c| puts "[continuous] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
67
+
68
+ measuring.call("Starting")
65
69
  sym = get_id_set(symbol: symbol, id: id)
66
70
  id = sym[:id]
67
71
  id_path = "#{config[:data_path]}/daily/#{id}"
68
- available_contracts = Dir["#{id_path}/*.csv"].map { |x| x.split('/').last.split('.').first }
69
- available_contracts.sort_by! { |x| x[-7] }.sort_by! { |x| x[-6..-5] }
70
- data = []
71
- available_contracts.each do |c|
72
- provide_daily(id: id, config: config, contract: c).each do |x|
73
- data << x
72
+ c_file = "#{id_path}/continuous.csv"
73
+
74
+ # instead of using the provide_daily methods above, for this bulk operation a 'continuous.csv' is created
75
+ # this boosts from 4.5sec to 0.3sec
76
+ rewriting = force_rewrite or not(File.exist?(c_file)) or (Time.now - File.mtime(c_file) > 8.days)
77
+ if rewriting
78
+ puts "In daily+continuous: Rewriting #{c_file} #{force_rewrite ? "forcibly" : "due to fileage"}.".light_yellow
79
+ `rm #{c_file}; find #{id_path} | xargs cat 2>/dev/null | grep -v '0,0,0,0' | sort -t, -k2 | cut -d, -f1,2,7,8 > #{c_file}`
80
+ end
81
+ loading = lambda do
82
+ data = CSV.read(c_file).map do |row|
83
+ r = { contract: row[0],
84
+ date: row[1],
85
+ volume: row[2].to_i,
86
+ oi: row[3].to_i
87
+ }
88
+ end
89
+
90
+ measuring.call("Finished retrieving dailies.")
91
+ result = []
92
+ data.group_by { |x| x[:date] }.map do |k, v|
93
+ v.map { |x| x.delete(:date) }
94
+ result << {
95
+ date: k,
96
+ volume: v.map { |x| x[:volume] }.reduce(:+),
97
+ oi: v.map { |x| x[:oi] }.reduce(:+)
98
+ }
99
+ result.last[:contracts] = v
74
100
  end
101
+ result
75
102
  end
76
- result = []
77
- data.sort_by { |x| x[:date] }.group_by { |x| x[:date] }.map do |k, v|
78
- v.map { |x| x.delete(:date) }
79
- result << {
80
- date: k,
81
- volume: v.map { |x| x[:volume] }.reduce(:+),
82
- oi: v.map { |x| x[:oi] }.reduce(:+)
83
- }
84
- result.last[:contracts] = v
103
+ constname = "CONTINUOUS_#{symbol}".to_sym
104
+ if rewriting or not Cotcube::Bardata.const_defined?( constname)
105
+ Cotcube::Bardata.const_set constname, loading.call
85
106
  end
86
- date.nil? ? result : result.select { |x| x[:date] == date }.first
107
+ measuring.call("Finished processing")
108
+ date.nil? ? Cotcube::Bardata.const_get(constname) : Cotcube::Bardata.const_get(constname).find { |x| x[:date] == date }
87
109
  end
88
110
 
89
111
  def continuous_ml(symbol: nil, id: nil, base: nil)
@@ -116,20 +138,27 @@ module Cotcube
116
138
  config: init,
117
139
  selector: :volume,
118
140
  human: false,
141
+ measure: nil,
119
142
  filter: nil)
143
+
144
+ raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
145
+ measuring = lambda {|c| puts "[continuous_overview] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
146
+
120
147
  raise ArgumentError, 'Selector must be either :volume or :oi' unless selector.is_a?(Symbol) &&
121
- %i[volume oi].include?(selector)
148
+ %i[volume oi].include?(selector)
122
149
 
150
+ measuring.call("Starting")
123
151
  sym = get_id_set(symbol: symbol, id: id)
124
152
  id = sym[:id]
125
153
  # noinspection RubyNilAnalysis
126
- data = continuous(id: id, config: config).map do |x|
154
+ data = continuous(id: id, config: config, measure: measure).map do |x|
127
155
  {
128
156
  date: x[:date],
129
157
  volume: x[:contracts].sort_by { |z| - z[:volume] }[0..4].compact.reject { |z| z[:volume].zero? },
130
158
  oi: x[:contracts].sort_by { |z| - z[:oi] }[0..4].compact.reject { |z| z[:oi].zero? }
131
159
  }
132
160
  end
161
+ measuring.call("Retrieved continuous")
133
162
  data.reject! { |x| x[selector].empty? }
134
163
  result = data.group_by { |x| x[selector].first[:contract] }
135
164
  result.each_key do |key|
@@ -149,31 +178,44 @@ module Cotcube
149
178
  }\t#{v.last[:date]
150
179
  }\t#{format('%4d', (Date.parse(v.last[:date]) - Date.parse(v.first[:date])))
151
180
  }\t#{result[k].map do |x|
152
- x[:volume].select do
153
- x[:contract] == k
154
- end
155
- end.size
181
+ x[:volume].select do
182
+ x[:contract] == k
183
+ end
184
+ end.size
156
185
  }"
157
186
  # rubocop:enable Layout/ClosingParenthesisIndentation
158
187
  end
159
188
  end
189
+ measuring.call("Finished processing")
160
190
  result
161
191
  end
162
192
 
163
193
  def continuous_table(symbol: nil, id: nil,
164
194
  selector: :volume,
165
195
  filter: nil,
196
+ date: Date.today,
197
+ measure: nil,
198
+ debuglevel: 1,
166
199
  debug: false)
200
+ if debug.is_a?(Integer)
201
+ debuglevel = debug
202
+ debug = debuglevel > 0 ? true : false
203
+ end
204
+
205
+ raise ArgumentError, ':measure, if given, must be a Time object (e.g. Time.now)' unless [NilClass, Time].include? measure.class
206
+ measuring = lambda {|c| puts "[continuous_table] Time measured until '#{c}': #{(Time.now.to_f - measure.to_f).round(2)}sec" unless measure.nil? }
207
+
167
208
  raise ArgumentError, 'Selector must be either :volume or :oi' unless selector.is_a?(Symbol) &&
168
- %i[volume oi].include?(selector)
209
+ %i[volume oi].include?(selector)
169
210
 
211
+ measuring.call("Entering function")
170
212
  sym = get_id_set(symbol: symbol, id: id)
171
213
  if %w[R6 BJ GE].include? sym[:symbol]
172
- puts "Rejecting to process symbol '#{sym[:symbol]}'."
173
- return
214
+ puts "Rejecting to process symbol '#{sym[:symbol]}'.".light_red
215
+ return []
174
216
  end
175
217
  id = sym[:id]
176
- dfm = lambda do |x, y = Date.today.year|
218
+ dfm = lambda do |x, y = date.year|
177
219
  k = Date.strptime("#{y} #{x.negative? ? x + 366 : x}", '%Y %j')
178
220
  k -= 1 while [0, 6].include?(k.wday)
179
221
  k.strftime('%a, %Y-%m-%d')
@@ -181,13 +223,15 @@ module Cotcube
181
223
  puts "#{sym[:symbol]}\t#{x}\t#{y}"
182
224
  end
183
225
 
184
- ytoday = Date.today.yday
185
- data = continuous_overview(id: id, selector: selector, filter: filter, human: false, config: init)
186
- .reject { |k, _| k[-2..].to_i == Date.today.year % 2000 }
187
- .group_by { |k, _| k[2] }
188
- output_sent = 0
226
+ ytoday = date.yday
227
+ data = continuous_overview(id: id, selector: selector, filter: filter, human: false, config: init, measure: measure)
228
+ .reject { |k, _| k[-2..].to_i == date.year % 2000 }
229
+ .group_by { |k, _| k[2] }
230
+ measuring.call("Retrieved continous_overview")
231
+ output_sent = []
232
+ early_year=nil
189
233
  data.keys.sort.each do |month|
190
- puts "Processing #{sym[:symbol]}#{month}" if debug
234
+ puts "Processing #{sym[:symbol]}#{month}" if debuglevel > 1
191
235
  v0 = data[month]
192
236
  ydays = v0.map { |_, v1| Date.parse(v1.last[:date]).yday }
193
237
  fdays = v0.map { |_, v1| Date.parse(v1.first[:date]).yday }.sort
@@ -207,10 +251,9 @@ module Cotcube
207
251
  else
208
252
  :white
209
253
  end
210
- if debug || (color != :white)
211
- # rubocop:disable Layout/ClosingParenthesisIndentation
212
- puts "#{sym[:symbol]
213
- }#{month
254
+ # rubocop:disable Layout/ClosingParenthesisIndentation
255
+ output = "#{sym[:symbol]
256
+ }#{month
214
257
  }\t#{format '%12s', sym[:type]
215
258
  }\t#{ytoday
216
259
  }/#{fday
@@ -220,22 +263,27 @@ module Cotcube
220
263
  }: #{dfm.call(yavg)
221
264
  }\t#{format '%5d', ydays.max
222
265
  }: #{dfm.call(ydays.max)}".colorize(color)
223
- output_sent += 1
224
- end
225
- next unless debug
266
+ if debug || (color != :white)
267
+ puts output
268
+ output_sent << "#{sym[:symbol]}#{month}" unless color == :white
269
+ end
270
+ early_year ||= output
271
+ next unless (debug and debuglevel >= 2)
226
272
 
227
- v0.each do |contract, v1|
228
- puts "\t#{contract
273
+ v0.each do |contract, v1|
274
+ puts "\t#{contract
229
275
  }\t#{v1.first[:date]
230
276
  }\t#{Date.parse(v1.last[:date]).strftime('%a')
231
277
  }, #{v1.last[:date]
232
278
  } (#{Date.parse(v1.last[:date]).yday}"
233
- # rubocop:enable Layout/ClosingParenthesisIndentation
234
- end
279
+ # rubocop:enable Layout/ClosingParenthesisIndentation
280
+ end
235
281
  end
236
- case output_sent
282
+ case output_sent.size
237
283
  when 0
238
284
  puts "WARNING: No output was sent for symbol '#{sym[:symbol]}'.".colorize(:light_yellow)
285
+ puts " Assuming late-year-processing.".light_yellow
286
+ puts early_year.light_green
239
287
  when 1
240
288
  # all ok
241
289
  true
@@ -243,7 +291,8 @@ module Cotcube
243
291
  puts "---- #{output_sent} for #{sym[:symbol]} ---------------"
244
292
 
245
293
  end
246
- nil
294
+ measuring.call("Finished processing")
295
+ output_sent
247
296
  end
248
297
  end
249
298
  end
@@ -8,9 +8,9 @@ module Cotcube
8
8
  SYMBOL_EXAMPLES
9
9
  else
10
10
  CSV
11
- .read(config[:symbols_file], headers: %i[id symbol ticksize power months type bcf reports name])
11
+ .read(config[:symbols_file], headers: %i[id symbol ticksize power months type bcf reports format name])
12
12
  .map(&:to_h)
13
- .map { |row| %i[ticksize power bcf].each { |z| row[z] = row[z].to_f }; row } # rubocop:disable Style/Semicolon
13
+ .map { |row| %i[ticksize power bcf].each { |z| row[z] = row[z].to_f }; row[:format] = "%#{row[:format]}f"; row } # rubocop:disable Style/Semicolon
14
14
  .reject { |row| row[:id].nil? }
15
15
  .tap { |all| all.select! { |x| x[:type] == type } unless type.nil? }
16
16
  .tap { |all| all.select! { |x| x[:symbol] == symbol } unless symbol.nil? }
@@ -44,7 +44,11 @@ module Cotcube
44
44
  days = provide_cached contract: contract, symbol: symbol, id: id, config: config, filter: filter,
45
45
  range: range, force_recent: force_recent
46
46
  dailies = provide_daily contract: contract, symbol: symbol, id: id, config: config, range: range
47
- dailies[..-2] + days.select { |d| d[:datetime] > dailies[-2][:datetime] }
47
+ if ((days.last[:datetime] > dailies.last[:datetime]) rescue false)
48
+ dailies[..-2] + days.select { |d| d[:datetime] > dailies[-2][:datetime] }
49
+ else
50
+ dailies
51
+ end
48
52
  else
49
53
  raise ArgumentError, "Unsupported or unknown interval '#{interval}' in Bardata.provide"
50
54
  end
@@ -5,7 +5,6 @@ module Cotcube
5
5
  module Bardata
6
6
  # this is an analysis tool to investigate actual ranges of an underlying symbol
7
7
  # it is in particular no true range or average true range, as a 'true range' can only be applied to
8
- # a steady series, what changing contracts definitely aren't
9
8
  #
10
9
  # The result printed / returned is a table, containing a matrix of rows:
11
10
  # 1. size: the amount of values evaluated
@@ -22,18 +21,26 @@ module Cotcube
22
21
  # 3.a-c) same with days reduced to months (c: 12 months)
23
22
  #
24
23
  # NOTE: there is now a new method Cotcube::Helpers.simple_series_stats, that should be used in favor.
25
- def range_matrix(symbol: nil, id: nil, print: false, dim: 0.05)
24
+ def range_matrix(symbol: nil, id: nil, base: nil, print: false, dim: 0.05, days_only: false, last_n: 60, &block)
26
25
  # rubocop:disable Style/MultilineBlockChain
26
+ symbol ||= base.last[:contract][0..1] if id.nil?
27
27
  sym = get_id_set(symbol: symbol, id: id)
28
28
  source = {}
29
29
  target = {}
30
- source[:days] = Cotcube::Bardata.continuous_actual_ml symbol: symbol
30
+ if base.nil?
31
+ ml = (Cotcube::Bardata.continuous_actual_ml symbol: symbol)&.last[:contract]
32
+ source[:days] = Cotcube::Bardata.provide contract: ml
33
+ else
34
+ source[:days] = base
35
+ end
31
36
  source[:weeks] = Cotcube::Helpers.reduce bars: source[:days], to: :weeks
32
37
  source[:months] = Cotcube::Helpers.reduce bars: source[:days], to: :months
33
38
 
34
39
  %i[days weeks months].each do |period|
40
+ next if days_only and %i[weeks months].include? period
41
+ p yield(source[period].first)
35
42
  source[period].map! do |x|
36
- x[:range] = ((x[:high] - x[:low]) / sym[:ticksize]).round
43
+ x[:range] = block_given? ? yield(x) : (((x[:high] - x[:low]) / sym[:ticksize]).round)
37
44
  x
38
45
  end
39
46
  target[period] = {}
@@ -80,14 +87,18 @@ module Cotcube
80
87
 
81
88
  range = case period
82
89
  when :months
83
- -13..-2
90
+ -(last_n/15)..-2
84
91
  when :weeks
85
- -53..-2
92
+ -(last_n/4)..-2
86
93
  when :days
87
- -200..-1
94
+ -last_n..-1
88
95
  else
89
96
  raise ArgumentError, "Unsupported period: '#{period}'"
90
97
  end
98
+ if range.begin.abs > source[period].size
99
+ puts "WARNING: requested last_n = #{last_n} exceeds actual size (#{source[period].size}), adjusting...".light_yellow
100
+ range = (-source[period].size..range.end)
101
+ end
91
102
  custom = source[period][range]
92
103
  target[period][:rec_size] = custom.size
93
104
  target[period][:rec_avg] = (custom.map { |x| x[:range] }.reduce(:+) / custom.size).round
@@ -110,6 +121,7 @@ module Cotcube
110
121
  %w[size avg lower median upper max].each do |a|
111
122
  print "#{'%10s' % a} | " # rubocop:disable Style/FormatString
112
123
  %i[days weeks months].each do |b|
124
+ next if days_only and %i[weeks months].include? b
113
125
  %w[all dim rec].each do |c|
114
126
  print ('%8d' % target[b]["#{c}_#{a}".to_sym]).to_s # rubocop:disable Style/FormatString
115
127
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cotcube-bardata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.9.1
4
+ version: 0.1.12
5
5
  platform: ruby
6
6
  authors:
7
7
  - Benjamin L. Tischendorf
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-01-29 00:00:00.000000000 Z
11
+ date: 2021-03-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport