cotcube-bardata 0.1.1 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,133 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cotcube
4
+ # Missing top level documentation comment
5
+ module Bardata
6
+ # just reads bardata/daily/<id>/<contract>.csv
7
+ def provide_daily(contract:,
8
+ symbol: nil, id: nil,
9
+ timezone: Time.find_zone('America/Chicago'),
10
+ config: init)
11
+ contract = contract.to_s.upcase
12
+ unless contract.is_a?(String) && [3, 5].include?(contract.size)
13
+ raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
14
+ end
15
+
16
+ sym = get_id_set(symbol: symbol, id: id, contract: contract)
17
+ contract = contract[2..4] if contract.to_s.size == 5
18
+ id = sym[:id]
19
+ id_path = "#{config[:data_path]}/daily/#{id}"
20
+ data_file = "#{id_path}/#{contract}.csv"
21
+ raise "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)
22
+
23
+ raise "No data found for requested contract #{symbol}:#{contract} in #{id_path}." unless File.exist?(data_file)
24
+
25
+ data = CSV.read(data_file, headers: %i[contract date open high low close volume oi]).map do |row|
26
+ row = row.to_h
27
+ row.each do |k, _|
28
+ row[k] = row[k].to_f if %i[open high low close].include? k
29
+ row[k] = row[k].to_i if %i[volume oi].include? k
30
+ end
31
+ row[:datetime] = timezone.parse(row[:date])
32
+ row[:type] = :daily
33
+ row
34
+ end
35
+ data.pop if data.last[:high].zero?
36
+ data
37
+ end
38
+
39
+ # reads all files in bardata/daily/<id> and aggregates by date
40
+ # (what is a pre-stage of a continuous based on daily bars)
41
+ def continuous(symbol: nil, id: nil, config: init, date: nil)
42
+ sym = get_id_set(symbol: symbol, id: id)
43
+ id = sym[:id]
44
+ id_path = "#{config[:data_path]}/daily/#{id}"
45
+ available_contracts = Dir["#{id_path}/*.csv"].map { |x| x.split('/').last.split('.').first }
46
+ available_contracts.sort_by! { |x| x[-7] }.sort_by! { |x| x[-6..-5] }
47
+ data = []
48
+ available_contracts.each do |c|
49
+ provide_daily(id: id, config: config, contract: c).each do |x|
50
+ data << x
51
+ end
52
+ end
53
+ result = []
54
+ data.sort_by { |x| x[:date] }.group_by { |x| x[:date] }.map do |k, v|
55
+ v.map { |x| x.delete(:date) }
56
+ result << {
57
+ date: k,
58
+ volume: v.map { |x| x[:volume] }.reduce(:+),
59
+ oi: v.map { |x| x[:oi] }.reduce(:+)
60
+ }
61
+ result.last[:contracts] = v
62
+ end
63
+ date.nil? ? result : result.select { |x| x[:date] == date }.first
64
+ end
65
+
66
+ def continuous_ml(symbol: nil, id: nil, base: nil)
67
+ (base.nil? ? Cotcube::Bardata.continuous(symbol: symbol, id: id) : base).map do |x|
68
+ x[:ml] = x[:contracts].max_by { |z| z[:volume] }[:contract]
69
+ { date: x[:date], ml: x[:ml] }
70
+ end
71
+ end
72
+
73
+ # the method above delivers the most_liquid as it is found at the end of the day. D
74
+ # during trading, the work is done with data
75
+ # that is already one day old. This is is fixed here:
76
+ def continuous_actual_ml(symbol: nil, id: nil)
77
+ continuous = Cotcube::Bardata.continuous symbol: symbol, id: id
78
+ continuous_ml = Cotcube::Bardata.continuous_ml base: continuous
79
+ continuous_hash = continuous.to_h { |x| [x[:date], x[:contracts]] }
80
+ actual_ml = continuous_ml.pairwise { |a, b| { date: b[:date], ml: a[:ml] } }
81
+ actual_ml.map do |x|
82
+ r = continuous_hash[x[:date]].select { |z| x[:ml] == z[:contract] }.first
83
+ r = continuous_hash[x[:date]].min_by { |z| -z[:volume] } if r.nil?
84
+ r
85
+ end
86
+ end
87
+
88
+ # based on .continuous, this methods sorts the prepared dailies continuous for each date
89
+ # on either :volume (default) or :oi
90
+ # with this job done, it can provide the period for which a past contract was the most liquid
91
+ #
92
+ def continuous_overview(symbol: nil, id: nil, # rubocop:disable Metrics/ParameterLists
93
+ config: init,
94
+ selector: :volume,
95
+ human: false,
96
+ filter: nil)
97
+ raise ArgumentError, 'Selector must be either :volume or :oi' unless selector.is_a?(Symbol) &&
98
+ %i[volume oi].include?(selector)
99
+
100
+ sym = get_id_set(symbol: symbol, id: id)
101
+ id = sym[:id]
102
+ # noinspection RubyNilAnalysis
103
+ data = continuous(id: id, config: config).map do |x|
104
+ {
105
+ date: x[:date],
106
+ volume: x[:contracts].sort_by { |z| - z[:volume] }[0..4].compact.reject { |z| z[:volume].zero? },
107
+ oi: x[:contracts].sort_by { |z| - z[:oi] }[0..4].compact.reject { |z| z[:oi].zero? }
108
+ }
109
+ end
110
+ data.reject! { |x| x[selector].empty? }
111
+ result = data.group_by { |x| x[selector].first[:contract] }
112
+ if human
113
+ result.each do |k, v|
114
+ next unless filter.nil? || v.first[selector].first[:contract][2..4] =~ (/#{filter}/)
115
+
116
+ # rubocop:disable Layout/ClosingParenthesisIndentation
117
+ puts "#{k
118
+ }\t#{v.first[:date]
119
+ }\t#{v.last[:date]
120
+ }\t#{format('%4d', (Date.parse(v.last[:date]) - Date.parse(v.first[:date])))
121
+ }\t#{result[k].map do |x|
122
+ x[:volume].select do
123
+ x[:contract] == k
124
+ end
125
+ end.size
126
+ }"
127
+ # rubocop:enable Layout/ClosingParenthesisIndentation
128
+ end
129
+ end
130
+ result
131
+ end
132
+ end
133
+ end
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cotcube
4
+ # Missing top level documentation
5
+ module Bardata
6
+ def most_liquid_for(symbol: nil, id: nil, date: last_trade_date, config: init)
7
+ id = get_id_set(symbol: symbol, id: id, config: config)[:id]
8
+ provide_eods(id: id, dates: date, contracts_only: true).first
9
+ end
10
+
11
+ # the following method seems to be garbage. It is not used anywhere. It seems it's purpose
12
+ # was to retrieve a list of quarters that have not been fetched recently (--> :age)
13
+ def provide_most_liquids_by_eod(symbol: nil, id: nil, # rubocop:disable Metrics/ParameterLists
14
+ config: init,
15
+ date: last_trade_date,
16
+ filter: :volume_part,
17
+ age: 1.hour)
18
+ sym = get_id_set(symbol: symbol, id: id) if symbol || id
19
+ # noinspection RubyScope
20
+ eods = provide_eods(id: sym.nil? ? nil : sym[:id], config: config, dates: date, filter: filter)
21
+ result = []
22
+ eods.map do |eod|
23
+ symbol = eod[0..1]
24
+ contract = eod[2..4]
25
+ sym = symbols.select { |s| s[:symbol] == symbol.to_s.upcase }.first
26
+ quarter = "#{config[:data_path]}/quarters/#{sym[:id]}/#{contract}.csv"
27
+ if File.exist?(quarter)
28
+ # puts "#{quarter}: #{ Time.now } - #{File.mtime(quarter)} > #{age} : #{Time.now - File.mtime(quarter) > age}"
29
+ result << eod if Time.now - File.mtime(quarter) > age
30
+ else
31
+ result << eod
32
+ end
33
+ end
34
+ result
35
+ end
36
+
37
+ # provide a list of all eods for id/symbol or all symbols (default) for an
38
+ # array of dates (default: [last_trade_date])
39
+ #
40
+ # filter by :threshold*100% share on entire volume(default) or oi
41
+ #
42
+ # return full data or just the contract name (default)
43
+ def provide_eods(symbol: nil, # rubocop:disable Metrics/ParameterLists
44
+ id: nil,
45
+ contract: nil,
46
+ config: init,
47
+ # should accept either a date or date_alike or date string OR a range of 2 dates alike
48
+ # if omitted returns the eods of last trading date
49
+ dates: last_trade_date,
50
+ # set threshold to 0 to disable filtering at all.
51
+ # otherwise only contracts with partial of >= threshold are returned
52
+ threshold: 0.05,
53
+ # filter can be set to volume_part and oi_part.
54
+ # determines, which property is used for filtering.
55
+ filter: :volume_part,
56
+ # set to false to return the complete row instead
57
+ # of just the contracts matching filter and threshold
58
+ contracts_only: true,
59
+ quiet: false)
60
+ unless contract.nil? || (contract.is_a?(String) && [3, 5].include?(contract.size))
61
+ raise ArgumentError, "Contract '#{contract}' is bogus, should be like 'M21' or 'ESM21'"
62
+ end
63
+
64
+ symbol = contract[0..1] if contract.to_s.size == 5
65
+ sym = get_id_set(symbol: symbol, id: id, config: config) if symbol || id
66
+ # if no id can be clarified from given arguments, return all matching contracts from all available symbols
67
+ # raise ArgumentError, "Could not guess :id or :symbol from 'contract: #{contract}', please clarify." if id.nil?
68
+ raise ArgumentError, ':filter must be in [:volume_part, :oi_part]' unless %i[volume_part oi_part].include? filter
69
+
70
+ # noinspection RubyScope
71
+ ids = sym.nil? ? symbols.map { |x| x[:id] } : [sym[:id]]
72
+ dates = [dates] unless dates.is_a?(Array) || dates.nil?
73
+
74
+ id_path_get = ->(local_id) { "#{config[:data_path]}/eods/#{local_id}" }
75
+
76
+ process_date_for_id = lambda do |d, i|
77
+ # l_sym = symbols.select { |s| s[:id] == i }.first
78
+ # l_symbol = l_sym[:symbol]
79
+ id_path = id_path_get.call(i)
80
+ data_file = "#{id_path}/#{d}.csv"
81
+ raise "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)
82
+
83
+ unless File.exist?(data_file)
84
+ unless quiet
85
+ puts 'WARNING: No data found for requested id/symbol'\
86
+ " #{id}/#{symbol} in #{id_path} for #{d}.".colorize(:light_yellow)
87
+ end
88
+ return []
89
+ end
90
+ data = CSV.read(data_file, headers: %i[contract date open high low close volume oi]).map do |row|
91
+ row = row.to_h
92
+ row.each do |k, _|
93
+ row[k] = row[k].to_f if %i[open high low close].include? k
94
+ row[k] = row[k].to_i if %i[volume oi].include? k
95
+ end
96
+ row
97
+ end
98
+ all_volume = data.map { |x| x[:volume] }.reduce(:+)
99
+ all_oi = data.map { |x| x[:oi] }.reduce(:+)
100
+ data.map do |x|
101
+ x[:volume_part] = (x[:volume] / all_volume.to_f).round(4)
102
+ x[:oi_part] = (x[:oi] / all_oi.to_f).round(4)
103
+ end
104
+ data.select { |x| x[filter] >= threshold }.sort_by { |x| -x[filter] }.tap do |x|
105
+ if contracts_only
106
+ x.map! do |y|
107
+ y[:contract]
108
+ end
109
+ end
110
+ end
111
+ end
112
+ if dates
113
+ dates.map do |date|
114
+ ids.map { |local_id| process_date_for_id.call(date, local_id) }
115
+ end.flatten
116
+ else
117
+ raise ArgumentError,
118
+ 'Sorry, support for unlimited dates is not implemented yet. Please send array of dates or single date'
119
+ end
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,107 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cotcube
4
+ # Missing top level documentation comment
5
+ module Bardata
6
+ # small helper to select a specific full trading day from quarters (or reduced)
7
+ # this special handling is needed, as full trading days start '5pm CT yesterday'
8
+ def select_specific_date(date:, base:)
9
+ base.select do |d|
10
+ d[:day] == date.day and date.year == d[:datetime].year and (
11
+ if date.day > 1
12
+ date.month == d[:datetime].month
13
+ else
14
+ ((date.month == d[:datetime].month and d[:datetime].day == 1) or
15
+ (date.month == d[:datetime].month + 1 and d[:datetime].day > 25))
16
+ end
17
+ )
18
+ end
19
+ end
20
+
21
+ # diminishes a given base of bars to fit into a given range (DO NOT CONFUSE with trading_hours)
22
+ # note that the last bar is simply required to _start_ within the given range, not to end withing
23
+ def extended_select_for_range(base:,
24
+ range: ('1900-01-01'...'2100-01-01'),
25
+ timezone: Time.find_zone('America/Chicago'),
26
+ quiet: false)
27
+
28
+ starting = range.begin
29
+ starting = timezone.parse(starting) if starting.is_a? String
30
+ ending = range.end
31
+ ending = timezone.parse(ending) if ending.is_a? String
32
+ puts "#{starting}\t#{ending}" unless quiet
33
+ if starting.hour.zero? && starting.min.zero? && ending.hour.zero? && ending.min.zero?
34
+ unless quiet
35
+ puts 'WARNING: When sending midnight, full trading day'\
36
+ ' is assumed (starting 5 pm CT yesterday, ending 4 pm CT today)'.colorize(:light_yellow)
37
+ end
38
+ result = select_specific_date(date: starting, base: base)
39
+ result += base.select { |d| d[:datetime] > starting and d[:datetime] < ending.to_date }
40
+ result += select_specific_date(date: ending, base: base)
41
+ result.uniq!
42
+ else
43
+ result = base.select { |x| x[:datetime] >= starting and x[:datetime] < ending }
44
+ end
45
+ result
46
+ end
47
+
48
+ def get_id_set(symbol: nil, id: nil, contract: nil, config: init)
49
+ if contract.is_a?(String) && (contract.length == 5)
50
+ c_symbol = contract[0..1]
51
+ if (not symbol.nil?) && (symbol != c_symbol)
52
+ raise ArgumentError,
53
+ "Mismatch between given symbol #{symbol} and contract #{contract}"
54
+ end
55
+
56
+ symbol = c_symbol
57
+ end
58
+
59
+ unless symbol.nil?
60
+ sym = symbols.select { |s| s[:symbol] == symbol.to_s.upcase }.first
61
+ if sym.nil? || sym[:id].nil?
62
+ raise ArgumentError,
63
+ "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}"
64
+ end
65
+ raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if (not id.nil?) && (sym[:id] != id)
66
+
67
+ return sym
68
+ end
69
+ unless id.nil?
70
+ sym = symbols.select { |s| s[:id] == id.to_s }.first
71
+ if sym.nil? || sym[:id].nil?
72
+ raise ArgumentError,
73
+ "Could not find match in #{config[:symbols_file]} for given id #{id}"
74
+ end
75
+ return sym
76
+ end
77
+ raise ArgumentError, 'Need :id, :symbol or valid :contract '
78
+ end
79
+
80
+ def compare(contract:, format: '%5.2f')
81
+ format = "%#{format}" unless format[0] == '%'
82
+ daily = provide(contract: contract, interval: :daily)
83
+ full = provide(contract: contract, interval: :days, filter: :full)
84
+ rth = provide(contract: contract, interval: :days, filter: :rth)
85
+ rth_dates = rth.map { |x| x[:datetime] }
86
+ daily.select! { |x| rth_dates.include? x[:datetime].to_datetime }
87
+ full.select! { |x| rth_dates.include? x[:datetime].to_datetime }
88
+
89
+ printer = lambda { |z|
90
+ # rubocop:disable Layout/ClosingParenthesisIndentation
91
+ "#{z[:datetime].strftime('%m-%d') # rubocop:disable Layout/IndentationWidth
92
+ }\t#{format format, z[:open]
93
+ }\t#{format format, z[:high]
94
+ }\t#{format format, z[:low]
95
+ }\t#{format format, z[:close]
96
+ }\t#{format '%7d', z[:volume]}"
97
+ # rubocop:enable Layout/ClosingParenthesisIndentation
98
+ }
99
+ daily.each_with_index do |_x, i|
100
+ puts "DAILY #{printer.call daily[i]}"
101
+ puts "FULL #{printer.call full[i]}"
102
+ puts "RTH #{printer.call rth[i]}"
103
+ puts ' '
104
+ end
105
+ end
106
+ end
107
+ end
@@ -1,80 +1,77 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Cotcube
4
+ # Missing top level documentation comment
4
5
  module Bardata
5
-
6
-
7
- def symbols(config: init)
6
+ def symbols(config: init, type: nil, symbol: nil)
8
7
  if config[:symbols_file].nil?
9
8
  SYMBOL_EXAMPLES
10
9
  else
11
10
  CSV
12
- .read(config[:symbols_file], headers: %i{ id symbol ticksize power months type bcf reports name})
13
- .map{|row| row.to_h }
14
- .map{|row| [ :ticksize, :power, :bcf ].each {|z| row[z] = row[z].to_f}; row }
15
- .reject{|row| row[:id].nil? }
11
+ .read(config[:symbols_file], headers: %i[id symbol ticksize power months type bcf reports name])
12
+ .map(&:to_h)
13
+ .map { |row| %i[ticksize power bcf].each { |z| row[z] = row[z].to_f }; row } # rubocop:disable Style/Semicolon
14
+ .reject { |row| row[:id].nil? }
15
+ .tap { |all| all.select! { |x| x[:type] == type } unless type.nil? }
16
+ .tap { |all| all.select! { |x| x[:symbol] == symbol } unless symbol.nil? }
16
17
  end
17
18
  end
18
19
 
19
20
  def config_prefix
20
- os = Gem::Platform.local.os
21
+ os = Gem::Platform.local.os
21
22
  case os
22
23
  when 'linux'
23
24
  ''
24
25
  when 'freebsd'
25
26
  '/usr/local'
26
27
  else
27
- raise RuntimeError, 'unknown architecture'
28
+ raise 'unknown architecture'
28
29
  end
29
30
  end
30
31
 
31
32
  def config_path
32
- config_prefix + '/etc/cotcube'
33
+ "#{config_prefix}/etc/cotcube"
33
34
  end
34
35
 
35
- def init(config_file_name: 'bardata.yml', debug: false)
36
+ def init(config_file_name: 'bardata.yml')
36
37
  name = 'bardata'
37
38
  config_file = config_path + "/#{config_file_name}"
38
39
 
39
- if File.exist?(config_file)
40
- config = YAML.load(File.read config_file).transform_keys(&:to_sym)
41
- else
42
- config = {}
43
- end
40
+ config = if File.exist?(config_file)
41
+ YAML.safe_load(File.read(config_file)).transform_keys(&:to_sym)
42
+ else
43
+ {}
44
+ end
44
45
 
45
- defaults = {
46
- data_path: config_prefix + '/var/cotcube/' + name,
46
+ defaults = {
47
+ data_path: "#{config_prefix}/var/cotcube/#{name}"
47
48
  }
48
49
 
49
-
50
50
  config = defaults.merge(config)
51
51
 
52
-
53
52
  # part 2 of init process: Prepare directories
54
53
 
55
54
  save_create_directory = lambda do |directory_name|
56
55
  unless Dir.exist?(directory_name)
57
56
  begin
58
57
  `mkdir -p #{directory_name}`
59
- unless $?.exitstatus.zero?
58
+ unless $CHILD_STATUS.exitstatus.zero?
60
59
  puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
61
60
  exit 1 unless defined?(IRB)
62
61
  end
63
- rescue
62
+ rescue StandardError
64
63
  puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
65
64
  exit 1 unless defined?(IRB)
66
65
  end
67
66
  end
68
67
  end
69
- ['',:daily,:quarters].each do |path|
70
- dir = "#{config[:data_path]}#{path == '' ? '' : '/'}#{path.to_s}"
68
+ ['', :daily, :quarters, :eods, :trading_hours, :cached].each do |path|
69
+ dir = "#{config[:data_path]}#{path == '' ? '' : '/'}#{path}"
71
70
  save_create_directory.call(dir)
72
71
  end
73
72
 
74
73
  # eventually return config
75
74
  config
76
75
  end
77
-
78
76
  end
79
77
  end
80
-