yf_as_dataframe 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rubocop.yml +13 -0
- data/CHANGELOG.rst +0 -0
- data/CODE_OF_CONDUCT.md +15 -0
- data/Gemfile +10 -0
- data/LICENSE.txt +21 -0
- data/README.md +299 -0
- data/Rakefile +8 -0
- data/chart.png +0 -0
- data/lib/yf_as_dataframe/analysis.rb +68 -0
- data/lib/yf_as_dataframe/financials.rb +304 -0
- data/lib/yf_as_dataframe/fundamentals.rb +53 -0
- data/lib/yf_as_dataframe/holders.rb +253 -0
- data/lib/yf_as_dataframe/multi.rb +238 -0
- data/lib/yf_as_dataframe/price_history.rb +2045 -0
- data/lib/yf_as_dataframe/price_technical.rb +579 -0
- data/lib/yf_as_dataframe/quote.rb +343 -0
- data/lib/yf_as_dataframe/ticker.rb +380 -0
- data/lib/yf_as_dataframe/tickers.rb +50 -0
- data/lib/yf_as_dataframe/utils.rb +354 -0
- data/lib/yf_as_dataframe/version.rb +3 -0
- data/lib/yf_as_dataframe/yf_connection.rb +304 -0
- data/lib/yf_as_dataframe/yfinance_exception.rb +15 -0
- data/lib/yf_as_dataframe.rb +24 -0
- metadata +139 -0
@@ -0,0 +1,380 @@
|
|
1
|
+
class YfAsDataframe
|
2
|
+
class Ticker
|
3
|
+
ROOT_URL = 'https://finance.yahoo.com'.freeze
|
4
|
+
BASE_URL = 'https://query2.finance.yahoo.com'.freeze
|
5
|
+
|
6
|
+
include YfAsDataframe::YfConnection
|
7
|
+
|
8
|
+
attr_accessor :tz, :proxy, :isin, :timeout
|
9
|
+
attr_reader :error_message, :ticker
|
10
|
+
|
11
|
+
class YahooFinanceException < Exception
|
12
|
+
end
|
13
|
+
|
14
|
+
class SymbolNotFoundException < YahooFinanceException
|
15
|
+
end
|
16
|
+
|
17
|
+
def initialize(ticker)
|
18
|
+
@proxy = nil
|
19
|
+
@timeout = 30
|
20
|
+
@tz = TZInfo::Timezone.get('America/New_York')
|
21
|
+
|
22
|
+
@isin = nil
|
23
|
+
@news = []
|
24
|
+
@shares = nil
|
25
|
+
|
26
|
+
@earnings_dates = {}
|
27
|
+
@expirations = {}
|
28
|
+
@underlying = {}
|
29
|
+
|
30
|
+
@ticker = (YfAsDataframe::Utils.is_isin(ticker.upcase) ? YfAsDataframe::Utils.get_ticker_by_isin(ticker.upcase, nil, @session) : ticker).upcase
|
31
|
+
|
32
|
+
yfconn_initialize
|
33
|
+
end
|
34
|
+
|
35
|
+
include YfAsDataframe::PriceHistory
|
36
|
+
include YfAsDataframe::Analysis
|
37
|
+
include YfAsDataframe::Fundamentals
|
38
|
+
include YfAsDataframe::Holders
|
39
|
+
include YfAsDataframe::Quote
|
40
|
+
include YfAsDataframe::Financials
|
41
|
+
|
42
|
+
alias_method :symbol, :ticker
|
43
|
+
|
44
|
+
def symbol; @ticker; end
|
45
|
+
|
46
|
+
def shares_full(start: nil, fin: nil)
|
47
|
+
logger = Rails.logger
|
48
|
+
|
49
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} start = #{start.inspect}, fin = #{fin.inspect}" }
|
50
|
+
|
51
|
+
if start
|
52
|
+
start_ts = YfAsDataframe::Utils.parse_user_dt(start, tz)
|
53
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} start_ts = #{start_ts}" }
|
54
|
+
start = Time.at(start_ts).in_time_zone(tz)
|
55
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} start = #{start.inspect}, fin = #{fin.inspect}" }
|
56
|
+
end
|
57
|
+
if fin
|
58
|
+
end_ts = YfAsDataframe::Utils.parse_user_dt(fin, tz)
|
59
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} end_ts = #{end_ts}" }
|
60
|
+
fin = Time.at(end_ts).in_time_zone(tz)
|
61
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} start = #{start.inspect}, fin = #{fin.inspect}" }
|
62
|
+
end
|
63
|
+
|
64
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} start = #{start.inspect}, fin = #{fin.inspect}" }
|
65
|
+
|
66
|
+
dt_now = DateTime.now.in_time_zone(tz)
|
67
|
+
fin ||= dt_now
|
68
|
+
start ||= (fin - 548.days).midnight
|
69
|
+
|
70
|
+
if start >= fin
|
71
|
+
logger.error("Start date (#{start}) must be before end (#{fin})")
|
72
|
+
return nil
|
73
|
+
end
|
74
|
+
|
75
|
+
ts_url_base = "https://query2.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/#{@ticker}?symbol=#{@ticker}"
|
76
|
+
shares_url = "#{ts_url_base}&period1=#{start.to_i}&period2=#{fin.tomorrow.midnight.to_i}"
|
77
|
+
|
78
|
+
begin
|
79
|
+
json_data = get(shares_url).parsed_response
|
80
|
+
rescue #_json.JSONDecodeError, requests.exceptions.RequestException
|
81
|
+
logger.error("#{@ticker}: Yahoo web request for share count failed")
|
82
|
+
return nil
|
83
|
+
end
|
84
|
+
|
85
|
+
fail = json_data["finance"]["error"]["code"] == "Bad Request" rescue false
|
86
|
+
if fail
|
87
|
+
logger.error("#{@ticker}: Yahoo web request for share count failed")
|
88
|
+
return nil
|
89
|
+
end
|
90
|
+
|
91
|
+
shares_data = json_data["timeseries"]["result"]
|
92
|
+
|
93
|
+
return nil if !shares_data[0].key?("shares_out")
|
94
|
+
|
95
|
+
timestamps = shares_data[0]["timestamp"].map{|t| Time.at(t).to_datetime }
|
96
|
+
|
97
|
+
df = Polars::DataFrame.new(
|
98
|
+
{
|
99
|
+
'Timestamps': timestamps,
|
100
|
+
"Shares": shares_data[0]["shares_out"]
|
101
|
+
}
|
102
|
+
)
|
103
|
+
|
104
|
+
return df
|
105
|
+
end
|
106
|
+
|
107
|
+
def shares
|
108
|
+
return @shares unless @shares.nil?
|
109
|
+
|
110
|
+
full_shares = shares_full(start: DateTime.now.utc.to_date-548.days, fin: DateTime.now.utc.to_date)
|
111
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} full_shares = #{full_shares.inspect}" }
|
112
|
+
|
113
|
+
# if shares.nil?
|
114
|
+
# # Requesting 18 months failed, so fallback to shares which should include last year
|
115
|
+
# shares = @ticker.get_shares()
|
116
|
+
|
117
|
+
# if shares.nil?
|
118
|
+
full_shares = full_shares['Shares'] if full_shares.is_a?(Polars::DataFrame)
|
119
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} full_shares = #{full_shares.inspect}" }
|
120
|
+
@shares = full_shares[-1].to_i
|
121
|
+
# end
|
122
|
+
# return @shares
|
123
|
+
end
|
124
|
+
|
125
|
+
def news()
|
126
|
+
return @news unless @news.empty?
|
127
|
+
|
128
|
+
url = "#{BASE_URL}/v1/finance/search?q=#{@ticker}"
|
129
|
+
data = get(url).parsed_response
|
130
|
+
if data.include?("Will be right back")
|
131
|
+
raise RuntimeError("*** YAHOO! FINANCE IS CURRENTLY DOWN! ***\nOur engineers are working quickly to resolve the issue. Thank you for your patience.")
|
132
|
+
end
|
133
|
+
|
134
|
+
@news = {}
|
135
|
+
data['news'].each do |item|
|
136
|
+
@news[item['title']] = item['link']
|
137
|
+
end
|
138
|
+
|
139
|
+
return @news
|
140
|
+
end
|
141
|
+
|
142
|
+
def earnings_dates(limit = 12)
|
143
|
+
# """
|
144
|
+
# Get earning dates (future and historic)
|
145
|
+
# :param limit: max amount of upcoming and recent earnings dates to return.
|
146
|
+
# Default value 12 should return next 4 quarters and last 8 quarters.
|
147
|
+
# Increase if more history is needed.
|
148
|
+
|
149
|
+
# :return: Polars dataframe
|
150
|
+
# """
|
151
|
+
return @earnings_dates[limit] if @earnings_dates && @earnings_dates[limit]
|
152
|
+
|
153
|
+
logger = Rails.logger
|
154
|
+
|
155
|
+
page_size = [limit, 100].min # YF caps at 100, don't go higher
|
156
|
+
page_offset = 0
|
157
|
+
dates = nil
|
158
|
+
# while true
|
159
|
+
url = "#{ROOT_URL}/calendar/earnings?symbol=#{@ticker}&offset=#{page_offset}&size=#{page_size}"
|
160
|
+
data = get(url).parsed_response # @data.cache_get(url: url).text
|
161
|
+
|
162
|
+
if data.include?("Will be right back")
|
163
|
+
raise RuntimeError, "*** YAHOO! FINANCE IS CURRENTLY DOWN! ***\nOur engineers are working quickly to resolve the issue. Thank you for your patience."
|
164
|
+
end
|
165
|
+
|
166
|
+
csv = ''
|
167
|
+
doc = Nokogiri::HTML(data)
|
168
|
+
tbl = doc.xpath("//table").first
|
169
|
+
tbl.search('tr').each do |tr|
|
170
|
+
cells = tr.search('th, td')
|
171
|
+
csv += CSV.generate_line(cells)
|
172
|
+
end
|
173
|
+
csv = CSV.parse(csv)
|
174
|
+
|
175
|
+
df = {}
|
176
|
+
(0..csv[0].length-1).each{|i| df[csv[0][i]] = csv[1..-1].transpose[i] }
|
177
|
+
dates = Polars::DataFrame.new(df)
|
178
|
+
# end
|
179
|
+
|
180
|
+
# Drop redundant columns
|
181
|
+
dates = dates.drop(["Symbol", "Company"]) #, axis: 1)
|
182
|
+
|
183
|
+
# Convert types
|
184
|
+
["EPS Estimate", "Reported EPS", "Surprise(%)"].each do |cn|
|
185
|
+
s = Polars::Series.new([Float::NAN] * (dates.shape.first))
|
186
|
+
(0..(dates.shape.first-1)).to_a.each {|i| s[i] = dates[cn][i].to_f unless dates[cn][i] == '-' }
|
187
|
+
dates.replace(cn, s)
|
188
|
+
end
|
189
|
+
|
190
|
+
# Convert % to range 0->1:
|
191
|
+
dates["Surprise(%)"] *= 0.01
|
192
|
+
|
193
|
+
# Parse earnings date string
|
194
|
+
s = Polars::Series.new(dates['Earnings Date'].map{|t| Time.at(t.to_datetime.to_i).to_datetime }, dtype: :i64)
|
195
|
+
dates.replace('Earnings Date', s)
|
196
|
+
|
197
|
+
|
198
|
+
@earnings_dates[limit] = dates
|
199
|
+
|
200
|
+
dates
|
201
|
+
end
|
202
|
+
|
203
|
+
def option_chain(date = nil, tz = nil)
|
204
|
+
options = if date.nil?
|
205
|
+
download_options
|
206
|
+
else
|
207
|
+
download_options if @expirations.empty? || date.nil?
|
208
|
+
raise "Expiration `#{date}` cannot be found. Available expirations are: [#{@expirations.keys.join(', ')}]" unless @expirations.key?(date)
|
209
|
+
|
210
|
+
download_options(@expirations[date])
|
211
|
+
end
|
212
|
+
|
213
|
+
df = OpenStruct.new(
|
214
|
+
calls: _options_to_df(options['calls'], tz),
|
215
|
+
puts: _options_to_df(options['puts'], tz),
|
216
|
+
underlying: options['underlying']
|
217
|
+
)
|
218
|
+
end
|
219
|
+
|
220
|
+
def options
|
221
|
+
download_options if @expirations.empty?
|
222
|
+
@expirations.keys
|
223
|
+
end
|
224
|
+
|
225
|
+
alias_method :option_expiration_dates, :options
|
226
|
+
|
227
|
+
def is_valid_timezone(tz)
|
228
|
+
begin
|
229
|
+
_tz.timezone(tz)
|
230
|
+
rescue UnknownTimeZoneError
|
231
|
+
return false
|
232
|
+
end
|
233
|
+
return true
|
234
|
+
end
|
235
|
+
|
236
|
+
def to_s
|
237
|
+
"yfinance.Ticker object <#{ticker}>"
|
238
|
+
end
|
239
|
+
|
240
|
+
def download_options(date = nil)
|
241
|
+
url = date.nil? ? "#{BASE_URL}/v7/finance/options/#{@ticker}" : "#{BASE_URL}/v7/finance/options/#{@ticker}?date=#{date}"
|
242
|
+
|
243
|
+
response = get(url).parsed_response #Net::HTTP.get(uri)
|
244
|
+
|
245
|
+
if response['optionChain'].key?('result') #r.dig('optionChain', 'result')&.any?
|
246
|
+
response['optionChain']['result'][0]['expirationDates'].each do |exp|
|
247
|
+
@expirations[Time.at(exp).utc.strftime('%Y-%m-%d')] = exp
|
248
|
+
end
|
249
|
+
|
250
|
+
@underlying = response['optionChain']['result'][0]['quote'] || {}
|
251
|
+
|
252
|
+
opt = response['optionChain']['result'][0]['options'] || []
|
253
|
+
|
254
|
+
return opt.empty? ? {} : opt[0].merge('underlying' => @underlying)
|
255
|
+
end
|
256
|
+
{}
|
257
|
+
end
|
258
|
+
|
259
|
+
def isin()
|
260
|
+
return @isin if !@isin.nil?
|
261
|
+
|
262
|
+
# ticker = @ticker.upcase
|
263
|
+
|
264
|
+
if ticker.include?("-") || ticker.include?("^")
|
265
|
+
@isin = '-'
|
266
|
+
return @isin
|
267
|
+
end
|
268
|
+
|
269
|
+
q = ticker
|
270
|
+
@info ||= info
|
271
|
+
return nil if @info.nil?
|
272
|
+
|
273
|
+
# q = @info['quoteType'].try(:[],'shortName') # if @info.key?("shortName")
|
274
|
+
|
275
|
+
url = "https://markets.businessinsider.com/ajax/SearchController_Suggest?max_results=25&query=#{(q)}"
|
276
|
+
data = get(url).parsed_response
|
277
|
+
|
278
|
+
search_str = "\"#{ticker}|"
|
279
|
+
if !data.include?(search_str)
|
280
|
+
if data.downcase.include?(q.downcase)
|
281
|
+
search_str = '"|'
|
282
|
+
if !data.include?(search_str)
|
283
|
+
@isin = '-'
|
284
|
+
return @isin
|
285
|
+
end
|
286
|
+
else
|
287
|
+
@isin = '-'
|
288
|
+
return @isin
|
289
|
+
end
|
290
|
+
end
|
291
|
+
|
292
|
+
@isin = data.split(search_str)[1].split('"')[0].split('|')[0]
|
293
|
+
return @isin
|
294
|
+
end
|
295
|
+
|
296
|
+
|
297
|
+
|
298
|
+
|
299
|
+
|
300
|
+
|
301
|
+
|
302
|
+
|
303
|
+
|
304
|
+
|
305
|
+
|
306
|
+
|
307
|
+
private
|
308
|
+
|
309
|
+
# def _lazy_load_price_history
|
310
|
+
# @price_history ||= PriceHistory.new(@ticker, _get_ticker_tz(@proxy, timeout: 10), @data)
|
311
|
+
# end
|
312
|
+
|
313
|
+
alias_method :_get_ticker_tz, :tz
|
314
|
+
# def _get_ticker_tz(proxy=nil, timeout=nil)
|
315
|
+
# return @tz
|
316
|
+
# end
|
317
|
+
|
318
|
+
# def _fetch_ticker_tz(proxy, timeout)
|
319
|
+
# proxy ||= @proxy
|
320
|
+
|
321
|
+
# params = {"range": "1d", "interval": "1d"}
|
322
|
+
|
323
|
+
# url = "#{BASE_URL}/v8/finance/chart/#{@ticker}"
|
324
|
+
|
325
|
+
# begin
|
326
|
+
# data = @data.cache_get(url: url, params: params, proxy: proxy, timeout: timeout)
|
327
|
+
# data = data.json()
|
328
|
+
# rescue Exception => e
|
329
|
+
# Rails.logger.error("Failed to get ticker '#{@ticker}' reason: #{e}")
|
330
|
+
# return nil
|
331
|
+
# end
|
332
|
+
|
333
|
+
# error = data.get('chart', {}).get('error', nil)
|
334
|
+
# if error
|
335
|
+
# Rails.logger.debug("Got error from yahoo api for ticker #{@ticker}, Error: #{error}")
|
336
|
+
# else
|
337
|
+
# begin
|
338
|
+
# return data["chart"]["result"][0]["meta"]["exchangeTimezoneName"]
|
339
|
+
# rescue Exception => err
|
340
|
+
# Rails.logger.error("Could not get exchangeTimezoneName for ticker '#{@ticker}' reason: #{err}")
|
341
|
+
# Rails.logger.debug("Got response: ")
|
342
|
+
# Rails.logger.debug("-------------")
|
343
|
+
# Rails.logger.debug(" #{data}")
|
344
|
+
# Rails.logger.debug("-------------")
|
345
|
+
# end
|
346
|
+
# end
|
347
|
+
|
348
|
+
# return nil
|
349
|
+
# end
|
350
|
+
|
351
|
+
def _options_to_df(opt, tz = nil)
|
352
|
+
data = opt.map do |o|
|
353
|
+
{
|
354
|
+
contractSymbol: o['contractSymbol'],
|
355
|
+
lastTradeDate: DateTime.strptime(o['lastTradeDate'].to_s, '%s').new_offset(0),
|
356
|
+
strike: o['strike'],
|
357
|
+
lastPrice: o['lastPrice'],
|
358
|
+
bid: o['bid'],
|
359
|
+
ask: o['ask'],
|
360
|
+
change: o['change'],
|
361
|
+
percentChange: o['percentChange'],
|
362
|
+
volume: o['volume'],
|
363
|
+
openInterest: o['openInterest'],
|
364
|
+
impliedVolatility: o['impliedVolatility'],
|
365
|
+
inTheMoney: o['inTheMoney'],
|
366
|
+
contractSize: o['contractSize'],
|
367
|
+
currency: o['currency']
|
368
|
+
}
|
369
|
+
end
|
370
|
+
|
371
|
+
if tz
|
372
|
+
data.each do |d|
|
373
|
+
d[:lastTradeDate] = d[:lastTradeDate].new_offset(tz)
|
374
|
+
end
|
375
|
+
end
|
376
|
+
|
377
|
+
data
|
378
|
+
end
|
379
|
+
end
|
380
|
+
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
class YfAsDataframe
|
2
|
+
class Tickers
|
3
|
+
def initialize(tickers, session = nil)
|
4
|
+
tickers = tickers.is_a?(Array) ? tickers : tickers.split(',')
|
5
|
+
@symbols = tickers.map(&:upcase)
|
6
|
+
@tickers = @symbols.each_with_object({}) do |ticker, hash|
|
7
|
+
hash[ticker] = Ticker.new(ticker, session: session)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def to_s
|
12
|
+
"yfinance.Tickers object <#{@symbols.join(', ')}>"
|
13
|
+
end
|
14
|
+
|
15
|
+
def history(period: "1mo", interval: "1d", start: nil, fin: nil, prepost: false,
|
16
|
+
actions: true, auto_adjust: true, repair: false, proxy: nil,
|
17
|
+
threads: true, group_by: 'column', progress: true, timeout: 10, **kwargs)
|
18
|
+
download(period: period, interval: interval, start: start, fin: fin, prepost: prepost,
|
19
|
+
actions: actions, auto_adjust: auto_adjust, repair: repair, proxy: proxy,
|
20
|
+
threads: threads, group_by: group_by, progress: progress, timeout: timeout, **kwargs)
|
21
|
+
end
|
22
|
+
|
23
|
+
def download(period: "1mo", interval: "1d", start: nil, fin: nil, prepost: false,
|
24
|
+
actions: true, auto_adjust: true, repair: false, proxy: nil,
|
25
|
+
threads: true, group_by: 'column', progress: true, timeout: 10, **kwargs)
|
26
|
+
data = Multi.download(@symbols, start: start, fin: fin, actions: actions,
|
27
|
+
auto_adjust: auto_adjust, repair: repair, period: period,
|
28
|
+
interval: interval, prepost: prepost, proxy: proxy,
|
29
|
+
group_by: 'ticker', threads: threads, progress: progress,
|
30
|
+
timeout: timeout, **kwargs)
|
31
|
+
|
32
|
+
@symbols.each do |symbol|
|
33
|
+
@tickers[symbol]._history = data[symbol] if @tickers[symbol]
|
34
|
+
end
|
35
|
+
|
36
|
+
if group_by == 'column'
|
37
|
+
data.columns = data.columns.swaplevel(0, 1)
|
38
|
+
data.sort_index(level: 0, axis: 1, inplace: true)
|
39
|
+
end
|
40
|
+
|
41
|
+
data
|
42
|
+
end
|
43
|
+
|
44
|
+
def news
|
45
|
+
@symbols.each_with_object({}) do |ticker, hash|
|
46
|
+
hash[ticker] = Ticker.new(ticker).news
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|