yf_as_dataframe 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.rubocop.yml +13 -0
- data/CHANGELOG.rst +0 -0
- data/CODE_OF_CONDUCT.md +15 -0
- data/Gemfile +10 -0
- data/LICENSE.txt +21 -0
- data/README.md +299 -0
- data/Rakefile +8 -0
- data/chart.png +0 -0
- data/lib/yf_as_dataframe/analysis.rb +68 -0
- data/lib/yf_as_dataframe/financials.rb +304 -0
- data/lib/yf_as_dataframe/fundamentals.rb +53 -0
- data/lib/yf_as_dataframe/holders.rb +253 -0
- data/lib/yf_as_dataframe/multi.rb +238 -0
- data/lib/yf_as_dataframe/price_history.rb +2045 -0
- data/lib/yf_as_dataframe/price_technical.rb +579 -0
- data/lib/yf_as_dataframe/quote.rb +343 -0
- data/lib/yf_as_dataframe/ticker.rb +380 -0
- data/lib/yf_as_dataframe/tickers.rb +50 -0
- data/lib/yf_as_dataframe/utils.rb +354 -0
- data/lib/yf_as_dataframe/version.rb +3 -0
- data/lib/yf_as_dataframe/yf_connection.rb +304 -0
- data/lib/yf_as_dataframe/yfinance_exception.rb +15 -0
- data/lib/yf_as_dataframe.rb +24 -0
- metadata +139 -0
@@ -0,0 +1,343 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
|
3
|
+
class YfAsDataframe
|
4
|
+
module Quote
|
5
|
+
extend ActiveSupport::Concern
|
6
|
+
|
7
|
+
def self.included(base) # built-in Ruby hook for modules
|
8
|
+
base.class_eval do
|
9
|
+
original_method = instance_method(:initialize)
|
10
|
+
define_method(:initialize) do |*args, &block|
|
11
|
+
original_method.bind(self).call(*args, &block)
|
12
|
+
initialize_quote # (your module code here)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def initialize_quote
|
18
|
+
@info = nil
|
19
|
+
@retired_info = nil
|
20
|
+
@sustainability = nil
|
21
|
+
@recommendations = nil
|
22
|
+
@upgrades_downgrades = nil
|
23
|
+
@calendar = nil
|
24
|
+
|
25
|
+
@already_scraped = false
|
26
|
+
@already_fetched = false
|
27
|
+
@already_fetched_complementary = false
|
28
|
+
end
|
29
|
+
|
30
|
+
def info #(self)
|
31
|
+
if @info.nil?
|
32
|
+
_fetch_info() #(@proxy)
|
33
|
+
_fetch_complementary() #(@proxy)
|
34
|
+
end
|
35
|
+
return @info
|
36
|
+
end
|
37
|
+
|
38
|
+
def sustainability
|
39
|
+
raise YFNotImplementedError.new('sustainability') if @sustainability.nil?
|
40
|
+
return @sustainability
|
41
|
+
end
|
42
|
+
|
43
|
+
def recommendations
|
44
|
+
Polars::Config.set_tbl_rows(-1)
|
45
|
+
if @recommendations.nil?
|
46
|
+
|
47
|
+
result = _fetch(['recommendationTrend']).parsed_response
|
48
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} result = #{result.inspect}" }
|
49
|
+
# if result.nil?
|
50
|
+
# @recommendations = YfAsDataframe::Utils.empty_df() #Polars::DataFrame()
|
51
|
+
# else
|
52
|
+
begin
|
53
|
+
data = result["quoteSummary"]["result"][0]["recommendationTrend"]["trend"]
|
54
|
+
rescue KeyError, IndexError => e
|
55
|
+
raise YfinDataException(f"Failed to parse json response from Yahoo Finance: #{e.result}")
|
56
|
+
end
|
57
|
+
@recommendations = Polars::DataFrame.new(data)
|
58
|
+
# end
|
59
|
+
end
|
60
|
+
return @recommendations
|
61
|
+
end
|
62
|
+
|
63
|
+
alias_method :recommendation_summary, :recommendations
|
64
|
+
alias_method :recommendations_summary, :recommendations
|
65
|
+
|
66
|
+
def upgrades_downgrades
|
67
|
+
Polars::Config.set_tbl_rows(-1)
|
68
|
+
if @upgrades_downgrades.nil?
|
69
|
+
result = _fetch(['upgradeDowngradeHistory']).parsed_response
|
70
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} result = #{result.inspect}" }
|
71
|
+
|
72
|
+
# if result.nil?
|
73
|
+
# @upgrades_downgrades = YfAsDataframe::Utils.empty_df() #Polars::DataFrame()
|
74
|
+
# else
|
75
|
+
begin
|
76
|
+
data = result["quoteSummary"]["result"][0]["upgradeDowngradeHistory"]["history"]
|
77
|
+
|
78
|
+
raise YfinDataException("No upgrade/downgrade history found for #{ticker.symbol}") if (data).length.zero?
|
79
|
+
|
80
|
+
df = Polars::DataFrame.new(data)
|
81
|
+
df.rename({"epochGradeDate" => "GradeDate", 'firm' => 'Firm', 'toGrade' => 'ToGrade', 'fromGrade' => 'FromGrade', 'action' => 'Action'})
|
82
|
+
# df.set_index('GradeDate', inplace=true)
|
83
|
+
# df.index = pd.to_datetime(df.index, unit='s')
|
84
|
+
@upgrades_downgrades = df
|
85
|
+
rescue KeyError, IndexError => e
|
86
|
+
raise YfinDataException("Failed to parse json response from Yahoo Finance: #{e.result}")
|
87
|
+
end
|
88
|
+
# end
|
89
|
+
end
|
90
|
+
return @upgrades_downgrades
|
91
|
+
end
|
92
|
+
|
93
|
+
def calendar
|
94
|
+
self._fetch_calendar() if @calendar.nil?
|
95
|
+
return @calendar
|
96
|
+
end
|
97
|
+
|
98
|
+
def valid_modules()
|
99
|
+
return QUOTE_SUMMARY_VALID_MODULES
|
100
|
+
end
|
101
|
+
|
102
|
+
# quote_methods = [:info, :sustainability, :recommendations, :recommendations_summary, :recommendation_summary, \
|
103
|
+
# :upgrades_downgrades, :calendar]
|
104
|
+
# quote_methods.each do |meth|
|
105
|
+
# # define_method "get_#{meth}".to_sym do
|
106
|
+
# # data = @quote.send(meth.to_sym)
|
107
|
+
# # return data
|
108
|
+
# # end
|
109
|
+
# alias_method "get_#{meth}".to_sym, meth
|
110
|
+
# end
|
111
|
+
|
112
|
+
|
113
|
+
|
114
|
+
|
115
|
+
|
116
|
+
|
117
|
+
private
|
118
|
+
|
119
|
+
def _fetch(modules) #(self, proxy, modules: list)
|
120
|
+
# raise YahooFinanceException("Should provide a list of modules, see available modules using `valid_modules`") if !modules.is_a?(Array)
|
121
|
+
|
122
|
+
modules = modules.intersection(QUOTE_SUMMARY_VALID_MODULES) #[m for m in modules if m in quote_summary_valid_modules])
|
123
|
+
|
124
|
+
raise YahooFinanceException("No valid modules provided.") if modules.empty?
|
125
|
+
|
126
|
+
params_dict = {"modules": modules.join(','), "corsDomain": "finance.yahoo.com", "formatted": "false", "symbol": symbol}
|
127
|
+
|
128
|
+
begin
|
129
|
+
result = get_raw_json(QUOTE_SUMMARY_URL + "/#{symbol}", user_agent_headers=user_agent_headers, params=params_dict)
|
130
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} result = #{result.inspect}" }
|
131
|
+
rescue Exception => e
|
132
|
+
Rails.logger.error("ERROR: #{e.message}")
|
133
|
+
return nil
|
134
|
+
end
|
135
|
+
return result
|
136
|
+
end
|
137
|
+
|
138
|
+
def _format(k, v)
|
139
|
+
v2 = nil
|
140
|
+
if isinstance(v, dict) && "raw".in?(v) && "fmt".in?(v)
|
141
|
+
v2 = k.in?(["regularMarketTime", "postMarketTime"]) ? v["fmt"] : v["raw"]
|
142
|
+
elsif isinstance(v, list)
|
143
|
+
v2 = v.map{|vv| _format(nil, vv)}
|
144
|
+
elsif isinstance(v, dict)
|
145
|
+
v2 = v.items().map{|k,x| _format(k,x) } #{k: _format(k, x) for k, x in v.items()}
|
146
|
+
elsif isinstance(v, str)
|
147
|
+
v2 = v.replace("\xa0", " ")
|
148
|
+
else
|
149
|
+
v2 = v
|
150
|
+
end
|
151
|
+
return v2
|
152
|
+
|
153
|
+
query1_info.items().each do |k,v|
|
154
|
+
query1_info[k] = _format(k, v)
|
155
|
+
@info = query1_info
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def _fetch_complementary() #(proxy) # (self, proxy)
|
160
|
+
return if @already_fetched_complementary
|
161
|
+
|
162
|
+
# self._scrape(proxy) # decrypt broken
|
163
|
+
self._fetch_info() #(proxy)
|
164
|
+
return if @info.nil?
|
165
|
+
|
166
|
+
# Complementary key-statistics. For now just want 'trailing PEG ratio'
|
167
|
+
keys = ["trailingPegRatio"] #{"trailingPegRatio"}
|
168
|
+
if keys
|
169
|
+
# Simplified the original scrape code for key-statistics. Very expensive for fetching
|
170
|
+
# just one value, best if scraping most/all:
|
171
|
+
#
|
172
|
+
# p = _re.compile(r'root\.App\.main = (.*);')
|
173
|
+
# url = 'https://finance.yahoo.com/quote/{}/key-statistics?p={}'.format(self._ticker.ticker, self._ticker.ticker)
|
174
|
+
# try:
|
175
|
+
# r = session.get(url, headers=utils.user_agent_headers)
|
176
|
+
# data = _json.loads(p.findall(r.text)[0])
|
177
|
+
# key_stats = data['context']['dispatcher']['stores']['QuoteTimeSeriesStore']["timeSeries"]
|
178
|
+
# for k in keys:
|
179
|
+
# if k not in key_stats or len(key_stats[k])==0:
|
180
|
+
# # Yahoo website prints N/A, indicates Yahoo lacks necessary data to calculate
|
181
|
+
# v = nil
|
182
|
+
# else:
|
183
|
+
# # Select most recent (last) raw value in list:
|
184
|
+
# v = key_stats[k][-1]["reportedValue"]["raw"]
|
185
|
+
# self._info[k] = v
|
186
|
+
# except Exception:
|
187
|
+
# raise
|
188
|
+
# pass
|
189
|
+
#
|
190
|
+
# For just one/few variable is faster to query directly:
|
191
|
+
url = "https://query1.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/#{symbol}?symbol=#{symbol}"
|
192
|
+
keys.each { |k| url += "&type=" + k }
|
193
|
+
|
194
|
+
# Request 6 months of data
|
195
|
+
start = (DateTime.now.utc.midnight - 6.months).to_i #datetime.timedelta(days=365 // 2)
|
196
|
+
# start = int(start.timestamp())
|
197
|
+
|
198
|
+
ending = DateTime.now.utc.tomorrow.midnight.to_i
|
199
|
+
# ending = int(ending.timestamp())
|
200
|
+
url += "&period1=#{start}&period2=#{ending}"
|
201
|
+
|
202
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} url = #{url}" }
|
203
|
+
json_str = get(url).parsed_response # , proxy=proxy).parsed_response #@data.cache_get(url=url, proxy=proxy).text
|
204
|
+
json_data = json_str #json.loads(json_str)
|
205
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} json_data = #{json_data.inspect}" }
|
206
|
+
json_result = json_data.try(:[],"timeseries") or json_data.try(:[], "finance")
|
207
|
+
unless json_result["error"].nil?
|
208
|
+
raise YfinException("Failed to parse json response from Yahoo Finance: #{json_result["error"]}")
|
209
|
+
|
210
|
+
keys.each do |k|
|
211
|
+
keydict = json_result["result"][0]
|
212
|
+
|
213
|
+
@info[k] = k.in?(keydict) ? keydict[k][-1]["reportedValue"]["raw"] : nil
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
@already_fetched_complementary = true
|
219
|
+
end
|
220
|
+
|
221
|
+
def _fetch_calendar
|
222
|
+
begin
|
223
|
+
# secFilings return too old data, so not requesting it for now
|
224
|
+
result = self._fetch(['calendarEvents']) #(@proxy, modules=['calendarEvents'])
|
225
|
+
if result.nil?
|
226
|
+
@calendar = {}
|
227
|
+
return
|
228
|
+
end
|
229
|
+
|
230
|
+
rescue KeyError, IndexError => e
|
231
|
+
raise YfinDataException("Failed to parse json response from Yahoo Finance: #{e.result}")
|
232
|
+
rescue => e
|
233
|
+
@calendar = {} #dict()
|
234
|
+
_events = result["quoteSummary"]["result"][0]["calendarEvents"]
|
235
|
+
if 'dividendDate'.in?(_events)
|
236
|
+
@calendar['Dividend Date'] = datetime.datetime.fromtimestamp(_events['dividendDate']).date()
|
237
|
+
if 'exDividendDate'.in?(_events)
|
238
|
+
@calendar['Ex-Dividend Date'] = datetime.datetime.fromtimestamp(_events['exDividendDate']).date()
|
239
|
+
# splits = _events.get('splitDate') # need to check later, i will add code for this if found data
|
240
|
+
earnings = _events.get('earnings')
|
241
|
+
if !earnings.nil?
|
242
|
+
@calendar['Earnings Date'] = earnings.get('earningsDate', []).map{|d| d.to_date } # [datetime.datetime.fromtimestamp(d).date() for d in earnings.get('earningsDate', [])]
|
243
|
+
@calendar['Earnings High'] = earnings.get('earningsHigh', nil)
|
244
|
+
@calendar['Earnings Low'] = earnings.get('earningsLow', nil)
|
245
|
+
@calendar['Earnings Average'] = earnings.get('earningsAverage', nil)
|
246
|
+
@calendar['Revenue High'] = earnings.get('revenueHigh', nil)
|
247
|
+
@calendar['Revenue Low'] = earnings.get('revenueLow', nil)
|
248
|
+
@calendar['Revenue Average'] = earnings.get('revenueAverage', nil)
|
249
|
+
# Likely need to decipher and restore the following
|
250
|
+
# except
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
254
|
+
end
|
255
|
+
end
|
256
|
+
|
257
|
+
def _fetch_info()
|
258
|
+
return if @already_fetched
|
259
|
+
|
260
|
+
modules = ['financialData', 'quoteType', 'defaultKeyStatistics', 'assetProfile', 'summaryDetail']
|
261
|
+
|
262
|
+
result = _fetch(modules)
|
263
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} result = #{result.inspect}" }
|
264
|
+
if result.parsed_response.nil?
|
265
|
+
@info = {}
|
266
|
+
return
|
267
|
+
end
|
268
|
+
|
269
|
+
result["quoteSummary"]["result"][0]["symbol"] = symbol
|
270
|
+
|
271
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} result[quoteSummary][result] = #{result["quoteSummary"]["result"].inspect}" }
|
272
|
+
# Rails.logger.info { "#{__FILE__}:#{__LINE__} result[quoteSummary][result].first.keys = #{result["quoteSummary"]["result"].first.keys.inspect}" }
|
273
|
+
query1_info = result["quoteSummary"]["result"].first
|
274
|
+
|
275
|
+
# Likely need to decipher and restore the following
|
276
|
+
# query1_info = next(
|
277
|
+
# (info for info in result.get("quoteSummary", {}).get("result", []) if info["symbol"] == @symbol),
|
278
|
+
# nil,
|
279
|
+
# )
|
280
|
+
|
281
|
+
# Most keys that appear in multiple dicts have same value. Except 'maxAge' because
|
282
|
+
# Yahoo not consistent with days vs seconds. Fix it here:
|
283
|
+
|
284
|
+
query1_info.keys.each {|k|
|
285
|
+
query1_info[k]["maxAge"] = 86400 if "maxAge".in?(query1_info[k]) && query1_info[k]["maxAge"] == 1
|
286
|
+
}
|
287
|
+
|
288
|
+
# Likely need to decipher and restore the following
|
289
|
+
# query1_info = {
|
290
|
+
# k1: v1
|
291
|
+
# for k, v in query1_info.items()
|
292
|
+
# if isinstance(v, dict)
|
293
|
+
# for k1, v1 in v.items()
|
294
|
+
# if v1
|
295
|
+
# }
|
296
|
+
# # recursively format but only because of 'companyOfficers'
|
297
|
+
|
298
|
+
@info = query1_info
|
299
|
+
@already_fetched = true
|
300
|
+
end
|
301
|
+
|
302
|
+
|
303
|
+
BASE_URL = 'https://query2.finance.yahoo.com'
|
304
|
+
QUOTE_SUMMARY_URL = "#{BASE_URL}/v10/finance/quoteSummary"
|
305
|
+
|
306
|
+
QUOTE_SUMMARY_VALID_MODULES = [
|
307
|
+
"summaryProfile", # contains general information about the company
|
308
|
+
"summaryDetail", # prices + volume + market cap + etc
|
309
|
+
"assetProfile", # summaryProfile + company officers
|
310
|
+
"fundProfile",
|
311
|
+
"price", # current prices
|
312
|
+
"quoteType", # quoteType
|
313
|
+
"esgScores", # Environmental, social, and governance (ESG) scores, sustainability and ethical performance of companies
|
314
|
+
"incomeStatementHistory",
|
315
|
+
"incomeStatementHistoryQuarterly",
|
316
|
+
"balanceSheetHistory",
|
317
|
+
"balanceSheetHistoryQuarterly",
|
318
|
+
"cashFlowStatementHistory",
|
319
|
+
"cashFlowStatementHistoryQuarterly",
|
320
|
+
"defaultKeyStatistics", # KPIs (PE, enterprise value, EPS, EBITA, and more)
|
321
|
+
"financialData", # Financial KPIs (revenue, gross margins, operating cash flow, free cash flow, and more)
|
322
|
+
"calendarEvents", # future earnings date
|
323
|
+
"secFilings", # SEC filings, such as 10K and 10Q reports
|
324
|
+
"upgradeDowngradeHistory", # upgrades and downgrades that analysts have given a company's stock
|
325
|
+
"institutionOwnership", # institutional ownership, holders and shares outstanding
|
326
|
+
"fundOwnership", # mutual fund ownership, holders and shares outstanding
|
327
|
+
"majorDirectHolders",
|
328
|
+
"majorHoldersBreakdown",
|
329
|
+
"insiderTransactions", # insider transactions, such as the number of shares bought and sold by company executives
|
330
|
+
"insiderHolders", # insider holders, such as the number of shares held by company executives
|
331
|
+
"netSharePurchaseActivity", # net share purchase activity, such as the number of shares bought and sold by company executives
|
332
|
+
"earnings", # earnings history
|
333
|
+
"earningsHistory",
|
334
|
+
"earningsTrend", # earnings trend
|
335
|
+
"industryTrend",
|
336
|
+
"indexTrend",
|
337
|
+
"sectorTrend",
|
338
|
+
"recommendationTrend",
|
339
|
+
"futuresChain",
|
340
|
+
].freeze
|
341
|
+
|
342
|
+
end
|
343
|
+
end
|