yf_as_dataframe 0.2.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,343 @@
1
+ require 'httparty'
2
+
3
+ class YfAsDataframe
4
+ module Quote
5
+ extend ActiveSupport::Concern
6
+
7
+ def self.included(base) # built-in Ruby hook for modules
8
+ base.class_eval do
9
+ original_method = instance_method(:initialize)
10
+ define_method(:initialize) do |*args, &block|
11
+ original_method.bind(self).call(*args, &block)
12
+ initialize_quote # (your module code here)
13
+ end
14
+ end
15
+ end
16
+
17
+ def initialize_quote
18
+ @info = nil
19
+ @retired_info = nil
20
+ @sustainability = nil
21
+ @recommendations = nil
22
+ @upgrades_downgrades = nil
23
+ @calendar = nil
24
+
25
+ @already_scraped = false
26
+ @already_fetched = false
27
+ @already_fetched_complementary = false
28
+ end
29
+
30
+ def info #(self)
31
+ if @info.nil?
32
+ _fetch_info() #(@proxy)
33
+ _fetch_complementary() #(@proxy)
34
+ end
35
+ return @info
36
+ end
37
+
38
+ def sustainability
39
+ raise YFNotImplementedError.new('sustainability') if @sustainability.nil?
40
+ return @sustainability
41
+ end
42
+
43
+ def recommendations
44
+ Polars::Config.set_tbl_rows(-1)
45
+ if @recommendations.nil?
46
+
47
+ result = _fetch(['recommendationTrend']).parsed_response
48
+ # Rails.logger.info { "#{__FILE__}:#{__LINE__} result = #{result.inspect}" }
49
+ # if result.nil?
50
+ # @recommendations = YfAsDataframe::Utils.empty_df() #Polars::DataFrame()
51
+ # else
52
+ begin
53
+ data = result["quoteSummary"]["result"][0]["recommendationTrend"]["trend"]
54
+ rescue KeyError, IndexError => e
55
+ raise YfinDataException(f"Failed to parse json response from Yahoo Finance: #{e.result}")
56
+ end
57
+ @recommendations = Polars::DataFrame.new(data)
58
+ # end
59
+ end
60
+ return @recommendations
61
+ end
62
+
63
+ alias_method :recommendation_summary, :recommendations
64
+ alias_method :recommendations_summary, :recommendations
65
+
66
+ def upgrades_downgrades
67
+ Polars::Config.set_tbl_rows(-1)
68
+ if @upgrades_downgrades.nil?
69
+ result = _fetch(['upgradeDowngradeHistory']).parsed_response
70
+ # Rails.logger.info { "#{__FILE__}:#{__LINE__} result = #{result.inspect}" }
71
+
72
+ # if result.nil?
73
+ # @upgrades_downgrades = YfAsDataframe::Utils.empty_df() #Polars::DataFrame()
74
+ # else
75
+ begin
76
+ data = result["quoteSummary"]["result"][0]["upgradeDowngradeHistory"]["history"]
77
+
78
+ raise YfinDataException("No upgrade/downgrade history found for #{ticker.symbol}") if (data).length.zero?
79
+
80
+ df = Polars::DataFrame.new(data)
81
+ df.rename({"epochGradeDate" => "GradeDate", 'firm' => 'Firm', 'toGrade' => 'ToGrade', 'fromGrade' => 'FromGrade', 'action' => 'Action'})
82
+ # df.set_index('GradeDate', inplace=true)
83
+ # df.index = pd.to_datetime(df.index, unit='s')
84
+ @upgrades_downgrades = df
85
+ rescue KeyError, IndexError => e
86
+ raise YfinDataException("Failed to parse json response from Yahoo Finance: #{e.result}")
87
+ end
88
+ # end
89
+ end
90
+ return @upgrades_downgrades
91
+ end
92
+
93
+ def calendar
94
+ self._fetch_calendar() if @calendar.nil?
95
+ return @calendar
96
+ end
97
+
98
+ def valid_modules()
99
+ return QUOTE_SUMMARY_VALID_MODULES
100
+ end
101
+
102
+ # quote_methods = [:info, :sustainability, :recommendations, :recommendations_summary, :recommendation_summary, \
103
+ # :upgrades_downgrades, :calendar]
104
+ # quote_methods.each do |meth|
105
+ # # define_method "get_#{meth}".to_sym do
106
+ # # data = @quote.send(meth.to_sym)
107
+ # # return data
108
+ # # end
109
+ # alias_method "get_#{meth}".to_sym, meth
110
+ # end
111
+
112
+
113
+
114
+
115
+
116
+
117
+ private
118
+
119
+ def _fetch(modules) #(self, proxy, modules: list)
120
+ # raise YahooFinanceException("Should provide a list of modules, see available modules using `valid_modules`") if !modules.is_a?(Array)
121
+
122
+ modules = modules.intersection(QUOTE_SUMMARY_VALID_MODULES) #[m for m in modules if m in quote_summary_valid_modules])
123
+
124
+ raise YahooFinanceException("No valid modules provided.") if modules.empty?
125
+
126
+ params_dict = {"modules": modules.join(','), "corsDomain": "finance.yahoo.com", "formatted": "false", "symbol": symbol}
127
+
128
+ begin
129
+ result = get_raw_json(QUOTE_SUMMARY_URL + "/#{symbol}", user_agent_headers=user_agent_headers, params=params_dict)
130
+ # Rails.logger.info { "#{__FILE__}:#{__LINE__} result = #{result.inspect}" }
131
+ rescue Exception => e
132
+ Rails.logger.error("ERROR: #{e.message}")
133
+ return nil
134
+ end
135
+ return result
136
+ end
137
+
138
+ def _format(k, v)
139
+ v2 = nil
140
+ if isinstance(v, dict) && "raw".in?(v) && "fmt".in?(v)
141
+ v2 = k.in?(["regularMarketTime", "postMarketTime"]) ? v["fmt"] : v["raw"]
142
+ elsif isinstance(v, list)
143
+ v2 = v.map{|vv| _format(nil, vv)}
144
+ elsif isinstance(v, dict)
145
+ v2 = v.items().map{|k,x| _format(k,x) } #{k: _format(k, x) for k, x in v.items()}
146
+ elsif isinstance(v, str)
147
+ v2 = v.replace("\xa0", " ")
148
+ else
149
+ v2 = v
150
+ end
151
+ return v2
152
+
153
+ query1_info.items().each do |k,v|
154
+ query1_info[k] = _format(k, v)
155
+ @info = query1_info
156
+ end
157
+ end
158
+
159
+ def _fetch_complementary() #(proxy) # (self, proxy)
160
+ return if @already_fetched_complementary
161
+
162
+ # self._scrape(proxy) # decrypt broken
163
+ self._fetch_info() #(proxy)
164
+ return if @info.nil?
165
+
166
+ # Complementary key-statistics. For now just want 'trailing PEG ratio'
167
+ keys = ["trailingPegRatio"] #{"trailingPegRatio"}
168
+ if keys
169
+ # Simplified the original scrape code for key-statistics. Very expensive for fetching
170
+ # just one value, best if scraping most/all:
171
+ #
172
+ # p = _re.compile(r'root\.App\.main = (.*);')
173
+ # url = 'https://finance.yahoo.com/quote/{}/key-statistics?p={}'.format(self._ticker.ticker, self._ticker.ticker)
174
+ # try:
175
+ # r = session.get(url, headers=utils.user_agent_headers)
176
+ # data = _json.loads(p.findall(r.text)[0])
177
+ # key_stats = data['context']['dispatcher']['stores']['QuoteTimeSeriesStore']["timeSeries"]
178
+ # for k in keys:
179
+ # if k not in key_stats or len(key_stats[k])==0:
180
+ # # Yahoo website prints N/A, indicates Yahoo lacks necessary data to calculate
181
+ # v = nil
182
+ # else:
183
+ # # Select most recent (last) raw value in list:
184
+ # v = key_stats[k][-1]["reportedValue"]["raw"]
185
+ # self._info[k] = v
186
+ # except Exception:
187
+ # raise
188
+ # pass
189
+ #
190
+ # For just one/few variable is faster to query directly:
191
+ url = "https://query1.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/#{symbol}?symbol=#{symbol}"
192
+ keys.each { |k| url += "&type=" + k }
193
+
194
+ # Request 6 months of data
195
+ start = (DateTime.now.utc.midnight - 6.months).to_i #datetime.timedelta(days=365 // 2)
196
+ # start = int(start.timestamp())
197
+
198
+ ending = DateTime.now.utc.tomorrow.midnight.to_i
199
+ # ending = int(ending.timestamp())
200
+ url += "&period1=#{start}&period2=#{ending}"
201
+
202
+ # Rails.logger.info { "#{__FILE__}:#{__LINE__} url = #{url}" }
203
+ json_str = get(url).parsed_response # , proxy=proxy).parsed_response #@data.cache_get(url=url, proxy=proxy).text
204
+ json_data = json_str #json.loads(json_str)
205
+ # Rails.logger.info { "#{__FILE__}:#{__LINE__} json_data = #{json_data.inspect}" }
206
+ json_result = json_data.try(:[],"timeseries") or json_data.try(:[], "finance")
207
+ unless json_result["error"].nil?
208
+ raise YfinException("Failed to parse json response from Yahoo Finance: #{json_result["error"]}")
209
+
210
+ keys.each do |k|
211
+ keydict = json_result["result"][0]
212
+
213
+ @info[k] = k.in?(keydict) ? keydict[k][-1]["reportedValue"]["raw"] : nil
214
+ end
215
+ end
216
+ end
217
+
218
+ @already_fetched_complementary = true
219
+ end
220
+
221
+ def _fetch_calendar
222
+ begin
223
+ # secFilings return too old data, so not requesting it for now
224
+ result = self._fetch(['calendarEvents']) #(@proxy, modules=['calendarEvents'])
225
+ if result.nil?
226
+ @calendar = {}
227
+ return
228
+ end
229
+
230
+ rescue KeyError, IndexError => e
231
+ raise YfinDataException("Failed to parse json response from Yahoo Finance: #{e.result}")
232
+ rescue => e
233
+ @calendar = {} #dict()
234
+ _events = result["quoteSummary"]["result"][0]["calendarEvents"]
235
+ if 'dividendDate'.in?(_events)
236
+ @calendar['Dividend Date'] = datetime.datetime.fromtimestamp(_events['dividendDate']).date()
237
+ if 'exDividendDate'.in?(_events)
238
+ @calendar['Ex-Dividend Date'] = datetime.datetime.fromtimestamp(_events['exDividendDate']).date()
239
+ # splits = _events.get('splitDate') # need to check later, i will add code for this if found data
240
+ earnings = _events.get('earnings')
241
+ if !earnings.nil?
242
+ @calendar['Earnings Date'] = earnings.get('earningsDate', []).map{|d| d.to_date } # [datetime.datetime.fromtimestamp(d).date() for d in earnings.get('earningsDate', [])]
243
+ @calendar['Earnings High'] = earnings.get('earningsHigh', nil)
244
+ @calendar['Earnings Low'] = earnings.get('earningsLow', nil)
245
+ @calendar['Earnings Average'] = earnings.get('earningsAverage', nil)
246
+ @calendar['Revenue High'] = earnings.get('revenueHigh', nil)
247
+ @calendar['Revenue Low'] = earnings.get('revenueLow', nil)
248
+ @calendar['Revenue Average'] = earnings.get('revenueAverage', nil)
249
+ # Likely need to decipher and restore the following
250
+ # except
251
+ end
252
+ end
253
+ end
254
+ end
255
+ end
256
+
257
+ def _fetch_info()
258
+ return if @already_fetched
259
+
260
+ modules = ['financialData', 'quoteType', 'defaultKeyStatistics', 'assetProfile', 'summaryDetail']
261
+
262
+ result = _fetch(modules)
263
+ # Rails.logger.info { "#{__FILE__}:#{__LINE__} result = #{result.inspect}" }
264
+ if result.parsed_response.nil?
265
+ @info = {}
266
+ return
267
+ end
268
+
269
+ result["quoteSummary"]["result"][0]["symbol"] = symbol
270
+
271
+ # Rails.logger.info { "#{__FILE__}:#{__LINE__} result[quoteSummary][result] = #{result["quoteSummary"]["result"].inspect}" }
272
+ # Rails.logger.info { "#{__FILE__}:#{__LINE__} result[quoteSummary][result].first.keys = #{result["quoteSummary"]["result"].first.keys.inspect}" }
273
+ query1_info = result["quoteSummary"]["result"].first
274
+
275
+ # Likely need to decipher and restore the following
276
+ # query1_info = next(
277
+ # (info for info in result.get("quoteSummary", {}).get("result", []) if info["symbol"] == @symbol),
278
+ # nil,
279
+ # )
280
+
281
+ # Most keys that appear in multiple dicts have same value. Except 'maxAge' because
282
+ # Yahoo not consistent with days vs seconds. Fix it here:
283
+
284
+ query1_info.keys.each {|k|
285
+ query1_info[k]["maxAge"] = 86400 if "maxAge".in?(query1_info[k]) && query1_info[k]["maxAge"] == 1
286
+ }
287
+
288
+ # Likely need to decipher and restore the following
289
+ # query1_info = {
290
+ # k1: v1
291
+ # for k, v in query1_info.items()
292
+ # if isinstance(v, dict)
293
+ # for k1, v1 in v.items()
294
+ # if v1
295
+ # }
296
+ # # recursively format but only because of 'companyOfficers'
297
+
298
+ @info = query1_info
299
+ @already_fetched = true
300
+ end
301
+
302
+
303
+ BASE_URL = 'https://query2.finance.yahoo.com'
304
+ QUOTE_SUMMARY_URL = "#{BASE_URL}/v10/finance/quoteSummary"
305
+
306
+ QUOTE_SUMMARY_VALID_MODULES = [
307
+ "summaryProfile", # contains general information about the company
308
+ "summaryDetail", # prices + volume + market cap + etc
309
+ "assetProfile", # summaryProfile + company officers
310
+ "fundProfile",
311
+ "price", # current prices
312
+ "quoteType", # quoteType
313
+ "esgScores", # Environmental, social, and governance (ESG) scores, sustainability and ethical performance of companies
314
+ "incomeStatementHistory",
315
+ "incomeStatementHistoryQuarterly",
316
+ "balanceSheetHistory",
317
+ "balanceSheetHistoryQuarterly",
318
+ "cashFlowStatementHistory",
319
+ "cashFlowStatementHistoryQuarterly",
320
+ "defaultKeyStatistics", # KPIs (PE, enterprise value, EPS, EBITA, and more)
321
+ "financialData", # Financial KPIs (revenue, gross margins, operating cash flow, free cash flow, and more)
322
+ "calendarEvents", # future earnings date
323
+ "secFilings", # SEC filings, such as 10K and 10Q reports
324
+ "upgradeDowngradeHistory", # upgrades and downgrades that analysts have given a company's stock
325
+ "institutionOwnership", # institutional ownership, holders and shares outstanding
326
+ "fundOwnership", # mutual fund ownership, holders and shares outstanding
327
+ "majorDirectHolders",
328
+ "majorHoldersBreakdown",
329
+ "insiderTransactions", # insider transactions, such as the number of shares bought and sold by company executives
330
+ "insiderHolders", # insider holders, such as the number of shares held by company executives
331
+ "netSharePurchaseActivity", # net share purchase activity, such as the number of shares bought and sold by company executives
332
+ "earnings", # earnings history
333
+ "earningsHistory",
334
+ "earningsTrend", # earnings trend
335
+ "industryTrend",
336
+ "indexTrend",
337
+ "sectorTrend",
338
+ "recommendationTrend",
339
+ "futuresChain",
340
+ ].freeze
341
+
342
+ end
343
+ end