yf_as_dataframe 0.2.15

data/lib/yf_as_dataframe/utils.rb

@@ -0,0 +1,354 @@
+require 'polars-df'
+
+class YfAsDataframe
+  class Utils
+    BASE_URL = 'https://query1.finance.yahoo.com'
+
+    class << self
+      attr_accessor :logger
+    end
+
+    def self.get_all_by_isin(isin, proxy: nil, session: nil)
+      raise ArgumentError, 'Invalid ISIN number' unless is_isin(isin)
+
+      session ||= Net::HTTP
+      url = "#{BASE_URL}/v1/finance/search?q=#{isin}"
+      data = session.get(URI(url), 'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36', 'Accept' => 'application/json')
+      data = JSON.parse(data.body)
+      ticker = data['quotes'][0] || {}
+      {
+        'ticker' => {
+          'symbol' => ticker['symbol'],
+          'shortname' => ticker['shortname'],
+          'longname' => ticker['longname'],
+          'type' => ticker['quoteType'],
+          'exchange' => ticker['exchDisp']
+        },
+        'news' => data['news'] || []
+      }
+    rescue StandardError
+      {}
+    end
+
+    def self.get_ticker_by_isin(isin, proxy: nil, session: nil)
+      data = get_all_by_isin(isin, proxy: proxy, session: session)
+      data.dig('ticker', 'symbol') || ''
+    end
+
+    def self.get_info_by_isin(isin, proxy: nil, session: nil)
+      data = get_all_by_isin(isin, proxy: proxy, session: session)
+      data['ticker'] || {}
+    end
+
+    def self.get_news_by_isin(isin, proxy: nil, session: nil)
+      data = get_all_by_isin(isin, proxy: proxy, session: session)
+      data['news'] || {}
+    end
+
+    def self.empty_df(index = nil)
+      # index ||= []
+      empty = Polars::DataFrame.new({
+        'Timestamps' => DateTime.new(2000,1,1,0,0,0),
+        'Open' => Float::NAN, 'High' => Float::NAN, 'Low' => Float::NAN,
+        'Close' => Float::NAN, 'Adj Close' => Float::NAN, 'Volume' => Float::NAN
+      })
+      # empty = index.each_with_object({}) { |i, h| h[i] = empty }
+      # empty['Date'] = 'Date'
+      empty
+    end
+
+    def self.empty_earnings_dates_df
+      {
+        'Symbol' => 'Symbol', 'Company' => 'Company', 'Earnings Date' => 'Earnings Date',
+        'EPS Estimate' => 'EPS Estimate', 'Reported EPS' => 'Reported EPS', 'Surprise(%)' => 'Surprise(%)'
+      }
+    end
+
+    def self.build_template(data)
+      template_ttm_order = []
+      template_annual_order = []
+      template_order = []
+      level_detail = []
+
+      def traverse(node, level)
+        return if level > 5
+
+        template_ttm_order << "trailing#{node['key']}"
+        template_annual_order << "annual#{node['key']}"
+        template_order << node['key']
+        level_detail << level
+        return unless node['children']
+
+        node['children'].each { |child| traverse(child, level + 1) }
+      end
+
+      data['template'].each { |key| traverse(key, 0) }
+
+      [template_ttm_order, template_annual_order, template_order, level_detail]
+    end
+
+    def self.retrieve_financial_details(data)
+      ttm_dicts = []
+      annual_dicts = []
+
+      data['timeSeries'].each do |key, timeseries|
+        next unless timeseries
+
+        time_series_dict = { 'index' => key }
+        timeseries.each do |each|
+          next unless each
+
+          time_series_dict[each['asOfDate']] = each['reportedValue']
+        end
+        if key.include?('trailing')
+          ttm_dicts << time_series_dict
+        elsif key.include?('annual')
+          annual_dicts << time_series_dict
+        end
+      end
+
+      [ttm_dicts, annual_dicts]
+    end
+
+    def self.format_annual_financial_statement(level_detail, annual_dicts, annual_order, ttm_dicts = nil, ttm_order = nil)
+      annual = annual_dicts.each_with_object({}) { |d, h| h[d['index']] = d }
+      annual = annual_order.each_with_object({}) { |k, h| h[k] = annual[k] }
+      annual = annual.transform_keys { |k| k.gsub('annual', '') }
+
+      if ttm_dicts && ttm_order
+        ttm = ttm_dicts.each_with_object({}) { |d, h| h[d['index']] = d }
+        ttm = ttm_order.each_with_object({}) { |k, h| h[k] = ttm[k] }
+        ttm = ttm.transform_keys { |k| k.gsub('trailing', '') }
+        statement = annual.merge(ttm)
+      else
+        statement = annual
+      end
+
+      statement = statement.transform_keys { |k| camel2title(k) }
+      statement.transform_values { |v| v.transform_keys { |k| camel2title(k) } }
+    end
+
+    def self.format_quarterly_financial_statement(statement, level_detail, order)
+      statement = order.each_with_object({}) { |k, h| h[k] = statement[k] }
+      statement = statement.transform_keys { |k| camel2title(k) }
+      statement.transform_values { |v| v.transform_keys { |k| camel2title(k) } }
+    end
+
+    def self.camel2title(strings, sep: ' ', acronyms: nil)
+      raise TypeError, "camel2title() 'strings' argument must be iterable of strings" unless strings.is_a?(Enumerable)
+      raise TypeError, "camel2title() 'strings' argument must be iterable of strings" unless strings.all? { |s| s.is_a?(String) }
+      raise ValueError, "camel2title() 'sep' argument = '#{sep}' must be single character" unless sep.is_a?(String) && sep.length == 1
+      raise ValueError, "camel2title() 'sep' argument = '#{sep}' cannot be alpha-numeric" if sep.match?(/[a-zA-Z0-9]/)
+      raise ValueError, "camel2title() 'sep' argument = '#{sep}' cannot be special character" if sep != Regexp.escape(sep) && !%w[ -].include?(sep)
+
+      if acronyms.nil?
+        pat = /([a-z])([A-Z])/
+        rep = '\1' + sep + '\2'
+        strings.map { |s| s.gsub(pat, rep).capitalize }
+      else
+        raise TypeError, "camel2title() 'acronyms' argument must be iterable of strings" unless acronyms.is_a?(Enumerable)
+        raise TypeError, "camel2title() 'acronyms' argument must be iterable of strings" unless acronyms.all? { |a| a.is_a?(String) }
+        acronyms.each do |a|
+          raise ValueError, "camel2title() 'acronyms' argument must only contain upper-case, but '#{a}' detected" unless a.match?(/^[A-Z]+$/)
+        end
+
+        pat = /([a-z])([A-Z])/
+        rep = '\1' + sep + '\2'
+        strings = strings.map { |s| s.gsub(pat, rep) }
+
+        acronyms.each do |a|
+          pat = /(#{a})([A-Z][a-z])/
+          rep = '\1' + sep + '\2'
+          strings = strings.map { |s| s.gsub(pat, rep) }
+        end
+
+        strings.map do |s|
+          s.split(sep).map do |w|
+            if acronyms.include?(w)
+              w
+            else
+              w.capitalize
+            end
+          end.join(sep)
+        end
+      end
+    end
+
+    def self.snake_case_2_camelCase(s)
+      s.split('_').first + s.split('_')[1..].map(&:capitalize).join
+    end
+
+    # def self.parse_quotes(data)
+    #   timestamps = data['timestamp']
+    #   ohlc = data['indicators']['quote'][0]
+    #   volumes = ohlc['volume']
+    #   opens = ohlc['open']
+    #   closes = ohlc['close']
+    #   lows = ohlc['low']
+    #   highs = ohlc['high']
+
+    #   adjclose = closes
+    #   adjclose = data['indicators']['adjclose'][0]['adjclose'] if data['indicators']['adjclose']
+
+    #   quotes = {
+    #     'Open' => opens,
+    #     'High' => highs,
+    #     'Low' => lows,
+    #     'Close' => closes,
+    #     'Adj Close' => adjclose,
+    #     'Volume' => volumes
+    #   }
+
+    #   quotes.each { |k, v| quotes[k] = v.map { |x| x.nil? ? Float::NAN : x } }
+    #   quotes['Date'] = timestamps.map { |x| Time.at(x).to_datetime }
+
+    #   quotes
+    # end
+
+    # def self.auto_adjust(data)
+    #   ratio = data['Adj Close'] / data['Close']
+    #   data['Adj Open'] = data['Open'] * ratio
+    #   data['Adj High'] = data['High'] * ratio
+    #   data['Adj Low'] = data['Low'] * ratio
+
+    #   data.delete('Open')
+    #   data.delete('High')
+    #   data.delete('Low')
+    #   data.delete('Close')
+
+    #   data['Open'] = data.delete('Adj Open')
+    #   data['High'] = data.delete('Adj High')
+    #   data['Low'] = data.delete('Adj Low')
+
+    #   data
+    # end
+
+    # def self.back_adjust(data)
+    #   ratio = data['Adj Close'] / data['Close']
+    #   data['Adj Open'] = data['Open'] * ratio
+    #   data['Adj High'] = data['High'] * ratio
+    #   data['Adj Low'] = data['Low'] * ratio
+
+    #   data.delete('Open')
+    #   data.delete('High')
+    #   data.delete('Low')
+    #   data.delete('Adj Close')
+
+    #   data['Open'] = data.delete('Adj Open')
+    #   data['High'] = data.delete('Adj High')
+    #   data['Low'] = data.delete('Adj Low')
+
+    #   data
+    # end
+
+    def self.is_isin(string)
+      /^[A-Z]{2}[A-Z0-9]{9}[0-9]$/.match?(string)
+    end
+
+    def self.parse_user_dt(dt, exchange_tz)
+      if dt.is_a?(Integer)
+        Time.at(dt)
+      elsif dt.is_a?(String)
+        dt = DateTime.strptime(dt.to_s, '%Y-%m-%d') 
+      elsif dt.is_a?(Date)
+        dt = dt.to_datetime 
+      elsif dt.is_a?(DateTime) && dt.zone.nil?
+        dt = dt.in_time_zone(exchange_tz)
+      end
+      dt.to_i
+    end
+
+    def self.interval_to_timedelta(interval)
+      case interval
+      when '1mo'
+        1.month
+      when '2mo'
+        2.months
+      when '3mo'
+        3.months
+      when '6mo'
+        6.months
+      when '9mo'
+        9.months
+      when '12mo'
+        1.year
+      when '1y'
+        1.year
+      when '2y'
+        2.year
+      when '3y'
+        3.year
+      when '4y'
+        4.year
+      when '5y'
+        5.year
+      when '1wk'
+        1.week
+      when '2wk'
+        2.week
+      when '3wk'
+        3.week
+      when '4wk'
+        4.week
+      else
+        Rails.logger.warn { "#{__FILE__}:#{__LINE__} #{interval} not a recognized interval" }
+        interval
+      end
+    end
+
+    # def _interval_to_timedelta(interval)
+    #   if interval == "1mo"
+    #     return ActiveSupport::Duration.new(months: 1)
+    #   elsif interval == "3mo"
+    #     return ActiveSupport::Duration.new(months: 3)
+    #   elsif interval == "1y"
+    #     return ActiveSupport::Duration.new(years: 1)
+    #   elsif interval == "1wk"
+    #     return 7.days
+    #   else
+    #     return ActiveSupport::Duration.parse(interval)
+    #   end
+    # end    
+  end
+end
+
+# module Yfin
+#   class << self
+#     attr_accessor :logger
+#   end
+
+#   self.logger = Logger.new(STDOUT)
+#   self.logger.level = Logger::WARN
+# end
+
+def attributes(obj)
+  disallowed_names = Set.new(obj.class.instance_methods(false).map(&:to_s))
+  obj.instance_variables.each_with_object({}) do |var, h|
+    name = var.to_s[1..]
+    next if name.start_with?('_') || disallowed_names.include?(name)
+
+    h[name] = obj.instance_variable_get(var)
+  end
+end
+
+def print_once(msg)
+  puts msg
+end
+
+def get_yf_logger
+  # Yfin.logger
+  Rails.logger
+end
+
+def setup_debug_formatting
+  logger = get_yf_logger
+
+  return unless logger.level == Logger::DEBUG
+
+  logger.formatter = MultiLineFormatter.new('%(levelname)-8s %(message)s')
+end
+
+def enable_debug_mode
+  Rails.logger.level = Logger::DEBUG
+  setup_debug_formatting
+end

data/lib/yf_as_dataframe/version.rb

@@ -0,0 +1,3 @@
+class YfAsDataframe
+  VERSION = "0.2.15"
+end

data/lib/yf_as_dataframe/yf_connection.rb

@@ -0,0 +1,304 @@
+# require 'requests'
+# require 'requests_cache'
+require 'thread'
+require 'date'
+require 'nokogiri'
+require 'zache'
+require 'httparty'
+
+class YfAsDataframe
+  module YfConnection
+    extend ActiveSupport::Concern
+    # extend HTTParty
+
+    # """
+    # Have one place to retrieve data from Yahoo API in order to ease caching and speed up operations.
+    # """
+    @@user_agent_headers = {
+      'User-Agent' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
+      # 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36'
+    }
+    @@proxy = nil
+
+    cattr_accessor :user_agent_headers, :proxy
+
+    def yfconn_initialize
+      # Rails.logger.info { "#{__FILE__}:#{__LINE__} here"}
+      begin
+        @@zache = ::Zache.new
+        @@session_is_caching = true
+      rescue NoMethodError
+        # Not caching
+        @@session_is_caching = false
+      end
+
+      @@crumb = nil
+      @@cookie = nil
+      @@cookie_strategy = 'basic'
+      @@cookie_lock = ::Mutex.new()
+    end
+
+
+    def get(url, headers=nil, params=nil)
+      # Important: treat input arguments as immutable.
+      # Rails.logger.info { "#{__FILE__}:#{__LINE__} url = #{url}, headers = #{headers}, params=#{params.inspect}" }
+
+      headers ||= {}
+      params ||= {}
+      params.merge!(crumb: @@crumb) unless @@crumb.nil?
+      cookie, crumb, strategy = _get_cookie_and_crumb()
+      crumbs = !crumb.nil? ? {'crumb' => crumb} : {}
+
+      request_args = {
+        url: url,
+        params: params.merge(crumbs),
+        headers: headers || {}
+      }
+
+      proxy = _get_proxy
+      ::HTTParty.http_proxy(addr = proxy.split(':').first, port = proxy.split(':').second.split('/').first) unless proxy.nil?
+
+      cookie_hash = ::HTTParty::CookieHash.new
+      cookie_hash.add_cookies(@@cookie)
+      options = { headers: headers.dup.merge(@@user_agent_headers).merge({ 'cookie' => cookie_hash.to_cookie_string, 'crumb' => crumb })} #,  debug_output: STDOUT }
+
+      u = (request_args[:url]).dup.to_s
+      joiner = ('?'.in?(request_args[:url]) ? '&' : '?')
+      u += (joiner + CGI.unescape(request_args[:params].to_query)) unless request_args[:params].empty?
+
+      # Rails.logger.info { "#{__FILE__}:#{__LINE__} u=#{u}, options = #{options.inspect}" }
+      response = ::HTTParty.get(u, options)
+      # Rails.logger.info { "#{__FILE__}:#{__LINE__} response=#{response.inspect}" }
+
+      return response
+    end
+
+    alias_method :cache_get, :get
+
+
+    def get_raw_json(url, user_agent_headers=nil, params=nil)
+      # Rails.logger.info { "#{__FILE__}:#{__LINE__} url = #{url.inspect}" }
+      response = get(url, user_agent_headers, params)
+      # Rails.logger.info { "#{__FILE__}:#{__LINE__} response = #{response.inspect}" }
+      # response.raise_for_status()
+      return response   #.json()
+    end
+
+
+
+
+
+
+
+    private
+
+
+    def _get_proxy
+      # setup proxy in requests format
+      proxy = nil
+      unless proxy.nil?
+        proxy = {"https" => @@proxy["https"]} if @@proxy.is_a?(Hash) && @@proxy.include?("https")
+      end
+
+      return proxy
+    end
+
+    def _set_cookie_strategy(strategy, have_lock=false)
+      return if strategy == @@cookie_strategy
+
+      if !have_lock
+        @@cookie_lock.synchronize do
+          @@cookie_strategy = strategy
+          @@cookie = nil
+          @@crumb = nil
+        end
+      end
+    end
+
+    def _get_cookie_and_crumb()
+      cookie, crumb, strategy = nil, nil, nil
+      # puts "cookie_mode = '#{@@cookie_strategy}'"
+
+      @@cookie_lock.synchronize do
+        if @@cookie_strategy == 'csrf'
+          crumb = _get_crumb_csrf()
+          if crumb.nil?
+            # Fail
+            _set_cookie_strategy('basic', have_lock=true)
+            cookie, crumb = __get_cookie_and_crumb_basic()
+            # Rails.logger.info { "#{__FILE__}:#{__LINE__} cookie = #{cookie}, crumb = #{crumb}" }
+          end
+        else
+          # Fallback strategy
+          cookie, crumb = __get_cookie_and_crumb_basic()
+          # Rails.logger.info { "#{__FILE__}:#{__LINE__} cookie = #{cookie}, crumb = #{crumb}" }
+          if cookie.nil? || crumb.nil?
+            # Fail
+            _set_cookie_strategy('csrf', have_lock=true)
+            crumb = _get_crumb_csrf()
+          end
+        end
+        strategy = @@cookie_strategy
+      end
+
+      # Rails.logger.info { "#{__FILE__}:#{__LINE__} cookie = #{cookie}, crumb = #{crumb}, strategy=#{strategy}" }
+      return cookie, crumb, strategy
+    end
+
+    def __get_cookie_and_crumb_basic()
+      cookie = _get_cookie_basic()
+      crumb = _get_crumb_basic()
+      return cookie, crumb
+    end
+
+    def _get_cookie_basic()
+      @@cookie ||= _load_cookie_basic()
+      return @@cookie unless @@cookie.nil? || @@cookie.length.zero?
+
+      headers = @@user_agent_headers.dup
+
+      response = HTTParty.get('https://fc.yahoo.com', headers) #.merge(debug_output: STDOUT))
+
+      cookie = response.headers['set-cookie']
+      cookies ||= ''
+      cookies += cookie.split(';').first
+      @@cookie = cookies;
+
+      _save_cookie_basic(@@cookie)
+
+      return @@cookie
+    end
+
+    def _get_crumb_basic()
+      return @@crumb unless @@crumb.nil?
+      return nil if (cookie = _get_cookie_basic()).nil?
+
+      cookie_hash = ::HTTParty::CookieHash.new
+      cookie_hash.add_cookies(cookie)
+      options = {headers: @@user_agent_headers.dup.merge(
+                   { 'cookie' => cookie_hash.to_cookie_string }
+      )} #,  debug_output: STDOUT }
+
+      crumb_response = ::HTTParty.get('https://query1.finance.yahoo.com/v1/test/getcrumb', options)
+      @@crumb = crumb_response.parsed_response
+
+      return (@@crumb.nil? || '<html>'.in?(@@crumb)) ? nil : @@crumb
+    end
+
+    def _get_cookie_csrf()
+      return true unless @@cookie.nil?
+      return (@@cookie = true) if _load_session_cookies()
+
+      base_args = {
+        headers: @@user_agent_headers,
+        # proxies: proxy,
+      }
+
+      get_args = base_args.merge({url: 'https://guce.yahoo.com/consent'})
+
+      get_args[:expire_after] = @expire_after if @session_is_caching
+      response = @session.get(**get_args)
+
+      soup = ::Nokogiri::HTML(response.content, 'html.parser')
+      csrfTokenInput = soup.find('input', attrs: {'name': 'csrfToken'})
+
+      # puts 'Failed to find "csrfToken" in response'
+      return false if csrfTokenInput.nil?
+
+      csrfToken = csrfTokenInput['value']
+      # puts "csrfToken = #{csrfToken}"
+      sessionIdInput = soup.find('input', attrs: {'name': 'sessionId'})
+      sessionId = sessionIdInput['value']
+      # puts "sessionId='#{sessionId}"
+
+      originalDoneUrl = 'https://finance.yahoo.com/'
+      namespace = 'yahoo'
+      data = {
+        'agree': ['agree', 'agree'],
+        'consentUUID': 'default',
+        'sessionId': sessionId,
+        'csrfToken': csrfToken,
+        'originalDoneUrl': originalDoneUrl,
+        'namespace': namespace,
+      }
+      post_args = base_args.merge(
+        {
+          url: "https://consent.yahoo.com/v2/collectConsent?sessionId=#{sessionId}",
+          data: data
+        }
+      )
+      get_args = base_args.merge(
+        {
+          url: "https://guce.yahoo.com/copyConsent?sessionId=#{sessionId}",
+          data: data
+        }
+      )
+      if @session_is_caching
+        post_args[:expire_after] = @expire_after
+        get_args[:expire_after] = @expire_after
+      end
+      @session.post(**post_args)
+      @session.get(**get_args)
+
+      @@cookie = true
+      _save_session_cookies()
+
+      return true
+    end
+
+    def _get_crumb_csrf()
+      # Credit goes to @bot-unit #1729
+
+      # puts 'reusing crumb'
+      return @@crumb unless @@crumb.nil?
+      # This cookie stored in session
+      return nil unless _get_cookie_csrf().present?
+
+      get_args = {
+        url: 'https://query2.finance.yahoo.com/v1/test/getcrumb',
+        headers: @@user_agent_headers
+      }
+
+      get_args[:expire_after] = @expire_after if @session_is_caching
+      r = @session.get(**get_args)
+
+      @@crumb = r.text
+
+      # puts "Didn't receive crumb"
+      return nil if @@crumb.nil? || '<html>'.in?(@@crumb) || @@crumb.length.zero?
+      return @@crumb
+    end
+
+
+
+
+
+    def _save_session_cookies()
+      begin
+        @@zache.put(:csrf, @session.cookies, lifetime: 60 * 60 * 24)
+      rescue Exception
+        return false
+      end
+      return true
+    end
+
+    def _load_session_cookies()
+      return false if @@zache.expired?(:csrf)
+      @session.cookies = @@zache.get(:csrf)
+    end
+
+    def _save_cookie_basic(cookie)
+      begin
+        @@zache.put(:basic, cookie, lifetime: 60*60*24)
+      rescue Exception
+        return false
+      end
+      return true
+    end
+
+    def _load_cookie_basic()
+      @@zache.put(:basic, nil, lifetime: 1) unless @@zache.exists?(:basic, dirty: false)
+      return @@zache.expired?(:basic) ? nil : @@zache.get(:basic)
+    end
+  end
+end

data/lib/yf_as_dataframe/yfinance_exception.rb

@@ -0,0 +1,15 @@
+class YfAsDataframe
+  class YfinanceException < StandardError
+    attr_reader :msg
+  end
+
+  class YfinDataException < YfinanceException
+  end
+
+  class YFNotImplementedError < NotImplementedError
+    def initialize(str)
+      @msg = "Have not implemented fetching \"#{str}\" from Yahoo API"
+      Rails.logger.warn { @msg }
+    end
+  end
+end

data/lib/yf_as_dataframe.rb

@@ -0,0 +1,24 @@
+class YfAsDataframe
+end
+
+# frozen_string_literal: true
+
+require_relative 'yf_as_dataframe/version'
+require_relative 'yf_as_dataframe/utils'
+require_relative 'yf_as_dataframe/yfinance_exception'
+require_relative 'yf_as_dataframe/yf_connection'
+require_relative 'yf_as_dataframe/price_technical'
+require_relative 'yf_as_dataframe/price_history'
+require_relative 'yf_as_dataframe/quote'
+require_relative 'yf_as_dataframe/analysis'
+require_relative 'yf_as_dataframe/fundamentals'
+require_relative 'yf_as_dataframe/financials'
+require_relative 'yf_as_dataframe/holders'
+require_relative 'yf_as_dataframe/ticker'
+require_relative "yf_as_dataframe/version"
+
+class YfAsDataframe
+
+  extend YfAsDataframe::PriceTechnical
+end
+