yf_as_dataframe 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 13496d1eaf3e5ce09c9477de83957acb3d2183214eec2b0dedabdf8b612bd80b
-  data.tar.gz: 604c01e5bba073e9350146c6636c6ce29f1ac62a7d2a9a4e6e9338a33f1a65e0
+  metadata.gz: ef9b7d363312088694d19ce68cd538579f9b3f2b93f57cf5e111f3247cd87b00
+  data.tar.gz: 5bab2fa2aa65c4b6496025aa13f86c15d5df49deeeff2f6b60db07818d6fe1c8
 SHA512:
-  metadata.gz: e758fafbc0396c7582a9ad0654c94d7721afd2457a403408376be07fa7ace874887a83499c4507a122f1d027a0126da31c9447cb5b61a3d9fc3ab190e7e36f96
-  data.tar.gz: 74a1263cd148add7c56417b4912052471ae6c49fc4163daacc5d5ce3988e53135802f4ec7912aeef372e735fec470072921f15f536d628e5449315fb31ea3142
+  metadata.gz: '0019c0c2f4293f3c1b3684bff81f2e0a78eb614c505031aaa77c4ba541c88def5189ed8d96310dc6b7217f3651f37f7ea9dbb64c05009a2286d12856027368b2'
+  data.tar.gz: 39b3d46681bbc24409315878a8722e563266553e9b81592051404ef53df6aef397c0248ab1fc724ea6edfc5f6d647e2fb89de6bf675b975aeeb592ae7244f66c

data/README.md

@@ -12,7 +12,7 @@
 Yahoo, Inc.**
 
 yf_as_dataframe is **not** affiliated, endorsed, or vetted by Yahoo, Inc. It is
-an open-source tool that uses Yahoo's publicly available APIs, and is
+an open-source tool that uses Yahoo's publicly available APIs, and is **only**
 intended for research and educational purposes.
 
 **You should refer to Yahoo!'s terms of use**
@@ -216,7 +216,9 @@ ls -la /usr/local/bin/curl_*
 
 ### Custom Installation Directory
 
-If you have curl-impersonate installed in a different directory, you can set the `CURL_IMPERSONATE_DIR` environment variable:
+The codebase will look for the location of the curl-impersonate binaries per the `CURL_IMPERSONATE_DIR` environment variable;
+if it is not assigned, the default location of the binaries is  `/usr/local/bin`.
+The code will randomly select one of the binaries (expected to be named "curl_chrome*", "curl_ff*", "curl_edge*", etc.) for its communications with the servers.
 
 ```bash
 # Set custom directory
@@ -226,8 +228,6 @@ export CURL_IMPERSONATE_DIR="/opt/curl-impersonate/bin"
 CURL_IMPERSONATE_DIR="/opt/curl-impersonate/bin" ruby your_script.rb
 ```
 
-The default directory is `/usr/local/bin` if the environment variable is not set.
-
 ### Configuration (Optional)
 
 You can configure the curl-impersonate behavior if needed:
@@ -250,14 +250,6 @@ puts "Available: #{executables.length} executables"
 puts "Using directory: #{YfAsDataframe::CurlImpersonateIntegration.executable_directory}"
 ```
 
-### How It Works
-
-1. **Automatic Detection**: Dynamically finds curl-impersonate executables in the configured directory
-2. **Default Behavior**: Uses curl-impersonate for all requests by default
-3. **Seamless Fallback**: Falls back to HTTParty if curl-impersonate fails
-4. **Browser Rotation**: Randomly selects from Chrome, Firefox, Edge, and Safari configurations
-5. **Zero Interface Changes**: All existing method signatures remain the same
-
 For more detailed information, see [MINIMAL_INTEGRATION.md](MINIMAL_INTEGRATION.md).
 
 ---

data/lib/yf_as_dataframe/curl_impersonate_integration.rb

@@ -1,20 +1,24 @@
 require 'open3'
 require 'json'
 require 'ostruct'
+require 'timeout'
 
 class YfAsDataframe
   module CurlImpersonateIntegration
     # Configuration
     @curl_impersonate_enabled = true
     @curl_impersonate_fallback = true
-    @curl_impersonate_timeout = 5
+    @curl_impersonate_timeout = 30  # Increased from 5 to 30 seconds
+    @curl_impersonate_connect_timeout = 10  # New: connection timeout
     @curl_impersonate_retries = 2
     @curl_impersonate_retry_delay = 1
+    @curl_impersonate_process_timeout = 60  # New: process timeout protection
 
     class << self
       attr_accessor :curl_impersonate_enabled, :curl_impersonate_fallback, 
-                   :curl_impersonate_timeout, :curl_impersonate_retries, 
-                   :curl_impersonate_retry_delay
+                   :curl_impersonate_timeout, :curl_impersonate_connect_timeout,
+                   :curl_impersonate_retries, :curl_impersonate_retry_delay,
+                   :curl_impersonate_process_timeout
     end
 
     # Get the curl-impersonate executable directory from environment variable or default
@@ -50,51 +54,76 @@ class YfAsDataframe
       available.sample
     end
 
-    # Make a curl-impersonate request
-    def self.make_request(url, headers: {}, params: {}, timeout: nil)
+    # Make a curl-impersonate request with improved timeout handling
+    def self.make_request(url, headers: {}, params: {}, timeout: nil, retries: nil)
       executable_info = get_random_executable
       return nil unless executable_info
 
       timeout ||= @curl_impersonate_timeout
+      retries ||= @curl_impersonate_retries
 
-      # Build command
-      cmd = [executable_info[:path], "--max-time", timeout.to_s]
-      
-      # Add headers
-      headers.each do |key, value|
-        cmd.concat(["-H", "#{key}: #{value}"])
-      end
-
-      # Add query parameters
+      cmd = [
+        executable_info[:path],
+        "--max-time", timeout.to_s,
+        "--connect-timeout", @curl_impersonate_connect_timeout.to_s,
+        "--retry", retries.to_s,
+        "--retry-delay", @curl_impersonate_retry_delay.to_s,
+        "--retry-max-time", (timeout * 2).to_s,
+        "--fail",
+        "--silent",
+        "--show-error"
+      ]
+      headers.each { |key, value| cmd.concat(["-H", "#{key}: #{value}"]) }
       unless params.empty?
         query_string = params.map { |k, v| "#{k}=#{v}" }.join('&')
         separator = url.include?('?') ? '&' : '?'
         url = "#{url}#{separator}#{query_string}"
       end
-
-      # Add URL
       cmd << url
 
-      # Debug output
       puts "DEBUG: curl-impersonate command: #{cmd.join(' ')}"
       puts "DEBUG: curl-impersonate timeout: #{timeout} seconds"
 
-      # Execute
-      stdout, stderr, status = Open3.capture3(*cmd)
-      
-      puts "DEBUG: curl-impersonate stdout: #{stdout[0..200]}..." if stdout && !stdout.empty?
-      puts "DEBUG: curl-impersonate stderr: #{stderr}" if stderr && !stderr.empty?
+      begin
+        stdout_str = ''
+        stderr_str = ''
+        status = nil
+        Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
+          stdin.close
+          pid = wait_thr.pid
+          done = false
+          monitor = Thread.new do
+            sleep(timeout + 10)
+            unless done
+              puts "DEBUG: Killing curl-impersonate PID \\#{pid} after timeout"
+              Process.kill('TERM', pid) rescue nil
+              sleep(1)
+              Process.kill('KILL', pid) rescue nil if wait_thr.alive?
+            end
+          end
+          stdout_str = stdout.read
+          stderr_str = stderr.read
+          status = wait_thr.value
+          done = true
+          monitor.kill
+        end
+        puts "DEBUG: curl-impersonate stdout: #{stdout_str[0..200]}..." if stdout_str && !stdout_str.empty?
+        puts "DEBUG: curl-impersonate stderr: #{stderr_str}" if stderr_str && !stderr_str.empty?
       puts "DEBUG: curl-impersonate status: #{status.exitstatus}"
-      
       if status.success?
-        # Create a response object similar to HTTParty
         response = OpenStruct.new
-        response.body = stdout
+          response.body = stdout_str
         response.code = 200
         response.define_singleton_method(:success?) { true }
-        response.parsed_response = parse_json_if_possible(stdout)
+          response.parsed_response = parse_json_if_possible(stdout_str)
         response
       else
+          error_message = "curl failed with code \\#{status.exitstatus}: \\#{stderr_str}"
+          puts "DEBUG: curl-impersonate failed with error: \\#{error_message}"
+          nil
+        end
+      rescue => e
+        puts "DEBUG: curl-impersonate exception: \\#{e.message}"
         nil
       end
     end

data/lib/yf_as_dataframe/holders.rb

@@ -1,4 +1,5 @@
 require 'logger'
+require 'open-uri'
 
 class YfAsDataframe
   module Holders

data/lib/yf_as_dataframe/multi.rb

@@ -119,7 +119,7 @@ class YfAsDataframe
       @shared::_PROGRESS_BAR.completed if progress
 
       unless @shared::_ERRORS.empty?
-        logger.error("\n#{@shared::_ERRORS.length} Failed download#{@shared::_ERRORS.length > 1 ? 's' : ''}:")
+        # logger.error("\n#{@shared::_ERRORS.length} Failed download#{@shared::_ERRORS.length > 1 ? 's' : ''}:")
 
         errors = {}
         @shared::_ERRORS.each do |ticker, err|
@@ -127,9 +127,9 @@ class YfAsDataframe
           errors[err] ||= []
           errors[err] << ticker
         end
-        errors.each do |err, tickers|
-          logger.error("#{tickers.join(', ')}: #{err}")
-        end
+        # errors.each do |err, tickers|
+        #   logger.error("#{tickers.join(', ')}: #{err}")
+        # end
 
         tbs = {}
         @shared::_TRACEBACKS.each do |ticker, tb|
@@ -137,9 +137,9 @@ class YfAsDataframe
           tbs[tb] ||= []
           tbs[tb] << ticker
         end
-        tbs.each do |tb, tickers|
-          logger.debug("#{tickers.join(', ')}: #{tb}")
-        end
+        # tbs.each do |tb, tickers|
+        #   logger.debug("#{tickers.join(', ')}: #{tb}")
+        # end
       end
 
       if ignore_tz

data/lib/yf_as_dataframe/price_history.rb

@@ -719,7 +719,7 @@ class YfAsDataframe
         # startDt = quotes.index[0].floor('D')
         startDt = quotes['Timestamps'].to_a.map(&:to_date).min
         # Rails.logger.info { "#{__FILE__}:#{__LINE__} startDt = #{startDt.inspect}" }
-        endDt = !fin.nil? && !(fin.respond_to?(:empty?) && fin.empty?) ? fin.to_date : Time.at((Time.now + 1.day).to_i).to_i
+        endDt = !fin.nil? && !(fin.respond_to?(:empty?) && fin.empty?) ? fin.to_date : (Time.now + 86400).to_date
 
         # Rails.logger.info { "#{__FILE__}:#{__LINE__} @history[events][dividends] = #{@history['events']["dividends"].inspect}" }
         # divi = {}
@@ -731,32 +731,32 @@ class YfAsDataframe
         # Rails.logger.info { "#{__FILE__}:#{__LINE__} ts = #{ts.inspect}" }
         @history['events']["dividends"].select{|k,v| 
           Time.at(k.to_i).utc.to_date >= startDt && Time.at(k.to_i).utc.to_date <= endDt }.each{|k,v| 
-            d[ts.index(Time.at(k.to_i).utc)] = v['amount'].to_f} unless @history.try(:[],'events').try(:[],"dividends").nil?
+            d[ts.index(Time.at(k.to_i).utc)] = v['amount'].to_f} unless @history.dig('events', 'dividends').nil?
         df['Dividends'] = Polars::Series.new(d)
         # Rails.logger.info { "#{__FILE__}:#{__LINE__} df = #{df.inspect}" }
 
         # caga = {}
         # @history['events']["capital gains"].select{|k,v| 
         #   Time.at(k.to_i).utc.to_date >= startDt  && Time.at(k.to_i).utc.to_date <= endDt }.each{|k,v| 
-        #     caga['date'] = v['amount']} unless @history.try(:[],'events').try(:[],"capital gains").nil?
+        #     caga['date'] = v['amount']} unless @history.dig('events', 'capital gains').nil?
         # capital_gains = capital_gains.loc[startDt:] if capital_gains.shape.first > 0
         # Rails.logger.info { "#{__FILE__}:#{__LINE__} caga = #{caga.inspect}" }
         d = [0.0] * df.length
         @history['events']["capital gains"].select{|k,v| 
           Time.at(k.to_i).utc.to_date >= startDt  && Time.at(k.to_i).utc.to_date <= endDt }.each{|k,v| 
-            d[ts.index(Time.at(k.to_i).utc)] = v['amount'].to_f} unless @history.try(:[],'events').try(:[],"capital gains").nil?
+            d[ts.index(Time.at(k.to_i).utc)] = v['amount'].to_f} unless @history.dig('events', 'capital gains').nil?
         df['Capital Gains'] = Polars::Series.new(d)
 
         # splits = splits.loc[startDt:] if splits.shape[0] > 0
         # stspl = {}
         # @history['events']['stock splits'].select{|k,v| 
         #   Time.at(k.to_i).utc.to_date >= startDt  && Time.at(k.to_i).utc.to_date <= endDt }.each{|k,v| 
-        #     stspl['date'] = v['numerator'].to_f/v['denominator'].to_f} unless @history.try(:[],'events').try(:[],"stock splits").nil?
+        #     stspl['date'] = v['numerator'].to_f/v['denominator'].to_f} unless @history.dig('events', 'capital gains').nil?
         # Rails.logger.info { "#{__FILE__}:#{__LINE__} stspl = #{stspl.inspect}" }
         d = [0.0] * df.length
         @history['events']["capital gains"].select{|k,v| 
           Time.at(k.to_i).utc.to_date >= startDt  && Time.at(k.to_i).utc.to_date <= endDt }.each{|k,v| 
-            d[ts.index(Time.at(k.to_i).utc)] = v['numerator'].to_f/v['denominator'].to_f} unless @history.try(:[],'events').try(:[],"capital gains").nil?
+            d[ts.index(Time.at(k.to_i).utc)] = v['numerator'].to_f/v['denominator'].to_f} unless @history.dig('events', 'capital gains').nil?
         df['Stock Splits'] = Polars::Series.new(d)
       end
 

data/lib/yf_as_dataframe/ticker.rb

@@ -54,21 +54,21 @@ class YfAsDataframe
       if start
         start_ts = YfAsDataframe::Utils.parse_user_dt(start, tz)
         # Rails.logger.info { "#{__FILE__}:#{__LINE__} start_ts = #{start_ts}" }
-        start = Time.at(start_ts).in_time_zone(tz)
+        start = Time.at(start_ts)
         # Rails.logger.info { "#{__FILE__}:#{__LINE__} start = #{start.inspect}, fin = #{fin.inspect}" } 
       end
       if fin
         end_ts = YfAsDataframe::Utils.parse_user_dt(fin, tz)
         # Rails.logger.info { "#{__FILE__}:#{__LINE__} end_ts = #{end_ts}" }
-        fin = Time.at(end_ts).in_time_zone(tz)
+        fin = Time.at(end_ts)
         # Rails.logger.info { "#{__FILE__}:#{__LINE__} start = #{start.inspect}, fin = #{fin.inspect}" } 
       end
 
       # Rails.logger.info { "#{__FILE__}:#{__LINE__} start = #{start.inspect}, fin = #{fin.inspect}" } 
 
-      dt_now = Time.now.in_time_zone(tz)
+      dt_now = Time.now
       fin ||= dt_now
-      start ||= (fin - 548.days).midnight
+      start ||= Time.new(fin.year, fin.month, fin.day) - 548*24*60*60
 
       if start >= fin
         logger.error("Start date (#{start}) must be before end (#{fin})")
@@ -76,7 +76,7 @@ class YfAsDataframe
       end
 
       ts_url_base = "https://query2.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/#{@ticker}?symbol=#{@ticker}"
-      shares_url = "#{ts_url_base}&period1=#{start.to_i}&period2=#{fin.tomorrow.midnight.to_i}"
+      shares_url = "#{ts_url_base}&period1=#{Time.new(start.year, start.month, start.day).to_i}&period2=#{Time.new((fin + 86400).year, (fin + 86400).month, (fin + 86400).day).to_i}"
 
       begin
         json_data = get(shares_url).parsed_response
@@ -95,7 +95,7 @@ class YfAsDataframe
 
       return nil if !shares_data[0].key?("shares_out")
 
-      timestamps = shares_data[0]["timestamp"].map{|t| Time.at(t).to_datetime }
+      timestamps = shares_data[0]["timestamp"].map{|t| Time.at(t) }
 
       df = Polars::DataFrame.new(
         {

data/lib/yf_as_dataframe/utils.rb

@@ -247,13 +247,17 @@ class YfAsDataframe
 
     def self.parse_user_dt(dt, exchange_tz)
       if dt.is_a?(Integer)
-        Time.at(dt)
+        return Time.at(dt)
       elsif dt.is_a?(String)
-        dt = DateTime.strptime(dt.to_s, '%Y-%m-%d') 
+        dt = DateTime.strptime(dt.to_s, '%Y-%m-%d')
       elsif dt.is_a?(Date)
-        dt = dt.to_datetime 
-      elsif dt.is_a?(DateTime) && dt.zone.nil?
-        dt = dt.in_time_zone(exchange_tz)
+        dt = dt.to_datetime
+      end
+      # If it's a DateTime, convert to Time
+      if dt.is_a?(DateTime)
+        # If zone is nil, try to set it, else just convert
+        dt = dt.in_time_zone(exchange_tz) if dt.zone.nil? && dt.respond_to?(:in_time_zone)
+        dt = dt.to_time
       end
       dt.to_i
     end
@@ -334,7 +338,7 @@ class YfAsDataframe
       when '4wk'
         28.days
       else
-        Logger.new(STDOUT).warn { "#{__FILE__}:#{__LINE__} #{interval} not a recognized interval" }
+        # Logger.new(STDOUT).warn { "#{__FILE__}:#{__LINE__} #{interval} not a recognized interval" }
         interval
       end
     end

data/lib/yf_as_dataframe/version.rb

@@ -1,3 +1,3 @@
 class YfAsDataframe
-  VERSION = "0.4.0"
+  VERSION = "0.4.1"
 end

data/lib/yf_as_dataframe/yf_connection.rb

@@ -265,7 +265,7 @@ class YfAsDataframe
             @@cookie = nil
             # Clear curl-impersonate executables cache to force re-selection
             CurlImpersonateIntegration.instance_variable_set(:@available_executables, nil)
-            warn "[yf_as_dataframe] Retrying crumb fetch (attempt #{attempt + 1}/3)"
+            # warn "[yf_as_dataframe] Retrying crumb fetch (attempt #{attempt + 1}/3)"
             # Add delay between retries to be respectful of rate limits
             sleep(2 ** attempt) # Exponential backoff: 2s, 4s, 8s
           end
@@ -283,20 +283,20 @@ class YfAsDataframe
 
           # Validate crumb: must be short, alphanumeric, no spaces, not an error message
           if crumb_valid?(@@crumb)
-            warn "[yf_as_dataframe] Successfully fetched valid crumb on attempt #{attempt + 1}"
+            # warn "[yf_as_dataframe] Successfully fetched valid crumb on attempt #{attempt + 1}"
             return @@crumb
           else
-            warn "[yf_as_dataframe] Invalid crumb received on attempt #{attempt + 1}: '#{@@crumb.inspect}'"
+            # warn "[yf_as_dataframe] Invalid crumb received on attempt #{attempt + 1}: '#{@@crumb.inspect}'"
             @@crumb = nil
           end
         rescue => e
-          warn "[yf_as_dataframe] Error fetching crumb on attempt #{attempt + 1}: #{e.message}"
+          # warn "[yf_as_dataframe] Error fetching crumb on attempt #{attempt + 1}: #{e.message}"
           @@crumb = nil
         end
       end
       
       # All attempts failed
-      warn "[yf_as_dataframe] Failed to fetch valid crumb after 3 attempts"
+      # warn "[yf_as_dataframe] Failed to fetch valid crumb after 3 attempts"
       raise "Could not fetch a valid Yahoo Finance crumb after 3 attempts"
     end
 
@@ -441,7 +441,7 @@ class YfAsDataframe
     def refresh_session_if_needed
       return unless session_needs_refresh?
       
-      warn "[yf_as_dataframe] Refreshing session (age: #{session_age} seconds, requests: #{@@request_count})"
+      # warn "[yf_as_dataframe] Refreshing session (age: #{session_age} seconds, requests: #{@@request_count})"
       refresh_session
     end
 
@@ -461,7 +461,7 @@ class YfAsDataframe
       @@crumb = nil
       @@session_created_at = Time.now
       @@request_count = 0
-      warn "[yf_as_dataframe] Session refreshed"
+      # warn "[yf_as_dataframe] Session refreshed"
     end
 
     # Circuit breaker methods
@@ -472,7 +472,7 @@ class YfAsDataframe
       when :open
         if Time.now - @@last_failure_time > @@circuit_breaker_timeout
           @@circuit_breaker_state = :half_open
-          warn "[yf_as_dataframe] Circuit breaker transitioning to half-open"
+          # warn "[yf_as_dataframe] Circuit breaker transitioning to half-open"
           true
         else
           false
@@ -490,7 +490,7 @@ class YfAsDataframe
         @@circuit_breaker_state = :open
         # Exponential backoff: 60s, 120s, 240s, 480s, etc.
         @@circuit_breaker_timeout = @@circuit_breaker_base_timeout * (2 ** (@@failure_count - @@circuit_breaker_threshold))
-        warn "[yf_as_dataframe] Circuit breaker opened after #{@@failure_count} failures (timeout: #{@@circuit_breaker_timeout}s)"
+        # warn "[yf_as_dataframe] Circuit breaker opened after #{@@failure_count} failures (timeout: #{@@circuit_breaker_timeout}s)"
       end
     end
 
@@ -499,7 +499,7 @@ class YfAsDataframe
         @@circuit_breaker_state = :closed
         @@failure_count = 0
         @@circuit_breaker_timeout = @@circuit_breaker_base_timeout
-        warn "[yf_as_dataframe] Circuit breaker closed after successful request"
+        # warn "[yf_as_dataframe] Circuit breaker closed after successful request"
       elsif @@circuit_breaker_state == :closed
         # Reset failure count on success
         @@failure_count = 0
@@ -538,10 +538,10 @@ class YfAsDataframe
       begin
         data = JSON.parse(json_blob)
         crumb = data.dig('context', 'dispatcher', 'stores', 'CrumbStore', 'crumb')
-        warn "[yf_as_dataframe] Scraped crumb from quote page: #{crumb.inspect}"
+        # warn "[yf_as_dataframe] Scraped crumb from quote page: #{crumb.inspect}"
         return crumb
       rescue => e
-        warn "[yf_as_dataframe] Failed to parse crumb from quote page: #{e.message}"
+        # warn "[yf_as_dataframe] Failed to parse crumb from quote page: #{e.message}"
         return nil
       end
     end

data/lib/yf_as_dataframe/yf_connection_minimal_patch.rb

@@ -22,9 +22,16 @@ class YfAsDataframe
           # Prepare headers and params as in original method
           headers ||= {}
           params ||= {}
-          params.merge!(crumb: @@crumb) unless @@crumb.nil?
-          cookie, crumb, strategy = _get_cookie_and_crumb()
-          crumbs = !crumb.nil? ? {'crumb' => crumb} : {}
+          
+          # Only fetch crumb for /v7/finance/download endpoint
+          crumb_needed = url.include?('/v7/finance/download')
+          if crumb_needed
+            crumb = get_crumb_scrape_quote_page(params[:symbol] || params['symbol'])
+            params.merge!(crumb: crumb) unless crumb.nil?
+          end
+          
+          cookie, _, strategy = _get_cookie_and_crumb(crumb_needed)
+          crumbs = {} # crumb logic handled above if needed
 
           # Prepare headers for curl-impersonate
           curl_headers = headers.dup.merge(@@user_agent_headers)
@@ -39,12 +46,13 @@ class YfAsDataframe
           # Add crumb if available
           curl_headers['crumb'] = crumb if crumb
 
-          # Make curl-impersonate request
+          # Make curl-impersonate request with improved timeout handling
           response = CurlImpersonateIntegration.make_request(
             url, 
             headers: curl_headers, 
             params: params.merge(crumbs),
-            timeout: CurlImpersonateIntegration.curl_impersonate_timeout
+            timeout: CurlImpersonateIntegration.curl_impersonate_timeout,
+            retries: CurlImpersonateIntegration.curl_impersonate_retries
           )
 
           if response && response.success?
@@ -56,7 +64,7 @@ class YfAsDataframe
         rescue => e
           # Log error but continue to fallback
           puts "DEBUG: curl-impersonate exception: #{e.message}"
-          warn "curl-impersonate request failed: #{e.message}" if $VERBOSE
+          # warn "curl-impersonate request failed: #{e.message}" if $VERBOSE
         end
       else
         puts "DEBUG: curl-impersonate is disabled, skipping to fallback"
@@ -89,9 +97,33 @@ class YfAsDataframe
         CurlImpersonateIntegration.curl_impersonate_timeout = timeout
       end
 
+      def set_curl_impersonate_connect_timeout(timeout)
+        CurlImpersonateIntegration.curl_impersonate_connect_timeout = timeout
+      end
+
+      def set_curl_impersonate_process_timeout(timeout)
+        CurlImpersonateIntegration.curl_impersonate_process_timeout = timeout
+      end
+
+      def set_curl_impersonate_retries(retries)
+        CurlImpersonateIntegration.curl_impersonate_retries = retries
+      end
+
       def get_available_curl_impersonate_executables
         CurlImpersonateIntegration.available_executables
       end
+
+      def get_curl_impersonate_config
+        {
+          enabled: CurlImpersonateIntegration.curl_impersonate_enabled,
+          fallback: CurlImpersonateIntegration.curl_impersonate_fallback,
+          timeout: CurlImpersonateIntegration.curl_impersonate_timeout,
+          connect_timeout: CurlImpersonateIntegration.curl_impersonate_connect_timeout,
+          process_timeout: CurlImpersonateIntegration.curl_impersonate_process_timeout,
+          retries: CurlImpersonateIntegration.curl_impersonate_retries,
+          retry_delay: CurlImpersonateIntegration.curl_impersonate_retry_delay
+        }
+      end
     end
   end
 end 

data/lib/yf_as_dataframe/yfinance_exception.rb

@@ -11,7 +11,7 @@ class YfAsDataframe
   class YFNotImplementedError < NotImplementedError
     def initialize(str)
       @msg = "Have not implemented fetching \"#{str}\" from Yahoo API"
-      Logger.new(STDOUT).warn { @msg }
+      # Logger.new(STDOUT).warn { @msg }
     end
   end
 end

data/smoke_test.rb

@@ -0,0 +1,64 @@
+# require "bundler/setup"
+require "yf_as_dataframe"
+
+def print_section(title)
+  puts "\n=== #{title} ==="
+end
+
+begin
+  print_section("Ticker Creation")
+  msft = YfAsDataframe::Ticker.new("MSFT")
+  puts "Ticker created: #{msft.ticker}"
+
+  print_section("Price History")
+  hist = msft.history(period: "1mo")
+  puts "History DataFrame shape: #{hist.shape}" if hist
+
+  print_section("Meta Information")
+  meta = msft.history_metadata
+  puts "Meta: #{meta.inspect}"
+
+  print_section("Actions")
+  puts "Dividends: #{msft.dividends.inspect}"
+  puts "Splits: #{msft.splits.inspect}"
+
+  print_section("Share Count")
+  shares = msft.shares_full(start: "2022-01-01", fin: nil)
+  puts "Shares DataFrame shape: #{shares.shape}" if shares
+
+  print_section("Financials")
+  puts "Income Statement: #{msft.income_stmt.inspect}"
+  puts "Balance Sheet: #{msft.balance_sheet.inspect}"
+  puts "Cash Flow: #{msft.cashflow.inspect}"
+
+  print_section("Holders")
+  puts "Major Holders: #{msft.major_holders.inspect}"
+  puts "Institutional Holders: #{msft.institutional_holders.inspect}"
+
+  print_section("Recommendations")
+  puts "Recommendations: #{msft.recommendations.inspect}"
+
+  print_section("Earnings Dates")
+  puts "Earnings Dates: #{msft.earnings_dates.inspect}"
+
+  print_section("ISIN")
+  puts "ISIN: #{msft.isin.inspect}"
+
+  print_section("Options")
+  puts "Options: #{msft.options.inspect}"
+
+  print_section("News")
+  puts "News: #{msft.news.inspect}"
+
+  print_section("Technical Indicator Example")
+  if hist
+    ad = YfAsDataframe.ad(hist)
+    puts "AD indicator: #{ad.inspect}"
+  end
+
+  puts "\nAll tests completed successfully!"
+
+rescue => e
+  puts "\nTest failed: #{e.class} - #{e.message}"
+  puts e.backtrace.first(10)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yf_as_dataframe
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Bill McKinnon
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-06-25 00:00:00.000000000 Z
+date: 2025-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tzinfo
@@ -156,8 +156,7 @@ files:
 - lib/yf_as_dataframe/yf_connection.rb
 - lib/yf_as_dataframe/yf_connection_minimal_patch.rb
 - lib/yf_as_dataframe/yfinance_exception.rb
-- quick_test.rb
-- test_minimal_integration.rb
+- smoke_test.rb
 homepage: https://www.github.com/bmck/yf_as_dataframe
 licenses:
 - MIT

data/quick_test.rb

@@ -1,143 +0,0 @@
-#!/usr/bin/env ruby
-
-# Quick test for minimal curl-impersonate integration
-# This test verifies the integration without making actual HTTP requests
-
-puts "=== Quick Curl-Impersonate Integration Test ==="
-puts
-
-# Test 1: Check curl-impersonate integration module
-puts "1. Testing curl-impersonate integration module..."
-begin
-  require_relative 'lib/yf_as_dataframe/curl_impersonate_integration'
-  
-  executables = YfAsDataframe::CurlImpersonateIntegration.available_executables
-  if executables.empty?
-    puts "   ❌ No curl-impersonate executables found!"
-    exit 1
-  else
-    puts "   ✅ Found #{executables.length} curl-impersonate executables"
-    puts "   Sample: #{executables.first[:executable]} (#{executables.first[:browser]})"
-  end
-rescue => e
-  puts "   ❌ Error loading integration module: #{e.message}"
-  exit 1
-end
-
-puts
-
-# Test 2: Test executable selection
-puts "2. Testing executable selection..."
-begin
-  executable = YfAsDataframe::CurlImpersonateIntegration.get_random_executable
-  if executable
-    puts "   ✅ Random executable selected: #{executable[:executable]} (#{executable[:browser]})"
-  else
-    puts "   ❌ No executable selected"
-  end
-rescue => e
-  puts "   ❌ Error selecting executable: #{e.message}"
-end
-
-puts
-
-# Test 3: Test environment variable functionality
-puts "3. Testing environment variable functionality..."
-begin
-  default_dir = YfAsDataframe::CurlImpersonateIntegration.executable_directory
-  puts "   ✅ Default directory: #{default_dir}"
-  
-  # Test with a custom directory (should still use default if not set)
-  old_env = ENV['CURL_IMPERSONATE_DIR']
-  ENV['CURL_IMPERSONATE_DIR'] = '/nonexistent/path'
-  
-  # Clear the cached executables to force re-discovery
-  YfAsDataframe::CurlImpersonateIntegration.instance_variable_set(:@available_executables, nil)
-  
-  custom_dir = YfAsDataframe::CurlImpersonateIntegration.executable_directory
-  puts "   ✅ Custom directory (set): #{custom_dir}"
-  
-  # Restore original environment
-  if old_env
-    ENV['CURL_IMPERSONATE_DIR'] = old_env
-  else
-    ENV.delete('CURL_IMPERSONATE_DIR')
-  end
-  
-  # Clear cache again
-  YfAsDataframe::CurlImpersonateIntegration.instance_variable_set(:@available_executables, nil)
-  
-  restored_dir = YfAsDataframe::CurlImpersonateIntegration.executable_directory
-  puts "   ✅ Restored directory: #{restored_dir}"
-  
-rescue => e
-  puts "   ❌ Error testing environment variable: #{e.message}"
-end
-
-puts
-
-# Test 4: Test minimal patch loading
-puts "4. Testing minimal patch structure..."
-begin
-  # This would normally require the full YfConnection class
-  # For this test, we'll just verify the patch file loads
-  require_relative 'lib/yf_as_dataframe/curl_impersonate_integration'
-  require_relative 'lib/yf_as_dataframe/yf_connection_minimal_patch'
-  
-  puts "   ✅ Minimal patch files load successfully"
-  puts "   ✅ Integration module is available"
-rescue => e
-  puts "   ❌ Error loading minimal patch: #{e.message}"
-end
-
-puts
-
-# Test 5: Test configuration
-puts "5. Testing configuration..."
-begin
-  puts "   ✅ Configuration methods available:"
-  puts "     - enable_curl_impersonate"
-  puts "     - enable_curl_impersonate_fallback" 
-  puts "     - set_curl_impersonate_timeout"
-  puts "     - get_available_curl_impersonate_executables"
-  
-  # Test setting configuration
-  YfAsDataframe::CurlImpersonateIntegration.curl_impersonate_timeout = 20
-  puts "   ✅ Configuration can be modified"
-rescue => e
-  puts "   ❌ Error with configuration: #{e.message}"
-end
-
-puts
-
-# Test 6: Test command building (without execution)
-puts "6. Testing command building..."
-begin
-  executable = YfAsDataframe::CurlImpersonateIntegration.get_random_executable
-  if executable
-    # Build a command without executing it
-    cmd = [executable[:path], "--max-time", "5", "https://httpbin.org/get"]
-    puts "   ✅ Command built: #{cmd.join(' ')}"
-  else
-    puts "   ❌ Could not build command"
-  end
-rescue => e
-  puts "   ❌ Error building command: #{e.message}"
-end
-
-puts
-puts "=== Quick Test Summary ==="
-puts "✅ Integration module loads successfully"
-puts "✅ Executables are detected"
-puts "✅ Environment variable functionality works"
-puts "✅ Configuration works"
-puts "✅ Patch files load without errors"
-puts
-puts "The minimal curl-impersonate integration is ready for use!"
-puts
-puts "To integrate with your code:"
-puts "require 'yf_as_dataframe/curl_impersonate_integration'"
-puts "require 'yf_as_dataframe/yf_connection_minimal_patch'"
-puts
-puts "Environment variable support:"
-puts "export CURL_IMPERSONATE_DIR='/custom/path'  # Optional" 

data/test_minimal_integration.rb

@@ -1,121 +0,0 @@
-#!/usr/bin/env ruby
-
-# Test script for minimal curl-impersonate integration
-# This tests the approach where curl-impersonate is the default behavior
-
-puts "=== Minimal Curl-Impersonate Integration Test ==="
-puts
-
-# Test 1: Check curl-impersonate integration module
-puts "1. Testing curl-impersonate integration module..."
-begin
-  require_relative 'lib/yf_as_dataframe/curl_impersonate_integration'
-  
-  executables = YfAsDataframe::CurlImpersonateIntegration.available_executables
-  if executables.empty?
-    puts "   ❌ No curl-impersonate executables found!"
-    exit 1
-  else
-    puts "   ✅ Found #{executables.length} curl-impersonate executables"
-    puts "   Sample: #{executables.first[:executable]} (#{executables.first[:browser]})"
-  end
-rescue => e
-  puts "   ❌ Error loading integration module: #{e.message}"
-  exit 1
-end
-
-puts
-
-# Test 2: Test direct curl-impersonate request with short timeout
-puts "2. Testing direct curl-impersonate request..."
-begin
-  response = YfAsDataframe::CurlImpersonateIntegration.make_request(
-    "https://httpbin.org/get",
-    headers: { "User-Agent" => "Test-Agent" },
-    timeout: 10  # 10 second timeout
-  )
-  
-  if response && response.success?
-    puts "   ✅ Direct curl-impersonate request successful"
-    puts "   Response length: #{response.body.length} characters"
-  else
-    puts "   ❌ Direct curl-impersonate request failed"
-  end
-rescue => e
-  puts "   ❌ Error with direct request: #{e.message}"
-end
-
-puts
-
-# Test 3: Test minimal patch (without full gem)
-puts "3. Testing minimal patch structure..."
-begin
-  # This would normally require the full YfConnection class
-  # For this test, we'll just verify the patch file loads
-  require_relative 'lib/yf_as_dataframe/curl_impersonate_integration'
-  require_relative 'lib/yf_as_dataframe/yf_connection_minimal_patch'
-  
-  puts "   ✅ Minimal patch files load successfully"
-  puts "   ✅ Integration module is available"
-rescue => e
-  puts "   ❌ Error loading minimal patch: #{e.message}"
-end
-
-puts
-
-# Test 4: Test configuration methods
-puts "4. Testing configuration methods..."
-begin
-  # Test configuration (these would work with the full YfConnection class)
-  puts "   ✅ Configuration methods available:"
-  puts "     - enable_curl_impersonate"
-  puts "     - enable_curl_impersonate_fallback" 
-  puts "     - set_curl_impersonate_timeout"
-  puts "     - get_available_curl_impersonate_executables"
-rescue => e
-  puts "   ❌ Error with configuration: #{e.message}"
-end
-
-puts
-
-# Test 5: Test with Yahoo Finance endpoint with short timeout
-puts "5. Testing Yahoo Finance endpoint..."
-begin
-  response = YfAsDataframe::CurlImpersonateIntegration.make_request(
-    "https://query1.finance.yahoo.com/v8/finance/chart/MSFT",
-    params: { "interval" => "1d", "range" => "1d" },
-    timeout: 15  # 15 second timeout
-  )
-  
-  if response && response.success?
-    puts "   ✅ Yahoo Finance request successful"
-    puts "   Response length: #{response.body.length} characters"
-    
-    if response.body.strip.start_with?('{') && response.body.include?('"chart"')
-      puts "   ✅ Response appears to be valid Yahoo Finance JSON"
-    else
-      puts "   ⚠️  Response format unexpected"
-    end
-  else
-    puts "   ❌ Yahoo Finance request failed"
-  end
-rescue => e
-  puts "   ❌ Error with Yahoo Finance: #{e.message}"
-end
-
-puts
-puts "=== Test Summary ==="
-puts "The minimal curl-impersonate integration is ready."
-puts
-puts "To use with the full gem:"
-puts "1. Add the two integration files to lib/yf_as_dataframe/"
-puts "2. Add require statements to your code"
-puts "3. Your existing code will automatically use curl-impersonate"
-puts
-puts "Files needed:"
-puts "- lib/yf_as_dataframe/curl_impersonate_integration.rb"
-puts "- lib/yf_as_dataframe/yf_connection_minimal_patch.rb"
-puts
-puts "Integration code:"
-puts "require 'yf_as_dataframe/curl_impersonate_integration'"
-puts "require 'yf_as_dataframe/yf_connection_minimal_patch'"