yf_as_dataframe 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33f890b28552ff173c4aa6949a073d5e5dae98d9531a00b6b7c4b3028a5cb493
-  data.tar.gz: 7056aee7a785f649493200065a3cf8ef77ad06cecd89c1b446d64a1277efed68
+  metadata.gz: 13496d1eaf3e5ce09c9477de83957acb3d2183214eec2b0dedabdf8b612bd80b
+  data.tar.gz: 604c01e5bba073e9350146c6636c6ce29f1ac62a7d2a9a4e6e9338a33f1a65e0
 SHA512:
-  metadata.gz: 7bbc4573fa5cd0dcbb76de1026551bd49909d71355184cd2fafe52ec155daba4a2503659fec21eddb2cfe170baec08dfc8d16b53ebcdb0df6f895bd61ad75423
-  data.tar.gz: 8559c84bfd359ed3b7d4fba8d87fc201a4207a95067f5afa664b882b30adccfb98c4f5cbbe34e05a644a28aa05449d2205ec8c1906da2e6864d0ee67c1a65d86
+  metadata.gz: e758fafbc0396c7582a9ad0654c94d7721afd2457a403408376be07fa7ace874887a83499c4507a122f1d027a0126da31c9447cb5b61a3d9fc3ab190e7e36f96
+  data.tar.gz: 74a1263cd148add7c56417b4912052471ae6c49fc4163daacc5d5ce3988e53135802f4ec7912aeef372e735fec470072921f15f536d628e5449315fb31ea3142

data/Gemfile.lock

@@ -0,0 +1,99 @@
+PATH
+  remote: .
+  specs:
+    yf_as_dataframe (0.4.0)
+      activesupport
+      httparty
+      nokogiri
+      polars-df (~> 0.12.0)
+      tulirb
+      tzinfo
+      tzinfo-data
+      zache
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (7.2.2.1)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+    ast (2.4.3)
+    base64 (0.3.0)
+    benchmark (0.4.1)
+    bigdecimal (3.2.2)
+    concurrent-ruby (1.3.5)
+    connection_pool (2.5.3)
+    csv (3.3.5)
+    drb (2.2.3)
+    httparty (0.23.1)
+      csv
+      mini_mime (>= 1.0.0)
+      multi_xml (>= 0.5.2)
+    i18n (1.14.7)
+      concurrent-ruby (~> 1.0)
+    json (2.12.2)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    logger (1.7.0)
+    mini_mime (1.1.5)
+    minitest (5.25.5)
+    multi_xml (0.7.1)
+      bigdecimal (~> 3.1)
+    nokogiri (1.18.8-arm64-darwin)
+      racc (~> 1.4)
+    parallel (1.27.0)
+    parser (3.3.8.0)
+      ast (~> 2.4.1)
+      racc
+    polars-df (0.12.0-arm64-darwin)
+      bigdecimal
+    prism (1.4.0)
+    racc (1.8.1)
+    rainbow (3.1.1)
+    rake (13.3.0)
+    regexp_parser (2.10.0)
+    rubocop (1.77.0)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (~> 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.45.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.45.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    ruby-progressbar (1.13.0)
+    securerandom (0.4.1)
+    tulirb (1.0.0)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2025.2)
+      tzinfo (>= 1.0.0)
+    unicode-display_width (3.1.4)
+      unicode-emoji (~> 4.0, >= 4.0.4)
+    unicode-emoji (4.0.4)
+    zache (0.15.0)
+
+PLATFORMS
+  arm64-darwin-23
+
+DEPENDENCIES
+  rake (~> 13.0)
+  rubocop (~> 1.21)
+  yf_as_dataframe!
+
+BUNDLED WITH
+   2.6.9

data/MINIMAL_INTEGRATION.md

@@ -0,0 +1,227 @@
+# Minimal Curl-Impersonate Integration
+
+## Overview
+
+This is a minimal integration that makes curl-impersonate the **default behavior** for all Yahoo Finance requests. No changes to your existing code are required - curl-impersonate is used automatically to bypass TLS fingerprinting.
+
+## Installation
+
+### 1. Install curl-impersonate
+
+```bash
+# macOS
+brew tap shakacode/brew
+brew install curl-impersonate
+
+# Verify installation
+ls -la /usr/local/bin/curl_*
+```
+
+### 2. Custom Installation Directory (Optional)
+
+If you have curl-impersonate installed in a different directory, you can set the `CURL_IMPERSONATE_DIR` environment variable:
+
+```bash
+# Set custom directory
+export CURL_IMPERSONATE_DIR="/opt/curl-impersonate/bin"
+
+# Or set it for a single command
+CURL_IMPERSONATE_DIR="/opt/curl-impersonate/bin" ruby your_script.rb
+```
+
+The default directory is `/usr/local/bin` if the environment variable is not set.
+
+### 3. Add Integration Files
+
+Copy these two files to your project's `lib/yf_as_dataframe/` directory:
+
+1. `lib/yf_as_dataframe/curl_impersonate_integration.rb`
+2. `lib/yf_as_dataframe/yf_connection_minimal_patch.rb`
+
+### 4. Enable Integration
+
+Add this single line to your code **before** any Yahoo Finance requests:
+
+```ruby
+require 'yf_as_dataframe/curl_impersonate_integration'
+require 'yf_as_dataframe/yf_connection_minimal_patch'
+```
+
+## Usage
+
+### Default Behavior (Recommended)
+
+Your existing code works exactly as before, but now uses curl-impersonate automatically:
+
+```ruby
+require 'yf_as_dataframe'
+require 'yf_as_dataframe/curl_impersonate_integration'
+require 'yf_as_dataframe/yf_connection_minimal_patch'
+
+# Your existing code - no changes needed!
+msft = YfAsDataframe::Ticker.new("MSFT")
+hist = msft.history(period: "1mo")  # Uses curl-impersonate automatically
+puts "Retrieved #{hist.length} data points"
+```
+
+### Configuration (Optional)
+
+You can configure the behavior if needed:
+
+```ruby
+# Disable curl-impersonate (use HTTParty only)
+YfAsDataframe::YfConnection.enable_curl_impersonate(false)
+
+# Disable fallback (fail if curl-impersonate fails)
+YfAsDataframe::YfConnection.enable_curl_impersonate_fallback(false)
+
+# Set timeout
+YfAsDataframe::YfConnection.set_curl_impersonate_timeout(45)
+
+# Check available executables
+executables = YfAsDataframe::YfConnection.get_available_curl_impersonate_executables
+puts "Available: #{executables.length} executables"
+
+# Check which directory is being used
+puts "Using directory: #{YfAsDataframe::CurlImpersonateIntegration.executable_directory}"
+```
+
+## How It Works
+
+1. **Automatic Detection**: Dynamically finds curl-impersonate executables in the configured directory
+2. **Default Behavior**: Uses curl-impersonate for all requests by default
+3. **Seamless Fallback**: Falls back to HTTParty if curl-impersonate fails
+4. **Zero Interface Changes**: All existing method signatures remain the same
+
+## Key Features
+
+### ✅ **Zero Code Changes**
+- Your existing code works exactly as before
+- No new method names to learn
+- No changes to method signatures
+
+### ✅ **Automatic Browser Rotation**
+- Randomly selects from available curl-impersonate executables
+- Supports Chrome, Firefox, Edge, and Safari configurations
+- Automatically adapts to new browser versions
+
+### ✅ **Robust Fallback**
+- Falls back to HTTParty if curl-impersonate fails
+- Configurable fallback behavior
+- Maintains compatibility with existing code
+
+### ✅ **Dynamic Discovery**
+- Automatically finds curl-impersonate executables
+- Configurable directory via environment variable
+- Works with any curl-impersonate installation
+
+### ✅ **Environment Variable Support**
+- Set `CURL_IMPERSONATE_DIR` to customize installation directory
+- Defaults to `/usr/local/bin` if not set
+- Supports both persistent and per-command configuration
+
+## Example
+
+```ruby
+require 'yf_as_dataframe'
+require 'yf_as_dataframe/curl_impersonate_integration'
+require 'yf_as_dataframe/yf_connection_minimal_patch'
+
+# Check what's available
+executables = YfAsDataframe::YfConnection.get_available_curl_impersonate_executables
+puts "Found #{executables.length} curl-impersonate executables"
+
+# Check which directory is being used
+puts "Using directory: #{YfAsDataframe::CurlImpersonateIntegration.executable_directory}"
+
+# Use as normal - curl-impersonate is used automatically
+msft = YfAsDataframe::Ticker.new("MSFT")
+
+begin
+  # These all use curl-impersonate automatically
+  hist = msft.history(period: "1mo")
+  info = msft.info
+  actions = msft.actions
+  
+  puts "✅ All requests successful using curl-impersonate"
+  puts "History: #{hist.length} data points"
+  puts "Company: #{info['longName']}"
+  puts "Actions: #{actions.length} items"
+rescue => e
+  puts "❌ Error: #{e.message}"
+end
+```
+
+## Troubleshooting
+
+### "No curl-impersonate executables found"
+```bash
+# Check if executables exist in default location
+ls -la /usr/local/bin/curl_*
+
+# Check if executables exist in custom location
+ls -la $CURL_IMPERSONATE_DIR/curl_*
+
+# If not found, reinstall curl-impersonate
+brew reinstall curl-impersonate
+```
+
+### Permission errors
+```bash
+sudo chmod +x /usr/local/bin/curl_*
+# or
+sudo chmod +x $CURL_IMPERSONATE_DIR/curl_*
+```
+
+### Still getting blocked
+```ruby
+# Try disabling fallback to see if curl-impersonate is working
+YfAsDataframe::YfConnection.enable_curl_impersonate_fallback(false)
+
+# Check available executables
+executables = YfAsDataframe::YfConnection.get_available_curl_impersonate_executables
+puts executables.map { |e| "#{e[:browser]} #{e[:executable]}" }
+
+# Check which directory is being used
+puts "Directory: #{YfAsDataframe::CurlImpersonateIntegration.executable_directory}"
+```
+
+## Configuration Options
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `curl_impersonate_enabled` | `true` | Use curl-impersonate for requests |
+| `curl_impersonate_fallback` | `true` | Fall back to HTTParty if curl-impersonate fails |
+| `curl_impersonate_timeout` | `30` | Timeout in seconds for curl-impersonate requests |
+| `CURL_IMPERSONATE_DIR` | `/usr/local/bin` | Directory containing curl-impersonate executables |
+
+## Benefits
+
+1. **Immediate Solution**: Bypasses TLS fingerprinting immediately
+2. **Zero Learning Curve**: No new APIs or methods to learn
+3. **Future-Proof**: Automatically adapts to new curl-impersonate versions
+4. **Robust**: Multiple fallback strategies ensure reliability
+5. **Minimal**: Only two small files to add
+6. **Flexible**: Configurable installation directory via environment variable
+
+## Comparison with Previous Approach
+
+| Aspect | Previous Approach | Minimal Approach |
+|--------|------------------|------------------|
+| Interface Changes | New method names | No changes |
+| Learning Curve | High | Zero |
+| Integration | Complex | Simple |
+| Default Behavior | HTTParty | curl-impersonate |
+| Configuration | Required | Optional |
+| Files to Add | 3 files | 2 files |
+| Directory Config | Hardcoded | Environment variable |
+
+## Next Steps
+
+1. **Install curl-impersonate** following the instructions above
+2. **Set CURL_IMPERSONATE_DIR** if using a custom installation directory
+3. **Add the two integration files** to your project
+4. **Add the require statements** to your code
+5. **Test with your existing code** - it should work immediately
+
+That's it! Your existing Yahoo Finance scraping code will now automatically use curl-impersonate to bypass TLS fingerprinting. 

data/README.md

@@ -197,6 +197,71 @@ YfAsDataframe.zlema(df, column: 'Adj Close', window: 5)
 
 ---
 
+## TLS Fingerprinting Protection
+
+**New in v0.4.0**: This gem now includes built-in support for [curl-impersonate](https://github.com/lwthiker/curl-impersonate). The curl-impersonate integration is **enabled by default** in v0.4.0+. Existing code will automatically use curl-impersonate to bypass TLS fingerprinting:
+
+### Installation Requirements
+
+To use the TLS fingerprinting protection, you need to install curl-impersonate:
+
+```bash
+# macOS
+brew tap shakacode/brew
+brew install curl-impersonate
+
+# Verify installation
+ls -la /usr/local/bin/curl_*
+```
+
+### Custom Installation Directory
+
+If you have curl-impersonate installed in a different directory, you can set the `CURL_IMPERSONATE_DIR` environment variable:
+
+```bash
+# Set custom directory
+export CURL_IMPERSONATE_DIR="/opt/curl-impersonate/bin"
+
+# Or set it for a single command
+CURL_IMPERSONATE_DIR="/opt/curl-impersonate/bin" ruby your_script.rb
+```
+
+The default directory is `/usr/local/bin` if the environment variable is not set.
+
+### Configuration (Optional)
+
+You can configure the curl-impersonate behavior if needed:
+
+```ruby
+# Disable curl-impersonate (use HTTParty only)
+YfAsDataframe::YfConnection.enable_curl_impersonate(false)
+
+# Disable fallback (fail if curl-impersonate fails)
+YfAsDataframe::YfConnection.enable_curl_impersonate_fallback(false)
+
+# Set timeout
+YfAsDataframe::YfConnection.set_curl_impersonate_timeout(45)
+
+# Check available executables
+executables = YfAsDataframe::YfConnection.get_available_curl_impersonate_executables
+puts "Available: #{executables.length} executables"
+
+# Check which directory is being used
+puts "Using directory: #{YfAsDataframe::CurlImpersonateIntegration.executable_directory}"
+```
+
+### How It Works
+
+1. **Automatic Detection**: Dynamically finds curl-impersonate executables in the configured directory
+2. **Default Behavior**: Uses curl-impersonate for all requests by default
+3. **Seamless Fallback**: Falls back to HTTParty if curl-impersonate fails
+4. **Browser Rotation**: Randomly selects from Chrome, Firefox, Edge, and Safari configurations
+5. **Zero Interface Changes**: All existing method signatures remain the same
+
+For more detailed information, see [MINIMAL_INTEGRATION.md](MINIMAL_INTEGRATION.md).
+
+---
+
 ## Graphing
 
 To graph any of the series using [Vega](https://github.com/ankane/vega-ruby), per the information [here](https://github.com/ankane/vega-ruby#exporting-charts-experimental), you will need to run 

data/lib/yf_as_dataframe/curl_impersonate_integration.rb

@@ -0,0 +1,110 @@
+require 'open3'
+require 'json'
+require 'ostruct'
+
+class YfAsDataframe
+  module CurlImpersonateIntegration
+    # Configuration
+    @curl_impersonate_enabled = true
+    @curl_impersonate_fallback = true
+    @curl_impersonate_timeout = 5
+    @curl_impersonate_retries = 2
+    @curl_impersonate_retry_delay = 1
+
+    class << self
+      attr_accessor :curl_impersonate_enabled, :curl_impersonate_fallback, 
+                   :curl_impersonate_timeout, :curl_impersonate_retries, 
+                   :curl_impersonate_retry_delay
+    end
+
+    # Get the curl-impersonate executable directory from environment variable or default
+    def self.executable_directory
+      ENV['CURL_IMPERSONATE_DIR'] || '/usr/local/bin'
+    end
+
+    # Find available curl-impersonate executables
+    def self.available_executables
+      @available_executables ||= begin
+        executables = []
+        Dir.glob(File.join(executable_directory, "curl_*")).each do |path|
+          executable = File.basename(path)
+          if executable.start_with?('curl_')
+            browser_type = case executable
+                          when /^curl_chrome/ then :chrome
+                          when /^curl_ff/ then :firefox
+                          when /^curl_edge/ then :edge
+                          when /^curl_safari/ then :safari
+                          else :unknown
+                          end
+            executables << { path: path, executable: executable, browser: browser_type }
+          end
+        end
+        executables
+      end
+    end
+
+    # Get a random executable
+    def self.get_random_executable
+      available = available_executables
+      return nil if available.empty?
+      available.sample
+    end
+
+    # Make a curl-impersonate request
+    def self.make_request(url, headers: {}, params: {}, timeout: nil)
+      executable_info = get_random_executable
+      return nil unless executable_info
+
+      timeout ||= @curl_impersonate_timeout
+
+      # Build command
+      cmd = [executable_info[:path], "--max-time", timeout.to_s]
+      
+      # Add headers
+      headers.each do |key, value|
+        cmd.concat(["-H", "#{key}: #{value}"])
+      end
+
+      # Add query parameters
+      unless params.empty?
+        query_string = params.map { |k, v| "#{k}=#{v}" }.join('&')
+        separator = url.include?('?') ? '&' : '?'
+        url = "#{url}#{separator}#{query_string}"
+      end
+
+      # Add URL
+      cmd << url
+
+      # Debug output
+      puts "DEBUG: curl-impersonate command: #{cmd.join(' ')}"
+      puts "DEBUG: curl-impersonate timeout: #{timeout} seconds"
+
+      # Execute
+      stdout, stderr, status = Open3.capture3(*cmd)
+      
+      puts "DEBUG: curl-impersonate stdout: #{stdout[0..200]}..." if stdout && !stdout.empty?
+      puts "DEBUG: curl-impersonate stderr: #{stderr}" if stderr && !stderr.empty?
+      puts "DEBUG: curl-impersonate status: #{status.exitstatus}"
+      
+      if status.success?
+        # Create a response object similar to HTTParty
+        response = OpenStruct.new
+        response.body = stdout
+        response.code = 200
+        response.define_singleton_method(:success?) { true }
+        response.parsed_response = parse_json_if_possible(stdout)
+        response
+      else
+        nil
+      end
+    end
+
+    private
+
+    def self.parse_json_if_possible(response_body)
+      JSON.parse(response_body)
+    rescue JSON::ParserError
+      response_body
+    end
+  end
+end 

data/lib/yf_as_dataframe/financials.rb

@@ -1,4 +1,5 @@
 require 'polars-df'
+require 'logger'
 
 class YfAsDataframe
   module Financials
@@ -111,7 +112,7 @@ class YfAsDataframe
       ts_url_base = "https://query2.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/#{symbol}?symbol=#{symbol}"
       url = ts_url_base + "&type=" + ts_keys.map { |k| "#{timescale}#{k}" }.join(",")
       start_dt = DateTime.new(2016, 12, 31)
-      end_dt = DateTime.now.tomorrow.midnight
+      end_dt = Time.now.tomorrow.midnight
       url += "&period1=#{start_dt.to_i}&period2=#{end_dt.to_i}"
 
       json_str = get(url).parsed_response
@@ -160,7 +161,7 @@ class YfAsDataframe
         statement = _create_financials_table(nam, timescale)
         return statement unless statement.nil?
       rescue Yfin::YfinDataException => e
-        Rails.logger.error {"#{@symbol}: Failed to create #{nam} financials table for reason: #{e}"}
+        Logger.new(STDOUT).error {"#{@symbol}: Failed to create #{nam} financials table for reason: #{e}"}
       end
       Polars::DataFrame.new()
     end

data/lib/yf_as_dataframe/holders.rb

@@ -1,3 +1,5 @@
+require 'logger'
+
 class YfAsDataframe
   module Holders
     extend ActiveSupport::Concern
@@ -97,7 +99,7 @@ class YfAsDataframe
         result = get_raw_json(QUOTE_SUMMARY_URL + "/#{symbol}", user_agent_headers=user_agent_headers, params=params_dict)
         # Rails.logger.info { "#{__FILE__}:#{__LINE__} result = #{result.inspect}" }
       rescue Exception => e
-        Rails.logger.error("ERROR: #{e.message}")
+        Logger.new(STDOUT).error("ERROR: #{e.message}")
         return nil
       end
       return result
@@ -133,7 +135,7 @@ class YfAsDataframe
 
     def _parse_result(result)
       data = result.parsed_response['quoteSummary']['result'].first #.dig('quoteSummary', 'result', 0)
-      Rails.logger.info { "#{__FILE__}:#{__LINE__} data = #{data.inspect}" }
+      Logger.new(STDOUT).info { "#{__FILE__}:#{__LINE__} data = #{data.inspect}" }
       _parse_institution_ownership(data['institutionOwnership'])
       _parse_fund_ownership(data['fundOwnership'])
       _parse_major_holders_breakdown(data['majorHoldersBreakdown'])

data/lib/yf_as_dataframe/multi.rb

@@ -1,4 +1,5 @@
 require 'polars-df'
+require 'logger'
 
 class YfAsDataframe
   class Multi
@@ -59,7 +60,7 @@ class YfAsDataframe
       #     session: None or Session
       #         Optional. Pass your own session object to be used for all requests
       # """
-      logger = Rails.logger
+      logger = Logger.new(STDOUT)
 
       if show_errors
         YfAsDataframe::Utils.print_once("yfinance: download(show_errors=#{show_errors}) argument is deprecated and will be removed in future version. Do this instead: logging.getLogger('yfinance').setLevel(logging.ERROR)")