yf_as_dataframe 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -12
- data/lib/yf_as_dataframe/curl_impersonate_integration.rb +64 -35
- data/lib/yf_as_dataframe/financials.rb +2 -2
- data/lib/yf_as_dataframe/holders.rb +3 -2
- data/lib/yf_as_dataframe/multi.rb +17 -16
- data/lib/yf_as_dataframe/price_history.rb +17 -17
- data/lib/yf_as_dataframe/quote.rb +1 -1
- data/lib/yf_as_dataframe/ticker.rb +11 -11
- data/lib/yf_as_dataframe/utils.rb +12 -8
- data/lib/yf_as_dataframe/version.rb +1 -1
- data/lib/yf_as_dataframe/yf_connection.rb +12 -12
- data/lib/yf_as_dataframe/yf_connection_minimal_patch.rb +50 -19
- data/lib/yf_as_dataframe/yfinance_exception.rb +1 -1
- metadata +3 -5
- data/quick_test.rb +0 -143
- data/test_minimal_integration.rb +0 -121
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cb178f1a6feb462ac539fccad4d2440f2754b55b4ec0742a965b48bf4aa72fb1
|
4
|
+
data.tar.gz: 189dd9688dd80eeb85f8d2154f3a1d9f260f8fcb30ad7c3f086afb1485d0303a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 042f92be7a2842fb89210a84415ab810cb3d9f96d902f2ba1aa4c10da646012f35dda9e257b02cb0dd2486a432817e489b4d041a054ca291d5a4f8a13f824544
|
7
|
+
data.tar.gz: 023f06de418a640ab2ae3b01997a0f1e8865d519cf0082d6bf48349350958437f40aa0087773800d9edd386ea9e907ae7949d00cb569ca0abc3801bd979c2bdf
|
data/README.md
CHANGED
@@ -12,7 +12,7 @@
|
|
12
12
|
Yahoo, Inc.**
|
13
13
|
|
14
14
|
yf_as_dataframe is **not** affiliated, endorsed, or vetted by Yahoo, Inc. It is
|
15
|
-
an open-source tool that uses Yahoo's publicly available APIs, and is
|
15
|
+
an open-source tool that uses Yahoo's publicly available APIs, and is **only**
|
16
16
|
intended for research and educational purposes.
|
17
17
|
|
18
18
|
**You should refer to Yahoo!'s terms of use**
|
@@ -216,7 +216,9 @@ ls -la /usr/local/bin/curl_*
|
|
216
216
|
|
217
217
|
### Custom Installation Directory
|
218
218
|
|
219
|
-
|
219
|
+
The codebase will look for the location of the curl-impersonate binaries per the `CURL_IMPERSONATE_DIR` environment variable;
|
220
|
+
if it is not assigned, the default location of the binaries is `/usr/local/bin`.
|
221
|
+
The code will randomly select one of the binaries (expected to be named "curl_chrome*", "curl_ff*", "curl_edge*", etc.) for its communications with the servers.
|
220
222
|
|
221
223
|
```bash
|
222
224
|
# Set custom directory
|
@@ -226,8 +228,6 @@ export CURL_IMPERSONATE_DIR="/opt/curl-impersonate/bin"
|
|
226
228
|
CURL_IMPERSONATE_DIR="/opt/curl-impersonate/bin" ruby your_script.rb
|
227
229
|
```
|
228
230
|
|
229
|
-
The default directory is `/usr/local/bin` if the environment variable is not set.
|
230
|
-
|
231
231
|
### Configuration (Optional)
|
232
232
|
|
233
233
|
You can configure the curl-impersonate behavior if needed:
|
@@ -250,14 +250,6 @@ puts "Available: #{executables.length} executables"
|
|
250
250
|
puts "Using directory: #{YfAsDataframe::CurlImpersonateIntegration.executable_directory}"
|
251
251
|
```
|
252
252
|
|
253
|
-
### How It Works
|
254
|
-
|
255
|
-
1. **Automatic Detection**: Dynamically finds curl-impersonate executables in the configured directory
|
256
|
-
2. **Default Behavior**: Uses curl-impersonate for all requests by default
|
257
|
-
3. **Seamless Fallback**: Falls back to HTTParty if curl-impersonate fails
|
258
|
-
4. **Browser Rotation**: Randomly selects from Chrome, Firefox, Edge, and Safari configurations
|
259
|
-
5. **Zero Interface Changes**: All existing method signatures remain the same
|
260
|
-
|
261
253
|
For more detailed information, see [MINIMAL_INTEGRATION.md](MINIMAL_INTEGRATION.md).
|
262
254
|
|
263
255
|
---
|
@@ -1,20 +1,24 @@
|
|
1
1
|
require 'open3'
|
2
2
|
require 'json'
|
3
3
|
require 'ostruct'
|
4
|
+
require 'timeout'
|
4
5
|
|
5
6
|
class YfAsDataframe
|
6
7
|
module CurlImpersonateIntegration
|
7
8
|
# Configuration
|
8
9
|
@curl_impersonate_enabled = true
|
9
10
|
@curl_impersonate_fallback = true
|
10
|
-
@curl_impersonate_timeout = 5
|
11
|
+
@curl_impersonate_timeout = 30 # Increased from 5 to 30 seconds
|
12
|
+
@curl_impersonate_connect_timeout = 10 # New: connection timeout
|
11
13
|
@curl_impersonate_retries = 2
|
12
14
|
@curl_impersonate_retry_delay = 1
|
15
|
+
@curl_impersonate_process_timeout = 60 # New: process timeout protection
|
13
16
|
|
14
17
|
class << self
|
15
18
|
attr_accessor :curl_impersonate_enabled, :curl_impersonate_fallback,
|
16
|
-
:curl_impersonate_timeout, :
|
17
|
-
:curl_impersonate_retry_delay
|
19
|
+
:curl_impersonate_timeout, :curl_impersonate_connect_timeout,
|
20
|
+
:curl_impersonate_retries, :curl_impersonate_retry_delay,
|
21
|
+
:curl_impersonate_process_timeout
|
18
22
|
end
|
19
23
|
|
20
24
|
# Get the curl-impersonate executable directory from environment variable or default
|
@@ -50,51 +54,76 @@ class YfAsDataframe
|
|
50
54
|
available.sample
|
51
55
|
end
|
52
56
|
|
53
|
-
# Make a curl-impersonate request
|
54
|
-
def self.make_request(url, headers: {}, params: {}, timeout: nil)
|
57
|
+
# Make a curl-impersonate request with improved timeout handling
|
58
|
+
def self.make_request(url, headers: {}, params: {}, timeout: nil, retries: nil)
|
55
59
|
executable_info = get_random_executable
|
56
60
|
return nil unless executable_info
|
57
61
|
|
58
62
|
timeout ||= @curl_impersonate_timeout
|
63
|
+
retries ||= @curl_impersonate_retries
|
59
64
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
65
|
+
cmd = [
|
66
|
+
executable_info[:path],
|
67
|
+
"--max-time", timeout.to_s,
|
68
|
+
"--connect-timeout", @curl_impersonate_connect_timeout.to_s,
|
69
|
+
"--retry", retries.to_s,
|
70
|
+
"--retry-delay", @curl_impersonate_retry_delay.to_s,
|
71
|
+
"--retry-max-time", (timeout * 2).to_s,
|
72
|
+
"--fail",
|
73
|
+
"--silent",
|
74
|
+
"--show-error"
|
75
|
+
]
|
76
|
+
headers.each { |key, value| cmd.concat(["-H", "#{key}: #{value}"]) }
|
69
77
|
unless params.empty?
|
70
78
|
query_string = params.map { |k, v| "#{k}=#{v}" }.join('&')
|
71
79
|
separator = url.include?('?') ? '&' : '?'
|
72
80
|
url = "#{url}#{separator}#{query_string}"
|
73
81
|
end
|
74
|
-
|
75
|
-
# Add URL
|
76
82
|
cmd << url
|
77
83
|
|
78
|
-
#
|
79
|
-
puts "DEBUG: curl-impersonate
|
80
|
-
puts "DEBUG: curl-impersonate timeout: #{timeout} seconds"
|
84
|
+
# puts "DEBUG: curl-impersonate command: #{cmd.join(' ')}"
|
85
|
+
# puts "DEBUG: curl-impersonate timeout: #{timeout} seconds"
|
81
86
|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
87
|
+
begin
|
88
|
+
stdout_str = ''
|
89
|
+
stderr_str = ''
|
90
|
+
status = nil
|
91
|
+
Open3.popen3(*cmd) do |stdin, stdout, stderr, wait_thr|
|
92
|
+
stdin.close
|
93
|
+
pid = wait_thr.pid
|
94
|
+
done = false
|
95
|
+
monitor = Thread.new do
|
96
|
+
sleep(timeout + 10)
|
97
|
+
unless done
|
98
|
+
# puts "DEBUG: Killing curl-impersonate PID \\#{pid} after timeout"
|
99
|
+
Process.kill('TERM', pid) rescue nil
|
100
|
+
sleep(1)
|
101
|
+
Process.kill('KILL', pid) rescue nil if wait_thr.alive?
|
102
|
+
end
|
103
|
+
end
|
104
|
+
stdout_str = stdout.read
|
105
|
+
stderr_str = stderr.read
|
106
|
+
status = wait_thr.value
|
107
|
+
done = true
|
108
|
+
monitor.kill
|
109
|
+
end
|
110
|
+
# puts "DEBUG: curl-impersonate stdout: #{stdout_str[0..200]}..." if stdout_str && !stdout_str.empty?
|
111
|
+
# puts "DEBUG: curl-impersonate stderr: #{stderr_str}" if stderr_str && !stderr_str.empty?
|
112
|
+
# puts "DEBUG: curl-impersonate status: #{status.exitstatus}"
|
113
|
+
if status.success?
|
114
|
+
response = OpenStruct.new
|
115
|
+
response.body = stdout_str
|
116
|
+
response.code = 200
|
117
|
+
response.define_singleton_method(:success?) { true }
|
118
|
+
response.parsed_response = parse_json_if_possible(stdout_str)
|
119
|
+
response
|
120
|
+
else
|
121
|
+
# puts "DEBUG: curl-impersonate failed with error: \\#{error_message}"
|
122
|
+
error_message = "curl failed with code \\#{status.exitstatus}: \\#{stderr_str}"
|
123
|
+
nil
|
124
|
+
end
|
125
|
+
rescue => e
|
126
|
+
# puts "DEBUG: curl-impersonate exception: \\#{e.message}"
|
98
127
|
nil
|
99
128
|
end
|
100
129
|
end
|
@@ -143,7 +143,7 @@ class YfAsDataframe
|
|
143
143
|
|
144
144
|
df = Polars::DataFrame.new(df)
|
145
145
|
timestamps.map{|ts| Time.at(ts).utc.to_date.to_s }.each do |t|
|
146
|
-
puts t
|
146
|
+
# puts t
|
147
147
|
df.replace(t, Polars::Series.new(df[t].cast(Polars::String)))
|
148
148
|
end
|
149
149
|
df
|
@@ -161,7 +161,7 @@ class YfAsDataframe
|
|
161
161
|
statement = _create_financials_table(nam, timescale)
|
162
162
|
return statement unless statement.nil?
|
163
163
|
rescue Yfin::YfinDataException => e
|
164
|
-
Logger.new(STDOUT).error {"#{@symbol}: Failed to create #{nam} financials table for reason: #{e}"}
|
164
|
+
# Logger.new(STDOUT).error {"#{@symbol}: Failed to create #{nam} financials table for reason: #{e}"}
|
165
165
|
end
|
166
166
|
Polars::DataFrame.new()
|
167
167
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'logger'
|
2
|
+
require 'open-uri'
|
2
3
|
|
3
4
|
class YfAsDataframe
|
4
5
|
module Holders
|
@@ -99,7 +100,7 @@ class YfAsDataframe
|
|
99
100
|
result = get_raw_json(QUOTE_SUMMARY_URL + "/#{symbol}", user_agent_headers=user_agent_headers, params=params_dict)
|
100
101
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} result = #{result.inspect}" }
|
101
102
|
rescue Exception => e
|
102
|
-
Logger.new(STDOUT).error("ERROR: #{e.message}")
|
103
|
+
# Logger.new(STDOUT).error("ERROR: #{e.message}")
|
103
104
|
return nil
|
104
105
|
end
|
105
106
|
return result
|
@@ -135,7 +136,7 @@ class YfAsDataframe
|
|
135
136
|
|
136
137
|
def _parse_result(result)
|
137
138
|
data = result.parsed_response['quoteSummary']['result'].first #.dig('quoteSummary', 'result', 0)
|
138
|
-
Logger.new(STDOUT).info { "#{__FILE__}:#{__LINE__} data = #{data.inspect}" }
|
139
|
+
# Logger.new(STDOUT).info { "#{__FILE__}:#{__LINE__} data = #{data.inspect}" }
|
139
140
|
_parse_institution_ownership(data['institutionOwnership'])
|
140
141
|
_parse_fund_ownership(data['fundOwnership'])
|
141
142
|
_parse_major_holders_breakdown(data['majorHoldersBreakdown'])
|
@@ -62,19 +62,20 @@ class YfAsDataframe
|
|
62
62
|
# """
|
63
63
|
logger = Logger.new(STDOUT)
|
64
64
|
|
65
|
+
# YfAsDataframe::Utils.print_once("yfinance: download(show_errors=#{show_errors}) argument is deprecated and will be removed in future version. Do this instead: logging.getLogger('yfinance').setLevel(logging.ERROR)")
|
66
|
+
|
65
67
|
if show_errors
|
66
|
-
YfAsDataframe::Utils.print_once("yfinance: download(show_errors=#{show_errors}) argument is deprecated and will be removed in future version. Do this instead: logging.getLogger('yfinance').setLevel(logging.
|
67
|
-
logger.level = Logger::
|
68
|
+
# YfAsDataframe::Utils.print_once("yfinance: download(show_errors=#{show_errors}) argument is deprecated and will be removed in future version. Do this instead to suppress error messages: logging.getLogger('yfinance').setLevel(logging.CRITICAL)")
|
69
|
+
# logger.level = Logger::CRITICAL
|
68
70
|
else
|
69
|
-
|
70
|
-
logger.level = Logger::CRITICAL
|
71
|
+
# logger.level = Logger::CRITICAL
|
71
72
|
end
|
72
73
|
|
73
|
-
if logger.debug?
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
end
|
74
|
+
# if logger.debug?
|
75
|
+
# threads = false if threads
|
76
|
+
# logger.debug('Disabling multithreading because DEBUG logging enabled')
|
77
|
+
# progress = false if progress
|
78
|
+
# end
|
78
79
|
|
79
80
|
ignore_tz = interval[1..-1].match?(/[mh]/) ? false : true if ignore_tz.nil?
|
80
81
|
|
@@ -119,7 +120,7 @@ class YfAsDataframe
|
|
119
120
|
@shared::_PROGRESS_BAR.completed if progress
|
120
121
|
|
121
122
|
unless @shared::_ERRORS.empty?
|
122
|
-
logger.error("\n#{@shared::_ERRORS.length} Failed download#{@shared::_ERRORS.length > 1 ? 's' : ''}:")
|
123
|
+
# logger.error("\n#{@shared::_ERRORS.length} Failed download#{@shared::_ERRORS.length > 1 ? 's' : ''}:")
|
123
124
|
|
124
125
|
errors = {}
|
125
126
|
@shared::_ERRORS.each do |ticker, err|
|
@@ -127,9 +128,9 @@ class YfAsDataframe
|
|
127
128
|
errors[err] ||= []
|
128
129
|
errors[err] << ticker
|
129
130
|
end
|
130
|
-
errors.each do |err, tickers|
|
131
|
-
|
132
|
-
end
|
131
|
+
# errors.each do |err, tickers|
|
132
|
+
# logger.error("#{tickers.join(', ')}: #{err}")
|
133
|
+
# end
|
133
134
|
|
134
135
|
tbs = {}
|
135
136
|
@shared::_TRACEBACKS.each do |ticker, tb|
|
@@ -137,9 +138,9 @@ class YfAsDataframe
|
|
137
138
|
tbs[tb] ||= []
|
138
139
|
tbs[tb] << ticker
|
139
140
|
end
|
140
|
-
tbs.each do |tb, tickers|
|
141
|
-
|
142
|
-
end
|
141
|
+
# tbs.each do |tb, tickers|
|
142
|
+
# logger.debug("#{tickers.join(', ')}: #{tb}")
|
143
|
+
# end
|
143
144
|
end
|
144
145
|
|
145
146
|
if ignore_tz
|
@@ -35,7 +35,7 @@ class YfAsDataframe
|
|
35
35
|
def history(period: "1mo", interval: "1d", start: nil, fin: nil, prepost: false,
|
36
36
|
actions: true, auto_adjust: true, back_adjust: false, repair: false, keepna: false,
|
37
37
|
rounding: false, raise_errors: false, returns: false)
|
38
|
-
logger = Logger.new(STDOUT) # Replace Rails.logger with standard Ruby logger
|
38
|
+
# logger = Logger.new(STDOUT) # Replace Rails.logger with standard Ruby logger
|
39
39
|
start_user = start
|
40
40
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} here" }
|
41
41
|
end_user = fin || Time.now
|
@@ -75,7 +75,7 @@ class YfAsDataframe
|
|
75
75
|
if raise_errors
|
76
76
|
raise Exception.new("#{ticker}: #{err_msg}")
|
77
77
|
else
|
78
|
-
logger.error("#{ticker}: #{err_msg}")
|
78
|
+
# logger.error("#{ticker}: #{err_msg}")
|
79
79
|
end
|
80
80
|
if @reconstruct_start_interval && @reconstruct_start_interval == interval
|
81
81
|
@reconstruct_start_interval = nil
|
@@ -572,7 +572,7 @@ class YfAsDataframe
|
|
572
572
|
if raise_errors
|
573
573
|
raise Exception.new("#{@ticker}: #{err_msg}")
|
574
574
|
else
|
575
|
-
Logger.new(STDOUT).error("#{@ticker}: #{err_msg}")
|
575
|
+
# Logger.new(STDOUT).error("#{@ticker}: #{err_msg}")
|
576
576
|
end
|
577
577
|
return YfAsDataframe::Utils.empty_df
|
578
578
|
end
|
@@ -617,8 +617,8 @@ class YfAsDataframe
|
|
617
617
|
def _get_data(ticker, params, fin, raise_errors)
|
618
618
|
url = "https://query2.finance.yahoo.com/v8/finance/chart/#{CGI.escape ticker}"
|
619
619
|
# url = "https://query1.finance.yahoo.com/v7/finance/download/#{ticker}" ... Deprecated
|
620
|
-
logger = Logger.new(STDOUT)
|
621
|
-
|
620
|
+
# logger = Logger.new(STDOUT)
|
621
|
+
# logger.info { "#{__FILE__}:#{__LINE__} url = #{url}" }
|
622
622
|
data = nil
|
623
623
|
# get_fn = @data.method(:get)
|
624
624
|
|
@@ -631,9 +631,9 @@ class YfAsDataframe
|
|
631
631
|
end
|
632
632
|
|
633
633
|
begin
|
634
|
-
logger.info { "#{__FILE__}:#{__LINE__} url = #{url}, params = #{params.inspect}" }
|
634
|
+
# logger.info { "#{__FILE__}:#{__LINE__} url = #{url}, params = #{params.inspect}" }
|
635
635
|
data = get(url, nil, params).parsed_response
|
636
|
-
logger.info { "#{__FILE__}:#{__LINE__} data = #{data.inspect}" }
|
636
|
+
# logger.info { "#{__FILE__}:#{__LINE__} data = #{data.inspect}" }
|
637
637
|
|
638
638
|
# Validate response before processing
|
639
639
|
unless validate_yahoo_response(data)
|
@@ -647,10 +647,10 @@ class YfAsDataframe
|
|
647
647
|
|
648
648
|
# Use standard Ruby Hash
|
649
649
|
data = data.is_a?(Hash) ? data : JSON.parse(data.to_s) rescue data
|
650
|
-
logger.info { "#{__FILE__}:#{__LINE__} data = #{data.inspect}" }
|
650
|
+
# logger.info { "#{__FILE__}:#{__LINE__} data = #{data.inspect}" }
|
651
651
|
rescue Exception => e
|
652
|
-
logger.error { "#{__FILE__}:#{__LINE__} Exception caught: #{e.message}" }
|
653
|
-
logger.error { "#{__FILE__}:#{__LINE__} Exception backtrace: #{e.backtrace.first(5).join("\n")}" }
|
652
|
+
# logger.error { "#{__FILE__}:#{__LINE__} Exception caught: #{e.message}" }
|
653
|
+
# logger.error { "#{__FILE__}:#{__LINE__} Exception backtrace: #{e.backtrace.first(5).join("\n")}" }
|
654
654
|
raise if raise_errors
|
655
655
|
end
|
656
656
|
|
@@ -719,7 +719,7 @@ class YfAsDataframe
|
|
719
719
|
# startDt = quotes.index[0].floor('D')
|
720
720
|
startDt = quotes['Timestamps'].to_a.map(&:to_date).min
|
721
721
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} startDt = #{startDt.inspect}" }
|
722
|
-
endDt = !fin.nil? && !(fin.respond_to?(:empty?) && fin.empty?) ? fin.to_date :
|
722
|
+
endDt = !fin.nil? && !(fin.respond_to?(:empty?) && fin.empty?) ? fin.to_date : (Time.now + 86400).to_date
|
723
723
|
|
724
724
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} @history[events][dividends] = #{@history['events']["dividends"].inspect}" }
|
725
725
|
# divi = {}
|
@@ -731,32 +731,32 @@ class YfAsDataframe
|
|
731
731
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} ts = #{ts.inspect}" }
|
732
732
|
@history['events']["dividends"].select{|k,v|
|
733
733
|
Time.at(k.to_i).utc.to_date >= startDt && Time.at(k.to_i).utc.to_date <= endDt }.each{|k,v|
|
734
|
-
d[ts.index(Time.at(k.to_i).utc)] = v['amount'].to_f} unless @history.
|
734
|
+
d[ts.index(Time.at(k.to_i).utc)] = v['amount'].to_f} unless @history.dig('events', 'dividends').nil?
|
735
735
|
df['Dividends'] = Polars::Series.new(d)
|
736
736
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} df = #{df.inspect}" }
|
737
737
|
|
738
738
|
# caga = {}
|
739
739
|
# @history['events']["capital gains"].select{|k,v|
|
740
740
|
# Time.at(k.to_i).utc.to_date >= startDt && Time.at(k.to_i).utc.to_date <= endDt }.each{|k,v|
|
741
|
-
# caga['date'] = v['amount']} unless @history.
|
741
|
+
# caga['date'] = v['amount']} unless @history.dig('events', 'capital gains').nil?
|
742
742
|
# capital_gains = capital_gains.loc[startDt:] if capital_gains.shape.first > 0
|
743
743
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} caga = #{caga.inspect}" }
|
744
744
|
d = [0.0] * df.length
|
745
745
|
@history['events']["capital gains"].select{|k,v|
|
746
746
|
Time.at(k.to_i).utc.to_date >= startDt && Time.at(k.to_i).utc.to_date <= endDt }.each{|k,v|
|
747
|
-
d[ts.index(Time.at(k.to_i).utc)] = v['amount'].to_f} unless @history.
|
747
|
+
d[ts.index(Time.at(k.to_i).utc)] = v['amount'].to_f} unless @history.dig('events', 'capital gains').nil?
|
748
748
|
df['Capital Gains'] = Polars::Series.new(d)
|
749
749
|
|
750
750
|
# splits = splits.loc[startDt:] if splits.shape[0] > 0
|
751
751
|
# stspl = {}
|
752
752
|
# @history['events']['stock splits'].select{|k,v|
|
753
753
|
# Time.at(k.to_i).utc.to_date >= startDt && Time.at(k.to_i).utc.to_date <= endDt }.each{|k,v|
|
754
|
-
# stspl['date'] = v['numerator'].to_f/v['denominator'].to_f} unless @history.
|
754
|
+
# stspl['date'] = v['numerator'].to_f/v['denominator'].to_f} unless @history.dig('events', 'capital gains').nil?
|
755
755
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} stspl = #{stspl.inspect}" }
|
756
756
|
d = [0.0] * df.length
|
757
757
|
@history['events']["capital gains"].select{|k,v|
|
758
758
|
Time.at(k.to_i).utc.to_date >= startDt && Time.at(k.to_i).utc.to_date <= endDt }.each{|k,v|
|
759
|
-
d[ts.index(Time.at(k.to_i).utc)] = v['numerator'].to_f/v['denominator'].to_f} unless @history.
|
759
|
+
d[ts.index(Time.at(k.to_i).utc)] = v['numerator'].to_f/v['denominator'].to_f} unless @history.dig('events', 'capital gains').nil?
|
760
760
|
df['Stock Splits'] = Polars::Series.new(d)
|
761
761
|
end
|
762
762
|
|
@@ -1085,7 +1085,7 @@ class YfAsDataframe
|
|
1085
1085
|
# quotes.sort_index!(inplace: true)
|
1086
1086
|
|
1087
1087
|
if interval.downcase == "30m"
|
1088
|
-
logger.debug("#{ticker}: resampling 30m OHLC from 15m")
|
1088
|
+
# logger.debug("#{ticker}: resampling 30m OHLC from 15m")
|
1089
1089
|
quotes2 = quotes.resample('30T')
|
1090
1090
|
quotes = Polars::DataFrame.new(index: quotes2.last.index, data: {
|
1091
1091
|
'Open' => quotes2['Open'].first,
|
@@ -130,7 +130,7 @@ class YfAsDataframe
|
|
130
130
|
result = get_raw_json(QUOTE_SUMMARY_URL + "/#{symbol}", user_agent_headers=user_agent_headers, params=params_dict)
|
131
131
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} result = #{result.inspect}" }
|
132
132
|
rescue Exception => e
|
133
|
-
Logger.new(STDOUT).error("ERROR: #{e.message}")
|
133
|
+
# Logger.new(STDOUT).error("ERROR: #{e.message}")
|
134
134
|
return nil
|
135
135
|
end
|
136
136
|
return result
|
@@ -47,47 +47,47 @@ class YfAsDataframe
|
|
47
47
|
def symbol; @ticker; end
|
48
48
|
|
49
49
|
def shares_full(start: nil, fin: nil)
|
50
|
-
logger = Logger.new(STDOUT)
|
50
|
+
# logger = Logger.new(STDOUT)
|
51
51
|
|
52
52
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} start = #{start.inspect}, fin = #{fin.inspect}" }
|
53
53
|
|
54
54
|
if start
|
55
55
|
start_ts = YfAsDataframe::Utils.parse_user_dt(start, tz)
|
56
56
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} start_ts = #{start_ts}" }
|
57
|
-
start = Time.at(start_ts)
|
57
|
+
start = Time.at(start_ts)
|
58
58
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} start = #{start.inspect}, fin = #{fin.inspect}" }
|
59
59
|
end
|
60
60
|
if fin
|
61
61
|
end_ts = YfAsDataframe::Utils.parse_user_dt(fin, tz)
|
62
62
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} end_ts = #{end_ts}" }
|
63
|
-
fin = Time.at(end_ts)
|
63
|
+
fin = Time.at(end_ts)
|
64
64
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} start = #{start.inspect}, fin = #{fin.inspect}" }
|
65
65
|
end
|
66
66
|
|
67
67
|
# Rails.logger.info { "#{__FILE__}:#{__LINE__} start = #{start.inspect}, fin = #{fin.inspect}" }
|
68
68
|
|
69
|
-
dt_now = Time.now
|
69
|
+
dt_now = Time.now
|
70
70
|
fin ||= dt_now
|
71
|
-
start ||= (fin - 548
|
71
|
+
start ||= Time.new(fin.year, fin.month, fin.day) - 548*24*60*60
|
72
72
|
|
73
73
|
if start >= fin
|
74
|
-
logger.error("Start date (#{start}) must be before end (#{fin})")
|
74
|
+
# logger.error("Start date (#{start}) must be before end (#{fin})")
|
75
75
|
return nil
|
76
76
|
end
|
77
77
|
|
78
78
|
ts_url_base = "https://query2.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/#{@ticker}?symbol=#{@ticker}"
|
79
|
-
shares_url = "#{ts_url_base}&period1=#{start.to_i}&period2=#{fin.
|
79
|
+
shares_url = "#{ts_url_base}&period1=#{Time.new(start.year, start.month, start.day).to_i}&period2=#{Time.new((fin + 86400).year, (fin + 86400).month, (fin + 86400).day).to_i}"
|
80
80
|
|
81
81
|
begin
|
82
82
|
json_data = get(shares_url).parsed_response
|
83
83
|
rescue #_json.JSONDecodeError, requests.exceptions.RequestException
|
84
|
-
logger.error("#{@ticker}: Yahoo web request for share count failed")
|
84
|
+
# logger.error("#{@ticker}: Yahoo web request for share count failed")
|
85
85
|
return nil
|
86
86
|
end
|
87
87
|
|
88
88
|
fail = json_data["finance"]["error"]["code"] == "Bad Request" rescue false
|
89
89
|
if fail
|
90
|
-
logger.error("#{@ticker}: Yahoo web request for share count failed")
|
90
|
+
# logger.error("#{@ticker}: Yahoo web request for share count failed")
|
91
91
|
return nil
|
92
92
|
end
|
93
93
|
|
@@ -95,7 +95,7 @@ class YfAsDataframe
|
|
95
95
|
|
96
96
|
return nil if !shares_data[0].key?("shares_out")
|
97
97
|
|
98
|
-
timestamps = shares_data[0]["timestamp"].map{|t| Time.at(t)
|
98
|
+
timestamps = shares_data[0]["timestamp"].map{|t| Time.at(t) }
|
99
99
|
|
100
100
|
df = Polars::DataFrame.new(
|
101
101
|
{
|
@@ -153,7 +153,7 @@ class YfAsDataframe
|
|
153
153
|
# """
|
154
154
|
return @earnings_dates[limit] if @earnings_dates && @earnings_dates[limit]
|
155
155
|
|
156
|
-
logger = Logger.new(STDOUT)
|
156
|
+
# logger = Logger.new(STDOUT)
|
157
157
|
|
158
158
|
page_size = [limit, 100].min # YF caps at 100, don't go higher
|
159
159
|
page_offset = 0
|
@@ -247,13 +247,17 @@ class YfAsDataframe
|
|
247
247
|
|
248
248
|
def self.parse_user_dt(dt, exchange_tz)
|
249
249
|
if dt.is_a?(Integer)
|
250
|
-
Time.at(dt)
|
250
|
+
return Time.at(dt)
|
251
251
|
elsif dt.is_a?(String)
|
252
|
-
dt = DateTime.strptime(dt.to_s, '%Y-%m-%d')
|
252
|
+
dt = DateTime.strptime(dt.to_s, '%Y-%m-%d')
|
253
253
|
elsif dt.is_a?(Date)
|
254
|
-
dt = dt.to_datetime
|
255
|
-
|
256
|
-
|
254
|
+
dt = dt.to_datetime
|
255
|
+
end
|
256
|
+
# If it's a DateTime, convert to Time
|
257
|
+
if dt.is_a?(DateTime)
|
258
|
+
# If zone is nil, try to set it, else just convert
|
259
|
+
dt = dt.in_time_zone(exchange_tz) if dt.zone.nil? && dt.respond_to?(:in_time_zone)
|
260
|
+
dt = dt.to_time
|
257
261
|
end
|
258
262
|
dt.to_i
|
259
263
|
end
|
@@ -334,7 +338,7 @@ class YfAsDataframe
|
|
334
338
|
when '4wk'
|
335
339
|
28.days
|
336
340
|
else
|
337
|
-
Logger.new(STDOUT).warn { "#{__FILE__}:#{__LINE__} #{interval} not a recognized interval" }
|
341
|
+
# Logger.new(STDOUT).warn { "#{__FILE__}:#{__LINE__} #{interval} not a recognized interval" }
|
338
342
|
interval
|
339
343
|
end
|
340
344
|
end
|
@@ -375,7 +379,7 @@ def attributes(obj)
|
|
375
379
|
end
|
376
380
|
|
377
381
|
def print_once(msg)
|
378
|
-
puts msg
|
382
|
+
# puts msg
|
379
383
|
end
|
380
384
|
|
381
385
|
def get_yf_logger
|
@@ -388,7 +392,7 @@ def setup_debug_formatting
|
|
388
392
|
|
389
393
|
return unless logger.level == Logger::DEBUG
|
390
394
|
|
391
|
-
|
395
|
+
# logger.formatter = MultiLineFormatter.new('%(levelname)-8s %(message)s')
|
392
396
|
end
|
393
397
|
|
394
398
|
def enable_debug_mode
|
@@ -265,7 +265,7 @@ class YfAsDataframe
|
|
265
265
|
@@cookie = nil
|
266
266
|
# Clear curl-impersonate executables cache to force re-selection
|
267
267
|
CurlImpersonateIntegration.instance_variable_set(:@available_executables, nil)
|
268
|
-
warn "[yf_as_dataframe] Retrying crumb fetch (attempt #{attempt + 1}/3)"
|
268
|
+
# warn "[yf_as_dataframe] Retrying crumb fetch (attempt #{attempt + 1}/3)"
|
269
269
|
# Add delay between retries to be respectful of rate limits
|
270
270
|
sleep(2 ** attempt) # Exponential backoff: 2s, 4s, 8s
|
271
271
|
end
|
@@ -283,20 +283,20 @@ class YfAsDataframe
|
|
283
283
|
|
284
284
|
# Validate crumb: must be short, alphanumeric, no spaces, not an error message
|
285
285
|
if crumb_valid?(@@crumb)
|
286
|
-
warn "[yf_as_dataframe] Successfully fetched valid crumb on attempt #{attempt + 1}"
|
286
|
+
# warn "[yf_as_dataframe] Successfully fetched valid crumb on attempt #{attempt + 1}"
|
287
287
|
return @@crumb
|
288
288
|
else
|
289
|
-
warn "[yf_as_dataframe] Invalid crumb received on attempt #{attempt + 1}: '#{@@crumb.inspect}'"
|
289
|
+
# warn "[yf_as_dataframe] Invalid crumb received on attempt #{attempt + 1}: '#{@@crumb.inspect}'"
|
290
290
|
@@crumb = nil
|
291
291
|
end
|
292
292
|
rescue => e
|
293
|
-
warn "[yf_as_dataframe] Error fetching crumb on attempt #{attempt + 1}: #{e.message}"
|
293
|
+
# warn "[yf_as_dataframe] Error fetching crumb on attempt #{attempt + 1}: #{e.message}"
|
294
294
|
@@crumb = nil
|
295
295
|
end
|
296
296
|
end
|
297
297
|
|
298
298
|
# All attempts failed
|
299
|
-
warn "[yf_as_dataframe] Failed to fetch valid crumb after 3 attempts"
|
299
|
+
# warn "[yf_as_dataframe] Failed to fetch valid crumb after 3 attempts"
|
300
300
|
raise "Could not fetch a valid Yahoo Finance crumb after 3 attempts"
|
301
301
|
end
|
302
302
|
|
@@ -441,7 +441,7 @@ class YfAsDataframe
|
|
441
441
|
def refresh_session_if_needed
|
442
442
|
return unless session_needs_refresh?
|
443
443
|
|
444
|
-
warn "[yf_as_dataframe] Refreshing session (age: #{session_age} seconds, requests: #{@@request_count})"
|
444
|
+
# warn "[yf_as_dataframe] Refreshing session (age: #{session_age} seconds, requests: #{@@request_count})"
|
445
445
|
refresh_session
|
446
446
|
end
|
447
447
|
|
@@ -461,7 +461,7 @@ class YfAsDataframe
|
|
461
461
|
@@crumb = nil
|
462
462
|
@@session_created_at = Time.now
|
463
463
|
@@request_count = 0
|
464
|
-
warn "[yf_as_dataframe] Session refreshed"
|
464
|
+
# warn "[yf_as_dataframe] Session refreshed"
|
465
465
|
end
|
466
466
|
|
467
467
|
# Circuit breaker methods
|
@@ -472,7 +472,7 @@ class YfAsDataframe
|
|
472
472
|
when :open
|
473
473
|
if Time.now - @@last_failure_time > @@circuit_breaker_timeout
|
474
474
|
@@circuit_breaker_state = :half_open
|
475
|
-
warn "[yf_as_dataframe] Circuit breaker transitioning to half-open"
|
475
|
+
# warn "[yf_as_dataframe] Circuit breaker transitioning to half-open"
|
476
476
|
true
|
477
477
|
else
|
478
478
|
false
|
@@ -490,7 +490,7 @@ class YfAsDataframe
|
|
490
490
|
@@circuit_breaker_state = :open
|
491
491
|
# Exponential backoff: 60s, 120s, 240s, 480s, etc.
|
492
492
|
@@circuit_breaker_timeout = @@circuit_breaker_base_timeout * (2 ** (@@failure_count - @@circuit_breaker_threshold))
|
493
|
-
warn "[yf_as_dataframe] Circuit breaker opened after #{@@failure_count} failures (timeout: #{@@circuit_breaker_timeout}s)"
|
493
|
+
# warn "[yf_as_dataframe] Circuit breaker opened after #{@@failure_count} failures (timeout: #{@@circuit_breaker_timeout}s)"
|
494
494
|
end
|
495
495
|
end
|
496
496
|
|
@@ -499,7 +499,7 @@ class YfAsDataframe
|
|
499
499
|
@@circuit_breaker_state = :closed
|
500
500
|
@@failure_count = 0
|
501
501
|
@@circuit_breaker_timeout = @@circuit_breaker_base_timeout
|
502
|
-
warn "[yf_as_dataframe] Circuit breaker closed after successful request"
|
502
|
+
# warn "[yf_as_dataframe] Circuit breaker closed after successful request"
|
503
503
|
elsif @@circuit_breaker_state == :closed
|
504
504
|
# Reset failure count on success
|
505
505
|
@@failure_count = 0
|
@@ -538,10 +538,10 @@ class YfAsDataframe
|
|
538
538
|
begin
|
539
539
|
data = JSON.parse(json_blob)
|
540
540
|
crumb = data.dig('context', 'dispatcher', 'stores', 'CrumbStore', 'crumb')
|
541
|
-
warn "[yf_as_dataframe] Scraped crumb from quote page: #{crumb.inspect}"
|
541
|
+
# warn "[yf_as_dataframe] Scraped crumb from quote page: #{crumb.inspect}"
|
542
542
|
return crumb
|
543
543
|
rescue => e
|
544
|
-
warn "[yf_as_dataframe] Failed to parse crumb from quote page: #{e.message}"
|
544
|
+
# warn "[yf_as_dataframe] Failed to parse crumb from quote page: #{e.message}"
|
545
545
|
return nil
|
546
546
|
end
|
547
547
|
end
|
@@ -12,19 +12,26 @@ class YfAsDataframe
|
|
12
12
|
# Override get method to use curl-impersonate by default
|
13
13
|
def get(url, headers=nil, params=nil)
|
14
14
|
# Debug output
|
15
|
-
puts "DEBUG: curl_impersonate_enabled = #{CurlImpersonateIntegration.curl_impersonate_enabled}"
|
16
|
-
puts "DEBUG: curl_impersonate_fallback = #{CurlImpersonateIntegration.curl_impersonate_fallback}"
|
15
|
+
# puts "DEBUG: curl_impersonate_enabled = #{CurlImpersonateIntegration.curl_impersonate_enabled}"
|
16
|
+
# puts "DEBUG: curl_impersonate_fallback = #{CurlImpersonateIntegration.curl_impersonate_fallback}"
|
17
17
|
|
18
18
|
# Try curl-impersonate first if enabled
|
19
19
|
if CurlImpersonateIntegration.curl_impersonate_enabled
|
20
|
-
puts "DEBUG: Trying curl-impersonate..."
|
20
|
+
# puts "DEBUG: Trying curl-impersonate..."
|
21
21
|
begin
|
22
22
|
# Prepare headers and params as in original method
|
23
23
|
headers ||= {}
|
24
24
|
params ||= {}
|
25
|
-
|
26
|
-
|
27
|
-
|
25
|
+
|
26
|
+
# Only fetch crumb for /v7/finance/download endpoint
|
27
|
+
crumb_needed = url.include?('/v7/finance/download')
|
28
|
+
if crumb_needed
|
29
|
+
crumb = get_crumb_scrape_quote_page(params[:symbol] || params['symbol'])
|
30
|
+
params.merge!(crumb: crumb) unless crumb.nil?
|
31
|
+
end
|
32
|
+
|
33
|
+
cookie, _, strategy = _get_cookie_and_crumb(crumb_needed)
|
34
|
+
crumbs = {} # crumb logic handled above if needed
|
28
35
|
|
29
36
|
# Prepare headers for curl-impersonate
|
30
37
|
curl_headers = headers.dup.merge(@@user_agent_headers)
|
@@ -39,36 +46,36 @@ class YfAsDataframe
|
|
39
46
|
# Add crumb if available
|
40
47
|
curl_headers['crumb'] = crumb if crumb
|
41
48
|
|
42
|
-
# Make curl-impersonate request
|
49
|
+
# Make curl-impersonate request with improved timeout handling
|
43
50
|
response = CurlImpersonateIntegration.make_request(
|
44
51
|
url,
|
45
52
|
headers: curl_headers,
|
46
53
|
params: params.merge(crumbs),
|
47
|
-
timeout: CurlImpersonateIntegration.curl_impersonate_timeout
|
54
|
+
timeout: CurlImpersonateIntegration.curl_impersonate_timeout,
|
55
|
+
retries: CurlImpersonateIntegration.curl_impersonate_retries
|
48
56
|
)
|
49
57
|
|
50
|
-
if response && response.
|
51
|
-
puts "DEBUG: curl-impersonate succeeded"
|
58
|
+
if response && !response.empty?
|
59
|
+
# puts "DEBUG: curl-impersonate succeeded"
|
52
60
|
return response
|
53
61
|
else
|
54
|
-
puts "DEBUG: curl-impersonate returned nil or failed"
|
62
|
+
# puts "DEBUG: curl-impersonate returned nil or failed"
|
55
63
|
end
|
56
64
|
rescue => e
|
57
|
-
#
|
58
|
-
|
59
|
-
warn "curl-impersonate request failed: #{e.message}" if $VERBOSE
|
65
|
+
# puts "DEBUG: curl-impersonate exception: #{e.message}"
|
66
|
+
# warn "curl-impersonate request failed: #{e.message}" if $VERBOSE
|
60
67
|
end
|
61
68
|
else
|
62
|
-
puts "DEBUG: curl-impersonate is disabled, skipping to fallback"
|
69
|
+
# puts "DEBUG: curl-impersonate is disabled, skipping to fallback"
|
63
70
|
end
|
64
71
|
|
65
72
|
# Fallback to original HTTParty method
|
66
73
|
if CurlImpersonateIntegration.curl_impersonate_fallback
|
67
|
-
puts "DEBUG: Using HTTParty fallback"
|
68
|
-
|
74
|
+
# puts "DEBUG: Using HTTParty fallback"
|
75
|
+
return HTTParty.get(url, headers: headers).body
|
69
76
|
else
|
70
|
-
puts "DEBUG: Fallback is disabled, but forcing fallback anyway"
|
71
|
-
|
77
|
+
# puts "DEBUG: Fallback is disabled, but forcing fallback anyway"
|
78
|
+
return HTTParty.get(url, headers: headers).body
|
72
79
|
end
|
73
80
|
end
|
74
81
|
|
@@ -89,9 +96,33 @@ class YfAsDataframe
|
|
89
96
|
CurlImpersonateIntegration.curl_impersonate_timeout = timeout
|
90
97
|
end
|
91
98
|
|
99
|
+
def set_curl_impersonate_connect_timeout(timeout)
|
100
|
+
CurlImpersonateIntegration.curl_impersonate_connect_timeout = timeout
|
101
|
+
end
|
102
|
+
|
103
|
+
def set_curl_impersonate_process_timeout(timeout)
|
104
|
+
CurlImpersonateIntegration.curl_impersonate_process_timeout = timeout
|
105
|
+
end
|
106
|
+
|
107
|
+
def set_curl_impersonate_retries(retries)
|
108
|
+
CurlImpersonateIntegration.curl_impersonate_retries = retries
|
109
|
+
end
|
110
|
+
|
92
111
|
def get_available_curl_impersonate_executables
|
93
112
|
CurlImpersonateIntegration.available_executables
|
94
113
|
end
|
114
|
+
|
115
|
+
def get_curl_impersonate_config
|
116
|
+
{
|
117
|
+
enabled: CurlImpersonateIntegration.curl_impersonate_enabled,
|
118
|
+
fallback: CurlImpersonateIntegration.curl_impersonate_fallback,
|
119
|
+
timeout: CurlImpersonateIntegration.curl_impersonate_timeout,
|
120
|
+
connect_timeout: CurlImpersonateIntegration.curl_impersonate_connect_timeout,
|
121
|
+
process_timeout: CurlImpersonateIntegration.curl_impersonate_process_timeout,
|
122
|
+
retries: CurlImpersonateIntegration.curl_impersonate_retries,
|
123
|
+
retry_delay: CurlImpersonateIntegration.curl_impersonate_retry_delay
|
124
|
+
}
|
125
|
+
end
|
95
126
|
end
|
96
127
|
end
|
97
128
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: yf_as_dataframe
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bill McKinnon
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2025-
|
11
|
+
date: 2025-07-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: tzinfo
|
@@ -156,8 +156,6 @@ files:
|
|
156
156
|
- lib/yf_as_dataframe/yf_connection.rb
|
157
157
|
- lib/yf_as_dataframe/yf_connection_minimal_patch.rb
|
158
158
|
- lib/yf_as_dataframe/yfinance_exception.rb
|
159
|
-
- quick_test.rb
|
160
|
-
- test_minimal_integration.rb
|
161
159
|
homepage: https://www.github.com/bmck/yf_as_dataframe
|
162
160
|
licenses:
|
163
161
|
- MIT
|
@@ -180,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
180
178
|
- !ruby/object:Gem::Version
|
181
179
|
version: '0'
|
182
180
|
requirements: []
|
183
|
-
rubygems_version: 3.
|
181
|
+
rubygems_version: 3.1.6
|
184
182
|
signing_key:
|
185
183
|
specification_version: 4
|
186
184
|
summary: A shameless port of python's yfinance module to ruby
|
data/quick_test.rb
DELETED
@@ -1,143 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# Quick test for minimal curl-impersonate integration
|
4
|
-
# This test verifies the integration without making actual HTTP requests
|
5
|
-
|
6
|
-
puts "=== Quick Curl-Impersonate Integration Test ==="
|
7
|
-
puts
|
8
|
-
|
9
|
-
# Test 1: Check curl-impersonate integration module
|
10
|
-
puts "1. Testing curl-impersonate integration module..."
|
11
|
-
begin
|
12
|
-
require_relative 'lib/yf_as_dataframe/curl_impersonate_integration'
|
13
|
-
|
14
|
-
executables = YfAsDataframe::CurlImpersonateIntegration.available_executables
|
15
|
-
if executables.empty?
|
16
|
-
puts " ❌ No curl-impersonate executables found!"
|
17
|
-
exit 1
|
18
|
-
else
|
19
|
-
puts " ✅ Found #{executables.length} curl-impersonate executables"
|
20
|
-
puts " Sample: #{executables.first[:executable]} (#{executables.first[:browser]})"
|
21
|
-
end
|
22
|
-
rescue => e
|
23
|
-
puts " ❌ Error loading integration module: #{e.message}"
|
24
|
-
exit 1
|
25
|
-
end
|
26
|
-
|
27
|
-
puts
|
28
|
-
|
29
|
-
# Test 2: Test executable selection
|
30
|
-
puts "2. Testing executable selection..."
|
31
|
-
begin
|
32
|
-
executable = YfAsDataframe::CurlImpersonateIntegration.get_random_executable
|
33
|
-
if executable
|
34
|
-
puts " ✅ Random executable selected: #{executable[:executable]} (#{executable[:browser]})"
|
35
|
-
else
|
36
|
-
puts " ❌ No executable selected"
|
37
|
-
end
|
38
|
-
rescue => e
|
39
|
-
puts " ❌ Error selecting executable: #{e.message}"
|
40
|
-
end
|
41
|
-
|
42
|
-
puts
|
43
|
-
|
44
|
-
# Test 3: Test environment variable functionality
|
45
|
-
puts "3. Testing environment variable functionality..."
|
46
|
-
begin
|
47
|
-
default_dir = YfAsDataframe::CurlImpersonateIntegration.executable_directory
|
48
|
-
puts " ✅ Default directory: #{default_dir}"
|
49
|
-
|
50
|
-
# Test with a custom directory (should still use default if not set)
|
51
|
-
old_env = ENV['CURL_IMPERSONATE_DIR']
|
52
|
-
ENV['CURL_IMPERSONATE_DIR'] = '/nonexistent/path'
|
53
|
-
|
54
|
-
# Clear the cached executables to force re-discovery
|
55
|
-
YfAsDataframe::CurlImpersonateIntegration.instance_variable_set(:@available_executables, nil)
|
56
|
-
|
57
|
-
custom_dir = YfAsDataframe::CurlImpersonateIntegration.executable_directory
|
58
|
-
puts " ✅ Custom directory (set): #{custom_dir}"
|
59
|
-
|
60
|
-
# Restore original environment
|
61
|
-
if old_env
|
62
|
-
ENV['CURL_IMPERSONATE_DIR'] = old_env
|
63
|
-
else
|
64
|
-
ENV.delete('CURL_IMPERSONATE_DIR')
|
65
|
-
end
|
66
|
-
|
67
|
-
# Clear cache again
|
68
|
-
YfAsDataframe::CurlImpersonateIntegration.instance_variable_set(:@available_executables, nil)
|
69
|
-
|
70
|
-
restored_dir = YfAsDataframe::CurlImpersonateIntegration.executable_directory
|
71
|
-
puts " ✅ Restored directory: #{restored_dir}"
|
72
|
-
|
73
|
-
rescue => e
|
74
|
-
puts " ❌ Error testing environment variable: #{e.message}"
|
75
|
-
end
|
76
|
-
|
77
|
-
puts
|
78
|
-
|
79
|
-
# Test 4: Test minimal patch loading
|
80
|
-
puts "4. Testing minimal patch structure..."
|
81
|
-
begin
|
82
|
-
# This would normally require the full YfConnection class
|
83
|
-
# For this test, we'll just verify the patch file loads
|
84
|
-
require_relative 'lib/yf_as_dataframe/curl_impersonate_integration'
|
85
|
-
require_relative 'lib/yf_as_dataframe/yf_connection_minimal_patch'
|
86
|
-
|
87
|
-
puts " ✅ Minimal patch files load successfully"
|
88
|
-
puts " ✅ Integration module is available"
|
89
|
-
rescue => e
|
90
|
-
puts " ❌ Error loading minimal patch: #{e.message}"
|
91
|
-
end
|
92
|
-
|
93
|
-
puts
|
94
|
-
|
95
|
-
# Test 5: Test configuration
|
96
|
-
puts "5. Testing configuration..."
|
97
|
-
begin
|
98
|
-
puts " ✅ Configuration methods available:"
|
99
|
-
puts " - enable_curl_impersonate"
|
100
|
-
puts " - enable_curl_impersonate_fallback"
|
101
|
-
puts " - set_curl_impersonate_timeout"
|
102
|
-
puts " - get_available_curl_impersonate_executables"
|
103
|
-
|
104
|
-
# Test setting configuration
|
105
|
-
YfAsDataframe::CurlImpersonateIntegration.curl_impersonate_timeout = 20
|
106
|
-
puts " ✅ Configuration can be modified"
|
107
|
-
rescue => e
|
108
|
-
puts " ❌ Error with configuration: #{e.message}"
|
109
|
-
end
|
110
|
-
|
111
|
-
puts
|
112
|
-
|
113
|
-
# Test 6: Test command building (without execution)
|
114
|
-
puts "6. Testing command building..."
|
115
|
-
begin
|
116
|
-
executable = YfAsDataframe::CurlImpersonateIntegration.get_random_executable
|
117
|
-
if executable
|
118
|
-
# Build a command without executing it
|
119
|
-
cmd = [executable[:path], "--max-time", "5", "https://httpbin.org/get"]
|
120
|
-
puts " ✅ Command built: #{cmd.join(' ')}"
|
121
|
-
else
|
122
|
-
puts " ❌ Could not build command"
|
123
|
-
end
|
124
|
-
rescue => e
|
125
|
-
puts " ❌ Error building command: #{e.message}"
|
126
|
-
end
|
127
|
-
|
128
|
-
puts
|
129
|
-
puts "=== Quick Test Summary ==="
|
130
|
-
puts "✅ Integration module loads successfully"
|
131
|
-
puts "✅ Executables are detected"
|
132
|
-
puts "✅ Environment variable functionality works"
|
133
|
-
puts "✅ Configuration works"
|
134
|
-
puts "✅ Patch files load without errors"
|
135
|
-
puts
|
136
|
-
puts "The minimal curl-impersonate integration is ready for use!"
|
137
|
-
puts
|
138
|
-
puts "To integrate with your code:"
|
139
|
-
puts "require 'yf_as_dataframe/curl_impersonate_integration'"
|
140
|
-
puts "require 'yf_as_dataframe/yf_connection_minimal_patch'"
|
141
|
-
puts
|
142
|
-
puts "Environment variable support:"
|
143
|
-
puts "export CURL_IMPERSONATE_DIR='/custom/path' # Optional"
|
data/test_minimal_integration.rb
DELETED
@@ -1,121 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
# Test script for minimal curl-impersonate integration
|
4
|
-
# This tests the approach where curl-impersonate is the default behavior
|
5
|
-
|
6
|
-
puts "=== Minimal Curl-Impersonate Integration Test ==="
|
7
|
-
puts
|
8
|
-
|
9
|
-
# Test 1: Check curl-impersonate integration module
|
10
|
-
puts "1. Testing curl-impersonate integration module..."
|
11
|
-
begin
|
12
|
-
require_relative 'lib/yf_as_dataframe/curl_impersonate_integration'
|
13
|
-
|
14
|
-
executables = YfAsDataframe::CurlImpersonateIntegration.available_executables
|
15
|
-
if executables.empty?
|
16
|
-
puts " ❌ No curl-impersonate executables found!"
|
17
|
-
exit 1
|
18
|
-
else
|
19
|
-
puts " ✅ Found #{executables.length} curl-impersonate executables"
|
20
|
-
puts " Sample: #{executables.first[:executable]} (#{executables.first[:browser]})"
|
21
|
-
end
|
22
|
-
rescue => e
|
23
|
-
puts " ❌ Error loading integration module: #{e.message}"
|
24
|
-
exit 1
|
25
|
-
end
|
26
|
-
|
27
|
-
puts
|
28
|
-
|
29
|
-
# Test 2: Test direct curl-impersonate request with short timeout
|
30
|
-
puts "2. Testing direct curl-impersonate request..."
|
31
|
-
begin
|
32
|
-
response = YfAsDataframe::CurlImpersonateIntegration.make_request(
|
33
|
-
"https://httpbin.org/get",
|
34
|
-
headers: { "User-Agent" => "Test-Agent" },
|
35
|
-
timeout: 10 # 10 second timeout
|
36
|
-
)
|
37
|
-
|
38
|
-
if response && response.success?
|
39
|
-
puts " ✅ Direct curl-impersonate request successful"
|
40
|
-
puts " Response length: #{response.body.length} characters"
|
41
|
-
else
|
42
|
-
puts " ❌ Direct curl-impersonate request failed"
|
43
|
-
end
|
44
|
-
rescue => e
|
45
|
-
puts " ❌ Error with direct request: #{e.message}"
|
46
|
-
end
|
47
|
-
|
48
|
-
puts
|
49
|
-
|
50
|
-
# Test 3: Test minimal patch (without full gem)
|
51
|
-
puts "3. Testing minimal patch structure..."
|
52
|
-
begin
|
53
|
-
# This would normally require the full YfConnection class
|
54
|
-
# For this test, we'll just verify the patch file loads
|
55
|
-
require_relative 'lib/yf_as_dataframe/curl_impersonate_integration'
|
56
|
-
require_relative 'lib/yf_as_dataframe/yf_connection_minimal_patch'
|
57
|
-
|
58
|
-
puts " ✅ Minimal patch files load successfully"
|
59
|
-
puts " ✅ Integration module is available"
|
60
|
-
rescue => e
|
61
|
-
puts " ❌ Error loading minimal patch: #{e.message}"
|
62
|
-
end
|
63
|
-
|
64
|
-
puts
|
65
|
-
|
66
|
-
# Test 4: Test configuration methods
|
67
|
-
puts "4. Testing configuration methods..."
|
68
|
-
begin
|
69
|
-
# Test configuration (these would work with the full YfConnection class)
|
70
|
-
puts " ✅ Configuration methods available:"
|
71
|
-
puts " - enable_curl_impersonate"
|
72
|
-
puts " - enable_curl_impersonate_fallback"
|
73
|
-
puts " - set_curl_impersonate_timeout"
|
74
|
-
puts " - get_available_curl_impersonate_executables"
|
75
|
-
rescue => e
|
76
|
-
puts " ❌ Error with configuration: #{e.message}"
|
77
|
-
end
|
78
|
-
|
79
|
-
puts
|
80
|
-
|
81
|
-
# Test 5: Test with Yahoo Finance endpoint with short timeout
|
82
|
-
puts "5. Testing Yahoo Finance endpoint..."
|
83
|
-
begin
|
84
|
-
response = YfAsDataframe::CurlImpersonateIntegration.make_request(
|
85
|
-
"https://query1.finance.yahoo.com/v8/finance/chart/MSFT",
|
86
|
-
params: { "interval" => "1d", "range" => "1d" },
|
87
|
-
timeout: 15 # 15 second timeout
|
88
|
-
)
|
89
|
-
|
90
|
-
if response && response.success?
|
91
|
-
puts " ✅ Yahoo Finance request successful"
|
92
|
-
puts " Response length: #{response.body.length} characters"
|
93
|
-
|
94
|
-
if response.body.strip.start_with?('{') && response.body.include?('"chart"')
|
95
|
-
puts " ✅ Response appears to be valid Yahoo Finance JSON"
|
96
|
-
else
|
97
|
-
puts " ⚠️ Response format unexpected"
|
98
|
-
end
|
99
|
-
else
|
100
|
-
puts " ❌ Yahoo Finance request failed"
|
101
|
-
end
|
102
|
-
rescue => e
|
103
|
-
puts " ❌ Error with Yahoo Finance: #{e.message}"
|
104
|
-
end
|
105
|
-
|
106
|
-
puts
|
107
|
-
puts "=== Test Summary ==="
|
108
|
-
puts "The minimal curl-impersonate integration is ready."
|
109
|
-
puts
|
110
|
-
puts "To use with the full gem:"
|
111
|
-
puts "1. Add the two integration files to lib/yf_as_dataframe/"
|
112
|
-
puts "2. Add require statements to your code"
|
113
|
-
puts "3. Your existing code will automatically use curl-impersonate"
|
114
|
-
puts
|
115
|
-
puts "Files needed:"
|
116
|
-
puts "- lib/yf_as_dataframe/curl_impersonate_integration.rb"
|
117
|
-
puts "- lib/yf_as_dataframe/yf_connection_minimal_patch.rb"
|
118
|
-
puts
|
119
|
-
puts "Integration code:"
|
120
|
-
puts "require 'yf_as_dataframe/curl_impersonate_integration'"
|
121
|
-
puts "require 'yf_as_dataframe/yf_connection_minimal_patch'"
|