zenrows 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/zenrows/proxy.rb +30 -6
- data/lib/zenrows/version.rb +1 -1
- data/sig/zenrows/proxy.rbs +1 -0
- data/test/zenrows/hooks_thread_safety_test.rb +146 -0
- data/test/zenrows/proxy_test.rb +50 -0
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 90e01f2bc9ea6afa174e3e0f5c94856c3ec6767b987da51ff61eebd8dda07184
|
|
4
|
+
data.tar.gz: 6d755c3e3822639e8b7f3b16e258a06e372dd71d4438adb883ff75fbbc7d0223
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b387107cb2b90c8395180764f8bcc50b025722068eb0eb755eeeb169d19044e3cfdebcd0a3ba13ffc5654403013972803a62afeb2964f074bbaeb5ab8e5273d9
|
|
7
|
+
data.tar.gz: ec6134a37c804d3f2cef3919211f38dc0d17f7c617823c9c157ecee10cbd74062f348fabefda1fb6ee60039d672136c23d44ae250668544789ca7491aff52f03
|
data/lib/zenrows/proxy.rb
CHANGED
|
@@ -25,6 +25,9 @@ module Zenrows
|
|
|
25
25
|
# Valid sticky TTL values for HTTP proxy
|
|
26
26
|
VALID_STICKY_TTL = %w[30s 5m 30m 1h 1d].freeze
|
|
27
27
|
|
|
28
|
+
# Valid mode values for adaptive stealth mode
|
|
29
|
+
VALID_MODES = %w[auto].freeze
|
|
30
|
+
|
|
28
31
|
# Valid region codes for HTTP proxy
|
|
29
32
|
VALID_REGIONS = {
|
|
30
33
|
"africa" => "af",
|
|
@@ -82,6 +85,7 @@ module Zenrows
|
|
|
82
85
|
# @option options [String] :device Device emulation ('mobile' or 'desktop')
|
|
83
86
|
# @option options [Boolean] :antibot Enhanced antibot bypass mode
|
|
84
87
|
# @option options [String] :session_ttl Session duration ('30s', '5m', '30m', '1h', '1d')
|
|
88
|
+
# @option options [String] :mode Adaptive stealth mode ('auto')
|
|
85
89
|
# @return [Hash] Proxy configuration with :host, :port, :username, :password
|
|
86
90
|
# @raise [WaitTimeError] if wait time exceeds 3 minutes
|
|
87
91
|
# @raise [ArgumentError] if session_ttl is invalid
|
|
@@ -125,14 +129,34 @@ module Zenrows
|
|
|
125
129
|
def build_params(opts)
|
|
126
130
|
params = {}
|
|
127
131
|
|
|
132
|
+
# Adaptive stealth mode handling
|
|
133
|
+
if opts[:mode]
|
|
134
|
+
mode = opts[:mode].to_s.downcase
|
|
135
|
+
unless VALID_MODES.include?(mode)
|
|
136
|
+
raise ArgumentError, "Invalid mode: #{mode}. Valid values: #{VALID_MODES.join(", ")}"
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# mode=auto manages js_render and premium_proxy - warn if user tries to set them
|
|
140
|
+
if mode == "auto"
|
|
141
|
+
if opts[:js_render]
|
|
142
|
+
warn "[Zenrows] Warning: js_render is ignored when mode=auto (auto-managed)"
|
|
143
|
+
end
|
|
144
|
+
if opts[:premium_proxy]
|
|
145
|
+
warn "[Zenrows] Warning: premium_proxy is ignored when mode=auto (auto-managed)"
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
params[:mode] = mode
|
|
150
|
+
end
|
|
151
|
+
|
|
128
152
|
# Custom headers must be enabled first if we'll use headers
|
|
129
153
|
params[:custom_headers] = true if opts[:custom_headers]
|
|
130
154
|
|
|
131
|
-
# JavaScript rendering
|
|
132
|
-
params[:js_render] = true if opts[:js_render]
|
|
155
|
+
# JavaScript rendering (skip if mode=auto manages it)
|
|
156
|
+
params[:js_render] = true if opts[:js_render] && !params[:mode]
|
|
133
157
|
|
|
134
|
-
# Premium proxy
|
|
135
|
-
params[:premium_proxy] = true if opts[:premium_proxy]
|
|
158
|
+
# Premium proxy (skip if mode=auto manages it)
|
|
159
|
+
params[:premium_proxy] = true if opts[:premium_proxy] && !params[:mode]
|
|
136
160
|
|
|
137
161
|
# Wait time handling
|
|
138
162
|
if opts[:wait]
|
|
@@ -204,8 +228,8 @@ module Zenrows
|
|
|
204
228
|
params[:session_ttl] = ttl
|
|
205
229
|
end
|
|
206
230
|
|
|
207
|
-
# Auto-enable js_render if needed
|
|
208
|
-
params[:js_render] = true if requires_js_render?(params)
|
|
231
|
+
# Auto-enable js_render if needed (skip if mode=auto manages it)
|
|
232
|
+
params[:js_render] = true if requires_js_render?(params) && !params[:mode]
|
|
209
233
|
|
|
210
234
|
params
|
|
211
235
|
end
|
data/lib/zenrows/version.rb
CHANGED
data/sig/zenrows/proxy.rbs
CHANGED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "test_helper"
|
|
4
|
+
|
|
5
|
+
class HooksThreadSafetyTest < Minitest::Test
|
|
6
|
+
def setup
|
|
7
|
+
@hooks = Zenrows::Hooks.new
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def test_concurrent_registration
|
|
11
|
+
threads = 10.times.map do |i|
|
|
12
|
+
Thread.new do
|
|
13
|
+
10.times do |j|
|
|
14
|
+
@hooks.register(:on_response) { |_r, _c| "thread_#{i}_#{j}" }
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
threads.each(&:join)
|
|
20
|
+
|
|
21
|
+
# Should have 100 callbacks registered without corruption
|
|
22
|
+
refute_predicate @hooks, :empty?
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def test_concurrent_hook_execution
|
|
26
|
+
counter = 0
|
|
27
|
+
mutex = Mutex.new
|
|
28
|
+
|
|
29
|
+
@hooks.register(:on_response) do |_r, _c|
|
|
30
|
+
mutex.synchronize { counter += 1 }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
threads = 20.times.map do
|
|
34
|
+
Thread.new do
|
|
35
|
+
50.times { @hooks.run(:on_response, nil, {}) }
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
threads.each(&:join)
|
|
40
|
+
|
|
41
|
+
assert_equal 1000, counter
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def test_concurrent_registration_and_execution
|
|
45
|
+
execution_count = 0
|
|
46
|
+
mutex = Mutex.new
|
|
47
|
+
|
|
48
|
+
@hooks.register(:on_response) do |_r, _c|
|
|
49
|
+
mutex.synchronize { execution_count += 1 }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
registrars = 5.times.map do
|
|
53
|
+
Thread.new do
|
|
54
|
+
10.times do
|
|
55
|
+
@hooks.register(:on_response) do |_r, _c|
|
|
56
|
+
mutex.synchronize { execution_count += 1 }
|
|
57
|
+
end
|
|
58
|
+
sleep 0.001
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
executors = 5.times.map do
|
|
64
|
+
Thread.new do
|
|
65
|
+
20.times do
|
|
66
|
+
@hooks.run(:on_response, nil, {})
|
|
67
|
+
sleep 0.001
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
(registrars + executors).each(&:join)
|
|
73
|
+
|
|
74
|
+
assert_operator execution_count, :>, 0
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def test_concurrent_subscriber_addition
|
|
78
|
+
subscribers = 10.times.map do |i|
|
|
79
|
+
subscriber = Object.new
|
|
80
|
+
subscriber.define_singleton_method(:on_response) { |_r, _c| "sub_#{i}" }
|
|
81
|
+
subscriber
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
threads = subscribers.map do |sub|
|
|
85
|
+
Thread.new { @hooks.add_subscriber(sub) }
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
threads.each(&:join)
|
|
89
|
+
|
|
90
|
+
refute_predicate @hooks, :empty?
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def test_concurrent_around_hooks
|
|
94
|
+
@hooks.register(:around_request) do |_ctx, &block|
|
|
95
|
+
block.call
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
threads = 10.times.map do
|
|
99
|
+
Thread.new do
|
|
100
|
+
@hooks.run_around({}) { :result }
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
results = threads.map(&:value)
|
|
105
|
+
|
|
106
|
+
assert results.all? { |r| r == :result }
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def test_dup_is_thread_safe
|
|
110
|
+
@hooks.register(:on_response) { |_r, _c| "original" }
|
|
111
|
+
|
|
112
|
+
copies = []
|
|
113
|
+
mutex = Mutex.new
|
|
114
|
+
|
|
115
|
+
threads = 10.times.map do
|
|
116
|
+
Thread.new do
|
|
117
|
+
copy = @hooks.dup
|
|
118
|
+
mutex.synchronize { copies << copy }
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
threads.each(&:join)
|
|
123
|
+
|
|
124
|
+
assert_equal 10, copies.size
|
|
125
|
+
copies.each { |copy| refute_predicate copy, :empty? }
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def test_empty_check_during_concurrent_registration
|
|
129
|
+
results = []
|
|
130
|
+
mutex = Mutex.new
|
|
131
|
+
|
|
132
|
+
threads = 10.times.map do
|
|
133
|
+
Thread.new do
|
|
134
|
+
was_empty = @hooks.empty?
|
|
135
|
+
@hooks.register(:on_response) { |_r, _c| nil }
|
|
136
|
+
mutex.synchronize { results << was_empty }
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
threads.each(&:join)
|
|
141
|
+
|
|
142
|
+
# At least one thread should have seen empty? as true
|
|
143
|
+
assert_includes results, true
|
|
144
|
+
refute_predicate @hooks, :empty?
|
|
145
|
+
end
|
|
146
|
+
end
|
data/test/zenrows/proxy_test.rb
CHANGED
|
@@ -150,4 +150,54 @@ class ProxyTest < Minitest::Test
|
|
|
150
150
|
@proxy.build(session_ttl: "invalid")
|
|
151
151
|
end
|
|
152
152
|
end
|
|
153
|
+
|
|
154
|
+
def test_build_with_mode_auto
|
|
155
|
+
config = @proxy.build(mode: "auto")
|
|
156
|
+
|
|
157
|
+
assert_includes config[:password], "mode=auto"
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def test_mode_auto_does_not_include_js_render_or_premium_proxy
|
|
161
|
+
config = @proxy.build(mode: "auto")
|
|
162
|
+
|
|
163
|
+
refute_includes config[:password], "js_render"
|
|
164
|
+
refute_includes config[:password], "premium_proxy"
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def test_mode_auto_with_other_options
|
|
168
|
+
config = @proxy.build(mode: "auto", session_id: 123, wait_for: ".content")
|
|
169
|
+
|
|
170
|
+
assert_includes config[:password], "mode=auto"
|
|
171
|
+
assert_includes config[:password], "session_id=123"
|
|
172
|
+
assert_includes config[:password], "wait_for=.content"
|
|
173
|
+
refute_includes config[:password], "js_render"
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def test_mode_auto_warns_when_js_render_set
|
|
177
|
+
_out, err = capture_io do
|
|
178
|
+
config = @proxy.build(mode: "auto", js_render: true)
|
|
179
|
+
|
|
180
|
+
assert_includes config[:password], "mode=auto"
|
|
181
|
+
refute_includes config[:password], "js_render"
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
assert_match(/js_render is ignored/, err)
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def test_mode_auto_warns_when_premium_proxy_set
|
|
188
|
+
_out, err = capture_io do
|
|
189
|
+
config = @proxy.build(mode: "auto", premium_proxy: true)
|
|
190
|
+
|
|
191
|
+
assert_includes config[:password], "mode=auto"
|
|
192
|
+
refute_includes config[:password], "premium_proxy"
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
assert_match(/premium_proxy is ignored/, err)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def test_raises_on_invalid_mode
|
|
199
|
+
assert_raises ArgumentError do
|
|
200
|
+
@proxy.build(mode: "invalid")
|
|
201
|
+
end
|
|
202
|
+
end
|
|
153
203
|
end
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: zenrows
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ernest Bursa
|
|
8
8
|
bindir: exe
|
|
9
9
|
cert_chain: []
|
|
10
|
-
date:
|
|
10
|
+
date: 2026-01-03 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
13
|
name: http
|
|
@@ -104,6 +104,7 @@ files:
|
|
|
104
104
|
- test/zenrows/hooks/context_test.rb
|
|
105
105
|
- test/zenrows/hooks/log_subscriber_test.rb
|
|
106
106
|
- test/zenrows/hooks_test.rb
|
|
107
|
+
- test/zenrows/hooks_thread_safety_test.rb
|
|
107
108
|
- test/zenrows/instrumented_client_test.rb
|
|
108
109
|
- test/zenrows/js_instructions_test.rb
|
|
109
110
|
- test/zenrows/proxy_test.rb
|