zenrows 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/README.md +40 -0
- data/lib/zenrows/api_client.rb +70 -7
- data/lib/zenrows/backends/base.rb +31 -1
- data/lib/zenrows/backends/http_rb.rb +10 -2
- data/lib/zenrows/backends/net_http.rb +10 -2
- data/lib/zenrows/client.rb +86 -3
- data/lib/zenrows/configuration.rb +111 -0
- data/lib/zenrows/hooks/context.rb +142 -0
- data/lib/zenrows/hooks/log_subscriber.rb +124 -0
- data/lib/zenrows/hooks.rb +213 -0
- data/lib/zenrows/instrumented_client.rb +187 -0
- data/lib/zenrows/version.rb +1 -1
- data/lib/zenrows.rb +4 -0
- data/sig/zenrows/api_client.rbs +4 -1
- data/sig/zenrows/backends/base.rbs +4 -1
- data/sig/zenrows/client.rbs +2 -1
- data/sig/zenrows/configuration.rbs +9 -0
- data/sig/zenrows/hook_configurator.rbs +9 -0
- data/sig/zenrows/hooks/context.rbs +6 -0
- data/sig/zenrows/hooks/log_subscriber.rbs +15 -0
- data/sig/zenrows/hooks.rbs +23 -0
- data/sig/zenrows/instrumented_client.rbs +22 -0
- data/test/test_helper.rb +42 -0
- data/test/zenrows/client_hooks_test.rb +105 -0
- data/test/zenrows/configuration_hooks_test.rb +101 -0
- data/test/zenrows/hooks/context_test.rb +150 -0
- data/test/zenrows/hooks/log_subscriber_test.rb +105 -0
- data/test/zenrows/hooks_test.rb +215 -0
- data/test/zenrows/instrumented_client_test.rb +153 -0
- metadata +18 -3
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "test_helper"
|
|
4
|
+
|
|
5
|
+
class HooksTest < Minitest::Test
|
|
6
|
+
def setup
|
|
7
|
+
@hooks = Zenrows::Hooks.new
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def test_register_with_block
|
|
11
|
+
called = false
|
|
12
|
+
@hooks.register(:on_response) { called = true }
|
|
13
|
+
|
|
14
|
+
@hooks.run(:on_response, nil, {})
|
|
15
|
+
|
|
16
|
+
assert called
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def test_register_with_callable
|
|
20
|
+
called = false
|
|
21
|
+
callable = ->(resp, ctx) { called = true }
|
|
22
|
+
@hooks.register(:on_response, callable)
|
|
23
|
+
|
|
24
|
+
@hooks.run(:on_response, nil, {})
|
|
25
|
+
|
|
26
|
+
assert called
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def test_register_raises_on_invalid_event
|
|
30
|
+
assert_raises ArgumentError do
|
|
31
|
+
@hooks.register(:invalid_event) {}
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def test_register_raises_without_handler
|
|
36
|
+
assert_raises ArgumentError do
|
|
37
|
+
@hooks.register(:on_response)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def test_register_raises_on_non_callable
|
|
42
|
+
assert_raises ArgumentError do
|
|
43
|
+
@hooks.register(:on_response, "not callable")
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def test_multiple_callbacks_for_same_event
|
|
48
|
+
results = []
|
|
49
|
+
@hooks.register(:on_response) { results << 1 }
|
|
50
|
+
@hooks.register(:on_response) { results << 2 }
|
|
51
|
+
|
|
52
|
+
@hooks.run(:on_response, nil, {})
|
|
53
|
+
|
|
54
|
+
assert_equal [1, 2], results
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def test_add_subscriber
|
|
58
|
+
subscriber = Object.new
|
|
59
|
+
def subscriber.on_response(resp, ctx)
|
|
60
|
+
@called = true
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def subscriber.called?
|
|
64
|
+
@called
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
@hooks.add_subscriber(subscriber)
|
|
68
|
+
@hooks.run(:on_response, nil, {})
|
|
69
|
+
|
|
70
|
+
assert_predicate subscriber, :called?
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def test_subscriber_with_multiple_methods
|
|
74
|
+
subscriber = Object.new
|
|
75
|
+
subscriber.instance_variable_set(:@calls, [])
|
|
76
|
+
def subscriber.before_request(ctx)
|
|
77
|
+
@calls << :before
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def subscriber.on_response(resp, ctx)
|
|
81
|
+
@calls << :response
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def subscriber.calls
|
|
85
|
+
@calls
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
@hooks.add_subscriber(subscriber)
|
|
89
|
+
@hooks.run(:before_request, {})
|
|
90
|
+
@hooks.run(:on_response, nil, {})
|
|
91
|
+
|
|
92
|
+
assert_equal [:before, :response], subscriber.calls
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def test_empty_when_no_hooks
|
|
96
|
+
assert_empty @hooks
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def test_not_empty_after_register
|
|
100
|
+
@hooks.register(:on_response) {}
|
|
101
|
+
|
|
102
|
+
refute_empty @hooks
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def test_not_empty_after_add_subscriber
|
|
106
|
+
@hooks.add_subscriber(Object.new)
|
|
107
|
+
|
|
108
|
+
refute_empty @hooks
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def test_dup_creates_independent_copy
|
|
112
|
+
@hooks.register(:on_response) {}
|
|
113
|
+
|
|
114
|
+
copy = @hooks.dup
|
|
115
|
+
copy.register(:on_error) {}
|
|
116
|
+
|
|
117
|
+
# Original should not have on_error
|
|
118
|
+
refute_empty @hooks
|
|
119
|
+
refute_empty copy
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def test_merge_combines_hooks
|
|
123
|
+
other = Zenrows::Hooks.new
|
|
124
|
+
results = []
|
|
125
|
+
|
|
126
|
+
@hooks.register(:on_response) { results << 1 }
|
|
127
|
+
other.register(:on_response) { results << 2 }
|
|
128
|
+
|
|
129
|
+
@hooks.merge(other)
|
|
130
|
+
@hooks.run(:on_response, nil, {})
|
|
131
|
+
|
|
132
|
+
assert_equal [1, 2], results
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def test_merge_with_nil_is_noop
|
|
136
|
+
@hooks.register(:on_response) {}
|
|
137
|
+
@hooks.merge(nil)
|
|
138
|
+
|
|
139
|
+
refute_empty @hooks
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def test_around_request_wraps_block
|
|
143
|
+
order = []
|
|
144
|
+
|
|
145
|
+
@hooks.register(:around_request) do |ctx, &block|
|
|
146
|
+
order << :before
|
|
147
|
+
result = block.call
|
|
148
|
+
order << :after
|
|
149
|
+
result
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
result = @hooks.run_around({}) do
|
|
153
|
+
order << :inside
|
|
154
|
+
:result
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
assert_equal [:before, :inside, :after], order
|
|
158
|
+
assert_equal :result, result
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def test_multiple_around_hooks_chain
|
|
162
|
+
order = []
|
|
163
|
+
|
|
164
|
+
@hooks.register(:around_request) do |ctx, &block|
|
|
165
|
+
order << :outer_before
|
|
166
|
+
result = block.call
|
|
167
|
+
order << :outer_after
|
|
168
|
+
result
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
@hooks.register(:around_request) do |ctx, &block|
|
|
172
|
+
order << :inner_before
|
|
173
|
+
result = block.call
|
|
174
|
+
order << :inner_after
|
|
175
|
+
result
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
@hooks.run_around({}) do
|
|
179
|
+
order << :core
|
|
180
|
+
:result
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
assert_equal [:outer_before, :inner_before, :core, :inner_after, :outer_after], order
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def test_run_around_without_hooks
|
|
187
|
+
result = @hooks.run_around({}) { :result }
|
|
188
|
+
|
|
189
|
+
assert_equal :result, result
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def test_valid_events
|
|
193
|
+
assert_equal [:before_request, :after_request, :on_response, :on_error, :around_request],
|
|
194
|
+
Zenrows::Hooks::EVENTS
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def test_run_passes_arguments_to_callbacks
|
|
198
|
+
received_args = nil
|
|
199
|
+
@hooks.register(:on_response) { |resp, ctx| received_args = [resp, ctx] }
|
|
200
|
+
|
|
201
|
+
@hooks.run(:on_response, :response, {key: :value})
|
|
202
|
+
|
|
203
|
+
assert_equal [:response, {key: :value}], received_args
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def test_returns_self_for_chaining
|
|
207
|
+
result = @hooks.register(:on_response) {}
|
|
208
|
+
|
|
209
|
+
assert_same @hooks, result
|
|
210
|
+
|
|
211
|
+
result = @hooks.add_subscriber(Object.new)
|
|
212
|
+
|
|
213
|
+
assert_same @hooks, result
|
|
214
|
+
end
|
|
215
|
+
end
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "test_helper"
|
|
4
|
+
|
|
5
|
+
class InstrumentedClientTest < Minitest::Test
|
|
6
|
+
def setup
|
|
7
|
+
@hooks = Zenrows::Hooks.new
|
|
8
|
+
@mock_http = MockHttpClient.new
|
|
9
|
+
@context_base = {options: {js_render: true}, backend: :http_rb}
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def test_get_delegates_to_http_client
|
|
13
|
+
client = Zenrows::InstrumentedClient.new(@mock_http, hooks: @hooks, context_base: @context_base)
|
|
14
|
+
client.get("https://example.com")
|
|
15
|
+
|
|
16
|
+
assert_equal [[:get, "https://example.com", {}]], @mock_http.calls
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def test_post_delegates_to_http_client
|
|
20
|
+
client = Zenrows::InstrumentedClient.new(@mock_http, hooks: @hooks, context_base: @context_base)
|
|
21
|
+
client.post("https://example.com", body: "data")
|
|
22
|
+
|
|
23
|
+
assert_equal [[:post, "https://example.com", {body: "data"}]], @mock_http.calls
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def test_runs_before_request_hook
|
|
27
|
+
before_called = false
|
|
28
|
+
@hooks.register(:before_request) { |ctx| before_called = true }
|
|
29
|
+
|
|
30
|
+
client = Zenrows::InstrumentedClient.new(@mock_http, hooks: @hooks, context_base: @context_base)
|
|
31
|
+
client.get("https://example.com")
|
|
32
|
+
|
|
33
|
+
assert before_called
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def test_runs_on_response_hook
|
|
37
|
+
received_response = nil
|
|
38
|
+
received_context = nil
|
|
39
|
+
@hooks.register(:on_response) do |resp, ctx|
|
|
40
|
+
received_response = resp
|
|
41
|
+
received_context = ctx
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
client = Zenrows::InstrumentedClient.new(@mock_http, hooks: @hooks, context_base: @context_base)
|
|
45
|
+
client.get("https://example.com")
|
|
46
|
+
|
|
47
|
+
assert_instance_of MockResponse, received_response
|
|
48
|
+
assert_equal :get, received_context[:method]
|
|
49
|
+
assert_equal "https://example.com", received_context[:url]
|
|
50
|
+
assert_equal "example.com", received_context[:host]
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def test_runs_after_request_hook
|
|
54
|
+
after_called = false
|
|
55
|
+
@hooks.register(:after_request) { |ctx| after_called = true }
|
|
56
|
+
|
|
57
|
+
client = Zenrows::InstrumentedClient.new(@mock_http, hooks: @hooks, context_base: @context_base)
|
|
58
|
+
client.get("https://example.com")
|
|
59
|
+
|
|
60
|
+
assert after_called
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def test_runs_on_error_hook_and_reraises
|
|
64
|
+
error = StandardError.new("Connection failed")
|
|
65
|
+
received_error = nil
|
|
66
|
+
received_context = nil
|
|
67
|
+
|
|
68
|
+
@hooks.register(:on_error) do |err, ctx|
|
|
69
|
+
received_error = err
|
|
70
|
+
received_context = ctx
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Create a mock that raises
|
|
74
|
+
error_http = Object.new
|
|
75
|
+
error_http.define_singleton_method(:get) { |*| raise error }
|
|
76
|
+
|
|
77
|
+
client = Zenrows::InstrumentedClient.new(error_http, hooks: @hooks, context_base: @context_base)
|
|
78
|
+
|
|
79
|
+
raised = assert_raises StandardError do
|
|
80
|
+
client.get("https://example.com")
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
assert_equal error, raised
|
|
84
|
+
assert_equal error, received_error
|
|
85
|
+
assert_equal "https://example.com", received_context[:url]
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def test_runs_around_request_hook
|
|
89
|
+
order = []
|
|
90
|
+
|
|
91
|
+
@hooks.register(:around_request) do |ctx, &block|
|
|
92
|
+
order << :before
|
|
93
|
+
result = block.call
|
|
94
|
+
order << :after
|
|
95
|
+
result
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
client = Zenrows::InstrumentedClient.new(@mock_http, hooks: @hooks, context_base: @context_base)
|
|
99
|
+
client.get("https://example.com")
|
|
100
|
+
|
|
101
|
+
assert_equal [:before, :after], order
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def test_hook_execution_order
|
|
105
|
+
order = []
|
|
106
|
+
|
|
107
|
+
@hooks.register(:before_request) { order << :before }
|
|
108
|
+
@hooks.register(:around_request) do |ctx, &block|
|
|
109
|
+
order << :around_before
|
|
110
|
+
result = block.call
|
|
111
|
+
order << :around_after
|
|
112
|
+
result
|
|
113
|
+
end
|
|
114
|
+
@hooks.register(:on_response) { order << :on_response }
|
|
115
|
+
@hooks.register(:after_request) { order << :after }
|
|
116
|
+
|
|
117
|
+
client = Zenrows::InstrumentedClient.new(@mock_http, hooks: @hooks, context_base: @context_base)
|
|
118
|
+
client.get("https://example.com")
|
|
119
|
+
|
|
120
|
+
assert_equal [:before, :around_before, :on_response, :around_after, :after], order
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def test_method_missing_delegates_to_http
|
|
124
|
+
client = Zenrows::InstrumentedClient.new(@mock_http, hooks: @hooks, context_base: @context_base)
|
|
125
|
+
result = client.custom_method(:arg1, :arg2)
|
|
126
|
+
|
|
127
|
+
assert_equal :result, result
|
|
128
|
+
assert_equal [[:custom_method, :arg1, :arg2]], @mock_http.calls
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def test_respond_to_missing
|
|
132
|
+
client = Zenrows::InstrumentedClient.new(@mock_http, hooks: @hooks, context_base: @context_base)
|
|
133
|
+
|
|
134
|
+
assert_respond_to client, :custom_method
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def test_context_includes_zenrows_headers
|
|
138
|
+
received_headers = nil
|
|
139
|
+
@hooks.register(:on_response) { |resp, ctx| received_headers = ctx[:zenrows_headers] }
|
|
140
|
+
|
|
141
|
+
response = MockResponse.new(200, {
|
|
142
|
+
"Concurrency-Limit" => "25",
|
|
143
|
+
"X-Request-Cost" => "5.0"
|
|
144
|
+
})
|
|
145
|
+
@mock_http.stub_response(:get, "https://example.com", response)
|
|
146
|
+
|
|
147
|
+
client = Zenrows::InstrumentedClient.new(@mock_http, hooks: @hooks, context_base: @context_base)
|
|
148
|
+
client.get("https://example.com")
|
|
149
|
+
|
|
150
|
+
assert_equal 25, received_headers[:concurrency_limit]
|
|
151
|
+
assert_in_delta(5.0, received_headers[:request_cost])
|
|
152
|
+
end
|
|
153
|
+
end
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: zenrows
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ernest Bursa
|
|
8
8
|
bindir: exe
|
|
9
9
|
cert_chain: []
|
|
10
|
-
date:
|
|
10
|
+
date: 2025-12-30 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
13
|
name: http
|
|
@@ -66,6 +66,10 @@ files:
|
|
|
66
66
|
- lib/zenrows/configuration.rb
|
|
67
67
|
- lib/zenrows/css_extractor.rb
|
|
68
68
|
- lib/zenrows/errors.rb
|
|
69
|
+
- lib/zenrows/hooks.rb
|
|
70
|
+
- lib/zenrows/hooks/context.rb
|
|
71
|
+
- lib/zenrows/hooks/log_subscriber.rb
|
|
72
|
+
- lib/zenrows/instrumented_client.rb
|
|
69
73
|
- lib/zenrows/js_instructions.rb
|
|
70
74
|
- lib/zenrows/proxy.rb
|
|
71
75
|
- lib/zenrows/railtie.rb
|
|
@@ -83,13 +87,24 @@ files:
|
|
|
83
87
|
- sig/zenrows/configuration.rbs
|
|
84
88
|
- sig/zenrows/css_extractor.rbs
|
|
85
89
|
- sig/zenrows/errors.rbs
|
|
90
|
+
- sig/zenrows/hook_configurator.rbs
|
|
91
|
+
- sig/zenrows/hooks.rbs
|
|
92
|
+
- sig/zenrows/hooks/context.rbs
|
|
93
|
+
- sig/zenrows/hooks/log_subscriber.rbs
|
|
94
|
+
- sig/zenrows/instrumented_client.rbs
|
|
86
95
|
- sig/zenrows/js_instructions.rbs
|
|
87
96
|
- sig/zenrows/proxy.rbs
|
|
88
97
|
- test/test_helper.rb
|
|
89
98
|
- test/zenrows/api_client_test.rb
|
|
90
99
|
- test/zenrows/api_response_test.rb
|
|
100
|
+
- test/zenrows/client_hooks_test.rb
|
|
91
101
|
- test/zenrows/client_test.rb
|
|
102
|
+
- test/zenrows/configuration_hooks_test.rb
|
|
92
103
|
- test/zenrows/css_extractor_test.rb
|
|
104
|
+
- test/zenrows/hooks/context_test.rb
|
|
105
|
+
- test/zenrows/hooks/log_subscriber_test.rb
|
|
106
|
+
- test/zenrows/hooks_test.rb
|
|
107
|
+
- test/zenrows/instrumented_client_test.rb
|
|
93
108
|
- test/zenrows/js_instructions_test.rb
|
|
94
109
|
- test/zenrows/proxy_test.rb
|
|
95
110
|
- test/zenrows_test.rb
|
|
@@ -116,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
116
131
|
- !ruby/object:Gem::Version
|
|
117
132
|
version: '0'
|
|
118
133
|
requirements: []
|
|
119
|
-
rubygems_version:
|
|
134
|
+
rubygems_version: 3.6.2
|
|
120
135
|
specification_version: 4
|
|
121
136
|
summary: Ruby client for ZenRows web scraping API via proxy mode
|
|
122
137
|
test_files: []
|