zenrows 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.standard.yml +8 -0
- data/.yardopts +10 -0
- data/CHANGELOG.md +28 -0
- data/CLAUDE.md +63 -0
- data/LICENSE.txt +21 -0
- data/README.md +174 -0
- data/Rakefile +15 -0
- data/lib/zenrows/backends/base.rb +95 -0
- data/lib/zenrows/backends/http_rb.rb +59 -0
- data/lib/zenrows/client.rb +159 -0
- data/lib/zenrows/configuration.rb +136 -0
- data/lib/zenrows/errors.rb +74 -0
- data/lib/zenrows/js_instructions.rb +267 -0
- data/lib/zenrows/proxy.rb +226 -0
- data/lib/zenrows/railtie.rb +25 -0
- data/lib/zenrows/version.rb +5 -0
- data/lib/zenrows.rb +67 -0
- data/plan.md +430 -0
- data/sig/zenrows.rbs +4 -0
- data/test/test_helper.rb +7 -0
- data/test/zenrows/client_test.rb +83 -0
- data/test/zenrows/js_instructions_test.rb +140 -0
- data/test/zenrows/proxy_test.rb +114 -0
- data/test/zenrows_test.rb +43 -0
- metadata +99 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "test_helper"
|
|
4
|
+
|
|
5
|
+
class JsInstructionsTest < Minitest::Test
|
|
6
|
+
def test_build_with_block
|
|
7
|
+
instructions = Zenrows::JsInstructions.build do
|
|
8
|
+
click ".button"
|
|
9
|
+
wait 1000
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
assert_equal 2, instructions.size
|
|
13
|
+
refute instructions.empty?
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def test_click
|
|
17
|
+
instructions = Zenrows::JsInstructions.build { click ".submit" }
|
|
18
|
+
|
|
19
|
+
assert_equal [{click: ".submit"}], instructions.to_a
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def test_wait
|
|
23
|
+
instructions = Zenrows::JsInstructions.build { wait 2000 }
|
|
24
|
+
|
|
25
|
+
assert_equal [{wait: 2000}], instructions.to_a
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def test_wait_for
|
|
29
|
+
instructions = Zenrows::JsInstructions.build { wait_for ".content" }
|
|
30
|
+
|
|
31
|
+
assert_equal [{wait_for: ".content"}], instructions.to_a
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def test_wait_event
|
|
35
|
+
instructions = Zenrows::JsInstructions.build { wait_event :networkidle }
|
|
36
|
+
|
|
37
|
+
assert_equal [{wait_event: "networkidle"}], instructions.to_a
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def test_fill
|
|
41
|
+
instructions = Zenrows::JsInstructions.build { fill "#email", "test@example.com" }
|
|
42
|
+
|
|
43
|
+
assert_equal [{fill: ["#email", "test@example.com"]}], instructions.to_a
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def test_check_and_uncheck
|
|
47
|
+
instructions = Zenrows::JsInstructions.build do
|
|
48
|
+
check "#terms"
|
|
49
|
+
uncheck "#newsletter"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
assert_equal [{check: "#terms"}, {uncheck: "#newsletter"}], instructions.to_a
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def test_select_option
|
|
56
|
+
instructions = Zenrows::JsInstructions.build { select_option "#country", "US" }
|
|
57
|
+
|
|
58
|
+
assert_equal [{select_option: ["#country", "US"]}], instructions.to_a
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def test_scroll_y
|
|
62
|
+
instructions = Zenrows::JsInstructions.build { scroll_y 1500 }
|
|
63
|
+
|
|
64
|
+
assert_equal [{scroll_y: 1500}], instructions.to_a
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def test_scroll_x
|
|
68
|
+
instructions = Zenrows::JsInstructions.build { scroll_x 500 }
|
|
69
|
+
|
|
70
|
+
assert_equal [{scroll_x: 500}], instructions.to_a
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def test_scroll_to
|
|
74
|
+
instructions = Zenrows::JsInstructions.build { scroll_to :bottom }
|
|
75
|
+
|
|
76
|
+
assert_equal [{scroll_to: "bottom"}], instructions.to_a
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def test_evaluate
|
|
80
|
+
instructions = Zenrows::JsInstructions.build { evaluate "console.log('test')" }
|
|
81
|
+
|
|
82
|
+
assert_equal [{evaluate: "console.log('test')"}], instructions.to_a
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def test_frame_click
|
|
86
|
+
instructions = Zenrows::JsInstructions.build { frame_click "#iframe", ".button" }
|
|
87
|
+
|
|
88
|
+
assert_equal [{frame_click: ["#iframe", ".button"]}], instructions.to_a
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def test_frame_wait_for
|
|
92
|
+
instructions = Zenrows::JsInstructions.build { frame_wait_for "#iframe", ".loaded" }
|
|
93
|
+
|
|
94
|
+
assert_equal [{frame_wait_for: ["#iframe", ".loaded"]}], instructions.to_a
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def test_frame_fill
|
|
98
|
+
instructions = Zenrows::JsInstructions.build { frame_fill "#iframe", "#input", "value" }
|
|
99
|
+
|
|
100
|
+
assert_equal [{frame_fill: ["#iframe", "#input", "value"]}], instructions.to_a
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def test_frame_evaluate
|
|
104
|
+
instructions = Zenrows::JsInstructions.build { frame_evaluate "iframe-name", "console.log('hi')" }
|
|
105
|
+
|
|
106
|
+
assert_equal [{frame_evaluate: ["iframe-name", "console.log('hi')"]}], instructions.to_a
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def test_to_json
|
|
110
|
+
instructions = Zenrows::JsInstructions.build do
|
|
111
|
+
click ".btn"
|
|
112
|
+
wait 500
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
json = instructions.to_json
|
|
116
|
+
assert_equal '[{"click":".btn"},{"wait":500}]', json
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def test_chaining
|
|
120
|
+
instructions = Zenrows::JsInstructions.new
|
|
121
|
+
.click(".btn")
|
|
122
|
+
.wait(1000)
|
|
123
|
+
.scroll_to(:bottom)
|
|
124
|
+
|
|
125
|
+
assert_equal 3, instructions.size
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def test_complex_flow
|
|
129
|
+
instructions = Zenrows::JsInstructions.build do
|
|
130
|
+
wait_for ".login-form"
|
|
131
|
+
fill "#email", "user@example.com"
|
|
132
|
+
fill "#password", "secret123"
|
|
133
|
+
click "#submit"
|
|
134
|
+
wait 2000
|
|
135
|
+
wait_for ".dashboard"
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
assert_equal 6, instructions.size
|
|
139
|
+
end
|
|
140
|
+
end
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "test_helper"
|
|
4
|
+
|
|
5
|
+
class ProxyTest < Minitest::Test
|
|
6
|
+
def setup
|
|
7
|
+
@proxy = Zenrows::Proxy.new(
|
|
8
|
+
api_key: "test_api_key",
|
|
9
|
+
host: "superproxy.zenrows.com",
|
|
10
|
+
port: 1337
|
|
11
|
+
)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def test_build_basic_config
|
|
15
|
+
config = @proxy.build
|
|
16
|
+
|
|
17
|
+
assert_equal "superproxy.zenrows.com", config[:host]
|
|
18
|
+
assert_equal 1337, config[:port]
|
|
19
|
+
assert_equal "test_api_key", config[:username]
|
|
20
|
+
assert_equal "", config[:password]
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def test_build_with_js_render
|
|
24
|
+
config = @proxy.build(js_render: true)
|
|
25
|
+
|
|
26
|
+
assert_includes config[:password], "js_render=true"
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def test_build_with_premium_proxy
|
|
30
|
+
config = @proxy.build(premium_proxy: true)
|
|
31
|
+
|
|
32
|
+
assert_includes config[:password], "premium_proxy=true"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def test_build_with_proxy_country_enables_premium
|
|
36
|
+
config = @proxy.build(proxy_country: "us")
|
|
37
|
+
|
|
38
|
+
assert_includes config[:password], "proxy_country=us"
|
|
39
|
+
assert_includes config[:password], "premium_proxy=true"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def test_build_with_wait_boolean
|
|
43
|
+
config = @proxy.build(wait: true)
|
|
44
|
+
|
|
45
|
+
assert_includes config[:password], "wait=15000"
|
|
46
|
+
assert_includes config[:password], "js_render=true"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def test_build_with_wait_integer
|
|
50
|
+
config = @proxy.build(wait: 5000)
|
|
51
|
+
|
|
52
|
+
assert_includes config[:password], "wait=5000"
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def test_build_with_wait_for_selector
|
|
56
|
+
config = @proxy.build(wait_for: ".content")
|
|
57
|
+
|
|
58
|
+
assert_includes config[:password], "wait_for=.content"
|
|
59
|
+
assert_includes config[:password], "js_render=true"
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def test_build_with_session_id_true
|
|
63
|
+
config = @proxy.build(session_id: true)
|
|
64
|
+
|
|
65
|
+
assert_match(/session_id=\d+/, config[:password])
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def test_build_with_session_id_string
|
|
69
|
+
config = @proxy.build(session_id: "my_session")
|
|
70
|
+
|
|
71
|
+
assert_includes config[:password], "session_id=my_session"
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def test_build_with_screenshot
|
|
75
|
+
config = @proxy.build(screenshot: true)
|
|
76
|
+
|
|
77
|
+
assert_includes config[:password], "screenshot=true"
|
|
78
|
+
assert_includes config[:password], "js_render=true"
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def test_build_with_json_response
|
|
82
|
+
config = @proxy.build(json_response: true)
|
|
83
|
+
|
|
84
|
+
assert_includes config[:password], "json_response=true"
|
|
85
|
+
assert_includes config[:password], "js_render=true"
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def test_build_with_block_resources
|
|
89
|
+
config = @proxy.build(block_resources: "image,media,font")
|
|
90
|
+
|
|
91
|
+
assert_includes config[:password], "block_resources=image,media,font"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def test_build_url
|
|
95
|
+
url = @proxy.build_url(js_render: true)
|
|
96
|
+
|
|
97
|
+
assert_equal "http://test_api_key:js_render=true@superproxy.zenrows.com:1337", url
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def test_build_array
|
|
101
|
+
arr = @proxy.build_array(js_render: true)
|
|
102
|
+
|
|
103
|
+
assert_equal "superproxy.zenrows.com", arr[0]
|
|
104
|
+
assert_equal 1337, arr[1]
|
|
105
|
+
assert_equal "test_api_key", arr[2]
|
|
106
|
+
assert_includes arr[3], "js_render=true"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def test_raises_on_excessive_wait_time
|
|
110
|
+
assert_raises Zenrows::WaitTimeError do
|
|
111
|
+
@proxy.build(wait: 200_000)
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "test_helper"
|
|
4
|
+
|
|
5
|
+
class ZenrowsTest < Minitest::Test
|
|
6
|
+
def setup
|
|
7
|
+
Zenrows.reset_configuration!
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def test_that_it_has_a_version_number
|
|
11
|
+
refute_nil Zenrows::VERSION
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def test_configure_with_block
|
|
15
|
+
Zenrows.configure do |config|
|
|
16
|
+
config.api_key = "test_key"
|
|
17
|
+
config.host = "proxy.example.com"
|
|
18
|
+
config.port = 8080
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
assert_equal "test_key", Zenrows.configuration.api_key
|
|
22
|
+
assert_equal "proxy.example.com", Zenrows.configuration.host
|
|
23
|
+
assert_equal 8080, Zenrows.configuration.port
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def test_default_configuration
|
|
27
|
+
config = Zenrows.configuration
|
|
28
|
+
|
|
29
|
+
assert_nil config.api_key
|
|
30
|
+
assert_equal "superproxy.zenrows.com", config.host
|
|
31
|
+
assert_equal 1337, config.port
|
|
32
|
+
assert_equal 5, config.connect_timeout
|
|
33
|
+
assert_equal 180, config.read_timeout
|
|
34
|
+
assert_equal :http_rb, config.backend
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def test_reset_configuration
|
|
38
|
+
Zenrows.configure { |c| c.api_key = "test" }
|
|
39
|
+
Zenrows.reset_configuration!
|
|
40
|
+
|
|
41
|
+
assert_nil Zenrows.configuration.api_key
|
|
42
|
+
end
|
|
43
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: zenrows
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Ernest Bursa
|
|
8
|
+
bindir: exe
|
|
9
|
+
cert_chain: []
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
|
+
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: http
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - "~>"
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '5.0'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - "~>"
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '5.0'
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: addressable
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - "~>"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '2.8'
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - "~>"
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '2.8'
|
|
40
|
+
description: A Ruby gem for ZenRows web scraping proxy with multi-backend support
|
|
41
|
+
(http.rb primary), JavaScript rendering, premium proxies, and comprehensive browser
|
|
42
|
+
automation via JS instructions.
|
|
43
|
+
email:
|
|
44
|
+
- ernest.bursa@fourthwall.com
|
|
45
|
+
executables: []
|
|
46
|
+
extensions: []
|
|
47
|
+
extra_rdoc_files: []
|
|
48
|
+
files:
|
|
49
|
+
- ".standard.yml"
|
|
50
|
+
- ".yardopts"
|
|
51
|
+
- CHANGELOG.md
|
|
52
|
+
- CLAUDE.md
|
|
53
|
+
- LICENSE.txt
|
|
54
|
+
- README.md
|
|
55
|
+
- Rakefile
|
|
56
|
+
- lib/zenrows.rb
|
|
57
|
+
- lib/zenrows/backends/base.rb
|
|
58
|
+
- lib/zenrows/backends/http_rb.rb
|
|
59
|
+
- lib/zenrows/client.rb
|
|
60
|
+
- lib/zenrows/configuration.rb
|
|
61
|
+
- lib/zenrows/errors.rb
|
|
62
|
+
- lib/zenrows/js_instructions.rb
|
|
63
|
+
- lib/zenrows/proxy.rb
|
|
64
|
+
- lib/zenrows/railtie.rb
|
|
65
|
+
- lib/zenrows/version.rb
|
|
66
|
+
- plan.md
|
|
67
|
+
- sig/zenrows.rbs
|
|
68
|
+
- test/test_helper.rb
|
|
69
|
+
- test/zenrows/client_test.rb
|
|
70
|
+
- test/zenrows/js_instructions_test.rb
|
|
71
|
+
- test/zenrows/proxy_test.rb
|
|
72
|
+
- test/zenrows_test.rb
|
|
73
|
+
homepage: https://github.com/swistaczek/zenrows-rb
|
|
74
|
+
licenses:
|
|
75
|
+
- MIT
|
|
76
|
+
metadata:
|
|
77
|
+
homepage_uri: https://github.com/swistaczek/zenrows-rb
|
|
78
|
+
source_code_uri: https://github.com/swistaczek/zenrows-rb
|
|
79
|
+
changelog_uri: https://github.com/swistaczek/zenrows-rb/blob/main/CHANGELOG.md
|
|
80
|
+
documentation_uri: https://rubydoc.info/gems/zenrows
|
|
81
|
+
rubygems_mfa_required: 'true'
|
|
82
|
+
rdoc_options: []
|
|
83
|
+
require_paths:
|
|
84
|
+
- lib
|
|
85
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
86
|
+
requirements:
|
|
87
|
+
- - ">="
|
|
88
|
+
- !ruby/object:Gem::Version
|
|
89
|
+
version: 3.1.0
|
|
90
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
91
|
+
requirements:
|
|
92
|
+
- - ">="
|
|
93
|
+
- !ruby/object:Gem::Version
|
|
94
|
+
version: '0'
|
|
95
|
+
requirements: []
|
|
96
|
+
rubygems_version: 4.0.3
|
|
97
|
+
specification_version: 4
|
|
98
|
+
summary: Ruby client for ZenRows web scraping API via proxy mode
|
|
99
|
+
test_files: []
|