ferrum 0.13 → 0.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +13 -2
- data/lib/ferrum/browser/client.rb +2 -1
- data/lib/ferrum/browser/command.rb +4 -0
- data/lib/ferrum/browser/options/chrome.rb +9 -2
- data/lib/ferrum/browser/process.rb +1 -1
- data/lib/ferrum/browser/web_socket.rb +1 -1
- data/lib/ferrum/browser.rb +8 -2
- data/lib/ferrum/cookies/cookie.rb +57 -0
- data/lib/ferrum/cookies.rb +39 -3
- data/lib/ferrum/frame.rb +1 -0
- data/lib/ferrum/network/exchange.rb +20 -2
- data/lib/ferrum/network/intercepted_request.rb +3 -12
- data/lib/ferrum/network/request.rb +15 -40
- data/lib/ferrum/network/request_params.rb +57 -0
- data/lib/ferrum/network/response.rb +25 -5
- data/lib/ferrum/network.rb +35 -10
- data/lib/ferrum/node.rb +21 -1
- data/lib/ferrum/page/screenshot.rb +6 -0
- data/lib/ferrum/page.rb +11 -17
- data/lib/ferrum/version.rb +1 -1
- metadata +4 -101
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6cbef2f4caac663a8b39afc37b94e46f00c9471cdaea36a1781dcfa99488a546
|
4
|
+
data.tar.gz: 58d8f68e84138b71de15ac33368d4cd4f8992db4ab7f321a3e3f03530968502b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a621c8e99fe1ec5e183ad54681da4a6bf09105fc66477c37b76b59cb1842f679ac679690514de1829fbbe18af4b81e0e01a64b564a48db9c49af151fc5b64e51
|
7
|
+
data.tar.gz: cc3089d1b9a81d43e8dbeb356b0d677d5377bc1d1d4667e0da9f389034e044ef0497ced614b39812cff53dd70930ec1f66e473f2a30a7edad825dd3424df7e54
|
data/README.md
CHANGED
@@ -134,6 +134,7 @@ In docker as root you must pass the no-sandbox browser option:
|
|
134
134
|
Ferrum::Browser.new(browser_options: { 'no-sandbox': nil })
|
135
135
|
```
|
136
136
|
|
137
|
+
It has also been reported that the Chrome process repeatedly crashes when running inside a Docker container on an M1 Mac preventing Ferrum from working. Ferrum should work as expected when deployed to a Docker container on a non-M1 Mac.
|
137
138
|
|
138
139
|
## Customization
|
139
140
|
|
@@ -144,7 +145,8 @@ Ferrum::Browser.new(options)
|
|
144
145
|
```
|
145
146
|
|
146
147
|
* options `Hash`
|
147
|
-
* `:headless` (Boolean) - Set browser as headless or not, `true` by default.
|
148
|
+
* `:headless` (String | Boolean) - Set browser as headless or not, `true` by default. You can set `"new"` to support
|
149
|
+
[new headless mode](https://developer.chrome.com/articles/new-headless/).
|
148
150
|
* `:xvfb` (Boolean) - Run browser in a virtual framebuffer, `false` by default.
|
149
151
|
* `:window_size` (Array) - The dimensions of the browser window in which to
|
150
152
|
test, expressed as a 2-element array, e.g. [1024, 768]. Default: [1024, 768]
|
@@ -595,9 +597,16 @@ Activates offline mode for a page.
|
|
595
597
|
|
596
598
|
```ruby
|
597
599
|
browser.network.offline_mode
|
598
|
-
browser.go_to("https://github.com/") # => Ferrum::StatusError (Request to https://github.com/ failed
|
600
|
+
browser.go_to("https://github.com/") # => Ferrum::StatusError (Request to https://github.com/ failed(net::ERR_INTERNET_DISCONNECTED))
|
599
601
|
```
|
600
602
|
|
603
|
+
#### cache(disable: `Boolean`)
|
604
|
+
|
605
|
+
Toggles ignoring cache for each request. If true, cache will not be used.
|
606
|
+
|
607
|
+
```ruby
|
608
|
+
browser.network.cache(disable: true)
|
609
|
+
```
|
601
610
|
|
602
611
|
## Proxy
|
603
612
|
|
@@ -1128,6 +1137,8 @@ frame.at_css("//a[text() = 'Log in']") # => Node
|
|
1128
1137
|
#### evaluate
|
1129
1138
|
#### selected : `Array<Node>`
|
1130
1139
|
#### select
|
1140
|
+
#### scroll_into_view
|
1141
|
+
#### in_viewport?(of: `Node | nil`) : `Boolean`
|
1131
1142
|
|
1132
1143
|
(chainable) Selects options by passed attribute.
|
1133
1144
|
|
@@ -84,7 +84,8 @@ module Ferrum
|
|
84
84
|
case error["message"]
|
85
85
|
# Node has disappeared while we were trying to get it
|
86
86
|
when "No node with given id found",
|
87
|
-
"Could not find node with given id"
|
87
|
+
"Could not find node with given id",
|
88
|
+
"Inspected target navigated or closed"
|
88
89
|
raise NodeNotFoundError, error
|
89
90
|
# Context is lost, page is reloading
|
90
91
|
when "Cannot find context with specified id"
|
@@ -19,8 +19,9 @@ module Ferrum
|
|
19
19
|
"keep-alive-for-test" => nil,
|
20
20
|
"disable-popup-blocking" => nil,
|
21
21
|
"disable-extensions" => nil,
|
22
|
+
"disable-component-extensions-with-background-pages" => nil,
|
22
23
|
"disable-hang-monitor" => nil,
|
23
|
-
"disable-features" => "site-per-process,TranslateUI",
|
24
|
+
"disable-features" => "site-per-process,IsolateOrigins,TranslateUI",
|
24
25
|
"disable-translate" => nil,
|
25
26
|
"disable-background-networking" => nil,
|
26
27
|
"enable-features" => "NetworkService,NetworkServiceInProcess",
|
@@ -32,6 +33,7 @@ module Ferrum
|
|
32
33
|
"disable-ipc-flooding-protection" => nil,
|
33
34
|
"disable-prompt-on-repost" => nil,
|
34
35
|
"disable-renderer-backgrounding" => nil,
|
36
|
+
"disable-site-isolation-trials" => nil,
|
35
37
|
"force-color-profile" => "srgb",
|
36
38
|
"metrics-recording-only" => nil,
|
37
39
|
"safebrowsing-disable-auto-update" => nil,
|
@@ -74,7 +76,12 @@ module Ferrum
|
|
74
76
|
end
|
75
77
|
|
76
78
|
def merge_default(flags, options)
|
77
|
-
defaults =
|
79
|
+
defaults = case options.headless
|
80
|
+
when false
|
81
|
+
except("headless", "disable-gpu")
|
82
|
+
when "new"
|
83
|
+
except("headless").merge("headless" => "new")
|
84
|
+
end
|
78
85
|
|
79
86
|
defaults ||= DEFAULT_OPTIONS
|
80
87
|
defaults.merge(flags)
|
@@ -137,7 +137,7 @@ module Ferrum
|
|
137
137
|
output = ""
|
138
138
|
start = Utils::ElapsedTime.monotonic_time
|
139
139
|
max_time = start + timeout
|
140
|
-
regexp = %r{DevTools listening on (ws://.*)}
|
140
|
+
regexp = %r{DevTools listening on (ws://.*[a-zA-Z0-9-]{36})}
|
141
141
|
while (now = Utils::ElapsedTime.monotonic_time) < max_time
|
142
142
|
begin
|
143
143
|
output += read_io.read_nonblock(512)
|
data/lib/ferrum/browser.rb
CHANGED
@@ -22,7 +22,7 @@ module Ferrum
|
|
22
22
|
body doctype content=
|
23
23
|
headers cookies network
|
24
24
|
mouse keyboard
|
25
|
-
screenshot pdf mhtml viewport_size
|
25
|
+
screenshot pdf mhtml viewport_size device_pixel_ratio
|
26
26
|
frames frame_by main_frame
|
27
27
|
evaluate evaluate_on evaluate_async execute evaluate_func
|
28
28
|
add_script_tag add_style_tag bypass_csp
|
@@ -177,7 +177,7 @@ module Ferrum
|
|
177
177
|
block_given? ? yield(page) : page
|
178
178
|
ensure
|
179
179
|
if block_given?
|
180
|
-
page
|
180
|
+
page&.close
|
181
181
|
context.dispose if new_context
|
182
182
|
end
|
183
183
|
end
|
@@ -237,6 +237,8 @@ module Ferrum
|
|
237
237
|
end
|
238
238
|
|
239
239
|
def quit
|
240
|
+
return unless @client
|
241
|
+
|
240
242
|
@client.close
|
241
243
|
@process.stop
|
242
244
|
@client = @process = @contexts = nil
|
@@ -262,6 +264,10 @@ module Ferrum
|
|
262
264
|
VersionInfo.new(command("Browser.getVersion"))
|
263
265
|
end
|
264
266
|
|
267
|
+
def headless_new?
|
268
|
+
process&.command&.headless_new?
|
269
|
+
end
|
270
|
+
|
265
271
|
private
|
266
272
|
|
267
273
|
def start
|
@@ -113,6 +113,38 @@ module Ferrum
|
|
113
113
|
Time.at(attributes["expires"]) if attributes["expires"].positive?
|
114
114
|
end
|
115
115
|
|
116
|
+
#
|
117
|
+
# The priority of the cookie.
|
118
|
+
#
|
119
|
+
# @return [String]
|
120
|
+
#
|
121
|
+
def priority
|
122
|
+
@attributes["priority"]
|
123
|
+
end
|
124
|
+
|
125
|
+
#
|
126
|
+
# @return [Boolean]
|
127
|
+
#
|
128
|
+
def sameparty?
|
129
|
+
@attributes["sameParty"]
|
130
|
+
end
|
131
|
+
|
132
|
+
alias same_party? sameparty?
|
133
|
+
|
134
|
+
#
|
135
|
+
# @return [String]
|
136
|
+
#
|
137
|
+
def source_scheme
|
138
|
+
@attributes["sourceScheme"]
|
139
|
+
end
|
140
|
+
|
141
|
+
#
|
142
|
+
# @return [Integer]
|
143
|
+
#
|
144
|
+
def source_port
|
145
|
+
@attributes["sourcePort"]
|
146
|
+
end
|
147
|
+
|
116
148
|
#
|
117
149
|
# Compares different cookie objects.
|
118
150
|
#
|
@@ -121,6 +153,31 @@ module Ferrum
|
|
121
153
|
def ==(other)
|
122
154
|
other.class == self.class && other.attributes == attributes
|
123
155
|
end
|
156
|
+
|
157
|
+
#
|
158
|
+
# Converts the cookie back into a raw cookie String.
|
159
|
+
#
|
160
|
+
# @return [String]
|
161
|
+
# The raw cookie string.
|
162
|
+
#
|
163
|
+
def to_s
|
164
|
+
string = String.new("#{@attributes['name']}=#{@attributes['value']}")
|
165
|
+
|
166
|
+
@attributes.each do |key, value|
|
167
|
+
case key
|
168
|
+
when "name", "value" # no-op
|
169
|
+
when "domain" then string << "; Domain=#{value}"
|
170
|
+
when "path" then string << "; Path=#{value}"
|
171
|
+
when "expires" then string << "; Expires=#{Time.at(value).httpdate}"
|
172
|
+
when "httpOnly" then string << "; httpOnly" if value
|
173
|
+
when "secure" then string << "; Secure" if value
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
string
|
178
|
+
end
|
179
|
+
|
180
|
+
alias to_h attributes
|
124
181
|
end
|
125
182
|
end
|
126
183
|
end
|
data/lib/ferrum/cookies.rb
CHANGED
@@ -4,10 +4,34 @@ require "ferrum/cookies/cookie"
|
|
4
4
|
|
5
5
|
module Ferrum
|
6
6
|
class Cookies
|
7
|
+
include Enumerable
|
8
|
+
|
7
9
|
def initialize(page)
|
8
10
|
@page = page
|
9
11
|
end
|
10
12
|
|
13
|
+
#
|
14
|
+
# Enumerates over all cookies.
|
15
|
+
#
|
16
|
+
# @yield [cookie]
|
17
|
+
# The given block will be passed each cookie.
|
18
|
+
#
|
19
|
+
# @yieldparam [Cookie] cookie
|
20
|
+
# A cookie in the browser.
|
21
|
+
#
|
22
|
+
# @return [Enumerator]
|
23
|
+
# If no block is given, an Enumerator object will be returned.
|
24
|
+
#
|
25
|
+
def each
|
26
|
+
return enum_for(__method__) unless block_given?
|
27
|
+
|
28
|
+
cookies = @page.command("Network.getAllCookies")["cookies"]
|
29
|
+
|
30
|
+
cookies.each do |c|
|
31
|
+
yield Cookie.new(c)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
11
35
|
#
|
12
36
|
# Returns cookies hash.
|
13
37
|
#
|
@@ -22,8 +46,9 @@ module Ferrum
|
|
22
46
|
# # }
|
23
47
|
#
|
24
48
|
def all
|
25
|
-
|
26
|
-
|
49
|
+
each.to_h do |cookie|
|
50
|
+
[cookie.name, cookie]
|
51
|
+
end
|
27
52
|
end
|
28
53
|
|
29
54
|
#
|
@@ -44,7 +69,7 @@ module Ferrum
|
|
44
69
|
# # }>
|
45
70
|
#
|
46
71
|
def [](name)
|
47
|
-
|
72
|
+
find { |cookie| cookie.name == name }
|
48
73
|
end
|
49
74
|
|
50
75
|
#
|
@@ -53,23 +78,34 @@ module Ferrum
|
|
53
78
|
# @param [Hash{Symbol => Object}, Cookie] options
|
54
79
|
#
|
55
80
|
# @option options [String] :name
|
81
|
+
# The cookie param name.
|
56
82
|
#
|
57
83
|
# @option options [String] :value
|
84
|
+
# The cookie param value.
|
58
85
|
#
|
59
86
|
# @option options [String] :domain
|
87
|
+
# The domain the cookie belongs to.
|
60
88
|
#
|
61
89
|
# @option options [String] :path
|
90
|
+
# The path that the cookie is bound to.
|
62
91
|
#
|
63
92
|
# @option options [Integer] :expires
|
93
|
+
# When the cookie will expire.
|
64
94
|
#
|
65
95
|
# @option options [Integer] :size
|
96
|
+
# The size of the cookie.
|
66
97
|
#
|
67
98
|
# @option options [Boolean] :httponly
|
99
|
+
# Specifies whether the cookie `HttpOnly`.
|
68
100
|
#
|
69
101
|
# @option options [Boolean] :secure
|
102
|
+
# Specifies whether the cookie is marked as `Secure`.
|
70
103
|
#
|
71
104
|
# @option options [String] :samesite
|
105
|
+
# Specifies whether the cookie is `SameSite`.
|
72
106
|
#
|
107
|
+
# @option options [Boolean] :session
|
108
|
+
# Specifies whether the cookie is a session cookie.
|
73
109
|
#
|
74
110
|
# @example
|
75
111
|
# browser.cookies.set(name: "stealth", value: "omg", domain: "google.com") # => true
|
data/lib/ferrum/frame.rb
CHANGED
@@ -51,7 +51,7 @@ module Ferrum
|
|
51
51
|
# @return [Boolean]
|
52
52
|
#
|
53
53
|
def navigation_request?(frame_id)
|
54
|
-
request
|
54
|
+
request&.type?(:document) && request&.frame_id == frame_id
|
55
55
|
end
|
56
56
|
|
57
57
|
#
|
@@ -79,7 +79,7 @@ module Ferrum
|
|
79
79
|
# @return [Boolean]
|
80
80
|
#
|
81
81
|
def finished?
|
82
|
-
blocked? ||
|
82
|
+
blocked? || response&.loaded? || !error.nil?
|
83
83
|
end
|
84
84
|
|
85
85
|
#
|
@@ -100,6 +100,24 @@ module Ferrum
|
|
100
100
|
!intercepted_request.nil?
|
101
101
|
end
|
102
102
|
|
103
|
+
#
|
104
|
+
# Determines if the exchange is XHR.
|
105
|
+
#
|
106
|
+
# @return [Boolean]
|
107
|
+
#
|
108
|
+
def xhr?
|
109
|
+
!!request&.xhr?
|
110
|
+
end
|
111
|
+
|
112
|
+
#
|
113
|
+
# Determines if the exchange is a redirect.
|
114
|
+
#
|
115
|
+
# @return [Boolean]
|
116
|
+
#
|
117
|
+
def redirect?
|
118
|
+
response&.redirect?
|
119
|
+
end
|
120
|
+
|
103
121
|
#
|
104
122
|
# Returns request's URL.
|
105
123
|
#
|
@@ -1,10 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "ferrum/network/request_params"
|
3
4
|
require "base64"
|
4
5
|
|
5
6
|
module Ferrum
|
6
7
|
class Network
|
7
8
|
class InterceptedRequest
|
9
|
+
include RequestParams
|
10
|
+
|
8
11
|
attr_accessor :request_id, :frame_id, :resource_type, :network_id, :status
|
9
12
|
|
10
13
|
def initialize(page, params)
|
@@ -54,18 +57,6 @@ module Ferrum
|
|
54
57
|
@page.command("Fetch.failRequest", requestId: request_id, errorReason: "BlockedByClient")
|
55
58
|
end
|
56
59
|
|
57
|
-
def url
|
58
|
-
@request["url"]
|
59
|
-
end
|
60
|
-
|
61
|
-
def method
|
62
|
-
@request["method"]
|
63
|
-
end
|
64
|
-
|
65
|
-
def headers
|
66
|
-
@request["headers"]
|
67
|
-
end
|
68
|
-
|
69
60
|
def initial_priority
|
70
61
|
@request["initialPriority"]
|
71
62
|
end
|
@@ -1,5 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "ferrum/network/request_params"
|
3
4
|
require "time"
|
4
5
|
|
5
6
|
module Ferrum
|
@@ -9,6 +10,8 @@ module Ferrum
|
|
9
10
|
# object.
|
10
11
|
#
|
11
12
|
class Request
|
13
|
+
include RequestParams
|
14
|
+
|
12
15
|
#
|
13
16
|
# Initializes the request object.
|
14
17
|
#
|
@@ -51,48 +54,21 @@ module Ferrum
|
|
51
54
|
end
|
52
55
|
|
53
56
|
#
|
54
|
-
#
|
55
|
-
#
|
56
|
-
# @return [String]
|
57
|
+
# Determines if the request is XHR.
|
57
58
|
#
|
58
|
-
|
59
|
-
@params["frameId"]
|
60
|
-
end
|
61
|
-
|
62
|
-
#
|
63
|
-
# The URL for the request.
|
64
|
-
#
|
65
|
-
# @return [String]
|
66
|
-
#
|
67
|
-
def url
|
68
|
-
@request["url"]
|
69
|
-
end
|
70
|
-
|
71
|
-
#
|
72
|
-
# The URL fragment for the request.
|
73
|
-
#
|
74
|
-
# @return [String, nil]
|
59
|
+
# @return [Boolean]
|
75
60
|
#
|
76
|
-
def
|
77
|
-
|
61
|
+
def xhr?
|
62
|
+
type?("xhr")
|
78
63
|
end
|
79
64
|
|
80
65
|
#
|
81
|
-
# The request
|
66
|
+
# The frame ID of the request.
|
82
67
|
#
|
83
68
|
# @return [String]
|
84
69
|
#
|
85
|
-
def
|
86
|
-
@
|
87
|
-
end
|
88
|
-
|
89
|
-
#
|
90
|
-
# The request headers.
|
91
|
-
#
|
92
|
-
# @return [Hash{String => String}]
|
93
|
-
#
|
94
|
-
def headers
|
95
|
-
@request["headers"]
|
70
|
+
def frame_id
|
71
|
+
@params["frameId"]
|
96
72
|
end
|
97
73
|
|
98
74
|
#
|
@@ -105,15 +81,14 @@ module Ferrum
|
|
105
81
|
end
|
106
82
|
|
107
83
|
#
|
108
|
-
#
|
84
|
+
# Converts the request to a Hash.
|
109
85
|
#
|
110
|
-
# @return [String
|
111
|
-
# The
|
86
|
+
# @return [Hash{String => Object}]
|
87
|
+
# The params of the request.
|
112
88
|
#
|
113
|
-
def
|
114
|
-
@
|
89
|
+
def to_h
|
90
|
+
@params
|
115
91
|
end
|
116
|
-
alias body post_data
|
117
92
|
end
|
118
93
|
end
|
119
94
|
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Ferrum
|
4
|
+
class Network
|
5
|
+
#
|
6
|
+
# Common methods used by both {Request} and {InterceptedRequest}.
|
7
|
+
#
|
8
|
+
module RequestParams
|
9
|
+
#
|
10
|
+
# The URL for the request.
|
11
|
+
#
|
12
|
+
# @return [String]
|
13
|
+
#
|
14
|
+
def url
|
15
|
+
@request["url"]
|
16
|
+
end
|
17
|
+
|
18
|
+
#
|
19
|
+
# The URL fragment for the request.
|
20
|
+
#
|
21
|
+
# @return [String, nil]
|
22
|
+
#
|
23
|
+
def url_fragment
|
24
|
+
@request["urlFragment"]
|
25
|
+
end
|
26
|
+
|
27
|
+
#
|
28
|
+
# The request method.
|
29
|
+
#
|
30
|
+
# @return [String]
|
31
|
+
#
|
32
|
+
def method
|
33
|
+
@request["method"]
|
34
|
+
end
|
35
|
+
|
36
|
+
#
|
37
|
+
# The request headers.
|
38
|
+
#
|
39
|
+
# @return [Hash{String => String}]
|
40
|
+
#
|
41
|
+
def headers
|
42
|
+
@request["headers"]
|
43
|
+
end
|
44
|
+
|
45
|
+
#
|
46
|
+
# The optional HTTP `POST` form data.
|
47
|
+
#
|
48
|
+
# @return [String, nil]
|
49
|
+
# The HTTP `POST` form data.
|
50
|
+
#
|
51
|
+
def post_data
|
52
|
+
@request["postData"]
|
53
|
+
end
|
54
|
+
alias body post_data
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -18,8 +18,13 @@ module Ferrum
|
|
18
18
|
# @return [Hash{String => Object}]
|
19
19
|
attr_reader :params
|
20
20
|
|
21
|
+
# The response is fully loaded by the browser.
|
21
22
|
#
|
22
|
-
#
|
23
|
+
# @return [Boolean]
|
24
|
+
attr_writer :loaded
|
25
|
+
|
26
|
+
#
|
27
|
+
# Initializes the responses object.
|
23
28
|
#
|
24
29
|
# @param [Page] page
|
25
30
|
# The page associated with the network response.
|
@@ -121,9 +126,8 @@ module Ferrum
|
|
121
126
|
#
|
122
127
|
def body
|
123
128
|
@body ||= begin
|
124
|
-
body, encoded = @page
|
125
|
-
|
126
|
-
.values_at("body", "base64Encoded")
|
129
|
+
body, encoded = @page.command("Network.getResponseBody", requestId: id)
|
130
|
+
.values_at("body", "base64Encoded")
|
127
131
|
encoded ? Base64.decode64(body) : body
|
128
132
|
end
|
129
133
|
end
|
@@ -135,8 +139,22 @@ module Ferrum
|
|
135
139
|
@page.network.response == self
|
136
140
|
end
|
137
141
|
|
142
|
+
# The response is fully loaded by the browser or not.
|
143
|
+
#
|
144
|
+
# @return [Boolean]
|
145
|
+
def loaded?
|
146
|
+
@loaded
|
147
|
+
end
|
148
|
+
|
149
|
+
# Whether the response is a redirect.
|
150
|
+
#
|
151
|
+
# @return [Boolean]
|
152
|
+
def redirect?
|
153
|
+
params.key?("redirectResponse")
|
154
|
+
end
|
155
|
+
|
138
156
|
#
|
139
|
-
#
|
157
|
+
# Compares the response's ID to another response's ID.
|
140
158
|
#
|
141
159
|
# @return [Boolean]
|
142
160
|
# Indicates whether the response has the same ID as the other response
|
@@ -154,6 +172,8 @@ module Ferrum
|
|
154
172
|
def inspect
|
155
173
|
%(#<#{self.class} @params=#{@params.inspect} @response=#{@response.inspect}>)
|
156
174
|
end
|
175
|
+
|
176
|
+
alias to_h params
|
157
177
|
end
|
158
178
|
end
|
159
179
|
end
|
data/lib/ferrum/network.rb
CHANGED
@@ -11,9 +11,10 @@ module Ferrum
|
|
11
11
|
class Network
|
12
12
|
CLEAR_TYPE = %i[traffic cache].freeze
|
13
13
|
AUTHORIZE_TYPE = %i[server proxy].freeze
|
14
|
-
|
15
|
-
|
16
|
-
|
14
|
+
REQUEST_STAGES = %i[Request Response].freeze
|
15
|
+
RESOURCE_TYPES = %i[Document Stylesheet Image Media Font Script TextTrack
|
16
|
+
XHR Fetch Prefetch EventSource WebSocket Manifest
|
17
|
+
SignedExchange Ping CSPViolationReport Preflight Other].freeze
|
17
18
|
AUTHORIZE_BLOCK_MISSING = "Block is missing, call `authorize(...) { |r| r.continue } " \
|
18
19
|
"or subscribe to `on(:request)` events before calling it"
|
19
20
|
AUTHORIZE_TYPE_WRONG = ":type should be in #{AUTHORIZE_TYPE}"
|
@@ -187,11 +188,20 @@ module Ferrum
|
|
187
188
|
# end
|
188
189
|
# browser.go_to("https://google.com")
|
189
190
|
#
|
190
|
-
def intercept(pattern: "*", resource_type: nil)
|
191
|
+
def intercept(pattern: "*", resource_type: nil, request_stage: nil, handle_auth_requests: true)
|
191
192
|
pattern = { urlPattern: pattern }
|
192
|
-
pattern[:resourceType] = resource_type if resource_type && RESOURCE_TYPES.include?(resource_type.to_s)
|
193
193
|
|
194
|
-
|
194
|
+
if resource_type && RESOURCE_TYPES.none?(resource_type.to_sym)
|
195
|
+
raise ArgumentError, "Unknown resource type '#{resource_type}' must be #{RESOURCE_TYPES.join(' | ')}"
|
196
|
+
end
|
197
|
+
|
198
|
+
if request_stage && REQUEST_STAGES.none?(request_stage.to_sym)
|
199
|
+
raise ArgumentError, "Unknown request stage '#{request_stage}' must be #{REQUEST_STAGES.join(' | ')}"
|
200
|
+
end
|
201
|
+
|
202
|
+
pattern[:resourceType] = resource_type if resource_type
|
203
|
+
pattern[:requestStage] = request_stage if request_stage
|
204
|
+
@page.command("Fetch.enable", patterns: [pattern], handleAuthRequests: handle_auth_requests)
|
195
205
|
end
|
196
206
|
|
197
207
|
#
|
@@ -323,13 +333,23 @@ module Ferrum
|
|
323
333
|
#
|
324
334
|
# @example
|
325
335
|
# browser.network.offline_mode
|
326
|
-
# browser.go_to("https://github.com/")
|
327
|
-
#
|
336
|
+
# browser.go_to("https://github.com/")
|
337
|
+
# # => Request to https://github.com/ failed (net::ERR_INTERNET_DISCONNECTED) (Ferrum::StatusError)
|
328
338
|
#
|
329
339
|
def offline_mode
|
330
340
|
emulate_network_conditions(offline: true, latency: 0, download_throughput: 0, upload_throughput: 0)
|
331
341
|
end
|
332
342
|
|
343
|
+
#
|
344
|
+
# Toggles ignoring cache for each request. If true, cache will not be used.
|
345
|
+
#
|
346
|
+
# @example
|
347
|
+
# browser.network.cache(disable: true)
|
348
|
+
#
|
349
|
+
def cache(disable:)
|
350
|
+
@page.command("Network.setCacheDisabled", cacheDisabled: disable)
|
351
|
+
end
|
352
|
+
|
333
353
|
private
|
334
354
|
|
335
355
|
def subscribe_request_will_be_sent
|
@@ -352,6 +372,7 @@ module Ferrum
|
|
352
372
|
if params["redirectResponse"]
|
353
373
|
previous_exchange = select(request.id)[-2]
|
354
374
|
response = Network::Response.new(@page, params)
|
375
|
+
response.loaded = true
|
355
376
|
previous_exchange.response = response
|
356
377
|
end
|
357
378
|
|
@@ -374,8 +395,12 @@ module Ferrum
|
|
374
395
|
|
375
396
|
def subscribe_loading_finished
|
376
397
|
@page.on("Network.loadingFinished") do |params|
|
377
|
-
|
378
|
-
|
398
|
+
response = select(params["requestId"]).last&.response
|
399
|
+
|
400
|
+
if response
|
401
|
+
response.loaded = true
|
402
|
+
response.body_size = params["encodedDataLength"]
|
403
|
+
end
|
379
404
|
end
|
380
405
|
end
|
381
406
|
|
data/lib/ferrum/node.rb
CHANGED
@@ -88,6 +88,26 @@ module Ferrum
|
|
88
88
|
raise NotImplementedError
|
89
89
|
end
|
90
90
|
|
91
|
+
def scroll_into_view
|
92
|
+
tap { page.command("DOM.scrollIntoViewIfNeeded", nodeId: node_id) }
|
93
|
+
end
|
94
|
+
|
95
|
+
def in_viewport?(of: nil)
|
96
|
+
function = <<~JS
|
97
|
+
function(element, scope) {
|
98
|
+
const rect = element.getBoundingClientRect();
|
99
|
+
const [height, width] = scope
|
100
|
+
? [scope.offsetHeight, scope.offsetWidth]
|
101
|
+
: [window.innerHeight, window.innerWidth];
|
102
|
+
return rect.top >= 0 &&
|
103
|
+
rect.left >= 0 &&
|
104
|
+
rect.bottom <= height &&
|
105
|
+
rect.right <= width;
|
106
|
+
}
|
107
|
+
JS
|
108
|
+
page.evaluate_func(function, self, of)
|
109
|
+
end
|
110
|
+
|
91
111
|
def select_file(value)
|
92
112
|
page.command("DOM.setFileInputFiles", slowmoable: true, nodeId: node_id, files: Array(value))
|
93
113
|
end
|
@@ -208,7 +228,7 @@ module Ferrum
|
|
208
228
|
|
209
229
|
def content_quads
|
210
230
|
quads = page.command("DOM.getContentQuads", nodeId: node_id)["quads"]
|
211
|
-
raise CoordinatesNotFoundError, "Node is either not visible or not an HTMLElement" if quads.
|
231
|
+
raise CoordinatesNotFoundError, "Node is either not visible or not an HTMLElement" if quads.empty?
|
212
232
|
|
213
233
|
quads
|
214
234
|
end
|
data/lib/ferrum/page.rb
CHANGED
@@ -114,14 +114,8 @@ module Ferrum
|
|
114
114
|
options = { url: combine_url!(url) }
|
115
115
|
options.merge!(referrer: referrer) if referrer
|
116
116
|
response = command("Page.navigate", wait: GOTO_WAIT, **options)
|
117
|
-
|
118
|
-
if
|
119
|
-
net::ERR_NAME_RESOLUTION_FAILED
|
120
|
-
net::ERR_INTERNET_DISCONNECTED
|
121
|
-
net::ERR_CONNECTION_TIMED_OUT
|
122
|
-
net::ERR_FILE_NOT_FOUND].include?(response["errorText"])
|
123
|
-
raise StatusError, options[:url]
|
124
|
-
end
|
117
|
+
error_text = response["errorText"]
|
118
|
+
raise StatusError.new(options[:url], "Request to #{options[:url]} failed (#{error_text})") if error_text
|
125
119
|
|
126
120
|
response["frameId"]
|
127
121
|
rescue TimeoutError
|
@@ -151,9 +145,8 @@ module Ferrum
|
|
151
145
|
command("Emulation.setDeviceMetricsOverride", slowmoable: true,
|
152
146
|
width: width,
|
153
147
|
height: height,
|
154
|
-
deviceScaleFactor:
|
155
|
-
mobile: false
|
156
|
-
fitWindow: false)
|
148
|
+
deviceScaleFactor: 0,
|
149
|
+
mobile: false)
|
157
150
|
end
|
158
151
|
|
159
152
|
#
|
@@ -325,6 +318,10 @@ module Ferrum
|
|
325
318
|
use_proxy? && @proxy_user && @proxy_password
|
326
319
|
end
|
327
320
|
|
321
|
+
def document_node_id
|
322
|
+
command("DOM.getDocument", depth: 0).dig("root", "nodeId")
|
323
|
+
end
|
324
|
+
|
328
325
|
private
|
329
326
|
|
330
327
|
def subscribe
|
@@ -350,7 +347,7 @@ module Ferrum
|
|
350
347
|
on(:dialog) do |dialog, _index, total|
|
351
348
|
if total == 1
|
352
349
|
warn "Dialog was shown but you didn't provide `on(:dialog)` callback, accepting it by default. " \
|
353
|
-
"Please take a look at https://github.com/rubycdp/ferrum#
|
350
|
+
"Please take a look at https://github.com/rubycdp/ferrum#dialogs"
|
354
351
|
dialog.accept
|
355
352
|
end
|
356
353
|
end
|
@@ -393,7 +390,8 @@ module Ferrum
|
|
393
390
|
resize(width: width, height: height)
|
394
391
|
|
395
392
|
response = command("Page.getNavigationHistory")
|
396
|
-
|
393
|
+
transition_type = response.dig("entries", 0, "transitionType")
|
394
|
+
return if transition_type == "auto_toplevel"
|
397
395
|
|
398
396
|
# If we create page by clicking links, submitting forms and so on it
|
399
397
|
# opens a new window for which `frameStoppedLoading` event never
|
@@ -441,10 +439,6 @@ module Ferrum
|
|
441
439
|
(nil_or_relative ? @browser.base_url.join(url.to_s) : url).to_s
|
442
440
|
end
|
443
441
|
|
444
|
-
def document_node_id
|
445
|
-
command("DOM.getDocument", depth: 0).dig("root", "nodeId")
|
446
|
-
end
|
447
|
-
|
448
442
|
def ws_url
|
449
443
|
"ws://#{@browser.process.host}:#{@browser.process.port}/devtools/page/#{@target_id}"
|
450
444
|
end
|
data/lib/ferrum/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ferrum
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '0.
|
4
|
+
version: '0.14'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dmitry Vorotilin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-09-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: addressable
|
@@ -72,104 +72,6 @@ dependencies:
|
|
72
72
|
- - "<"
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: '0.8'
|
75
|
-
- !ruby/object:Gem::Dependency
|
76
|
-
name: chunky_png
|
77
|
-
requirement: !ruby/object:Gem::Requirement
|
78
|
-
requirements:
|
79
|
-
- - "~>"
|
80
|
-
- !ruby/object:Gem::Version
|
81
|
-
version: '1.3'
|
82
|
-
type: :development
|
83
|
-
prerelease: false
|
84
|
-
version_requirements: !ruby/object:Gem::Requirement
|
85
|
-
requirements:
|
86
|
-
- - "~>"
|
87
|
-
- !ruby/object:Gem::Version
|
88
|
-
version: '1.3'
|
89
|
-
- !ruby/object:Gem::Dependency
|
90
|
-
name: image_size
|
91
|
-
requirement: !ruby/object:Gem::Requirement
|
92
|
-
requirements:
|
93
|
-
- - "~>"
|
94
|
-
- !ruby/object:Gem::Version
|
95
|
-
version: '2.0'
|
96
|
-
type: :development
|
97
|
-
prerelease: false
|
98
|
-
version_requirements: !ruby/object:Gem::Requirement
|
99
|
-
requirements:
|
100
|
-
- - "~>"
|
101
|
-
- !ruby/object:Gem::Version
|
102
|
-
version: '2.0'
|
103
|
-
- !ruby/object:Gem::Dependency
|
104
|
-
name: pdf-reader
|
105
|
-
requirement: !ruby/object:Gem::Requirement
|
106
|
-
requirements:
|
107
|
-
- - "~>"
|
108
|
-
- !ruby/object:Gem::Version
|
109
|
-
version: '2.2'
|
110
|
-
type: :development
|
111
|
-
prerelease: false
|
112
|
-
version_requirements: !ruby/object:Gem::Requirement
|
113
|
-
requirements:
|
114
|
-
- - "~>"
|
115
|
-
- !ruby/object:Gem::Version
|
116
|
-
version: '2.2'
|
117
|
-
- !ruby/object:Gem::Dependency
|
118
|
-
name: puma
|
119
|
-
requirement: !ruby/object:Gem::Requirement
|
120
|
-
requirements:
|
121
|
-
- - "~>"
|
122
|
-
- !ruby/object:Gem::Version
|
123
|
-
version: '4.1'
|
124
|
-
type: :development
|
125
|
-
prerelease: false
|
126
|
-
version_requirements: !ruby/object:Gem::Requirement
|
127
|
-
requirements:
|
128
|
-
- - "~>"
|
129
|
-
- !ruby/object:Gem::Version
|
130
|
-
version: '4.1'
|
131
|
-
- !ruby/object:Gem::Dependency
|
132
|
-
name: rake
|
133
|
-
requirement: !ruby/object:Gem::Requirement
|
134
|
-
requirements:
|
135
|
-
- - "~>"
|
136
|
-
- !ruby/object:Gem::Version
|
137
|
-
version: '13.0'
|
138
|
-
type: :development
|
139
|
-
prerelease: false
|
140
|
-
version_requirements: !ruby/object:Gem::Requirement
|
141
|
-
requirements:
|
142
|
-
- - "~>"
|
143
|
-
- !ruby/object:Gem::Version
|
144
|
-
version: '13.0'
|
145
|
-
- !ruby/object:Gem::Dependency
|
146
|
-
name: rspec
|
147
|
-
requirement: !ruby/object:Gem::Requirement
|
148
|
-
requirements:
|
149
|
-
- - "~>"
|
150
|
-
- !ruby/object:Gem::Version
|
151
|
-
version: '3.8'
|
152
|
-
type: :development
|
153
|
-
prerelease: false
|
154
|
-
version_requirements: !ruby/object:Gem::Requirement
|
155
|
-
requirements:
|
156
|
-
- - "~>"
|
157
|
-
- !ruby/object:Gem::Version
|
158
|
-
version: '3.8'
|
159
|
-
- !ruby/object:Gem::Dependency
|
160
|
-
name: sinatra
|
161
|
-
requirement: !ruby/object:Gem::Requirement
|
162
|
-
requirements:
|
163
|
-
- - "~>"
|
164
|
-
- !ruby/object:Gem::Version
|
165
|
-
version: '2.0'
|
166
|
-
type: :development
|
167
|
-
prerelease: false
|
168
|
-
version_requirements: !ruby/object:Gem::Requirement
|
169
|
-
requirements:
|
170
|
-
- - "~>"
|
171
|
-
- !ruby/object:Gem::Version
|
172
|
-
version: '2.0'
|
173
75
|
description: Ferrum allows you to control headless Chrome browser
|
174
76
|
email:
|
175
77
|
- d.vorotilin@gmail.com
|
@@ -212,6 +114,7 @@ files:
|
|
212
114
|
- lib/ferrum/network/exchange.rb
|
213
115
|
- lib/ferrum/network/intercepted_request.rb
|
214
116
|
- lib/ferrum/network/request.rb
|
117
|
+
- lib/ferrum/network/request_params.rb
|
215
118
|
- lib/ferrum/network/response.rb
|
216
119
|
- lib/ferrum/node.rb
|
217
120
|
- lib/ferrum/page.rb
|
@@ -252,7 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
252
155
|
- !ruby/object:Gem::Version
|
253
156
|
version: '0'
|
254
157
|
requirements: []
|
255
|
-
rubygems_version: 3.
|
158
|
+
rubygems_version: 3.4.13
|
256
159
|
signing_key:
|
257
160
|
specification_version: 4
|
258
161
|
summary: Ruby headless Chrome driver
|