ferrum 0.12 → 0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +28 -22
- data/lib/ferrum/browser/client.rb +6 -5
- data/lib/ferrum/browser/command.rb +9 -6
- data/lib/ferrum/browser/options/base.rb +1 -4
- data/lib/ferrum/browser/options/chrome.rb +22 -10
- data/lib/ferrum/browser/options/firefox.rb +3 -6
- data/lib/ferrum/browser/options.rb +84 -0
- data/lib/ferrum/browser/process.rb +6 -7
- data/lib/ferrum/browser/version_info.rb +71 -0
- data/lib/ferrum/browser/web_socket.rb +1 -1
- data/lib/ferrum/browser/xvfb.rb +1 -1
- data/lib/ferrum/browser.rb +184 -64
- data/lib/ferrum/context.rb +3 -2
- data/lib/ferrum/contexts.rb +2 -2
- data/lib/ferrum/cookies/cookie.rb +183 -0
- data/lib/ferrum/cookies.rb +122 -49
- data/lib/ferrum/dialog.rb +30 -0
- data/lib/ferrum/frame/dom.rb +177 -0
- data/lib/ferrum/frame/runtime.rb +41 -61
- data/lib/ferrum/frame.rb +91 -3
- data/lib/ferrum/headers.rb +28 -0
- data/lib/ferrum/keyboard.rb +45 -2
- data/lib/ferrum/mouse.rb +84 -0
- data/lib/ferrum/network/exchange.rb +104 -5
- data/lib/ferrum/network/intercepted_request.rb +3 -12
- data/lib/ferrum/network/request.rb +58 -19
- data/lib/ferrum/network/request_params.rb +57 -0
- data/lib/ferrum/network/response.rb +106 -4
- data/lib/ferrum/network.rb +193 -8
- data/lib/ferrum/node.rb +21 -1
- data/lib/ferrum/page/animation.rb +16 -0
- data/lib/ferrum/page/frames.rb +66 -11
- data/lib/ferrum/page/screenshot.rb +97 -0
- data/lib/ferrum/page/tracing.rb +26 -0
- data/lib/ferrum/page.rb +158 -45
- data/lib/ferrum/proxy.rb +91 -2
- data/lib/ferrum/target.rb +6 -4
- data/lib/ferrum/version.rb +1 -1
- metadata +7 -101
data/lib/ferrum/browser.rb
CHANGED
@@ -6,16 +6,14 @@ require "ferrum/page"
|
|
6
6
|
require "ferrum/proxy"
|
7
7
|
require "ferrum/contexts"
|
8
8
|
require "ferrum/browser/xvfb"
|
9
|
+
require "ferrum/browser/options"
|
9
10
|
require "ferrum/browser/process"
|
10
11
|
require "ferrum/browser/client"
|
11
12
|
require "ferrum/browser/binary"
|
13
|
+
require "ferrum/browser/version_info"
|
12
14
|
|
13
15
|
module Ferrum
|
14
16
|
class Browser
|
15
|
-
DEFAULT_TIMEOUT = ENV.fetch("FERRUM_DEFAULT_TIMEOUT", 5).to_i
|
16
|
-
WINDOW_SIZE = [1024, 768].freeze
|
17
|
-
BASE_URL_SCHEMA = %w[http https].freeze
|
18
|
-
|
19
17
|
extend Forwardable
|
20
18
|
delegate %i[default_context] => :contexts
|
21
19
|
delegate %i[targets create_target page pages windows] => :default_context
|
@@ -24,7 +22,7 @@ module Ferrum
|
|
24
22
|
body doctype content=
|
25
23
|
headers cookies network
|
26
24
|
mouse keyboard
|
27
|
-
screenshot pdf mhtml viewport_size
|
25
|
+
screenshot pdf mhtml viewport_size device_pixel_ratio
|
28
26
|
frames frame_by main_frame
|
29
27
|
evaluate evaluate_on evaluate_async execute evaluate_func
|
30
28
|
add_script_tag add_style_tag bypass_csp
|
@@ -32,66 +30,146 @@ module Ferrum
|
|
32
30
|
playback_rate playback_rate=] => :page
|
33
31
|
delegate %i[default_user_agent] => :process
|
34
32
|
|
35
|
-
attr_reader :client, :process, :contexts, :
|
36
|
-
|
37
|
-
:proxy_server
|
38
|
-
attr_writer :timeout
|
33
|
+
attr_reader :client, :process, :contexts, :options, :window_size, :base_url
|
34
|
+
attr_accessor :timeout
|
39
35
|
|
36
|
+
#
|
37
|
+
# Initializes the browser.
|
38
|
+
#
|
39
|
+
# @param [Hash{Symbol => Object}, nil] options
|
40
|
+
# Additional browser options.
|
41
|
+
#
|
42
|
+
# @option options [Boolean] :headless (true)
|
43
|
+
# Set browser as headless or not.
|
44
|
+
#
|
45
|
+
# @option options [Boolean] :xvfb (false)
|
46
|
+
# Run browser in a virtual framebuffer.
|
47
|
+
#
|
48
|
+
# @option options [(Integer, Integer)] :window_size ([1024, 768])
|
49
|
+
# The dimensions of the browser window in which to test, expressed as a
|
50
|
+
# 2-element array, e.g. `[1024, 768]`.
|
51
|
+
#
|
52
|
+
# @option options [Array<String, Hash>] :extensions
|
53
|
+
# An array of paths to files or JS source code to be preloaded into the
|
54
|
+
# browser e.g.: `["/path/to/script.js", { source: "window.secret = 'top'" }]`
|
55
|
+
#
|
56
|
+
# @option options [#puts] :logger
|
57
|
+
# When present, debug output is written to this object.
|
58
|
+
#
|
59
|
+
# @option options [Integer, Float] :slowmo
|
60
|
+
# Set a delay in seconds to wait before sending command.
|
61
|
+
# Useful companion of headless option, so that you have time to see
|
62
|
+
# changes.
|
63
|
+
#
|
64
|
+
# @option options [Numeric] :timeout (5)
|
65
|
+
# The number of seconds we'll wait for a response when communicating with
|
66
|
+
# browser.
|
67
|
+
#
|
68
|
+
# @option options [Boolean] :js_errors
|
69
|
+
# When true, JavaScript errors get re-raised in Ruby.
|
70
|
+
#
|
71
|
+
# @option options [Boolean] :pending_connection_errors (true)
|
72
|
+
# When main frame is still waiting for slow responses while timeout is
|
73
|
+
# reached {PendingConnectionsError} is raised. It's better to figure out
|
74
|
+
# why you have slow responses and fix or block them rather than turn this
|
75
|
+
# setting off.
|
76
|
+
#
|
77
|
+
# @option options [:chrome, :firefox] :browser_name (:chrome)
|
78
|
+
# Sets the browser's name. **Note:** only experimental support for
|
79
|
+
# `:firefox` for now.
|
80
|
+
#
|
81
|
+
# @option options [String] :browser_path
|
82
|
+
# Path to Chrome binary, you can also set ENV variable as
|
83
|
+
# `BROWSER_PATH=some/path/chrome bundle exec rspec`.
|
84
|
+
#
|
85
|
+
# @option options [Hash] :browser_options
|
86
|
+
# Additional command line options, [see them all](https://peter.sh/experiments/chromium-command-line-switches/)
|
87
|
+
# e.g. `{ "ignore-certificate-errors" => nil }`
|
88
|
+
#
|
89
|
+
# @option options [Boolean] :ignore_default_browser_options
|
90
|
+
# Ferrum has a number of default options it passes to the browser,
|
91
|
+
# if you set this to `true` then only options you put in
|
92
|
+
# `:browser_options` will be passed to the browser, except required ones
|
93
|
+
# of course.
|
94
|
+
#
|
95
|
+
# @option options [Integer] :port
|
96
|
+
# Remote debugging port for headless Chrome.
|
97
|
+
#
|
98
|
+
# @option options [String] :host
|
99
|
+
# Remote debugging address for headless Chrome.
|
100
|
+
#
|
101
|
+
# @option options [String] :url
|
102
|
+
# URL for a running instance of Chrome. If this is set, a browser process
|
103
|
+
# will not be spawned.
|
104
|
+
#
|
105
|
+
# @option options [Integer] :process_timeout
|
106
|
+
# How long to wait for the Chrome process to respond on startup.
|
107
|
+
#
|
108
|
+
# @option options [Integer] :ws_max_receive_size
|
109
|
+
# How big messages to accept from Chrome over the web socket, in bytes.
|
110
|
+
# Defaults to 64MB. Incoming messages larger this will cause a
|
111
|
+
# {Ferrum::DeadBrowserError}.
|
112
|
+
#
|
113
|
+
# @option options [Hash] :proxy
|
114
|
+
# Specify proxy settings, [read more](https://github.com/rubycdp/ferrum#proxy).
|
115
|
+
#
|
116
|
+
# @option options [String] :save_path
|
117
|
+
# Path to save attachments with [Content-Disposition](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition)
|
118
|
+
# header.
|
119
|
+
#
|
120
|
+
# @option options [Hash] :env
|
121
|
+
# Environment variables you'd like to pass through to the process.
|
122
|
+
#
|
40
123
|
def initialize(options = nil)
|
41
|
-
options
|
42
|
-
|
43
|
-
@client = nil
|
44
|
-
@window_size = options.fetch(:window_size, WINDOW_SIZE)
|
45
|
-
@original_window_size = @window_size
|
46
|
-
|
47
|
-
@options = Hash(options.merge(window_size: @window_size))
|
48
|
-
@logger, @timeout, @ws_max_receive_size =
|
49
|
-
@options.values_at(:logger, :timeout, :ws_max_receive_size)
|
50
|
-
@js_errors = @options.fetch(:js_errors, false)
|
51
|
-
|
52
|
-
if @options[:proxy]
|
53
|
-
@proxy_options = @options[:proxy]
|
54
|
-
|
55
|
-
if @proxy_options[:server]
|
56
|
-
@proxy_server = Proxy.start(**@proxy_options.slice(:host, :port, :user, :password))
|
57
|
-
@proxy_options.merge!(host: @proxy_server.host, port: @proxy_server.port)
|
58
|
-
end
|
59
|
-
|
60
|
-
@options[:browser_options] ||= {}
|
61
|
-
address = "#{@proxy_options[:host]}:#{@proxy_options[:port]}"
|
62
|
-
@options[:browser_options].merge!("proxy-server" => address)
|
63
|
-
@options[:browser_options].merge!("proxy-bypass-list" => @proxy_options[:bypass]) if @proxy_options[:bypass]
|
64
|
-
end
|
65
|
-
|
66
|
-
@pending_connection_errors = @options.fetch(:pending_connection_errors, true)
|
67
|
-
@slowmo = @options[:slowmo].to_f
|
68
|
-
|
69
|
-
self.base_url = @options[:base_url] if @options.key?(:base_url)
|
70
|
-
|
71
|
-
if ENV.fetch("FERRUM_DEBUG", nil) && !@logger
|
72
|
-
$stdout.sync = true
|
73
|
-
@logger = $stdout
|
74
|
-
@options[:logger] = @logger
|
75
|
-
end
|
124
|
+
@options = Options.new(options)
|
125
|
+
@client = @process = @contexts = nil
|
76
126
|
|
77
|
-
@options.
|
127
|
+
@timeout = @options.timeout
|
128
|
+
@window_size = @options.window_size
|
129
|
+
@base_url = @options.base_url if @options.base_url
|
78
130
|
|
79
131
|
start
|
80
132
|
end
|
81
133
|
|
134
|
+
#
|
135
|
+
# Sets the base URL.
|
136
|
+
#
|
137
|
+
# @param [String] value
|
138
|
+
# The new base URL value.
|
139
|
+
#
|
140
|
+
# @raise [ArgumentError] when path is not absolute or doesn't include schema
|
141
|
+
#
|
142
|
+
# @return [Addressable::URI]
|
143
|
+
# The parsed base URI value.
|
144
|
+
#
|
82
145
|
def base_url=(value)
|
83
|
-
|
84
|
-
unless BASE_URL_SCHEMA.include?(parsed.normalized_scheme)
|
85
|
-
raise "Set `base_url` should be absolute and include schema: #{BASE_URL_SCHEMA}"
|
86
|
-
end
|
87
|
-
|
88
|
-
@base_url = parsed
|
146
|
+
@base_url = options.parse_base_url(value)
|
89
147
|
end
|
90
148
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
149
|
+
#
|
150
|
+
# Creates a new page.
|
151
|
+
#
|
152
|
+
# @param [Boolean] new_context
|
153
|
+
# Whether to create a page in a new context or not.
|
154
|
+
#
|
155
|
+
# @param [Hash] proxy
|
156
|
+
# Whether to use proxy for a page. The page will be created in a new context if so.
|
157
|
+
#
|
158
|
+
# @return [Ferrum::Page]
|
159
|
+
# Created page.
|
160
|
+
#
|
161
|
+
def create_page(new_context: false, proxy: nil)
|
162
|
+
page = if new_context || proxy
|
163
|
+
params = {}
|
164
|
+
|
165
|
+
if proxy
|
166
|
+
options.parse_proxy(proxy)
|
167
|
+
params.merge!(proxyServer: "#{proxy[:host]}:#{proxy[:port]}")
|
168
|
+
params.merge!(proxyBypassList: proxy[:bypass]) if proxy[:bypass]
|
169
|
+
end
|
170
|
+
|
171
|
+
context = contexts.create(**params)
|
172
|
+
context.create_page(proxy: proxy)
|
95
173
|
else
|
96
174
|
default_context.create_page
|
97
175
|
end
|
@@ -99,25 +177,34 @@ module Ferrum
|
|
99
177
|
block_given? ? yield(page) : page
|
100
178
|
ensure
|
101
179
|
if block_given?
|
102
|
-
page
|
180
|
+
page&.close
|
103
181
|
context.dispose if new_context
|
104
182
|
end
|
105
183
|
end
|
106
184
|
|
107
185
|
def extensions
|
108
|
-
@extensions ||= Array(
|
186
|
+
@extensions ||= Array(options.extensions).map do |ext|
|
109
187
|
(ext.is_a?(Hash) && ext[:source]) || File.read(ext)
|
110
188
|
end
|
111
189
|
end
|
112
190
|
|
191
|
+
#
|
192
|
+
# Evaluate JavaScript to modify things before a page load.
|
193
|
+
#
|
194
|
+
# @param [String] expression
|
195
|
+
# The JavaScript to add to each new document.
|
196
|
+
#
|
197
|
+
# @example
|
198
|
+
# browser.evaluate_on_new_document <<~JS
|
199
|
+
# Object.defineProperty(navigator, "languages", {
|
200
|
+
# get: function() { return ["tlh"]; }
|
201
|
+
# });
|
202
|
+
# JS
|
203
|
+
#
|
113
204
|
def evaluate_on_new_document(expression)
|
114
205
|
extensions << expression
|
115
206
|
end
|
116
207
|
|
117
|
-
def timeout
|
118
|
-
@timeout || DEFAULT_TIMEOUT
|
119
|
-
end
|
120
|
-
|
121
208
|
def command(*args)
|
122
209
|
@client.command(*args)
|
123
210
|
rescue DeadBrowserError
|
@@ -125,8 +212,22 @@ module Ferrum
|
|
125
212
|
raise
|
126
213
|
end
|
127
214
|
|
215
|
+
#
|
216
|
+
# Closes browser tabs opened by the `Browser` instance.
|
217
|
+
#
|
218
|
+
# @example
|
219
|
+
# # connect to a long-running Chrome process
|
220
|
+
# browser = Ferrum::Browser.new(url: 'http://localhost:9222')
|
221
|
+
#
|
222
|
+
# browser.go_to("https://github.com/")
|
223
|
+
#
|
224
|
+
# # clean up, lest the tab stays there hanging forever
|
225
|
+
# browser.reset
|
226
|
+
#
|
227
|
+
# browser.quit
|
228
|
+
#
|
128
229
|
def reset
|
129
|
-
@window_size =
|
230
|
+
@window_size = options.window_size
|
130
231
|
contexts.reset
|
131
232
|
end
|
132
233
|
|
@@ -136,6 +237,8 @@ module Ferrum
|
|
136
237
|
end
|
137
238
|
|
138
239
|
def quit
|
240
|
+
return unless @client
|
241
|
+
|
139
242
|
@client.close
|
140
243
|
@process.stop
|
141
244
|
@client = @process = @contexts = nil
|
@@ -150,12 +253,29 @@ module Ferrum
|
|
150
253
|
command("Browser.crash")
|
151
254
|
end
|
152
255
|
|
256
|
+
#
|
257
|
+
# Gets the version information from the browser.
|
258
|
+
#
|
259
|
+
# @return [VersionInfo]
|
260
|
+
#
|
261
|
+
# @since 0.13
|
262
|
+
#
|
263
|
+
def version
|
264
|
+
VersionInfo.new(command("Browser.getVersion"))
|
265
|
+
end
|
266
|
+
|
267
|
+
def headless_new?
|
268
|
+
process&.command&.headless_new?
|
269
|
+
end
|
270
|
+
|
153
271
|
private
|
154
272
|
|
155
273
|
def start
|
156
274
|
Utils::ElapsedTime.start
|
157
|
-
@process = Process.start(
|
158
|
-
@client = Client.new(
|
275
|
+
@process = Process.start(options)
|
276
|
+
@client = Client.new(@process.ws_url, self,
|
277
|
+
logger: options.logger,
|
278
|
+
ws_max_receive_size: options.ws_max_receive_size)
|
159
279
|
@contexts = Contexts.new(self)
|
160
280
|
end
|
161
281
|
end
|
data/lib/ferrum/context.rb
CHANGED
data/lib/ferrum/contexts.rb
CHANGED
@@ -23,8 +23,8 @@ module Ferrum
|
|
23
23
|
context
|
24
24
|
end
|
25
25
|
|
26
|
-
def create
|
27
|
-
response = @browser.command("Target.createBrowserContext")
|
26
|
+
def create(**options)
|
27
|
+
response = @browser.command("Target.createBrowserContext", **options)
|
28
28
|
context_id = response["browserContextId"]
|
29
29
|
context = Context.new(@browser, self, context_id)
|
30
30
|
@contexts[context_id] = context
|
@@ -0,0 +1,183 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Ferrum
|
4
|
+
class Cookies
|
5
|
+
#
|
6
|
+
# Represents a [cookie value](https://chromedevtools.github.io/devtools-protocol/1-3/Network/#type-Cookie).
|
7
|
+
#
|
8
|
+
class Cookie
|
9
|
+
# The parsed JSON attributes.
|
10
|
+
#
|
11
|
+
# @return [Hash{String => [String, Boolean, nil]}]
|
12
|
+
attr_reader :attributes
|
13
|
+
|
14
|
+
#
|
15
|
+
# Initializes the cookie.
|
16
|
+
#
|
17
|
+
# @param [Hash{String => String}] attributes
|
18
|
+
# The parsed JSON attributes.
|
19
|
+
#
|
20
|
+
def initialize(attributes)
|
21
|
+
@attributes = attributes
|
22
|
+
end
|
23
|
+
|
24
|
+
#
|
25
|
+
# The cookie's name.
|
26
|
+
#
|
27
|
+
# @return [String]
|
28
|
+
#
|
29
|
+
def name
|
30
|
+
attributes["name"]
|
31
|
+
end
|
32
|
+
|
33
|
+
#
|
34
|
+
# The cookie's value.
|
35
|
+
#
|
36
|
+
# @return [String]
|
37
|
+
#
|
38
|
+
def value
|
39
|
+
attributes["value"]
|
40
|
+
end
|
41
|
+
|
42
|
+
#
|
43
|
+
# The cookie's domain.
|
44
|
+
#
|
45
|
+
# @return [String]
|
46
|
+
#
|
47
|
+
def domain
|
48
|
+
attributes["domain"]
|
49
|
+
end
|
50
|
+
|
51
|
+
#
|
52
|
+
# The cookie's path.
|
53
|
+
#
|
54
|
+
# @return [String]
|
55
|
+
#
|
56
|
+
def path
|
57
|
+
attributes["path"]
|
58
|
+
end
|
59
|
+
|
60
|
+
#
|
61
|
+
# The `sameSite` configuration.
|
62
|
+
#
|
63
|
+
# @return ["Strict", "Lax", "None", nil]
|
64
|
+
#
|
65
|
+
def samesite
|
66
|
+
attributes["sameSite"]
|
67
|
+
end
|
68
|
+
alias same_site samesite
|
69
|
+
|
70
|
+
#
|
71
|
+
# The cookie's size.
|
72
|
+
#
|
73
|
+
# @return [Integer]
|
74
|
+
#
|
75
|
+
def size
|
76
|
+
attributes["size"]
|
77
|
+
end
|
78
|
+
|
79
|
+
#
|
80
|
+
# Specifies whether the cookie is secure or not.
|
81
|
+
#
|
82
|
+
# @return [Boolean]
|
83
|
+
#
|
84
|
+
def secure?
|
85
|
+
attributes["secure"]
|
86
|
+
end
|
87
|
+
|
88
|
+
#
|
89
|
+
# Specifies whether the cookie is HTTP-only or not.
|
90
|
+
#
|
91
|
+
# @return [Boolean]
|
92
|
+
#
|
93
|
+
def httponly?
|
94
|
+
attributes["httpOnly"]
|
95
|
+
end
|
96
|
+
alias http_only? httponly?
|
97
|
+
|
98
|
+
#
|
99
|
+
# Specifies whether the cookie is a session cookie or not.
|
100
|
+
#
|
101
|
+
# @return [Boolean]
|
102
|
+
#
|
103
|
+
def session?
|
104
|
+
attributes["session"]
|
105
|
+
end
|
106
|
+
|
107
|
+
#
|
108
|
+
# Specifies when the cookie will expire.
|
109
|
+
#
|
110
|
+
# @return [Time, nil]
|
111
|
+
#
|
112
|
+
def expires
|
113
|
+
Time.at(attributes["expires"]) if attributes["expires"].positive?
|
114
|
+
end
|
115
|
+
|
116
|
+
#
|
117
|
+
# The priority of the cookie.
|
118
|
+
#
|
119
|
+
# @return [String]
|
120
|
+
#
|
121
|
+
def priority
|
122
|
+
@attributes["priority"]
|
123
|
+
end
|
124
|
+
|
125
|
+
#
|
126
|
+
# @return [Boolean]
|
127
|
+
#
|
128
|
+
def sameparty?
|
129
|
+
@attributes["sameParty"]
|
130
|
+
end
|
131
|
+
|
132
|
+
alias same_party? sameparty?
|
133
|
+
|
134
|
+
#
|
135
|
+
# @return [String]
|
136
|
+
#
|
137
|
+
def source_scheme
|
138
|
+
@attributes["sourceScheme"]
|
139
|
+
end
|
140
|
+
|
141
|
+
#
|
142
|
+
# @return [Integer]
|
143
|
+
#
|
144
|
+
def source_port
|
145
|
+
@attributes["sourcePort"]
|
146
|
+
end
|
147
|
+
|
148
|
+
#
|
149
|
+
# Compares different cookie objects.
|
150
|
+
#
|
151
|
+
# @return [Boolean]
|
152
|
+
#
|
153
|
+
def ==(other)
|
154
|
+
other.class == self.class && other.attributes == attributes
|
155
|
+
end
|
156
|
+
|
157
|
+
#
|
158
|
+
# Converts the cookie back into a raw cookie String.
|
159
|
+
#
|
160
|
+
# @return [String]
|
161
|
+
# The raw cookie string.
|
162
|
+
#
|
163
|
+
def to_s
|
164
|
+
string = String.new("#{@attributes['name']}=#{@attributes['value']}")
|
165
|
+
|
166
|
+
@attributes.each do |key, value|
|
167
|
+
case key
|
168
|
+
when "name", "value" # no-op
|
169
|
+
when "domain" then string << "; Domain=#{value}"
|
170
|
+
when "path" then string << "; Path=#{value}"
|
171
|
+
when "expires" then string << "; Expires=#{Time.at(value).httpdate}"
|
172
|
+
when "httpOnly" then string << "; httpOnly" if value
|
173
|
+
when "secure" then string << "; Secure" if value
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
string
|
178
|
+
end
|
179
|
+
|
180
|
+
alias to_h attributes
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|