ferrum 0.11 → 0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +1 -1
- data/README.md +174 -30
- data/lib/ferrum/browser/binary.rb +46 -0
- data/lib/ferrum/browser/client.rb +17 -16
- data/lib/ferrum/browser/command.rb +10 -12
- data/lib/ferrum/browser/options/base.rb +2 -11
- data/lib/ferrum/browser/options/chrome.rb +29 -18
- data/lib/ferrum/browser/options/firefox.rb +13 -9
- data/lib/ferrum/browser/options.rb +84 -0
- data/lib/ferrum/browser/process.rb +45 -40
- data/lib/ferrum/browser/subscriber.rb +1 -3
- data/lib/ferrum/browser/version_info.rb +71 -0
- data/lib/ferrum/browser/web_socket.rb +9 -12
- data/lib/ferrum/browser/xvfb.rb +4 -8
- data/lib/ferrum/browser.rb +193 -47
- data/lib/ferrum/context.rb +9 -4
- data/lib/ferrum/contexts.rb +12 -10
- data/lib/ferrum/cookies/cookie.rb +126 -0
- data/lib/ferrum/cookies.rb +93 -55
- data/lib/ferrum/dialog.rb +30 -0
- data/lib/ferrum/errors.rb +115 -0
- data/lib/ferrum/frame/dom.rb +177 -0
- data/lib/ferrum/frame/runtime.rb +58 -75
- data/lib/ferrum/frame.rb +118 -23
- data/lib/ferrum/headers.rb +30 -2
- data/lib/ferrum/keyboard.rb +56 -13
- data/lib/ferrum/mouse.rb +92 -7
- data/lib/ferrum/network/auth_request.rb +7 -2
- data/lib/ferrum/network/exchange.rb +97 -12
- data/lib/ferrum/network/intercepted_request.rb +10 -8
- data/lib/ferrum/network/request.rb +69 -0
- data/lib/ferrum/network/response.rb +85 -3
- data/lib/ferrum/network.rb +285 -36
- data/lib/ferrum/node.rb +69 -23
- data/lib/ferrum/page/animation.rb +16 -1
- data/lib/ferrum/page/frames.rb +111 -30
- data/lib/ferrum/page/screenshot.rb +142 -65
- data/lib/ferrum/page/stream.rb +38 -0
- data/lib/ferrum/page/tracing.rb +97 -0
- data/lib/ferrum/page.rb +224 -60
- data/lib/ferrum/proxy.rb +147 -0
- data/lib/ferrum/{rbga.rb → rgba.rb} +4 -2
- data/lib/ferrum/target.rb +7 -4
- data/lib/ferrum/utils/attempt.rb +20 -0
- data/lib/ferrum/utils/elapsed_time.rb +27 -0
- data/lib/ferrum/utils/platform.rb +28 -0
- data/lib/ferrum/version.rb +1 -1
- data/lib/ferrum.rb +4 -146
- metadata +63 -51
data/lib/ferrum/page.rb
CHANGED
@@ -10,6 +10,8 @@ require "ferrum/network"
|
|
10
10
|
require "ferrum/page/frames"
|
11
11
|
require "ferrum/page/screenshot"
|
12
12
|
require "ferrum/page/animation"
|
13
|
+
require "ferrum/page/tracing"
|
14
|
+
require "ferrum/page/stream"
|
13
15
|
require "ferrum/browser/client"
|
14
16
|
|
15
17
|
module Ferrum
|
@@ -32,40 +34,82 @@ module Ferrum
|
|
32
34
|
|
33
35
|
extend Forwardable
|
34
36
|
delegate %i[at_css at_xpath css xpath
|
35
|
-
current_url current_title url title body doctype
|
36
|
-
execution_id evaluate evaluate_on evaluate_async execute evaluate_func
|
37
|
+
current_url current_title url title body doctype content=
|
38
|
+
execution_id execution_id! evaluate evaluate_on evaluate_async execute evaluate_func
|
37
39
|
add_script_tag add_style_tag] => :main_frame
|
38
40
|
|
39
|
-
include
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
41
|
+
include Animation
|
42
|
+
include Screenshot
|
43
|
+
include Frames
|
44
|
+
include Stream
|
45
|
+
|
46
|
+
attr_accessor :referrer, :timeout
|
47
|
+
attr_reader :target_id, :browser, :event, :tracing
|
48
|
+
|
49
|
+
# Mouse object.
|
50
|
+
#
|
51
|
+
# @return [Mouse]
|
52
|
+
attr_reader :mouse
|
53
|
+
|
54
|
+
# Keyboard object.
|
55
|
+
#
|
56
|
+
# @return [Keyboard]
|
57
|
+
attr_reader :keyboard
|
58
|
+
|
59
|
+
# Network object.
|
60
|
+
#
|
61
|
+
# @return [Network]
|
62
|
+
attr_reader :network
|
63
|
+
|
64
|
+
# Headers object.
|
65
|
+
#
|
66
|
+
# @return [Headers]
|
67
|
+
attr_reader :headers
|
68
|
+
|
69
|
+
# Cookie store.
|
70
|
+
#
|
71
|
+
# @return [Cookies]
|
72
|
+
attr_reader :cookies
|
73
|
+
|
74
|
+
def initialize(target_id, browser, proxy: nil)
|
75
|
+
@frames = Concurrent::Map.new
|
48
76
|
@main_frame = Frame.new(nil, self)
|
49
|
-
@
|
77
|
+
@browser = browser
|
78
|
+
@target_id = target_id
|
79
|
+
@timeout = @browser.timeout
|
50
80
|
@event = Event.new.tap(&:set)
|
81
|
+
self.proxy = proxy
|
51
82
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
83
|
+
@client = Browser::Client.new(ws_url, self,
|
84
|
+
logger: @browser.options.logger,
|
85
|
+
ws_max_receive_size: @browser.options.ws_max_receive_size,
|
86
|
+
id_starts_with: 1000)
|
56
87
|
|
57
|
-
@mouse
|
58
|
-
@
|
88
|
+
@mouse = Mouse.new(self)
|
89
|
+
@keyboard = Keyboard.new(self)
|
90
|
+
@headers = Headers.new(self)
|
91
|
+
@cookies = Cookies.new(self)
|
59
92
|
@network = Network.new(self)
|
93
|
+
@tracing = Tracing.new(self)
|
60
94
|
|
61
95
|
subscribe
|
62
96
|
prepare_page
|
63
97
|
end
|
64
98
|
|
65
|
-
def
|
66
|
-
@browser.
|
99
|
+
def context
|
100
|
+
@browser.contexts.find_by(target_id: target_id)
|
67
101
|
end
|
68
102
|
|
103
|
+
#
|
104
|
+
# Navigates the page to a URL.
|
105
|
+
#
|
106
|
+
# @param [String, nil] url
|
107
|
+
# The URL to navigate to. The url should include scheme unless you set
|
108
|
+
# `{Browser#base_url = url}` when configuring driver.
|
109
|
+
#
|
110
|
+
# @example
|
111
|
+
# browser.go_to("https://github.com/")
|
112
|
+
#
|
69
113
|
def go_to(url = nil)
|
70
114
|
options = { url: combine_url!(url) }
|
71
115
|
options.merge!(referrer: referrer) if referrer
|
@@ -74,17 +118,20 @@ module Ferrum
|
|
74
118
|
if %w[net::ERR_NAME_NOT_RESOLVED
|
75
119
|
net::ERR_NAME_RESOLUTION_FAILED
|
76
120
|
net::ERR_INTERNET_DISCONNECTED
|
77
|
-
net::ERR_CONNECTION_TIMED_OUT
|
121
|
+
net::ERR_CONNECTION_TIMED_OUT
|
122
|
+
net::ERR_FILE_NOT_FOUND].include?(response["errorText"])
|
78
123
|
raise StatusError, options[:url]
|
79
124
|
end
|
125
|
+
|
80
126
|
response["frameId"]
|
81
127
|
rescue TimeoutError
|
82
|
-
if @browser.pending_connection_errors
|
83
|
-
pendings = network.traffic.select(&:pending?).map
|
128
|
+
if @browser.options.pending_connection_errors
|
129
|
+
pendings = network.traffic.select(&:pending?).map(&:url).compact
|
84
130
|
raise PendingConnectionsError.new(options[:url], pendings) unless pendings.empty?
|
85
131
|
end
|
86
132
|
end
|
87
133
|
alias goto go_to
|
134
|
+
alias go go_to
|
88
135
|
|
89
136
|
def close
|
90
137
|
@headers.clear
|
@@ -109,27 +156,83 @@ module Ferrum
|
|
109
156
|
fitWindow: false)
|
110
157
|
end
|
111
158
|
|
159
|
+
#
|
160
|
+
# The current position of the browser window.
|
161
|
+
#
|
162
|
+
# @return [(Integer, Integer)]
|
163
|
+
# The left, top coordinates of the browser window.
|
164
|
+
#
|
165
|
+
# @example
|
166
|
+
# browser.position # => [10, 20]
|
167
|
+
#
|
112
168
|
def position
|
113
169
|
@browser.command("Browser.getWindowBounds", windowId: window_id).fetch("bounds").values_at("left", "top")
|
114
170
|
end
|
115
171
|
|
116
|
-
|
117
|
-
|
172
|
+
#
|
173
|
+
# Sets the position of the browser window.
|
174
|
+
#
|
175
|
+
# @param [Hash{Symbol => Object}] options
|
176
|
+
#
|
177
|
+
# @option options [Integer] :left
|
178
|
+
# The number of pixels from the left-hand side of the screen.
|
179
|
+
#
|
180
|
+
# @option options [Integer] :top
|
181
|
+
# The number of pixels from the top of the screen.
|
182
|
+
#
|
183
|
+
# @example
|
184
|
+
# browser.position = { left: 10, top: 20 }
|
185
|
+
#
|
186
|
+
def position=(options)
|
187
|
+
@browser.command("Browser.setWindowBounds",
|
188
|
+
windowId: window_id,
|
189
|
+
bounds: { left: options[:left], top: options[:top] })
|
118
190
|
end
|
119
191
|
|
192
|
+
#
|
193
|
+
# Reloads the current page.
|
194
|
+
#
|
195
|
+
# @example
|
196
|
+
# browser.go_to("https://github.com/")
|
197
|
+
# browser.refresh
|
198
|
+
#
|
120
199
|
def refresh
|
121
200
|
command("Page.reload", wait: timeout, slowmoable: true)
|
122
201
|
end
|
123
|
-
|
124
|
-
|
202
|
+
alias reload refresh
|
203
|
+
|
204
|
+
#
|
205
|
+
# Stop all navigations and loading pending resources on the page.
|
206
|
+
#
|
207
|
+
# @example
|
208
|
+
# browser.go_to("https://github.com/")
|
209
|
+
# browser.stop
|
210
|
+
#
|
125
211
|
def stop
|
126
212
|
command("Page.stopLoading", slowmoable: true)
|
127
213
|
end
|
128
214
|
|
215
|
+
#
|
216
|
+
# Navigates to the previous URL in the browser's history.
|
217
|
+
#
|
218
|
+
# @example
|
219
|
+
# browser.go_to("https://github.com/")
|
220
|
+
# browser.at_xpath("//a").click
|
221
|
+
# browser.back
|
222
|
+
#
|
129
223
|
def back
|
130
224
|
history_navigate(delta: -1)
|
131
225
|
end
|
132
226
|
|
227
|
+
#
|
228
|
+
# Navigates to the next URL in the browser's history.
|
229
|
+
#
|
230
|
+
# @example
|
231
|
+
# browser.go_to("https://github.com/")
|
232
|
+
# browser.at_xpath("//a").click
|
233
|
+
# browser.back
|
234
|
+
# browser.forward
|
235
|
+
#
|
133
236
|
def forward
|
134
237
|
history_navigate(delta: 1)
|
135
238
|
end
|
@@ -140,8 +243,21 @@ module Ferrum
|
|
140
243
|
@event.set
|
141
244
|
end
|
142
245
|
|
143
|
-
|
144
|
-
|
246
|
+
#
|
247
|
+
# Enables/disables CSP bypass.
|
248
|
+
#
|
249
|
+
# @param [Boolean] enabled
|
250
|
+
#
|
251
|
+
# @return [Boolean]
|
252
|
+
#
|
253
|
+
# @example
|
254
|
+
# browser.bypass_csp # => true
|
255
|
+
# browser.go_to("https://github.com/ruby-concurrency/concurrent-ruby/blob/master/docs-source/promises.in.md")
|
256
|
+
# browser.refresh
|
257
|
+
# browser.add_script_tag(content: "window.__injected = 42")
|
258
|
+
# browser.evaluate("window.__injected") # => 42
|
259
|
+
#
|
260
|
+
def bypass_csp(enabled: true)
|
145
261
|
command("Page.setBypassCSP", enabled: enabled)
|
146
262
|
enabled
|
147
263
|
end
|
@@ -155,16 +271,17 @@ module Ferrum
|
|
155
271
|
end
|
156
272
|
|
157
273
|
def command(method, wait: 0, slowmoable: false, **params)
|
158
|
-
iteration = @event.reset if wait
|
159
|
-
sleep(@browser.slowmo) if slowmoable && @browser.slowmo
|
274
|
+
iteration = @event.reset if wait.positive?
|
275
|
+
sleep(@browser.options.slowmo) if slowmoable && @browser.options.slowmo.positive?
|
160
276
|
result = @client.command(method, params)
|
161
277
|
|
162
|
-
if wait
|
163
|
-
|
164
|
-
|
165
|
-
|
278
|
+
if wait.positive?
|
279
|
+
# Wait a bit after command and check if iteration has
|
280
|
+
# changed which means there was some network event for
|
281
|
+
# the main frame and it started to load new content.
|
282
|
+
@event.wait(wait)
|
166
283
|
if iteration != @event.iteration
|
167
|
-
set = @event.wait(
|
284
|
+
set = @event.wait(timeout)
|
168
285
|
raise TimeoutError unless set
|
169
286
|
end
|
170
287
|
end
|
@@ -200,22 +317,41 @@ module Ferrum
|
|
200
317
|
@client.subscribed?(event)
|
201
318
|
end
|
202
319
|
|
320
|
+
def use_proxy?
|
321
|
+
@proxy_host && @proxy_port
|
322
|
+
end
|
323
|
+
|
324
|
+
def use_authorized_proxy?
|
325
|
+
use_proxy? && @proxy_user && @proxy_password
|
326
|
+
end
|
327
|
+
|
203
328
|
private
|
204
329
|
|
205
330
|
def subscribe
|
206
331
|
frames_subscribe
|
207
332
|
network.subscribe
|
208
333
|
|
209
|
-
if @browser.logger
|
334
|
+
if @browser.options.logger
|
210
335
|
on("Runtime.consoleAPICalled") do |params|
|
211
|
-
params["args"].each { |r| @browser.logger.puts(r["value"]) }
|
336
|
+
params["args"].each { |r| @browser.options.logger.puts(r["value"]) }
|
212
337
|
end
|
213
338
|
end
|
214
339
|
|
215
|
-
if @browser.js_errors
|
340
|
+
if @browser.options.js_errors
|
216
341
|
on("Runtime.exceptionThrown") do |params|
|
217
|
-
# FIXME https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
|
218
|
-
Thread.main.raise JavaScriptError.new(
|
342
|
+
# FIXME: https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
|
343
|
+
Thread.main.raise JavaScriptError.new(
|
344
|
+
params.dig("exceptionDetails", "exception"),
|
345
|
+
params.dig("exceptionDetails", "stackTrace")
|
346
|
+
)
|
347
|
+
end
|
348
|
+
end
|
349
|
+
|
350
|
+
on(:dialog) do |dialog, _index, total|
|
351
|
+
if total == 1
|
352
|
+
warn "Dialog was shown but you didn't provide `on(:dialog)` callback, accepting it by default. " \
|
353
|
+
"Please take a look at https://github.com/rubycdp/ferrum#dialog"
|
354
|
+
dialog.accept
|
219
355
|
end
|
220
356
|
end
|
221
357
|
end
|
@@ -228,8 +364,23 @@ module Ferrum
|
|
228
364
|
command("Log.enable")
|
229
365
|
command("Network.enable")
|
230
366
|
|
231
|
-
if
|
232
|
-
|
367
|
+
if use_authorized_proxy?
|
368
|
+
network.authorize(user: @proxy_user,
|
369
|
+
password: @proxy_password,
|
370
|
+
type: :proxy) do |request, _index, _total|
|
371
|
+
request.continue
|
372
|
+
end
|
373
|
+
end
|
374
|
+
|
375
|
+
if @browser.options.save_path
|
376
|
+
unless Pathname.new(@browser.options.save_path).absolute?
|
377
|
+
raise Error, "supply absolute path for `:save_path` option"
|
378
|
+
end
|
379
|
+
|
380
|
+
@browser.command("Browser.setDownloadBehavior",
|
381
|
+
browserContextId: context.id,
|
382
|
+
downloadPath: @browser.options.save_path,
|
383
|
+
behavior: "allow", eventsEnabled: true)
|
233
384
|
end
|
234
385
|
|
235
386
|
@browser.extensions.each do |extension|
|
@@ -242,14 +393,14 @@ module Ferrum
|
|
242
393
|
resize(width: width, height: height)
|
243
394
|
|
244
395
|
response = command("Page.getNavigationHistory")
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
396
|
+
return unless response.dig("entries", 0, "transitionType") != "typed"
|
397
|
+
|
398
|
+
# If we create page by clicking links, submitting forms and so on it
|
399
|
+
# opens a new window for which `frameStoppedLoading` event never
|
400
|
+
# occurs and thus search for nodes cannot be completed. Here we check
|
401
|
+
# the history and if the transitionType for example `link` then
|
402
|
+
# content is already loaded and we can try to get the document.
|
403
|
+
document_node_id
|
253
404
|
end
|
254
405
|
|
255
406
|
def inject_extensions
|
@@ -260,7 +411,7 @@ module Ferrum
|
|
260
411
|
# We also evaluate script just in case because
|
261
412
|
# `Page.addScriptToEvaluateOnNewDocument` doesn't work in popups.
|
262
413
|
command("Runtime.evaluate", expression: extension,
|
263
|
-
|
414
|
+
executionContextId: execution_id!,
|
264
415
|
returnByValue: true)
|
265
416
|
end
|
266
417
|
end
|
@@ -268,13 +419,15 @@ module Ferrum
|
|
268
419
|
def history_navigate(delta:)
|
269
420
|
history = command("Page.getNavigationHistory")
|
270
421
|
index, entries = history.values_at("currentIndex", "entries")
|
422
|
+
entry = entries[index + delta]
|
271
423
|
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
424
|
+
return unless entry
|
425
|
+
|
426
|
+
# Potential wait because of network event
|
427
|
+
command("Page.navigateToHistoryEntry",
|
428
|
+
wait: Mouse::CLICK_WAIT,
|
429
|
+
slowmoable: true,
|
430
|
+
entryId: entry["id"])
|
278
431
|
end
|
279
432
|
|
280
433
|
def combine_url!(url_or_path)
|
@@ -288,8 +441,19 @@ module Ferrum
|
|
288
441
|
(nil_or_relative ? @browser.base_url.join(url.to_s) : url).to_s
|
289
442
|
end
|
290
443
|
|
291
|
-
def
|
292
|
-
|
444
|
+
def document_node_id
|
445
|
+
command("DOM.getDocument", depth: 0).dig("root", "nodeId")
|
446
|
+
end
|
447
|
+
|
448
|
+
def ws_url
|
449
|
+
"ws://#{@browser.process.host}:#{@browser.process.port}/devtools/page/#{@target_id}"
|
450
|
+
end
|
451
|
+
|
452
|
+
def proxy=(options)
|
453
|
+
@proxy_host = options&.[](:host) || @browser.options.proxy&.[](:host)
|
454
|
+
@proxy_port = options&.[](:port) || @browser.options.proxy&.[](:port)
|
455
|
+
@proxy_user = options&.[](:user) || @browser.options.proxy&.[](:user)
|
456
|
+
@proxy_password = options&.[](:password) || @browser.options.proxy&.[](:password)
|
293
457
|
end
|
294
458
|
end
|
295
459
|
end
|
data/lib/ferrum/proxy.rb
ADDED
@@ -0,0 +1,147 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "tempfile"
|
4
|
+
require "webrick"
|
5
|
+
require "webrick/httpproxy"
|
6
|
+
|
7
|
+
module Ferrum
|
8
|
+
class Proxy
|
9
|
+
def self.start(**args)
|
10
|
+
new(**args).tap(&:start)
|
11
|
+
end
|
12
|
+
|
13
|
+
attr_reader :host, :port, :user, :password
|
14
|
+
|
15
|
+
def initialize(host: "127.0.0.1", port: 0, user: nil, password: nil)
|
16
|
+
@file = nil
|
17
|
+
@host = host
|
18
|
+
@port = port
|
19
|
+
@user = user
|
20
|
+
@password = password
|
21
|
+
end
|
22
|
+
|
23
|
+
def start
|
24
|
+
options = {
|
25
|
+
ProxyURI: nil, ServerType: Thread,
|
26
|
+
Logger: Logger.new(IO::NULL), AccessLog: [],
|
27
|
+
BindAddress: host, Port: port
|
28
|
+
}
|
29
|
+
|
30
|
+
if user && password
|
31
|
+
@file = Tempfile.new("htpasswd")
|
32
|
+
htpasswd = WEBrick::HTTPAuth::Htpasswd.new(@file.path)
|
33
|
+
htpasswd.set_passwd "Proxy Realm", user, password
|
34
|
+
htpasswd.flush
|
35
|
+
authenticator = WEBrick::HTTPAuth::ProxyBasicAuth.new(Realm: "Proxy Realm",
|
36
|
+
UserDB: htpasswd,
|
37
|
+
Logger: Logger.new(IO::NULL))
|
38
|
+
options.merge!(ProxyAuthProc: authenticator.method(:authenticate).to_proc)
|
39
|
+
end
|
40
|
+
|
41
|
+
@server = HTTPProxyServer.new(**options)
|
42
|
+
@server.start
|
43
|
+
at_exit { stop }
|
44
|
+
|
45
|
+
@port = @server.config[:Port]
|
46
|
+
end
|
47
|
+
|
48
|
+
def rotate(host:, port:, user: nil, password: nil)
|
49
|
+
credentials = "#{user}:#{password}@" if user && password
|
50
|
+
proxy_uri = "schema://#{credentials}#{host}:#{port}"
|
51
|
+
@server.config[:ProxyURI] = URI.parse(proxy_uri)
|
52
|
+
end
|
53
|
+
|
54
|
+
def stop
|
55
|
+
@file&.unlink
|
56
|
+
@server.shutdown
|
57
|
+
end
|
58
|
+
|
59
|
+
# Fix hanging proxy at exit
|
60
|
+
class HTTPProxyServer < WEBrick::HTTPProxyServer
|
61
|
+
# rubocop:disable all
|
62
|
+
def do_CONNECT(req, res)
|
63
|
+
# Proxy Authentication
|
64
|
+
proxy_auth(req, res)
|
65
|
+
|
66
|
+
ua = Thread.current[:WEBrickSocket] # User-Agent
|
67
|
+
raise WEBrick::HTTPStatus::InternalServerError,
|
68
|
+
"[BUG] cannot get socket" unless ua
|
69
|
+
|
70
|
+
host, port = req.unparsed_uri.split(":", 2)
|
71
|
+
# Proxy authentication for upstream proxy server
|
72
|
+
if proxy = proxy_uri(req, res)
|
73
|
+
proxy_request_line = "CONNECT #{host}:#{port} HTTP/1.0"
|
74
|
+
if proxy.userinfo
|
75
|
+
credentials = "Basic " + [proxy.userinfo].pack("m0")
|
76
|
+
end
|
77
|
+
host, port = proxy.host, proxy.port
|
78
|
+
end
|
79
|
+
|
80
|
+
begin
|
81
|
+
@logger.debug("CONNECT: upstream proxy is `#{host}:#{port}'.")
|
82
|
+
os = TCPSocket.new(host, port) # origin server
|
83
|
+
|
84
|
+
if proxy
|
85
|
+
@logger.debug("CONNECT: sending a Request-Line")
|
86
|
+
os << proxy_request_line << CRLF
|
87
|
+
@logger.debug("CONNECT: > #{proxy_request_line}")
|
88
|
+
if credentials
|
89
|
+
@logger.debug("CONNECT: sending credentials")
|
90
|
+
os << "Proxy-Authorization: " << credentials << CRLF
|
91
|
+
end
|
92
|
+
os << CRLF
|
93
|
+
proxy_status_line = os.gets(LF)
|
94
|
+
@logger.debug("CONNECT: read Status-Line from the upstream server")
|
95
|
+
@logger.debug("CONNECT: < #{proxy_status_line}")
|
96
|
+
if %r{^HTTP/\d+\.\d+\s+200\s*} =~ proxy_status_line
|
97
|
+
while line = os.gets(LF)
|
98
|
+
break if /\A(#{CRLF}|#{LF})\z/om =~ line
|
99
|
+
end
|
100
|
+
else
|
101
|
+
raise WEBrick::HTTPStatus::BadGateway
|
102
|
+
end
|
103
|
+
end
|
104
|
+
@logger.debug("CONNECT #{host}:#{port}: succeeded")
|
105
|
+
res.status = WEBrick::HTTPStatus::RC_OK
|
106
|
+
rescue => ex
|
107
|
+
@logger.debug("CONNECT #{host}:#{port}: failed `#{ex.message}'")
|
108
|
+
res.set_error(ex)
|
109
|
+
raise WEBrick::HTTPStatus::EOFError
|
110
|
+
ensure
|
111
|
+
# At exit os variable sometimes can be nil which results in hanging forever
|
112
|
+
raise WEBrick::HTTPStatus::EOFError unless os
|
113
|
+
|
114
|
+
if handler = @config[:ProxyContentHandler]
|
115
|
+
handler.call(req, res)
|
116
|
+
end
|
117
|
+
res.send_response(ua)
|
118
|
+
access_log(@config, req, res)
|
119
|
+
|
120
|
+
# Should clear request-line not to send the response twice.
|
121
|
+
# see: HTTPServer#run
|
122
|
+
req.parse(NullReader) rescue nil
|
123
|
+
end
|
124
|
+
|
125
|
+
begin
|
126
|
+
while fds = IO::select([ua, os])
|
127
|
+
if fds[0].member?(ua)
|
128
|
+
buf = ua.readpartial(1024);
|
129
|
+
@logger.debug("CONNECT: #{buf.bytesize} byte from User-Agent")
|
130
|
+
os.write(buf)
|
131
|
+
elsif fds[0].member?(os)
|
132
|
+
buf = os.readpartial(1024);
|
133
|
+
@logger.debug("CONNECT: #{buf.bytesize} byte from #{host}:#{port}")
|
134
|
+
ua.write(buf)
|
135
|
+
end
|
136
|
+
end
|
137
|
+
rescue
|
138
|
+
os.close
|
139
|
+
@logger.debug("CONNECT #{host}:#{port}: closed")
|
140
|
+
end
|
141
|
+
|
142
|
+
raise WEBrick::HTTPStatus::EOFError
|
143
|
+
end
|
144
|
+
# rubocop:enable all
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Ferrum
|
2
4
|
class RGBA
|
3
5
|
def initialize(red, green, blue, alpha)
|
@@ -23,13 +25,13 @@ module Ferrum
|
|
23
25
|
end
|
24
26
|
|
25
27
|
def validate_color(value)
|
26
|
-
return if value
|
28
|
+
return if value.is_a?(Integer) && Range.new(0, 255).include?(value)
|
27
29
|
|
28
30
|
raise ArgumentError, "Wrong value of #{value} should be Integer from 0 to 255"
|
29
31
|
end
|
30
32
|
|
31
33
|
def validate_alpha
|
32
|
-
return if alpha
|
34
|
+
return if alpha.is_a?(Float) && Range.new(0.0, 1.0).include?(alpha)
|
33
35
|
|
34
36
|
raise ArgumentError,
|
35
37
|
"Wrong alpha value #{alpha} should be Float between 0.0 (fully transparent) and 1.0 (fully opaque)"
|
data/lib/ferrum/target.rb
CHANGED
@@ -9,6 +9,7 @@ module Ferrum
|
|
9
9
|
attr_writer :page
|
10
10
|
|
11
11
|
def initialize(browser, params = nil)
|
12
|
+
@page = nil
|
12
13
|
@browser = browser
|
13
14
|
@params = params
|
14
15
|
end
|
@@ -22,10 +23,12 @@ module Ferrum
|
|
22
23
|
end
|
23
24
|
|
24
25
|
def page
|
25
|
-
@page ||=
|
26
|
-
|
27
|
-
|
28
|
-
|
26
|
+
@page ||= build_page
|
27
|
+
end
|
28
|
+
|
29
|
+
def build_page(**options)
|
30
|
+
maybe_sleep_if_new_window
|
31
|
+
Page.new(id, @browser, **options)
|
29
32
|
end
|
30
33
|
|
31
34
|
def id
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Ferrum
|
4
|
+
module Utils
|
5
|
+
module Attempt
|
6
|
+
module_function
|
7
|
+
|
8
|
+
def with_retry(errors:, max:, wait:)
|
9
|
+
attempts ||= 1
|
10
|
+
yield
|
11
|
+
rescue *Array(errors)
|
12
|
+
raise if attempts >= max
|
13
|
+
|
14
|
+
attempts += 1
|
15
|
+
sleep(wait)
|
16
|
+
retry
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "concurrent-ruby"
|
4
|
+
|
5
|
+
module Ferrum
|
6
|
+
module Utils
|
7
|
+
module ElapsedTime
|
8
|
+
module_function
|
9
|
+
|
10
|
+
def start
|
11
|
+
@start ||= monotonic_time
|
12
|
+
end
|
13
|
+
|
14
|
+
def elapsed_time(start = nil)
|
15
|
+
monotonic_time - (start || @start)
|
16
|
+
end
|
17
|
+
|
18
|
+
def monotonic_time
|
19
|
+
Concurrent.monotonic_time
|
20
|
+
end
|
21
|
+
|
22
|
+
def timeout?(start, timeout)
|
23
|
+
elapsed_time(start) > timeout
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Ferrum
|
4
|
+
module Utils
|
5
|
+
module Platform
|
6
|
+
module_function
|
7
|
+
|
8
|
+
def name
|
9
|
+
return :mac if mac?
|
10
|
+
return :windows if windows?
|
11
|
+
|
12
|
+
:linux
|
13
|
+
end
|
14
|
+
|
15
|
+
def windows?
|
16
|
+
RbConfig::CONFIG["host_os"] =~ /mingw|mswin|cygwin/
|
17
|
+
end
|
18
|
+
|
19
|
+
def mac?
|
20
|
+
RbConfig::CONFIG["host_os"] =~ /darwin/
|
21
|
+
end
|
22
|
+
|
23
|
+
def mri?
|
24
|
+
defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|