ferrum 0.6 → 0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +296 -38
- data/lib/ferrum.rb +29 -4
- data/lib/ferrum/browser.rb +14 -8
- data/lib/ferrum/browser/client.rb +19 -10
- data/lib/ferrum/browser/command.rb +57 -0
- data/lib/ferrum/browser/options/base.rb +46 -0
- data/lib/ferrum/browser/options/chrome.rb +73 -0
- data/lib/ferrum/browser/options/firefox.rb +34 -0
- data/lib/ferrum/browser/process.rb +53 -107
- data/lib/ferrum/browser/subscriber.rb +5 -1
- data/lib/ferrum/browser/web_socket.rb +23 -4
- data/lib/ferrum/browser/xvfb.rb +37 -0
- data/lib/ferrum/cookies.rb +7 -0
- data/lib/ferrum/dialog.rb +2 -2
- data/lib/ferrum/frame.rb +23 -5
- data/lib/ferrum/frame/dom.rb +38 -41
- data/lib/ferrum/frame/runtime.rb +54 -33
- data/lib/ferrum/headers.rb +1 -1
- data/lib/ferrum/keyboard.rb +3 -3
- data/lib/ferrum/mouse.rb +14 -3
- data/lib/ferrum/network.rb +60 -16
- data/lib/ferrum/network/exchange.rb +24 -21
- data/lib/ferrum/network/intercepted_request.rb +12 -3
- data/lib/ferrum/network/response.rb +4 -0
- data/lib/ferrum/node.rb +59 -26
- data/lib/ferrum/page.rb +45 -17
- data/lib/ferrum/page/frames.rb +11 -19
- data/lib/ferrum/page/screenshot.rb +3 -3
- data/lib/ferrum/target.rb +10 -2
- data/lib/ferrum/version.rb +1 -1
- metadata +10 -5
data/lib/ferrum/page.rb
CHANGED
@@ -13,6 +13,8 @@ require "ferrum/browser/client"
|
|
13
13
|
|
14
14
|
module Ferrum
|
15
15
|
class Page
|
16
|
+
GOTO_WAIT = ENV.fetch("FERRUM_GOTO_WAIT", 0.1).to_f
|
17
|
+
|
16
18
|
class Event < Concurrent::Event
|
17
19
|
def iteration
|
18
20
|
synchronize { @iteration }
|
@@ -29,7 +31,7 @@ module Ferrum
|
|
29
31
|
|
30
32
|
extend Forwardable
|
31
33
|
delegate %i[at_css at_xpath css xpath
|
32
|
-
current_url current_title url title body doctype
|
34
|
+
current_url current_title url title body doctype set_content
|
33
35
|
execution_id evaluate evaluate_on evaluate_async execute
|
34
36
|
add_script_tag add_style_tag] => :main_frame
|
35
37
|
|
@@ -42,13 +44,14 @@ module Ferrum
|
|
42
44
|
|
43
45
|
def initialize(target_id, browser)
|
44
46
|
@frames = {}
|
47
|
+
@main_frame = Frame.new(nil, self)
|
45
48
|
@target_id, @browser = target_id, browser
|
46
49
|
@event = Event.new.tap(&:set)
|
47
50
|
|
48
51
|
host = @browser.process.host
|
49
52
|
port = @browser.process.port
|
50
53
|
ws_url = "ws://#{host}:#{port}/devtools/page/#{@target_id}"
|
51
|
-
@client = Browser::Client.new(browser, ws_url, 1000)
|
54
|
+
@client = Browser::Client.new(browser, ws_url, id_starts_with: 1000)
|
52
55
|
|
53
56
|
@mouse, @keyboard = Mouse.new(self), Keyboard.new(self)
|
54
57
|
@headers, @cookies = Headers.new(self), Cookies.new(self)
|
@@ -65,7 +68,7 @@ module Ferrum
|
|
65
68
|
def goto(url = nil)
|
66
69
|
options = { url: combine_url!(url) }
|
67
70
|
options.merge!(referrer: referrer) if referrer
|
68
|
-
response = command("Page.navigate", wait:
|
71
|
+
response = command("Page.navigate", wait: GOTO_WAIT, **options)
|
69
72
|
# https://cs.chromium.org/chromium/src/net/base/net_error_list.h
|
70
73
|
if %w[net::ERR_NAME_NOT_RESOLVED
|
71
74
|
net::ERR_NAME_RESOLUTION_FAILED
|
@@ -74,6 +77,9 @@ module Ferrum
|
|
74
77
|
raise StatusError, options[:url]
|
75
78
|
end
|
76
79
|
response["frameId"]
|
80
|
+
rescue TimeoutError
|
81
|
+
pendings = network.traffic.select(&:pending?).map { |e| e.request.url }
|
82
|
+
raise StatusError.new(options[:url], pendings) unless pendings.empty?
|
77
83
|
end
|
78
84
|
|
79
85
|
def close
|
@@ -94,7 +100,8 @@ module Ferrum
|
|
94
100
|
@browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { width: width, height: height, windowState: "normal" })
|
95
101
|
end
|
96
102
|
|
97
|
-
command("Emulation.setDeviceMetricsOverride",
|
103
|
+
command("Emulation.setDeviceMetricsOverride", slowmoable: true,
|
104
|
+
width: width,
|
98
105
|
height: height,
|
99
106
|
deviceScaleFactor: 1,
|
100
107
|
mobile: false,
|
@@ -102,7 +109,12 @@ module Ferrum
|
|
102
109
|
end
|
103
110
|
|
104
111
|
def refresh
|
105
|
-
command("Page.reload", wait: timeout)
|
112
|
+
command("Page.reload", wait: timeout, slowmoable: true)
|
113
|
+
end
|
114
|
+
alias_method :reload, :refresh
|
115
|
+
|
116
|
+
def stop
|
117
|
+
command("Page.stopLoading", slowmoable: true)
|
106
118
|
end
|
107
119
|
|
108
120
|
def back
|
@@ -113,12 +125,31 @@ module Ferrum
|
|
113
125
|
history_navigate(delta: 1)
|
114
126
|
end
|
115
127
|
|
116
|
-
def
|
128
|
+
def wait_for_reload(sec = 1)
|
129
|
+
@event.reset if @event.set?
|
130
|
+
@event.wait(sec)
|
131
|
+
@event.set
|
132
|
+
end
|
133
|
+
|
134
|
+
def bypass_csp(value = true)
|
135
|
+
enabled = !!value
|
136
|
+
command("Page.setBypassCSP", enabled: enabled)
|
137
|
+
enabled
|
138
|
+
end
|
139
|
+
|
140
|
+
def command(method, wait: 0, slowmoable: false, **params)
|
117
141
|
iteration = @event.reset if wait > 0
|
142
|
+
sleep(@browser.slowmo) if slowmoable && @browser.slowmo > 0
|
118
143
|
result = @client.command(method, params)
|
144
|
+
|
119
145
|
if wait > 0
|
120
|
-
@event.wait(wait)
|
121
|
-
|
146
|
+
@event.wait(wait) # Wait a bit after command and check if iteration has
|
147
|
+
# changed which means there was some network event for
|
148
|
+
# the main frame and it started to load new content.
|
149
|
+
if iteration != @event.iteration
|
150
|
+
set = @event.wait(@browser.timeout)
|
151
|
+
raise TimeoutError unless set
|
152
|
+
end
|
122
153
|
end
|
123
154
|
result
|
124
155
|
end
|
@@ -133,6 +164,9 @@ module Ferrum
|
|
133
164
|
when :request
|
134
165
|
@client.on("Fetch.requestPaused") do |params, index, total|
|
135
166
|
request = Network::InterceptedRequest.new(self, params)
|
167
|
+
exchange = network.select(request.network_id).last
|
168
|
+
exchange ||= network.build_exchange(request.network_id)
|
169
|
+
exchange.intercepted_request = request
|
136
170
|
block.call(request, index, total)
|
137
171
|
end
|
138
172
|
when :auth
|
@@ -163,14 +197,6 @@ module Ferrum
|
|
163
197
|
Thread.main.raise JavaScriptError.new(params.dig("exceptionDetails", "exception"))
|
164
198
|
end
|
165
199
|
end
|
166
|
-
|
167
|
-
on("Page.domContentEventFired") do |params|
|
168
|
-
# `frameStoppedLoading` doesn't occur if status isn't success
|
169
|
-
if network.status != 200
|
170
|
-
@event.set
|
171
|
-
get_document_id
|
172
|
-
end
|
173
|
-
end
|
174
200
|
end
|
175
201
|
|
176
202
|
def prepare_page
|
@@ -224,7 +250,9 @@ module Ferrum
|
|
224
250
|
|
225
251
|
if entry = entries[index + delta]
|
226
252
|
# Potential wait because of network event
|
227
|
-
command("Page.navigateToHistoryEntry", wait: Mouse::CLICK_WAIT,
|
253
|
+
command("Page.navigateToHistoryEntry", wait: Mouse::CLICK_WAIT,
|
254
|
+
slowmoable: true,
|
255
|
+
entryId: entry["id"])
|
228
256
|
end
|
229
257
|
end
|
230
258
|
|
data/lib/ferrum/page/frames.rb
CHANGED
@@ -11,11 +11,9 @@ module Ferrum
|
|
11
11
|
@frames.values
|
12
12
|
end
|
13
13
|
|
14
|
-
def frame_by(id: nil,
|
14
|
+
def frame_by(id: nil, name: nil)
|
15
15
|
if id
|
16
16
|
@frames[id]
|
17
|
-
elsif execution_id
|
18
|
-
frames.find { |f| f.execution_id == execution_id }
|
19
17
|
elsif name
|
20
18
|
frames.find { |f| f.name == name }
|
21
19
|
else
|
@@ -42,18 +40,6 @@ module Ferrum
|
|
42
40
|
frame.name = name unless name.to_s.empty?
|
43
41
|
end
|
44
42
|
|
45
|
-
on("Page.frameScheduledNavigation") do |params|
|
46
|
-
frame = @frames[params["frameId"]]
|
47
|
-
frame.state = :scheduled_navigation
|
48
|
-
@event.reset
|
49
|
-
end
|
50
|
-
|
51
|
-
on("Page.frameClearedScheduledNavigation") do |params|
|
52
|
-
frame = @frames[params["frameId"]]
|
53
|
-
frame.state = :cleared_scheduled_navigation
|
54
|
-
@event.set if idling?
|
55
|
-
end
|
56
|
-
|
57
43
|
on("Page.frameStoppedLoading") do |params|
|
58
44
|
# `DOM.performSearch` doesn't work without getting #document node first.
|
59
45
|
# It returns node with nodeId 1 and nodeType 9 from which descend the
|
@@ -87,21 +73,27 @@ module Ferrum
|
|
87
73
|
on("Runtime.executionContextCreated") do |params|
|
88
74
|
context_id = params.dig("context", "id")
|
89
75
|
frame_id = params.dig("context", "auxData", "frameId")
|
76
|
+
|
77
|
+
unless @main_frame.id
|
78
|
+
@main_frame.id = frame_id
|
79
|
+
@frames[frame_id] = @main_frame
|
80
|
+
end
|
81
|
+
|
90
82
|
frame = @frames[frame_id] || Frame.new(frame_id, self)
|
91
|
-
frame.
|
83
|
+
frame.set_execution_id(context_id)
|
92
84
|
|
93
|
-
@main_frame ||= frame
|
94
85
|
@frames[frame_id] ||= frame
|
95
86
|
end
|
96
87
|
|
97
88
|
on("Runtime.executionContextDestroyed") do |params|
|
98
89
|
execution_id = params["executionContextId"]
|
99
|
-
frame =
|
100
|
-
frame.
|
90
|
+
frame = frames.find { |f| f.execution_id?(execution_id) }
|
91
|
+
frame.reset_execution_id
|
101
92
|
end
|
102
93
|
|
103
94
|
on("Runtime.executionContextsCleared") do
|
104
95
|
@frames.delete_if { |_, f| !f.main? }
|
96
|
+
@main_frame.reset_execution_id
|
105
97
|
end
|
106
98
|
end
|
107
99
|
|
@@ -48,8 +48,8 @@ module Ferrum
|
|
48
48
|
|
49
49
|
def document_size
|
50
50
|
evaluate <<~JS
|
51
|
-
[document.documentElement.
|
52
|
-
document.documentElement.
|
51
|
+
[document.documentElement.scrollWidth,
|
52
|
+
document.documentElement.scrollHeight]
|
53
53
|
JS
|
54
54
|
end
|
55
55
|
|
@@ -136,7 +136,7 @@ module Ferrum
|
|
136
136
|
|
137
137
|
def capture_screenshot(options, full)
|
138
138
|
maybe_resize_fullscreen(full) do
|
139
|
-
command("Page.captureScreenshot", options)
|
139
|
+
command("Page.captureScreenshot", **options)
|
140
140
|
end.fetch("data")
|
141
141
|
end
|
142
142
|
|
data/lib/ferrum/target.rb
CHANGED
@@ -4,6 +4,10 @@ module Ferrum
|
|
4
4
|
class Target
|
5
5
|
NEW_WINDOW_WAIT = ENV.fetch("FERRUM_NEW_WINDOW_WAIT", 0.3).to_f
|
6
6
|
|
7
|
+
# You can create page yourself and assign it to target, used in cuprite
|
8
|
+
# where we enhance page class and build page ourselves.
|
9
|
+
attr_writer :page
|
10
|
+
|
7
11
|
def initialize(browser, params = nil)
|
8
12
|
@browser = browser
|
9
13
|
@params = params
|
@@ -19,8 +23,7 @@ module Ferrum
|
|
19
23
|
|
20
24
|
def page
|
21
25
|
@page ||= begin
|
22
|
-
|
23
|
-
sleep(NEW_WINDOW_WAIT) if window?
|
26
|
+
maybe_sleep_if_new_window
|
24
27
|
Page.new(id, @browser)
|
25
28
|
end
|
26
29
|
end
|
@@ -52,5 +55,10 @@ module Ferrum
|
|
52
55
|
def window?
|
53
56
|
!!opener_id
|
54
57
|
end
|
58
|
+
|
59
|
+
def maybe_sleep_if_new_window
|
60
|
+
# Dirty hack because new window doesn't have events at all
|
61
|
+
sleep(NEW_WINDOW_WAIT) if window?
|
62
|
+
end
|
55
63
|
end
|
56
64
|
end
|
data/lib/ferrum/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ferrum
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: '0.
|
4
|
+
version: '0.9'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dmitry Vorotilin
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-07-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: websocket-driver
|
@@ -64,14 +64,14 @@ dependencies:
|
|
64
64
|
requirements:
|
65
65
|
- - "~>"
|
66
66
|
- !ruby/object:Gem::Version
|
67
|
-
version: '2.
|
67
|
+
version: '2.5'
|
68
68
|
type: :runtime
|
69
69
|
prerelease: false
|
70
70
|
version_requirements: !ruby/object:Gem::Requirement
|
71
71
|
requirements:
|
72
72
|
- - "~>"
|
73
73
|
- !ruby/object:Gem::Version
|
74
|
-
version: '2.
|
74
|
+
version: '2.5'
|
75
75
|
- !ruby/object:Gem::Dependency
|
76
76
|
name: rake
|
77
77
|
requirement: !ruby/object:Gem::Requirement
|
@@ -182,9 +182,14 @@ files:
|
|
182
182
|
- lib/ferrum.rb
|
183
183
|
- lib/ferrum/browser.rb
|
184
184
|
- lib/ferrum/browser/client.rb
|
185
|
+
- lib/ferrum/browser/command.rb
|
186
|
+
- lib/ferrum/browser/options/base.rb
|
187
|
+
- lib/ferrum/browser/options/chrome.rb
|
188
|
+
- lib/ferrum/browser/options/firefox.rb
|
185
189
|
- lib/ferrum/browser/process.rb
|
186
190
|
- lib/ferrum/browser/subscriber.rb
|
187
191
|
- lib/ferrum/browser/web_socket.rb
|
192
|
+
- lib/ferrum/browser/xvfb.rb
|
188
193
|
- lib/ferrum/context.rb
|
189
194
|
- lib/ferrum/contexts.rb
|
190
195
|
- lib/ferrum/cookies.rb
|
@@ -228,7 +233,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
228
233
|
- !ruby/object:Gem::Version
|
229
234
|
version: '0'
|
230
235
|
requirements: []
|
231
|
-
rubygems_version: 3.
|
236
|
+
rubygems_version: 3.1.2
|
232
237
|
signing_key:
|
233
238
|
specification_version: 4
|
234
239
|
summary: Ruby headless Chrome driver
|