ferrum 0.6.1 → 0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/ferrum/page.rb CHANGED
@@ -13,6 +13,8 @@ require "ferrum/browser/client"
13
13
 
14
14
  module Ferrum
15
15
  class Page
16
+ GOTO_WAIT = ENV.fetch("FERRUM_GOTO_WAIT", 0.1).to_f
17
+
16
18
  class Event < Concurrent::Event
17
19
  def iteration
18
20
  synchronize { @iteration }
@@ -29,8 +31,8 @@ module Ferrum
29
31
 
30
32
  extend Forwardable
31
33
  delegate %i[at_css at_xpath css xpath
32
- current_url current_title url title body doctype
33
- execution_id evaluate evaluate_on evaluate_async execute
34
+ current_url current_title url title body doctype set_content
35
+ execution_id evaluate evaluate_on evaluate_async execute evaluate_func
34
36
  add_script_tag add_style_tag] => :main_frame
35
37
 
36
38
  include Frames, Screenshot
@@ -42,13 +44,14 @@ module Ferrum
42
44
 
43
45
  def initialize(target_id, browser)
44
46
  @frames = {}
47
+ @main_frame = Frame.new(nil, self)
45
48
  @target_id, @browser = target_id, browser
46
49
  @event = Event.new.tap(&:set)
47
50
 
48
51
  host = @browser.process.host
49
52
  port = @browser.process.port
50
53
  ws_url = "ws://#{host}:#{port}/devtools/page/#{@target_id}"
51
- @client = Browser::Client.new(browser, ws_url, 1000)
54
+ @client = Browser::Client.new(browser, ws_url, id_starts_with: 1000)
52
55
 
53
56
  @mouse, @keyboard = Mouse.new(self), Keyboard.new(self)
54
57
  @headers, @cookies = Headers.new(self), Cookies.new(self)
@@ -62,10 +65,10 @@ module Ferrum
62
65
  @browser.timeout
63
66
  end
64
67
 
65
- def goto(url = nil)
68
+ def go_to(url = nil)
66
69
  options = { url: combine_url!(url) }
67
70
  options.merge!(referrer: referrer) if referrer
68
- response = command("Page.navigate", wait: timeout, **options)
71
+ response = command("Page.navigate", wait: GOTO_WAIT, **options)
69
72
  # https://cs.chromium.org/chromium/src/net/base/net_error_list.h
70
73
  if %w[net::ERR_NAME_NOT_RESOLVED
71
74
  net::ERR_NAME_RESOLUTION_FAILED
@@ -74,7 +77,13 @@ module Ferrum
74
77
  raise StatusError, options[:url]
75
78
  end
76
79
  response["frameId"]
80
+ rescue TimeoutError
81
+ if @browser.pending_connection_errors
82
+ pendings = network.traffic.select(&:pending?).map { |e| e.request.url }
83
+ raise PendingConnectionsError.new(options[:url], pendings) unless pendings.empty?
84
+ end
77
85
  end
86
+ alias goto go_to
78
87
 
79
88
  def close
80
89
  @headers.clear
@@ -83,18 +92,16 @@ module Ferrum
83
92
  end
84
93
 
85
94
  def resize(width: nil, height: nil, fullscreen: false)
86
- result = @browser.command("Browser.getWindowForTarget", targetId: @target_id)
87
- @window_id, @bounds = result.values_at("windowId", "bounds")
88
-
89
95
  if fullscreen
90
96
  width, height = document_size
91
- @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { windowState: "fullscreen" })
97
+ set_window_bounds(windowState: "fullscreen")
92
98
  else
93
- @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { windowState: "normal" })
94
- @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { width: width, height: height, windowState: "normal" })
99
+ set_window_bounds(windowState: "normal")
100
+ set_window_bounds(width: width, height: height)
95
101
  end
96
102
 
97
- command("Emulation.setDeviceMetricsOverride", width: width,
103
+ command("Emulation.setDeviceMetricsOverride", slowmoable: true,
104
+ width: width,
98
105
  height: height,
99
106
  deviceScaleFactor: 1,
100
107
  mobile: false,
@@ -102,7 +109,12 @@ module Ferrum
102
109
  end
103
110
 
104
111
  def refresh
105
- command("Page.reload", wait: timeout)
112
+ command("Page.reload", wait: timeout, slowmoable: true)
113
+ end
114
+ alias_method :reload, :refresh
115
+
116
+ def stop
117
+ command("Page.stopLoading", slowmoable: true)
106
118
  end
107
119
 
108
120
  def back
@@ -113,12 +125,39 @@ module Ferrum
113
125
  history_navigate(delta: 1)
114
126
  end
115
127
 
116
- def command(method, wait: 0, **params)
128
+ def wait_for_reload(sec = 1)
129
+ @event.reset if @event.set?
130
+ @event.wait(sec)
131
+ @event.set
132
+ end
133
+
134
+ def bypass_csp(value = true)
135
+ enabled = !!value
136
+ command("Page.setBypassCSP", enabled: enabled)
137
+ enabled
138
+ end
139
+
140
+ def window_id
141
+ @browser.command("Browser.getWindowForTarget", targetId: @target_id)["windowId"]
142
+ end
143
+
144
+ def set_window_bounds(bounds = {})
145
+ @browser.command("Browser.setWindowBounds", windowId: window_id, bounds: bounds)
146
+ end
147
+
148
+ def command(method, wait: 0, slowmoable: false, **params)
117
149
  iteration = @event.reset if wait > 0
150
+ sleep(@browser.slowmo) if slowmoable && @browser.slowmo > 0
118
151
  result = @client.command(method, params)
152
+
119
153
  if wait > 0
120
- @event.wait(wait)
121
- @event.wait(@browser.timeout) if iteration != @event.iteration
154
+ @event.wait(wait) # Wait a bit after command and check if iteration has
155
+ # changed which means there was some network event for
156
+ # the main frame and it started to load new content.
157
+ if iteration != @event.iteration
158
+ set = @event.wait(@browser.timeout)
159
+ raise TimeoutError unless set
160
+ end
122
161
  end
123
162
  result
124
163
  end
@@ -133,6 +172,9 @@ module Ferrum
133
172
  when :request
134
173
  @client.on("Fetch.requestPaused") do |params, index, total|
135
174
  request = Network::InterceptedRequest.new(self, params)
175
+ exchange = network.select(request.network_id).last
176
+ exchange ||= network.build_exchange(request.network_id)
177
+ exchange.intercepted_request = request
136
178
  block.call(request, index, total)
137
179
  end
138
180
  when :auth
@@ -145,6 +187,10 @@ module Ferrum
145
187
  end
146
188
  end
147
189
 
190
+ def subscribed?(event)
191
+ @client.subscribed?(event)
192
+ end
193
+
148
194
  private
149
195
 
150
196
  def subscribe
@@ -163,14 +209,6 @@ module Ferrum
163
209
  Thread.main.raise JavaScriptError.new(params.dig("exceptionDetails", "exception"))
164
210
  end
165
211
  end
166
-
167
- on("Page.domContentEventFired") do |params|
168
- # `frameStoppedLoading` doesn't occur if status isn't success
169
- if network.status != 200
170
- @event.set
171
- get_document_id
172
- end
173
- end
174
212
  end
175
213
 
176
214
  def prepare_page
@@ -224,7 +262,9 @@ module Ferrum
224
262
 
225
263
  if entry = entries[index + delta]
226
264
  # Potential wait because of network event
227
- command("Page.navigateToHistoryEntry", wait: Mouse::CLICK_WAIT, entryId: entry["id"])
265
+ command("Page.navigateToHistoryEntry", wait: Mouse::CLICK_WAIT,
266
+ slowmoable: true,
267
+ entryId: entry["id"])
228
268
  end
229
269
  end
230
270
 
@@ -233,7 +273,7 @@ module Ferrum
233
273
  nil_or_relative = url.nil? || url.relative?
234
274
 
235
275
  if nil_or_relative && !@browser.base_url
236
- raise "Set :base_url browser's option or use absolute url in `goto`, you passed: #{url_or_path}"
276
+ raise "Set :base_url browser's option or use absolute url in `go_to`, you passed: #{url_or_path}"
237
277
  end
238
278
 
239
279
  (nil_or_relative ? @browser.base_url.join(url.to_s) : url).to_s
@@ -40,18 +40,6 @@ module Ferrum
40
40
  frame.name = name unless name.to_s.empty?
41
41
  end
42
42
 
43
- on("Page.frameScheduledNavigation") do |params|
44
- frame = @frames[params["frameId"]]
45
- frame.state = :scheduled_navigation
46
- @event.reset
47
- end
48
-
49
- on("Page.frameClearedScheduledNavigation") do |params|
50
- frame = @frames[params["frameId"]]
51
- frame.state = :cleared_scheduled_navigation
52
- @event.set if idling?
53
- end
54
-
55
43
  on("Page.frameStoppedLoading") do |params|
56
44
  # `DOM.performSearch` doesn't work without getting #document node first.
57
45
  # It returns node with nodeId 1 and nodeType 9 from which descend the
@@ -85,21 +73,30 @@ module Ferrum
85
73
  on("Runtime.executionContextCreated") do |params|
86
74
  context_id = params.dig("context", "id")
87
75
  frame_id = params.dig("context", "auxData", "frameId")
76
+
77
+ unless @main_frame.id
78
+ root_frame = command("Page.getFrameTree").dig("frameTree", "frame", "id")
79
+ if frame_id == root_frame
80
+ @main_frame.id = frame_id
81
+ @frames[frame_id] = @main_frame
82
+ end
83
+ end
84
+
88
85
  frame = @frames[frame_id] || Frame.new(frame_id, self)
89
- frame.execution_id = context_id
86
+ frame.set_execution_id(context_id)
90
87
 
91
- @main_frame ||= frame
92
88
  @frames[frame_id] ||= frame
93
89
  end
94
90
 
95
91
  on("Runtime.executionContextDestroyed") do |params|
96
92
  execution_id = params["executionContextId"]
97
93
  frame = frames.find { |f| f.execution_id?(execution_id) }
98
- frame.execution_id = nil
94
+ frame.reset_execution_id
99
95
  end
100
96
 
101
97
  on("Runtime.executionContextsCleared") do
102
98
  @frames.delete_if { |_, f| !f.main? }
99
+ @main_frame.reset_execution_id
103
100
  end
104
101
  end
105
102
 
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "ferrum/rbga"
4
+
3
5
  module Ferrum
4
6
  class Page
5
7
  module Screenshot
@@ -24,19 +26,33 @@ module Ferrum
24
26
  A6: { width: 4.13, height: 5.83 },
25
27
  }.freeze
26
28
 
29
+ STREAM_CHUNK = 128 * 1024
30
+
27
31
  def screenshot(**opts)
28
32
  path, encoding = common_options(**opts)
29
33
  options = screenshot_options(path, **opts)
30
- data = capture_screenshot(options, opts[:full])
34
+ data = capture_screenshot(options, opts[:full], opts[:background_color])
31
35
  return data if encoding == :base64
32
- save_file(path, data)
36
+
37
+ bin = Base64.decode64(data)
38
+ save_file(path, bin)
33
39
  end
34
40
 
35
41
  def pdf(**opts)
36
42
  path, encoding = common_options(**opts)
37
- options = pdf_options(**opts)
38
- data = command("Page.printToPDF", **options).fetch("data")
39
- return data if encoding == :base64
43
+ options = pdf_options(**opts).merge(transferMode: "ReturnAsStream")
44
+ handle = command("Page.printToPDF", **options).fetch("stream")
45
+
46
+ if path
47
+ stream_to_file(handle, path: path)
48
+ else
49
+ stream_to_memory(handle, encoding: encoding)
50
+ end
51
+ end
52
+
53
+ def mhtml(path: nil)
54
+ data = command("Page.captureSnapshot", format: :mhtml).fetch("data")
55
+ return data if path.nil?
40
56
  save_file(path, data)
41
57
  end
42
58
 
@@ -48,17 +64,39 @@ module Ferrum
48
64
 
49
65
  def document_size
50
66
  evaluate <<~JS
51
- [document.documentElement.offsetWidth,
52
- document.documentElement.offsetHeight]
67
+ [document.documentElement.scrollWidth,
68
+ document.documentElement.scrollHeight]
53
69
  JS
54
70
  end
55
71
 
56
72
  private
57
73
 
58
74
  def save_file(path, data)
59
- bin = Base64.decode64(data)
60
- return bin unless path
61
- File.open(path.to_s, "wb") { |f| f.write(bin) }
75
+ return data unless path
76
+ File.open(path.to_s, "wb") { |f| f.write(data) }
77
+ end
78
+
79
+ def stream_to_file(handle, path:)
80
+ File.open(path, "wb") { |f| stream_to(handle, f) }
81
+ true
82
+ end
83
+
84
+ def stream_to_memory(handle, encoding:)
85
+ data = String.new("") # Mutable string has << and compatible to File
86
+ stream_to(handle, data)
87
+ encoding == :base64 ? Base64.encode64(data) : data
88
+ end
89
+
90
+ def stream_to(handle, output)
91
+ loop do
92
+ result = command("IO.read", handle: handle, size: STREAM_CHUNK)
93
+
94
+ data_chunk = result["data"]
95
+ data_chunk = Base64.decode64(data_chunk) if result["base64Encoded"]
96
+ output << data_chunk
97
+
98
+ break if result["eof"]
99
+ end
62
100
  end
63
101
 
64
102
  def common_options(encoding: :base64, path: nil, **_)
@@ -134,9 +172,11 @@ module Ferrum
134
172
  option.to_s.gsub(/(?:_|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }.to_sym
135
173
  end
136
174
 
137
- def capture_screenshot(options, full)
175
+ def capture_screenshot(options, full, background_color)
138
176
  maybe_resize_fullscreen(full) do
139
- command("Page.captureScreenshot", options)
177
+ with_background_color(background_color) do
178
+ command("Page.captureScreenshot", **options)
179
+ end
140
180
  end.fetch("data")
141
181
  end
142
182
 
@@ -150,6 +190,18 @@ module Ferrum
150
190
  ensure
151
191
  resize(width: width, height: height) if full
152
192
  end
193
+
194
+ def with_background_color(color)
195
+ if color
196
+ raise ArgumentError, "Accept Ferrum::RGBA class only" unless color.is_a?(RGBA)
197
+
198
+ command("Emulation.setDefaultBackgroundColorOverride", color: color.to_h)
199
+ end
200
+
201
+ yield
202
+ ensure
203
+ command("Emulation.setDefaultBackgroundColorOverride") if color
204
+ end
153
205
  end
154
206
  end
155
207
  end
@@ -0,0 +1,38 @@
1
+ module Ferrum
2
+ class RGBA
3
+ def initialize(red, green, blue, alpha)
4
+ self.red = red
5
+ self.green = green
6
+ self.blue = blue
7
+ self.alpha = alpha
8
+
9
+ validate
10
+ end
11
+
12
+ def to_h
13
+ { r: red, g: green, b: blue, a: alpha }
14
+ end
15
+
16
+ private
17
+
18
+ attr_accessor :red, :green, :blue, :alpha
19
+
20
+ def validate
21
+ [red, green, blue].each(&method(:validate_color))
22
+ validate_alpha
23
+ end
24
+
25
+ def validate_color(value)
26
+ return if value && value.is_a?(Integer) && Range.new(0, 255).include?(value)
27
+
28
+ raise ArgumentError, "Wrong value of #{value} should be Integer from 0 to 255"
29
+ end
30
+
31
+ def validate_alpha
32
+ return if alpha && alpha.is_a?(Float) && Range.new(0.0, 1.0).include?(alpha)
33
+
34
+ raise ArgumentError,
35
+ "Wrong alpha value #{alpha} should be Float between 0.0 (fully transparent) and 1.0 (fully opaque)"
36
+ end
37
+ end
38
+ end
data/lib/ferrum/target.rb CHANGED
@@ -4,6 +4,10 @@ module Ferrum
4
4
  class Target
5
5
  NEW_WINDOW_WAIT = ENV.fetch("FERRUM_NEW_WINDOW_WAIT", 0.3).to_f
6
6
 
7
+ # You can create page yourself and assign it to target, used in cuprite
8
+ # where we enhance page class and build page ourselves.
9
+ attr_writer :page
10
+
7
11
  def initialize(browser, params = nil)
8
12
  @browser = browser
9
13
  @params = params
@@ -19,8 +23,7 @@ module Ferrum
19
23
 
20
24
  def page
21
25
  @page ||= begin
22
- # Dirty hack because new window doesn't have events at all
23
- sleep(NEW_WINDOW_WAIT) if window?
26
+ maybe_sleep_if_new_window
24
27
  Page.new(id, @browser)
25
28
  end
26
29
  end
@@ -52,5 +55,10 @@ module Ferrum
52
55
  def window?
53
56
  !!opener_id
54
57
  end
58
+
59
+ def maybe_sleep_if_new_window
60
+ # Dirty hack because new window doesn't have events at all
61
+ sleep(NEW_WINDOW_WAIT) if window?
62
+ end
55
63
  end
56
64
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Ferrum
4
- VERSION = "0.6.1"
4
+ VERSION = "0.10"
5
5
  end