ferrum 0.6.1 → 0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -40,7 +40,7 @@ module Ferrum
40
40
 
41
41
  def set_overrides(user_agent: nil, accept_language: nil, platform: nil)
42
42
  options = Hash.new
43
- options[:userAgent] = user_agent if user_agent
43
+ options[:userAgent] = user_agent || @page.browser.default_user_agent
44
44
  options[:acceptLanguage] = accept_language if accept_language
45
45
  options[:platform] if platform
46
46
 
@@ -33,13 +33,13 @@ module Ferrum
33
33
  def down(key)
34
34
  key = normalize_keys(Array(key))
35
35
  type = key[:text] ? "keyDown" : "rawKeyDown"
36
- @page.command("Input.dispatchKeyEvent", type: type, **key)
36
+ @page.command("Input.dispatchKeyEvent", slowmoable: true, type: type, **key)
37
37
  self
38
38
  end
39
39
 
40
40
  def up(key)
41
41
  key = normalize_keys(Array(key))
42
- @page.command("Input.dispatchKeyEvent", type: "keyUp", **key)
42
+ @page.command("Input.dispatchKeyEvent", slowmoable: true, type: "keyUp", **key)
43
43
  self
44
44
  end
45
45
 
@@ -49,7 +49,7 @@ module Ferrum
49
49
  keys.each do |key|
50
50
  type = key[:text] ? "keyDown" : "rawKeyDown"
51
51
  @page.command("Input.dispatchKeyEvent", type: type, **key)
52
- @page.command("Input.dispatchKeyEvent", type: "keyUp", **key)
52
+ @page.command("Input.dispatchKeyEvent", slowmoable: true, type: "keyUp", **key)
53
53
  end
54
54
 
55
55
  self
data/lib/ferrum/mouse.rb CHANGED
@@ -32,10 +32,21 @@ module Ferrum
32
32
  tap { mouse_event(type: "mouseReleased", **options) }
33
33
  end
34
34
 
35
- # FIXME: steps
36
35
  def move(x:, y:, steps: 1)
36
+ from_x, from_y = @x, @y
37
37
  @x, @y = x, y
38
- @page.command("Input.dispatchMouseEvent", type: "mouseMoved", x: @x, y: @y)
38
+
39
+ steps.times do |i|
40
+ new_x = from_x + (@x - from_x) * ((i + 1) / steps.to_f)
41
+ new_y = from_y + (@y - from_y) * ((i + 1) / steps.to_f)
42
+
43
+ @page.command("Input.dispatchMouseEvent",
44
+ slowmoable: true,
45
+ type: "mouseMoved",
46
+ x: new_x.to_i,
47
+ y: new_y.to_i)
48
+ end
49
+
39
50
  self
40
51
  end
41
52
 
@@ -45,7 +56,7 @@ module Ferrum
45
56
  button = validate_button(button)
46
57
  options = { x: @x, y: @y, type: type, button: button, clickCount: count }
47
58
  options.merge!(modifiers: modifiers) if modifiers
48
- @page.command("Input.dispatchMouseEvent", wait: wait, **options)
59
+ @page.command("Input.dispatchMouseEvent", wait: wait, slowmoable: true, **options)
49
60
  end
50
61
 
51
62
  def validate_button(button)
@@ -3,6 +3,9 @@
3
3
  require "ferrum/network/exchange"
4
4
  require "ferrum/network/intercepted_request"
5
5
  require "ferrum/network/auth_request"
6
+ require "ferrum/network/error"
7
+ require "ferrum/network/request"
8
+ require "ferrum/network/response"
6
9
 
7
10
  module Ferrum
8
11
  class Network
@@ -20,6 +23,31 @@ module Ferrum
20
23
  @exchange = nil
21
24
  end
22
25
 
26
+ def wait_for_idle(connections: 0, duration: 0.05, timeout: @page.browser.timeout)
27
+ start = Ferrum.monotonic_time
28
+
29
+ until idle?(connections)
30
+ raise TimeoutError if Ferrum.timeout?(start, timeout)
31
+ sleep(duration)
32
+ end
33
+ end
34
+
35
+ def idle?(connections = 0)
36
+ pending_connections <= connections
37
+ end
38
+
39
+ def total_connections
40
+ @traffic.size
41
+ end
42
+
43
+ def finished_connections
44
+ @traffic.count(&:finished?)
45
+ end
46
+
47
+ def pending_connections
48
+ total_connections - finished_connections
49
+ end
50
+
23
51
  def request
24
52
  @exchange&.request
25
53
  end
@@ -55,19 +83,21 @@ module Ferrum
55
83
  @page.command("Fetch.enable", handleAuthRequests: true, patterns: [pattern])
56
84
  end
57
85
 
58
- def authorize(user:, password:, type: :server)
86
+ def authorize(user:, password:, type: :server, &block)
59
87
  unless AUTHORIZE_TYPE.include?(type)
60
88
  raise ArgumentError, ":type should be in #{AUTHORIZE_TYPE}"
61
89
  end
62
90
 
91
+ if !block_given? && !@page.subscribed?("Fetch.requestPaused")
92
+ raise ArgumentError, "Block is missing, call `authorize(...) { |r| r.continue } or subscribe to `on(:request)` events before calling it"
93
+ end
94
+
63
95
  @authorized_ids ||= {}
64
96
  @authorized_ids[type] ||= []
65
97
 
66
98
  intercept
67
99
 
68
- @page.on(:request) do |request|
69
- request.continue
70
- end
100
+ @page.on(:request, &block)
71
101
 
72
102
  @page.on(:auth) do |request, index, total|
73
103
  if request.auth_challenge?(type)
@@ -87,38 +117,69 @@ module Ferrum
87
117
 
88
118
  def subscribe
89
119
  @page.on("Network.requestWillBeSent") do |params|
120
+ request = Network::Request.new(params)
121
+
122
+ # We can build exchange in two places, here on the event or when request
123
+ # is interrupted. So we have to be careful when to create new one. We
124
+ # create new exchange only if there's no with such id or there's but
125
+ # it's filled with request which means this one is new but has response
126
+ # for a redirect. So we assign response from the params to previous
127
+ # exchange and build new exchange to assign this request to it.
128
+ exchange = select(request.id).last
129
+ exchange = build_exchange(request.id) unless exchange&.blank?
130
+
90
131
  # On redirects Chrome doesn't change `requestId` and there's no
91
132
  # `Network.responseReceived` event for such request. If there's already
92
133
  # exchange object with this id then we got redirected and params has
93
134
  # `redirectResponse` key which contains the response.
94
- if exchange = first_by(params["requestId"])
95
- exchange.build_response(params)
135
+ if params["redirectResponse"]
136
+ previous_exchange = select(request.id)[-2]
137
+ response = Network::Response.new(@page, params)
138
+ previous_exchange.response = response
96
139
  end
97
140
 
98
- exchange = Network::Exchange.new(@page, params)
99
- @exchange = exchange if exchange.navigation_request?(@page.main_frame.id)
100
- @traffic << exchange
141
+ exchange.request = request
142
+
143
+ if exchange.navigation_request?(@page.main_frame.id)
144
+ @exchange = exchange
145
+ end
101
146
  end
102
147
 
103
148
  @page.on("Network.responseReceived") do |params|
104
- if exchange = last_by(params["requestId"])
105
- exchange.build_response(params)
149
+ if exchange = select(params["requestId"]).last
150
+ response = Network::Response.new(@page, params)
151
+ exchange.response = response
106
152
  end
107
153
  end
108
154
 
109
155
  @page.on("Network.loadingFinished") do |params|
110
- exchange = last_by(params["requestId"])
156
+ exchange = select(params["requestId"]).last
111
157
  if exchange && exchange.response
112
158
  exchange.response.body_size = params["encodedDataLength"]
113
159
  end
114
160
  end
115
161
 
162
+ @page.on("Network.loadingFailed") do |params|
163
+ exchange = select(params["requestId"]).last
164
+ exchange.error ||= Network::Error.new
165
+
166
+ exchange.error.id = params["requestId"]
167
+ exchange.error.type = params["type"]
168
+ exchange.error.error_text = params["errorText"]
169
+ exchange.error.monotonic_time = params["timestamp"]
170
+ exchange.error.canceled = params["canceled"]
171
+ end
172
+
116
173
  @page.on("Log.entryAdded") do |params|
117
174
  entry = params["entry"] || {}
118
- if entry["source"] == "network" &&
119
- entry["level"] == "error" &&
120
- exchange = last_by(entry["networkRequestId"])
121
- exchange.build_error(entry)
175
+ if entry["source"] == "network" && entry["level"] == "error"
176
+ exchange = select(entry["networkRequestId"]).last
177
+ exchange.error ||= Network::Error.new
178
+
179
+ exchange.error.id = entry["networkRequestId"]
180
+ exchange.error.url = entry["url"]
181
+ exchange.error.description = entry["text"]
182
+ exchange.error.timestamp = entry["timestamp"]
122
183
  end
123
184
  end
124
185
  end
@@ -135,12 +196,12 @@ module Ferrum
135
196
  end
136
197
  end
137
198
 
138
- def first_by(request_id)
139
- @traffic.find { |e| e.request.id == request_id }
199
+ def select(request_id)
200
+ @traffic.select { |e| e.id == request_id }
140
201
  end
141
202
 
142
- def last_by(request_id)
143
- @traffic.select { |e| e.request.id == request_id }.last
203
+ def build_exchange(id)
204
+ Network::Exchange.new(@page, id).tap { |e| @traffic << e }
144
205
  end
145
206
  end
146
207
  end
@@ -3,24 +3,17 @@
3
3
  module Ferrum
4
4
  class Network
5
5
  class Error
6
- def initialize(data)
7
- @data = data
8
- end
9
-
10
- def id
11
- @data["networkRequestId"]
12
- end
13
-
14
- def url
15
- @data["url"]
16
- end
6
+ attr_writer :canceled
7
+ attr_reader :time, :timestamp
8
+ attr_accessor :id, :url, :type, :error_text, :monotonic_time, :description
17
9
 
18
- def description
19
- @data["text"]
10
+ def canceled?
11
+ @canceled
20
12
  end
21
13
 
22
- def time
23
- @time ||= Time.strptime(@data["timestamp"].to_s, "%s")
14
+ def timestamp=(value)
15
+ @timestamp = value
16
+ @time = Time.strptime((value / 1000).to_s, "%s")
24
17
  end
25
18
  end
26
19
  end
@@ -1,39 +1,37 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "ferrum/network/error"
4
- require "ferrum/network/request"
5
- require "ferrum/network/response"
6
-
7
3
  module Ferrum
8
4
  class Network
9
5
  class Exchange
10
- attr_reader :request, :response, :error
6
+ attr_reader :id
7
+ attr_accessor :intercepted_request
8
+ attr_accessor :request, :response, :error
11
9
 
12
- def initialize(page, params)
13
- @page = page
14
- @response = @error = nil
15
- build_request(params)
10
+ def initialize(page, id)
11
+ @page, @id = page, id
12
+ @intercepted_request = nil
13
+ @request = @response = @error = nil
16
14
  end
17
15
 
18
- def build_request(params)
19
- @request = Network::Request.new(params)
16
+ def navigation_request?(frame_id)
17
+ request.type?(:document) &&
18
+ request.frame_id == frame_id
20
19
  end
21
20
 
22
- def build_response(params)
23
- @response = Network::Response.new(@page, params)
21
+ def blank?
22
+ !request
24
23
  end
25
24
 
26
- def build_error(params)
27
- @error = Network::Error.new(params)
25
+ def blocked?
26
+ intercepted_request && intercepted_request.status?(:aborted)
28
27
  end
29
28
 
30
- def navigation_request?(frame_id)
31
- request.type?(:document) &&
32
- request.frame_id == frame_id
29
+ def finished?
30
+ blocked? || response || error
33
31
  end
34
32
 
35
- def blocked?
36
- response.nil?
33
+ def pending?
34
+ !finished?
37
35
  end
38
36
 
39
37
  def to_a
@@ -41,7 +39,12 @@ module Ferrum
41
39
  end
42
40
 
43
41
  def inspect
44
- %(#<#{self.class} @id=#{@id.inspect} @request=#{@request.inspect} @response=#{@response.inspect} @error=#{@error.inspect}>)
42
+ "#<#{self.class} "\
43
+ "@id=#{@id.inspect} "\
44
+ "@intercepted_request=#{@intercepted_request.inspect} "\
45
+ "@request=#{@request.inspect} "\
46
+ "@response=#{@response.inspect} "\
47
+ "@error=#{@error.inspect}>"
45
48
  end
46
49
  end
47
50
  end
@@ -5,14 +5,20 @@ require "base64"
5
5
  module Ferrum
6
6
  class Network
7
7
  class InterceptedRequest
8
- attr_accessor :request_id, :frame_id, :resource_type
8
+ attr_accessor :request_id, :frame_id, :resource_type, :network_id, :status
9
9
 
10
10
  def initialize(page, params)
11
+ @status = nil
11
12
  @page, @params = page, params
12
13
  @request_id = params["requestId"]
13
14
  @frame_id = params["frameId"]
14
15
  @resource_type = params["resourceType"]
15
16
  @request = params["request"]
17
+ @network_id = params["networkId"]
18
+ end
19
+
20
+ def status?(value)
21
+ @status == value.to_sym
16
22
  end
17
23
 
18
24
  def navigation_request?
@@ -25,7 +31,7 @@ module Ferrum
25
31
 
26
32
  def respond(**options)
27
33
  has_body = options.has_key?(:body)
28
- headers = has_body ? { "content-length" => options.fetch(:body, '').length } : {}
34
+ headers = has_body ? { "content-length" => options.fetch(:body, "").length } : {}
29
35
  headers = headers.merge(options.fetch(:responseHeaders, {}))
30
36
 
31
37
  options = {responseCode: 200}.merge(options)
@@ -33,17 +39,20 @@ module Ferrum
33
39
  requestId: request_id,
34
40
  responseHeaders: header_array(headers),
35
41
  })
36
- options = options.merge(body: Base64.encode64(options.fetch(:body, '')).strip) if has_body
42
+ options = options.merge(body: Base64.strict_encode64(options.fetch(:body, ""))) if has_body
37
43
 
44
+ @status = :responded
38
45
  @page.command("Fetch.fulfillRequest", **options)
39
46
  end
40
47
 
41
48
  def continue(**options)
42
49
  options = options.merge(requestId: request_id)
50
+ @status = :continued
43
51
  @page.command("Fetch.continueRequest", **options)
44
52
  end
45
53
 
46
54
  def abort
55
+ @status = :aborted
47
56
  @page.command("Fetch.failRequest", requestId: request_id, errorReason: "BlockedByClient")
48
57
  end
49
58
 
@@ -34,6 +34,10 @@ module Ferrum
34
34
  def headers_size
35
35
  @response["encodedDataLength"]
36
36
  end
37
+
38
+ def type
39
+ @params["type"]
40
+ end
37
41
 
38
42
  def content_type
39
43
  @content_type ||= headers.find { |k, _| k.downcase == "content-type" }&.last&.sub(/;.*\z/, "")
data/lib/ferrum/node.rb CHANGED
@@ -2,6 +2,9 @@
2
2
 
3
3
  module Ferrum
4
4
  class Node
5
+ MOVING_WAIT = ENV.fetch("FERRUM_NODE_MOVING_WAIT", 0.01).to_f
6
+ MOVING_ATTEMPTS = ENV.fetch("FERRUM_NODE_MOVING_ATTEMPTS", 50).to_i
7
+
5
8
  attr_reader :page, :target_id, :node_id, :description, :tag_name
6
9
 
7
10
  def initialize(frame, target_id, node_id, description)
@@ -24,7 +27,7 @@ module Ferrum
24
27
  end
25
28
 
26
29
  def focus
27
- tap { page.command("DOM.focus", nodeId: node_id) }
30
+ tap { page.command("DOM.focus", slowmoable: true, nodeId: node_id) }
28
31
  end
29
32
 
30
33
  def blur
@@ -37,22 +40,23 @@ module Ferrum
37
40
 
38
41
  # mode: (:left | :right | :double)
39
42
  # keys: (:alt, (:ctrl | :control), (:meta | :command), :shift)
40
- # offset: { :x, :y }
41
- def click(mode: :left, keys: [], offset: {})
42
- x, y = find_position(offset[:x], offset[:y])
43
+ # offset: { :x, :y, :position (:top | :center) }
44
+ def click(mode: :left, keys: [], offset: {}, delay: 0)
45
+ x, y = find_position(**offset)
43
46
  modifiers = page.keyboard.modifiers(keys)
44
47
 
45
48
  case mode
46
49
  when :right
47
50
  page.mouse.move(x: x, y: y)
48
51
  page.mouse.down(button: :right, modifiers: modifiers)
52
+ sleep(delay)
49
53
  page.mouse.up(button: :right, modifiers: modifiers)
50
54
  when :double
51
55
  page.mouse.move(x: x, y: y)
52
56
  page.mouse.down(modifiers: modifiers, count: 2)
53
57
  page.mouse.up(modifiers: modifiers, count: 2)
54
58
  when :left
55
- page.mouse.click(x: x, y: y, modifiers: modifiers)
59
+ page.mouse.click(x: x, y: y, modifiers: modifiers, delay: delay)
56
60
  end
57
61
 
58
62
  self
@@ -63,7 +67,7 @@ module Ferrum
63
67
  end
64
68
 
65
69
  def select_file(value)
66
- page.command("DOM.setFileInputFiles", nodeId: node_id, files: Array(value))
70
+ page.command("DOM.setFileInputFiles", slowmoable: true, nodeId: node_id, files: Array(value))
67
71
  end
68
72
 
69
73
  def at_xpath(selector)
@@ -119,35 +123,75 @@ module Ferrum
119
123
  %(#<#{self.class} @target_id=#{@target_id.inspect} @node_id=#{@node_id} @description=#{@description.inspect}>)
120
124
  end
121
125
 
122
- def find_position(offset_x = nil, offset_y = nil)
123
- quads = get_content_quads
124
- offset_x, offset_y = offset_x.to_i, offset_y.to_i
126
+ def find_position(x: nil, y: nil, position: :top)
127
+ prev = get_content_quads
128
+
129
+ # FIXME: Case when a few quads returned
130
+ points = Ferrum.with_attempts(errors: NodeIsMovingError, max: MOVING_ATTEMPTS, wait: 0) do
131
+ sleep(MOVING_WAIT)
132
+ current = get_content_quads
133
+
134
+ if current != prev
135
+ error = NodeIsMovingError.new(self, prev, current)
136
+ prev = current
137
+ raise(error)
138
+ end
139
+
140
+ current
141
+ end.map { |q| to_points(q) }.first
142
+
143
+ get_position(points, x, y, position)
144
+ rescue Ferrum::BrowserError => e
145
+ return raise unless e.message&.include?("Could not compute content quads")
146
+
147
+ find_position_via_js
148
+ end
149
+
150
+ private
151
+
152
+ def find_position_via_js
153
+ [
154
+ evaluate("this.getBoundingClientRect().left + window.pageXOffset + (this.offsetWidth / 2)"), # x
155
+ evaluate("this.getBoundingClientRect().top + window.pageYOffset + (this.offsetHeight / 2)") # y
156
+ ]
157
+ end
125
158
 
126
- if offset_x > 0 || offset_y > 0
127
- point = quads.first
128
- [point[:x] + offset_x, point[:y] + offset_y]
159
+ def get_content_quads
160
+ quads = page.command("DOM.getContentQuads", nodeId: node_id)["quads"]
161
+ raise "Node is either not visible or not an HTMLElement" if quads.size == 0
162
+ quads
163
+ end
164
+
165
+ def get_position(points, offset_x, offset_y, position)
166
+ x = y = nil
167
+
168
+ if offset_x && offset_y && position == :top
169
+ point = points.first
170
+ x = point[:x] + offset_x.to_i
171
+ y = point[:y] + offset_y.to_i
129
172
  else
130
- x, y = quads.inject([0, 0]) do |memo, point|
173
+ x, y = points.inject([0, 0]) do |memo, point|
131
174
  [memo[0] + point[:x],
132
175
  memo[1] + point[:y]]
133
176
  end
134
- [x / 4, y / 4]
177
+
178
+ x = x / 4
179
+ y = y / 4
135
180
  end
136
- end
137
181
 
138
- private
182
+ if offset_x && offset_y && position == :center
183
+ x = x + offset_x.to_i
184
+ y = y + offset_y.to_i
185
+ end
139
186
 
140
- def get_content_quads
141
- result = page.command("DOM.getContentQuads", nodeId: node_id)
142
- raise "Node is either not visible or not an HTMLElement" if result["quads"].size == 0
187
+ [x, y]
188
+ end
143
189
 
144
- # FIXME: Case when a few quads returned
145
- result["quads"].map do |quad|
146
- [{x: quad[0], y: quad[1]},
147
- {x: quad[2], y: quad[3]},
148
- {x: quad[4], y: quad[5]},
149
- {x: quad[6], y: quad[7]}]
150
- end.first
190
+ def to_points(quad)
191
+ [{x: quad[0], y: quad[1]},
192
+ {x: quad[2], y: quad[3]},
193
+ {x: quad[4], y: quad[5]},
194
+ {x: quad[6], y: quad[7]}]
151
195
  end
152
196
  end
153
197
  end