ferrum 0.8 → 0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +1 -1
- data/README.md +200 -75
- data/lib/ferrum.rb +36 -8
- data/lib/ferrum/browser.rb +12 -5
- data/lib/ferrum/browser/client.rb +6 -0
- data/lib/ferrum/browser/command.rb +26 -25
- data/lib/ferrum/browser/options/base.rb +46 -0
- data/lib/ferrum/browser/options/chrome.rb +74 -0
- data/lib/ferrum/browser/options/firefox.rb +34 -0
- data/lib/ferrum/browser/process.rb +23 -11
- data/lib/ferrum/browser/subscriber.rb +5 -1
- data/lib/ferrum/browser/web_socket.rb +17 -1
- data/lib/ferrum/browser/xvfb.rb +37 -0
- data/lib/ferrum/context.rb +10 -6
- data/lib/ferrum/contexts.rb +4 -2
- data/lib/ferrum/frame.rb +1 -0
- data/lib/ferrum/frame/dom.rb +30 -30
- data/lib/ferrum/frame/runtime.rb +62 -63
- data/lib/ferrum/network.rb +23 -6
- data/lib/ferrum/network/error.rb +8 -15
- data/lib/ferrum/network/intercepted_request.rb +1 -1
- data/lib/ferrum/node.rb +61 -17
- data/lib/ferrum/page.rb +40 -13
- data/lib/ferrum/page/animation.rb +16 -0
- data/lib/ferrum/page/frames.rb +10 -2
- data/lib/ferrum/page/screenshot.rb +64 -12
- data/lib/ferrum/rbga.rb +38 -0
- data/lib/ferrum/version.rb +1 -1
- metadata +19 -10
- data/lib/ferrum/browser/chrome.rb +0 -76
- data/lib/ferrum/browser/firefox.rb +0 -34
@@ -13,7 +13,7 @@ module Ferrum
|
|
13
13
|
|
14
14
|
def initialize
|
15
15
|
super
|
16
|
-
@on = Hash.new { |h, k| h[k] =
|
16
|
+
@on = Concurrent::Hash.new { |h, k| h[k] = Concurrent::Array.new }
|
17
17
|
end
|
18
18
|
|
19
19
|
def on(event, &block)
|
@@ -21,6 +21,10 @@ module Ferrum
|
|
21
21
|
true
|
22
22
|
end
|
23
23
|
|
24
|
+
def subscribed?(event)
|
25
|
+
@on.key?(event)
|
26
|
+
end
|
27
|
+
|
24
28
|
def call(message)
|
25
29
|
method, params = message.values_at("method", "params")
|
26
30
|
total = @on[method].size
|
@@ -8,6 +8,7 @@ module Ferrum
|
|
8
8
|
class Browser
|
9
9
|
class WebSocket
|
10
10
|
WEBSOCKET_BUG_SLEEP = 0.01
|
11
|
+
SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
|
11
12
|
|
12
13
|
attr_reader :url, :messages
|
13
14
|
|
@@ -20,6 +21,10 @@ module Ferrum
|
|
20
21
|
@driver = ::WebSocket::Driver.client(self, max_length: max_receive_size)
|
21
22
|
@messages = Queue.new
|
22
23
|
|
24
|
+
if SKIP_LOGGING_SCREENSHOTS
|
25
|
+
@screenshot_commands = Concurrent::Hash.new
|
26
|
+
end
|
27
|
+
|
23
28
|
@driver.on(:open, &method(:on_open))
|
24
29
|
@driver.on(:message, &method(:on_message))
|
25
30
|
@driver.on(:close, &method(:on_close))
|
@@ -50,7 +55,14 @@ module Ferrum
|
|
50
55
|
def on_message(event)
|
51
56
|
data = JSON.parse(event.data)
|
52
57
|
@messages.push(data)
|
53
|
-
|
58
|
+
|
59
|
+
output = event.data
|
60
|
+
if SKIP_LOGGING_SCREENSHOTS && @screenshot_commands[data["id"]]
|
61
|
+
@screenshot_commands.delete(data["id"])
|
62
|
+
output.sub!(/{"data":"(.*)"}/, %("Set FERRUM_LOGGING_SCREENSHOTS=true to see screenshots in Base64"))
|
63
|
+
end
|
64
|
+
|
65
|
+
@logger&.puts(" ◀ #{Ferrum.elapsed_time} #{output}\n")
|
54
66
|
end
|
55
67
|
|
56
68
|
def on_close(_event)
|
@@ -59,6 +71,10 @@ module Ferrum
|
|
59
71
|
end
|
60
72
|
|
61
73
|
def send_message(data)
|
74
|
+
if SKIP_LOGGING_SCREENSHOTS
|
75
|
+
@screenshot_commands[data[:id]] = true
|
76
|
+
end
|
77
|
+
|
62
78
|
json = data.to_json
|
63
79
|
@driver.text(json)
|
64
80
|
@logger&.puts("\n\n▶ #{Ferrum.elapsed_time} #{json}")
|
@@ -0,0 +1,37 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Ferrum
|
4
|
+
class Browser
|
5
|
+
class Xvfb
|
6
|
+
NOT_FOUND = "Could not find an executable for the Xvfb. Try to install " \
|
7
|
+
"it with your package manager".freeze
|
8
|
+
|
9
|
+
def self.start(*args)
|
10
|
+
new(*args).tap(&:start)
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.xvfb_path
|
14
|
+
Cliver.detect("Xvfb")
|
15
|
+
end
|
16
|
+
|
17
|
+
attr_reader :screen_size, :display_id, :pid
|
18
|
+
|
19
|
+
def initialize(options)
|
20
|
+
@path = self.class.xvfb_path
|
21
|
+
raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
|
22
|
+
|
23
|
+
@screen_size = options.fetch(:window_size, [1024, 768]).join("x") + "x24"
|
24
|
+
@display_id = (Time.now.to_f * 1000).to_i % 100_000_000
|
25
|
+
end
|
26
|
+
|
27
|
+
def start
|
28
|
+
@pid = ::Process.spawn("#{@path} :#{display_id} -screen 0 #{screen_size}")
|
29
|
+
::Process.detach(@pid)
|
30
|
+
end
|
31
|
+
|
32
|
+
def to_env
|
33
|
+
{ "DISPLAY" => ":#{display_id}" }
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
data/lib/ferrum/context.rb
CHANGED
@@ -10,7 +10,7 @@ module Ferrum
|
|
10
10
|
|
11
11
|
def initialize(browser, contexts, id)
|
12
12
|
@browser, @contexts, @id = browser, contexts, id
|
13
|
-
@targets = Concurrent::
|
13
|
+
@targets = Concurrent::Map.new
|
14
14
|
@pendings = Concurrent::MVar.new
|
15
15
|
end
|
16
16
|
|
@@ -42,19 +42,19 @@ module Ferrum
|
|
42
42
|
end
|
43
43
|
|
44
44
|
def create_target
|
45
|
-
|
46
|
-
|
47
|
-
|
45
|
+
@browser.command("Target.createTarget",
|
46
|
+
browserContextId: @id,
|
47
|
+
url: "about:blank")
|
48
48
|
target = @pendings.take(@browser.timeout)
|
49
49
|
raise NoSuchTargetError unless target.is_a?(Target)
|
50
|
-
@targets
|
50
|
+
@targets.put_if_absent(target.id, target)
|
51
51
|
target
|
52
52
|
end
|
53
53
|
|
54
54
|
def add_target(params)
|
55
55
|
target = Target.new(@browser, params)
|
56
56
|
if target.window?
|
57
|
-
@targets
|
57
|
+
@targets.put_if_absent(target.id, target)
|
58
58
|
else
|
59
59
|
@pendings.put(target, @browser.timeout)
|
60
60
|
end
|
@@ -72,6 +72,10 @@ module Ferrum
|
|
72
72
|
@contexts.dispose(@id)
|
73
73
|
end
|
74
74
|
|
75
|
+
def has_target?(target_id)
|
76
|
+
!!@targets[target_id]
|
77
|
+
end
|
78
|
+
|
75
79
|
def inspect
|
76
80
|
%(#<#{self.class} @id=#{@id.inspect} @targets=#{@targets.inspect} @default_target=#{@default_target.inspect}>)
|
77
81
|
end
|
data/lib/ferrum/contexts.rb
CHANGED
@@ -7,7 +7,7 @@ module Ferrum
|
|
7
7
|
attr_reader :contexts
|
8
8
|
|
9
9
|
def initialize(browser)
|
10
|
-
@contexts = Concurrent::
|
10
|
+
@contexts = Concurrent::Map.new
|
11
11
|
@browser = browser
|
12
12
|
subscribe
|
13
13
|
discover
|
@@ -18,7 +18,9 @@ module Ferrum
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def find_by(target_id:)
|
21
|
-
|
21
|
+
context = nil
|
22
|
+
@contexts.each_value { |c| context = c if c.has_target?(target_id) }
|
23
|
+
context
|
22
24
|
end
|
23
25
|
|
24
26
|
def create
|
data/lib/ferrum/frame.rb
CHANGED
data/lib/ferrum/frame/dom.rb
CHANGED
@@ -37,54 +37,54 @@ module Ferrum
|
|
37
37
|
end
|
38
38
|
|
39
39
|
def xpath(selector, within: nil)
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
40
|
+
expr = <<~JS
|
41
|
+
function(selector, within) {
|
42
|
+
let results = [];
|
43
|
+
within ||= document
|
44
44
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
45
|
+
let xpath = document.evaluate(selector, within, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
46
|
+
for (let i = 0; i < xpath.snapshotLength; i++) {
|
47
|
+
results.push(xpath.snapshotItem(i));
|
48
|
+
}
|
49
49
|
|
50
|
-
|
50
|
+
return results;
|
51
|
+
}
|
51
52
|
JS
|
52
53
|
|
53
|
-
|
54
|
+
evaluate_func(expr, selector, within)
|
54
55
|
end
|
55
56
|
|
56
57
|
def at_xpath(selector, within: nil)
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
58
|
+
expr = <<~JS
|
59
|
+
function(selector, within) {
|
60
|
+
within ||= document
|
61
|
+
let xpath = document.evaluate(selector, within, null, XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, null);
|
62
|
+
return xpath.snapshotItem(0);
|
63
|
+
}
|
63
64
|
JS
|
64
|
-
|
65
|
-
evaluate_async(code, @page.timeout, selector, within)
|
65
|
+
evaluate_func(expr, selector, within)
|
66
66
|
end
|
67
67
|
|
68
68
|
def css(selector, within: nil)
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
69
|
+
expr = <<~JS
|
70
|
+
function(selector, within) {
|
71
|
+
within ||= document
|
72
|
+
return Array.from(within.querySelectorAll(selector));
|
73
|
+
}
|
74
74
|
JS
|
75
75
|
|
76
|
-
|
76
|
+
evaluate_func(expr, selector, within)
|
77
77
|
end
|
78
78
|
|
79
79
|
def at_css(selector, within: nil)
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
80
|
+
expr = <<~JS
|
81
|
+
function(selector, within) {
|
82
|
+
within ||= document
|
83
|
+
return within.querySelector(selector);
|
84
|
+
}
|
85
85
|
JS
|
86
86
|
|
87
|
-
|
87
|
+
evaluate_func(expr, selector, within)
|
88
88
|
end
|
89
89
|
end
|
90
90
|
end
|
data/lib/ferrum/frame/runtime.rb
CHANGED
@@ -1,36 +1,21 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "singleton"
|
4
|
+
|
3
5
|
module Ferrum
|
6
|
+
class CyclicObject
|
7
|
+
include Singleton
|
8
|
+
|
9
|
+
def inspect
|
10
|
+
%(#<#{self.class} JavaScript object that cannot be represented in Ruby>)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
4
14
|
class Frame
|
5
15
|
module Runtime
|
6
16
|
INTERMITTENT_ATTEMPTS = ENV.fetch("FERRUM_INTERMITTENT_ATTEMPTS", 6).to_i
|
7
17
|
INTERMITTENT_SLEEP = ENV.fetch("FERRUM_INTERMITTENT_SLEEP", 0.1).to_f
|
8
18
|
|
9
|
-
EXECUTE_OPTIONS = {
|
10
|
-
returnByValue: true,
|
11
|
-
functionDeclaration: %(function() { %s })
|
12
|
-
}.freeze
|
13
|
-
DEFAULT_OPTIONS = {
|
14
|
-
functionDeclaration: %(function() { return %s })
|
15
|
-
}.freeze
|
16
|
-
EVALUATE_ASYNC_OPTIONS = {
|
17
|
-
awaitPromise: true,
|
18
|
-
functionDeclaration: %(
|
19
|
-
function() {
|
20
|
-
return new Promise((__resolve, __reject) => {
|
21
|
-
try {
|
22
|
-
arguments[arguments.length] = r => __resolve(r);
|
23
|
-
arguments.length = arguments.length + 1;
|
24
|
-
setTimeout(() => __reject(new Error("timed out promise")), %s);
|
25
|
-
%s
|
26
|
-
} catch(error) {
|
27
|
-
__reject(error);
|
28
|
-
}
|
29
|
-
});
|
30
|
-
}
|
31
|
-
)
|
32
|
-
}.freeze
|
33
|
-
|
34
19
|
SCRIPT_SRC_TAG = <<~JS
|
35
20
|
const script = document.createElement("script");
|
36
21
|
script.src = arguments[0];
|
@@ -61,37 +46,45 @@ module Ferrum
|
|
61
46
|
JS
|
62
47
|
|
63
48
|
def evaluate(expression, *args)
|
64
|
-
|
49
|
+
expression = "function() { return %s }" % expression
|
50
|
+
call(expression: expression, arguments: args)
|
65
51
|
end
|
66
52
|
|
67
|
-
def evaluate_async(expression,
|
68
|
-
|
53
|
+
def evaluate_async(expression, wait, *args)
|
54
|
+
template = <<~JS
|
55
|
+
function() {
|
56
|
+
return new Promise((__f, __r) => {
|
57
|
+
try {
|
58
|
+
arguments[arguments.length] = r => __f(r);
|
59
|
+
arguments.length = arguments.length + 1;
|
60
|
+
setTimeout(() => __r(new Error("timed out promise")), %s);
|
61
|
+
%s
|
62
|
+
} catch(error) {
|
63
|
+
__r(error);
|
64
|
+
}
|
65
|
+
});
|
66
|
+
}
|
67
|
+
JS
|
68
|
+
|
69
|
+
expression = template % [wait * 1000, expression]
|
70
|
+
call(expression: expression, arguments: args, awaitPromise: true)
|
69
71
|
end
|
70
72
|
|
71
73
|
def execute(expression, *args)
|
72
|
-
|
74
|
+
expression = "function() { %s }" % expression
|
75
|
+
call(expression: expression, arguments: args, handle: false, returnByValue: true)
|
73
76
|
true
|
74
77
|
end
|
75
78
|
|
76
|
-
def
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
Ferrum.with_attempts(errors: errors, max: attempts, wait: sleep) do
|
81
|
-
response = @page.command("DOM.resolveNode", nodeId: node.node_id)
|
82
|
-
object_id = response.dig("object", "objectId")
|
83
|
-
options = DEFAULT_OPTIONS.merge(objectId: object_id)
|
84
|
-
options[:functionDeclaration] = options[:functionDeclaration] % expression
|
85
|
-
options.merge!(returnByValue: by_value)
|
86
|
-
|
87
|
-
response = @page.command("Runtime.callFunctionOn",
|
88
|
-
wait: wait, slowmoable: true,
|
89
|
-
**options)
|
90
|
-
handle_error(response)
|
91
|
-
response = response["result"]
|
79
|
+
def evaluate_func(expression, *args, on: nil)
|
80
|
+
call(expression: expression, arguments: args, on: on)
|
81
|
+
end
|
92
82
|
|
93
|
-
|
94
|
-
|
83
|
+
def evaluate_on(node:, expression:, by_value: true, wait: 0)
|
84
|
+
options = { handle: true }
|
85
|
+
expression = "function() { return %s }" % expression
|
86
|
+
options = { handle: false, returnByValue: true } if by_value
|
87
|
+
call(expression: expression, on: node, wait: wait, **options)
|
95
88
|
end
|
96
89
|
|
97
90
|
def add_script_tag(url: nil, path: nil, content: nil, type: "text/javascript")
|
@@ -124,27 +117,31 @@ module Ferrum
|
|
124
117
|
|
125
118
|
private
|
126
119
|
|
127
|
-
def call(
|
120
|
+
def call(expression:, arguments: [], on: nil, wait: 0, handle: true, **options)
|
128
121
|
errors = [NodeNotFoundError, NoExecutionContextError]
|
129
122
|
attempts, sleep = INTERMITTENT_ATTEMPTS, INTERMITTENT_SLEEP
|
130
123
|
|
131
124
|
Ferrum.with_attempts(errors: errors, max: attempts, wait: sleep) do
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
125
|
+
params = options.dup
|
126
|
+
|
127
|
+
if on
|
128
|
+
response = @page.command("DOM.resolveNode", nodeId: on.node_id)
|
129
|
+
object_id = response.dig("object", "objectId")
|
130
|
+
params = params.merge(objectId: object_id)
|
131
|
+
end
|
132
|
+
|
133
|
+
if params[:executionContextId].nil? && params[:objectId].nil?
|
138
134
|
params = params.merge(executionContextId: execution_id)
|
139
135
|
end
|
140
136
|
|
141
137
|
response = @page.command("Runtime.callFunctionOn",
|
142
|
-
slowmoable: true,
|
143
|
-
**params
|
138
|
+
wait: wait, slowmoable: true,
|
139
|
+
**params.merge(functionDeclaration: expression,
|
140
|
+
arguments: prepare_args(arguments)))
|
144
141
|
handle_error(response)
|
145
142
|
response = response["result"]
|
146
143
|
|
147
|
-
handle ? handle_response(response) : response
|
144
|
+
handle ? handle_response(response) : response.dig("value")
|
148
145
|
end
|
149
146
|
end
|
150
147
|
|
@@ -215,7 +212,7 @@ module Ferrum
|
|
215
212
|
|
216
213
|
def reduce_props(object_id, to)
|
217
214
|
if cyclic?(object_id).dig("result", "value")
|
218
|
-
return
|
215
|
+
return to.is_a?(Array) ? [cyclic_object] : cyclic_object
|
219
216
|
else
|
220
217
|
props = @page.command("Runtime.getProperties", ownProperties: true, objectId: object_id)
|
221
218
|
props["result"].reduce(to) do |memo, prop|
|
@@ -237,15 +234,13 @@ module Ferrum
|
|
237
234
|
return false;
|
238
235
|
}
|
239
236
|
|
240
|
-
|
241
|
-
function detectCycle(obj) {
|
237
|
+
function detectCycle(obj, seen) {
|
242
238
|
if (typeof obj === "object") {
|
243
239
|
if (seen.indexOf(obj) !== -1) {
|
244
240
|
return true;
|
245
241
|
}
|
246
|
-
seen.push(obj);
|
247
242
|
for (let key in obj) {
|
248
|
-
if (obj.hasOwnProperty(key) && detectCycle(obj[key])) {
|
243
|
+
if (obj.hasOwnProperty(key) && detectCycle(obj[key], seen.concat([obj]))) {
|
249
244
|
return true;
|
250
245
|
}
|
251
246
|
}
|
@@ -254,11 +249,15 @@ module Ferrum
|
|
254
249
|
return false;
|
255
250
|
}
|
256
251
|
|
257
|
-
return detectCycle(this);
|
252
|
+
return detectCycle(this, []);
|
258
253
|
}
|
259
254
|
JS
|
260
255
|
)
|
261
256
|
end
|
257
|
+
|
258
|
+
def cyclic_object
|
259
|
+
CyclicObject.instance
|
260
|
+
end
|
262
261
|
end
|
263
262
|
end
|
264
263
|
end
|