ferrum 0.8 → 0.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/LICENSE +1 -1
- data/README.md +200 -75
- data/lib/ferrum.rb +36 -8
- data/lib/ferrum/browser.rb +12 -5
- data/lib/ferrum/browser/client.rb +6 -0
- data/lib/ferrum/browser/command.rb +26 -25
- data/lib/ferrum/browser/options/base.rb +46 -0
- data/lib/ferrum/browser/options/chrome.rb +74 -0
- data/lib/ferrum/browser/options/firefox.rb +34 -0
- data/lib/ferrum/browser/process.rb +23 -11
- data/lib/ferrum/browser/subscriber.rb +5 -1
- data/lib/ferrum/browser/web_socket.rb +17 -1
- data/lib/ferrum/browser/xvfb.rb +37 -0
- data/lib/ferrum/context.rb +10 -6
- data/lib/ferrum/contexts.rb +4 -2
- data/lib/ferrum/frame.rb +1 -0
- data/lib/ferrum/frame/dom.rb +30 -30
- data/lib/ferrum/frame/runtime.rb +62 -63
- data/lib/ferrum/network.rb +23 -6
- data/lib/ferrum/network/error.rb +8 -15
- data/lib/ferrum/network/intercepted_request.rb +1 -1
- data/lib/ferrum/node.rb +61 -17
- data/lib/ferrum/page.rb +40 -13
- data/lib/ferrum/page/animation.rb +16 -0
- data/lib/ferrum/page/frames.rb +10 -2
- data/lib/ferrum/page/screenshot.rb +64 -12
- data/lib/ferrum/rbga.rb +38 -0
- data/lib/ferrum/version.rb +1 -1
- metadata +19 -10
- data/lib/ferrum/browser/chrome.rb +0 -76
- data/lib/ferrum/browser/firefox.rb +0 -34
data/lib/ferrum.rb
CHANGED
@@ -10,14 +10,20 @@ module Ferrum
|
|
10
10
|
class NotImplementedError < Error; end
|
11
11
|
|
12
12
|
class StatusError < Error
|
13
|
+
def initialize(url, message = nil)
|
14
|
+
super(message || "Request to #{url} failed to reach server, check DNS and server status")
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
class PendingConnectionsError < StatusError
|
19
|
+
attr_reader :pendings
|
20
|
+
|
13
21
|
def initialize(url, pendings = [])
|
14
|
-
|
15
|
-
"Request to #{url} failed to reach server, check DNS and/or server status"
|
16
|
-
else
|
17
|
-
"Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
|
18
|
-
end
|
22
|
+
@pendings = pendings
|
19
23
|
|
20
|
-
|
24
|
+
message = "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
|
25
|
+
|
26
|
+
super(url, message)
|
21
27
|
end
|
22
28
|
end
|
23
29
|
|
@@ -37,8 +43,11 @@ module Ferrum
|
|
37
43
|
end
|
38
44
|
|
39
45
|
class ProcessTimeoutError < Error
|
40
|
-
|
41
|
-
|
46
|
+
attr_reader :output
|
47
|
+
|
48
|
+
def initialize(timeout, output)
|
49
|
+
@output = output
|
50
|
+
super("Browser did not produce websocket url within #{timeout} seconds, try to increase `:process_timeout`. See https://github.com/rubycdp/ferrum#customization")
|
42
51
|
end
|
43
52
|
end
|
44
53
|
|
@@ -48,6 +57,25 @@ module Ferrum
|
|
48
57
|
end
|
49
58
|
end
|
50
59
|
|
60
|
+
class NodeMovingError < Error
|
61
|
+
def initialize(node, prev, current)
|
62
|
+
@node, @prev, @current = node, prev, current
|
63
|
+
super(message)
|
64
|
+
end
|
65
|
+
|
66
|
+
def message
|
67
|
+
"#{@node.inspect} that you're trying to click is moving, hence " \
|
68
|
+
"we cannot. Previosuly it was at #{@prev.inspect} but now at " \
|
69
|
+
"#{@current.inspect}."
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
class CoordinatesNotFoundError < Error
|
74
|
+
def initialize(message = "Could not compute content quads")
|
75
|
+
super
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
51
79
|
class BrowserError < Error
|
52
80
|
attr_reader :response
|
53
81
|
|
data/lib/ferrum/browser.rb
CHANGED
@@ -4,6 +4,7 @@ require "base64"
|
|
4
4
|
require "forwardable"
|
5
5
|
require "ferrum/page"
|
6
6
|
require "ferrum/contexts"
|
7
|
+
require "ferrum/browser/xvfb"
|
7
8
|
require "ferrum/browser/process"
|
8
9
|
require "ferrum/browser/client"
|
9
10
|
|
@@ -16,19 +17,20 @@ module Ferrum
|
|
16
17
|
extend Forwardable
|
17
18
|
delegate %i[default_context] => :contexts
|
18
19
|
delegate %i[targets create_target create_page page pages windows] => :default_context
|
19
|
-
delegate %i[
|
20
|
+
delegate %i[go_to back forward refresh reload stop wait_for_reload
|
20
21
|
at_css at_xpath css xpath current_url current_title url title
|
21
22
|
body doctype set_content
|
22
23
|
headers cookies network
|
23
24
|
mouse keyboard
|
24
|
-
screenshot pdf viewport_size
|
25
|
+
screenshot pdf mhtml viewport_size
|
25
26
|
frames frame_by main_frame
|
26
|
-
evaluate evaluate_on evaluate_async execute
|
27
|
+
evaluate evaluate_on evaluate_async execute evaluate_func
|
27
28
|
add_script_tag add_style_tag bypass_csp
|
28
|
-
on
|
29
|
+
on goto position position=
|
30
|
+
playback_rate playback_rate=] => :page
|
29
31
|
delegate %i[default_user_agent] => :process
|
30
32
|
|
31
|
-
attr_reader :client, :process, :contexts, :logger, :js_errors,
|
33
|
+
attr_reader :client, :process, :contexts, :logger, :js_errors, :pending_connection_errors,
|
32
34
|
:slowmo, :base_url, :options, :window_size, :ws_max_receive_size
|
33
35
|
attr_writer :timeout
|
34
36
|
|
@@ -43,6 +45,7 @@ module Ferrum
|
|
43
45
|
@logger, @timeout, @ws_max_receive_size =
|
44
46
|
@options.values_at(:logger, :timeout, :ws_max_receive_size)
|
45
47
|
@js_errors = @options.fetch(:js_errors, false)
|
48
|
+
@pending_connection_errors = @options.fetch(:pending_connection_errors, true)
|
46
49
|
@slowmo = @options[:slowmo].to_f
|
47
50
|
|
48
51
|
if @options.key?(:base_url)
|
@@ -75,6 +78,10 @@ module Ferrum
|
|
75
78
|
end
|
76
79
|
end
|
77
80
|
|
81
|
+
def evaluate_on_new_document(expression)
|
82
|
+
extensions << expression
|
83
|
+
end
|
84
|
+
|
78
85
|
def timeout
|
79
86
|
@timeout || DEFAULT_TIMEOUT
|
80
87
|
end
|
@@ -58,6 +58,10 @@ module Ferrum
|
|
58
58
|
end
|
59
59
|
end
|
60
60
|
|
61
|
+
def subscribed?(event)
|
62
|
+
[@interruptor, @subscriber].any? { |s| s.subscribed?(event) }
|
63
|
+
end
|
64
|
+
|
61
65
|
def close
|
62
66
|
@ws.close
|
63
67
|
# Give a thread some time to handle a tail of messages
|
@@ -86,6 +90,8 @@ module Ferrum
|
|
86
90
|
raise NoExecutionContextError.new(error)
|
87
91
|
when "No target with given id found"
|
88
92
|
raise NoSuchPageError
|
93
|
+
when /Could not compute content quads/
|
94
|
+
raise CoordinatesNotFoundError
|
89
95
|
else
|
90
96
|
raise BrowserError.new(error)
|
91
97
|
end
|
@@ -3,53 +3,54 @@
|
|
3
3
|
module Ferrum
|
4
4
|
class Browser
|
5
5
|
class Command
|
6
|
-
BROWSER_HOST = "127.0.0.1"
|
7
|
-
BROWSER_PORT = "0"
|
8
6
|
NOT_FOUND = "Could not find an executable for the browser. Try to make " \
|
9
|
-
"it available on the PATH or set environment
|
7
|
+
"it available on the PATH or set environment variable for " \
|
10
8
|
"example BROWSER_PATH=\"/usr/bin/chrome\"".freeze
|
11
9
|
|
12
10
|
# Currently only these browsers support CDP:
|
13
11
|
# https://github.com/cyrus-and/chrome-remote-interface#implementations
|
14
12
|
def self.build(options, user_data_dir)
|
15
|
-
case options[:browser_name]
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
13
|
+
defaults = case options[:browser_name]
|
14
|
+
when :firefox
|
15
|
+
Options::Firefox.options
|
16
|
+
when :chrome, :opera, :edge, nil
|
17
|
+
Options::Chrome.options
|
18
|
+
else
|
19
|
+
raise NotImplementedError, "not supported browser"
|
20
|
+
end
|
21
|
+
|
22
|
+
new(defaults, options, user_data_dir)
|
23
23
|
end
|
24
24
|
|
25
|
-
attr_reader :
|
25
|
+
attr_reader :defaults, :path, :options
|
26
26
|
|
27
|
-
def initialize(options, user_data_dir)
|
27
|
+
def initialize(defaults, options, user_data_dir)
|
28
28
|
@flags = {}
|
29
|
+
@defaults = defaults
|
29
30
|
@options, @user_data_dir = options, user_data_dir
|
30
|
-
@path = options[:browser_path] || ENV["BROWSER_PATH"] || detect_path
|
31
|
+
@path = options[:browser_path] || ENV["BROWSER_PATH"] || defaults.detect_path
|
31
32
|
raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
|
33
|
+
merge_options
|
34
|
+
end
|
32
35
|
|
33
|
-
|
36
|
+
def xvfb?
|
37
|
+
!!options[:xvfb]
|
34
38
|
end
|
35
39
|
|
36
40
|
def to_a
|
37
|
-
[path] + flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
|
41
|
+
[path] + @flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
|
38
42
|
end
|
39
43
|
|
40
44
|
private
|
41
45
|
|
42
|
-
def
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
.find { |b| p = Cliver.detect(b) and break(p) }
|
46
|
+
def merge_options
|
47
|
+
@flags = defaults.merge_required(@flags, options, @user_data_dir)
|
48
|
+
|
49
|
+
unless options[:ignore_default_browser_options]
|
50
|
+
@flags = defaults.merge_default(@flags, options)
|
48
51
|
end
|
49
|
-
end
|
50
52
|
|
51
|
-
|
52
|
-
raise NotImplementedError
|
53
|
+
@flags.merge!(options.fetch(:browser_options, {}))
|
53
54
|
end
|
54
55
|
end
|
55
56
|
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "singleton"
|
4
|
+
|
5
|
+
module Ferrum
|
6
|
+
class Browser
|
7
|
+
module Options
|
8
|
+
class Base
|
9
|
+
BROWSER_HOST = "127.0.0.1"
|
10
|
+
BROWSER_PORT = "0"
|
11
|
+
|
12
|
+
include Singleton
|
13
|
+
|
14
|
+
def self.options
|
15
|
+
instance
|
16
|
+
end
|
17
|
+
|
18
|
+
def to_h
|
19
|
+
self.class::DEFAULT_OPTIONS
|
20
|
+
end
|
21
|
+
|
22
|
+
def except(*keys)
|
23
|
+
to_h.reject { |n, _| keys.include?(n) }
|
24
|
+
end
|
25
|
+
|
26
|
+
def detect_path
|
27
|
+
if Ferrum.mac?
|
28
|
+
self.class::MAC_BIN_PATH.find { |n| File.exist?(n) }
|
29
|
+
else
|
30
|
+
self.class::LINUX_BIN_PATH.find do |name|
|
31
|
+
path = Cliver.detect(name) and break(path)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def merge_required(flags, options, user_data_dir)
|
37
|
+
raise NotImplementedError
|
38
|
+
end
|
39
|
+
|
40
|
+
def merge_default(flags, options)
|
41
|
+
raise NotImplementedError
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Ferrum
|
4
|
+
class Browser
|
5
|
+
module Options
|
6
|
+
class Chrome < Base
|
7
|
+
DEFAULT_OPTIONS = {
|
8
|
+
"headless" => nil,
|
9
|
+
"disable-gpu" => nil,
|
10
|
+
"hide-scrollbars" => nil,
|
11
|
+
"mute-audio" => nil,
|
12
|
+
"enable-automation" => nil,
|
13
|
+
"disable-web-security" => nil,
|
14
|
+
"disable-session-crashed-bubble" => nil,
|
15
|
+
"disable-breakpad" => nil,
|
16
|
+
"disable-sync" => nil,
|
17
|
+
"no-first-run" => nil,
|
18
|
+
"use-mock-keychain" => nil,
|
19
|
+
"keep-alive-for-test" => nil,
|
20
|
+
"disable-popup-blocking" => nil,
|
21
|
+
"disable-extensions" => nil,
|
22
|
+
"disable-hang-monitor" => nil,
|
23
|
+
"disable-features" => "site-per-process,TranslateUI",
|
24
|
+
"disable-translate" => nil,
|
25
|
+
"disable-background-networking" => nil,
|
26
|
+
"enable-features" => "NetworkService,NetworkServiceInProcess",
|
27
|
+
"disable-background-timer-throttling" => nil,
|
28
|
+
"disable-backgrounding-occluded-windows" => nil,
|
29
|
+
"disable-client-side-phishing-detection" => nil,
|
30
|
+
"disable-default-apps" => nil,
|
31
|
+
"disable-dev-shm-usage" => nil,
|
32
|
+
"disable-ipc-flooding-protection" => nil,
|
33
|
+
"disable-prompt-on-repost" => nil,
|
34
|
+
"disable-renderer-backgrounding" => nil,
|
35
|
+
"force-color-profile" => "srgb",
|
36
|
+
"metrics-recording-only" => nil,
|
37
|
+
"safebrowsing-disable-auto-update" => nil,
|
38
|
+
"password-store" => "basic",
|
39
|
+
"no-startup-window" => nil,
|
40
|
+
# Note: --no-sandbox is not needed if you properly setup a user in the container.
|
41
|
+
# https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
|
42
|
+
# "no-sandbox" => nil,
|
43
|
+
}.freeze
|
44
|
+
|
45
|
+
MAC_BIN_PATH = [
|
46
|
+
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
47
|
+
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
|
48
|
+
].freeze
|
49
|
+
LINUX_BIN_PATH = %w[chromium google-chrome-unstable google-chrome-beta
|
50
|
+
google-chrome chrome chromium-browser
|
51
|
+
google-chrome-stable].freeze
|
52
|
+
|
53
|
+
def merge_required(flags, options, user_data_dir)
|
54
|
+
port = options.fetch(:port, BROWSER_PORT)
|
55
|
+
host = options.fetch(:host, BROWSER_HOST)
|
56
|
+
flags.merge("remote-debugging-port" => port,
|
57
|
+
"remote-debugging-address" => host,
|
58
|
+
# Doesn't work on MacOS, so we need to set it by CDP
|
59
|
+
"window-size" => options[:window_size].join(","),
|
60
|
+
"user-data-dir" => user_data_dir)
|
61
|
+
end
|
62
|
+
|
63
|
+
def merge_default(flags, options)
|
64
|
+
unless options.fetch(:headless, true)
|
65
|
+
defaults = except("headless", "disable-gpu")
|
66
|
+
end
|
67
|
+
|
68
|
+
defaults ||= DEFAULT_OPTIONS
|
69
|
+
defaults.merge(flags)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Ferrum
|
4
|
+
class Browser
|
5
|
+
module Options
|
6
|
+
class Firefox < Base
|
7
|
+
DEFAULT_OPTIONS = {
|
8
|
+
"headless" => nil,
|
9
|
+
}.freeze
|
10
|
+
|
11
|
+
MAC_BIN_PATH = [
|
12
|
+
"/Applications/Firefox.app/Contents/MacOS/firefox-bin"
|
13
|
+
].freeze
|
14
|
+
LINUX_BIN_PATH = %w[firefox].freeze
|
15
|
+
|
16
|
+
def merge_required(flags, options, user_data_dir)
|
17
|
+
port = options.fetch(:port, BROWSER_PORT)
|
18
|
+
host = options.fetch(:host, BROWSER_HOST)
|
19
|
+
flags.merge("remote-debugger" => "#{host}:#{port}",
|
20
|
+
"profile" => user_data_dir)
|
21
|
+
end
|
22
|
+
|
23
|
+
def merge_default(flags, options)
|
24
|
+
unless options.fetch(:headless, true)
|
25
|
+
defaults = except("headless")
|
26
|
+
end
|
27
|
+
|
28
|
+
defaults ||= DEFAULT_OPTIONS
|
29
|
+
defaults.merge(flags)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -6,20 +6,21 @@ require "json"
|
|
6
6
|
require "addressable"
|
7
7
|
require "tmpdir"
|
8
8
|
require "forwardable"
|
9
|
+
require "ferrum/browser/options/base"
|
10
|
+
require "ferrum/browser/options/chrome"
|
11
|
+
require "ferrum/browser/options/firefox"
|
9
12
|
require "ferrum/browser/command"
|
10
|
-
require "ferrum/browser/chrome"
|
11
|
-
require "ferrum/browser/firefox"
|
12
13
|
|
13
14
|
module Ferrum
|
14
15
|
class Browser
|
15
16
|
class Process
|
16
17
|
KILL_TIMEOUT = 2
|
17
18
|
WAIT_KILLED = 0.05
|
18
|
-
PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT",
|
19
|
+
PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
|
19
20
|
|
20
21
|
attr_reader :host, :port, :ws_url, :pid, :command,
|
21
22
|
:default_user_agent, :browser_version, :protocol_version,
|
22
|
-
:v8_version, :webkit_version
|
23
|
+
:v8_version, :webkit_version, :xvfb
|
23
24
|
|
24
25
|
|
25
26
|
extend Forwardable
|
@@ -63,11 +64,13 @@ module Ferrum
|
|
63
64
|
return
|
64
65
|
end
|
65
66
|
|
67
|
+
@pid = @xvfb = @user_data_dir = nil
|
66
68
|
@logger = options[:logger]
|
67
69
|
@process_timeout = options.fetch(:process_timeout, PROCESS_TIMEOUT)
|
68
70
|
|
69
|
-
tmpdir = Dir.mktmpdir
|
71
|
+
tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
|
70
72
|
ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
|
73
|
+
@user_data_dir = tmpdir
|
71
74
|
@command = Command.build(options, tmpdir)
|
72
75
|
end
|
73
76
|
|
@@ -81,7 +84,12 @@ module Ferrum
|
|
81
84
|
process_options[:pgroup] = true unless Ferrum.windows?
|
82
85
|
process_options[:out] = process_options[:err] = write_io
|
83
86
|
|
84
|
-
@
|
87
|
+
if @command.xvfb?
|
88
|
+
@xvfb = Xvfb.start(@command.options)
|
89
|
+
ObjectSpace.define_finalizer(self, self.class.process_killer(@xvfb.pid))
|
90
|
+
end
|
91
|
+
|
92
|
+
@pid = ::Process.spawn(Hash(@xvfb&.to_env), *@command.to_a, process_options)
|
85
93
|
ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
|
86
94
|
|
87
95
|
parse_ws_url(read_io, @process_timeout)
|
@@ -92,7 +100,12 @@ module Ferrum
|
|
92
100
|
end
|
93
101
|
|
94
102
|
def stop
|
95
|
-
|
103
|
+
if @pid
|
104
|
+
kill(@pid)
|
105
|
+
kill(@xvfb.pid) if @xvfb&.pid
|
106
|
+
@pid = nil
|
107
|
+
end
|
108
|
+
|
96
109
|
remove_user_data_dir if @user_data_dir
|
97
110
|
ObjectSpace.undefine_finalizer(self)
|
98
111
|
end
|
@@ -104,9 +117,8 @@ module Ferrum
|
|
104
117
|
|
105
118
|
private
|
106
119
|
|
107
|
-
def kill
|
108
|
-
self.class.process_killer(
|
109
|
-
@pid = nil
|
120
|
+
def kill(pid)
|
121
|
+
self.class.process_killer(pid).call
|
110
122
|
end
|
111
123
|
|
112
124
|
def remove_user_data_dir
|
@@ -134,7 +146,7 @@ module Ferrum
|
|
134
146
|
|
135
147
|
unless ws_url
|
136
148
|
@logger.puts(output) if @logger
|
137
|
-
raise ProcessTimeoutError.new(timeout)
|
149
|
+
raise ProcessTimeoutError.new(timeout, output)
|
138
150
|
end
|
139
151
|
end
|
140
152
|
|