ferrum 0.8 → 0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +1 -1
- data/README.md +200 -75
- data/lib/ferrum.rb +36 -8
- data/lib/ferrum/browser.rb +12 -5
- data/lib/ferrum/browser/client.rb +6 -0
- data/lib/ferrum/browser/command.rb +26 -25
- data/lib/ferrum/browser/options/base.rb +46 -0
- data/lib/ferrum/browser/options/chrome.rb +74 -0
- data/lib/ferrum/browser/options/firefox.rb +34 -0
- data/lib/ferrum/browser/process.rb +23 -11
- data/lib/ferrum/browser/subscriber.rb +5 -1
- data/lib/ferrum/browser/web_socket.rb +17 -1
- data/lib/ferrum/browser/xvfb.rb +37 -0
- data/lib/ferrum/context.rb +10 -6
- data/lib/ferrum/contexts.rb +4 -2
- data/lib/ferrum/frame.rb +1 -0
- data/lib/ferrum/frame/dom.rb +30 -30
- data/lib/ferrum/frame/runtime.rb +62 -63
- data/lib/ferrum/network.rb +23 -6
- data/lib/ferrum/network/error.rb +8 -15
- data/lib/ferrum/network/intercepted_request.rb +1 -1
- data/lib/ferrum/node.rb +61 -17
- data/lib/ferrum/page.rb +40 -13
- data/lib/ferrum/page/animation.rb +16 -0
- data/lib/ferrum/page/frames.rb +10 -2
- data/lib/ferrum/page/screenshot.rb +64 -12
- data/lib/ferrum/rbga.rb +38 -0
- data/lib/ferrum/version.rb +1 -1
- metadata +19 -10
- data/lib/ferrum/browser/chrome.rb +0 -76
- data/lib/ferrum/browser/firefox.rb +0 -34
data/lib/ferrum.rb
CHANGED
@@ -10,14 +10,20 @@ module Ferrum
|
|
10
10
|
class NotImplementedError < Error; end
|
11
11
|
|
12
12
|
class StatusError < Error
|
13
|
+
def initialize(url, message = nil)
|
14
|
+
super(message || "Request to #{url} failed to reach server, check DNS and server status")
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
class PendingConnectionsError < StatusError
|
19
|
+
attr_reader :pendings
|
20
|
+
|
13
21
|
def initialize(url, pendings = [])
|
14
|
-
|
15
|
-
"Request to #{url} failed to reach server, check DNS and/or server status"
|
16
|
-
else
|
17
|
-
"Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
|
18
|
-
end
|
22
|
+
@pendings = pendings
|
19
23
|
|
20
|
-
|
24
|
+
message = "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
|
25
|
+
|
26
|
+
super(url, message)
|
21
27
|
end
|
22
28
|
end
|
23
29
|
|
@@ -37,8 +43,11 @@ module Ferrum
|
|
37
43
|
end
|
38
44
|
|
39
45
|
class ProcessTimeoutError < Error
|
40
|
-
|
41
|
-
|
46
|
+
attr_reader :output
|
47
|
+
|
48
|
+
def initialize(timeout, output)
|
49
|
+
@output = output
|
50
|
+
super("Browser did not produce websocket url within #{timeout} seconds, try to increase `:process_timeout`. See https://github.com/rubycdp/ferrum#customization")
|
42
51
|
end
|
43
52
|
end
|
44
53
|
|
@@ -48,6 +57,25 @@ module Ferrum
|
|
48
57
|
end
|
49
58
|
end
|
50
59
|
|
60
|
+
class NodeMovingError < Error
|
61
|
+
def initialize(node, prev, current)
|
62
|
+
@node, @prev, @current = node, prev, current
|
63
|
+
super(message)
|
64
|
+
end
|
65
|
+
|
66
|
+
def message
|
67
|
+
"#{@node.inspect} that you're trying to click is moving, hence " \
|
68
|
+
"we cannot. Previosuly it was at #{@prev.inspect} but now at " \
|
69
|
+
"#{@current.inspect}."
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
class CoordinatesNotFoundError < Error
|
74
|
+
def initialize(message = "Could not compute content quads")
|
75
|
+
super
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
51
79
|
class BrowserError < Error
|
52
80
|
attr_reader :response
|
53
81
|
|
data/lib/ferrum/browser.rb
CHANGED
@@ -4,6 +4,7 @@ require "base64"
|
|
4
4
|
require "forwardable"
|
5
5
|
require "ferrum/page"
|
6
6
|
require "ferrum/contexts"
|
7
|
+
require "ferrum/browser/xvfb"
|
7
8
|
require "ferrum/browser/process"
|
8
9
|
require "ferrum/browser/client"
|
9
10
|
|
@@ -16,19 +17,20 @@ module Ferrum
|
|
16
17
|
extend Forwardable
|
17
18
|
delegate %i[default_context] => :contexts
|
18
19
|
delegate %i[targets create_target create_page page pages windows] => :default_context
|
19
|
-
delegate %i[
|
20
|
+
delegate %i[go_to back forward refresh reload stop wait_for_reload
|
20
21
|
at_css at_xpath css xpath current_url current_title url title
|
21
22
|
body doctype set_content
|
22
23
|
headers cookies network
|
23
24
|
mouse keyboard
|
24
|
-
screenshot pdf viewport_size
|
25
|
+
screenshot pdf mhtml viewport_size
|
25
26
|
frames frame_by main_frame
|
26
|
-
evaluate evaluate_on evaluate_async execute
|
27
|
+
evaluate evaluate_on evaluate_async execute evaluate_func
|
27
28
|
add_script_tag add_style_tag bypass_csp
|
28
|
-
on
|
29
|
+
on goto position position=
|
30
|
+
playback_rate playback_rate=] => :page
|
29
31
|
delegate %i[default_user_agent] => :process
|
30
32
|
|
31
|
-
attr_reader :client, :process, :contexts, :logger, :js_errors,
|
33
|
+
attr_reader :client, :process, :contexts, :logger, :js_errors, :pending_connection_errors,
|
32
34
|
:slowmo, :base_url, :options, :window_size, :ws_max_receive_size
|
33
35
|
attr_writer :timeout
|
34
36
|
|
@@ -43,6 +45,7 @@ module Ferrum
|
|
43
45
|
@logger, @timeout, @ws_max_receive_size =
|
44
46
|
@options.values_at(:logger, :timeout, :ws_max_receive_size)
|
45
47
|
@js_errors = @options.fetch(:js_errors, false)
|
48
|
+
@pending_connection_errors = @options.fetch(:pending_connection_errors, true)
|
46
49
|
@slowmo = @options[:slowmo].to_f
|
47
50
|
|
48
51
|
if @options.key?(:base_url)
|
@@ -75,6 +78,10 @@ module Ferrum
|
|
75
78
|
end
|
76
79
|
end
|
77
80
|
|
81
|
+
def evaluate_on_new_document(expression)
|
82
|
+
extensions << expression
|
83
|
+
end
|
84
|
+
|
78
85
|
def timeout
|
79
86
|
@timeout || DEFAULT_TIMEOUT
|
80
87
|
end
|
@@ -58,6 +58,10 @@ module Ferrum
|
|
58
58
|
end
|
59
59
|
end
|
60
60
|
|
61
|
+
def subscribed?(event)
|
62
|
+
[@interruptor, @subscriber].any? { |s| s.subscribed?(event) }
|
63
|
+
end
|
64
|
+
|
61
65
|
def close
|
62
66
|
@ws.close
|
63
67
|
# Give a thread some time to handle a tail of messages
|
@@ -86,6 +90,8 @@ module Ferrum
|
|
86
90
|
raise NoExecutionContextError.new(error)
|
87
91
|
when "No target with given id found"
|
88
92
|
raise NoSuchPageError
|
93
|
+
when /Could not compute content quads/
|
94
|
+
raise CoordinatesNotFoundError
|
89
95
|
else
|
90
96
|
raise BrowserError.new(error)
|
91
97
|
end
|
@@ -3,53 +3,54 @@
|
|
3
3
|
module Ferrum
|
4
4
|
class Browser
|
5
5
|
class Command
|
6
|
-
BROWSER_HOST = "127.0.0.1"
|
7
|
-
BROWSER_PORT = "0"
|
8
6
|
NOT_FOUND = "Could not find an executable for the browser. Try to make " \
|
9
|
-
"it available on the PATH or set environment
|
7
|
+
"it available on the PATH or set environment variable for " \
|
10
8
|
"example BROWSER_PATH=\"/usr/bin/chrome\"".freeze
|
11
9
|
|
12
10
|
# Currently only these browsers support CDP:
|
13
11
|
# https://github.com/cyrus-and/chrome-remote-interface#implementations
|
14
12
|
def self.build(options, user_data_dir)
|
15
|
-
case options[:browser_name]
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
13
|
+
defaults = case options[:browser_name]
|
14
|
+
when :firefox
|
15
|
+
Options::Firefox.options
|
16
|
+
when :chrome, :opera, :edge, nil
|
17
|
+
Options::Chrome.options
|
18
|
+
else
|
19
|
+
raise NotImplementedError, "not supported browser"
|
20
|
+
end
|
21
|
+
|
22
|
+
new(defaults, options, user_data_dir)
|
23
23
|
end
|
24
24
|
|
25
|
-
attr_reader :
|
25
|
+
attr_reader :defaults, :path, :options
|
26
26
|
|
27
|
-
def initialize(options, user_data_dir)
|
27
|
+
def initialize(defaults, options, user_data_dir)
|
28
28
|
@flags = {}
|
29
|
+
@defaults = defaults
|
29
30
|
@options, @user_data_dir = options, user_data_dir
|
30
|
-
@path = options[:browser_path] || ENV["BROWSER_PATH"] || detect_path
|
31
|
+
@path = options[:browser_path] || ENV["BROWSER_PATH"] || defaults.detect_path
|
31
32
|
raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
|
33
|
+
merge_options
|
34
|
+
end
|
32
35
|
|
33
|
-
|
36
|
+
def xvfb?
|
37
|
+
!!options[:xvfb]
|
34
38
|
end
|
35
39
|
|
36
40
|
def to_a
|
37
|
-
[path] + flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
|
41
|
+
[path] + @flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
|
38
42
|
end
|
39
43
|
|
40
44
|
private
|
41
45
|
|
42
|
-
def
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
.find { |b| p = Cliver.detect(b) and break(p) }
|
46
|
+
def merge_options
|
47
|
+
@flags = defaults.merge_required(@flags, options, @user_data_dir)
|
48
|
+
|
49
|
+
unless options[:ignore_default_browser_options]
|
50
|
+
@flags = defaults.merge_default(@flags, options)
|
48
51
|
end
|
49
|
-
end
|
50
52
|
|
51
|
-
|
52
|
-
raise NotImplementedError
|
53
|
+
@flags.merge!(options.fetch(:browser_options, {}))
|
53
54
|
end
|
54
55
|
end
|
55
56
|
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "singleton"
|
4
|
+
|
5
|
+
module Ferrum
|
6
|
+
class Browser
|
7
|
+
module Options
|
8
|
+
class Base
|
9
|
+
BROWSER_HOST = "127.0.0.1"
|
10
|
+
BROWSER_PORT = "0"
|
11
|
+
|
12
|
+
include Singleton
|
13
|
+
|
14
|
+
def self.options
|
15
|
+
instance
|
16
|
+
end
|
17
|
+
|
18
|
+
def to_h
|
19
|
+
self.class::DEFAULT_OPTIONS
|
20
|
+
end
|
21
|
+
|
22
|
+
def except(*keys)
|
23
|
+
to_h.reject { |n, _| keys.include?(n) }
|
24
|
+
end
|
25
|
+
|
26
|
+
def detect_path
|
27
|
+
if Ferrum.mac?
|
28
|
+
self.class::MAC_BIN_PATH.find { |n| File.exist?(n) }
|
29
|
+
else
|
30
|
+
self.class::LINUX_BIN_PATH.find do |name|
|
31
|
+
path = Cliver.detect(name) and break(path)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def merge_required(flags, options, user_data_dir)
|
37
|
+
raise NotImplementedError
|
38
|
+
end
|
39
|
+
|
40
|
+
def merge_default(flags, options)
|
41
|
+
raise NotImplementedError
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Ferrum
|
4
|
+
class Browser
|
5
|
+
module Options
|
6
|
+
class Chrome < Base
|
7
|
+
DEFAULT_OPTIONS = {
|
8
|
+
"headless" => nil,
|
9
|
+
"disable-gpu" => nil,
|
10
|
+
"hide-scrollbars" => nil,
|
11
|
+
"mute-audio" => nil,
|
12
|
+
"enable-automation" => nil,
|
13
|
+
"disable-web-security" => nil,
|
14
|
+
"disable-session-crashed-bubble" => nil,
|
15
|
+
"disable-breakpad" => nil,
|
16
|
+
"disable-sync" => nil,
|
17
|
+
"no-first-run" => nil,
|
18
|
+
"use-mock-keychain" => nil,
|
19
|
+
"keep-alive-for-test" => nil,
|
20
|
+
"disable-popup-blocking" => nil,
|
21
|
+
"disable-extensions" => nil,
|
22
|
+
"disable-hang-monitor" => nil,
|
23
|
+
"disable-features" => "site-per-process,TranslateUI",
|
24
|
+
"disable-translate" => nil,
|
25
|
+
"disable-background-networking" => nil,
|
26
|
+
"enable-features" => "NetworkService,NetworkServiceInProcess",
|
27
|
+
"disable-background-timer-throttling" => nil,
|
28
|
+
"disable-backgrounding-occluded-windows" => nil,
|
29
|
+
"disable-client-side-phishing-detection" => nil,
|
30
|
+
"disable-default-apps" => nil,
|
31
|
+
"disable-dev-shm-usage" => nil,
|
32
|
+
"disable-ipc-flooding-protection" => nil,
|
33
|
+
"disable-prompt-on-repost" => nil,
|
34
|
+
"disable-renderer-backgrounding" => nil,
|
35
|
+
"force-color-profile" => "srgb",
|
36
|
+
"metrics-recording-only" => nil,
|
37
|
+
"safebrowsing-disable-auto-update" => nil,
|
38
|
+
"password-store" => "basic",
|
39
|
+
"no-startup-window" => nil,
|
40
|
+
# Note: --no-sandbox is not needed if you properly setup a user in the container.
|
41
|
+
# https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
|
42
|
+
# "no-sandbox" => nil,
|
43
|
+
}.freeze
|
44
|
+
|
45
|
+
MAC_BIN_PATH = [
|
46
|
+
"/Applications/Chromium.app/Contents/MacOS/Chromium",
|
47
|
+
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
|
48
|
+
].freeze
|
49
|
+
LINUX_BIN_PATH = %w[chromium google-chrome-unstable google-chrome-beta
|
50
|
+
google-chrome chrome chromium-browser
|
51
|
+
google-chrome-stable].freeze
|
52
|
+
|
53
|
+
def merge_required(flags, options, user_data_dir)
|
54
|
+
port = options.fetch(:port, BROWSER_PORT)
|
55
|
+
host = options.fetch(:host, BROWSER_HOST)
|
56
|
+
flags.merge("remote-debugging-port" => port,
|
57
|
+
"remote-debugging-address" => host,
|
58
|
+
# Doesn't work on MacOS, so we need to set it by CDP
|
59
|
+
"window-size" => options[:window_size].join(","),
|
60
|
+
"user-data-dir" => user_data_dir)
|
61
|
+
end
|
62
|
+
|
63
|
+
def merge_default(flags, options)
|
64
|
+
unless options.fetch(:headless, true)
|
65
|
+
defaults = except("headless", "disable-gpu")
|
66
|
+
end
|
67
|
+
|
68
|
+
defaults ||= DEFAULT_OPTIONS
|
69
|
+
defaults.merge(flags)
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Ferrum
|
4
|
+
class Browser
|
5
|
+
module Options
|
6
|
+
class Firefox < Base
|
7
|
+
DEFAULT_OPTIONS = {
|
8
|
+
"headless" => nil,
|
9
|
+
}.freeze
|
10
|
+
|
11
|
+
MAC_BIN_PATH = [
|
12
|
+
"/Applications/Firefox.app/Contents/MacOS/firefox-bin"
|
13
|
+
].freeze
|
14
|
+
LINUX_BIN_PATH = %w[firefox].freeze
|
15
|
+
|
16
|
+
def merge_required(flags, options, user_data_dir)
|
17
|
+
port = options.fetch(:port, BROWSER_PORT)
|
18
|
+
host = options.fetch(:host, BROWSER_HOST)
|
19
|
+
flags.merge("remote-debugger" => "#{host}:#{port}",
|
20
|
+
"profile" => user_data_dir)
|
21
|
+
end
|
22
|
+
|
23
|
+
def merge_default(flags, options)
|
24
|
+
unless options.fetch(:headless, true)
|
25
|
+
defaults = except("headless")
|
26
|
+
end
|
27
|
+
|
28
|
+
defaults ||= DEFAULT_OPTIONS
|
29
|
+
defaults.merge(flags)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -6,20 +6,21 @@ require "json"
|
|
6
6
|
require "addressable"
|
7
7
|
require "tmpdir"
|
8
8
|
require "forwardable"
|
9
|
+
require "ferrum/browser/options/base"
|
10
|
+
require "ferrum/browser/options/chrome"
|
11
|
+
require "ferrum/browser/options/firefox"
|
9
12
|
require "ferrum/browser/command"
|
10
|
-
require "ferrum/browser/chrome"
|
11
|
-
require "ferrum/browser/firefox"
|
12
13
|
|
13
14
|
module Ferrum
|
14
15
|
class Browser
|
15
16
|
class Process
|
16
17
|
KILL_TIMEOUT = 2
|
17
18
|
WAIT_KILLED = 0.05
|
18
|
-
PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT",
|
19
|
+
PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
|
19
20
|
|
20
21
|
attr_reader :host, :port, :ws_url, :pid, :command,
|
21
22
|
:default_user_agent, :browser_version, :protocol_version,
|
22
|
-
:v8_version, :webkit_version
|
23
|
+
:v8_version, :webkit_version, :xvfb
|
23
24
|
|
24
25
|
|
25
26
|
extend Forwardable
|
@@ -63,11 +64,13 @@ module Ferrum
|
|
63
64
|
return
|
64
65
|
end
|
65
66
|
|
67
|
+
@pid = @xvfb = @user_data_dir = nil
|
66
68
|
@logger = options[:logger]
|
67
69
|
@process_timeout = options.fetch(:process_timeout, PROCESS_TIMEOUT)
|
68
70
|
|
69
|
-
tmpdir = Dir.mktmpdir
|
71
|
+
tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
|
70
72
|
ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
|
73
|
+
@user_data_dir = tmpdir
|
71
74
|
@command = Command.build(options, tmpdir)
|
72
75
|
end
|
73
76
|
|
@@ -81,7 +84,12 @@ module Ferrum
|
|
81
84
|
process_options[:pgroup] = true unless Ferrum.windows?
|
82
85
|
process_options[:out] = process_options[:err] = write_io
|
83
86
|
|
84
|
-
@
|
87
|
+
if @command.xvfb?
|
88
|
+
@xvfb = Xvfb.start(@command.options)
|
89
|
+
ObjectSpace.define_finalizer(self, self.class.process_killer(@xvfb.pid))
|
90
|
+
end
|
91
|
+
|
92
|
+
@pid = ::Process.spawn(Hash(@xvfb&.to_env), *@command.to_a, process_options)
|
85
93
|
ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
|
86
94
|
|
87
95
|
parse_ws_url(read_io, @process_timeout)
|
@@ -92,7 +100,12 @@ module Ferrum
|
|
92
100
|
end
|
93
101
|
|
94
102
|
def stop
|
95
|
-
|
103
|
+
if @pid
|
104
|
+
kill(@pid)
|
105
|
+
kill(@xvfb.pid) if @xvfb&.pid
|
106
|
+
@pid = nil
|
107
|
+
end
|
108
|
+
|
96
109
|
remove_user_data_dir if @user_data_dir
|
97
110
|
ObjectSpace.undefine_finalizer(self)
|
98
111
|
end
|
@@ -104,9 +117,8 @@ module Ferrum
|
|
104
117
|
|
105
118
|
private
|
106
119
|
|
107
|
-
def kill
|
108
|
-
self.class.process_killer(
|
109
|
-
@pid = nil
|
120
|
+
def kill(pid)
|
121
|
+
self.class.process_killer(pid).call
|
110
122
|
end
|
111
123
|
|
112
124
|
def remove_user_data_dir
|
@@ -134,7 +146,7 @@ module Ferrum
|
|
134
146
|
|
135
147
|
unless ws_url
|
136
148
|
@logger.puts(output) if @logger
|
137
|
-
raise ProcessTimeoutError.new(timeout)
|
149
|
+
raise ProcessTimeoutError.new(timeout, output)
|
138
150
|
end
|
139
151
|
end
|
140
152
|
|