ferrum 0.8 → 0.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/ferrum.rb CHANGED
@@ -10,14 +10,20 @@ module Ferrum
10
10
  class NotImplementedError < Error; end
11
11
 
12
12
  class StatusError < Error
13
+ def initialize(url, message = nil)
14
+ super(message || "Request to #{url} failed to reach server, check DNS and server status")
15
+ end
16
+ end
17
+
18
+ class PendingConnectionsError < StatusError
19
+ attr_reader :pendings
20
+
13
21
  def initialize(url, pendings = [])
14
- message = if pendings.empty?
15
- "Request to #{url} failed to reach server, check DNS and/or server status"
16
- else
17
- "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
18
- end
22
+ @pendings = pendings
19
23
 
20
- super(message)
24
+ message = "Request to #{url} reached server, but there are still pending connections: #{pendings.join(', ')}"
25
+
26
+ super(url, message)
21
27
  end
22
28
  end
23
29
 
@@ -37,8 +43,11 @@ module Ferrum
37
43
  end
38
44
 
39
45
  class ProcessTimeoutError < Error
40
- def initialize(timeout)
41
- super("Browser did not produce websocket url within #{timeout} seconds")
46
+ attr_reader :output
47
+
48
+ def initialize(timeout, output)
49
+ @output = output
50
+ super("Browser did not produce websocket url within #{timeout} seconds, try to increase `:process_timeout`. See https://github.com/rubycdp/ferrum#customization")
42
51
  end
43
52
  end
44
53
 
@@ -48,6 +57,25 @@ module Ferrum
48
57
  end
49
58
  end
50
59
 
60
+ class NodeMovingError < Error
61
+ def initialize(node, prev, current)
62
+ @node, @prev, @current = node, prev, current
63
+ super(message)
64
+ end
65
+
66
+ def message
67
+ "#{@node.inspect} that you're trying to click is moving, hence " \
68
+ "we cannot. Previosuly it was at #{@prev.inspect} but now at " \
69
+ "#{@current.inspect}."
70
+ end
71
+ end
72
+
73
+ class CoordinatesNotFoundError < Error
74
+ def initialize(message = "Could not compute content quads")
75
+ super
76
+ end
77
+ end
78
+
51
79
  class BrowserError < Error
52
80
  attr_reader :response
53
81
 
@@ -4,6 +4,7 @@ require "base64"
4
4
  require "forwardable"
5
5
  require "ferrum/page"
6
6
  require "ferrum/contexts"
7
+ require "ferrum/browser/xvfb"
7
8
  require "ferrum/browser/process"
8
9
  require "ferrum/browser/client"
9
10
 
@@ -16,19 +17,20 @@ module Ferrum
16
17
  extend Forwardable
17
18
  delegate %i[default_context] => :contexts
18
19
  delegate %i[targets create_target create_page page pages windows] => :default_context
19
- delegate %i[goto back forward refresh reload stop
20
+ delegate %i[go_to back forward refresh reload stop wait_for_reload
20
21
  at_css at_xpath css xpath current_url current_title url title
21
22
  body doctype set_content
22
23
  headers cookies network
23
24
  mouse keyboard
24
- screenshot pdf viewport_size
25
+ screenshot pdf mhtml viewport_size
25
26
  frames frame_by main_frame
26
- evaluate evaluate_on evaluate_async execute
27
+ evaluate evaluate_on evaluate_async execute evaluate_func
27
28
  add_script_tag add_style_tag bypass_csp
28
- on] => :page
29
+ on goto position position=
30
+ playback_rate playback_rate=] => :page
29
31
  delegate %i[default_user_agent] => :process
30
32
 
31
- attr_reader :client, :process, :contexts, :logger, :js_errors,
33
+ attr_reader :client, :process, :contexts, :logger, :js_errors, :pending_connection_errors,
32
34
  :slowmo, :base_url, :options, :window_size, :ws_max_receive_size
33
35
  attr_writer :timeout
34
36
 
@@ -43,6 +45,7 @@ module Ferrum
43
45
  @logger, @timeout, @ws_max_receive_size =
44
46
  @options.values_at(:logger, :timeout, :ws_max_receive_size)
45
47
  @js_errors = @options.fetch(:js_errors, false)
48
+ @pending_connection_errors = @options.fetch(:pending_connection_errors, true)
46
49
  @slowmo = @options[:slowmo].to_f
47
50
 
48
51
  if @options.key?(:base_url)
@@ -75,6 +78,10 @@ module Ferrum
75
78
  end
76
79
  end
77
80
 
81
+ def evaluate_on_new_document(expression)
82
+ extensions << expression
83
+ end
84
+
78
85
  def timeout
79
86
  @timeout || DEFAULT_TIMEOUT
80
87
  end
@@ -58,6 +58,10 @@ module Ferrum
58
58
  end
59
59
  end
60
60
 
61
+ def subscribed?(event)
62
+ [@interruptor, @subscriber].any? { |s| s.subscribed?(event) }
63
+ end
64
+
61
65
  def close
62
66
  @ws.close
63
67
  # Give a thread some time to handle a tail of messages
@@ -86,6 +90,8 @@ module Ferrum
86
90
  raise NoExecutionContextError.new(error)
87
91
  when "No target with given id found"
88
92
  raise NoSuchPageError
93
+ when /Could not compute content quads/
94
+ raise CoordinatesNotFoundError
89
95
  else
90
96
  raise BrowserError.new(error)
91
97
  end
@@ -3,53 +3,54 @@
3
3
  module Ferrum
4
4
  class Browser
5
5
  class Command
6
- BROWSER_HOST = "127.0.0.1"
7
- BROWSER_PORT = "0"
8
6
  NOT_FOUND = "Could not find an executable for the browser. Try to make " \
9
- "it available on the PATH or set environment varible for " \
7
+ "it available on the PATH or set environment variable for " \
10
8
  "example BROWSER_PATH=\"/usr/bin/chrome\"".freeze
11
9
 
12
10
  # Currently only these browsers support CDP:
13
11
  # https://github.com/cyrus-and/chrome-remote-interface#implementations
14
12
  def self.build(options, user_data_dir)
15
- case options[:browser_name]
16
- when :firefox
17
- Firefox
18
- when :chrome, :opera, :edge, nil
19
- Chrome
20
- else
21
- raise NotImplementedError, "not supported browser"
22
- end.new(options, user_data_dir)
13
+ defaults = case options[:browser_name]
14
+ when :firefox
15
+ Options::Firefox.options
16
+ when :chrome, :opera, :edge, nil
17
+ Options::Chrome.options
18
+ else
19
+ raise NotImplementedError, "not supported browser"
20
+ end
21
+
22
+ new(defaults, options, user_data_dir)
23
23
  end
24
24
 
25
- attr_reader :path, :flags, :options
25
+ attr_reader :defaults, :path, :options
26
26
 
27
- def initialize(options, user_data_dir)
27
+ def initialize(defaults, options, user_data_dir)
28
28
  @flags = {}
29
+ @defaults = defaults
29
30
  @options, @user_data_dir = options, user_data_dir
30
- @path = options[:browser_path] || ENV["BROWSER_PATH"] || detect_path
31
+ @path = options[:browser_path] || ENV["BROWSER_PATH"] || defaults.detect_path
31
32
  raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
33
+ merge_options
34
+ end
32
35
 
33
- combine_flags
36
+ def xvfb?
37
+ !!options[:xvfb]
34
38
  end
35
39
 
36
40
  def to_a
37
- [path] + flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
41
+ [path] + @flags.map { |k, v| v.nil? ? "--#{k}" : "--#{k}=#{v}" }
38
42
  end
39
43
 
40
44
  private
41
45
 
42
- def detect_path
43
- if Ferrum.mac?
44
- self.class::MAC_BIN_PATH.find { |b| File.exist?(b) }
45
- else
46
- self.class::LINUX_BIN_PATH
47
- .find { |b| p = Cliver.detect(b) and break(p) }
46
+ def merge_options
47
+ @flags = defaults.merge_required(@flags, options, @user_data_dir)
48
+
49
+ unless options[:ignore_default_browser_options]
50
+ @flags = defaults.merge_default(@flags, options)
48
51
  end
49
- end
50
52
 
51
- def combine_flags
52
- raise NotImplementedError
53
+ @flags.merge!(options.fetch(:browser_options, {}))
53
54
  end
54
55
  end
55
56
  end
@@ -0,0 +1,46 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "singleton"
4
+
5
+ module Ferrum
6
+ class Browser
7
+ module Options
8
+ class Base
9
+ BROWSER_HOST = "127.0.0.1"
10
+ BROWSER_PORT = "0"
11
+
12
+ include Singleton
13
+
14
+ def self.options
15
+ instance
16
+ end
17
+
18
+ def to_h
19
+ self.class::DEFAULT_OPTIONS
20
+ end
21
+
22
+ def except(*keys)
23
+ to_h.reject { |n, _| keys.include?(n) }
24
+ end
25
+
26
+ def detect_path
27
+ if Ferrum.mac?
28
+ self.class::MAC_BIN_PATH.find { |n| File.exist?(n) }
29
+ else
30
+ self.class::LINUX_BIN_PATH.find do |name|
31
+ path = Cliver.detect(name) and break(path)
32
+ end
33
+ end
34
+ end
35
+
36
+ def merge_required(flags, options, user_data_dir)
37
+ raise NotImplementedError
38
+ end
39
+
40
+ def merge_default(flags, options)
41
+ raise NotImplementedError
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ferrum
4
+ class Browser
5
+ module Options
6
+ class Chrome < Base
7
+ DEFAULT_OPTIONS = {
8
+ "headless" => nil,
9
+ "disable-gpu" => nil,
10
+ "hide-scrollbars" => nil,
11
+ "mute-audio" => nil,
12
+ "enable-automation" => nil,
13
+ "disable-web-security" => nil,
14
+ "disable-session-crashed-bubble" => nil,
15
+ "disable-breakpad" => nil,
16
+ "disable-sync" => nil,
17
+ "no-first-run" => nil,
18
+ "use-mock-keychain" => nil,
19
+ "keep-alive-for-test" => nil,
20
+ "disable-popup-blocking" => nil,
21
+ "disable-extensions" => nil,
22
+ "disable-hang-monitor" => nil,
23
+ "disable-features" => "site-per-process,TranslateUI",
24
+ "disable-translate" => nil,
25
+ "disable-background-networking" => nil,
26
+ "enable-features" => "NetworkService,NetworkServiceInProcess",
27
+ "disable-background-timer-throttling" => nil,
28
+ "disable-backgrounding-occluded-windows" => nil,
29
+ "disable-client-side-phishing-detection" => nil,
30
+ "disable-default-apps" => nil,
31
+ "disable-dev-shm-usage" => nil,
32
+ "disable-ipc-flooding-protection" => nil,
33
+ "disable-prompt-on-repost" => nil,
34
+ "disable-renderer-backgrounding" => nil,
35
+ "force-color-profile" => "srgb",
36
+ "metrics-recording-only" => nil,
37
+ "safebrowsing-disable-auto-update" => nil,
38
+ "password-store" => "basic",
39
+ "no-startup-window" => nil,
40
+ # Note: --no-sandbox is not needed if you properly setup a user in the container.
41
+ # https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
42
+ # "no-sandbox" => nil,
43
+ }.freeze
44
+
45
+ MAC_BIN_PATH = [
46
+ "/Applications/Chromium.app/Contents/MacOS/Chromium",
47
+ "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
48
+ ].freeze
49
+ LINUX_BIN_PATH = %w[chromium google-chrome-unstable google-chrome-beta
50
+ google-chrome chrome chromium-browser
51
+ google-chrome-stable].freeze
52
+
53
+ def merge_required(flags, options, user_data_dir)
54
+ port = options.fetch(:port, BROWSER_PORT)
55
+ host = options.fetch(:host, BROWSER_HOST)
56
+ flags.merge("remote-debugging-port" => port,
57
+ "remote-debugging-address" => host,
58
+ # Doesn't work on MacOS, so we need to set it by CDP
59
+ "window-size" => options[:window_size].join(","),
60
+ "user-data-dir" => user_data_dir)
61
+ end
62
+
63
+ def merge_default(flags, options)
64
+ unless options.fetch(:headless, true)
65
+ defaults = except("headless", "disable-gpu")
66
+ end
67
+
68
+ defaults ||= DEFAULT_OPTIONS
69
+ defaults.merge(flags)
70
+ end
71
+ end
72
+ end
73
+ end
74
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ferrum
4
+ class Browser
5
+ module Options
6
+ class Firefox < Base
7
+ DEFAULT_OPTIONS = {
8
+ "headless" => nil,
9
+ }.freeze
10
+
11
+ MAC_BIN_PATH = [
12
+ "/Applications/Firefox.app/Contents/MacOS/firefox-bin"
13
+ ].freeze
14
+ LINUX_BIN_PATH = %w[firefox].freeze
15
+
16
+ def merge_required(flags, options, user_data_dir)
17
+ port = options.fetch(:port, BROWSER_PORT)
18
+ host = options.fetch(:host, BROWSER_HOST)
19
+ flags.merge("remote-debugger" => "#{host}:#{port}",
20
+ "profile" => user_data_dir)
21
+ end
22
+
23
+ def merge_default(flags, options)
24
+ unless options.fetch(:headless, true)
25
+ defaults = except("headless")
26
+ end
27
+
28
+ defaults ||= DEFAULT_OPTIONS
29
+ defaults.merge(flags)
30
+ end
31
+ end
32
+ end
33
+ end
34
+ end
@@ -6,20 +6,21 @@ require "json"
6
6
  require "addressable"
7
7
  require "tmpdir"
8
8
  require "forwardable"
9
+ require "ferrum/browser/options/base"
10
+ require "ferrum/browser/options/chrome"
11
+ require "ferrum/browser/options/firefox"
9
12
  require "ferrum/browser/command"
10
- require "ferrum/browser/chrome"
11
- require "ferrum/browser/firefox"
12
13
 
13
14
  module Ferrum
14
15
  class Browser
15
16
  class Process
16
17
  KILL_TIMEOUT = 2
17
18
  WAIT_KILLED = 0.05
18
- PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 2).to_i
19
+ PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
19
20
 
20
21
  attr_reader :host, :port, :ws_url, :pid, :command,
21
22
  :default_user_agent, :browser_version, :protocol_version,
22
- :v8_version, :webkit_version
23
+ :v8_version, :webkit_version, :xvfb
23
24
 
24
25
 
25
26
  extend Forwardable
@@ -63,11 +64,13 @@ module Ferrum
63
64
  return
64
65
  end
65
66
 
67
+ @pid = @xvfb = @user_data_dir = nil
66
68
  @logger = options[:logger]
67
69
  @process_timeout = options.fetch(:process_timeout, PROCESS_TIMEOUT)
68
70
 
69
- tmpdir = Dir.mktmpdir
71
+ tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
70
72
  ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
73
+ @user_data_dir = tmpdir
71
74
  @command = Command.build(options, tmpdir)
72
75
  end
73
76
 
@@ -81,7 +84,12 @@ module Ferrum
81
84
  process_options[:pgroup] = true unless Ferrum.windows?
82
85
  process_options[:out] = process_options[:err] = write_io
83
86
 
84
- @pid = ::Process.spawn(*@command.to_a, process_options)
87
+ if @command.xvfb?
88
+ @xvfb = Xvfb.start(@command.options)
89
+ ObjectSpace.define_finalizer(self, self.class.process_killer(@xvfb.pid))
90
+ end
91
+
92
+ @pid = ::Process.spawn(Hash(@xvfb&.to_env), *@command.to_a, process_options)
85
93
  ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
86
94
 
87
95
  parse_ws_url(read_io, @process_timeout)
@@ -92,7 +100,12 @@ module Ferrum
92
100
  end
93
101
 
94
102
  def stop
95
- kill if @pid
103
+ if @pid
104
+ kill(@pid)
105
+ kill(@xvfb.pid) if @xvfb&.pid
106
+ @pid = nil
107
+ end
108
+
96
109
  remove_user_data_dir if @user_data_dir
97
110
  ObjectSpace.undefine_finalizer(self)
98
111
  end
@@ -104,9 +117,8 @@ module Ferrum
104
117
 
105
118
  private
106
119
 
107
- def kill
108
- self.class.process_killer(@pid).call
109
- @pid = nil
120
+ def kill(pid)
121
+ self.class.process_killer(pid).call
110
122
  end
111
123
 
112
124
  def remove_user_data_dir
@@ -134,7 +146,7 @@ module Ferrum
134
146
 
135
147
  unless ws_url
136
148
  @logger.puts(output) if @logger
137
- raise ProcessTimeoutError.new(timeout)
149
+ raise ProcessTimeoutError.new(timeout, output)
138
150
  end
139
151
  end
140
152