ferrum 0.19.2 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +60 -0
- data/lib/ferrum/browser/binary.rb +1 -2
- data/lib/ferrum/browser/options.rb +33 -12
- data/lib/ferrum/browser/process/output.rb +143 -0
- data/lib/ferrum/browser/process.rb +42 -69
- data/lib/ferrum/browser.rb +9 -23
- data/lib/ferrum/client/web_socket.rb +4 -15
- data/lib/ferrum/client.rb +1 -1
- data/lib/ferrum/console/location.rb +42 -0
- data/lib/ferrum/console/message.rb +230 -0
- data/lib/ferrum/console.rb +157 -0
- data/lib/ferrum/context.rb +23 -15
- data/lib/ferrum/contexts.rb +41 -18
- data/lib/ferrum/errors.rb +7 -5
- data/lib/ferrum/frame/dom.rb +50 -44
- data/lib/ferrum/frame/runtime/legacy.rb +336 -0
- data/lib/ferrum/frame/runtime.rb +416 -186
- data/lib/ferrum/frame/script.rb +289 -0
- data/lib/ferrum/frame.rb +33 -4
- data/lib/ferrum/interceptable.rb +19 -9
- data/lib/ferrum/isolated_world.rb +94 -0
- data/lib/ferrum/keyboard.rb +95 -8
- data/lib/ferrum/log/annotated.rb +77 -0
- data/lib/ferrum/log/default.rb +55 -0
- data/lib/ferrum/log/json.rb +32 -0
- data/lib/ferrum/log/paint.rb +28 -0
- data/lib/ferrum/log.rb +247 -0
- data/lib/ferrum/mouse.rb +16 -8
- data/lib/ferrum/node.rb +127 -52
- data/lib/ferrum/page/frames.rb +10 -3
- data/lib/ferrum/page/screenshot.rb +8 -7
- data/lib/ferrum/page.rb +106 -53
- data/lib/ferrum/proxy.rb +8 -7
- data/lib/ferrum/remote_object.rb +75 -0
- data/lib/ferrum/target.rb +7 -0
- data/lib/ferrum/utils/deprecate.rb +77 -0
- data/lib/ferrum/version.rb +1 -1
- data/lib/ferrum/worker.rb +8 -0
- data/lib/ferrum.rb +3 -0
- metadata +16 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 18ff9dece365985b7cb1033d7ec3c9913602d3da49d60ceaef432d1c569c1b86
|
|
4
|
+
data.tar.gz: 18b3d49c63a924a77526eff46cce0f2dea4696580ca19cc25fb9a640f9e4bf62
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: dd8a1c24c02303a636c557bc016fd4cb0fb4cd16cf4bfe0e3c3bb7f00f92adda3b1f4568b7692002e1b1e4b55a0e8d2e1772d840183ec9d0eecbad710470bce8
|
|
7
|
+
data.tar.gz: 164a77db028af9018846f9f3f5d1451755cf0d6ed495ea2c0424235b150fc388985c6b57742b9dbfbf88dd42c70823c24a18e8ca137906556fa4dda3d9beb8fe
|
data/README.md
CHANGED
|
@@ -35,6 +35,17 @@ Add this to your `Gemfile` and run `bundle install`.
|
|
|
35
35
|
gem "ferrum"
|
|
36
36
|
```
|
|
37
37
|
|
|
38
|
+
Or skip installing Chrome and run it in a container with our images, see
|
|
39
|
+
[Docker](https://docs.rubycdp.com/docs/ferrum/docker#our-images-recommended):
|
|
40
|
+
|
|
41
|
+
```shell
|
|
42
|
+
docker run --rm -p 3000:3000 ghcr.io/rubycdp/chromium
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
```ruby
|
|
46
|
+
browser = Ferrum::Browser.new(url: "http://localhost:3000")
|
|
47
|
+
```
|
|
48
|
+
|
|
38
49
|
## Quick Start
|
|
39
50
|
|
|
40
51
|
Navigate to a website and save a screenshot:
|
|
@@ -74,6 +85,55 @@ JS
|
|
|
74
85
|
browser.quit
|
|
75
86
|
```
|
|
76
87
|
|
|
88
|
+
Pass arguments by name, so the script can use parameters instead of digging
|
|
89
|
+
through `arguments[0]`:
|
|
90
|
+
|
|
91
|
+
```ruby
|
|
92
|
+
page.evaluate("a + b", a: 1, b: 2) # => 3
|
|
93
|
+
page.evaluate("el.getAttribute(name)", el: page.at_css("a"), name: "href")
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
A script that is a function or arrow declaration is used as-is and takes its
|
|
97
|
+
arguments positionally, in the order it declares them:
|
|
98
|
+
|
|
99
|
+
```ruby
|
|
100
|
+
page.evaluate(<<~JS, 1, 2)
|
|
101
|
+
function(a, b) {
|
|
102
|
+
const sum = a + b;
|
|
103
|
+
return sum * 3;
|
|
104
|
+
}
|
|
105
|
+
JS
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
`await` works directly, and a function declaration's returned promise is
|
|
109
|
+
awaited:
|
|
110
|
+
|
|
111
|
+
```ruby
|
|
112
|
+
page.evaluate("await fetch(url).then(r => r.text())", url: "/api")
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Pass `timeout:` (seconds, defaulting to the page timeout) to bound how long the
|
|
116
|
+
script may take before `Ferrum::ScriptTimeoutError` is raised. `timeout:` is the
|
|
117
|
+
only reserved keyword.
|
|
118
|
+
|
|
119
|
+
Use `#execute` when you only want the side effects, `#evaluate_handle` when you
|
|
120
|
+
want to keep a value in the browser and pass it back in later, and
|
|
121
|
+
`Ferrum::Node#evaluate` to run a script with `this` bound to an element:
|
|
122
|
+
|
|
123
|
+
```ruby
|
|
124
|
+
page.execute("window.scrollBy(0, 100)") # => true
|
|
125
|
+
|
|
126
|
+
list = page.evaluate_handle("document.querySelectorAll('li')")
|
|
127
|
+
page.evaluate("Array.from(nodes).map(n => n.textContent)", nodes: list)
|
|
128
|
+
|
|
129
|
+
page.at_css("input").evaluate("this.value")
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Scripts written for the old API keep working, and Ferrum warns, with a suggested
|
|
133
|
+
rewrite, about any whose result will change. The next release, 0.21, removes the
|
|
134
|
+
old behavior, so run your suite with `FERRUM_JS_API=new` to check it's ready. See
|
|
135
|
+
[upgrading to 0.21](docs/13-javascript.md#upgrading-to-021).
|
|
136
|
+
|
|
77
137
|
Do any mouse movements you like:
|
|
78
138
|
|
|
79
139
|
```ruby
|
|
@@ -68,7 +68,6 @@ module Ferrum
|
|
|
68
68
|
[paths, exts]
|
|
69
69
|
end
|
|
70
70
|
|
|
71
|
-
# rubocop:disable-next Style/CollectionCompact
|
|
72
71
|
def lazy_find(cmds)
|
|
73
72
|
cmds.lazy.map do |cmd, path, ext|
|
|
74
73
|
absolute_path = File.absolute_path(cmd)
|
|
@@ -79,7 +78,7 @@ module Ferrum
|
|
|
79
78
|
next if File.directory?(cmd)
|
|
80
79
|
|
|
81
80
|
cmd
|
|
82
|
-
end.
|
|
81
|
+
end.compact
|
|
83
82
|
end
|
|
84
83
|
end
|
|
85
84
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "ferrum/log"
|
|
4
|
+
|
|
3
5
|
module Ferrum
|
|
4
6
|
class Browser
|
|
5
7
|
#
|
|
@@ -16,13 +18,14 @@ module Ferrum
|
|
|
16
18
|
DEFAULT_TIMEOUT = ENV.fetch("FERRUM_DEFAULT_TIMEOUT", 5).to_i
|
|
17
19
|
DEFAULT_PROTOCOL_TIMEOUT = ENV.fetch("FERRUM_PROTOCOL_TIMEOUT", DEFAULT_TIMEOUT).to_i
|
|
18
20
|
PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
|
|
19
|
-
DEBUG_MODE = !ENV.fetch("FERRUM_DEBUG", nil).nil?
|
|
20
21
|
|
|
21
|
-
|
|
22
|
+
warn "Ferrum: FERRUM_DEBUG is deprecated, use FERRUM_LOG=* instead" if ENV.key?("FERRUM_DEBUG")
|
|
23
|
+
|
|
24
|
+
attr_reader :window_size, :logger, :log, :ws_max_receive_size,
|
|
22
25
|
:js_errors, :base_url, :slowmo, :pending_connection_errors,
|
|
23
26
|
:url, :ws_url, :env, :process_timeout, :browser_name, :browser_path,
|
|
24
27
|
:save_path, :proxy, :port, :host, :headless, :incognito, :dockerize, :browser_options,
|
|
25
|
-
:ignore_default_browser_options, :xvfb, :flatten
|
|
28
|
+
:ignore_default_browser_options, :xvfb, :flatten, :extensions
|
|
26
29
|
attr_accessor :timeout, :protocol_timeout, :default_user_agent
|
|
27
30
|
|
|
28
31
|
def initialize(options = nil)
|
|
@@ -51,10 +54,11 @@ module Ferrum
|
|
|
51
54
|
@ignore_default_browser_options = @options[:ignore_default_browser_options]
|
|
52
55
|
|
|
53
56
|
@proxy = validate_proxy(@options[:proxy])
|
|
54
|
-
@logger = parse_logger(@options[:logger])
|
|
57
|
+
@logger, @log = parse_logger(@options[:logger])
|
|
55
58
|
@base_url = parse_base_url(@options[:base_url]) if @options[:base_url]
|
|
56
59
|
@url = @options[:url].to_s if @options[:url]
|
|
57
60
|
@ws_url = @options[:ws_url].to_s if @options[:ws_url]
|
|
61
|
+
@extensions = Array(@options[:extensions]).map { |e| read_extension(e) }.freeze
|
|
58
62
|
|
|
59
63
|
@options = @options.merge(window_size: @window_size).freeze
|
|
60
64
|
@browser_options = @options.fetch(:browser_options, {}).freeze
|
|
@@ -73,15 +77,24 @@ module Ferrum
|
|
|
73
77
|
end
|
|
74
78
|
|
|
75
79
|
#
|
|
76
|
-
#
|
|
80
|
+
# Adds an extension to run in every new document, before the document's own scripts. Applies to pages and
|
|
81
|
+
# iframes attached after the call: pages already open keep the extensions they were attached with.
|
|
82
|
+
#
|
|
83
|
+
# @param [String, Hash] extension
|
|
84
|
+
# A path to a JS file, or `{ source: "..." }`.
|
|
77
85
|
#
|
|
78
86
|
# @return [Array<String>]
|
|
79
|
-
# JS source code
|
|
87
|
+
# JS source code of every extension.
|
|
80
88
|
#
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
89
|
+
# @raise [Errno::ENOENT]
|
|
90
|
+
# If the file doesn't exist.
|
|
91
|
+
#
|
|
92
|
+
# @example
|
|
93
|
+
# browser.add_extension("/path/to/stub.js")
|
|
94
|
+
# browser.add_extension(source: "window.secret = 'top'")
|
|
95
|
+
#
|
|
96
|
+
def add_extension(extension)
|
|
97
|
+
@extensions = [*@extensions, read_extension(extension)].freeze
|
|
85
98
|
end
|
|
86
99
|
|
|
87
100
|
#
|
|
@@ -119,10 +132,18 @@ module Ferrum
|
|
|
119
132
|
|
|
120
133
|
private
|
|
121
134
|
|
|
135
|
+
def read_extension(extension)
|
|
136
|
+
(extension.is_a?(Hash) && extension[:source]) || File.read(extension)
|
|
137
|
+
end
|
|
138
|
+
|
|
122
139
|
def parse_logger(logger)
|
|
123
|
-
|
|
140
|
+
value = ENV.fetch("FERRUM_LOG") { "*" if ENV.key?("FERRUM_DEBUG") }
|
|
141
|
+
settings = value ? Log.parse(value) : {}
|
|
142
|
+
logger ||= $stdout.tap { |s| s.sync = true } if value && settings
|
|
143
|
+
return [logger, nil] unless logger && settings
|
|
124
144
|
|
|
125
|
-
|
|
145
|
+
[logger, Log.new(logger, **settings, format: Log.parse_format(ENV.fetch("FERRUM_LOG_FORMAT", nil)),
|
|
146
|
+
color: Log.parse_color(ENV.fetch("FERRUM_LOG_COLOR", nil)))]
|
|
126
147
|
end
|
|
127
148
|
|
|
128
149
|
def parse_base_url(value)
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ferrum
|
|
4
|
+
class Browser
|
|
5
|
+
class Process
|
|
6
|
+
#
|
|
7
|
+
# What the browser process wrote to stdout and stderr: keeps the last
|
|
8
|
+
# {LIMIT} lines and writes each one to the `:browser` log channel. Owns
|
|
9
|
+
# the pipe the process writes to, which is read on its own thread for as
|
|
10
|
+
# long as the process lives, so it never fills up and blocks the browser.
|
|
11
|
+
#
|
|
12
|
+
class Output
|
|
13
|
+
LIMIT = 100
|
|
14
|
+
DRAIN_TIMEOUT = 0.5
|
|
15
|
+
|
|
16
|
+
def initialize(log = nil)
|
|
17
|
+
@log = log
|
|
18
|
+
@lines = []
|
|
19
|
+
@partial = String.new(encoding: Encoding::BINARY)
|
|
20
|
+
@mutex = Mutex.new
|
|
21
|
+
@io = @reader = nil
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
#
|
|
25
|
+
# Opens the pipe and yields its write end, to be given to the process
|
|
26
|
+
# as stdout and stderr. The write end is closed here afterwards, so
|
|
27
|
+
# reading hits the end of the pipe once the process exits.
|
|
28
|
+
#
|
|
29
|
+
# @yieldparam [IO] io
|
|
30
|
+
#
|
|
31
|
+
# @return [void]
|
|
32
|
+
#
|
|
33
|
+
def pipe
|
|
34
|
+
@io, writer = IO.pipe
|
|
35
|
+
yield writer
|
|
36
|
+
ensure
|
|
37
|
+
close_io(writer)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
#
|
|
41
|
+
# Reads the pipe until the output matches, the process exits or the
|
|
42
|
+
# time is up.
|
|
43
|
+
#
|
|
44
|
+
# @param [Regexp] regexp
|
|
45
|
+
#
|
|
46
|
+
# @param [Numeric] timeout
|
|
47
|
+
#
|
|
48
|
+
# @return [MatchData, nil]
|
|
49
|
+
#
|
|
50
|
+
def wait_for(regexp, timeout)
|
|
51
|
+
deadline = Utils::ElapsedTime.monotonic_time + timeout
|
|
52
|
+
while (now = Utils::ElapsedTime.monotonic_time) < deadline
|
|
53
|
+
begin
|
|
54
|
+
self << @io.read_nonblock(512)
|
|
55
|
+
rescue IO::WaitReadable
|
|
56
|
+
@io.wait_readable(deadline - now)
|
|
57
|
+
rescue EOFError
|
|
58
|
+
return
|
|
59
|
+
else
|
|
60
|
+
match = to_s.match(regexp)
|
|
61
|
+
return match if match
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
#
|
|
67
|
+
# Keeps reading the pipe on a new thread until it's closed.
|
|
68
|
+
#
|
|
69
|
+
# @return [Thread]
|
|
70
|
+
#
|
|
71
|
+
def follow
|
|
72
|
+
@reader = Utils::Thread.spawn do
|
|
73
|
+
loop { self << @io.readpartial(4096) }
|
|
74
|
+
rescue IOError
|
|
75
|
+
self << "\n" unless @partial.empty?
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
#
|
|
80
|
+
# Gives the reading thread {DRAIN_TIMEOUT} seconds to take what the
|
|
81
|
+
# process wrote last, then closes the pipe. The lines are kept.
|
|
82
|
+
#
|
|
83
|
+
# @return [void]
|
|
84
|
+
#
|
|
85
|
+
def close
|
|
86
|
+
@reader&.join(DRAIN_TIMEOUT)
|
|
87
|
+
close_io(@io)
|
|
88
|
+
@io = @reader = nil
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
#
|
|
92
|
+
# Adds a chunk of output, which can end in the middle of a line.
|
|
93
|
+
#
|
|
94
|
+
# @param [String] chunk
|
|
95
|
+
#
|
|
96
|
+
# @return [self]
|
|
97
|
+
#
|
|
98
|
+
def <<(chunk)
|
|
99
|
+
lines = @mutex.synchronize do
|
|
100
|
+
*complete, @partial = (@partial + chunk.b).split("\n", -1)
|
|
101
|
+
complete.map! { |line| utf8(line) }
|
|
102
|
+
@lines.concat(complete)
|
|
103
|
+
@lines.shift(@lines.size - LIMIT) if @lines.size > LIMIT
|
|
104
|
+
complete
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
lines.each { |line| @log&.browser(line) }
|
|
108
|
+
self
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
#
|
|
112
|
+
# The last lines, oldest first.
|
|
113
|
+
#
|
|
114
|
+
# @return [Array<String>]
|
|
115
|
+
#
|
|
116
|
+
def lines
|
|
117
|
+
@mutex.synchronize { @partial.empty? ? @lines.dup : [*@lines, utf8(@partial)] }
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
#
|
|
121
|
+
# The last lines joined by newlines.
|
|
122
|
+
#
|
|
123
|
+
# @return [String]
|
|
124
|
+
#
|
|
125
|
+
def to_s
|
|
126
|
+
lines.join("\n")
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
private
|
|
130
|
+
|
|
131
|
+
def utf8(line)
|
|
132
|
+
line.dup.force_encoding(Encoding::UTF_8).scrub.delete_suffix("\r")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def close_io(io)
|
|
136
|
+
io.close if io && !io.closed?
|
|
137
|
+
rescue IOError
|
|
138
|
+
raise unless RUBY_ENGINE == "jruby"
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
@@ -8,6 +8,7 @@ require "forwardable"
|
|
|
8
8
|
require "ferrum/browser/options/base"
|
|
9
9
|
require "ferrum/browser/options/chrome"
|
|
10
10
|
require "ferrum/browser/process/killer"
|
|
11
|
+
require "ferrum/browser/process/output"
|
|
11
12
|
require "ferrum/browser/command"
|
|
12
13
|
require "ferrum/utils/elapsed_time"
|
|
13
14
|
require "ferrum/utils/platform"
|
|
@@ -27,6 +28,7 @@ module Ferrum
|
|
|
27
28
|
HTTP_SCHEMES = { "ws" => "http", "wss" => "https" }.freeze
|
|
28
29
|
WS_SCHEMES = { "http" => "ws", "https" => "wss" }.freeze
|
|
29
30
|
LOCAL_HOSTS = %w[0.0.0.0 :: 127.0.0.1 ::1 localhost].freeze
|
|
31
|
+
WS_URL = %r{DevTools listening on (ws://.*[a-zA-Z0-9-]{36})}
|
|
30
32
|
|
|
31
33
|
delegate path: :command
|
|
32
34
|
|
|
@@ -42,27 +44,25 @@ module Ferrum
|
|
|
42
44
|
new(*args).tap(&:start)
|
|
43
45
|
end
|
|
44
46
|
|
|
45
|
-
attr_reader :host, :port, :ws_url, :pid, :command,
|
|
46
|
-
|
|
47
|
-
|
|
47
|
+
attr_reader :host, :port, :ws_url, :pid, :command, :xvfb
|
|
48
|
+
|
|
49
|
+
# What the browser process wrote to stdout and stderr, empty for a
|
|
50
|
+
# browser Ferrum didn't launch.
|
|
51
|
+
#
|
|
52
|
+
# @return [Output]
|
|
53
|
+
attr_reader :output
|
|
48
54
|
|
|
49
55
|
def initialize(options)
|
|
50
56
|
@pid = @xvfb = @user_data_dir = nil
|
|
51
57
|
@requested_host = options.host
|
|
58
|
+
@log = options.log
|
|
59
|
+
@output = Output.new(@log)
|
|
52
60
|
|
|
53
61
|
if options.ws_url || options.url
|
|
54
|
-
|
|
55
|
-
# and use it as a ws_url, otherwise use what has been parsed.
|
|
56
|
-
endpoint = options.ws_url || options.url
|
|
57
|
-
response = parse_json_version(endpoint)
|
|
58
|
-
ws_url = options.ws_url || reachable_ws_url(response&.[]("webSocketDebuggerUrl"), endpoint)
|
|
59
|
-
raise NoWebSocketUrlError, endpoint unless ws_url
|
|
60
|
-
|
|
61
|
-
self.ws_url = ws_url
|
|
62
|
+
self.ws_url = options.ws_url || fetch_ws_url(options.url)
|
|
62
63
|
return
|
|
63
64
|
end
|
|
64
65
|
|
|
65
|
-
@logger = options.logger
|
|
66
66
|
@process_timeout = options.process_timeout
|
|
67
67
|
@env = Hash(options.env)
|
|
68
68
|
|
|
@@ -80,21 +80,13 @@ module Ferrum
|
|
|
80
80
|
return if ws_url
|
|
81
81
|
|
|
82
82
|
begin
|
|
83
|
-
read_io, write_io = IO.pipe
|
|
84
|
-
process_options = { in: File::NULL }
|
|
85
|
-
process_options[:pgroup] = true unless Utils::Platform.windows?
|
|
86
|
-
process_options[:out] = process_options[:err] = write_io
|
|
87
|
-
|
|
88
83
|
@xvfb = Xvfb.start(@command.options) if @command.xvfb?
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
parse_ws_url(read_io, @process_timeout)
|
|
95
|
-
parse_json_version(ws_url)
|
|
84
|
+
@output = Output.new(@log)
|
|
85
|
+
@output.pipe { |io| spawn_browser(io) }
|
|
86
|
+
parse_ws_url
|
|
87
|
+
@output.follow
|
|
96
88
|
ensure
|
|
97
|
-
|
|
89
|
+
@output.close unless ws_url
|
|
98
90
|
end
|
|
99
91
|
end
|
|
100
92
|
|
|
@@ -143,15 +135,20 @@ module Ferrum
|
|
|
143
135
|
"#<#{self.class} " \
|
|
144
136
|
"@user_data_dir=#{@user_data_dir.inspect} " \
|
|
145
137
|
"@command=#<#{@command.class}:#{@command.object_id}> " \
|
|
146
|
-
"@
|
|
147
|
-
"@ws_url=#{@ws_url.inspect} " \
|
|
148
|
-
"@v8_version=#{@v8_version.inspect} " \
|
|
149
|
-
"@browser_version=#{@browser_version.inspect} " \
|
|
150
|
-
"@webkit_version=#{@webkit_version.inspect}>"
|
|
138
|
+
"@ws_url=#{@ws_url.inspect}>"
|
|
151
139
|
end
|
|
152
140
|
|
|
153
141
|
private
|
|
154
142
|
|
|
143
|
+
def spawn_browser(io)
|
|
144
|
+
process_options = { in: File::NULL, out: io, err: io }
|
|
145
|
+
process_options[:pgroup] = true unless Utils::Platform.windows?
|
|
146
|
+
env = Hash(@xvfb&.to_env).merge(@env)
|
|
147
|
+
|
|
148
|
+
@pid = ::Process.spawn(env, *@command.to_a, process_options)
|
|
149
|
+
ObjectSpace.define_finalizer(self, Killer.finalizer([@pid, @xvfb&.pid], @user_data_dir))
|
|
150
|
+
end
|
|
151
|
+
|
|
155
152
|
def sync_stop
|
|
156
153
|
if @pid
|
|
157
154
|
Killer.kill(@pid)
|
|
@@ -159,6 +156,7 @@ module Ferrum
|
|
|
159
156
|
@pid = nil
|
|
160
157
|
end
|
|
161
158
|
|
|
159
|
+
@output.close
|
|
162
160
|
remove_user_data_dir if @user_data_dir
|
|
163
161
|
ObjectSpace.undefine_finalizer(self)
|
|
164
162
|
nil
|
|
@@ -174,11 +172,13 @@ module Ferrum
|
|
|
174
172
|
pid = @pid
|
|
175
173
|
xvfb_pid = @xvfb&.pid
|
|
176
174
|
user_data_dir = @user_data_dir
|
|
175
|
+
output = @output
|
|
177
176
|
@pid = @user_data_dir = nil
|
|
178
177
|
|
|
179
178
|
Utils::Thread.spawn do
|
|
180
179
|
Killer.kill(pid) if pid
|
|
181
180
|
Killer.kill(xvfb_pid) if xvfb_pid
|
|
181
|
+
output.close
|
|
182
182
|
Killer.remove_directory(user_data_dir) if user_data_dir
|
|
183
183
|
ObjectSpace.undefine_finalizer(self)
|
|
184
184
|
end
|
|
@@ -189,28 +189,11 @@ module Ferrum
|
|
|
189
189
|
@user_data_dir = nil
|
|
190
190
|
end
|
|
191
191
|
|
|
192
|
-
def parse_ws_url
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
max_time = start + timeout
|
|
196
|
-
regexp = %r{DevTools listening on (ws://.*[a-zA-Z0-9-]{36})}
|
|
197
|
-
while (now = Utils::ElapsedTime.monotonic_time) < max_time
|
|
198
|
-
begin
|
|
199
|
-
output += read_io.read_nonblock(512)
|
|
200
|
-
rescue IO::WaitReadable
|
|
201
|
-
read_io.wait_readable(max_time - now)
|
|
202
|
-
else
|
|
203
|
-
if output.match(regexp)
|
|
204
|
-
self.ws_url = rewrite_host(output.match(regexp)[1].strip)
|
|
205
|
-
break
|
|
206
|
-
end
|
|
207
|
-
end
|
|
208
|
-
end
|
|
209
|
-
|
|
210
|
-
return if ws_url
|
|
192
|
+
def parse_ws_url
|
|
193
|
+
match = @output.wait_for(WS_URL, @process_timeout)
|
|
194
|
+
raise ProcessTimeoutError.new(@process_timeout, @output.to_s) unless match
|
|
211
195
|
|
|
212
|
-
|
|
213
|
-
raise ProcessTimeoutError.new(timeout, output)
|
|
196
|
+
self.ws_url = rewrite_host(match[1].strip)
|
|
214
197
|
end
|
|
215
198
|
|
|
216
199
|
def ws_url=(url)
|
|
@@ -225,26 +208,16 @@ module Ferrum
|
|
|
225
208
|
uri.to_s
|
|
226
209
|
end
|
|
227
210
|
|
|
228
|
-
def
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
211
|
+
def fetch_ws_url(url)
|
|
212
|
+
response = ::Net::HTTP.get_response(URI(json_version_url(url).to_s))
|
|
213
|
+
status = "#{response.code} #{response.message}" unless response.is_a?(::Net::HTTPSuccess)
|
|
214
|
+
ws_url = begin
|
|
215
|
+
JSON.parse(response.body.to_s)["webSocketDebuggerUrl"]
|
|
216
|
+
rescue JSON::ParserError
|
|
217
|
+
nil
|
|
233
218
|
end
|
|
234
|
-
end
|
|
235
|
-
|
|
236
|
-
def parse_json_version(url)
|
|
237
|
-
response = JSON.parse(::Net::HTTP.get(URI(json_version_url(url).to_s)))
|
|
238
|
-
|
|
239
|
-
@v8_version = response["V8-Version"]
|
|
240
|
-
@browser_version = response["Browser"]
|
|
241
|
-
@webkit_version = response["WebKit-Version"]
|
|
242
|
-
@default_user_agent = response["User-Agent"]
|
|
243
|
-
@protocol_version = response["Protocol-Version"]
|
|
244
219
|
|
|
245
|
-
|
|
246
|
-
rescue JSON::ParserError
|
|
247
|
-
# nop
|
|
220
|
+
reachable_ws_url(ws_url, url) || raise(NoWebSocketUrlError.new(url, status))
|
|
248
221
|
end
|
|
249
222
|
|
|
250
223
|
def json_version_url(url)
|
data/lib/ferrum/browser.rb
CHANGED
|
@@ -29,15 +29,18 @@ module Ferrum
|
|
|
29
29
|
delegate %i[default_context] => :contexts
|
|
30
30
|
delegate %i[targets create_target page pages windows workers service_workers attach_target] => :default_context
|
|
31
31
|
delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
|
|
32
|
+
wait_for_navigation wait_for_navigation!
|
|
32
33
|
at_css at_xpath css xpath wait_for_selector current_url current_title url title
|
|
33
34
|
body doctype content=
|
|
34
|
-
headers cookies network accessibility downloads
|
|
35
|
+
headers cookies network console accessibility downloads
|
|
35
36
|
mouse keyboard
|
|
36
37
|
screenshot pdf mhtml viewport_size device_pixel_ratio
|
|
37
38
|
start_screencast stop_screencast
|
|
38
39
|
frames frame_by main_frame
|
|
39
|
-
evaluate
|
|
40
|
+
evaluate evaluate_handle execute
|
|
41
|
+
evaluate_on evaluate_async evaluate_func isolated_world
|
|
40
42
|
add_script_tag add_style_tag bypass_csp
|
|
43
|
+
evaluate_on_new_document remove_script_on_new_document
|
|
41
44
|
on position position=
|
|
42
45
|
playback_rate playback_rate=
|
|
43
46
|
disable_javascript set_viewport resize] => :page
|
|
@@ -45,7 +48,7 @@ module Ferrum
|
|
|
45
48
|
attr_reader :client, :process, :contexts, :options
|
|
46
49
|
|
|
47
50
|
delegate %i[timeout timeout= protocol_timeout protocol_timeout=
|
|
48
|
-
base_url base_url= default_user_agent default_user_agent= extensions] => :options
|
|
51
|
+
base_url base_url= default_user_agent default_user_agent= extensions add_extension] => :options
|
|
49
52
|
delegate %i[command] => :client
|
|
50
53
|
|
|
51
54
|
#
|
|
@@ -77,8 +80,8 @@ module Ferrum
|
|
|
77
80
|
# An array of paths to files or JS source code to be preloaded into the
|
|
78
81
|
# browser e.g.: `["/path/to/script.js", { source: "window.secret = 'top'" }]`
|
|
79
82
|
#
|
|
80
|
-
# @option options [#puts] :logger
|
|
81
|
-
# When present, debug output is written to this object.
|
|
83
|
+
# @option options [#puts, Logger] :logger
|
|
84
|
+
# When present, debug output is written to this object, see {Log}.
|
|
82
85
|
#
|
|
83
86
|
# @option options [Integer, Float] :slowmo
|
|
84
87
|
# Set a delay in seconds to wait before sending a command.
|
|
@@ -191,23 +194,6 @@ module Ferrum
|
|
|
191
194
|
end
|
|
192
195
|
end
|
|
193
196
|
|
|
194
|
-
#
|
|
195
|
-
# Evaluate JavaScript to modify things before a page load.
|
|
196
|
-
#
|
|
197
|
-
# @param [String] expression
|
|
198
|
-
# The JavaScript to add to each new document.
|
|
199
|
-
#
|
|
200
|
-
# @example
|
|
201
|
-
# browser.evaluate_on_new_document <<~JS
|
|
202
|
-
# Object.defineProperty(navigator, "languages", {
|
|
203
|
-
# get: function() { return ["tlh"]; }
|
|
204
|
-
# });
|
|
205
|
-
# JS
|
|
206
|
-
#
|
|
207
|
-
def evaluate_on_new_document(expression)
|
|
208
|
-
extensions << expression
|
|
209
|
-
end
|
|
210
|
-
|
|
211
197
|
#
|
|
212
198
|
# Closes browser tabs opened by the `Browser` instance.
|
|
213
199
|
#
|
|
@@ -309,9 +295,9 @@ module Ferrum
|
|
|
309
295
|
|
|
310
296
|
begin
|
|
311
297
|
@process.start
|
|
312
|
-
@options.default_user_agent = @process.default_user_agent
|
|
313
298
|
|
|
314
299
|
@client = Client.new(@process.ws_url, options)
|
|
300
|
+
@options.default_user_agent = version.user_agent
|
|
315
301
|
@contexts = Contexts.new(@client)
|
|
316
302
|
rescue StandardError
|
|
317
303
|
@process.stop
|
|
@@ -24,7 +24,6 @@ module Ferrum
|
|
|
24
24
|
class WebSocket
|
|
25
25
|
WEBSOCKET_BUG_SLEEP = 0.05
|
|
26
26
|
DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
|
|
27
|
-
SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
|
|
28
27
|
WRITE_CHUNK_SIZE = 64 * 1024
|
|
29
28
|
ESCAPE_OR_LONE_SURROGATE = /\\u[dD][89abAB]\h{2}\\u[dD][c-fC-F]\h{2}|\\u([dD][89a-fA-F]\h{2})|\\./
|
|
30
29
|
|
|
@@ -34,9 +33,9 @@ module Ferrum
|
|
|
34
33
|
# Why Ferrum closed the connection itself, `nil` if it didn't.
|
|
35
34
|
attr_reader :close_reason
|
|
36
35
|
|
|
37
|
-
def initialize(url, max_receive_size,
|
|
36
|
+
def initialize(url, max_receive_size, log, write_timeout = nil)
|
|
38
37
|
@url = url
|
|
39
|
-
@
|
|
38
|
+
@log = log
|
|
40
39
|
@write_timeout = write_timeout
|
|
41
40
|
uri = URI.parse(@url)
|
|
42
41
|
port = uri.port || DEFAULT_PORTS[uri.scheme]
|
|
@@ -60,8 +59,6 @@ module Ferrum
|
|
|
60
59
|
@messages = Queue.new
|
|
61
60
|
@outgoing = Queue.new
|
|
62
61
|
|
|
63
|
-
@screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
|
|
64
|
-
|
|
65
62
|
@driver.on(:open, &method(:on_open))
|
|
66
63
|
@driver.on(:message, &method(:on_message))
|
|
67
64
|
@driver.on(:close, &method(:on_close))
|
|
@@ -91,13 +88,7 @@ module Ferrum
|
|
|
91
88
|
#
|
|
92
89
|
def on_message(event)
|
|
93
90
|
data = safely_parse_json(event.data)
|
|
94
|
-
|
|
95
|
-
if SKIP_LOGGING_SCREENSHOTS && @screenshot_commands[data&.dig("id")]
|
|
96
|
-
@screenshot_commands.delete(data&.dig("id"))
|
|
97
|
-
output.sub!(/{"data":"[^"]*"}/, %("Set FERRUM_LOGGING_SCREENSHOTS=true to see screenshots in Base64"))
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
@logger&.puts(" ◀ #{Utils::ElapsedTime.elapsed_time} #{output}\n")
|
|
91
|
+
@log&.received(data, event.data, self)
|
|
101
92
|
|
|
102
93
|
# If we couldn't parse JSON data for some reason (parse error or deeply nested object) we
|
|
103
94
|
# don't push response to @messages. Worse that could happen we raise timeout error due to command didn't return
|
|
@@ -135,12 +126,10 @@ module Ferrum
|
|
|
135
126
|
# @return [void]
|
|
136
127
|
#
|
|
137
128
|
def send_message(data, timeout = nil)
|
|
138
|
-
@screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
|
|
139
|
-
|
|
140
129
|
json = data.to_json
|
|
141
130
|
deadline = Utils::ElapsedTime.monotonic_time + timeout if timeout
|
|
142
131
|
drive(deadline) { @driver.text(json) }
|
|
143
|
-
@
|
|
132
|
+
@log&.sent(data, self)
|
|
144
133
|
end
|
|
145
134
|
|
|
146
135
|
#
|
data/lib/ferrum/client.rb
CHANGED
|
@@ -160,7 +160,7 @@ module Ferrum
|
|
|
160
160
|
@ws_url = ws_url
|
|
161
161
|
@options = options
|
|
162
162
|
@pendings = Concurrent::Hash.new
|
|
163
|
-
@ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.
|
|
163
|
+
@ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.log, options.protocol_timeout)
|
|
164
164
|
@subscriber = Subscriber.new
|
|
165
165
|
|
|
166
166
|
start
|