ferrum 0.19.1 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +60 -0
  3. data/lib/ferrum/browser/binary.rb +1 -2
  4. data/lib/ferrum/browser/options.rb +33 -12
  5. data/lib/ferrum/browser/process/output.rb +143 -0
  6. data/lib/ferrum/browser/process.rb +42 -69
  7. data/lib/ferrum/browser.rb +9 -23
  8. data/lib/ferrum/client/web_socket.rb +93 -73
  9. data/lib/ferrum/client.rb +9 -5
  10. data/lib/ferrum/console/location.rb +42 -0
  11. data/lib/ferrum/console/message.rb +230 -0
  12. data/lib/ferrum/console.rb +157 -0
  13. data/lib/ferrum/context.rb +23 -15
  14. data/lib/ferrum/contexts.rb +41 -18
  15. data/lib/ferrum/errors.rb +7 -5
  16. data/lib/ferrum/frame/dom.rb +50 -44
  17. data/lib/ferrum/frame/runtime/legacy.rb +336 -0
  18. data/lib/ferrum/frame/runtime.rb +416 -186
  19. data/lib/ferrum/frame/script.rb +289 -0
  20. data/lib/ferrum/frame.rb +33 -4
  21. data/lib/ferrum/interceptable.rb +19 -9
  22. data/lib/ferrum/isolated_world.rb +94 -0
  23. data/lib/ferrum/keyboard.rb +95 -8
  24. data/lib/ferrum/log/annotated.rb +77 -0
  25. data/lib/ferrum/log/default.rb +55 -0
  26. data/lib/ferrum/log/json.rb +32 -0
  27. data/lib/ferrum/log/paint.rb +28 -0
  28. data/lib/ferrum/log.rb +247 -0
  29. data/lib/ferrum/mouse.rb +16 -8
  30. data/lib/ferrum/node.rb +127 -52
  31. data/lib/ferrum/page/frames.rb +10 -3
  32. data/lib/ferrum/page/screenshot.rb +8 -7
  33. data/lib/ferrum/page.rb +106 -53
  34. data/lib/ferrum/proxy.rb +8 -7
  35. data/lib/ferrum/remote_object.rb +75 -0
  36. data/lib/ferrum/target.rb +7 -0
  37. data/lib/ferrum/utils/deprecate.rb +77 -0
  38. data/lib/ferrum/version.rb +1 -1
  39. data/lib/ferrum/worker.rb +8 -0
  40. data/lib/ferrum.rb +3 -0
  41. metadata +16 -2
@@ -0,0 +1,77 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ferrum/log/paint"
4
+
5
+ module Ferrum
6
+ class Log
7
+ #
8
+ # Formats CDP messages one line each, the JSON whole after a header: the
9
+ # direction, the elapsed time, the command id and method. Received
10
+ # messages are indented under the commands sent. A response also
11
+ # names the method of its command and how long the answer took. Picked
12
+ # with `FERRUM_LOG_FORMAT=annotated`.
13
+ #
14
+ class Annotated
15
+ include Paint
16
+
17
+ def initialize(color: false)
18
+ @color = color
19
+ end
20
+
21
+ # @param [Hash{Symbol => Object}] message
22
+ #
23
+ # @param [Float] elapsed
24
+ #
25
+ # @return [String]
26
+ def sent(message, elapsed)
27
+ line(paint("▶", :cyan), time(elapsed), "##{message[:id]}", paint(message[:method], :bold),
28
+ json(message[:params]), session(message[:sessionId]))
29
+ end
30
+
31
+ # @param [Hash{String => Object}, nil] message
32
+ #
33
+ # @param [String] raw
34
+ #
35
+ # @param [Float] elapsed
36
+ #
37
+ # @param [Array(String, Float), nil] command
38
+ # The method of the command this message answers and how long the
39
+ # answer took, in seconds.
40
+ #
41
+ # @return [String]
42
+ def received(message, raw, elapsed, command = nil)
43
+ return line(" ", paint("◀", :magenta), time(elapsed), raw) unless message
44
+ return event(message, elapsed) unless message.key?("id")
45
+
46
+ method, duration = command
47
+ error = message["error"]
48
+ line(" ", paint("◀", error ? :red : :green), time(elapsed), "##{message['id']}", paint(method, :bold),
49
+ duration && "#{(duration * 1000).round}ms", json(error || message["result"]),
50
+ session(message["sessionId"]))
51
+ end
52
+
53
+ private
54
+
55
+ def event(message, elapsed)
56
+ line(" ", paint("◀", :magenta), time(elapsed), paint(message["method"], :bold), json(message["params"]),
57
+ session(message["sessionId"]))
58
+ end
59
+
60
+ def line(*parts)
61
+ parts.map(&:to_s).reject(&:empty?).join(" ")
62
+ end
63
+
64
+ def time(elapsed)
65
+ paint(format("%.3f", elapsed), :dim)
66
+ end
67
+
68
+ def json(value)
69
+ Log.dump(value) unless value.nil? || value == {}
70
+ end
71
+
72
+ def session(id)
73
+ paint("@#{id[0, 8]}", :dim) if id
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ferrum/log/paint"
4
+
5
+ module Ferrum
6
+ class Log
7
+ #
8
+ # Formats CDP messages as Ferrum always has: the direction, the elapsed
9
+ # time and the message as JSON. On a terminal the arrow is colored by
10
+ # kind, the time dimmed and the method bold.
11
+ #
12
+ class Default
13
+ include Paint
14
+
15
+ def initialize(color: false)
16
+ @color = color
17
+ end
18
+
19
+ # @param [Hash{Symbol => Object}] message
20
+ #
21
+ # @param [Float] elapsed
22
+ #
23
+ # @return [String]
24
+ def sent(message, elapsed)
25
+ "\n\n#{paint('▶', :cyan)} #{paint(elapsed, :dim)} #{payload(message, message[:method])}"
26
+ end
27
+
28
+ # @param [Hash{String => Object}, nil] message
29
+ #
30
+ # @param [String] raw
31
+ #
32
+ # @param [Float] elapsed
33
+ #
34
+ # @return [String]
35
+ def received(message, raw, elapsed, _command = nil)
36
+ arrow = if message.nil? || !message.key?("id") then :magenta
37
+ elsif message.key?("error") then :red
38
+ else :green
39
+ end
40
+ body = message ? payload(message, message["method"]) : raw
41
+
42
+ " #{paint('◀', arrow)} #{paint(elapsed, :dim)} #{body}\n"
43
+ end
44
+
45
+ private
46
+
47
+ def payload(message, method)
48
+ json = Log.dump(message)
49
+ return json unless method && color?
50
+
51
+ json.sub(%("method":"#{method}"), %("method":"#{paint(method, :bold)}"))
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ferrum
4
+ class Log
5
+ #
6
+ # Formats CDP messages as JSON alone, one message per line, for tools such
7
+ # as `jq`. Commands, responses and events tell apart by their keys: `id`
8
+ # and `method`, `id` and `result` or `error`, `method` alone. Never
9
+ # colored. Picked with `FERRUM_LOG_FORMAT=json`.
10
+ #
11
+ class Json
12
+ def initialize(**); end
13
+
14
+ # @param [Hash{Symbol => Object}] message
15
+ #
16
+ # @return [String]
17
+ def sent(message, _elapsed)
18
+ Log.dump(message)
19
+ end
20
+
21
+ # @param [Hash{String => Object}, nil] message
22
+ #
23
+ # @param [String] raw
24
+ # Written as is when the message couldn't be parsed.
25
+ #
26
+ # @return [String]
27
+ def received(message, raw, _elapsed, _command = nil)
28
+ message ? Log.dump(message) : raw
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,28 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ferrum
4
+ class Log
5
+ #
6
+ # Wraps text in basic ANSI colors, which terminal themes adapt to light
7
+ # and dark backgrounds, when the includer's `@color` is set.
8
+ #
9
+ module Paint
10
+ COLORS = { bold: 1, dim: 2, red: 31, green: 32, yellow: 33, magenta: 35, cyan: 36 }.freeze
11
+
12
+ # Whether text gets colored.
13
+ #
14
+ # @return [Boolean]
15
+ def color?
16
+ @color
17
+ end
18
+
19
+ private
20
+
21
+ def paint(text, color)
22
+ return text.to_s unless color && color?
23
+
24
+ "\e[#{COLORS.fetch(color)}m#{text}\e[0m"
25
+ end
26
+ end
27
+ end
28
+ end
data/lib/ferrum/log.rb ADDED
@@ -0,0 +1,247 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ferrum/log/paint"
4
+ require "ferrum/log/default"
5
+ require "ferrum/log/json"
6
+ require "ferrum/log/annotated"
7
+
8
+ module Ferrum
9
+ #
10
+ # Writes Ferrum's debug output to the `:logger` given to {Browser.new}, split
11
+ # into channels: `:protocol` (CDP messages) and `:browser` (the output of the
12
+ # Chrome process Ferrum launched).
13
+ #
14
+ # The logger can be an IO-like object responding to `puts`, or a `Logger`,
15
+ # which gets every line at debug level with `ferrum:<channel>` as program
16
+ # name. Nothing is built for a disabled channel or a level the `Logger` skips.
17
+ #
18
+ # `FERRUM_LOG` picks what is written, see {.parse}. `FERRUM_LOG_FORMAT` picks
19
+ # how CDP messages look: `json` writes them as JSON alone, see {Json}, and
20
+ # `annotated` puts the command id and method in front of each message and
21
+ # its response, see {Annotated}. Without it they look as they always have,
22
+ # see {Default}.
23
+ #
24
+ # Written to a terminal, lines are colored with the basic ANSI colors, which
25
+ # terminal themes adapt to light and dark backgrounds. Files, pipes and
26
+ # `Logger`s get plain text, and so does a terminal when `NO_COLOR` is set.
27
+ # `FERRUM_LOG_COLOR=always` or `never` overrides both.
28
+ #
29
+ # Logging never breaks the browser: an error while formatting or writing a
30
+ # line, e.g. a closed IO, is reported once and the line skipped.
31
+ #
32
+ class Log
33
+ include Paint
34
+
35
+ CHANNELS = %i[protocol browser].freeze
36
+ ALL = %w[* 1 true].freeze
37
+ OFF = %w[0 false].freeze
38
+ FORMATS = { default: Default, json: Json, annotated: Annotated }.freeze
39
+ VALUE_LENGTH = Integer(ENV.fetch("FERRUM_LOG_VALUE_LENGTH", 1024))
40
+
41
+ attr_reader :logger, :channels, :only, :skip
42
+
43
+ # Parses `FERRUM_LOG`, a comma-separated list of:
44
+ #
45
+ # * `*`, `1` or `true` - all channels
46
+ # * `protocol`, `browser` - that channel, `-browser` leaves it out
47
+ # * `Page.*`, `Runtime.evaluate` - only CDP messages whose method matches,
48
+ # which turns `protocol` on
49
+ # * `-Network.*` - leaves out CDP messages whose method matches
50
+ #
51
+ # Without a channel or CDP pattern to log, all channels are logged, so
52
+ # `-browser` means everything but the browser output. Exclusions win over
53
+ # inclusions, and responses follow the command they answer. `0`, `false`
54
+ # or an empty value turn logging off.
55
+ #
56
+ # @param [String, nil] value
57
+ #
58
+ # @return [Hash{Symbol => Array}, nil]
59
+ # The `channels:`, `only:` and `skip:` for {#initialize}, `nil` when off.
60
+ def self.parse(value)
61
+ items = value.to_s.split(",").map(&:strip).reject(&:empty?)
62
+ return if items.all? { |item| OFF.include?(item) }
63
+
64
+ patterns, names = items.partition { |item| item.include?(".") }
65
+ skip, only = patterns.partition { |pattern| pattern.start_with?("-") }
66
+ removed, added = names.partition { |name| name.start_with?("-") }
67
+
68
+ channels = added.flat_map { |name| parse_channel(name) }
69
+ channels << :protocol if only.any?
70
+ channels = CHANNELS if channels.empty?
71
+ channels -= removed.flat_map { |name| parse_channel(name.delete_prefix("-")) }
72
+
73
+ { channels: channels.uniq, only: only, skip: skip.map { |pattern| pattern.delete_prefix("-") } }
74
+ end
75
+
76
+ # Parses one channel item of `FERRUM_LOG`, warning about an unknown one.
77
+ #
78
+ # @param [String] name
79
+ #
80
+ # @return [Array<Symbol>]
81
+ def self.parse_channel(name)
82
+ return CHANNELS if ALL.include?(name)
83
+ return [name.to_sym] if CHANNELS.include?(name.to_sym)
84
+
85
+ warn "Ferrum: unknown FERRUM_LOG item #{name.inspect} is ignored"
86
+ []
87
+ end
88
+
89
+ # Parses `FERRUM_LOG_FORMAT`: `json`, `annotated` or `default`, which is
90
+ # also what an empty or unknown value means.
91
+ #
92
+ # @param [String, nil] value
93
+ #
94
+ # @return [Symbol]
95
+ def self.parse_format(value)
96
+ format = value.to_s.strip
97
+ return :default if format.empty?
98
+ return format.to_sym if FORMATS.key?(format.to_sym)
99
+
100
+ warn "Ferrum: unknown FERRUM_LOG_FORMAT #{format.inspect}, using default"
101
+ :default
102
+ end
103
+
104
+ # Parses `FERRUM_LOG_COLOR`, anything but `always` or `never` means `:auto`.
105
+ #
106
+ # @param [String, nil] value
107
+ #
108
+ # @return [Symbol]
109
+ def self.parse_color(value)
110
+ %w[always never].include?(value.to_s.strip) ? value.strip.to_sym : :auto
111
+ end
112
+
113
+ # Encodes a CDP message, or a part of it, as JSON with string values longer
114
+ # than {VALUE_LENGTH} shortened, e.g. Base64 screenshots.
115
+ # `FERRUM_LOG_VALUE_LENGTH=0` keeps them whole. Messages nest as deep as
116
+ # Chrome sends them, e.g. a DOM tree, so there's no nesting limit.
117
+ #
118
+ # @param [Object] value
119
+ #
120
+ # @return [String]
121
+ def self.dump(value)
122
+ JSON.generate(truncate(value), max_nesting: false)
123
+ end
124
+
125
+ # Shortens string values longer than {VALUE_LENGTH} anywhere in a CDP
126
+ # message.
127
+ #
128
+ # @param [Object] value
129
+ #
130
+ # @return [Object]
131
+ def self.truncate(value)
132
+ case value
133
+ when Hash then value.transform_values { |item| truncate(item) }
134
+ when Array then value.map { |item| truncate(item) }
135
+ when String
136
+ return value if VALUE_LENGTH.zero? || value.size <= VALUE_LENGTH
137
+
138
+ "#{value[0, VALUE_LENGTH]}... (#{value.size - VALUE_LENGTH} more characters)"
139
+ else value
140
+ end
141
+ end
142
+
143
+ def initialize(logger, channels: CHANNELS, only: [], skip: [], format: :default, color: :auto)
144
+ @logger = logger
145
+ @channels = channels
146
+ @only = only
147
+ @skip = skip
148
+ @color = case color
149
+ when :always then true
150
+ when :never then false
151
+ else logger.respond_to?(:tty?) && logger.tty? && ENV.fetch("NO_COLOR", "").empty?
152
+ end
153
+ @formatter = FORMATS.fetch(format).new(color: @color)
154
+ @commands = Concurrent::Map.new if format == :annotated || only.any? || skip.any?
155
+ @mutex = Mutex.new
156
+ end
157
+
158
+ # Whether the channel is written to the logger.
159
+ #
160
+ # @param [Symbol] channel
161
+ #
162
+ # @return [Boolean]
163
+ def enabled?(channel)
164
+ channels.include?(channel)
165
+ end
166
+
167
+ # Writes a CDP message sent to the browser.
168
+ #
169
+ # @param [Hash{Symbol => Object}] message
170
+ #
171
+ # @param [Object] connection
172
+ # What the message was sent over, command ids are unique per connection.
173
+ #
174
+ # @return [void]
175
+ def sent(message, connection = nil)
176
+ return unless enabled?(:protocol)
177
+
178
+ @commands&.[]=([connection, message[:id]], [message[:method], Utils::ElapsedTime.monotonic_time])
179
+ return unless show?(message[:method])
180
+
181
+ write(:protocol) { @formatter.sent(message, Utils::ElapsedTime.elapsed_time) }
182
+ end
183
+
184
+ # Writes a CDP message received from the browser.
185
+ #
186
+ # @param [Hash{String => Object}, nil] message
187
+ # The parsed message, `nil` when it couldn't be parsed.
188
+ #
189
+ # @param [String] raw
190
+ # The message as received.
191
+ #
192
+ # @param [Object] connection
193
+ # What the message was received over.
194
+ #
195
+ # @return [void]
196
+ def received(message, raw, connection = nil)
197
+ return unless enabled?(:protocol)
198
+
199
+ command = answered_command(message, connection)
200
+ return unless show?(message&.fetch("method", nil) || command&.first)
201
+
202
+ write(:protocol) { @formatter.received(message, raw, Utils::ElapsedTime.elapsed_time, command) }
203
+ end
204
+
205
+ # Writes a line of the browser process output.
206
+ #
207
+ # @param [String] line
208
+ #
209
+ # @return [void]
210
+ def browser(line)
211
+ write(:browser) { line }
212
+ end
213
+
214
+ private
215
+
216
+ def answered_command(message, connection)
217
+ return unless @commands && message&.key?("id")
218
+
219
+ method, sent_at = @commands.delete([connection, message["id"]])
220
+ [method, Utils::ElapsedTime.monotonic_time - sent_at] if method
221
+ end
222
+
223
+ def show?(method)
224
+ return true unless method
225
+
226
+ (only.empty? || matches?(only, method)) && !matches?(skip, method)
227
+ end
228
+
229
+ def matches?(patterns, method)
230
+ patterns.any? { |pattern| File.fnmatch?(pattern, method) }
231
+ end
232
+
233
+ def write(channel, &block)
234
+ return unless enabled?(channel)
235
+
236
+ if logger.respond_to?(:debug)
237
+ logger.debug("ferrum:#{channel}", &block)
238
+ else
239
+ line = block.call
240
+ @mutex.synchronize { logger.puts(line) }
241
+ end
242
+ rescue StandardError => e
243
+ warn "Ferrum: couldn't write to the logger, #{e.class}: #{e.message}" unless @write_failed
244
+ @write_failed = true
245
+ end
246
+ end
247
+ end
data/lib/ferrum/mouse.rb CHANGED
@@ -81,12 +81,13 @@ module Ferrum
81
81
  # @option options [Integer] :count (1)
82
82
  #
83
83
  # @option options [Integer] :modifiers
84
- # Bitfield for key modifiers. See`keyboard.modifiers`.
84
+ # Bitfield for key modifiers. See `keyboard.modifiers`.
85
+ # If omitted, uses the keys held by `keyboard.hold`, or none outside of it.
85
86
  #
86
87
  # @return [self]
87
88
  #
88
89
  def click(x:, y:, delay: 0, wait: CLICK_WAIT, **options)
89
- move(x: x, y: y)
90
+ move(x: x, y: y, modifiers: options[:modifiers])
90
91
  down(**options)
91
92
  sleep(delay)
92
93
  # Potential wait because if some network event is triggered then we have
@@ -107,7 +108,8 @@ module Ferrum
107
108
  # @option options [Integer] :count (1)
108
109
  #
109
110
  # @option options [Integer] :modifiers
110
- # Bitfield for key modifiers. See`keyboard.modifiers`.
111
+ # Bitfield for key modifiers. See `keyboard.modifiers`.
112
+ # If omitted, uses the keys held by `keyboard.hold`, or none outside of it.
111
113
  #
112
114
  # @return [self]
113
115
  #
@@ -127,7 +129,8 @@ module Ferrum
127
129
  # @option options [Integer] :count (1)
128
130
  #
129
131
  # @option options [Integer] :modifiers
130
- # Bitfield for key modifiers. See`keyboard.modifiers`.
132
+ # Bitfield for key modifiers. See `keyboard.modifiers`.
133
+ # If omitted, uses the keys held by `keyboard.hold`, or none outside of it.
131
134
  #
132
135
  # @return [self]
133
136
  #
@@ -145,9 +148,13 @@ module Ferrum
145
148
  # @param [Integer] steps
146
149
  # Sends intermediate mousemove events.
147
150
  #
151
+ # @param [Integer, nil] modifiers
152
+ # Bitfield for key modifiers. See `keyboard.modifiers`.
153
+ # If omitted, uses the keys held by `keyboard.hold`, or none outside of it.
154
+ #
148
155
  # @return [self]
149
156
  #
150
- def move(x:, y:, steps: 1)
157
+ def move(x:, y:, steps: 1, modifiers: nil)
151
158
  from_x = @x
152
159
  from_y = @y
153
160
  @x = x
@@ -164,7 +171,8 @@ module Ferrum
164
171
  x: new_x,
165
172
  y: new_y,
166
173
  button: button,
167
- buttons: @buttons)
174
+ buttons: @buttons,
175
+ modifiers: modifiers || @page.keyboard.held_modifiers)
168
176
  end
169
177
 
170
178
  self
@@ -175,8 +183,8 @@ module Ferrum
175
183
  def mouse_event(type:, button: :left, count: 1, modifiers: nil, wait: 0)
176
184
  button = validate_button(button)
177
185
  register_event_button(type, button)
178
- options = { x: @x, y: @y, type: type, button: button, buttons: @buttons, clickCount: count }
179
- options.merge!(modifiers: modifiers) if modifiers
186
+ options = { x: @x, y: @y, type: type, button: button, buttons: @buttons, clickCount: count,
187
+ modifiers: modifiers || @page.keyboard.held_modifiers }
180
188
  @page.command("Input.dispatchMouseEvent", wait: wait, slowmoable: true, **options)
181
189
  end
182
190