ferrum 0.10.1 → 0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cea2150084c494e0461e9a63b53ff215e34ed98f2bfe345458ae46c95f0f8f3
-  data.tar.gz: 46fad57af780b5a5b6ff9659125642d6b57674db386af34b868dc07d1164e2b6
+  metadata.gz: 9e4ca57662283c8c6b917aa3e2f1f06b154cb66ffdd8522756f55e09be84a423
+  data.tar.gz: 50d37d615399fe481634154d619d55872ee13b0cace511a1b147da38454247b5
 SHA512:
-  metadata.gz: 63d6104319d89bfa85897c946c01f6d44efd92de88204b4882ccea879e196dd80bebfd61b06a7566272dd6e22de1c306eb1385d17a6a72df739d94753adb0a41
-  data.tar.gz: ef333fd514ea94d674bf4ad51117f736c4d7982ca56981192fe636010e1f34df6c84d889ce2ce40bfeeb47534f1c912a5aec54efdad01e23b7d915ac1be88524
+  metadata.gz: 9cc514740ae8041b61a184bdeee0a44ce26b276f36af0358c6a00524f8e13b6db9ddda48acf5781c4122acaa4980579cf06053ffb420c2013e0b44177eab2571
+  data.tar.gz: cc00b6d3cb440db600241156354dcd92a6bf67c0402b9e2881f32ea507549d80f69b51011bb246baf9faf95f33e5596933ce569cd9b03f41c7cc066d215b6073

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019-2020 Dmitry Vorotilin
+Copyright (c) 2019-2022 Dmitry Vorotilin
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -3,7 +3,7 @@
 <img align="right"
      width="320" height="241"
      alt="Ferrum logo"
-     src="https://raw.githubusercontent.com/rubycdp/ferrum/master/logo.svg?sanitize=true">
+     src="https://raw.githubusercontent.com/rubycdp/ferrum/main/logo.svg?sanitize=true">
 
 #### As simple as Puppeteer, though even simpler.
 
@@ -23,12 +23,7 @@ going to crawl sites you better use Ferrum or
 [Vessel](https://github.com/rubycdp/vessel) because you crawl, not test.
 
 * [Vessel](https://github.com/rubycdp/vessel) high-level web crawling framework
-based on Ferrum. It looks like [Scrapy](https://scrapy.org/) except that it uses
-a real browser in order to grab data.
-
-Web design by [Evrone](https://evrone.com/), what else
-[we build with Ruby on Rails](https://evrone.com/ruby), what else
-[we do at Evrone](https://evrone.com/cases#case-studies).
+based on Ferrum and Mechanize.
 
 
 ## Index
@@ -40,7 +35,9 @@ Web design by [Evrone](https://evrone.com/), what else
 * [Navigation](https://github.com/rubycdp/ferrum#navigation)
 * [Finders](https://github.com/rubycdp/ferrum#finders)
 * [Screenshots](https://github.com/rubycdp/ferrum#screenshots)
+* [Cleaning Up](https://github.com/rubycdp/ferrum#cleaning-up)
 * [Network](https://github.com/rubycdp/ferrum#network)
+* [Proxy](https://github.com/rubycdp/ferrum#proxy)
 * [Mouse](https://github.com/rubycdp/ferrum#mouse)
 * [Keyboard](https://github.com/rubycdp/ferrum#keyboard)
 * [Cookies](https://github.com/rubycdp/ferrum#cookies)
@@ -48,7 +45,10 @@ Web design by [Evrone](https://evrone.com/), what else
 * [JavaScript](https://github.com/rubycdp/ferrum#javascript)
 * [Frames](https://github.com/rubycdp/ferrum#frames)
 * [Frame](https://github.com/rubycdp/ferrum#frame)
-* [Dialog](https://github.com/rubycdp/ferrum#dialog)
+* [Dialogs](https://github.com/rubycdp/ferrum#dialogs)
+* [Animation](https://github.com/rubycdp/ferrum#animation)
+* [Node](https://github.com/rubycdp/ferrum#node)
+* [Tracing](https://github.com/rubycdp/ferrum#tracing)
 * [Thread safety](https://github.com/rubycdp/ferrum#thread-safety)
 * [Development](https://github.com/rubycdp/ferrum#development)
 * [Contributing](https://github.com/rubycdp/ferrum#contributing)
@@ -59,7 +59,8 @@ Web design by [Evrone](https://evrone.com/), what else
 
 There's no official Chrome or Chromium package for Linux don't install it this
 way because it's either outdated or unofficial, both are bad. Download it from
-official [source](https://www.chromium.org/getting-involved/download-chromium).
+official source for [Chrome](https://www.google.com/chrome/) or
+[Chromium](https://www.chromium.org/getting-involved/download-chromium).
 Chrome binary should be in the `PATH` or `BROWSER_PATH` or you can pass it as an
 option to browser instance see `:browser_path` in
 [Customization](https://github.com/rubycdp/ferrum#customization).
@@ -171,15 +172,18 @@ Ferrum::Browser.new(options)
       options it passes to the browser, if you set this to `true` then only
       options you put in `:browser_options` will be passed to the browser,
       except required ones of course.
-  * `:port` (Integer) - Remote debugging port for headless Chrome
-  * `:host` (String) - Remote debugging address for headless Chrome
+  * `:port` (Integer) - Remote debugging port for headless Chrome.
+  * `:host` (String) - Remote debugging address for headless Chrome.
   * `:url` (String) - URL for a running instance of Chrome. If this is set, a
       browser process will not be spawned.
   * `:process_timeout` (Integer) - How long to wait for the Chrome process to
-      respond on startup
+      respond on startup.
   * `:ws_max_receive_size` (Integer) - How big messages to accept from Chrome
       over the web socket, in bytes. Defaults to 64MB. Incoming messages larger
       than this will cause a `Ferrum::DeadBrowserError`.
+  * `:proxy` (Hash) - Specify proxy settings, [read more](https://github.com/rubycdp/ferrum#proxy)
+  * `:save_path` (String) - Path to save attachments with [Content-Disposition](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition) header.
+  * `:env` (Hash) - Environment variables you'd like to pass through to the process
 
 
 ## Navigation
@@ -234,6 +238,25 @@ browser.go_to("https://github.com/")
 browser.stop
 ```
 
+#### position = \*\*options
+
+Set the position for the browser window
+
+* options `Hash`
+  * :left `Integer`
+  * :top `Integer`
+
+```ruby
+browser.position = { left: 10, top: 20 }
+```
+
+#### position : `Array<Integer>`
+
+Get the position for the browser window
+
+```ruby
+browser.position # => [10, 20]
+```
 
 ## Finders
 
@@ -386,9 +409,28 @@ browser.mhtml(path: "google.mhtml") # => 87742
 ```
 
 
+## Cleaning Up
+
+#### reset
+
+Closes browser tabs opened by the `Browser` instance.
+
+```ruby
+# connect to a long-running Chrome process
+browser = Ferrum::Browser.new(url: 'http://localhost:9222')
+
+browser.go_to("https://github.com/")
+
+# clean up, lest the tab stays there hanging forever
+browser.reset
+
+browser.quit
+```
+
+
 ## Network
 
-browser.network
+`browser.network`
 
 #### traffic `Array<Network::Exchange>`
 
@@ -521,17 +563,83 @@ end
 browser.network.authorize(user: "login", password: "pass", type: :proxy)
 
 browser.go_to("https://google.com")
-
 ```
 
 You used to call `authorize` method without block, but since it's implemented using request interception there could be
 a collision with another part of your code that also uses request interception, so that authorize allows the request
 while your code denies but it's too late. The block is mandatory now.
 
+#### emulate_network_conditions(\*\*options)
+
+Activates emulation of network conditions.
+
+* options `Hash`
+  * :offline `Boolean` emulate internet disconnection, `false` by default
+  * :latency `Integer` minimum latency from request sent to response headers received (ms), `0` by
+    default
+  * :download_throughput `Integer` maximal aggregated download throughput (bytes/sec), `-1`
+    by default, disables download throttling
+  * :upload_throughput `Integer` maximal aggregated upload throughput (bytes/sec), `-1`
+    by default, disables download throttling
+  * :connection_type `String` connection type if known, one of: none, cellular2g, cellular3g, cellular4g,
+    bluetooth, ethernet, wifi, wimax, other. `nil` by default
+
+```ruby
+browser.network.emulate_network_conditions(connection_type: "cellular2g")
+browser.go_to("https://github.com/")
+```
+
+#### offline_mode
+
+Activates offline mode for a page.
+
+```ruby
+browser.network.offline_mode
+browser.go_to("https://github.com/") # => Ferrum::StatusError (Request to https://github.com/ failed to reach server, check DNS and server status)
+```
+
+
+## Proxy
+
+You can set a proxy with the `proxy` option.
+
+```ruby
+browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800" })
+browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", user: "user", pasword: "pa$$" })
+```
+
+Chrome Devtools Protocol does not support changing proxies after the browser is launched. If you want to change proxies, you must restart your browser, which may not be convenient. There is a workaround. Ferrum provides a wrapper for a proxy server that can rotate proxies. We can run a proxy in the same process and rotate proxies inside this proxy server:
+
+```ruby
+browser = Ferrum::Browser.new(proxy: { server: true })
+
+browser.proxy_server.rotate(host: "x.x.x.x", port: 31337, user: "user", password: "password")
+browser.create_page(new_context: true) do |page|
+  page.go_to("https://api.ipify.org?format=json")
+  page.body # => "x.x.x.x"
+end
+
+browser.proxy_server.rotate(host: "y.y.y.y", port: 31337, user: "user", password: "password")
+browser.create_page(new_context: true) do |page|
+  page.go_to("https://api.ipify.org?format=json")
+  page.body # => "y.y.y.y"
+end
+```
+
+Make sure to create page in the new context, because Chrome doesn't break the connection with the proxy for `CONNECT`
+requests even if you close the page.
+
+You can specify semi-colon-separated list of hosts for which proxy shouldn't be used:
+
+```ruby
+browser = Ferrum::Browser.new(proxy: { host: "x.x.x.x", port: "8800", bypass: "*.google.com;*foo.com" })
+browser = Ferrum::Browser.new(proxy: { server: true, bypass: "*.google.com;*foo.com" })
+```
+
 
 ### Mouse
 
-browser.mouse
+`browser.mouse`
 
 #### scroll_to(x, y)
 
@@ -618,13 +726,12 @@ Returns bitfield for a given keys
 
 ## Cookies
 
-browser.cookies
+`browser.cookies`
 
 #### all : `Hash<String, Cookie>`
 
 Returns cookies hash
 
-
 ```ruby
 browser.cookies.all # => {"NID"=>#<Ferrum::Cookies::Cookie:0x0000558624b37a40 @attributes={"name"=>"NID", "value"=>"...", "domain"=>".google.com", "path"=>"/", "expires"=>1583211046.575681, "size"=>178, "httpOnly"=>true, "secure"=>false, "session"=>false}>}
 ```
@@ -639,11 +746,11 @@ Returns cookie
 browser.cookies["NID"] # => <Ferrum::Cookies::Cookie:0x0000558624b67a88 @attributes={"name"=>"NID", "value"=>"...", "domain"=>".google.com", "path"=>"/", "expires"=>1583211046.575681, "size"=>178, "httpOnly"=>true, "secure"=>false, "session"=>false}>
 ```
 
-#### set(\*\*options) : `Boolean`
+#### set(value) : `Boolean`
 
-Sets given values as cookie
+Sets a cookie
 
-* options `Hash`
+* value `Hash`
   * :name `String`
   * :value `String`
   * :domain `String`
@@ -655,6 +762,13 @@ Sets given values as cookie
 browser.cookies.set(name: "stealth", value: "omg", domain: "google.com") # => true
 ```
 
+* value `Cookie`
+
+```ruby
+nid_cookie = browser.cookies["NID"] # => <Ferrum::Cookies::Cookie:0x0000558624b67a88>
+browser.cookies.set(nid_cookie) # => true
+```
+
 #### remove(\*\*options) : `Boolean`
 
 Removes given cookie
@@ -678,7 +792,7 @@ browser.cookies.clear # => true
 
 ## Headers
 
-browser.headers
+`browser.headers`
 
 #### get : `Hash`
 
@@ -740,6 +854,20 @@ simple value.
 browser.execute(%(1 + 1)) # => true
 ```
 
+#### evaluate_on_new_document(expression)
+
+Evaluate JavaScript to modify things before a page load
+
+* expression `String` should be valid JavaScript
+
+```ruby
+browser.evaluate_on_new_document <<~JS
+  Object.defineProperty(navigator, "languages", {
+    get: function() { return ["tlh"]; }
+  });
+JS
+```
+
 #### add_script_tag(\*\*options) : `Boolean`
 
 * options `Hash`
@@ -763,9 +891,10 @@ browser.add_script_tag(url: "http://example.com/stylesheet.css") # => true
 browser.add_style_tag(content: "h1 { font-size: 40px; }") # => true
 
 ```
-#### bypass_csp(enabled) : `Boolean`
+#### bypass_csp(\*\*options) : `Boolean`
 
-* enabled `Boolean`, `true` by default
+* options `Hash`
+  * :enabled `Boolean`, `true` by default
 
 ```ruby
 browser.bypass_csp # => true
@@ -906,7 +1035,7 @@ browser.go_to("https://www.w3schools.com/tags/tag_frame.asp")
 browser.main_frame.doctype # => "<!DOCTYPE html>"
 ```
 
-#### set_content(html)
+#### content = html
 
 Sets a content of a given frame.
 
@@ -916,12 +1045,12 @@ Sets a content of a given frame.
 browser.go_to("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe")
 frame = browser.frames[1]
 frame.body # <html lang="en"><head><style>body {transition: opacity ease-in 0.2s; }...
-frame.set_content("<html><head></head><body><p>lol</p></body></html>")
+frame.content = "<html><head></head><body><p>lol</p></body></html>"
 frame.body # => <html><head></head><body><p>lol</p></body></html>
 ```
 
 
-## Dialog
+## Dialogs
 
 #### accept(text)
 
@@ -946,6 +1075,107 @@ browser.go_to("https://google.com")
 ```
 
 
+## Animation
+
+You can slow down or speed up CSS animations.
+
+#### playback_rate : `Integer`
+
+Returns playback rate for CSS animations, defaults to `1`.
+
+
+#### playback_rate = value
+
+Sets playback rate of CSS animations
+
+  * value `Integer`
+
+```ruby
+browser = Ferrum::Browser.new
+browser.playback_rate = 2000
+browser.go_to("https://google.com")
+browser.playback_rate # => 2000
+```
+
+
+## Node
+
+#### node? : `Boolean`
+#### frame_id
+#### frame  : `Frame`
+
+Returns [Frame](https://github.com/rubycdp/ferrum#frame) object for current node, you can keep using
+[Finders](https://github.com/rubycdp/ferrum#Finders) for that object:
+
+```ruby
+frame =  browser.at_xpath("//iframe").frame # => Frame
+frame.at_css("//a[text() = 'Log in']") # => Node
+```
+
+#### focus
+#### focusable?
+#### moving? : `Boolean`
+#### wait_for_stop_moving
+#### blur
+#### type
+#### click
+#### hover
+#### select_file
+#### at_xpath
+#### at_css
+#### xpath
+#### css
+#### text
+#### inner_text
+#### value
+#### property
+#### attribute
+#### evaluate
+#### selected : `Array<Node>`
+#### select
+
+(chainable) Selects options by passed attribute.
+
+```ruby
+browser.at_xpath("//*[select]").select(["1"]) # => Node (select)
+browser.at_xpath("//*[select]").select(["text"], by: :text) # => Node (select)
+```
+
+Accept string, array or strings:
+```ruby
+browser.at_xpath("//*[select]").select("1")
+browser.at_xpath("//*[select]").select("1", "2")
+browser.at_xpath("//*[select]").select(["1", "2"])
+```
+
+
+## Tracing
+
+You can use `tracing.record` to create a trace file which can be opened in Chrome DevTools or
+[timeline viewer](https://chromedevtools.github.io/timeline-viewer/).
+
+```ruby
+page.tracing.record(path: "trace.json") do
+  page.go_to("https://www.google.com")
+end
+```
+
+#### tracing.record(\*\*options) : `String`
+
+Accepts block, records trace and by default returns trace data from `Tracing.tracingComplete` event as output. When
+`path` is specified returns `true` and stores trace data into file.
+
+* options `Hash`
+  * :path `String` save data on the disk, `nil` by default
+  * :encoding `Symbol` `:base64` | `:binary` encode output as Base64 or plain text. `:binary` by default
+  * :timeout `Float` wait until file streaming finishes in the specified time or raise error, defaults to `nil`
+  * :screenshots `Boolean` capture screenshots in the trace, `false` by default
+  * :trace_config `Hash<String, Object>` config for
+    [trace](https://chromedevtools.github.io/devtools-protocol/tot/Tracing/#type-TraceConfig), for categories
+    see [getCategories](https://chromedevtools.github.io/devtools-protocol/tot/Tracing/#method-getCategories),
+    only one trace config can be active at a time per browser.
+
+
 ## Thread safety ##
 
 Ferrum is fully thread-safe. You can create one browser or a few as you wish and
@@ -1007,9 +1237,12 @@ browser.quit
 
 After checking out the repo, run `bundle install` to install dependencies.
 
-Then, run `bundle exec rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+Then, run `bundle exec rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will
+allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the
+version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version,
+push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 
 ## Contributing

data/lib/ferrum/browser/binary.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    module Binary
+      module_function
+
+      def find(commands)
+        enum(commands).first
+      end
+
+      def all(commands)
+        enum(commands).force
+      end
+
+      def enum(commands)
+        paths, exts = prepare_paths
+        cmds = Array(commands).product(paths, exts)
+        lazy_find(cmds)
+      end
+
+      def prepare_paths
+        exts = (ENV.key?("PATHEXT") ? ENV.fetch("PATHEXT").split(";") : []) << ""
+        paths = ENV["PATH"].split(File::PATH_SEPARATOR)
+        raise EmptyPathError if paths.empty?
+
+        [paths, exts]
+      end
+
+      # rubocop:disable Style/CollectionCompact
+      def lazy_find(cmds)
+        cmds.lazy.map do |cmd, path, ext|
+          absolute_path = File.absolute_path(cmd)
+          is_absolute_path = absolute_path == cmd
+          cmd = File.expand_path("#{cmd}#{ext}", path) unless is_absolute_path
+
+          next unless File.executable?(cmd)
+          next if File.directory?(cmd)
+
+          cmd
+        end.reject(&:nil?) # .compact isn't defined on Enumerator::Lazy
+      end
+      # rubocop:enable Style/CollectionCompact
+    end
+  end
+end

data/lib/ferrum/browser/client.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require "concurrent-ruby"
 require "ferrum/browser/subscriber"
 require "ferrum/browser/web_socket"
 
@@ -14,17 +13,18 @@ module Ferrum
         @command_id = id_starts_with
         @pendings = Concurrent::Hash.new
         @ws = WebSocket.new(ws_url, @browser.ws_max_receive_size, @browser.logger)
-        @subscriber, @interruptor = Subscriber.build(2)
+        @subscriber, @interrupter = Subscriber.build(2)
 
         @thread = Thread.new do
           Thread.current.abort_on_exception = true
-          if Thread.current.respond_to?(:report_on_exception=)
-            Thread.current.report_on_exception = true
-          end
+          Thread.current.report_on_exception = true if Thread.current.respond_to?(:report_on_exception=)
+
+          loop do
+            message = @ws.messages.pop
+            break unless message
 
-          while message = @ws.messages.pop
             if INTERRUPTIONS.include?(message["method"])
-              @interruptor.async.call(message)
+              @interrupter.async.call(message)
             elsif message.key?("method")
               @subscriber.async.call(message)
             else
@@ -44,6 +44,7 @@ module Ferrum
 
         raise DeadBrowserError if data.nil? && @ws.messages.closed?
         raise TimeoutError unless data
+
         error, response = data.values_at("error", "result")
         raise_browser_error(error) if error
         response
@@ -52,14 +53,14 @@ module Ferrum
       def on(event, &block)
         case event
         when *INTERRUPTIONS
-          @interruptor.on(event, &block)
+          @interrupter.on(event, &block)
         else
           @subscriber.on(event, &block)
         end
       end
 
       def subscribed?(event)
-        [@interruptor, @subscriber].any? { |s| s.subscribed?(event) }
+        [@interrupter, @subscriber].any? { |s| s.subscribed?(event) }
       end
 
       def close
@@ -84,14 +85,16 @@ module Ferrum
         # Node has disappeared while we were trying to get it
         when "No node with given id found",
              "Could not find node with given id"
-          raise NodeNotFoundError.new(error)
+          raise NodeNotFoundError, error
         # Context is lost, page is reloading
         when "Cannot find context with specified id"
-          raise NoExecutionContextError.new(error)
+          raise NoExecutionContextError, error
         when "No target with given id found"
           raise NoSuchPageError
+        when /Could not compute content quads/
+          raise CoordinatesNotFoundError
         else
-          raise BrowserError.new(error)
+          raise BrowserError, error
         end
       end
     end

data/lib/ferrum/browser/command.rb

@@ -5,7 +5,7 @@ module Ferrum
     class Command
       NOT_FOUND = "Could not find an executable for the browser. Try to make " \
                   "it available on the PATH or set environment variable for " \
-                  "example BROWSER_PATH=\"/usr/bin/chrome\"".freeze
+                  "example BROWSER_PATH=\"/usr/bin/chrome\""
 
       # Currently only these browsers support CDP:
       # https://github.com/cyrus-and/chrome-remote-interface#implementations
@@ -27,9 +27,11 @@ module Ferrum
       def initialize(defaults, options, user_data_dir)
         @flags = {}
         @defaults = defaults
-        @options, @user_data_dir = options, user_data_dir
-        @path = options[:browser_path] || ENV["BROWSER_PATH"] || defaults.detect_path
-        raise Cliver::Dependency::NotFound.new(NOT_FOUND) unless @path
+        @options = options
+        @user_data_dir = user_data_dir
+        @path = options[:browser_path] || ENV.fetch("BROWSER_PATH", nil) || defaults.detect_path
+        raise BinaryNotFoundError, NOT_FOUND unless @path
+
         merge_options
       end
 
@@ -45,10 +47,7 @@ module Ferrum
 
       def merge_options
         @flags = defaults.merge_required(@flags, options, @user_data_dir)
-
-        unless options[:ignore_default_browser_options]
-          @flags = defaults.merge_default(@flags, options)
-        end
+        @flags = defaults.merge_default(@flags, options) unless options[:ignore_default_browser_options]
 
         @flags.merge!(options.fetch(:browser_options, {}))
       end

data/lib/ferrum/browser/options/base.rb

@@ -24,13 +24,7 @@ module Ferrum
         end
 
         def detect_path
-          if Ferrum.mac?
-            self.class::MAC_BIN_PATH.find { |n| File.exist?(n) }
-          else
-            self.class::LINUX_BIN_PATH.find do |name|
-              path = Cliver.detect(name) and break(path)
-            end
-          end
+          Binary.find(self.class::PLATFORM_PATH[Utils::Platform.name])
         end
 
         def merge_required(flags, options, user_data_dir)