ferrum 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3062988d83fbb92a5b52b9037fdc73a66f79cefe7e8c46e776642c805b9595ba
+  data.tar.gz: 150abbfd5781c22988db1d15432a9ee45a0180c4ca81691ecdef1ca8febcfcd2
+SHA512:
+  metadata.gz: 6b2bb8783725639bc80103673725ad9d9573d58ab2ab276ab8a9347867a6782eca177195a9418f4dee64aa1b91aa1c196b423b2e097106aa78513df7659a4d93
+  data.tar.gz: 282beb51703405e569501a2e0c17f1a85aae7b03ee9342df90bca056232e9c60b9f31cbfdd27b8b947999731b759f0f7e39f1bc505c0ddad100af3f2dd52e8e5

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Dmitry Vorotilin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,28 @@
+# Ferrum - fearless Ruby Chrome/Chromium driver.
+
+As simple as Puppeteer, though even simpler. It is Ruby clean and high-level API
+to Chrome/Chromium through the DevTools Protocol. Runs headless by default,
+but you can configure it to run in a non-headless mode.
+
+Navigate to `example.com` and save a screenshot:
+
+```ruby
+browser = Ferrum::Browser.new
+browser.goto("https://example.com")
+browser.screenshot(path: "example.png")
+browser.quit
+```
+
+Interact with a page:
+
+```ruby
+browser = Ferrum::Browser.new
+browser.goto("https://google.com")
+input = browser.at_css("input[title='Search']")
+input.send_keys("Ruby headless driver for Capybara")
+input.send_keys(:Enter)
+browser.at_css("a > h3").text # => "machinio/cuprite: Headless Chrome driver for Capybara - GitHub"
+browser.quit
+```
+
+The README will be updated soon. Meanwhile take a look at specs.

data/lib/ferrum.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+Thread.abort_on_exception = true
+Thread.report_on_exception = true if Thread.respond_to?(:report_on_exception=)
+
+module Ferrum
+  require "ferrum/browser"
+  require "ferrum/node"
+  require "ferrum/errors"
+  require "ferrum/cookie"
+
+  class << self
+    def windows?
+      RbConfig::CONFIG["host_os"] =~ /mingw|mswin|cygwin/
+    end
+
+    def mac?
+      RbConfig::CONFIG["host_os"] =~ /darwin/
+    end
+
+    def mri?
+      defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby"
+    end
+  end
+end

data/lib/ferrum/browser.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+require "base64"
+require "forwardable"
+require "ferrum/page"
+require "ferrum/targets"
+require "ferrum/browser/api"
+require "ferrum/browser/process"
+require "ferrum/browser/client"
+
+module Ferrum
+  class Browser
+    TIMEOUT = 5
+    WINDOW_SIZE = [1024, 768].freeze
+    BASE_URL_SCHEMA = %w[http https].freeze
+
+    include API
+    extend Forwardable
+
+    attr_reader :headers, :window_size
+
+    delegate on: :@client
+    delegate %i(window_handle window_handles switch_to_window open_new_window
+                close_window within_window page) => :targets
+    delegate %i(goto status body at_css at_xpath css xpath text property attributes attribute select_file
+                value visible? disabled? network_traffic clear_network_traffic
+                path response_headers refresh click right_click double_click
+                hover set click_coordinates select trigger scroll_to send_keys
+                evaluate evaluate_on evaluate_async execute frame_url
+                frame_title switch_to_frame current_url title go_back
+                go_forward find_modal accept_confirm dismiss_confirm
+                accept_prompt dismiss_prompt reset_modals authorize
+                proxy_authorize) => :page
+
+    attr_reader :process, :logger, :js_errors, :slowmo, :base_url,
+                :url_blacklist, :url_whitelist, :options
+    attr_writer :timeout
+
+    def initialize(options = nil)
+      options ||= {}
+
+      @client = nil
+      @window_size = options.fetch(:window_size, WINDOW_SIZE)
+      @original_window_size = @window_size
+
+      @options = Hash(options.merge(window_size: @window_size))
+      @logger, @timeout = @options.values_at(:logger, :timeout)
+      @js_errors = @options.fetch(:js_errors, false)
+      @slowmo = @options[:slowmo].to_i
+
+      if @options.key?(:base_url)
+        self.base_url = @options[:base_url]
+      end
+
+      self.url_blacklist = @options[:url_blacklist]
+      self.url_whitelist = @options[:url_whitelist]
+
+      if ENV["FERRUM_DEBUG"] && !@logger
+        STDOUT.sync = true
+        @logger = STDOUT
+        @options[:logger] = @logger
+      end
+
+      @options.freeze
+
+      start
+    end
+
+    def base_url=(value)
+      parsed = Addressable::URI.parse(value)
+      unless BASE_URL_SCHEMA.include?(parsed.normalized_scheme)
+        raise "Set `base_url` should be absolute and include schema: #{BASE_URL_SCHEMA}"
+      end
+
+      @base_url = parsed
+    end
+
+    def extensions
+      @extensions ||= Array(@options[:extensions]).map { |p| File.read(p) }
+    end
+
+    def timeout
+      @timeout || TIMEOUT
+    end
+
+    def command(*args)
+      @client.command(*args)
+    rescue DeadBrowser
+      restart
+      raise
+    end
+
+    def set_overrides(user_agent: nil, accept_language: nil, platform: nil)
+      options = Hash.new
+      options[:userAgent] = user_agent if user_agent
+      options[:acceptLanguage] = accept_language if accept_language
+      options[:platform] if platform
+
+      page.command("Network.setUserAgentOverride", **options) if !options.empty?
+    end
+
+    def clear_memory_cache
+      page.command("Network.clearBrowserCache")
+    end
+
+    def reset
+      @headers = {}
+      @zoom_factor = nil
+      @window_size = @original_window_size
+      targets.reset
+    end
+
+    def restart
+      quit
+      start
+    end
+
+    def quit
+      @client.close
+      @process.stop
+      @client = @process = @targets = nil
+    end
+
+    def targets
+      @targets ||= Targets.new(self)
+    end
+
+    def resize(**options)
+      @window_size = [options[:width], options[:height]]
+      page.resize(**options)
+    end
+
+    def crash
+      command("Browser.crash")
+    end
+
+    private
+
+    def start
+      @headers = {}
+      @process = Process.start(@options)
+      @client = Client.new(self, @process.ws_url, 0, false)
+    end
+  end
+end

data/lib/ferrum/browser/api.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "ferrum/browser/api/cookie"
+require "ferrum/browser/api/header"
+require "ferrum/browser/api/screenshot"
+require "ferrum/browser/api/intercept"
+
+module Ferrum
+  class Browser
+    module API
+      include Cookie, Header, Screenshot, Intercept
+    end
+  end
+end

data/lib/ferrum/browser/api/cookie.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    module API
+      module Cookie
+        def cookies
+          cookies = page.command("Network.getAllCookies")["cookies"]
+          cookies.map { |c| [c["name"], ::Ferrum::Cookie.new(c)] }.to_h
+        end
+
+        def set_cookie(name: nil, value: nil, cookie: nil, **options)
+          cookie = options.dup
+          cookie[:name]   ||= name
+          cookie[:value]  ||= value
+          cookie[:domain] ||= default_domain
+
+          expires = cookie.delete(:expires).to_i
+          cookie[:expires] = expires if expires > 0
+
+          page.command("Network.setCookie", **cookie)
+        end
+
+        # Supports :url, :domain and :path options
+        def remove_cookie(name:, **options)
+          raise "Specify :domain or :url option" if !options[:domain] && !options[:url] && !default_domain
+
+          options = options.merge(name: name)
+          options[:domain] ||= default_domain
+
+          page.command("Network.deleteCookies", **options)
+        end
+
+        def clear_cookies
+          page.command("Network.clearBrowserCookies")
+        end
+
+        private
+
+        def default_domain
+          URI.parse(base_url).host if base_url
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/api/header.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    module API
+      module Header
+        def headers=(headers)
+          @headers = {}
+          add_headers(headers)
+        end
+
+        def add_headers(headers, permanent: true)
+          if headers["Referer"]
+            page.referrer = headers["Referer"]
+            headers.delete("Referer") unless permanent
+          end
+
+          @headers.merge!(headers)
+          user_agent = @headers["User-Agent"]
+          accept_language = @headers["Accept-Language"]
+
+          set_overrides(user_agent: user_agent, accept_language: accept_language)
+          page.command("Network.setExtraHTTPHeaders", headers: @headers)
+        end
+
+        def add_header(header, permanent: true)
+          add_headers(header, permanent: permanent)
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/api/intercept.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    module API
+      module Intercept
+        def url_whitelist=(wildcards)
+          @url_whitelist = prepare_wildcards(wildcards)
+          page.intercept_request("*") if @client && !@url_whitelist.empty?
+        end
+
+        def url_blacklist=(wildcards)
+          @url_blacklist = prepare_wildcards(wildcards)
+          page.intercept_request("*") if @client && !@url_blacklist.empty?
+        end
+
+        private
+
+        def prepare_wildcards(wc)
+          Array(wc).map do |wildcard|
+            if wildcard.is_a?(Regexp)
+              wildcard
+            else
+              wildcard = wildcard.gsub("*", ".*")
+              Regexp.new(wildcard, Regexp::IGNORECASE)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/api/screenshot.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Browser
+    module API
+      module Screenshot
+        def screenshot(**opts)
+          encoding, path, options = screenshot_options(**opts)
+
+          data = if options[:format].to_s == "pdf"
+            options = {}
+            options[:paperWidth] = @paper_size[:width].to_f if @paper_size
+            options[:paperHeight] = @paper_size[:height].to_f if @paper_size
+            options[:scale] = @zoom_factor if @zoom_factor
+            page.command("Page.printToPDF", **options)
+          else
+            page.command("Page.captureScreenshot", **options)
+          end.fetch("data")
+
+          return data if encoding == :base64
+
+          bin = Base64.decode64(data)
+          File.open(path.to_s, "wb") { |f| f.write(bin) }
+        end
+
+        def zoom_factor=(value)
+          @zoom_factor = value.to_f
+        end
+
+        def paper_size=(value)
+          @paper_size = value
+        end
+
+        private
+
+        def screenshot_options(encoding: :base64, format: nil, path: nil, **opts)
+          options = {}
+
+          encoding = :binary if path
+
+          if encoding == :binary && !path
+            raise "Not supported option `:path` #{path}. Should be path to file"
+          end
+
+          format ||= path ? File.extname(path).delete(".") : "png"
+          format = "jpeg" if format == "jpg"
+          raise "Not supported options `:format` #{format}. jpeg | png | pdf" if format !~ /jpeg|png|pdf/i
+          options.merge!(format: format)
+
+          options.merge!(quality: opts[:quality] ? opts[:quality] : 75) if format == "jpeg"
+
+          if !!opts[:full] && opts[:selector]
+            warn "Ignoring :selector in #screenshot since full: true was given at #{caller(1..1).first}"
+          end
+
+          if !!opts[:full]
+            width, height = page.evaluate("[document.documentElement.offsetWidth, document.documentElement.offsetHeight]")
+            options.merge!(clip: { x: 0, y: 0, width: width, height: height, scale: @zoom_factor || 1.0 }) if width > 0 && height > 0
+          elsif opts[:selector]
+            rect = page.evaluate("document.querySelector('#{opts[:selector]}').getBoundingClientRect()")
+            options.merge!(clip: { x: rect["x"], y: rect["y"], width: rect["width"], height: rect["height"], scale: @zoom_factor || 1.0 })
+          end
+
+          if @zoom_factor
+            if !options[:clip]
+              width, height = page.evaluate("[document.documentElement.clientWidth, document.documentElement.clientHeight]")
+              options[:clip] = { x: 0, y: 0, width: width, height: height }
+            end
+
+            options[:clip].merge!(scale: @zoom_factor)
+          end
+
+          [encoding, path, options]
+        end
+      end
+    end
+  end
+end

data/lib/ferrum/browser/client.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "concurrent-ruby"
+require "ferrum/browser/subscriber"
+require "ferrum/browser/web_socket"
+
+module Ferrum
+  class Browser
+    class Client
+      def initialize(browser, ws_url, start_id = 0, allow_slowmo = true)
+        @command_id = start_id
+        @pendings = Concurrent::Hash.new
+        @browser = browser
+        @slowmo = @browser.slowmo if allow_slowmo && @browser.slowmo > 0
+        @ws = WebSocket.new(ws_url, @browser.logger)
+        @subscriber = Subscriber.new
+
+        @thread = Thread.new do
+          while message = @ws.messages.pop
+            if message.key?("method")
+              @subscriber.async.call(message)
+            else
+              @pendings[message["id"]]&.set(message)
+            end
+          end
+        end
+      end
+
+      def command(method, params = {})
+        pending = Concurrent::IVar.new
+        message = build_message(method, params)
+        @pendings[message[:id]] = pending
+        sleep(@slowmo) if @slowmo
+        @ws.send_message(message)
+        data = pending.value!(@browser.timeout)
+        @pendings.delete(message[:id])
+
+        raise DeadBrowser if data.nil? && @ws.messages.closed?
+        raise TimeoutError unless data
+        error, response = data.values_at("error", "result")
+        raise BrowserError.new(error) if error
+        response
+      end
+
+      def on(event, &block)
+        @subscriber.on(event, &block)
+      end
+
+      def close
+        @ws.close
+        # Give a thread some time to handle a tail of messages
+        @pendings.clear
+        Timeout.timeout(1) { @thread.join }
+      rescue Timeout::Error
+        @thread.kill
+      end
+
+      private
+
+      def build_message(method, params)
+        { method: method, params: params }.merge(id: next_command_id)
+      end
+
+      def next_command_id
+        @command_id += 1
+      end
+    end
+  end
+end