rubium 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 124da1f21fad244bbdeb8adee803043896502e4c58964753bbba6ea8083df750
+  data.tar.gz: 13cda3fb2bd4f121700d08f03a8fbc3b15e38b785dca30074afa13121f4592b0
+SHA512:
+  metadata.gz: e724053ddf9d97bbaf77db2c3647e1988c5bbace1a56b2f55f72afde1758ace0aaabd1f9d1771647361f8a863a6c33c35a8b616a7b8c4bd4f61278a750b33af0
+  data.tar.gz: 8decfacf86c9751c6ae39e61cae9b8fa1b29a3cb769f8ac3e11e15122364173bf0d20e649e058fe150023e1a7ea38ac13fd919d14abddef6fbfa961d64ca6987

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in rubium.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Victor Afanasev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,81 @@
+# Rubium
+
+Rubium is a handy wrapper around [chrome_remote](https://github.com/cavalle/chrome_remote) gem. It adds browsers instances handling, and some Capybara-like methods. It is very lightweight (200 lines of code in the main `Rubium::Browser` class for now) and doens't use Selenium or Capybara.
+
+You can use Rubium as a lightweight alternative to Selenium/Capybara/Watir if you need to perform some operations (like web scraping) using Headless Chromium and Ruby. Of course, the API currently doesn't has a lot of methods to automate browser, but it has the most frequently used and basic ones.
+
+```ruby
+require 'rubium'
+
+browser = Rubium::Browser.new
+browser.visit("https://github.com/vifreefly/rubium")
+
+# Get current page response as string:
+browser.body
+
+# Get current page response as Nokogiri object:
+browser.current_response
+
+# Click to the some element (css selector):
+browser.click("some selector")
+
+# Get current cookies:
+browser.cookies
+
+# Fill in some field:
+browser.fill_in("some field selector", "Some text")
+
+# Tells if current response has provided css selector or not. You can
+# provide optional `wait:` argument (in seconds) to set the max wait time for the selector:
+browser.has_css?("some selector", wait: 1)
+
+# Tells if current response has provided text or not. You can
+# provide optional `wait:` argument (in seconds) to set the max wait time for the text:
+browser.has_text?("some text")
+
+# Evaluate some JS code on a new tab:
+browser.evaluate_on_new_document(File.read "browser_inject.js")
+
+# Evaluate JS code expression:
+browser.execute_script("JS code string")
+
+# Access chrome_remote client directly:
+browser.client
+
+# Close browser:
+browser.close
+
+# Restart browser:
+browser.restart!
+```
+
+There are some options which you can provide while creating browser instance:
+
+```ruby
+browser = Rubium::Browser.new(
+  debugging_port: 9222, # custom debugging port
+  headless: false, # Run browser in normal (not headless) mode
+  user_agent: "Some user agent", # Custom user-agent
+  proxy_server: "http://1.1.1.1:8080", # Set proxy
+)
+```
+
+You can provide custom Chrome binary path this way:
+
+```ruby
+Rubium.configure do |config|
+  config.chrome_path = "/path/to/chrome/binary"
+end
+```
+
+
+## Installation
+Rubium tested with `2.3.0` Ruby version and up.
+
+Rubium is in the alpha stage (and therefore will have breaking updates in the future), so it's recommended to hard-code latest gem version in your Gemfile, like: `gem 'rubium', '0.1.0'`.
+
+## Contribution
+Sure, feel free to fork and add new functionality.
+
+## License
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "rubium"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/rubium.rb

@@ -0,0 +1,42 @@
+require 'ostruct'
+require 'rubium/version'
+require 'rubium/browser'
+
+module Rubium
+  DEFAULT_PUPPETEER_ARGS = %w(
+    --disable-background-networking
+    --disable-background-timer-throttling
+    --disable-backgrounding-occluded-windows
+    --disable-breakpad
+    --disable-client-side-phishing-detection
+    --disable-default-apps
+    --disable-dev-shm-usage
+    --disable-extensions
+    --disable-features=site-per-process
+    --disable-hang-monitor
+    --disable-ipc-flooding-protection
+    --disable-popup-blocking
+    --disable-prompt-on-repost
+    --disable-renderer-backgrounding
+    --disable-sync
+    --disable-translate
+    --metrics-recording-only
+    --no-first-run
+    --safebrowsing-disable-auto-update
+    --enable-automation
+    --password-store=basic
+    --use-mock-keychain
+    --hide-scrollbars
+    --mute-audio
+    --no-sandbox
+    --disable-infobars
+  ).freeze
+
+  def self.configuration
+    @configuration ||= OpenStruct.new
+  end
+
+  def self.configure
+    yield(configuration)
+  end
+end

data/lib/rubium/browser.rb

@@ -0,0 +1,206 @@
+require 'chrome_remote'
+require 'nokogiri'
+require 'random-port'
+require 'cliver'
+require 'timeout'
+require 'securerandom'
+
+at_exit do
+  Rubium::Browser.running_pids.each { |pid| Process.kill("HUP", pid) }
+end
+
+module Rubium
+  class Browser
+    class ConfigurationError < StandardError; end
+
+    MAX_CONNECT_WAIT_TIME = 2
+
+    class << self
+      def ports_pool
+        @pool ||= RandomPort::Pool.new
+      end
+
+      def running_pids
+        @running_pids ||= []
+      end
+    end
+
+    attr_reader  :client, :devtools_url, :pid, :port, :options
+
+    def initialize(options = {})
+      @options = options
+      create_browser
+    end
+
+    def restart!
+      close
+      create_browser
+    end
+
+    def close
+      unless closed?
+        Process.kill("HUP", @pid)
+        self.class.running_pids.delete(@pid)
+        self.class.ports_pool.release(@port)
+
+        FileUtils.rm_rf(@data_dir) if Dir.exist?(@data_dir)
+        @closed = true
+      end
+    end
+
+    alias_method :destroy_driver!, :close
+
+    def closed?
+      @closed
+    end
+
+    def goto(url, wait: 30)
+      response = @client.send_cmd "Page.navigate", url: url
+
+      if wait
+        Timeout.timeout(wait) { @client.wait_for "Page.loadEventFired" }
+      else
+        response
+      end
+    end
+
+    alias_method :visit, :goto
+
+    def body
+      response = @client.send_cmd "Runtime.evaluate", expression: 'document.documentElement.outerHTML'
+      response.dig("result", "value")
+    end
+
+    def current_response
+      Nokogiri::HTML(body)
+    end
+
+    def has_xpath?(path, wait: 0)
+      timer = 0
+      until current_response.at_xpath(path)
+        return false if timer >= wait
+        timer += 0.2 and sleep 0.2
+      end
+
+      true
+    end
+
+    def has_css?(selector, wait: 0)
+      timer = 0
+      until current_response.at_css(selector)
+        return false if timer >= wait
+        timer += 0.2 and sleep 0.2
+      end
+
+      true
+    end
+
+    def has_text?(text, wait: 0)
+      timer = 0
+      until body&.include?(text)
+        return false if timer >= wait
+        timer += 0.2 and sleep 0.2
+      end
+
+      true
+    end
+
+    def click(selector)
+      @client.send_cmd "Runtime.evaluate", expression: <<~JS
+        document.querySelector("#{selector}").click();
+      JS
+    end
+
+    # https://github.com/cyrus-and/chrome-remote-interface/issues/226#issuecomment-320247756
+    # https://stackoverflow.com/a/18937620
+    def send_key_on(selector, key)
+      @client.send_cmd "Runtime.evaluate", expression: <<~JS
+        document.querySelector("#{selector}").dispatchEvent(
+          new KeyboardEvent("keydown", {
+            bubbles: true, cancelable: true, keyCode: #{key}
+          })
+        );
+      JS
+    end
+
+    # https://github.com/GoogleChrome/puppeteer/blob/master/lib/Page.js#L784
+    # https://stackoverflow.com/questions/46113267/how-to-use-evaluateonnewdocument-and-exposefunction
+    # https://chromedevtools.github.io/devtools-protocol/tot/Page/#method-addScriptToEvaluateOnNewDocument
+    def evaluate_on_new_document(script)
+      @client.send_cmd "Page.addScriptToEvaluateOnNewDocument", source: script
+    end
+
+    def cookies
+      response = @client.send_cmd "Network.getCookies"
+      response["cookies"]
+    end
+
+    def fill_in(selector, text)
+      execute_script <<~HEREDOC
+        document.querySelector("#{selector}").value = "#{text}"
+      HEREDOC
+    end
+
+    def execute_script(script)
+      @client.send_cmd "Runtime.evaluate", expression: script
+    end
+
+    private
+
+    def create_browser
+      @port = options[:debugging_port] || self.class.ports_pool.acquire
+      @data_dir = "/tmp/rubium_profile_#{SecureRandom.hex}"
+
+      chrome_path = Rubium.configuration.chrome_path ||
+        Cliver.detect("chromium-browser") ||
+        Cliver.detect("google-chrome")
+      raise ConfigurationError, "Can't find chrome executable" unless chrome_path
+
+      command = %W(
+        #{chrome_path} about:blank
+        --remote-debugging-port=#{@port}
+        --user-data-dir=#{@data_dir}
+      ) + DEFAULT_PUPPETEER_ARGS
+
+      command << "--headless" if ENV["HEADLESS"] != "false" && options[:headless] != false
+      command << "--window-size=#{options[:window_size].join(',')}" if options[:window_size]
+
+      if options[:user_agent]
+        user_agent = options[:user_agent].respond_to?(:call) ? options[:user_agent].call : options[:user_agent]
+        command << "--user-agent=#{user_agent}"
+      end
+
+      if options[:proxy_server]
+        proxy_server = options[:proxy_server].respond_to?(:call) ? options[:proxy_server].call : options[:proxy_server]
+        proxy_server = convert_proxy(proxy_server) unless proxy_server.include?("://")
+        command << "--proxy-server=#{proxy_server}"
+      end
+
+      @pid = spawn(*command, [:out, :err] => "/dev/null")
+      self.class.running_pids << @pid
+      @closed = false
+
+      counter = 0
+      begin
+        counter += 0.2 and sleep 0.2
+        @client = ChromeRemote.client(port: @port)
+      rescue Errno::ECONNREFUSED => e
+        counter < MAX_CONNECT_WAIT_TIME ? retry : raise(e)
+      end
+
+      @devtools_url = "http://localhost:#{@port}/"
+
+      # https://github.com/GoogleChrome/puppeteer/blob/master/lib/Page.js
+      @client.send_cmd "Target.setAutoAttach", autoAttach: true, waitForDebuggerOnStart: false
+      @client.send_cmd "Network.enable"
+      @client.send_cmd "Page.enable"
+
+      evaluate_on_new_document(options[:extension_code]) if options[:extension_code]
+    end
+
+    def convert_proxy(proxy_string)
+      ip, port, type, user, password = proxy_string.split(":")
+      "#{type}://#{ip}:#{port}"
+    end
+  end
+end

data/lib/rubium/version.rb

@@ -0,0 +1,3 @@
+module Rubium
+  VERSION = "0.1.0"
+end

data/rubium.gemspec

@@ -0,0 +1,32 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "rubium/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "rubium"
+  spec.version       = Rubium::VERSION
+  spec.authors       = ["Victor Afanasev"]
+  spec.email         = ["vicfreefly@gmail.com"]
+
+  spec.summary       = "Headless Chromium Ruby API based on ChromeRemote gem"
+  spec.description   = spec.summary
+  spec.homepage      = "https://github.com/vifreefly/rubium"
+  spec.license       = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "chrome_remote", "~> 0.2"
+  spec.add_dependency "cliver", "~> 0.3"
+  spec.add_dependency "random-port"
+  spec.add_dependency "nokogiri"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

metadata

@@ -0,0 +1,153 @@
+--- !ruby/object:Gem::Specification
+name: rubium
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Victor Afanasev
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-12-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: chrome_remote
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: cliver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: random-port
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+description: Headless Chromium Ruby API based on ChromeRemote gem
+email:
+- vicfreefly@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/rubium.rb
+- lib/rubium/browser.rb
+- lib/rubium/version.rb
+- rubium.gemspec
+homepage: https://github.com/vifreefly/rubium
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Headless Chromium Ruby API based on ChromeRemote gem
+test_files: []