rubium 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 124da1f21fad244bbdeb8adee803043896502e4c58964753bbba6ea8083df750
-  data.tar.gz: 13cda3fb2bd4f121700d08f03a8fbc3b15e38b785dca30074afa13121f4592b0
+  metadata.gz: e10835f0d5dff1baa1cc0e65f3a916372c9f97ecf02c80ae337b192aca99c082
+  data.tar.gz: 5b5d44b8d329282fe8dfef8d7867ed9d1d57174e7fdc4d1ecc07180ba720e491
 SHA512:
-  metadata.gz: e724053ddf9d97bbaf77db2c3647e1988c5bbace1a56b2f55f72afde1758ace0aaabd1f9d1771647361f8a863a6c33c35a8b616a7b8c4bd4f61278a750b33af0
-  data.tar.gz: 8decfacf86c9751c6ae39e61cae9b8fa1b29a3cb769f8ac3e11e15122364173bf0d20e649e058fe150023e1a7ea38ac13fd919d14abddef6fbfa961d64ca6987
+  metadata.gz: 56fdf04341dbf6b119d5682ad80d69beafe9147552bb26f53ba1c94e40ba509c7846d669de16b79f2ca667c18fceb8dc3d540d0922de934269ecc885f1a549c1
+  data.tar.gz: 611689e94f1846eb5b1f25a2d22b65666c34a28ab4d95239015f575c26f30f4e8dc9b4931aa8ebbb003a5a7f8a29224019036c3f49934d2f3e2c745fa63b4114

data/.gitignore

@@ -1,4 +1,5 @@
 /.bundle/
+/.claude/
 /.yardoc
 /_yardoc/
 /coverage/

data/CHANGELOG.md

@@ -0,0 +1,28 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.2.1] - 2025-12-07
+
+### Added
+- Added CHANGELOG.md to track version changes
+- Require fileutils to make sure fileutils included in newer ruby versions
+- Update dependencies 
+
+### Changed
+- Updated MAX_CONNECT_WAIT_TIME to 6 seconds
+- Added option to provide custom data-dir path
+
+## [0.2.0] - 2019-02-24
+
+- Added logging support
+- Added cookies and restart_after options
+- Added urls_blacklist and disable_images options
+- Various bug fixes and improvements
+
+## [0.1.0] - 2019-01-XX
+
+- Initial release

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2018 Victor Afanasev
+Copyright (c) 2025 Victor Afanasev
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -1,6 +1,8 @@
 # Rubium
 
-Rubium is a handy wrapper around [chrome_remote](https://github.com/cavalle/chrome_remote) gem. It adds browsers instances handling, and some Capybara-like methods. It is very lightweight (200 lines of code in the main `Rubium::Browser` class for now) and doens't use Selenium or Capybara.
+## Description
+
+Rubium is a handy wrapper around [chrome_remote](https://github.com/cavalle/chrome_remote) gem. It adds browsers instances handling, and some Capybara-like methods. It is very lightweight (250 lines of code in the main `Rubium::Browser` class for now) and doens't use Selenium or Capybara. Consider Rubium as a _very simple_ and _basic_ implementation of [Puppeteer](https://github.com/GoogleChrome/puppeteer) in Ruby language.
 
 You can use Rubium as a lightweight alternative to Selenium/Capybara/Watir if you need to perform some operations (like web scraping) using Headless Chromium and Ruby. Of course, the API currently doesn't has a lot of methods to automate browser, but it has the most frequently used and basic ones.
 
@@ -22,6 +24,12 @@ browser.click("some selector")
 # Get current cookies:
 browser.cookies
 
+# Set cookies (Array of hashes):
+browser.set_cookies([
+  { name: "some_cookie_name", value: "some_cookie_value", domain: ".some-cookie-domain.com" },
+  { name: "another_cookie_name", value: "another_cookie_value", domain: ".another-cookie-domain.com" }
+])
+
 # Fill in some field:
 browser.fill_in("some field selector", "Some text")
 
@@ -39,7 +47,8 @@ browser.evaluate_on_new_document(File.read "browser_inject.js")
 # Evaluate JS code expression:
 browser.execute_script("JS code string")
 
-# Access chrome_remote client directly:
+# Access chrome_remote client (instance of ChromeRemote class) directly:
+# See more here: https://github.com/cavalle/chrome_remote#using-the-chromeremote-api
 browser.client
 
 # Close browser:
@@ -49,18 +58,42 @@ browser.close
 browser.restart!
 ```
 
-There are some options which you can provide while creating browser instance:
+**There are some options** which you can provide while creating browser instance:
 
 ```ruby
 browser = Rubium::Browser.new(
-  debugging_port: 9222, # custom debugging port
-  headless: false, # Run browser in normal (not headless) mode
-  user_agent: "Some user agent", # Custom user-agent
-  proxy_server: "http://1.1.1.1:8080", # Set proxy
+  debugging_port: 9222,                  # custom debugging port. Default is any available port.
+  headless: false,                       # Run browser in normal (not headless) mode. Default is headless.
+  window_size: [1600, 900],              # Custom window size. Default is unset.
+  user_agent: "Some user agent",         # Custom user-agent.
+  proxy_server: "http://1.1.1.1:8080",   # Set proxy.
+  extension_code: "Some JS code string", # Inject custom JS code on each page. See above `evaluate_on_new_document`
+  cookies: [],                           # Set custom cookies, see above `set_cookies`
+  restart_after: 25,                     # Automatically restart browser after N processed requests
+  enable_logger: true,                   # Enable logger to log info about processing requests
+  max_timeout: 30,                       # How long to wait (in seconds) until page will be fully loaded. Default 60 sec.
+  urls_blacklist: ["*some-domain.com*"], # Skip all requests which match provided patterns (wildcard allowed).
+  disable_images: true                   # Do not download images.
+)
+```
+
+Note that for options `user_agent` and `proxy_server` you can provide `lambda` object instead of string:
+
+```ruby
+USER_AGENTS = ["Safari", "Mozilla", "IE", "Chrome"]
+PROXIES = ["http://1.1.1.1:8080", "http://2.2.2.2:8080", "http://3.3.3.3:8080"]
+
+browser = Rubium::Browser.new(
+  user_agent:   -> { USER_AGENTS.sample },
+  proxy_server: -> { PROXIES.sample },
+  restart_after: 25
 )
 ```
 
-You can provide custom Chrome binary path this way:
+> What for: Chrome doesn't provide an API to change proxies on the fly (after browser has been started). It is possible to set proxy while starting Chrome instance by providing CLI argument only. On the other hand, Rubium allows you to automatically restart browser (`restart_after` option) after N processed requests. On each restart, if options `user_agent` and/or `proxy_server` has lambda format, then lambda will be called to fetch fresh value. Thus it's possible to rotate proxies/user-agents without any much effort.
+
+
+**You can provide custom Chrome binary** path this way:
 
 ```ruby
 Rubium.configure do |config|
@@ -68,12 +101,18 @@ Rubium.configure do |config|
 end
 ```
 
+Common Chrome path example for MacOS:
+
+```ruby
+Rubium.configure do |config|
+  config.chrome_path = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+end
+```
+
 
 ## Installation
 Rubium tested with `2.3.0` Ruby version and up.
 
-Rubium is in the alpha stage (and therefore will have breaking updates in the future), so it's recommended to hard-code latest gem version in your Gemfile, like: `gem 'rubium', '0.1.0'`.
-
 ## Contribution
 Sure, feel free to fork and add new functionality.
 

data/lib/rubium/browser.rb

@@ -4,6 +4,8 @@ require 'random-port'
 require 'cliver'
 require 'timeout'
 require 'securerandom'
+require 'logger'
+require 'fileutils'
 
 at_exit do
   Rubium::Browser.running_pids.each { |pid| Process.kill("HUP", pid) }
@@ -13,7 +15,8 @@ module Rubium
   class Browser
     class ConfigurationError < StandardError; end
 
-    MAX_CONNECT_WAIT_TIME = 2
+    MAX_CONNECT_WAIT_TIME = 6
+    MAX_DEFAULT_TIMEOUT = 60
 
     class << self
       def ports_pool
@@ -25,25 +28,40 @@ module Rubium
       end
     end
 
-    attr_reader  :client, :devtools_url, :pid, :port, :options
+    attr_reader  :client, :devtools_url, :pid, :port, :options, :processed_requests_count, :logger
 
     def initialize(options = {})
       @options = options
+
+      if @options[:enable_logger]
+        @logger = Logger.new(STDOUT)
+        @logger.progname = self.class.to_s
+      end
+
       create_browser
     end
 
     def restart!
+      logger.info "Restarting..." if options[:enable_logger]
+
       close
       create_browser
     end
 
     def close
-      unless closed?
+      if closed?
+        logger.info "Browser already has been closed" if options[:enable_logger]
+      else
         Process.kill("HUP", @pid)
         self.class.running_pids.delete(@pid)
         self.class.ports_pool.release(@port)
 
-        FileUtils.rm_rf(@data_dir) if Dir.exist?(@data_dir)
+        # Delete temp profile directory, if there is no custom one
+        unless options[:data_dir]
+          FileUtils.rm_rf(@data_dir) if Dir.exist?(@data_dir)
+        end
+
+        logger.info "Closed browser" if options[:enable_logger]
         @closed = true
       end
     end
@@ -54,14 +72,28 @@ module Rubium
       @closed
     end
 
-    def goto(url, wait: 30)
+    def goto(url, wait: options[:max_timeout] || MAX_DEFAULT_TIMEOUT)
+      logger.info "Started request: #{url}" if options[:enable_logger]
+      if options[:restart_after] && processed_requests_count >= options[:restart_after]
+        restart!
+      end
+
       response = @client.send_cmd "Page.navigate", url: url
 
-      if wait
-        Timeout.timeout(wait) { @client.wait_for "Page.loadEventFired" }
-      else
-        response
+      # By default, after Page.navigate we should wait till page will load completely
+      # using Page.loadEventFired. But on some websites with Ajax navigation, Page.loadEventFired
+      # will stuck forever. In this case you can provide `wait: false` option to skip waiting.
+      if wait != false
+        # https://chromedevtools.github.io/devtools-protocol/tot/Page#event-frameStoppedLoading
+        Timeout.timeout(wait) do
+          @client.wait_for do |event_name, event_params|
+            event_name == "Page.frameStoppedLoading" && event_params["frameId"] == response["frameId"]
+          end
+        end
       end
+
+      @processed_requests_count += 1
+      logger.info "Finished request: #{url}" if options[:enable_logger]
     end
 
     alias_method :visit, :goto
@@ -106,21 +138,21 @@ module Rubium
     end
 
     def click(selector)
-      @client.send_cmd "Runtime.evaluate", expression: <<~JS
+      @client.send_cmd "Runtime.evaluate", expression: <<~js
         document.querySelector("#{selector}").click();
-      JS
+      js
     end
 
     # https://github.com/cyrus-and/chrome-remote-interface/issues/226#issuecomment-320247756
     # https://stackoverflow.com/a/18937620
     def send_key_on(selector, key)
-      @client.send_cmd "Runtime.evaluate", expression: <<~JS
+      @client.send_cmd "Runtime.evaluate", expression: <<~js
         document.querySelector("#{selector}").dispatchEvent(
           new KeyboardEvent("keydown", {
             bubbles: true, cancelable: true, keyCode: #{key}
           })
         );
-      JS
+      js
     end
 
     # https://github.com/GoogleChrome/puppeteer/blob/master/lib/Page.js#L784
@@ -130,15 +162,24 @@ module Rubium
       @client.send_cmd "Page.addScriptToEvaluateOnNewDocument", source: script
     end
 
+    ###
+
     def cookies
       response = @client.send_cmd "Network.getCookies"
       response["cookies"]
     end
 
+    # https://chromedevtools.github.io/devtools-protocol/tot/Network#method-setCookies
+    def set_cookies(cookies)
+      @client.send_cmd "Network.setCookies", cookies: cookies
+    end
+
+    ###
+
     def fill_in(selector, text)
-      execute_script <<~HEREDOC
+      execute_script <<~js
         document.querySelector("#{selector}").value = "#{text}"
-      HEREDOC
+      js
     end
 
     def execute_script(script)
@@ -148,8 +189,11 @@ module Rubium
     private
 
     def create_browser
+      @processed_requests_count = 0
+
       @port = options[:debugging_port] || self.class.ports_pool.acquire
-      @data_dir = "/tmp/rubium_profile_#{SecureRandom.hex}"
+
+      @data_dir = options[:data_dir] || "/tmp/rubium_profile_#{SecureRandom.hex}"
 
       chrome_path = Rubium.configuration.chrome_path ||
         Cliver.detect("chromium-browser") ||
@@ -196,6 +240,26 @@ module Rubium
       @client.send_cmd "Page.enable"
 
       evaluate_on_new_document(options[:extension_code]) if options[:extension_code]
+
+      set_cookies(options[:cookies]) if options[:cookies]
+
+      if options[:urls_blacklist] || options[:disable_images]
+        urls = []
+
+        if options[:urls_blacklist]
+          urls += options[:urls_blacklist]
+        end
+
+        if options[:disable_images]
+          urls += %w(jpg jpeg png gif swf svg tif).map { |ext| ["*.#{ext}", "*.#{ext}?*"] }.flatten
+          urls << "data:image*"
+        end
+
+        @client.send_cmd "Network.setBlockedURLs", urls: urls
+      end
+
+
+      logger.info "Opened browser" if options[:enable_logger]
     end
 
     def convert_proxy(proxy_string)

data/lib/rubium/version.rb

@@ -1,3 +1,3 @@
 module Rubium
-  VERSION = "0.1.0"
+  VERSION = "0.2.1"
 end

data/lib/rubium.rb

@@ -30,6 +30,7 @@ module Rubium
     --mute-audio
     --no-sandbox
     --disable-infobars
+    --disable-blink-features=AutomationControlled
   ).freeze
 
   def self.configuration

data/rubium.gemspec

@@ -21,12 +21,12 @@ Gem::Specification.new do |spec|
   end
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "chrome_remote", "~> 0.2"
+  spec.add_dependency "chrome_remote", "~> 0.3"
   spec.add_dependency "cliver", "~> 0.3"
   spec.add_dependency "random-port"
   spec.add_dependency "nokogiri"
 
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "bundler", "~> 2.4"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "minitest", "~> 5.22"
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: rubium
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Victor Afanasev
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-19 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chrome_remote
@@ -16,14 +15,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: cliver
   requirement: !ruby/object:Gem::Requirement
@@ -72,42 +71,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.22'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '5.22'
 description: Headless Chromium Ruby API based on ChromeRemote gem
 email:
 - vicfreefly@gmail.com
@@ -116,6 +115,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -130,7 +130,6 @@ homepage: https://github.com/vifreefly/rubium
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -145,9 +144,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Headless Chromium Ruby API based on ChromeRemote gem
 test_files: []