robro 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e88a67da2ab2954835ceb48c4780e7c7b919221bf071010566a375cc2b46bf59
-  data.tar.gz: bf448e4ec1218620392f80f3e0a68d5a735b7001ec53bedca348c69403572e43
+  metadata.gz: a1ed8475085bbc598b8d65eb0bd03913e6c1d898156bed192373c9dd4543e025
+  data.tar.gz: b233b37e22fff88c921a08a524a87515805de9327531ecdb8d636cf4ec33e8ea
 SHA512:
-  metadata.gz: e13f9f6c6d1b2b1213dce31b946e4788c6de82844bf743e761cb7b30cf07608b0fcc25245efc0a523b6e6e5011cf400425e2c2993e70c744e89ec88a3e54dd48
-  data.tar.gz: e73077cb7f59764e3355b294a5d44377d158026db6ecd36df0126eea3c87b3f31988300fc241b7d756c67858f2bd4c84a87e459f451968d8f0e4bd7fe853fae5
+  metadata.gz: e826a76fe9e7dce91249b2a0d5bd19495bebe6202e2ff55be5871b6acad9d8c8b3a4a160f03062b5a8b82f72f8c68ec4dfba3e8ce10b592f88fb316f9e0af003
+  data.tar.gz: b4de6981f340ebcc59bf0c709dde1546440a67e6a53137550a58f8d816e19ecac58cf0075da48029c66ec616036dae0e8a24fb626d8691560bd04aeb6732dc95

data/.gitignore

@@ -8,4 +8,3 @@
 /spec/reports/
 /tmp/
 /Gemfile.lock
-/.byebug_history

data/lib/robro/browser/helpers.rb

@@ -0,0 +1,36 @@
+module Robro
+  class Browser
+    module Helpers
+      def remove_elements(css:)
+        page.execute_script "document.querySelectorAll('#{css}').forEach(e => { e.remove() } )"
+      end
+
+      def click_through_overlay(element)
+        element_regexp = {
+          firefox: /^Element (<.*>) is not clickable at point \((\d+),(\d+)\) because another element (?<element><.*>) obscures it$/,
+          chrome: /^element click intercepted: Element (<.*>) is not clickable at point \((\d+), (\d+)\)\. Other element would receive the click: (?<element><.*>)/,
+        }
+        loop do
+          element.click(wait: 1)
+          break
+        rescue StandardError => e
+          m_element = element_regexp[Robro.browser.application].match e.message
+          unless m_element.nil?
+            m_id = /id=\"(?<id>[a-zA-Z0-9-]+)/.match m_element[:element]
+            unless m_id.nil?
+              overlay_element_id = m_id[:id]
+              Robro.logger.debug "Removing '#{overlay_element_id}' that obscures target element"
+              remove_elements css: "##{overlay_element_id}"
+            else
+              Robro.logger.error "Unable to find element ID that obscures target (#{element}) in: '#{m_id[:element]}'"
+              debugger
+            end
+          else
+            Robro.logger.error "Unable to find element that obscures target (#{element}) in: '#{e.message}'"
+            debugger
+          end
+        end
+      end
+    end
+  end
+end

data/lib/robro/browser.rb

@@ -2,32 +2,35 @@ require 'capybara'
 require 'capybara/dsl'
 require 'capybara-screenshot'
 
+require_relative 'browser/helpers'
+
 module Robro
   class Browser
     attr_reader :application
 
     include Capybara::DSL
+    include Robro::Browser::Helpers
 
-    def initialize(application)
+    def initialize(application, headless: false)
       @application = application.to_sym
-      @headless = false # FIXME ENV['GUI'].nil?
+      @headless = headless
 
       Capybara.configure do |config|
         config.run_server = false
         config.default_max_wait_time = 5
       end
 
-      # FIXME Register chrome_jack_headless
       require 'selenium-webdriver'
       Capybara.register_driver :chrome_jack do |app|
-        options = ::Selenium::WebDriver::Chrome::Options.new.tap do |opts|
-          opts.args << '--start-maximized'
-          opts.args << '--disable-blink-features'
-          opts.args << '--disable-blink-features=AutomationControlled'
-          opts.args << '--excludeSwitches=enable-automation'
-          opts.args << '--disable-gpu'
-        end
-        driver = Capybara::Selenium::Driver.new(app, browser: :chrome, capabilities: options)
+        options = ::Selenium::WebDriver::Chrome::Options.new
+        options.add_argument('--headless') if headless
+        options.add_argument('--start-maximized')
+        options.add_argument('--disable-blink-features')
+        options.add_argument('--disable-blink-features=AutomationControlled')
+        options.add_argument('--excludeSwitches=enable-automation')
+        options.add_argument('--disable-gpu')
+
+        driver = Capybara::Selenium::Driver.new(app, browser: :chrome, options: options)
         bridge = driver.browser.send(:bridge)
 
         path = '/session/:session_id/chromium/send_command'
@@ -104,15 +107,13 @@ module Robro
 
       driver_name = case @application
       when :firefox
-        'selenium'
+        @headless ? 'selenium_headless' : 'selenium'
       when :chrome
         'chrome_jack'
       else
         raise "Unsupported browser: '#{application}'"
       end
 
-      driver_name += '_headless' if @headless
-
       driver_name.to_sym
     end
   end

data/lib/robro/cli/helpers.rb

@@ -0,0 +1,24 @@
+require 'tty-cursor'
+
+require 'thor'
+
+module Robro
+  class CLI < Thor
+    module Helpers
+      def self.show_download_progress(changes, details)
+        if changes['filename']
+          puts "Downloading '#{details['filename']}' (size: #{details['size']})"
+          puts "  progress: 0% rate: --- remaining time: ---"
+        elsif changes['progress']
+          print TTY::Cursor.prev_line
+          print TTY::Cursor.clear_line
+          puts "  progress: #{details['progress']} rate: #{details['rate']} remaining time: #{details['remaining_time']}"
+        elsif changes['rate_average']
+          puts "  rate average: #{details['rate_average']}"
+        elsif changes['message']
+          puts "Download message: #{changes['message']}"
+        end
+      end
+    end
+  end
+end

data/lib/robro/cli.rb

@@ -4,8 +4,6 @@ require 'robro/user_scripts'
 require 'thor'
 require 'json'
 
-require 'active_support/inflector'
-
 module Robro
   class CLI < Thor
     desc 'browse URL', 'Start browser at URL, drop a shell'
@@ -18,22 +16,19 @@ module Robro
 
       quit = false
       until quit
-        UserScripts.all.each do |us_class|
-          us = us_class.new
-          next unless uri.host.start_with? *(us.supported_urls)
+        us = UserScripts.find_for uri: uri
 
-          unless us.nil?
-            puts "Commands for this URL: #{us.supported_url_commands}"
+        unless us.nil?
+          puts "Commands for this URL: #{us.supported_url_commands}"
 
-            us.supported_url_commands.each do |command|
-              define_singleton_method command do |*args|
-                us.send(command, *args)
-              end
+          us.supported_url_commands.each do |command|
+            define_singleton_method command do |*args|
+              us.send(command, *args)
             end
           end
         end
 
-        byebug
+        debugger
       end
     end
 
@@ -54,22 +49,20 @@ module Robro
       end
     end
 
-    UserScripts.all.each do |us_class|
-      us = us_class.new
-      us.supported_url_commands.each do |command|
-        desc "#{command} URL", command.to_s
-        method_option :browser, :type => :string, :description => 'Supported values: firefox or chromium', :default => 'chrome'
-        define_method command do |*args|
-          url = args.shift
-          raise Thor::Error, 'URL argument is missing' if url.nil?
+    UserScripts.all_commands.each do |command|
+      desc "#{command} URL", command.to_s
+      method_option :browser, :type => :string, :description => 'Supported values: firefox or chromium', :default => 'firefox'
+      define_method command do |*args|
+        url = args.shift
+        raise Thor::Error, 'URL argument is missing' if url.nil?
 
-          uri = validate_url(url)
+        uri = validate_url(url)
 
-          Robro.browser = Browser.new options[:browser]
-          Robro.browser.visit uri
+        Robro.browser = Browser.new options[:browser]
 
-          puts JSON.pretty_generate(us.send(command, *args))
-        end
+        result = UserScripts.execute command, uri
+
+        puts JSON.pretty_generate(result)
       end
     end
   end

data/lib/robro/download.rb

@@ -0,0 +1,123 @@
+require 'open3'
+
+module Robro
+  class Download
+    class Error < StandardError; end
+
+    attr_reader :details
+
+    def initialize(url)
+      @url = url.sub /^http:/, 'https:'
+      Robro.logger.debug "Attempt to download: #{@url}"
+    end
+
+    FILTERS = [
+      #--2020-11-10 22:30:31--  https://kubuntu.org/wp-content/uploads/2020/10/6ac0/GroovyBannerv4-1024x273.png
+      /^--\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}--  http/,
+      #Resolving kubuntu.org (kubuntu.org)... 162.213.33.214
+      /^Resolving /,
+      #Connecting to kubuntu.org (kubuntu.org)|162.213.33.214|:443... connected.
+      /^Connecting /,
+      #Reusing existing connection to kubuntu.org:443.
+      /^Reusing /,
+    ]
+
+    def run(&block)
+      @details = {}
+
+      result = false
+
+      Open3.popen2e(
+        { "LANG" => 'C'},
+        'wget', '--continue', '--content-disposition', '--progress=dot:mega', @url,
+      ) do |stdin, stdout_and_err, thread|
+        stdout_and_err.each do |line|
+          line.chomp!
+          next if line.empty?
+
+          data = extract_details line
+
+          unless data.nil?
+            data['filename'] = eval "\"#{data['filename']}\"" unless data['filename'].nil? # Handle escaped UTF-8 chars
+            @details.merge! data
+            block.call data, @details
+          else
+            puts "==='#{line}'===" unless filtred?(line)
+          end
+        end
+
+        exit_status = thread.value
+        @details.merge!({ exit_status: exit_status })
+
+        raise Error.new "Error while downloading: #{error}" unless error.nil?
+      end
+    end
+
+    def downloaded?
+      success?
+    end
+
+    def success?
+      error.nil?
+    end
+
+    def error
+      return "#{details['error_code']}: #{details['error_text']}" unless details['error_code'].nil?
+      return "#{details['error_text']}" unless details['error_text'].nil?
+      return "Command execution failed with exit code: #{details[:exit_status].exitstatus}" unless details[:exit_status].success?
+
+      nil
+    end
+
+    private
+
+    def filtred?(line)
+      FILTERS.each do |regexp|
+        match = regexp.match line
+        return true unless match.nil?
+      end
+
+      false
+    end
+
+    EXTRACTORS = [
+      /^Length: (?<size>\d+)/,
+      /^Saving to: '(?<filename>.*)'$/,
+      #  64512K ,,,,,,,. ........ ........ ........ ........ ........  2% 1.26M 37m23s
+      #  67584K ........ ........ ........ ........ ........ ........  2%  870K 42m29s
+      # 101376K ........ ........ ........ ........ ........ ........  4% 4.70M 8m56s
+      # 104448K ........ ........ ........ ........ ........ ........  4% 5.87M 8m41s
+      # 107520K ........ ........ ........ ........ ........ ........  4% 6.17M 8m27s
+      #2472960K ........ ........ ..                                 100% 5.80M=5m57s
+      /^.* [,. ]{8} [,. ]{8} [,. ]{8} [,. ]{8} [,. ]{8} [,. ]{8} *(?<progress>\d+%) (?<rate>.*)[ =](?<remaining_time>.*)$/,
+      #2020-11-08 09:29:55 ERROR 410: Gone.
+      /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} ERROR (?<error_code>\d+): (?<error_text>.*)$/,
+      #        [ skipping 79872K ]
+      /^ +\[ skipping (?<skipping>.*) \]/,
+      #2020-11-10 22:25:48 (1.19 MB/s) - 'GroovyBannerv4-1024x273.png' saved [278174/278174]
+      /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} \((?<rate_average>.*)\) - '.*' saved/,
+      #    The file is already fully retrieved; nothing to do.
+      /^    (?<message>The file is already fully retrieved; nothing to do\.)$/,
+      #HTTP request sent, awaiting response... 200 OK
+      /^HTTP request sent, awaiting response... (?<http_status_code>\d+) (?<http_status_text>.*)$/,
+      #wget: unable to resolve host address 'kuubuntu.org'
+      /^wget: (?<error_text>unable to resolve host address '.*')$/,
+      #Cannot write to 'kubuntu.iso' (Success).
+      /^(?<error_text>Cannot write to .*)$/,
+    ]
+
+    def extract_details(line)
+      result = {}
+      EXTRACTORS.each do |extractor|
+        result.merge! extract(extractor, line)
+      end
+      result == {} ? nil : result
+    end
+
+    def extract(regexp, line)
+      match = regexp.match line
+      return {} if match.nil?
+      match.named_captures
+    end
+  end
+end

data/lib/robro/user_scripts.rb

@@ -1,3 +1,5 @@
+require 'active_support/inflector'
+
 module Robro
   module UserScripts
     def self.all
@@ -8,7 +10,33 @@ module Robro
       end
     end
 
+    def self.all_commands
+      user_scripts = UserScripts.all.map(&:new)
+      user_scripts.map(&:supported_url_commands).flatten.uniq
+    end
+
+    def self.find_for(uri:)
+      user_scripts = UserScripts.all.map(&:new)
+      user_scripts.find { |us| uri.host.start_with? *(us.supported_urls) }
+    end
+
+    def self.execute(command, uri, *args)
+      us = find_for uri: uri
+
+      raise "No user scripts found for URI: #{uri}" if us.nil?
+
+      Robro.logger.info "Opening URL: #{uri}"
+      Robro.browser.visit uri
+
+      Robro.logger.info "User script: '#{us.class}' will process commnand '#{command}'"
+      us.send(command, uri)
+    end
+
     class Base
+      def browser
+        Robro.browser
+      end
+
       def commands
         raise NotImplementedError
       end

data/lib/robro/version.rb

@@ -1,3 +1,3 @@
 module Robro
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/lib/robro.rb

@@ -2,7 +2,7 @@ require "robro/browser"
 require "robro/version"
 
 require 'tty/logger'
-require 'byebug'
+require 'debug'
 
 module Robro
   class Error < StandardError; end
@@ -12,6 +12,7 @@ module Robro
       config.handlers = [
         [:console, { output: $stderr, level: :debug }],
       ]
+      config.metadata = %i[date time]
     end
   end
 

data/robro.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |spec|
 
   spec.summary       = %q{Robotized Browser}
   spec.description   = %q{Automate tasks that require a _real_ browser}
-  spec.homepage      = "https://github.com/opus-codium/robro"
+  spec.homepage      = "https://github.com/neomilium/robro"
   spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0")
 
   spec.metadata["homepage_uri"] = spec.homepage
@@ -29,8 +29,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "capybara-screenshot"
   spec.add_dependency "selenium-webdriver"
   spec.add_dependency "thor"
+  spec.add_dependency "tty-cursor"
   spec.add_dependency "tty-logger"
-  spec.add_development_dependency "byebug"
+  spec.add_development_dependency "debug"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: robro
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Romuald Conty
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-04-12 00:00:00.000000000 Z
+date: 2025-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -80,6 +80,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: tty-cursor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: tty-logger
   requirement: !ruby/object:Gem::Requirement
@@ -95,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: byebug
+  name: debug
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -169,17 +183,20 @@ files:
 - exe/robro
 - lib/robro.rb
 - lib/robro/browser.rb
+- lib/robro/browser/helpers.rb
 - lib/robro/cli.rb
+- lib/robro/cli/helpers.rb
+- lib/robro/download.rb
 - lib/robro/user_scripts.rb
 - lib/robro/version.rb
 - robro.gemspec
-homepage: https://github.com/opus-codium/robro
+homepage: https://github.com/neomilium/robro
 licenses: []
 metadata:
-  homepage_uri: https://github.com/opus-codium/robro
-  source_code_uri: https://github.com/opus-codium/robro.git
-  changelog_uri: https://github.com/opus-codium/robro/Changelog.md
-post_install_message: 
+  homepage_uri: https://github.com/neomilium/robro
+  source_code_uri: https://github.com/neomilium/robro.git
+  changelog_uri: https://github.com/neomilium/robro/Changelog.md
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -194,8 +211,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.3.15
+signing_key:
 specification_version: 4
 summary: Robotized Browser
 test_files: []