kimurai 1.4.0 → 2.0.0

data/lib/kimurai/browser_builder/selenium_firefox_builder.rb

@@ -4,201 +4,203 @@ require_relative '../capybara_configuration'
 require_relative '../capybara_ext/selenium/driver'
 require_relative '../capybara_ext/session'
 
-module Kimurai::BrowserBuilder
-  class SeleniumFirefoxBuilder
-    class << self
-      attr_accessor :virtual_display
-    end
+module Kimurai
+  module BrowserBuilder
+    class SeleniumFirefoxBuilder
+      class << self
+        attr_accessor :virtual_display
+      end
 
-    attr_reader :logger, :spider
+      attr_reader :logger, :spider
 
-    def initialize(config, spider:)
-      @config = config
-      @spider = spider
-      @logger = spider.logger
-    end
+      def initialize(config, spider:)
+        @config = config
+        @spider = spider
+        @logger = spider.logger
+      end
 
-    def build
-      # Register driver
-      Capybara.register_driver :selenium_firefox do |app|
-        # Create driver options
-        driver_options = Selenium::WebDriver::Firefox::Options.new
-        driver_options.profile = Selenium::WebDriver::Firefox::Profile.new
-        driver_options.profile["browser.link.open_newwindow"] = 3 # open windows in tabs
-        driver_options.profile["media.peerconnection.enabled"] = false # disable web rtc
-
-        # Proxy
-        if proxy = @config[:proxy].presence
-          proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
-          ip, port, type, user, password = proxy_string.split(":")
-
-          if user.nil? && password.nil?
-            driver_options.profile["network.proxy.type"] = 1
-            if type == "http"
-              driver_options.profile["network.proxy.http"] = ip
-              driver_options.profile["network.proxy.http_port"] = port.to_i
-              driver_options.profile["network.proxy.ssl"] = ip
-              driver_options.profile["network.proxy.ssl_port"] = port.to_i
-
-              logger.debug "BrowserBuilder (selenium_firefox): enabled http proxy, ip: #{ip}, port: #{port}"
-            elsif type == "socks5"
-              driver_options.profile["network.proxy.socks"] = ip
-              driver_options.profile["network.proxy.socks_port"] = port.to_i
-              driver_options.profile["network.proxy.socks_version"] = 5
-              driver_options.profile["network.proxy.socks_remote_dns"] = true
-
-              logger.debug "BrowserBuilder (selenium_firefox): enabled socks5 proxy, ip: #{ip}, port: #{port}"
+      def build
+        # Register driver
+        Capybara.register_driver :selenium_firefox do |app|
+          # Create driver options
+          driver_options = Selenium::WebDriver::Firefox::Options.new
+          driver_options.profile = Selenium::WebDriver::Firefox::Profile.new
+          driver_options.profile['browser.link.open_newwindow'] = 3 # open windows in tabs
+          driver_options.profile['media.peerconnection.enabled'] = false # disable web rtc
+
+          # Proxy
+          if (proxy = @config[:proxy].presence)
+            proxy_string = (proxy.instance_of?(Proc) ? proxy.call : proxy).strip
+            ip, port, type, user, password = proxy_string.split(':')
+
+            if user.nil? && password.nil?
+              driver_options.profile['network.proxy.type'] = 1
+              if type == 'http'
+                driver_options.profile['network.proxy.http'] = ip
+                driver_options.profile['network.proxy.http_port'] = port.to_i
+                driver_options.profile['network.proxy.ssl'] = ip
+                driver_options.profile['network.proxy.ssl_port'] = port.to_i
+
+                logger.debug "BrowserBuilder (selenium_firefox): enabled http proxy, ip: #{ip}, port: #{port}"
+              elsif type == 'socks5'
+                driver_options.profile['network.proxy.socks'] = ip
+                driver_options.profile['network.proxy.socks_port'] = port.to_i
+                driver_options.profile['network.proxy.socks_version'] = 5
+                driver_options.profile['network.proxy.socks_remote_dns'] = true
+
+                logger.debug "BrowserBuilder (selenium_firefox): enabled socks5 proxy, ip: #{ip}, port: #{port}"
+              else
+                logger.error "BrowserBuilder (selenium_firefox): wrong type of proxy: #{type}, skipped"
+              end
             else
-              logger.error "BrowserBuilder (selenium_firefox): wrong type of proxy: #{type}, skipped"
+              logger.error "BrowserBuilder (selenium_firefox): proxy with authentication doesn't supported by selenium, skipped"
             end
-          else
-            logger.error "BrowserBuilder (selenium_firefox): proxy with authentication doesn't supported by selenium, skipped"
           end
-        end
 
-        if proxy_bypass_list = @config[:proxy_bypass_list].presence
-          if proxy
-            driver_options.profile["network.proxy.no_proxies_on"] = proxy_bypass_list.join(", ")
-            logger.debug "BrowserBuilder (selenium_firefox): enabled proxy_bypass_list"
-          else
-            logger.error "BrowserBuilder (selenium_firefox): provide `proxy` to set proxy_bypass_list, skipped"
+          if (proxy_bypass_list = @config[:proxy_bypass_list].presence)
+            if proxy
+              driver_options.profile['network.proxy.no_proxies_on'] = proxy_bypass_list.join(', ')
+              logger.debug 'BrowserBuilder (selenium_firefox): enabled proxy_bypass_list'
+            else
+              logger.error 'BrowserBuilder (selenium_firefox): provide `proxy` to set proxy_bypass_list, skipped'
+            end
           end
-        end
 
-        # SSL
-        if @config[:ignore_ssl_errors].present?
-          driver_options.profile.secure_ssl = false
-          driver_options.profile.assume_untrusted_certificate_issuer = true
-          logger.debug "BrowserBuilder (selenium_firefox): enabled ignore_ssl_errors"
-        end
+          # SSL
+          if @config[:ignore_ssl_errors].present?
+            driver_options.profile.secure_ssl = false
+            driver_options.profile.assume_untrusted_certificate_issuer = true
+            logger.debug 'BrowserBuilder (selenium_firefox): enabled ignore_ssl_errors'
+          end
 
-        # Disable images
-        if @config[:disable_images].present?
-          driver_options.profile["permissions.default.image"] = 2
-          logger.debug "BrowserBuilder (selenium_firefox): enabled disable_images"
-        end
+          # Disable images
+          if @config[:disable_images].present?
+            driver_options.profile['permissions.default.image'] = 2
+            logger.debug 'BrowserBuilder (selenium_firefox): enabled disable_images'
+          end
 
-        # Headers
-        if @config[:headers].present?
-          logger.warn "BrowserBuilder: (selenium_firefox): custom headers doesn't supported by selenium, skipped"
-        end
+          # Headers
+          if @config[:headers].present?
+            logger.warn "BrowserBuilder: (selenium_firefox): custom headers doesn't supported by selenium, skipped"
+          end
 
-        if user_agent = @config[:user_agent].presence
-          user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
-          driver_options.profile["general.useragent.override"] = user_agent_string
-          logger.debug "BrowserBuilder (selenium_firefox): enabled custom user_agent"
-        end
+          if (user_agent = @config[:user_agent].presence)
+            user_agent_string = (user_agent.instance_of?(Proc) ? user_agent.call : user_agent).strip
+            driver_options.profile['general.useragent.override'] = user_agent_string
+            logger.debug 'BrowserBuilder (selenium_firefox): enabled custom user_agent'
+          end
 
-        # Headless mode
-        if ENV["HEADLESS"] != "false"
-          if @config[:headless_mode] == :virtual_display
-            if Gem::Platform.local.os == "linux"
-              unless self.class.virtual_display
-                require 'headless'
-                self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
-                self.class.virtual_display.start
+          # Headless mode
+          if ENV['HEADLESS'] != 'false'
+            if @config[:headless_mode] == :virtual_display
+              if Gem::Platform.local.os == 'linux'
+                unless self.class.virtual_display
+                  require 'headless'
+                  self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
+                  self.class.virtual_display.start
+                end
+
+                logger.debug 'BrowserBuilder (selenium_firefox): enabled virtual_display headless_mode'
+              else
+                logger.error 'BrowserBuilder (selenium_firefox): virtual_display headless_mode works only ' \
+                  'on Linux platform. Browser will run in normal mode. Set `native` mode instead.'
               end
-
-              logger.debug "BrowserBuilder (selenium_firefox): enabled virtual_display headless_mode"
             else
-              logger.error "BrowserBuilder (selenium_firefox): virtual_display headless_mode works only " \
-                "on Linux platform. Browser will run in normal mode. Set `native` mode instead."
+              driver_options.args << '--headless'
+              logger.debug 'BrowserBuilder (selenium_firefox): enabled native headless_mode'
             end
-          else
-            driver_options.args << "--headless"
-            logger.debug "BrowserBuilder (selenium_firefox): enabled native headless_mode"
           end
+
+          Capybara::Selenium::Driver.new(app, browser: :firefox, options: driver_options)
         end
 
-        Capybara::Selenium::Driver.new(app, browser: :firefox, options: driver_options)
-      end
+        # Create browser instance (Capybara session)
+        @browser = Capybara::Session.new(:selenium_firefox)
+        @browser.spider = spider
+        logger.debug 'BrowserBuilder (selenium_firefox): created browser instance'
 
-      # Create browser instance (Capybara session)
-      @browser = Capybara::Session.new(:selenium_firefox)
-      @browser.spider = spider
-      logger.debug "BrowserBuilder (selenium_firefox): created browser instance"
+        if @config[:extensions].present?
+          logger.error 'BrowserBuilder (selenium_firefox): `extensions` option not supported by Selenium, skipped'
+        end
 
-      if @config[:extensions].present?
-        logger.error "BrowserBuilder (selenium_firefox): `extensions` option not supported by Selenium, skipped"
-      end
+        # Window size
+        if (size = @config[:window_size].presence)
+          @browser.current_window.resize_to(*size)
+          logger.debug 'BrowserBuilder (selenium_firefox): enabled window_size'
+        end
 
-      # Window size
-      if size = @config[:window_size].presence
-        @browser.current_window.resize_to(*size)
-        logger.debug "BrowserBuilder (selenium_firefox): enabled window_size"
-      end
+        # Cookies
+        if (cookies = @config[:cookies].presence)
+          @browser.config.cookies = cookies
+          logger.debug 'BrowserBuilder (selenium_firefox): enabled custom cookies'
+        end
 
-      # Cookies
-      if cookies = @config[:cookies].presence
-        @browser.config.cookies = cookies
-        logger.debug "BrowserBuilder (selenium_firefox): enabled custom cookies"
-      end
+        # Browser instance options
+        # skip_request_errors
+        if (skip_errors = @config[:skip_request_errors].presence)
+          @browser.config.skip_request_errors = skip_errors
+          logger.debug 'BrowserBuilder (selenium_firefox): enabled skip_request_errors'
+        end
 
-      # Browser instance options
-      # skip_request_errors
-      if skip_errors = @config[:skip_request_errors].presence
-        @browser.config.skip_request_errors = skip_errors
-        logger.debug "BrowserBuilder (selenium_firefox): enabled skip_request_errors"
-      end
+        # retry_request_errors
+        if (retry_errors = @config[:retry_request_errors].presence)
+          @browser.config.retry_request_errors = retry_errors
+          logger.debug 'BrowserBuilder (selenium_firefox): enabled retry_request_errors'
+        end
 
-      # retry_request_errors
-      if retry_errors = @config[:retry_request_errors].presence
-        @browser.config.retry_request_errors = retry_errors
-        logger.debug "BrowserBuilder (selenium_firefox): enabled retry_request_errors"
-      end
+        # restart_if
+        if (requests_limit = @config.dig(:restart_if, :requests_limit).presence)
+          @browser.config.restart_if[:requests_limit] = requests_limit
+          logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.requests_limit >= #{requests_limit}"
+        end
 
-      # restart_if
-      if requests_limit = @config.dig(:restart_if, :requests_limit).presence
-        @browser.config.restart_if[:requests_limit] = requests_limit
-        logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.requests_limit >= #{requests_limit}"
-      end
+        if (memory_limit = @config.dig(:restart_if, :memory_limit).presence)
+          @browser.config.restart_if[:memory_limit] = memory_limit
+          logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.memory_limit >= #{memory_limit}"
+        end
 
-      if memory_limit = @config.dig(:restart_if, :memory_limit).presence
-        @browser.config.restart_if[:memory_limit] = memory_limit
-        logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.memory_limit >= #{memory_limit}"
-      end
+        # before_request clear_cookies
+        if @config.dig(:before_request, :clear_cookies)
+          @browser.config.before_request[:clear_cookies] = true
+          logger.debug 'BrowserBuilder (selenium_firefox): enabled before_request.clear_cookies'
+        end
 
-      # before_request clear_cookies
-      if @config.dig(:before_request, :clear_cookies)
-        @browser.config.before_request[:clear_cookies] = true
-        logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_cookies"
-      end
+        # before_request clear_and_set_cookies
+        if @config.dig(:before_request, :clear_and_set_cookies)
+          if (cookies = @config[:cookies].presence)
+            @browser.config.cookies = cookies
+            @browser.config.before_request[:clear_and_set_cookies] = true
+            logger.debug 'BrowserBuilder (selenium_firefox): enabled before_request.clear_and_set_cookies'
+          else
+            logger.error 'BrowserBuilder (selenium_firefox): cookies should be present to enable before_request.clear_and_set_cookies, skipped'
+          end
+        end
 
-      # before_request clear_and_set_cookies
-      if @config.dig(:before_request, :clear_and_set_cookies)
-        if cookies = @config[:cookies].presence
-          @browser.config.cookies = cookies
-          @browser.config.before_request[:clear_and_set_cookies] = true
-          logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_and_set_cookies"
-        else
-          logger.error "BrowserBuilder (selenium_firefox): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
+        # before_request change_user_agent
+        if @config.dig(:before_request, :change_user_agent)
+          logger.error 'BrowserBuilder (selenium_firefox): before_request.change_user_agent option not supported by Selenium, skipped'
         end
-      end
 
-      # before_request change_user_agent
-      if @config.dig(:before_request, :change_user_agent)
-        logger.error "BrowserBuilder (selenium_firefox): before_request.change_user_agent option not supported by Selenium, skipped"
-      end
+        # before_request change_proxy
+        if @config.dig(:before_request, :change_proxy)
+          logger.error 'BrowserBuilder (selenium_firefox): before_request.change_proxy option not supported by Selenium, skipped'
+        end
 
-      # before_request change_proxy
-      if @config.dig(:before_request, :change_proxy)
-        logger.error "BrowserBuilder (selenium_firefox): before_request.change_proxy option not supported by Selenium, skipped"
-      end
+        # before_request delay
+        if (delay = @config.dig(:before_request, :delay).presence)
+          @browser.config.before_request[:delay] = delay
+          logger.debug 'BrowserBuilder (selenium_firefox): enabled before_request.delay'
+        end
 
-      # before_request delay
-      if delay = @config.dig(:before_request, :delay).presence
-        @browser.config.before_request[:delay] = delay
-        logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.delay"
-      end
+        # encoding
+        if (encoding = @config[:encoding])
+          @browser.config.encoding = encoding
+          logger.debug "BrowserBuilder (selenium_firefox): enabled encoding: #{encoding}"
+        end
 
-      # encoding
-      if encoding = @config[:encoding]
-        @browser.config.encoding = encoding
-        logger.debug "BrowserBuilder (selenium_firefox): enabled encoding: #{encoding}"
+        # return Capybara session instance
+        @browser
       end
-
-      # return Capybara session instance
-      @browser
     end
   end
 end

data/lib/kimurai/browser_builder.rb

@@ -1,15 +1,9 @@
 module Kimurai
   module BrowserBuilder
     def self.build(engine, config = {}, spider:)
-      if config[:browser].present?
-        raise "++++++ BrowserBuilder: browser option is depricated. Now all sub-options inside " \
-          "`browser` should be placed right into `@config` hash, without `browser` parent key.\n" \
-          "See more here: https://github.com/vifreefly/kimuraframework/blob/master/CHANGELOG.md#breaking-changes-110 ++++++"
-      end
-
       begin
         require "kimurai/browser_builder/#{engine}_builder"
-      rescue LoadError => e
+      rescue LoadError
       end
 
       builder_class_name = "#{engine}_builder".classify

data/lib/kimurai/capybara_configuration.rb

@@ -3,7 +3,7 @@ require 'capybara'
 Capybara.configure do |config|
   config.run_server = false
   config.default_selector = :xpath
-  config.save_path = "tmp"
+  config.save_path = 'tmp'
   config.default_max_wait_time = 10
   config.ignore_hidden_elements = false
   config.threadsafe = true

data/lib/kimurai/capybara_ext/driver/base.rb

@@ -1,62 +1,66 @@
 require 'pathname'
 
-class Capybara::Driver::Base
-  attr_accessor :visited
-  attr_writer :requests, :responses
+module Capybara
+  module Driver
+    class Base
+      attr_accessor :visited
+      attr_writer :requests, :responses
 
-  def requests
-    @requests ||= 0
-  end
+      def requests
+        @requests ||= 0
+      end
 
-  def responses
-    @responses ||= 0
-  end
+      def responses
+        @responses ||= 0
+      end
 
-  def current_memory
-    driver_pid = pid
+      def current_memory
+        driver_pid = pid
 
-    all = (get_descendant_processes(driver_pid) << driver_pid).uniq
-    all.map { |pid| get_process_memory(pid) }.sum
-  end
+        all = (get_descendant_processes(driver_pid) << driver_pid).uniq
+        all.map { |pid| get_process_memory(pid) }.sum
+      end
 
-  private
+      private
 
-  def get_descendant_processes(base)
-    descendants = Hash.new { |ht, k| ht[k] = [k] }
-    Hash[*`ps -eo pid,ppid`.scan(/\d+/).map(&:to_i)].each do |pid, ppid|
-      descendants[ppid] << descendants[pid]
-    end
+      def get_descendant_processes(base)
+        descendants = Hash.new { |ht, k| ht[k] = [k] }
+        Hash[*`ps -eo pid,ppid`.scan(/\d+/).map(&:to_i)].each do |pid, ppid|
+          descendants[ppid] << descendants[pid]
+        end
 
-    descendants[base].flatten - [base]
-  end
+        descendants[base].flatten - [base]
+      end
 
-  # https://github.com/schneems/get_process_mem
-  # Note: for Linux takes PSS (not RSS) memory (I think PSS better fits in this case)
-  def get_process_memory(pid)
-    case @platform ||= Gem::Platform.local.os
-    when "linux"
-      begin
-        file = Pathname.new "/proc/#{pid}/smaps"
-        return 0 unless file.exist?
-
-        lines = file.each_line.select { |line| line.match(/^Pss/) }
-        return 0 if lines.empty?
-
-        lines.reduce(0) do |sum, line|
-          line.match(/(?<value>(\d*\.{0,1}\d+))\s+(?<unit>\w\w)/) do |m|
-            sum += m[:value].to_i
-          end
+      # https://github.com/schneems/get_process_mem
+      # Note: for Linux takes PSS (not RSS) memory (I think PSS better fits in this case)
+      def get_process_memory(pid)
+        case @platform ||= Gem::Platform.local.os
+        when 'linux'
+          begin
+            file = Pathname.new "/proc/#{pid}/smaps"
+            return 0 unless file.exist?
 
-          sum
+            lines = file.each_line.select { |line| line.match(/^Pss/) }
+            return 0 if lines.empty?
+
+            lines.reduce(0) do |sum, line|
+              line.match(/(?<value>(\d*\.{0,1}\d+))\s+(?<unit>\w\w)/) do |m|
+                sum += m[:value].to_i
+              end
+
+              sum
+            end
+          rescue Errno::EACCES
+            0
+          end
+        when 'darwin'
+          mem = `ps -o rss= -p #{pid}`.strip
+          mem.empty? ? 0 : mem.to_i
+        else
+          raise "Can't check process memory, wrong type of platform: #{@platform}"
         end
-      rescue Errno::EACCES
-        0
       end
-    when "darwin"
-      mem = `ps -o rss= -p #{pid}`.strip
-      mem.empty? ? 0 : mem.to_i
-    else
-      raise "Can't check process memory, wrong type of platform: #{@platform}"
     end
   end
 end

data/lib/kimurai/capybara_ext/mechanize/driver.rb

@@ -1,71 +1,72 @@
 require 'mechanize'
 require_relative '../driver/base'
 
-class Capybara::Mechanize::Driver
-  # Extend capybara-mechnize to support Poltergeist-like methods
-  # https://www.rubydoc.info/gems/poltergeist/Capybara/Poltergeist/Driver
+module Capybara
+  module Mechanize
+    class Driver
+      def set_proxy(ip, port, _type, user = nil, password = nil)
+        # type is always "http", "socks" is not supported (yet)
+        browser.agent.set_proxy(ip, port, user, password)
+      end
 
-  def set_proxy(ip, port, type, user = nil, password = nil)
-    # type is always "http", "socks" is not supported (yet)
-    browser.agent.set_proxy(ip, port, user, password)
-  end
+      ###
 
-  ###
+      def headers
+        browser.agent.request_headers
+      end
 
-  def headers
-    browser.agent.request_headers
-  end
+      def headers=(headers)
+        browser.agent.request_headers = headers
+      end
 
-  def headers=(headers)
-    browser.agent.request_headers = headers
-  end
+      def add_header(name, value)
+        browser.agent.request_headers[name] = value
+      end
 
-  def add_header(name, value)
-    browser.agent.request_headers[name] = value
-  end
+      ###
 
-  ###
+      def get_cookies
+        browser.agent.cookies
+      end
 
-  def get_cookies
-    browser.agent.cookies
-  end
-
-  def set_cookie(name, value, options = {})
-    options[:name]  ||= name
-    options[:value] ||= value
+      def set_cookie(name, value, options = {})
+        options[:name]  ||= name
+        options[:value] ||= value
 
-    cookie = Mechanize::Cookie.new(options.merge path: "/")
-    browser.agent.cookie_jar << cookie
-  end
+        cookie = Mechanize::Cookie.new(options.merge(path: '/'))
+        browser.agent.cookie_jar << cookie
+      end
 
-  def set_cookies(cookies)
-    cookies.each do |cookie|
-      set_cookie(cookie[:name], cookie[:value], cookie)
-    end
-  end
+      def set_cookies(cookies)
+        cookies.each do |cookie|
+          set_cookie(cookie[:name], cookie[:value], cookie)
+        end
+      end
 
-  def clear_cookies
-    browser.agent.cookie_jar.clear!
-  end
+      def clear_cookies
+        browser.agent.cookie_jar.clear!
+      end
 
-  ###
+      ###
 
-  def quit
-    browser.agent.shutdown
-  end
+      def quit
+        browser.agent.shutdown
+      end
 
-  ###
+      ###
 
-  # Reset parent method `current_memory` for mechanize (we can't measure memory of Mechanize driver)
-  def current_memory
-    nil
-  end
+      # Reset parent method `current_memory` for mechanize (we can't measure memory of Mechanize driver)
+      def current_memory
+        nil
+      end
 
-  def pid
-    nil
-  end
+      def pid
+        nil
+      end
 
-  def port
-    nil
+      def port
+        nil
+      end
+    end
   end
 end