kimurai_dynamic 1.4.1

data/lib/kimurai/browser_builder/selenium_chrome_builder.rb

@@ -0,0 +1,199 @@
+require 'capybara'
+require 'selenium-webdriver'
+require_relative '../capybara_configuration'
+require_relative '../capybara_ext/selenium/driver'
+require_relative '../capybara_ext/session'
+
+module Kimurai::BrowserBuilder
+  class SeleniumChromeBuilder
+    class << self
+      attr_accessor :virtual_display
+    end
+
+    attr_reader :logger, :spider
+
+    def initialize(config, spider:)
+      @config = config
+      @spider = spider
+      @logger = spider.logger
+    end
+
+    def build
+      # Register driver
+      Capybara.register_driver :selenium_chrome do |app|
+        # Create driver options
+        opts = { args: %w[--disable-gpu --no-sandbox --disable-translate] }
+
+        # Provide custom chrome browser path:
+        if chrome_path = Kimurai.configuration.selenium_chrome_path
+          opts.merge!(binary: chrome_path)
+        end
+
+        # See all options here: https://seleniumhq.github.io/selenium/docs/api/rb/Selenium/WebDriver/Chrome/Options.html
+        driver_options = Selenium::WebDriver::Chrome::Options.new(opts)
+
+        # Window size
+        if size = @config[:window_size].presence
+          driver_options.args << "--window-size=#{size.join(',')}"
+          logger.debug "BrowserBuilder (selenium_chrome): enabled window_size"
+        end
+
+        # Proxy
+        if proxy = @config[:proxy].presence
+          proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
+          ip, port, type, user, password = proxy_string.split(":")
+
+          if %w(http socks5).include?(type)
+            if user.nil? && password.nil?
+              driver_options.args << "--proxy-server=#{type}://#{ip}:#{port}"
+              logger.debug "BrowserBuilder (selenium_chrome): enabled #{type} proxy, ip: #{ip}, port: #{port}"
+            else
+              logger.error "BrowserBuilder (selenium_chrome): proxy with authentication doesn't supported by selenium, skipped"
+            end
+          else
+            logger.error "BrowserBuilder (selenium_chrome): wrong type of proxy: #{type}, skipped"
+          end
+        end
+
+        if proxy_bypass_list = @config[:proxy_bypass_list].presence
+          if proxy
+            driver_options.args << "--proxy-bypass-list=#{proxy_bypass_list.join(';')}"
+            logger.debug "BrowserBuilder (selenium_chrome): enabled proxy_bypass_list"
+          else
+            logger.error "BrowserBuilder (selenium_chrome): provide `proxy` to set proxy_bypass_list, skipped"
+          end
+        end
+
+        # SSL
+        if @config[:ignore_ssl_errors].present?
+          driver_options.args << "--ignore-certificate-errors"
+          driver_options.args << "--allow-insecure-localhost"
+          logger.debug "BrowserBuilder (selenium_chrome): enabled ignore_ssl_errors"
+        end
+
+        # Disable images
+        if @config[:disable_images].present?
+          driver_options.prefs["profile.managed_default_content_settings.images"] = 2
+          logger.debug "BrowserBuilder (selenium_chrome): enabled disable_images"
+        end
+
+        # Headers
+        if @config[:headers].present?
+          logger.warn "BrowserBuilder: (selenium_chrome): custom headers doesn't supported by selenium, skipped"
+        end
+
+        if user_agent = @config[:user_agent].presence
+          user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
+          driver_options.args << "--user-agent='#{user_agent_string}'"
+          logger.debug "BrowserBuilder (selenium_chrome): enabled custom user_agent"
+        end
+
+        # Headless mode
+        if ENV["HEADLESS"] != "false"
+          if @config[:headless_mode] == :virtual_display
+            if Gem::Platform.local.os == "linux"
+              unless self.class.virtual_display
+                require 'headless'
+                self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
+                self.class.virtual_display.start
+              end
+
+              logger.debug "BrowserBuilder (selenium_chrome): enabled virtual_display headless_mode"
+            else
+              logger.error "BrowserBuilder (selenium_chrome): virtual_display headless_mode works only " \
+                "on Linux platform. Browser will run in normal mode. Set `native` mode instead."
+            end
+          else
+            driver_options.args << "--headless"
+            logger.debug "BrowserBuilder (selenium_chrome): enabled native headless_mode"
+          end
+        end
+
+        chromedriver_path = Kimurai.configuration.chromedriver_path || "/usr/local/bin/chromedriver"
+        service = Selenium::WebDriver::Service.chrome(path: chromedriver_path)
+        Capybara::Selenium::Driver.new(app, browser: :chrome, options: driver_options, service: service)
+      end
+
+      # Create browser instance (Capybara session)
+      @browser = Capybara::Session.new(:selenium_chrome)
+      @browser.spider = spider
+      logger.debug "BrowserBuilder (selenium_chrome): created browser instance"
+
+      if @config[:extensions].present?
+        logger.error "BrowserBuilder (selenium_chrome): `extensions` option not supported by Selenium, skipped"
+      end
+
+      # Cookies
+      if cookies = @config[:cookies].presence
+        @browser.config.cookies = cookies
+        logger.debug "BrowserBuilder (selenium_chrome): enabled custom cookies"
+      end
+
+      # Browser instance options
+      # skip_request_errors
+      if skip_errors = @config[:skip_request_errors].presence
+        @browser.config.skip_request_errors = skip_errors
+        logger.debug "BrowserBuilder (selenium_chrome): enabled skip_request_errors"
+      end
+
+      # retry_request_errors
+      if retry_errors = @config[:retry_request_errors].presence
+        @browser.config.retry_request_errors = retry_errors
+        logger.debug "BrowserBuilder (selenium_chrome): enabled retry_request_errors"
+      end
+
+      # restart_if
+      if requests_limit = @config.dig(:restart_if, :requests_limit).presence
+        @browser.config.restart_if[:requests_limit] = requests_limit
+        logger.debug "BrowserBuilder (selenium_chrome): enabled restart_if.requests_limit >= #{requests_limit}"
+      end
+
+      if memory_limit = @config.dig(:restart_if, :memory_limit).presence
+        @browser.config.restart_if[:memory_limit] = memory_limit
+        logger.debug "BrowserBuilder (selenium_chrome): enabled restart_if.memory_limit >= #{memory_limit}"
+      end
+
+      # before_request clear_cookies
+      if @config.dig(:before_request, :clear_cookies)
+        @browser.config.before_request[:clear_cookies] = true
+        logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_cookies"
+      end
+
+      # before_request clear_and_set_cookies
+      if @config.dig(:before_request, :clear_and_set_cookies)
+        if cookies = @config[:cookies].presence
+          @browser.config.cookies = cookies
+          @browser.config.before_request[:clear_and_set_cookies] = true
+          logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.clear_and_set_cookies"
+        else
+          logger.error "BrowserBuilder (selenium_chrome): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
+        end
+      end
+
+      # before_request change_user_agent
+      if @config.dig(:before_request, :change_user_agent)
+        logger.error "BrowserBuilder (selenium_chrome): before_request.change_user_agent option not supported by Selenium, skipped"
+      end
+
+      # before_request change_proxy
+      if @config.dig(:before_request, :change_proxy)
+        logger.error "BrowserBuilder (selenium_chrome): before_request.change_proxy option not supported by Selenium, skipped"
+      end
+
+      # before_request delay
+      if delay = @config.dig(:before_request, :delay).presence
+        @browser.config.before_request[:delay] = delay
+        logger.debug "BrowserBuilder (selenium_chrome): enabled before_request.delay"
+      end
+
+      # encoding
+      if encoding = @config[:encoding]
+        @browser.config.encoding = encoding
+        logger.debug "BrowserBuilder (selenium_chrome): enabled encoding: #{encoding}"
+      end
+
+      # return Capybara session instance
+      @browser
+    end
+  end
+end

data/lib/kimurai/browser_builder/selenium_firefox_builder.rb

@@ -0,0 +1,204 @@
+require 'capybara'
+require 'selenium-webdriver'
+require_relative '../capybara_configuration'
+require_relative '../capybara_ext/selenium/driver'
+require_relative '../capybara_ext/session'
+
+module Kimurai::BrowserBuilder
+  class SeleniumFirefoxBuilder
+    class << self
+      attr_accessor :virtual_display
+    end
+
+    attr_reader :logger, :spider
+
+    def initialize(config, spider:)
+      @config = config
+      @spider = spider
+      @logger = spider.logger
+    end
+
+    def build
+      # Register driver
+      Capybara.register_driver :selenium_firefox do |app|
+        # Create driver options
+        driver_options = Selenium::WebDriver::Firefox::Options.new
+        driver_options.profile = Selenium::WebDriver::Firefox::Profile.new
+        driver_options.profile["browser.link.open_newwindow"] = 3 # open windows in tabs
+        driver_options.profile["media.peerconnection.enabled"] = false # disable web rtc
+
+        # Proxy
+        if proxy = @config[:proxy].presence
+          proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
+          ip, port, type, user, password = proxy_string.split(":")
+
+          if user.nil? && password.nil?
+            driver_options.profile["network.proxy.type"] = 1
+            if type == "http"
+              driver_options.profile["network.proxy.http"] = ip
+              driver_options.profile["network.proxy.http_port"] = port.to_i
+              driver_options.profile["network.proxy.ssl"] = ip
+              driver_options.profile["network.proxy.ssl_port"] = port.to_i
+
+              logger.debug "BrowserBuilder (selenium_firefox): enabled http proxy, ip: #{ip}, port: #{port}"
+            elsif type == "socks5"
+              driver_options.profile["network.proxy.socks"] = ip
+              driver_options.profile["network.proxy.socks_port"] = port.to_i
+              driver_options.profile["network.proxy.socks_version"] = 5
+              driver_options.profile["network.proxy.socks_remote_dns"] = true
+
+              logger.debug "BrowserBuilder (selenium_firefox): enabled socks5 proxy, ip: #{ip}, port: #{port}"
+            else
+              logger.error "BrowserBuilder (selenium_firefox): wrong type of proxy: #{type}, skipped"
+            end
+          else
+            logger.error "BrowserBuilder (selenium_firefox): proxy with authentication doesn't supported by selenium, skipped"
+          end
+        end
+
+        if proxy_bypass_list = @config[:proxy_bypass_list].presence
+          if proxy
+            driver_options.profile["network.proxy.no_proxies_on"] = proxy_bypass_list.join(", ")
+            logger.debug "BrowserBuilder (selenium_firefox): enabled proxy_bypass_list"
+          else
+            logger.error "BrowserBuilder (selenium_firefox): provide `proxy` to set proxy_bypass_list, skipped"
+          end
+        end
+
+        # SSL
+        if @config[:ignore_ssl_errors].present?
+          driver_options.profile.secure_ssl = false
+          driver_options.profile.assume_untrusted_certificate_issuer = true
+          logger.debug "BrowserBuilder (selenium_firefox): enabled ignore_ssl_errors"
+        end
+
+        # Disable images
+        if @config[:disable_images].present?
+          driver_options.profile["permissions.default.image"] = 2
+          logger.debug "BrowserBuilder (selenium_firefox): enabled disable_images"
+        end
+
+        # Headers
+        if @config[:headers].present?
+          logger.warn "BrowserBuilder: (selenium_firefox): custom headers doesn't supported by selenium, skipped"
+        end
+
+        if user_agent = @config[:user_agent].presence
+          user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
+          driver_options.profile["general.useragent.override"] = user_agent_string
+          logger.debug "BrowserBuilder (selenium_firefox): enabled custom user_agent"
+        end
+
+        # Headless mode
+        if ENV["HEADLESS"] != "false"
+          if @config[:headless_mode] == :virtual_display
+            if Gem::Platform.local.os == "linux"
+              unless self.class.virtual_display
+                require 'headless'
+                self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
+                self.class.virtual_display.start
+              end
+
+              logger.debug "BrowserBuilder (selenium_firefox): enabled virtual_display headless_mode"
+            else
+              logger.error "BrowserBuilder (selenium_firefox): virtual_display headless_mode works only " \
+                "on Linux platform. Browser will run in normal mode. Set `native` mode instead."
+            end
+          else
+            driver_options.args << "--headless"
+            logger.debug "BrowserBuilder (selenium_firefox): enabled native headless_mode"
+          end
+        end
+
+        Capybara::Selenium::Driver.new(app, browser: :firefox, options: driver_options)
+      end
+
+      # Create browser instance (Capybara session)
+      @browser = Capybara::Session.new(:selenium_firefox)
+      @browser.spider = spider
+      logger.debug "BrowserBuilder (selenium_firefox): created browser instance"
+
+      if @config[:extensions].present?
+        logger.error "BrowserBuilder (selenium_firefox): `extensions` option not supported by Selenium, skipped"
+      end
+
+      # Window size
+      if size = @config[:window_size].presence
+        @browser.current_window.resize_to(*size)
+        logger.debug "BrowserBuilder (selenium_firefox): enabled window_size"
+      end
+
+      # Cookies
+      if cookies = @config[:cookies].presence
+        @browser.config.cookies = cookies
+        logger.debug "BrowserBuilder (selenium_firefox): enabled custom cookies"
+      end
+
+      # Browser instance options
+      # skip_request_errors
+      if skip_errors = @config[:skip_request_errors].presence
+        @browser.config.skip_request_errors = skip_errors
+        logger.debug "BrowserBuilder (selenium_firefox): enabled skip_request_errors"
+      end
+
+      # retry_request_errors
+      if retry_errors = @config[:retry_request_errors].presence
+        @browser.config.retry_request_errors = retry_errors
+        logger.debug "BrowserBuilder (selenium_firefox): enabled retry_request_errors"
+      end
+
+      # restart_if
+      if requests_limit = @config.dig(:restart_if, :requests_limit).presence
+        @browser.config.restart_if[:requests_limit] = requests_limit
+        logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.requests_limit >= #{requests_limit}"
+      end
+
+      if memory_limit = @config.dig(:restart_if, :memory_limit).presence
+        @browser.config.restart_if[:memory_limit] = memory_limit
+        logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.memory_limit >= #{memory_limit}"
+      end
+
+      # before_request clear_cookies
+      if @config.dig(:before_request, :clear_cookies)
+        @browser.config.before_request[:clear_cookies] = true
+        logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_cookies"
+      end
+
+      # before_request clear_and_set_cookies
+      if @config.dig(:before_request, :clear_and_set_cookies)
+        if cookies = @config[:cookies].presence
+          @browser.config.cookies = cookies
+          @browser.config.before_request[:clear_and_set_cookies] = true
+          logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_and_set_cookies"
+        else
+          logger.error "BrowserBuilder (selenium_firefox): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
+        end
+      end
+
+      # before_request change_user_agent
+      if @config.dig(:before_request, :change_user_agent)
+        logger.error "BrowserBuilder (selenium_firefox): before_request.change_user_agent option not supported by Selenium, skipped"
+      end
+
+      # before_request change_proxy
+      if @config.dig(:before_request, :change_proxy)
+        logger.error "BrowserBuilder (selenium_firefox): before_request.change_proxy option not supported by Selenium, skipped"
+      end
+
+      # before_request delay
+      if delay = @config.dig(:before_request, :delay).presence
+        @browser.config.before_request[:delay] = delay
+        logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.delay"
+      end
+
+      # encoding
+      if encoding = @config[:encoding]
+        @browser.config.encoding = encoding
+        logger.debug "BrowserBuilder (selenium_firefox): enabled encoding: #{encoding}"
+      end
+
+      # return Capybara session instance
+      @browser
+    end
+  end
+end

data/lib/kimurai/browser_builder.rb

@@ -0,0 +1,20 @@
+module Kimurai
+  module BrowserBuilder
+    def self.build(engine, config = {}, spider:)
+      if config[:browser].present?
+        raise "++++++ BrowserBuilder: browser option is depricated. Now all sub-options inside " \
+          "`browser` should be placed right into `@config` hash, without `browser` parent key.\n" \
+          "See more here: https://github.com/vifreefly/kimuraframework/blob/master/CHANGELOG.md#breaking-changes-110 ++++++"
+      end
+
+      begin
+        require "kimurai/browser_builder/#{engine}_builder"
+      rescue LoadError => e
+      end
+
+      builder_class_name = "#{engine}_builder".classify
+      builder = "Kimurai::BrowserBuilder::#{builder_class_name}".constantize
+      builder.new(config, spider: spider).build
+    end
+  end
+end

data/lib/kimurai/capybara_configuration.rb

@@ -0,0 +1,10 @@
+require 'capybara'
+
+Capybara.configure do |config|
+  config.run_server = false
+  config.default_selector = :xpath
+  config.save_path = "tmp"
+  config.default_max_wait_time = 10
+  config.ignore_hidden_elements = false
+  config.threadsafe = true
+end

data/lib/kimurai/capybara_ext/driver/base.rb

@@ -0,0 +1,62 @@
+require 'pathname'
+
+class Capybara::Driver::Base
+  attr_accessor :visited
+  attr_writer :requests, :responses
+
+  def requests
+    @requests ||= 0
+  end
+
+  def responses
+    @responses ||= 0
+  end
+
+  def current_memory
+    driver_pid = pid
+
+    all = (get_descendant_processes(driver_pid) << driver_pid).uniq
+    all.map { |pid| get_process_memory(pid) }.sum
+  end
+
+  private
+
+  def get_descendant_processes(base)
+    descendants = Hash.new { |ht, k| ht[k] = [k] }
+    Hash[*`ps -eo pid,ppid`.scan(/\d+/).map(&:to_i)].each do |pid, ppid|
+      descendants[ppid] << descendants[pid]
+    end
+
+    descendants[base].flatten - [base]
+  end
+
+  # https://github.com/schneems/get_process_mem
+  # Note: for Linux takes PSS (not RSS) memory (I think PSS better fits in this case)
+  def get_process_memory(pid)
+    case @platform ||= Gem::Platform.local.os
+    when "linux"
+      begin
+        file = Pathname.new "/proc/#{pid}/smaps"
+        return 0 unless file.exist?
+
+        lines = file.each_line.select { |line| line.match(/^Pss/) }
+        return 0 if lines.empty?
+
+        lines.reduce(0) do |sum, line|
+          line.match(/(?<value>(\d*\.{0,1}\d+))\s+(?<unit>\w\w)/) do |m|
+            sum += m[:value].to_i
+          end
+
+          sum
+        end
+      rescue Errno::EACCES
+        0
+      end
+    when "darwin"
+      mem = `ps -o rss= -p #{pid}`.strip
+      mem.empty? ? 0 : mem.to_i
+    else
+      raise "Can't check process memory, wrong type of platform: #{@platform}"
+    end
+  end
+end

data/lib/kimurai/capybara_ext/mechanize/driver.rb

@@ -0,0 +1,71 @@
+require 'mechanize'
+require_relative '../driver/base'
+
+class Capybara::Mechanize::Driver
+  # Extend capybara-mechnize to support Poltergeist-like methods
+  # https://www.rubydoc.info/gems/poltergeist/Capybara/Poltergeist/Driver
+
+  def set_proxy(ip, port, type, user = nil, password = nil)
+    # type is always "http", "socks" is not supported (yet)
+    browser.agent.set_proxy(ip, port, user, password)
+  end
+
+  ###
+
+  def headers
+    browser.agent.request_headers
+  end
+
+  def headers=(headers)
+    browser.agent.request_headers = headers
+  end
+
+  def add_header(name, value)
+    browser.agent.request_headers[name] = value
+  end
+
+  ###
+
+  def get_cookies
+    browser.agent.cookies
+  end
+
+  def set_cookie(name, value, options = {})
+    options[:name]  ||= name
+    options[:value] ||= value
+
+    cookie = Mechanize::Cookie.new(options.merge path: "/")
+    browser.agent.cookie_jar << cookie
+  end
+
+  def set_cookies(cookies)
+    cookies.each do |cookie|
+      set_cookie(cookie[:name], cookie[:value], cookie)
+    end
+  end
+
+  def clear_cookies
+    browser.agent.cookie_jar.clear!
+  end
+
+  ###
+
+  def quit
+    browser.agent.shutdown
+  end
+
+  ###
+
+  # Reset parent method `current_memory` for mechanize (we can't measure memory of Mechanize driver)
+  def current_memory
+    nil
+  end
+
+  def pid
+    nil
+  end
+
+  def port
+    nil
+  end
+end

data/lib/kimurai/capybara_ext/poltergeist/driver.rb

@@ -0,0 +1,13 @@
+require_relative '../driver/base'
+
+module Capybara::Poltergeist
+  class Driver
+    def pid
+      client_pid
+    end
+
+    def port
+      server.port
+    end
+  end
+end

data/lib/kimurai/capybara_ext/selenium/driver.rb

@@ -0,0 +1,34 @@
+require_relative '../driver/base'
+
+class Capybara::Selenium::Driver
+  def get_cookies
+    browser.manage.all_cookies
+  end
+
+  def set_cookie(name, value, options = {})
+    options[:name]  ||= name
+    options[:value] ||= value
+
+    browser.manage.add_cookie(options)
+  end
+
+  def set_cookies(cookies)
+    cookies.each do |cookie|
+      set_cookie(cookie[:name], cookie[:value], cookie)
+    end
+  end
+
+  def clear_cookies
+    browser.manage.delete_all_cookies
+  end
+
+  ###
+
+  def pid
+    @pid ||= `lsof -i tcp:#{port} -t`.strip.to_i
+  end
+
+  def port
+    @port ||= browser.send(:bridge).instance_variable_get("@http").instance_variable_get("@server_url").port
+  end
+end

data/lib/kimurai/capybara_ext/session/config.rb

@@ -0,0 +1,22 @@
+module Capybara
+  class SessionConfig
+    attr_accessor :cookies, :proxy, :user_agent, :encoding
+    attr_writer :retry_request_errors, :skip_request_errors
+
+    def retry_request_errors
+      @retry_request_errors ||= []
+    end
+
+    def skip_request_errors
+      @skip_request_errors ||= []
+    end
+
+    def restart_if
+      @restart_if ||= {}
+    end
+
+    def before_request
+      @before_request ||= {}
+    end
+  end
+end