kimurai 1.0.0

data/lib/kimurai/browser_builder/poltergeist_phantomjs_builder.rb

@@ -0,0 +1,156 @@
+require 'capybara'
+require 'capybara/poltergeist'
+require_relative '../capybara_configuration'
+require_relative '../capybara_ext/poltergeist/driver'
+require_relative '../capybara_ext/session'
+
+module Kimurai
+  class BrowserBuilder
+    class PoltergeistPhantomJSBuilder
+      attr_reader :logger, :spider
+
+      def initialize(config, spider:)
+        @config = config
+        @spider = spider
+        @logger = spider.logger
+      end
+
+      def build
+        # Register driver
+        Capybara.register_driver :poltergeist_phantomjs do |app|
+          # Create driver options
+          driver_options = {
+            js_errors: false, debug: false, inspector: false, phantomjs_options: []
+          }
+
+          # Window size
+          if size = @config[:window_size].presence
+            driver_options[:window_size] = size
+            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled window_size"
+          end
+
+          # SSL
+          if ssl_cert_path = @config[:ssl_cert_path].presence
+            driver_options[:phantomjs_options] << "--ssl-certificates-path=#{ssl_cert_path}"
+            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom ssl_cert"
+          end
+
+          if @config[:ignore_ssl_errors].present?
+            driver_options[:phantomjs_options].push("--ignore-ssl-errors=yes", "--ssl-protocol=any")
+            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled ignore_ssl_errors"
+          end
+
+          # Disable images
+          if @config[:disable_images].present?
+            driver_options[:phantomjs_options] << "--load-images=no"
+            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled disable_images"
+          end
+
+          Capybara::Poltergeist::Driver.new(app, driver_options)
+        end
+
+        # Create browser instance (Capybara session)
+        @browser = Capybara::Session.new(:poltergeist_phantomjs)
+        @browser.spider = spider
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): created browser instance"
+
+        # Proxy
+        if proxy = @config[:proxy].presence
+          proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
+          ip, port, type = proxy_string.split(":")
+
+          @browser.set_proxy(*proxy_string.split(":"))
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled #{type} proxy, ip: #{ip}, port: #{port}"
+        end
+
+        # Headers
+        if headers = @config[:headers].presence
+          @browser.driver.headers = headers
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom headers"
+        end
+
+        if user_agent = @config[:user_agent].presence
+          user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
+
+          @browser.driver.add_header("User-Agent", user_agent_string)
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom user-agent"
+        end
+
+        # Cookies
+        if cookies = @config[:cookies].presence
+          cookies.each do |cookie|
+            @browser.driver.set_cookie(cookie[:name], cookie[:value], cookie)
+          end
+
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom cookies"
+        end
+
+        # Browser instance options
+        # retry_request_errors
+        if errors = @config.dig(:browser, :retry_request_errors).presence
+          @browser.config.retry_request_errors = errors
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled `browser retry_request_errors`"
+        end
+
+        # restart_if
+        if requests_limit = @config.dig(:browser, :restart_if, :requests_limit).presence
+          @browser.config.restart_if[:requests_limit] = requests_limit
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled `browser restart_if requests_limit` >= #{requests_limit}"
+        end
+
+        if memory_limit = @config.dig(:browser, :restart_if, :memory_limit).presence
+          @browser.config.restart_if[:memory_limit] = memory_limit
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled `browser restart_if memory_limit` >= #{memory_limit}"
+        end
+
+        # before_request clear_cookies
+        if @config.dig(:browser, :before_request, :clear_cookies)
+          @browser.config.before_request[:clear_cookies] = true
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled `browser before_request clear_cookies`"
+        end
+
+        # before_request clear_and_set_cookies
+        if @config.dig(:browser, :before_request, :clear_and_set_cookies)
+          if cookies = @config[:cookies].presence
+            @browser.config.cookies = cookies
+            @browser.config.before_request[:clear_and_set_cookies] = true
+            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled `browser before_request clear_and_set_cookies`"
+          else
+            logger.error "BrowserBuilder (poltergeist_phantomjs): `cookies` should be present to enable `browser before_request clear_and_set_cookies`, skipped"
+          end
+        end
+
+        # before_request change_user_agent
+        if @config.dig(:browser, :before_request, :change_user_agent)
+          if @config[:user_agent].present? && @config[:user_agent].class == Proc
+            @browser.config.user_agent = @config[:user_agent]
+            @browser.config.before_request[:change_user_agent] = true
+            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled `browser before_request change_user_agent`"
+          else
+            logger.error "BrowserBuilder (poltergeist_phantomjs): `user_agent` should be present and has lambda format to enable `browser before_request change_user_agent`, skipped"
+          end
+        end
+
+        # before_request change_proxy
+        if @config.dig(:browser, :before_request, :change_proxy)
+          if @config[:proxy].present? && @config[:proxy].class == Proc
+            @browser.config.proxy = @config[:proxy]
+            @browser.config.before_request[:change_proxy] = true
+            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled `browser before_request change_proxy`"
+          else
+            logger.error "BrowserBuilder (poltergeist_phantomjs): `proxy` should be present and has lambda format to enable `browser before_request change_proxy`, skipped"
+          end
+        end
+
+        # before_request delay
+        if delay = @config.dig(:browser, :before_request, :delay).presence
+          @browser.config.before_request[:delay] = delay
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled `browser before_request delay`"
+        end
+
+        # return Capybara session instance
+        @browser
+      end
+    end
+  end
+end

data/lib/kimurai/browser_builder/selenium_chrome_builder.rb

@@ -0,0 +1,178 @@
+require 'capybara'
+require 'selenium-webdriver'
+require_relative '../capybara_configuration'
+require_relative '../capybara_ext/selenium/driver'
+require_relative '../capybara_ext/session'
+
+module Kimurai
+  class BrowserBuilder
+    class SeleniumChromeBuilder
+      class << self
+        attr_accessor :virtual_display
+      end
+
+      attr_reader :logger, :spider
+
+      def initialize(config, spider:)
+        @config = config
+        @spider = spider
+        @logger = spider.logger
+      end
+
+      def build
+        # Register driver
+        Capybara.register_driver :selenium_chrome do |app|
+          # Create driver options
+          default_args = %w[--disable-gpu --no-sandbox --disable-translate]
+          driver_options = Selenium::WebDriver::Chrome::Options.new(args: default_args)
+
+          # Window size
+          if size = @config[:window_size].presence
+            driver_options.args << "--window-size=#{size.join(',')}"
+            logger.debug "BrowserBuilder (selenium_chrome): enabled window_size"
+          end
+
+          # Proxy
+          if proxy = @config[:proxy].presence
+            proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
+            ip, port, type, user, password = proxy_string.split(":")
+
+            if user.nil? && password.nil?
+              driver_options.args << "--proxy-server=#{type}://#{ip}:#{port}"
+              logger.debug "BrowserBuilder (selenium_chrome): enabled #{type} proxy, ip: #{ip}, port: #{port}"
+            else
+              logger.error "BrowserBuilder (selenium_chrome): proxy with authentication doesn't supported by selenium, skipped"
+            end
+          end
+
+          if proxy_bypass_list = @config[:proxy_bypass_list].presence
+            if proxy
+              driver_options.args << "--proxy-bypass-list=#{proxy_bypass_list.join(';')}"
+              logger.debug "BrowserBuilder (selenium_chrome): enabled proxy_bypass_list"
+            else
+              logger.error "BrowserBuilder (selenium_chrome): provide `proxy` to set proxy_bypass_list, skipped"
+            end
+          end
+
+          # SSL
+          if @config[:ignore_ssl_errors].present?
+            driver_options.args << "--ignore-certificate-errors"
+            driver_options.args << "--allow-insecure-localhost"
+            logger.debug "BrowserBuilder (selenium_chrome): enabled ignore_ssl_errors"
+          end
+
+          # Disable images
+          if @config[:disable_images].present?
+            driver_options.prefs["profile.managed_default_content_settings.images"] = 2
+            logger.debug "BrowserBuilder (selenium_chrome): enabled disable_images"
+          end
+
+          # Headers
+          if @config[:headers].present?
+            logger.warn "BrowserBuilder: (selenium_chrome): custom headers doesn't supported by selenium, skipped"
+          end
+
+          if user_agent = @config[:user_agent].presence
+            user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
+            driver_options.args << "--user-agent='#{user_agent_string}'"
+            logger.debug "BrowserBuilder (selenium_chrome): enabled custom user-agent"
+          end
+
+          # Headless mode
+          if ENV["HEADLESS"] != "false"
+            if @config[:headless_mode] == :virtual_display
+              if Gem::Platform.local.os == "linux"
+                unless self.class.virtual_display
+                  require 'headless'
+                  self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
+                  self.class.virtual_display.start
+                end
+
+                logger.debug "BrowserBuilder (selenium_chrome): enabled virtual_display headless_mode"
+              else
+                logger.error "BrowserBuilder (selenium_chrome): virtual_display headless_mode works only " \
+                  "on Linux platform. Browser will run in normal mode. Set `native` mode instead."
+              end
+            else
+              driver_options.args << "--headless"
+              logger.debug "BrowserBuilder (selenium_chrome): enabled native headless_mode"
+            end
+          end
+
+          Capybara::Selenium::Driver.new(app, browser: :chrome, options: driver_options, driver_path: "/usr/local/bin/chromedriver")
+        end
+
+        # Create browser instance (Capybara session)
+        @browser = Capybara::Session.new(:selenium_chrome)
+        @browser.spider = spider
+        logger.debug "BrowserBuilder (selenium_chrome): created browser instance"
+
+        # Window size
+        # if size = @config[:window_size].presence
+        #   @browser.current_window.resize_to(*size)
+        #   logger.debug "BrowserBuilder (selenium_chrome): enabled window_size"
+        # end
+
+        # Cookies
+        if cookies = @config[:cookies].presence
+          @browser.config.cookies = cookies
+          logger.debug "BrowserBuilder (selenium_chrome): enabled custom cookies"
+        end
+
+        # Browser instance options
+        # retry_request_errors
+        if errors = @config.dig(:browser, :retry_request_errors).presence
+          @browser.config.retry_request_errors = errors
+          logger.debug "BrowserBuilder (selenium_chrome): enabled `browser retry_request_errors`"
+        end
+
+        # restart_if
+        if requests_limit = @config.dig(:browser, :restart_if, :requests_limit).presence
+          @browser.config.restart_if[:requests_limit] = requests_limit
+          logger.debug "BrowserBuilder (selenium_chrome): enabled `browser restart_if requests_limit` >= #{requests_limit}"
+        end
+
+        if memory_limit = @config.dig(:browser, :restart_if, :memory_limit).presence
+          @browser.config.restart_if[:memory_limit] = memory_limit
+          logger.debug "BrowserBuilder (selenium_chrome): enabled `browser restart_if memory_limit` >= #{memory_limit}"
+        end
+
+        # before_request clear_cookies
+        if @config.dig(:browser, :before_request, :clear_cookies)
+          @browser.config.before_request[:clear_cookies] = true
+          logger.debug "BrowserBuilder (selenium_chrome): enabled `browser before_request clear_cookies`"
+        end
+
+        # before_request clear_and_set_cookies
+        if @config.dig(:browser, :before_request, :clear_and_set_cookies)
+          if cookies = @config[:cookies].presence
+            @browser.config.cookies = cookies
+            @browser.config.before_request[:clear_and_set_cookies] = true
+            logger.debug "BrowserBuilder (selenium_chrome): enabled `browser before_request clear_and_set_cookies`"
+          else
+            logger.error "BrowserBuilder (selenium_chrome): `cookies` should be present to enable `browser before_request clear_and_set_cookies`, skipped"
+          end
+        end
+
+        # before_request change_user_agent
+        if @config.dig(:browser, :before_request, :change_user_agent)
+          logger.error "BrowserBuilder (selenium_chrome): `browser before_request change_user_agent` option not supported by Selenium, skipped"
+        end
+
+        # before_request change_proxy
+        if @config.dig(:browser, :before_request, :change_proxy)
+          logger.error "BrowserBuilder (selenium_chrome): `browser before_request change_proxy` option not supported by Selenium, skipped"
+        end
+
+        # before_request delay
+        if delay = @config.dig(:browser, :before_request, :delay).presence
+          @browser.config.before_request[:delay] = delay
+          logger.debug "BrowserBuilder (selenium_chrome): enabled `browser before_request delay`"
+        end
+
+        # return Capybara session instance
+        @browser
+      end
+    end
+  end
+end

data/lib/kimurai/browser_builder/selenium_firefox_builder.rb

@@ -0,0 +1,185 @@
+require 'capybara'
+require 'selenium-webdriver'
+require_relative '../capybara_configuration'
+require_relative '../capybara_ext/selenium/driver'
+require_relative '../capybara_ext/session'
+
+module Kimurai
+  class BrowserBuilder
+    class SeleniumFirefoxBuilder
+      class << self
+        attr_accessor :virtual_display
+      end
+
+      attr_reader :logger, :spider
+
+      def initialize(config, spider:)
+        @config = config
+        @spider = spider
+        @logger = spider.logger
+      end
+
+      def build
+        # Register driver
+        Capybara.register_driver :selenium_firefox do |app|
+          # Create driver options
+          driver_options = Selenium::WebDriver::Firefox::Options.new
+          driver_options.profile = Selenium::WebDriver::Firefox::Profile.new
+          driver_options.profile["browser.link.open_newwindow"] = 3 # open windows in tabs
+          driver_options.profile["media.peerconnection.enabled"] = false # disable web rtc
+
+          # Proxy
+          if proxy = @config[:proxy].presence
+            proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
+            ip, port, type, user, password = proxy_string.split(":")
+
+            if user.nil? && password.nil?
+              driver_options.profile["network.proxy.type"] = 1
+              if type == "http"
+                driver_options.profile["network.proxy.http"] = ip
+                driver_options.profile["network.proxy.http_port"] = port.to_i
+                driver_options.profile["network.proxy.ssl"] = ip
+                driver_options.profile["network.proxy.ssl_port"] = port.to_i
+              elsif type == "socks5"
+                driver_options.profile["network.proxy.socks"] = ip
+                driver_options.profile["network.proxy.socks_port"] = port.to_i
+                driver_options.profile["network.proxy.socks_version"] = 5
+                driver_options.profile["network.proxy.socks_remote_dns"] = true
+              end
+              logger.debug "BrowserBuilder (selenium_firefox): enabled #{type} proxy, ip: #{ip}, port: #{port}"
+            else
+              logger.error "BrowserBuilder (selenium_firefox): proxy with authentication doesn't supported by selenium, skipped"
+            end
+          end
+
+          if proxy_bypass_list = @config[:proxy_bypass_list].presence
+            if proxy
+              driver_options.profile["network.proxy.no_proxies_on"] = proxy_bypass_list.join(", ")
+              logger.debug "BrowserBuilder (selenium_firefox): enabled proxy_bypass_list"
+            else
+              logger.error "BrowserBuilder (selenium_firefox): provide `proxy` to set proxy_bypass_list, skipped"
+            end
+          end
+
+          # SSL
+          if @config[:ignore_ssl_errors].present?
+            driver_options.profile.secure_ssl = false
+            driver_options.profile.assume_untrusted_certificate_issuer = true
+            logger.debug "BrowserBuilder (selenium_firefox): enabled ignore_ssl_errors"
+          end
+
+          # Disable images
+          if @config[:disable_images].present?
+            driver_options.profile["permissions.default.image"] = 2
+            logger.debug "BrowserBuilder (selenium_firefox): enabled disable_images"
+          end
+
+          # Headers
+          if @config[:headers].present?
+            logger.warn "BrowserBuilder: (selenium_firefox): custom headers doesn't supported by selenium, skipped"
+          end
+
+          if user_agent = @config[:user_agent].presence
+            user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
+            driver_options.profile["general.useragent.override"] = user_agent_string
+            logger.debug "BrowserBuilder (selenium_firefox): enabled custom user-agent"
+          end
+
+          # Headless mode
+          if ENV["HEADLESS"] != "false"
+            if @config[:headless_mode] == :virtual_display
+              if Gem::Platform.local.os == "linux"
+                unless self.class.virtual_display
+                  require 'headless'
+                  self.class.virtual_display = Headless.new(reuse: true, destroy_at_exit: false)
+                  self.class.virtual_display.start
+                end
+
+                logger.debug "BrowserBuilder (selenium_firefox): enabled virtual_display headless_mode"
+              else
+                logger.error "BrowserBuilder (selenium_firefox): virtual_display headless_mode works only " \
+                  "on Linux platform. Browser will run in normal mode. Set `native` mode instead."
+              end
+            else
+              driver_options.args << "--headless"
+              logger.debug "BrowserBuilder (selenium_firefox): enabled native headless_mode"
+            end
+          end
+
+          Capybara::Selenium::Driver.new(app, browser: :firefox, options: driver_options)
+        end
+
+        # Create browser instance (Capybara session)
+        @browser = Capybara::Session.new(:selenium_firefox)
+        @browser.spider = spider
+        logger.debug "BrowserBuilder (selenium_firefox): created browser instance"
+
+        # Window size
+        if size = @config[:window_size].presence
+          @browser.current_window.resize_to(*size)
+          logger.debug "BrowserBuilder (selenium_firefox): enabled window_size"
+        end
+
+        # Cookies
+        if cookies = @config[:cookies].presence
+          @browser.config.cookies = cookies
+          logger.debug "BrowserBuilder (selenium_firefox): enabled custom cookies"
+        end
+
+        # Browser instance options
+        # retry_request_errors
+        if errors = @config.dig(:browser, :retry_request_errors).presence
+          @browser.config.retry_request_errors = errors
+          logger.debug "BrowserBuilder (selenium_firefox): enabled `browser retry_request_errors`"
+        end
+
+        # restart_if
+        if requests_limit = @config.dig(:browser, :restart_if, :requests_limit).presence
+          @browser.config.restart_if[:requests_limit] = requests_limit
+          logger.debug "BrowserBuilder (selenium_firefox): enabled `browser restart_if requests_limit` >= #{requests_limit}"
+        end
+
+        if memory_limit = @config.dig(:browser, :restart_if, :memory_limit).presence
+          @browser.config.restart_if[:memory_limit] = memory_limit
+          logger.debug "BrowserBuilder (selenium_firefox): enabled `browser restart_if memory_limit` >= #{memory_limit}"
+        end
+
+        # before_request clear_cookies
+        if @config.dig(:browser, :before_request, :clear_cookies)
+          @browser.config.before_request[:clear_cookies] = true
+          logger.debug "BrowserBuilder (selenium_firefox): enabled `browser before_request clear_cookies`"
+        end
+
+        # before_request clear_and_set_cookies
+        if @config.dig(:browser, :before_request, :clear_and_set_cookies)
+          if cookies = @config[:cookies].presence
+            @browser.config.cookies = cookies
+            @browser.config.before_request[:clear_and_set_cookies] = true
+            logger.debug "BrowserBuilder (selenium_firefox): enabled `browser before_request clear_and_set_cookies`"
+          else
+            logger.error "BrowserBuilder (selenium_firefox): `cookies` should be present to enable `browser before_request clear_and_set_cookies`, skipped"
+          end
+        end
+
+        # before_request change_user_agent
+        if @config.dig(:browser, :before_request, :change_user_agent)
+          logger.error "BrowserBuilder (selenium_firefox): `browser before_request change_user_agent` option not supported by Selenium, skipped"
+        end
+
+        # before_request change_proxy
+        if @config.dig(:browser, :before_request, :change_proxy)
+          logger.error "BrowserBuilder (selenium_firefox): `browser before_request change_proxy` option not supported by Selenium, skipped"
+        end
+
+        # before_request delay
+        if delay = @config.dig(:browser, :before_request, :delay).presence
+          @browser.config.before_request[:delay] = delay
+          logger.debug "BrowserBuilder (selenium_firefox): enabled `browser before_request delay`"
+        end
+
+        # return Capybara session instance
+        @browser
+      end
+    end
+  end
+end