kimurai 1.3.2 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71ac47a69324914eaa098231d72b4af453e9e222d4fc7b19f0b2ef88449c7c18
-  data.tar.gz: e5ac63a13897f92118b3bbe33bc55f8f46e4970333ffeedf88c3f64e2df791e8
+  metadata.gz: 9efd6a3590e9df217a803d9e7c806b052d1c3c1cd9887294d7c736ee19fc8e7b
+  data.tar.gz: 14d14793e808f5122158ae315da53bbb7484772274cc332e96064300016a3974
 SHA512:
-  metadata.gz: 11555b24a707d857c37de7952654cc2d5cdc5d5c188cd15aa364b02126e3abeff71b65a86ddfae276fc8d2136f7b2e0b3b25afe7b798e9af1674bfb0b6796c03
-  data.tar.gz: '094fb4c7cac6a326c76758c07e3ca9e09ab36d68e8da3d80465493c93a6fa5a664e332ec55ea3d5b05f1d52dd3327bd732c43149db95b04e59bf2fe85f36ec15'
+  metadata.gz: 6224c7b4bccdbe92b610cd47fa04ed01dc9366fc7b12fb35edac218d71cdd24b7644fd7e5ee5b195d389395f6d147bc7ec8716c33e2d9227fde1cc1204b4a1e4
+  data.tar.gz: 2c6789e7c62dfe999b641cae172f28c0eb609860de4e123fd0305934ef251fedfa2e6a299d3705f395bdbd483d3cd1004ec1804a98a5517ad218a789d98702f5

data/CHANGELOG.md

@@ -1,4 +1,12 @@
 # CHANGELOG
+## 1.4.0
+### New
+* Add `encoding` config option (see [All available config options](https://github.com/vifreefly/kimuraframework#all-available-config-options))
+* Validate url before processing a request (Base#request_to)
+
+### Fixes
+* Fix console command bug (see [issue 21](https://github.com/vifreefly/kimuraframework/issues/21))
+
 ## 1.3.2
 ### Fixes
 * In the project template, set Ruby version as >= 2.5 (before was hard-coded to 2.5.1)

data/README.md

@@ -10,15 +10,12 @@
 > * The code was massively refactored for a [support](#using-kimurai-inside-existing-ruby-application) to run spiders multiple times from inside a single process. Now it's possible to run Kimurai spiders using background jobs like Sidekiq.
 > * `require 'kimurai'` doesn't require any gems except Active Support. Only when a particular spider [starts](#crawl-method), Capybara will be required with a specific driver.
 > * Although Kimurai [extends](lib/kimurai/capybara_ext) Capybara (all the magic happens inside [extended](lib/kimurai/capybara_ext/session.rb) `Capybara::Session#visit` method), session instances which were created manually will behave normally.
-> * No spaghetti code with `case/when/end` blocks anymore. All drivers [were extended](lib/kimurai/capybara_ext) to support unified methods for cookies, proxies, headers, etc.
-> * `selenium_url_to_set_cookies` @config option don't need anymore if you're use Selenium-like engine with custom cookies setting.
 > * Small changes in design (check the readme again to see what was changed)
 > * Stats database with a web dashboard were removed
-> * Again, massive refactor. Code now looks much better than it was before.
 
 <br>
 
-> Note: this readme is for `1.3.2` gem version. CHANGELOG [here](CHANGELOG.md).
+> Note: this readme is for `1.4.0` gem version. CHANGELOG [here](CHANGELOG.md).
 
 Kimurai is a modern web scraping framework written in Ruby which **works out of box with Headless Chromium/Firefox, PhantomJS**, or simple HTTP requests and **allows to scrape and interact with JavaScript rendered websites.**
 
@@ -1592,6 +1589,12 @@ end
   # Format: same like for `skip_request_errors` option.
   retry_request_errors: [Net::ReadTimeout],
 
+  # Handle page encoding while parsing html response using Nokogiri. There are two modes:
+  # Auto (`:auto`) (try to fetch correct encoding from <meta http-equiv="Content-Type"> or <meta charset> tags)
+  # Set required encoding manually, example: `encoding: "GB2312"` (Set required encoding manually)
+  # Default this option is unset.
+  encoding: nil,
+
   # Restart browser if one of the options is true:
   restart_if: {
     # Restart browser if provided memory limit (in kilobytes) is exceeded (works for all engines)
@@ -1740,8 +1743,6 @@ end
 ### Crawl
 To run a particular spider in the project, run: `$ bundle exec kimurai crawl example_spider`. Don't forget to add `bundle exec` before command to load required environment.
 
-You can provide an additional option `--continue` to use [persistence storage database](#persistence-database-for-the-storage) feature.
-
 ### List
 To list all project spiders, run: `$ bundle exec kimurai list`
 

data/lib/kimurai/base.rb

@@ -3,6 +3,8 @@ require_relative 'base/storage'
 
 module Kimurai
   class Base
+    class InvalidUrlError < StandardError; end
+
     # don't deep merge config's headers hash option
     DMERGE_EXCLUDE = [:headers]
 
@@ -171,7 +173,7 @@ module Kimurai
     attr_accessor :with_info
 
     def initialize(engine = self.class.engine, config: {})
-      @engine = engine
+      @engine = engine || self.class.engine
       @config = self.class.config.deep_merge_excl(config, DMERGE_EXCLUDE)
       @pipelines = self.class.pipelines.map do |pipeline_name|
         klass = Pipeline.descendants.find { |kl| kl.name == pipeline_name }
@@ -189,6 +191,8 @@ module Kimurai
     end
 
     def request_to(handler, delay = nil, url:, data: {}, response_type: :html)
+      raise InvalidUrlError, "Requested url is invalid: #{url}" unless URI.parse(url).kind_of?(URI::HTTP)
+
       if @config[:skip_duplicate_requests] && !unique_request?(url)
         add_event(:duplicate_requests) if self.with_info
         logger.warn "Spider: request_to: not unique url: #{url}, skipped" and return

data/lib/kimurai/browser_builder.rb

@@ -1,38 +1,20 @@
 module Kimurai
-  class BrowserBuilder
-    AVAILABLE_ENGINES = [
-      :mechanize,
-      :mechanize_standalone,
-      :poltergeist_phantomjs,
-      :selenium_firefox,
-      :selenium_chrome
-    ]
-
+  module BrowserBuilder
     def self.build(engine, config = {}, spider:)
-      unless AVAILABLE_ENGINES.include? engine
-        raise "BrowserBuilder: wrong name of engine, available engines: #{AVAILABLE_ENGINES.join(', ')}"
-      end
-
       if config[:browser].present?
         raise "++++++ BrowserBuilder: browser option is depricated. Now all sub-options inside " \
           "`browser` should be placed right into `@config` hash, without `browser` parent key.\n" \
           "See more here: https://github.com/vifreefly/kimuraframework/blob/master/CHANGELOG.md#breaking-changes-110 ++++++"
       end
 
-      case engine
-      when :mechanize
-        require_relative 'browser_builder/mechanize_builder'
-        MechanizeBuilder.new(config, spider: spider).build
-      when :selenium_chrome
-        require_relative 'browser_builder/selenium_chrome_builder'
-        SeleniumChromeBuilder.new(config, spider: spider).build
-      when :poltergeist_phantomjs
-        require_relative 'browser_builder/poltergeist_phantomjs_builder'
-        PoltergeistPhantomJSBuilder.new(config, spider: spider).build
-      when :selenium_firefox
-        require_relative 'browser_builder/selenium_firefox_builder'
-        SeleniumFirefoxBuilder.new(config, spider: spider).build
+      begin
+        require "kimurai/browser_builder/#{engine}_builder"
+      rescue LoadError => e
       end
+
+      builder_class_name = "#{engine}_builder".classify
+      builder = "Kimurai::BrowserBuilder::#{builder_class_name}".constantize
+      builder.new(config, spider: spider).build
     end
   end
 end

data/lib/kimurai/browser_builder/mechanize_builder.rb

@@ -4,147 +4,151 @@ require_relative '../capybara_configuration'
 require_relative '../capybara_ext/mechanize/driver'
 require_relative '../capybara_ext/session'
 
-module Kimurai
-  class BrowserBuilder
-    class MechanizeBuilder
-      attr_reader :logger, :spider
-
-      def initialize(config, spider:)
-        @config = config
-        @spider = spider
-        @logger = spider.logger
-      end
-
-      def build
-        # Register driver
-        Capybara.register_driver :mechanize do |app|
-          driver = Capybara::Mechanize::Driver.new("app")
-          # keep the history as small as possible (by default it's unlimited)
-          driver.configure { |a| a.history.max_size = 2 }
-          driver
-        end
+module Kimurai::BrowserBuilder
+  class MechanizeBuilder
+    attr_reader :logger, :spider
+
+    def initialize(config, spider:)
+      @config = config
+      @spider = spider
+      @logger = spider.logger
+    end
 
-        # Create browser instance (Capybara session)
-        @browser = Capybara::Session.new(:mechanize)
-        @browser.spider = spider
-        logger.debug "BrowserBuilder (mechanize): created browser instance"
+    def build
+      # Register driver
+      Capybara.register_driver :mechanize do |app|
+        driver = Capybara::Mechanize::Driver.new("app")
+        # keep the history as small as possible (by default it's unlimited)
+        driver.configure { |a| a.history.max_size = 2 }
+        driver
+      end
 
-        if @config[:extensions].present?
-          logger.error "BrowserBuilder (mechanize): `extensions` option not supported, skipped"
-        end
+      # Create browser instance (Capybara session)
+      @browser = Capybara::Session.new(:mechanize)
+      @browser.spider = spider
+      logger.debug "BrowserBuilder (mechanize): created browser instance"
 
-        # Proxy
-        if proxy = @config[:proxy].presence
-          proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
-          ip, port, type = proxy_string.split(":")
-
-          if type == "http"
-            @browser.driver.set_proxy(*proxy_string.split(":"))
-            logger.debug "BrowserBuilder (mechanize): enabled http proxy, ip: #{ip}, port: #{port}"
-          else
-            logger.error "BrowserBuilder (mechanize): can't set #{type} proxy (not supported), skipped"
-          end
-        end
+      if @config[:extensions].present?
+        logger.error "BrowserBuilder (mechanize): `extensions` option not supported, skipped"
+      end
 
-        # SSL
-        if ssl_cert_path = @config[:ssl_cert_path].presence
-          @browser.driver.browser.agent.http.ca_file = ssl_cert_path
-          logger.debug "BrowserBuilder (mechanize): enabled custom ssl_cert"
-        end
+      # Proxy
+      if proxy = @config[:proxy].presence
+        proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
+        ip, port, type = proxy_string.split(":")
 
-        if @config[:ignore_ssl_errors].present?
-          @browser.driver.browser.agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          logger.debug "BrowserBuilder (mechanize): enabled ignore_ssl_errors"
+        if type == "http"
+          @browser.driver.set_proxy(*proxy_string.split(":"))
+          logger.debug "BrowserBuilder (mechanize): enabled http proxy, ip: #{ip}, port: #{port}"
+        else
+          logger.error "BrowserBuilder (mechanize): can't set #{type} proxy (not supported), skipped"
         end
+      end
 
-        # Headers
-        if headers = @config[:headers].presence
-          @browser.driver.headers = headers
-          logger.debug "BrowserBuilder (mechanize): enabled custom headers"
-        end
+      # SSL
+      if ssl_cert_path = @config[:ssl_cert_path].presence
+        @browser.driver.browser.agent.http.ca_file = ssl_cert_path
+        logger.debug "BrowserBuilder (mechanize): enabled custom ssl_cert"
+      end
 
-        if user_agent = @config[:user_agent].presence
-          user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
+      if @config[:ignore_ssl_errors].present?
+        @browser.driver.browser.agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        logger.debug "BrowserBuilder (mechanize): enabled ignore_ssl_errors"
+      end
 
-          @browser.driver.add_header("User-Agent", user_agent_string)
-          logger.debug "BrowserBuilder (mechanize): enabled custom user_agent"
-        end
+      # Headers
+      if headers = @config[:headers].presence
+        @browser.driver.headers = headers
+        logger.debug "BrowserBuilder (mechanize): enabled custom headers"
+      end
 
-        # Cookies
-        if cookies = @config[:cookies].presence
-          cookies.each do |cookie|
-            @browser.driver.set_cookie(cookie[:name], cookie[:value], cookie)
-          end
+      if user_agent = @config[:user_agent].presence
+        user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
 
-          logger.debug "BrowserBuilder (mechanize): enabled custom cookies"
-        end
+        @browser.driver.add_header("User-Agent", user_agent_string)
+        logger.debug "BrowserBuilder (mechanize): enabled custom user_agent"
+      end
 
-        # Browser instance options
-        # skip_request_errors
-        if skip_errors = @config[:skip_request_errors].presence
-          @browser.config.skip_request_errors = skip_errors
-          logger.debug "BrowserBuilder (mechanize): enabled skip_request_errors"
+      # Cookies
+      if cookies = @config[:cookies].presence
+        cookies.each do |cookie|
+          @browser.driver.set_cookie(cookie[:name], cookie[:value], cookie)
         end
 
-        # retry_request_errors
-        if retry_errors = @config[:retry_request_errors].presence
-          @browser.config.retry_request_errors = retry_errors
-          logger.debug "BrowserBuilder (mechanize): enabled retry_request_errors"
-        end
+        logger.debug "BrowserBuilder (mechanize): enabled custom cookies"
+      end
 
-        # restart_if
-        if @config[:restart_if].present?
-          logger.warn "BrowserBuilder (mechanize): restart_if options not supported by Mechanize, skipped"
-        end
+      # Browser instance options
+      # skip_request_errors
+      if skip_errors = @config[:skip_request_errors].presence
+        @browser.config.skip_request_errors = skip_errors
+        logger.debug "BrowserBuilder (mechanize): enabled skip_request_errors"
+      end
 
-        # before_request clear_cookies
-        if @config.dig(:before_request, :clear_cookies)
-          @browser.config.before_request[:clear_cookies] = true
-          logger.debug "BrowserBuilder (mechanize): enabled before_request.clear_cookies"
-        end
+      # retry_request_errors
+      if retry_errors = @config[:retry_request_errors].presence
+        @browser.config.retry_request_errors = retry_errors
+        logger.debug "BrowserBuilder (mechanize): enabled retry_request_errors"
+      end
 
-        # before_request clear_and_set_cookies
-        if @config.dig(:before_request, :clear_and_set_cookies)
-          if cookies = @config[:cookies].presence
-            @browser.config.cookies = cookies
-            @browser.config.before_request[:clear_and_set_cookies] = true
-            logger.debug "BrowserBuilder (mechanize): enabled before_request.clear_and_set_cookies"
-          else
-            logger.error "BrowserBuilder (mechanize): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
-          end
-        end
+      # restart_if
+      if @config[:restart_if].present?
+        logger.warn "BrowserBuilder (mechanize): restart_if options not supported by Mechanize, skipped"
+      end
+
+      # before_request clear_cookies
+      if @config.dig(:before_request, :clear_cookies)
+        @browser.config.before_request[:clear_cookies] = true
+        logger.debug "BrowserBuilder (mechanize): enabled before_request.clear_cookies"
+      end
 
-        # before_request change_user_agent
-        if @config.dig(:before_request, :change_user_agent)
-          if @config[:user_agent].present? && @config[:user_agent].class == Proc
-            @browser.config.user_agent = @config[:user_agent]
-            @browser.config.before_request[:change_user_agent] = true
-            logger.debug "BrowserBuilder (mechanize): enabled before_request.change_user_agent"
-          else
-            logger.error "BrowserBuilder (mechanize): user_agent should be present and has lambda format to enable before_request.change_user_agent, skipped"
-          end
+      # before_request clear_and_set_cookies
+      if @config.dig(:before_request, :clear_and_set_cookies)
+        if cookies = @config[:cookies].presence
+          @browser.config.cookies = cookies
+          @browser.config.before_request[:clear_and_set_cookies] = true
+          logger.debug "BrowserBuilder (mechanize): enabled before_request.clear_and_set_cookies"
+        else
+          logger.error "BrowserBuilder (mechanize): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
         end
+      end
 
-        # before_request change_proxy
-        if @config.dig(:before_request, :change_proxy)
-          if @config[:proxy].present? && @config[:proxy].class == Proc
-            @browser.config.proxy = @config[:proxy]
-            @browser.config.before_request[:change_proxy] = true
-            logger.debug "BrowserBuilder (mechanize): enabled before_request.change_proxy"
-          else
-            logger.error "BrowserBuilder (mechanize): proxy should be present and has lambda format to enable before_request.change_proxy, skipped"
-          end
+      # before_request change_user_agent
+      if @config.dig(:before_request, :change_user_agent)
+        if @config[:user_agent].present? && @config[:user_agent].class == Proc
+          @browser.config.user_agent = @config[:user_agent]
+          @browser.config.before_request[:change_user_agent] = true
+          logger.debug "BrowserBuilder (mechanize): enabled before_request.change_user_agent"
+        else
+          logger.error "BrowserBuilder (mechanize): user_agent should be present and has lambda format to enable before_request.change_user_agent, skipped"
         end
+      end
 
-        # before_request delay
-        if delay = @config.dig(:before_request, :delay).presence
-          @browser.config.before_request[:delay] = delay
-          logger.debug "BrowserBuilder (mechanize): enabled before_request.delay"
+      # before_request change_proxy
+      if @config.dig(:before_request, :change_proxy)
+        if @config[:proxy].present? && @config[:proxy].class == Proc
+          @browser.config.proxy = @config[:proxy]
+          @browser.config.before_request[:change_proxy] = true
+          logger.debug "BrowserBuilder (mechanize): enabled before_request.change_proxy"
+        else
+          logger.error "BrowserBuilder (mechanize): proxy should be present and has lambda format to enable before_request.change_proxy, skipped"
         end
+      end
+
+      # before_request delay
+      if delay = @config.dig(:before_request, :delay).presence
+        @browser.config.before_request[:delay] = delay
+        logger.debug "BrowserBuilder (mechanize): enabled before_request.delay"
+      end
 
-        # return Capybara session instance
-        @browser
+      # encoding
+      if encoding = @config[:encoding]
+        @browser.config.encoding = encoding
+        logger.debug "BrowserBuilder (mechanize): enabled encoding: #{encoding}"
       end
+
+      # return Capybara session instance
+      @browser
     end
   end
 end

data/lib/kimurai/browser_builder/poltergeist_phantomjs_builder.rb

@@ -4,168 +4,172 @@ require_relative '../capybara_configuration'
 require_relative '../capybara_ext/poltergeist/driver'
 require_relative '../capybara_ext/session'
 
-module Kimurai
-  class BrowserBuilder
-    class PoltergeistPhantomJSBuilder
-      attr_reader :logger, :spider
-
-      def initialize(config, spider:)
-        @config = config
-        @spider = spider
-        @logger = spider.logger
-      end
+module Kimurai::BrowserBuilder
+  class PoltergeistPhantomjsBuilder
+    attr_reader :logger, :spider
+
+    def initialize(config, spider:)
+      @config = config
+      @spider = spider
+      @logger = spider.logger
+    end
 
-      def build
-        # Register driver
-        Capybara.register_driver :poltergeist_phantomjs do |app|
-          # Create driver options
-          driver_options = {
-            js_errors: false, debug: false, inspector: false, phantomjs_options: []
-          }
-
-          if extensions = @config[:extensions].presence
-            driver_options[:extensions] = extensions
-            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled extensions"
-          end
-
-          # Window size
-          if size = @config[:window_size].presence
-            driver_options[:window_size] = size
-            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled window_size"
-          end
-
-          # SSL
-          if ssl_cert_path = @config[:ssl_cert_path].presence
-            driver_options[:phantomjs_options] << "--ssl-certificates-path=#{ssl_cert_path}"
-            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom ssl_cert"
-          end
-
-          if @config[:ignore_ssl_errors].present?
-            driver_options[:phantomjs_options].push("--ignore-ssl-errors=yes", "--ssl-protocol=any")
-            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled ignore_ssl_errors"
-          end
-
-          # Disable images
-          if @config[:disable_images].present?
-            driver_options[:phantomjs_options] << "--load-images=no"
-            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled disable_images"
-          end
-
-          Capybara::Poltergeist::Driver.new(app, driver_options)
+    def build
+      # Register driver
+      Capybara.register_driver :poltergeist_phantomjs do |app|
+        # Create driver options
+        driver_options = {
+          js_errors: false, debug: false, inspector: false, phantomjs_options: []
+        }
+
+        if extensions = @config[:extensions].presence
+          driver_options[:extensions] = extensions
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled extensions"
         end
 
-        # Create browser instance (Capybara session)
-        @browser = Capybara::Session.new(:poltergeist_phantomjs)
-        @browser.spider = spider
-        logger.debug "BrowserBuilder (poltergeist_phantomjs): created browser instance"
-
-        # Proxy
-        if proxy = @config[:proxy].presence
-          proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
-          ip, port, type = proxy_string.split(":")
-
-          if %w(http socks5).include?(type)
-            @browser.driver.set_proxy(*proxy_string.split(":"))
-            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled #{type} proxy, ip: #{ip}, port: #{port}"
-          else
-            logger.error "BrowserBuilder (poltergeist_phantomjs): wrong type of proxy: #{type}, skipped"
-          end
+        # Window size
+        if size = @config[:window_size].presence
+          driver_options[:window_size] = size
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled window_size"
         end
 
-        # Headers
-        if headers = @config[:headers].presence
-          @browser.driver.headers = headers
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom headers"
+        # SSL
+        if ssl_cert_path = @config[:ssl_cert_path].presence
+          driver_options[:phantomjs_options] << "--ssl-certificates-path=#{ssl_cert_path}"
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom ssl_cert"
         end
 
-        if user_agent = @config[:user_agent].presence
-          user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
+        if @config[:ignore_ssl_errors].present?
+          driver_options[:phantomjs_options].push("--ignore-ssl-errors=yes", "--ssl-protocol=any")
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled ignore_ssl_errors"
+        end
 
-          @browser.driver.add_header("User-Agent", user_agent_string)
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom user_agent"
+        # Disable images
+        if @config[:disable_images].present?
+          driver_options[:phantomjs_options] << "--load-images=no"
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled disable_images"
         end
 
-        # Cookies
-        if cookies = @config[:cookies].presence
-          cookies.each do |cookie|
-            @browser.driver.set_cookie(cookie[:name], cookie[:value], cookie)
-          end
+        Capybara::Poltergeist::Driver.new(app, driver_options)
+      end
 
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom cookies"
+      # Create browser instance (Capybara session)
+      @browser = Capybara::Session.new(:poltergeist_phantomjs)
+      @browser.spider = spider
+      logger.debug "BrowserBuilder (poltergeist_phantomjs): created browser instance"
+
+      # Proxy
+      if proxy = @config[:proxy].presence
+        proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
+        ip, port, type = proxy_string.split(":")
+
+        if %w(http socks5).include?(type)
+          @browser.driver.set_proxy(*proxy_string.split(":"))
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled #{type} proxy, ip: #{ip}, port: #{port}"
+        else
+          logger.error "BrowserBuilder (poltergeist_phantomjs): wrong type of proxy: #{type}, skipped"
         end
+      end
 
-        # Browser instance options
-        # skip_request_errors
-        if skip_errors = @config[:skip_request_errors].presence
-          @browser.config.skip_request_errors = skip_errors
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled skip_request_errors"
-        end
+      # Headers
+      if headers = @config[:headers].presence
+        @browser.driver.headers = headers
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom headers"
+      end
 
-        # retry_request_errors
-        if retry_errors = @config[:retry_request_errors].presence
-          @browser.config.retry_request_errors = retry_errors
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled retry_request_errors"
-        end
+      if user_agent = @config[:user_agent].presence
+        user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
 
-        # restart_if
-        if requests_limit = @config.dig(:restart_if, :requests_limit).presence
-          @browser.config.restart_if[:requests_limit] = requests_limit
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled restart_if.requests_limit >= #{requests_limit}"
-        end
+        @browser.driver.add_header("User-Agent", user_agent_string)
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom user_agent"
+      end
 
-        if memory_limit = @config.dig(:restart_if, :memory_limit).presence
-          @browser.config.restart_if[:memory_limit] = memory_limit
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled restart_if.memory_limit >= #{memory_limit}"
+      # Cookies
+      if cookies = @config[:cookies].presence
+        cookies.each do |cookie|
+          @browser.driver.set_cookie(cookie[:name], cookie[:value], cookie)
         end
 
-        # before_request clear_cookies
-        if @config.dig(:before_request, :clear_cookies)
-          @browser.config.before_request[:clear_cookies] = true
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.clear_cookies"
-        end
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom cookies"
+      end
 
-        # before_request clear_and_set_cookies
-        if @config.dig(:before_request, :clear_and_set_cookies)
-          if cookies = @config[:cookies].presence
-            @browser.config.cookies = cookies
-            @browser.config.before_request[:clear_and_set_cookies] = true
-            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.clear_and_set_cookies"
-          else
-            logger.error "BrowserBuilder (poltergeist_phantomjs): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
-          end
-        end
+      # Browser instance options
+      # skip_request_errors
+      if skip_errors = @config[:skip_request_errors].presence
+        @browser.config.skip_request_errors = skip_errors
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled skip_request_errors"
+      end
 
-        # before_request change_user_agent
-        if @config.dig(:before_request, :change_user_agent)
-          if @config[:user_agent].present? && @config[:user_agent].class == Proc
-            @browser.config.user_agent = @config[:user_agent]
-            @browser.config.before_request[:change_user_agent] = true
-            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.change_user_agent"
-          else
-            logger.error "BrowserBuilder (poltergeist_phantomjs): user_agent should be present and has lambda format to enable before_request.change_user_agent, skipped"
-          end
+      # retry_request_errors
+      if retry_errors = @config[:retry_request_errors].presence
+        @browser.config.retry_request_errors = retry_errors
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled retry_request_errors"
+      end
+
+      # restart_if
+      if requests_limit = @config.dig(:restart_if, :requests_limit).presence
+        @browser.config.restart_if[:requests_limit] = requests_limit
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled restart_if.requests_limit >= #{requests_limit}"
+      end
+
+      if memory_limit = @config.dig(:restart_if, :memory_limit).presence
+        @browser.config.restart_if[:memory_limit] = memory_limit
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled restart_if.memory_limit >= #{memory_limit}"
+      end
+
+      # before_request clear_cookies
+      if @config.dig(:before_request, :clear_cookies)
+        @browser.config.before_request[:clear_cookies] = true
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.clear_cookies"
+      end
+
+      # before_request clear_and_set_cookies
+      if @config.dig(:before_request, :clear_and_set_cookies)
+        if cookies = @config[:cookies].presence
+          @browser.config.cookies = cookies
+          @browser.config.before_request[:clear_and_set_cookies] = true
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.clear_and_set_cookies"
+        else
+          logger.error "BrowserBuilder (poltergeist_phantomjs): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
         end
+      end
 
-        # before_request change_proxy
-        if @config.dig(:before_request, :change_proxy)
-          if @config[:proxy].present? && @config[:proxy].class == Proc
-            @browser.config.proxy = @config[:proxy]
-            @browser.config.before_request[:change_proxy] = true
-            logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.change_proxy"
-          else
-            logger.error "BrowserBuilder (poltergeist_phantomjs): proxy should be present and has lambda format to enable before_request.change_proxy, skipped"
-          end
+      # before_request change_user_agent
+      if @config.dig(:before_request, :change_user_agent)
+        if @config[:user_agent].present? && @config[:user_agent].class == Proc
+          @browser.config.user_agent = @config[:user_agent]
+          @browser.config.before_request[:change_user_agent] = true
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.change_user_agent"
+        else
+          logger.error "BrowserBuilder (poltergeist_phantomjs): user_agent should be present and has lambda format to enable before_request.change_user_agent, skipped"
         end
+      end
 
-        # before_request delay
-        if delay = @config.dig(:before_request, :delay).presence
-          @browser.config.before_request[:delay] = delay
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.delay"
+      # before_request change_proxy
+      if @config.dig(:before_request, :change_proxy)
+        if @config[:proxy].present? && @config[:proxy].class == Proc
+          @browser.config.proxy = @config[:proxy]
+          @browser.config.before_request[:change_proxy] = true
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.change_proxy"
+        else
+          logger.error "BrowserBuilder (poltergeist_phantomjs): proxy should be present and has lambda format to enable before_request.change_proxy, skipped"
         end
+      end
 
-        # return Capybara session instance
-        @browser
+      # before_request delay
+      if delay = @config.dig(:before_request, :delay).presence
+        @browser.config.before_request[:delay] = delay
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.delay"
       end
+
+      # encoding
+      if encoding = @config[:encoding]
+        @browser.config.encoding = encoding
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled encoding: #{encoding}"
+      end
+
+      # return Capybara session instance
+      @browser
     end
   end
 end