kimurai 1.3.2 → 2.0.0

data/Rakefile

@@ -1,10 +1,10 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
+require 'bundler/gem_tasks'
+require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
-  t.test_files = FileList["test/**/*_test.rb"]
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
 end
 
-task :default => :test
+task default: :test

data/bin/console

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
-
-require "bundler/setup"
-require "kimurai"
+require 'bundler/setup'
+require 'kimurai'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -10,5 +9,5 @@ require "kimurai"
 # require "pry"
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start(__FILE__)

data/exe/kimurai

@@ -1,5 +1,4 @@
 #!/usr/bin/env ruby
-
 require 'kimurai'
 require 'kimurai/cli'
 

data/kimurai.gemspec

@@ -1,48 +1,49 @@
-
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "kimurai/version"
+require 'kimurai/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "kimurai"
+  spec.name          = 'kimurai'
   spec.version       = Kimurai::VERSION
-  spec.authors       = ["Victor Afanasev"]
-  spec.email         = ["vicfreefly@gmail.com"]
+  spec.authors       = ['Victor Afanasev']
+  spec.email         = ['vicfreefly@gmail.com']
 
-  spec.summary       = "Modern web scraping framework written in Ruby and based on Capybara/Nokogiri"
-  spec.homepage      = "https://github.com/vifreefly/kimuraframework"
-  spec.license       = "MIT"
+  spec.summary       = 'Modern web scraping framework written in Ruby and based on Capybara/Nokogiri'
+  spec.homepage      = 'https://github.com/vifreefly/kimuraframework'
+  spec.license       = 'MIT'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = "kimurai"
-  spec.require_paths = ["lib"]
-  spec.required_ruby_version = ">= 2.5.0"
-
-  spec.add_dependency "thor"
-  spec.add_dependency "cliver"
-  spec.add_dependency "activesupport"
-  spec.add_dependency "murmurhash3"
-  spec.add_dependency "nokogiri"
-
-  spec.add_dependency "capybara", ">= 2.15", "< 4.0"
-  spec.add_dependency "capybara-mechanize"
-  spec.add_dependency "poltergeist"
-  spec.add_dependency "selenium-webdriver"
-
-  spec.add_dependency "headless"
-  spec.add_dependency "pmap"
-
-  spec.add_dependency "whenever"
-
-  spec.add_dependency "rbcat", "~> 0.2"
-  spec.add_dependency "pry"
-
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.bindir        = 'exe'
+  spec.executables   = 'kimurai'
+  spec.require_paths = ['lib']
+  spec.required_ruby_version = '>= 3.1.0'
+
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'cliver'
+  spec.add_dependency 'csv'
+  spec.add_dependency 'murmurhash3'
+  spec.add_dependency 'nokogiri'
+  spec.add_dependency 'ostruct'
+  spec.add_dependency 'thor'
+
+  # for capybara-mechanize compatibility
+  spec.add_dependency 'mutex_m'
+  spec.add_dependency 'nkf'
+  spec.add_dependency 'reline'
+
+  spec.add_dependency 'capybara', '~> 3.40'
+  spec.add_dependency 'capybara-mechanize', '~> 1.13'
+  spec.add_dependency 'selenium-webdriver', '~> 4.27'
+
+  spec.add_dependency 'headless'
+  spec.add_dependency 'pmap'
+
+  spec.add_dependency 'whenever'
+
+  spec.add_dependency 'pry'
+  spec.add_dependency 'rbcat', '~> 1.0'
 end

data/lib/kimurai/base/saver.rb

@@ -7,9 +7,7 @@ module Kimurai
       attr_reader :format, :path, :position, :append
 
       def initialize(path, format:, position: true, append: false)
-        unless %i(json pretty_json jsonlines csv).include?(format)
-          raise "SimpleSaver: wrong type of format: #{format}"
-        end
+        raise "SimpleSaver: wrong type of format: #{format}" unless %i[json pretty_json jsonlines csv].include?(format)
 
         @path = path
         @format = format
@@ -42,48 +40,48 @@ module Kimurai
       def save_to_json(item)
         data = JSON.generate([item])
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           file_content = File.read(path).sub(/\}\]\Z/, "\}\,")
-          File.open(path, "w") do |f|
-            f.write(file_content + data.sub(/\A\[/, ""))
+          File.open(path, 'w') do |f|
+            f.write(file_content + data.sub(/\A\[/, ''))
           end
         else
-          File.open(path, "w") { |f| f.write(data) }
+          File.open(path, 'w') { |f| f.write(data) }
         end
       end
 
       def save_to_pretty_json(item)
         data = JSON.pretty_generate([item])
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           file_content = File.read(path).sub(/\}\n\]\Z/, "\}\,\n")
-          File.open(path, "w") do |f|
-            f.write(file_content + data.sub(/\A\[\n/, ""))
+          File.open(path, 'w') do |f|
+            f.write(file_content + data.sub(/\A\[\n/, ''))
           end
         else
-          File.open(path, "w") { |f| f.write(data) }
+          File.open(path, 'w') { |f| f.write(data) }
         end
       end
 
       def save_to_jsonlines(item)
         data = JSON.generate(item)
 
-        if @index > 1 || append && File.exists?(path)
-          File.open(path, "a") { |file| file.write("\n" + data) }
+        if @index > 1 || append && File.exist?(path)
+          File.open(path, 'a') { |file| file.write("\n#{data}") }
         else
-          File.open(path, "w") { |file| file.write(data) }
+          File.open(path, 'w') { |file| file.write(data) }
         end
       end
 
       def save_to_csv(item)
         data = flatten_hash(item)
 
-        if @index > 1 || append && File.exists?(path)
-          CSV.open(path, "a+", force_quotes: true) do |csv|
+        if @index > 1 || append && File.exist?(path)
+          CSV.open(path, 'a+', force_quotes: true) do |csv|
             csv << data.values
           end
         else
-          CSV.open(path, "w", force_quotes: true) do |csv|
+          CSV.open(path, 'w', force_quotes: true) do |csv|
             csv << data.keys
             csv << data.values
           end
@@ -102,5 +100,3 @@ module Kimurai
     end
   end
 end
-
-

data/lib/kimurai/base/storage.rb

@@ -24,7 +24,7 @@ module Kimurai
       def add(scope, value)
         @mutex.synchronize do
           database[scope] ||= []
-          if value.kind_of?(Array)
+          if value.is_a?(Array)
             database[scope] += value
             database[scope].uniq!
           else

data/lib/kimurai/base.rb

@@ -1,19 +1,22 @@
+require 'English'
 require_relative 'base/saver'
 require_relative 'base/storage'
 
 module Kimurai
   class Base
+    class InvalidUrlError < StandardError; end
+
     # don't deep merge config's headers hash option
-    DMERGE_EXCLUDE = [:headers]
+    DMERGE_EXCLUDE = [:headers].freeze
 
     LoggerFormatter = proc do |severity, datetime, progname, msg|
       current_thread_id = Thread.current.object_id
-      thread_type = Thread.main == Thread.current ? "M" : "C"
-      output = "%s, [%s#%d] [%s: %s] %5s -- %s: %s\n"
-        .freeze % [severity[0..0], datetime, $$, thread_type, current_thread_id, severity, progname, msg]
+      thread_type = Thread.main == Thread.current ? 'M' : 'C'
+      output = format("%s, [%s#%d] [%s: %s] %5s -- %s: %s\n", severity[0..0], datetime, $PROCESS_ID, thread_type,
+                      current_thread_id, severity, progname, msg)
 
-      if Kimurai.configuration.colorize_logger != false && Kimurai.env == "development"
-        Rbcat.colorize(output, predefined: [:jsonhash, :logger])
+      if Kimurai.configuration.colorize_logger != false && Kimurai.env == 'development'
+        Rbcat.colorize(output, predefined: %i[jsonhash logger])
       else
         output
       end
@@ -49,11 +52,13 @@ module Kimurai
 
     def self.update(type, subtype)
       return unless @run_info
+
       @update_mutex.synchronize { @run_info[type][subtype] += 1 }
     end
 
     def self.add_event(scope, event)
       return unless @run_info
+
       @update_mutex.synchronize { @run_info[:events][scope][event] += 1 }
     end
 
@@ -91,9 +96,9 @@ module Kimurai
 
     def self.logger
       @logger ||= Kimurai.configuration.logger || begin
-        log_level = (ENV["LOG_LEVEL"] || Kimurai.configuration.log_level || "DEBUG").to_s.upcase
+        log_level = (ENV['LOG_LEVEL'] || Kimurai.configuration.log_level || 'DEBUG').to_s.upcase
         log_level = "Logger::#{log_level}".constantize
-        Logger.new(STDOUT, formatter: LoggerFormatter, level: log_level, progname: name)
+        Logger.new($stdout, formatter: LoggerFormatter, level: log_level, progname: name)
       end
     end
 
@@ -114,13 +119,13 @@ module Kimurai
       ###
 
       logger.info "Spider: started: #{name}"
-      open_spider if self.respond_to? :open_spider
+      open_spider if respond_to? :open_spider
 
-      spider = self.new
+      spider = new
       spider.with_info = true
       if start_urls
         start_urls.each do |start_url|
-          if start_url.class == Hash
+          if start_url.instance_of?(Hash)
             spider.request_to(:parse, start_url)
           else
             spider.request_to(:parse, url: start_url)
@@ -136,13 +141,13 @@ module Kimurai
       @run_info.merge!(status: :completed)
     ensure
       if spider
-        spider.browser.destroy_driver! if spider.instance_variable_get("@browser")
+        spider.browser.destroy_driver! if spider.instance_variable_get('@browser')
 
         stop_time  = Time.now
         total_time = (stop_time - @run_info[:start_time]).round(3)
         @run_info.merge!(stop_time: stop_time, running_time: total_time)
 
-        close_spider if self.respond_to? :close_spider
+        close_spider if respond_to? :close_spider
 
         message = "Spider: stopped: #{@run_info.merge(running_time: @run_info[:running_time]&.duration)}"
         failed? ? logger.fatal(message) : logger.info(message)
@@ -152,7 +157,7 @@ module Kimurai
     end
 
     def self.parse!(handler, *args, **request)
-      spider = self.new
+      spider = new
 
       if args.present?
         spider.public_send(handler, *args)
@@ -162,7 +167,7 @@ module Kimurai
         spider.public_send(handler)
       end
     ensure
-      spider.browser.destroy_driver! if spider.instance_variable_get("@browser")
+      spider.browser.destroy_driver! if spider.instance_variable_get('@browser')
     end
 
     ###
@@ -171,7 +176,7 @@ module Kimurai
     attr_accessor :with_info
 
     def initialize(engine = self.class.engine, config: {})
-      @engine = engine
+      @engine = engine || self.class.engine
       @config = self.class.config.deep_merge_excl(config, DMERGE_EXCLUDE)
       @pipelines = self.class.pipelines.map do |pipeline_name|
         klass = Pipeline.descendants.find { |kl| kl.name == pipeline_name }
@@ -189,15 +194,17 @@ module Kimurai
     end
 
     def request_to(handler, delay = nil, url:, data: {}, response_type: :html)
+      raise InvalidUrlError, "Requested url is invalid: #{url}" unless URI.parse(url).is_a?(URI::HTTP)
+
       if @config[:skip_duplicate_requests] && !unique_request?(url)
-        add_event(:duplicate_requests) if self.with_info
+        add_event(:duplicate_requests) if with_info
         logger.warn "Spider: request_to: not unique url: #{url}, skipped" and return
       end
 
       visited = delay ? browser.visit(url, delay: delay) : browser.visit(url)
       return unless visited
 
-      public_send(handler, browser.current_response(response_type), { url: url, data: data })
+      public_send(handler, browser.current_response(response_type), **{ url: url, data: data })
     end
 
     def console(response = nil, url: nil, data: {})
@@ -207,9 +214,9 @@ module Kimurai
     ###
 
     def storage
-      # Note: for `.crawl!` uses shared thread safe Storage instance,
+      # NOTE: for `.crawl!` uses shared thread safe Storage instance,
       # otherwise, each spider instance will have it's own Storage
-      @storage ||= self.with_info ? self.class.storage : Storage.new
+      @storage ||= with_info ? self.class.storage : Storage.new
     end
 
     def unique?(scope, value)
@@ -219,10 +226,10 @@ module Kimurai
     def save_to(path, item, format:, position: true, append: false)
       @savers[path] ||= begin
         options = { format: format, position: position, append: append }
-        if self.with_info
-          self.class.savers[path] ||= Saver.new(path, options)
+        if with_info
+          self.class.savers[path] ||= Saver.new(path, **options)
         else
-          Saver.new(path, options)
+          Saver.new(path, **options)
         end
       end
 
@@ -232,11 +239,8 @@ module Kimurai
     ###
 
     def add_event(scope = :custom, event)
-      unless self.with_info
-        raise "It's allowed to use `add_event` only while performing a full run (`.crawl!` method)"
-      end
+      self.class.add_event(scope, event) if with_info
 
-      self.class.add_event(scope, event)
       logger.info "Spider: new event (scope: #{scope}): #{event}" if scope == :custom
     end
 
@@ -250,35 +254,35 @@ module Kimurai
 
     def unique_request?(url)
       options = @config[:skip_duplicate_requests]
-      if options.class == Hash
+      if options.instance_of?(Hash)
         scope = options[:scope] || :requests_urls
         if options[:check_only]
           storage.include?(scope, url) ? false : true
         else
-          storage.unique?(scope, url) ? true : false
+          storage.unique?(scope, url) || false
         end
       else
-        storage.unique?(:requests_urls, url) ? true : false
+        storage.unique?(:requests_urls, url) || false
       end
     end
 
     def send_item(item, options = {})
       logger.debug "Pipeline: starting processing item through #{@pipelines.size} #{'pipeline'.pluralize(@pipelines.size)}..."
-      self.class.update(:items, :sent) if self.with_info
+      self.class.update(:items, :sent) if with_info
 
       @pipelines.each do |name, instance|
         item = options[name] ? instance.process_item(item, options: options[name]) : instance.process_item(item)
       end
-    rescue => e
+    rescue StandardError => e
       logger.error "Pipeline: dropped: #{e.inspect} (#{e.backtrace.first}), item: #{item}"
-      add_event(:drop_items_errors, e.inspect) if self.with_info
+      add_event(:drop_items_errors, e.inspect) if with_info
       false
     else
-      self.class.update(:items, :processed) if self.with_info
+      self.class.update(:items, :processed) if with_info
       logger.info "Pipeline: processed: #{JSON.generate(item)}"
       true
     ensure
-      if self.with_info
+      if with_info
         logger.info "Info: items: sent: #{self.class.items[:sent]}, processed: #{self.class.items[:processed]}"
       end
     end
@@ -296,10 +300,10 @@ module Kimurai
           Thread.current.abort_on_exception = true
 
           spider = self.class.new(engine, config: @config.deep_merge_excl(config, DMERGE_EXCLUDE))
-          spider.with_info = true if self.with_info
+          spider.with_info = true if with_info
 
           part.each do |url_data|
-            if url_data.class == Hash
+            if url_data.instance_of?(Hash)
               if url_data[:url].present? && url_data[:data].present?
                 spider.request_to(handler, delay, url_data)
               else
@@ -310,7 +314,7 @@ module Kimurai
             end
           end
         ensure
-          spider.browser.destroy_driver! if spider.instance_variable_get("@browser")
+          spider.browser.destroy_driver! if spider.instance_variable_get('@browser')
         end
 
         sleep 0.5

data/lib/kimurai/base_helper.rb

@@ -4,13 +4,14 @@ module Kimurai
 
     def absolute_url(url, base:)
       return unless url
-      URI.join(base, URI.escape(url)).to_s
+
+      URI.join(base, URI::DEFAULT_PARSER.escape(url)).to_s
     end
 
     def escape_url(url)
-      uri = URI.parse(url)
-    rescue URI::InvalidURIError => e
-      URI.parse(URI.escape url).to_s rescue url
+      URI.parse(url)
+    rescue URI::InvalidURIError
+      URI.parse(URI::DEFAULT_PARSER.escape(url)).to_s rescue url
     else
       url
     end

data/lib/kimurai/browser_builder/mechanize_builder.rb

@@ -5,7 +5,7 @@ require_relative '../capybara_ext/mechanize/driver'
 require_relative '../capybara_ext/session'
 
 module Kimurai
-  class BrowserBuilder
+  module BrowserBuilder
     class MechanizeBuilder
       attr_reader :logger, :spider
 
@@ -17,8 +17,8 @@ module Kimurai
 
       def build
         # Register driver
-        Capybara.register_driver :mechanize do |app|
-          driver = Capybara::Mechanize::Driver.new("app")
+        Capybara.register_driver :mechanize do |_app|
+          driver = Capybara::Mechanize::Driver.new('app')
           # keep the history as small as possible (by default it's unlimited)
           driver.configure { |a| a.history.max_size = 2 }
           driver
@@ -27,19 +27,19 @@ module Kimurai
         # Create browser instance (Capybara session)
         @browser = Capybara::Session.new(:mechanize)
         @browser.spider = spider
-        logger.debug "BrowserBuilder (mechanize): created browser instance"
+        logger.debug 'BrowserBuilder (mechanize): created browser instance'
 
         if @config[:extensions].present?
-          logger.error "BrowserBuilder (mechanize): `extensions` option not supported, skipped"
+          logger.error 'BrowserBuilder (mechanize): `extensions` option not supported, skipped'
         end
 
         # Proxy
-        if proxy = @config[:proxy].presence
-          proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
-          ip, port, type = proxy_string.split(":")
+        if (proxy = @config[:proxy].presence)
+          proxy_string = (proxy.instance_of?(Proc) ? proxy.call : proxy).strip
+          ip, port, type = proxy_string.split(':')
 
-          if type == "http"
-            @browser.driver.set_proxy(*proxy_string.split(":"))
+          if type == 'http'
+            @browser.driver.set_proxy(*proxy_string.split(':'))
             logger.debug "BrowserBuilder (mechanize): enabled http proxy, ip: #{ip}, port: #{port}"
           else
             logger.error "BrowserBuilder (mechanize): can't set #{type} proxy (not supported), skipped"
@@ -47,99 +47,105 @@ module Kimurai
         end
 
         # SSL
-        if ssl_cert_path = @config[:ssl_cert_path].presence
+        if (ssl_cert_path = @config[:ssl_cert_path].presence)
           @browser.driver.browser.agent.http.ca_file = ssl_cert_path
-          logger.debug "BrowserBuilder (mechanize): enabled custom ssl_cert"
+          logger.debug 'BrowserBuilder (mechanize): enabled custom ssl_cert'
         end
 
         if @config[:ignore_ssl_errors].present?
           @browser.driver.browser.agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
-          logger.debug "BrowserBuilder (mechanize): enabled ignore_ssl_errors"
+          logger.debug 'BrowserBuilder (mechanize): enabled ignore_ssl_errors'
         end
 
         # Headers
-        if headers = @config[:headers].presence
+        if (headers = @config[:headers].presence)
           @browser.driver.headers = headers
-          logger.debug "BrowserBuilder (mechanize): enabled custom headers"
+          logger.debug 'BrowserBuilder (mechanize): enabled custom headers'
         end
 
-        if user_agent = @config[:user_agent].presence
-          user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
+        if (user_agent = @config[:user_agent].presence)
+          user_agent_string = (user_agent.instance_of?(Proc) ? user_agent.call : user_agent).strip
 
-          @browser.driver.add_header("User-Agent", user_agent_string)
-          logger.debug "BrowserBuilder (mechanize): enabled custom user_agent"
+          @browser.driver.add_header('User-Agent', user_agent_string)
+          logger.debug 'BrowserBuilder (mechanize): enabled custom user_agent'
         end
 
         # Cookies
-        if cookies = @config[:cookies].presence
+        if (cookies = @config[:cookies].presence)
           cookies.each do |cookie|
             @browser.driver.set_cookie(cookie[:name], cookie[:value], cookie)
           end
 
-          logger.debug "BrowserBuilder (mechanize): enabled custom cookies"
+          logger.debug 'BrowserBuilder (mechanize): enabled custom cookies'
         end
 
         # Browser instance options
         # skip_request_errors
-        if skip_errors = @config[:skip_request_errors].presence
+        if (skip_errors = @config[:skip_request_errors].presence)
           @browser.config.skip_request_errors = skip_errors
-          logger.debug "BrowserBuilder (mechanize): enabled skip_request_errors"
+          logger.debug 'BrowserBuilder (mechanize): enabled skip_request_errors'
         end
 
         # retry_request_errors
-        if retry_errors = @config[:retry_request_errors].presence
+        if (retry_errors = @config[:retry_request_errors].presence)
           @browser.config.retry_request_errors = retry_errors
-          logger.debug "BrowserBuilder (mechanize): enabled retry_request_errors"
+          logger.debug 'BrowserBuilder (mechanize): enabled retry_request_errors'
         end
 
         # restart_if
         if @config[:restart_if].present?
-          logger.warn "BrowserBuilder (mechanize): restart_if options not supported by Mechanize, skipped"
+          logger.warn 'BrowserBuilder (mechanize): restart_if options not supported by Mechanize, skipped'
         end
 
         # before_request clear_cookies
         if @config.dig(:before_request, :clear_cookies)
           @browser.config.before_request[:clear_cookies] = true
-          logger.debug "BrowserBuilder (mechanize): enabled before_request.clear_cookies"
+          logger.debug 'BrowserBuilder (mechanize): enabled before_request.clear_cookies'
         end
 
         # before_request clear_and_set_cookies
         if @config.dig(:before_request, :clear_and_set_cookies)
-          if cookies = @config[:cookies].presence
+          if (cookies = @config[:cookies].presence)
             @browser.config.cookies = cookies
             @browser.config.before_request[:clear_and_set_cookies] = true
-            logger.debug "BrowserBuilder (mechanize): enabled before_request.clear_and_set_cookies"
+            logger.debug 'BrowserBuilder (mechanize): enabled before_request.clear_and_set_cookies'
           else
-            logger.error "BrowserBuilder (mechanize): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
+            logger.error 'BrowserBuilder (mechanize): cookies should be present to enable before_request.clear_and_set_cookies, skipped'
           end
         end
 
         # before_request change_user_agent
         if @config.dig(:before_request, :change_user_agent)
-          if @config[:user_agent].present? && @config[:user_agent].class == Proc
+          if @config[:user_agent].present? && @config[:user_agent].instance_of?(Proc)
             @browser.config.user_agent = @config[:user_agent]
             @browser.config.before_request[:change_user_agent] = true
-            logger.debug "BrowserBuilder (mechanize): enabled before_request.change_user_agent"
+            logger.debug 'BrowserBuilder (mechanize): enabled before_request.change_user_agent'
           else
-            logger.error "BrowserBuilder (mechanize): user_agent should be present and has lambda format to enable before_request.change_user_agent, skipped"
+            logger.error 'BrowserBuilder (mechanize): user_agent should be present and has lambda format to enable before_request.change_user_agent, skipped'
           end
         end
 
         # before_request change_proxy
         if @config.dig(:before_request, :change_proxy)
-          if @config[:proxy].present? && @config[:proxy].class == Proc
+          if @config[:proxy].present? && @config[:proxy].instance_of?(Proc)
             @browser.config.proxy = @config[:proxy]
             @browser.config.before_request[:change_proxy] = true
-            logger.debug "BrowserBuilder (mechanize): enabled before_request.change_proxy"
+            logger.debug 'BrowserBuilder (mechanize): enabled before_request.change_proxy'
           else
-            logger.error "BrowserBuilder (mechanize): proxy should be present and has lambda format to enable before_request.change_proxy, skipped"
+            logger.error 'BrowserBuilder (mechanize): proxy should be present and has lambda format to enable before_request.change_proxy, skipped'
           end
         end
 
         # before_request delay
-        if delay = @config.dig(:before_request, :delay).presence
+        if (delay = @config.dig(:before_request, :delay).presence)
           @browser.config.before_request[:delay] = delay
-          logger.debug "BrowserBuilder (mechanize): enabled before_request.delay"
+          logger.debug 'BrowserBuilder (mechanize): enabled before_request.delay'
+        end
+
+        # encoding
+        if (encoding = @config[:encoding])
+          @browser.config.encoding = encoding
+          logger.debug "BrowserBuilder (mechanize): enabled encoding: #{encoding}"
         end
 
         # return Capybara session instance