kimurai 1.4.0 → 2.0.0

data/Rakefile

@@ -1,10 +1,10 @@
-require "bundler/gem_tasks"
-require "rake/testtask"
+require 'bundler/gem_tasks'
+require 'rake/testtask'
 
 Rake::TestTask.new(:test) do |t|
-  t.libs << "test"
-  t.libs << "lib"
-  t.test_files = FileList["test/**/*_test.rb"]
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
 end
 
-task :default => :test
+task default: :test

data/bin/console

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
-
-require "bundler/setup"
-require "kimurai"
+require 'bundler/setup'
+require 'kimurai'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -10,5 +9,5 @@ require "kimurai"
 # require "pry"
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start(__FILE__)

data/exe/kimurai

@@ -1,5 +1,4 @@
 #!/usr/bin/env ruby
-
 require 'kimurai'
 require 'kimurai/cli'
 

data/kimurai.gemspec

@@ -1,48 +1,49 @@
-
-lib = File.expand_path("../lib", __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require "kimurai/version"
+require 'kimurai/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "kimurai"
+  spec.name          = 'kimurai'
   spec.version       = Kimurai::VERSION
-  spec.authors       = ["Victor Afanasev"]
-  spec.email         = ["vicfreefly@gmail.com"]
+  spec.authors       = ['Victor Afanasev']
+  spec.email         = ['vicfreefly@gmail.com']
 
-  spec.summary       = "Modern web scraping framework written in Ruby and based on Capybara/Nokogiri"
-  spec.homepage      = "https://github.com/vifreefly/kimuraframework"
-  spec.license       = "MIT"
+  spec.summary       = 'Modern web scraping framework written in Ruby and based on Capybara/Nokogiri'
+  spec.homepage      = 'https://github.com/vifreefly/kimuraframework'
+  spec.license       = 'MIT'
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = "kimurai"
-  spec.require_paths = ["lib"]
-  spec.required_ruby_version = ">= 2.5.0"
-
-  spec.add_dependency "thor"
-  spec.add_dependency "cliver"
-  spec.add_dependency "activesupport"
-  spec.add_dependency "murmurhash3"
-  spec.add_dependency "nokogiri"
-
-  spec.add_dependency "capybara", ">= 2.15", "< 4.0"
-  spec.add_dependency "capybara-mechanize"
-  spec.add_dependency "poltergeist"
-  spec.add_dependency "selenium-webdriver"
-
-  spec.add_dependency "headless"
-  spec.add_dependency "pmap"
-
-  spec.add_dependency "whenever"
-
-  spec.add_dependency "rbcat", "~> 0.2"
-  spec.add_dependency "pry"
-
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.bindir        = 'exe'
+  spec.executables   = 'kimurai'
+  spec.require_paths = ['lib']
+  spec.required_ruby_version = '>= 3.1.0'
+
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'cliver'
+  spec.add_dependency 'csv'
+  spec.add_dependency 'murmurhash3'
+  spec.add_dependency 'nokogiri'
+  spec.add_dependency 'ostruct'
+  spec.add_dependency 'thor'
+
+  # for capybara-mechanize compatibility
+  spec.add_dependency 'mutex_m'
+  spec.add_dependency 'nkf'
+  spec.add_dependency 'reline'
+
+  spec.add_dependency 'capybara', '~> 3.40'
+  spec.add_dependency 'capybara-mechanize', '~> 1.13'
+  spec.add_dependency 'selenium-webdriver', '~> 4.27'
+
+  spec.add_dependency 'headless'
+  spec.add_dependency 'pmap'
+
+  spec.add_dependency 'whenever'
+
+  spec.add_dependency 'pry'
+  spec.add_dependency 'rbcat', '~> 1.0'
 end

data/lib/kimurai/base/saver.rb

@@ -7,9 +7,7 @@ module Kimurai
       attr_reader :format, :path, :position, :append
 
       def initialize(path, format:, position: true, append: false)
-        unless %i(json pretty_json jsonlines csv).include?(format)
-          raise "SimpleSaver: wrong type of format: #{format}"
-        end
+        raise "SimpleSaver: wrong type of format: #{format}" unless %i[json pretty_json jsonlines csv].include?(format)
 
         @path = path
         @format = format
@@ -42,48 +40,48 @@ module Kimurai
       def save_to_json(item)
         data = JSON.generate([item])
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           file_content = File.read(path).sub(/\}\]\Z/, "\}\,")
-          File.open(path, "w") do |f|
-            f.write(file_content + data.sub(/\A\[/, ""))
+          File.open(path, 'w') do |f|
+            f.write(file_content + data.sub(/\A\[/, ''))
           end
         else
-          File.open(path, "w") { |f| f.write(data) }
+          File.open(path, 'w') { |f| f.write(data) }
         end
       end
 
       def save_to_pretty_json(item)
         data = JSON.pretty_generate([item])
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           file_content = File.read(path).sub(/\}\n\]\Z/, "\}\,\n")
-          File.open(path, "w") do |f|
-            f.write(file_content + data.sub(/\A\[\n/, ""))
+          File.open(path, 'w') do |f|
+            f.write(file_content + data.sub(/\A\[\n/, ''))
           end
         else
-          File.open(path, "w") { |f| f.write(data) }
+          File.open(path, 'w') { |f| f.write(data) }
         end
       end
 
       def save_to_jsonlines(item)
         data = JSON.generate(item)
 
-        if @index > 1 || append && File.exists?(path)
-          File.open(path, "a") { |file| file.write("\n" + data) }
+        if @index > 1 || append && File.exist?(path)
+          File.open(path, 'a') { |file| file.write("\n#{data}") }
         else
-          File.open(path, "w") { |file| file.write(data) }
+          File.open(path, 'w') { |file| file.write(data) }
         end
       end
 
       def save_to_csv(item)
         data = flatten_hash(item)
 
-        if @index > 1 || append && File.exists?(path)
-          CSV.open(path, "a+", force_quotes: true) do |csv|
+        if @index > 1 || append && File.exist?(path)
+          CSV.open(path, 'a+', force_quotes: true) do |csv|
             csv << data.values
           end
         else
-          CSV.open(path, "w", force_quotes: true) do |csv|
+          CSV.open(path, 'w', force_quotes: true) do |csv|
             csv << data.keys
             csv << data.values
           end
@@ -102,5 +100,3 @@ module Kimurai
     end
   end
 end
-
-

data/lib/kimurai/base/storage.rb

@@ -24,7 +24,7 @@ module Kimurai
       def add(scope, value)
         @mutex.synchronize do
           database[scope] ||= []
-          if value.kind_of?(Array)
+          if value.is_a?(Array)
             database[scope] += value
             database[scope].uniq!
           else

data/lib/kimurai/base.rb

@@ -1,3 +1,4 @@
+require 'English'
 require_relative 'base/saver'
 require_relative 'base/storage'
 
@@ -6,16 +7,16 @@ module Kimurai
     class InvalidUrlError < StandardError; end
 
     # don't deep merge config's headers hash option
-    DMERGE_EXCLUDE = [:headers]
+    DMERGE_EXCLUDE = [:headers].freeze
 
     LoggerFormatter = proc do |severity, datetime, progname, msg|
       current_thread_id = Thread.current.object_id
-      thread_type = Thread.main == Thread.current ? "M" : "C"
-      output = "%s, [%s#%d] [%s: %s] %5s -- %s: %s\n"
-        .freeze % [severity[0..0], datetime, $$, thread_type, current_thread_id, severity, progname, msg]
+      thread_type = Thread.main == Thread.current ? 'M' : 'C'
+      output = format("%s, [%s#%d] [%s: %s] %5s -- %s: %s\n", severity[0..0], datetime, $PROCESS_ID, thread_type,
+                      current_thread_id, severity, progname, msg)
 
-      if Kimurai.configuration.colorize_logger != false && Kimurai.env == "development"
-        Rbcat.colorize(output, predefined: [:jsonhash, :logger])
+      if Kimurai.configuration.colorize_logger != false && Kimurai.env == 'development'
+        Rbcat.colorize(output, predefined: %i[jsonhash logger])
       else
         output
       end
@@ -51,11 +52,13 @@ module Kimurai
 
     def self.update(type, subtype)
       return unless @run_info
+
       @update_mutex.synchronize { @run_info[type][subtype] += 1 }
     end
 
     def self.add_event(scope, event)
       return unless @run_info
+
       @update_mutex.synchronize { @run_info[:events][scope][event] += 1 }
     end
 
@@ -93,9 +96,9 @@ module Kimurai
 
     def self.logger
       @logger ||= Kimurai.configuration.logger || begin
-        log_level = (ENV["LOG_LEVEL"] || Kimurai.configuration.log_level || "DEBUG").to_s.upcase
+        log_level = (ENV['LOG_LEVEL'] || Kimurai.configuration.log_level || 'DEBUG').to_s.upcase
         log_level = "Logger::#{log_level}".constantize
-        Logger.new(STDOUT, formatter: LoggerFormatter, level: log_level, progname: name)
+        Logger.new($stdout, formatter: LoggerFormatter, level: log_level, progname: name)
       end
     end
 
@@ -116,13 +119,13 @@ module Kimurai
       ###
 
       logger.info "Spider: started: #{name}"
-      open_spider if self.respond_to? :open_spider
+      open_spider if respond_to? :open_spider
 
-      spider = self.new
+      spider = new
       spider.with_info = true
       if start_urls
         start_urls.each do |start_url|
-          if start_url.class == Hash
+          if start_url.instance_of?(Hash)
             spider.request_to(:parse, start_url)
           else
             spider.request_to(:parse, url: start_url)
@@ -138,13 +141,13 @@ module Kimurai
       @run_info.merge!(status: :completed)
     ensure
       if spider
-        spider.browser.destroy_driver! if spider.instance_variable_get("@browser")
+        spider.browser.destroy_driver! if spider.instance_variable_get('@browser')
 
         stop_time  = Time.now
         total_time = (stop_time - @run_info[:start_time]).round(3)
         @run_info.merge!(stop_time: stop_time, running_time: total_time)
 
-        close_spider if self.respond_to? :close_spider
+        close_spider if respond_to? :close_spider
 
         message = "Spider: stopped: #{@run_info.merge(running_time: @run_info[:running_time]&.duration)}"
         failed? ? logger.fatal(message) : logger.info(message)
@@ -154,7 +157,7 @@ module Kimurai
     end
 
     def self.parse!(handler, *args, **request)
-      spider = self.new
+      spider = new
 
       if args.present?
         spider.public_send(handler, *args)
@@ -164,7 +167,7 @@ module Kimurai
         spider.public_send(handler)
       end
     ensure
-      spider.browser.destroy_driver! if spider.instance_variable_get("@browser")
+      spider.browser.destroy_driver! if spider.instance_variable_get('@browser')
     end
 
     ###
@@ -191,17 +194,17 @@ module Kimurai
     end
 
     def request_to(handler, delay = nil, url:, data: {}, response_type: :html)
-      raise InvalidUrlError, "Requested url is invalid: #{url}" unless URI.parse(url).kind_of?(URI::HTTP)
+      raise InvalidUrlError, "Requested url is invalid: #{url}" unless URI.parse(url).is_a?(URI::HTTP)
 
       if @config[:skip_duplicate_requests] && !unique_request?(url)
-        add_event(:duplicate_requests) if self.with_info
+        add_event(:duplicate_requests) if with_info
         logger.warn "Spider: request_to: not unique url: #{url}, skipped" and return
       end
 
       visited = delay ? browser.visit(url, delay: delay) : browser.visit(url)
       return unless visited
 
-      public_send(handler, browser.current_response(response_type), { url: url, data: data })
+      public_send(handler, browser.current_response(response_type), **{ url: url, data: data })
     end
 
     def console(response = nil, url: nil, data: {})
@@ -211,9 +214,9 @@ module Kimurai
     ###
 
     def storage
-      # Note: for `.crawl!` uses shared thread safe Storage instance,
+      # NOTE: for `.crawl!` uses shared thread safe Storage instance,
       # otherwise, each spider instance will have it's own Storage
-      @storage ||= self.with_info ? self.class.storage : Storage.new
+      @storage ||= with_info ? self.class.storage : Storage.new
     end
 
     def unique?(scope, value)
@@ -223,10 +226,10 @@ module Kimurai
     def save_to(path, item, format:, position: true, append: false)
       @savers[path] ||= begin
         options = { format: format, position: position, append: append }
-        if self.with_info
-          self.class.savers[path] ||= Saver.new(path, options)
+        if with_info
+          self.class.savers[path] ||= Saver.new(path, **options)
         else
-          Saver.new(path, options)
+          Saver.new(path, **options)
         end
       end
 
@@ -236,11 +239,8 @@ module Kimurai
     ###
 
     def add_event(scope = :custom, event)
-      unless self.with_info
-        raise "It's allowed to use `add_event` only while performing a full run (`.crawl!` method)"
-      end
+      self.class.add_event(scope, event) if with_info
 
-      self.class.add_event(scope, event)
       logger.info "Spider: new event (scope: #{scope}): #{event}" if scope == :custom
     end
 
@@ -254,35 +254,35 @@ module Kimurai
 
     def unique_request?(url)
       options = @config[:skip_duplicate_requests]
-      if options.class == Hash
+      if options.instance_of?(Hash)
         scope = options[:scope] || :requests_urls
         if options[:check_only]
           storage.include?(scope, url) ? false : true
         else
-          storage.unique?(scope, url) ? true : false
+          storage.unique?(scope, url) || false
         end
       else
-        storage.unique?(:requests_urls, url) ? true : false
+        storage.unique?(:requests_urls, url) || false
       end
     end
 
     def send_item(item, options = {})
       logger.debug "Pipeline: starting processing item through #{@pipelines.size} #{'pipeline'.pluralize(@pipelines.size)}..."
-      self.class.update(:items, :sent) if self.with_info
+      self.class.update(:items, :sent) if with_info
 
       @pipelines.each do |name, instance|
         item = options[name] ? instance.process_item(item, options: options[name]) : instance.process_item(item)
       end
-    rescue => e
+    rescue StandardError => e
       logger.error "Pipeline: dropped: #{e.inspect} (#{e.backtrace.first}), item: #{item}"
-      add_event(:drop_items_errors, e.inspect) if self.with_info
+      add_event(:drop_items_errors, e.inspect) if with_info
       false
     else
-      self.class.update(:items, :processed) if self.with_info
+      self.class.update(:items, :processed) if with_info
       logger.info "Pipeline: processed: #{JSON.generate(item)}"
       true
     ensure
-      if self.with_info
+      if with_info
         logger.info "Info: items: sent: #{self.class.items[:sent]}, processed: #{self.class.items[:processed]}"
       end
     end
@@ -300,10 +300,10 @@ module Kimurai
           Thread.current.abort_on_exception = true
 
           spider = self.class.new(engine, config: @config.deep_merge_excl(config, DMERGE_EXCLUDE))
-          spider.with_info = true if self.with_info
+          spider.with_info = true if with_info
 
           part.each do |url_data|
-            if url_data.class == Hash
+            if url_data.instance_of?(Hash)
               if url_data[:url].present? && url_data[:data].present?
                 spider.request_to(handler, delay, url_data)
               else
@@ -314,7 +314,7 @@ module Kimurai
             end
           end
         ensure
-          spider.browser.destroy_driver! if spider.instance_variable_get("@browser")
+          spider.browser.destroy_driver! if spider.instance_variable_get('@browser')
         end
 
         sleep 0.5

data/lib/kimurai/base_helper.rb

@@ -4,13 +4,14 @@ module Kimurai
 
     def absolute_url(url, base:)
       return unless url
-      URI.join(base, URI.escape(url)).to_s
+
+      URI.join(base, URI::DEFAULT_PARSER.escape(url)).to_s
     end
 
     def escape_url(url)
-      uri = URI.parse(url)
-    rescue URI::InvalidURIError => e
-      URI.parse(URI.escape url).to_s rescue url
+      URI.parse(url)
+    rescue URI::InvalidURIError
+      URI.parse(URI::DEFAULT_PARSER.escape(url)).to_s rescue url
     else
       url
     end