kimurai 1.3.2 → 2.0.0

data/lib/kimurai/cli.rb

@@ -4,18 +4,22 @@ module Kimurai
   class CLI < Thor
     map %w[--version -v] => :__print_version
 
-    desc "generate", "Generator, available types: project, spider, schedule"
+    desc 'new PROJECT_NAME', 'Create a new Kimurai project'
+    def new(project_name)
+      raise 'Provide project name to generate a new project' unless project_name.present?
+
+      Generator.new.generate_project(project_name)
+    end
+
+    desc 'generate', 'Generator, available types: spider, schedule'
     def generate(generator_type, *args)
       case generator_type
-      when "project"
-        project_name = args.shift
-        raise "Provide project name to generate a new project" unless project_name.present?
-        Generator.new.generate_project(project_name)
-      when "spider"
+      when 'spider'
         spider_name = args.shift
-        raise "Provide spider name to generate a spider" unless spider_name.present?
+        raise 'Provide spider name to generate a spider' unless spider_name.present?
+
         Generator.new.generate_spider(spider_name, in_project: inside_project?)
-      when "schedule"
+      when 'schedule'
         Generator.new.generate_schedule
       else
         raise "Don't know this generator type: #{generator_type}"
@@ -24,82 +28,43 @@ module Kimurai
 
     ###
 
-    desc "setup", "Setup server"
-    option :port, aliases: :p, type: :string, banner: "Port for ssh connection"
-    option "ask-sudo", type: :boolean, banner: "Provide sudo password for a user to install system-wide packages"
-    option "ask-auth-pass", type: :boolean, banner: "Auth using password"
-    option "ssh-key-path", type: :string, banner: "Auth using ssh key"
-    option :local, type: :boolean, banner: "Run setup on a local machine (Ubuntu only)"
-    def setup(user_host)
-      command = AnsibleCommandBuilder.new(user_host, options, playbook: "setup").get
-
-      pid = spawn *command
-      Process.wait pid
-    end
-
-    desc "deploy", "Deploy project to the server and update cron schedule"
-    option :port, aliases: :p, type: :string, banner: "Port for ssh connection"
-    option "ask-auth-pass", type: :boolean, banner: "Auth using password"
-    option "ssh-key-path", type: :string, banner: "Auth using ssh key"
-    option "repo-url", type: :string, banner: "Repo url"
-    option "repo-key-path", type: :string, banner: "SSH key for a git repo"
-    def deploy(user_host)
-      if !`git status --short`.empty?
-        raise "Deploy: Please commit your changes first"
-      elsif `git remote`.empty?
-        raise "Deploy: Please add remote origin repository to your repo first"
-      elsif !`git rev-list master...origin/master`.empty?
-        raise "Deploy: Please push your commits to the remote origin repo first"
-      end
-
-      repo_url = options["repo-url"] ? options["repo-url"] : `git remote get-url origin`.strip
-      repo_name = repo_url[/\/([^\/]*)\.git/i, 1]
-
-      command = AnsibleCommandBuilder.new(user_host, options, playbook: "deploy",
-        vars: { repo_url: repo_url, repo_name: repo_name, repo_key_path: options["repo-key-path"] }
-      ).get
-
-      pid = spawn *command
-      Process.wait pid
-    end
-
-    ###
-
-    desc "crawl", "Run a particular spider by it's name"
+    desc 'crawl', "Run a particular spider by it's name"
     def crawl(spider_name)
       raise "Can't find Kimurai project" unless inside_project?
+
       require './config/boot'
 
-      unless klass = Kimurai.find_by_name(spider_name)
+      unless (klass = Kimurai.find_by_name(spider_name))
         raise "Can't find spider with name `#{spider_name}` in the project. " \
-          "To list all available spiders, run: `$ bundle exec kimurai list`"
+          'To list all available spiders, run: `$ bundle exec kimurai list`'
       end
 
       # Set time_zone if exists
-      if time_zone = Kimurai.configuration.time_zone
+      if (time_zone = Kimurai.configuration.time_zone)
         Kimurai.time_zone = time_zone
       end
 
       klass.crawl!
     end
 
-    desc "parse", "Parse url in the particular spider method"
-    option :url, type: :string, required: true, banner: "Url to pass to the method"
+    desc 'parse', 'Parse url in the particular spider method'
+    option :url, type: :string, required: true, banner: 'Url to pass to the method'
     def parse(spider_name, method_name)
       raise "Can't find Kimurai project" unless inside_project?
+
       require './config/boot'
 
-      unless klass = Kimurai.find_by_name(spider_name)
+      unless (klass = Kimurai.find_by_name(spider_name))
         raise "Can't find spider with name `#{spider_name}` in the project. " \
-          "To list all available spiders, run: `$ bundle exec kimurai list`"
+          'To list all available spiders, run: `$ bundle exec kimurai list`'
       end
 
-      klass.parse!(method_name, url: options["url"])
+      klass.parse!(method_name, url: options['url'])
     end
 
-    desc "console", "Start Kimurai console"
-    option :engine, type: :string, banner: "Engine to use"
-    option :url, type: :string, banner: "Url to process"
+    desc 'console', 'Start Kimurai console'
+    option :engine, type: :string, banner: 'Engine to use'
+    option :url, type: :string, banner: 'Url to process'
     def console(spider_name = nil)
       require 'pry'
       require './config/boot' if inside_project?
@@ -107,70 +72,72 @@ module Kimurai
       if spider_name
         raise "Can't find Kimurai project" unless inside_project?
 
-        unless klass = Kimurai.find_by_name(spider_name)
+        unless (klass = Kimurai.find_by_name(spider_name))
           raise "Can't find spider with name `#{spider_name}` in the project. " \
-            "To list all available spiders, run: `$ bundle exec kimurai list`"
+            'To list all available spiders, run: `$ bundle exec kimurai list`'
         end
       else
         klass = inside_project? ? ApplicationSpider : ::Kimurai::Base
       end
 
-      engine = options["engine"]&.delete(":")&.to_sym
-      klass.parse!(:console, engine, url: options["url"])
+      engine = options['engine']&.delete(':')&.to_sym
+      if options['url']
+        klass.new(engine).request_to(:console, url: options['url'])
+      else
+        klass.new(engine).public_send(:console)
+      end
     end
 
-    desc "list", "List all available spiders in the current project"
+    desc 'list', 'List all available spiders in the current project'
     def list
       raise "Can't find Kimurai project" unless inside_project?
+
       require './config/boot'
 
-      Kimurai.list.keys.each { |name| puts name }
+      Kimurai.list.keys.sort.each { |name| puts name }
     end
 
-    desc "runner", "Run all spiders in the project in queue"
-    option :include, type: :array, default: [], banner: "List of spiders to run"
-    option :exclude, type: :array, default: [], banner: "List of spiders to exclude from run"
-    option :jobs, aliases: :j, type: :numeric, default: 1, banner: "The number of concurrent jobs"
+    desc 'runner', 'Run all spiders in the project in queue'
+    option :include, type: :array, default: [], banner: 'List of spiders to run'
+    option :exclude, type: :array, default: [], banner: 'List of spiders to exclude from run'
+    option :jobs, aliases: :j, type: :numeric, default: 1, banner: 'The number of concurrent jobs'
     def runner
       raise "Can't find Kimurai project" unless inside_project?
 
-      jobs = options["jobs"]
-      raise "Jobs count can't be 0" if jobs == 0
+      jobs = options['jobs']
+      raise "Jobs count can't be 0" if jobs.zero?
 
       require './config/boot'
       require 'kimurai/runner'
 
-      spiders = options["include"].presence || Kimurai.list.keys
-      spiders -= options["exclude"]
+      spiders = options['include'].presence || Kimurai.list.keys
+      spiders -= options['exclude']
 
       Runner.new(spiders, jobs).run!
     end
 
-    desc "--version, -v", "Print the version"
+    desc '--version, -v', 'Print the version'
     def __print_version
       puts VERSION
     end
 
-    desc "dashboard", "Run dashboard"
+    desc 'dashboard', 'Run dashboard'
     def dashboard
       raise "Can't find Kimurai project" unless inside_project?
 
       require './config/boot'
-      if Object.const_defined?("Kimurai::Dashboard")
-        require 'kimurai/dashboard/app'
-        Kimurai::Dashboard::App.run!
-      else
-        raise "Kimurai::Dashboard is not defined"
-      end
+      raise 'Kimurai::Dashboard is not defined' unless Object.const_defined?('Kimurai::Dashboard')
+
+      require 'kimurai/dashboard/app'
+      Kimurai::Dashboard::App.run!
     end
 
     private
 
     def inside_project?
-      Dir.exists?("spiders") && File.exists?("./config/boot.rb")
+      Dir.exist?('spiders') && File.exist?('./config/boot.rb')
     end
   end
 end
 
 require_relative 'cli/generator'
-require_relative 'cli/ansible_command_builder'

data/lib/kimurai/core_ext/array.rb

@@ -1,8 +1,8 @@
 class Array
   def in_sorted_groups(number, fill_width = nil)
-    sorted_groups = Array.new(number) { |a| a = [] }
+    sorted_groups = Array.new(number) { |_a| [] }
 
-    self.in_groups_of(number, fill_width).each do |group|
+    in_groups_of(number, fill_width).each do |group|
       number.times do |i|
         group.fetch(i) rescue next
         sorted_groups[i] << group[i]

data/lib/kimurai/core_ext/hash.rb

@@ -1,5 +1,5 @@
 class Hash
   def deep_merge_excl(second, exclude)
-    self.merge(second.slice(*exclude)).deep_merge(second.except(*exclude))
+    merge(second.slice(*exclude)).deep_merge(second.except(*exclude))
   end
 end

data/lib/kimurai/core_ext/numeric.rb

@@ -1,16 +1,16 @@
 class Numeric
   # https://stackoverflow.com/a/1679963
   def duration
-    secs  = self.to_int
+    secs  = to_int
     mins  = secs / 60
     hours = mins / 60
     days  = hours / 24
 
-    if days > 0
+    if days.positive?
       "#{days}d, #{hours % 24}h"
-    elsif hours > 0
+    elsif hours.positive?
       "#{hours}h, #{mins % 60}m"
-    elsif mins > 0
+    elsif mins.positive?
       "#{mins}m, #{secs % 60}s"
     elsif secs >= 0
       "#{secs}s"

data/lib/kimurai/pipeline.rb

@@ -1,8 +1,9 @@
 module Kimurai
   class Pipeline
     class DropItemError < StandardError; end
+
     def self.name
-      self.to_s.sub(/.*?::/, "").underscore.to_sym
+      to_s.sub(/.*?::/, '').underscore.to_sym
     end
 
     include BaseHelper

data/lib/kimurai/runner.rb

@@ -19,17 +19,17 @@ module Kimurai
         spiders: @spiders
       }
 
-      if time_zone = Kimurai.configuration.time_zone
+      if (time_zone = Kimurai.configuration.time_zone)
         Kimurai.time_zone = time_zone
       end
 
-      ENV.store("SESSION_ID", @start_time.to_i.to_s)
-      ENV.store("RBCAT_COLORIZER", "false")
+      ENV.store('SESSION_ID', @start_time.to_i.to_s)
+      ENV.store('RBCAT_COLORIZER', 'false')
     end
 
     def run!(exception_on_fail: true)
       puts ">>> Runner: started: #{session_info}"
-      if at_start_callback = Kimurai.configuration.runner_at_start_callback
+      if (at_start_callback = Kimurai.configuration.runner_at_start_callback)
         at_start_callback.call(session_info)
       end
 
@@ -38,7 +38,7 @@ module Kimurai
         next unless running
 
         puts "> Runner: started spider: #{spider}, index: #{i}"
-        pid = spawn("bundle", "exec", "kimurai", "crawl", spider, [:out, :err] => "log/#{spider}.log")
+        pid = spawn('bundle', 'exec', 'kimurai', 'crawl', spider, %i[out err] => "log/#{spider}.log")
         Process.wait pid
 
         puts "< Runner: stopped spider: #{spider}, index: #{i}"
@@ -51,7 +51,7 @@ module Kimurai
     else
       session_info.merge!(status: :completed, stop_time: Time.now)
     ensure
-      if at_stop_callback = Kimurai.configuration.runner_at_stop_callback
+      if (at_stop_callback = Kimurai.configuration.runner_at_stop_callback)
         at_stop_callback.call(session_info)
       end
       puts "<<< Runner: stopped: #{session_info}"

data/lib/kimurai/template/Gemfile

@@ -1,10 +1,10 @@
 source 'https://rubygems.org'
 git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 
-ruby '>= 2.5'
+ruby '>= 3.1'
 
 # Framework
-gem 'kimurai', '~> 1.0'
+gem 'kimurai', '~> 2.0'
 
 # Require files in directory and child directories recursively
 gem 'require_all'

data/lib/kimurai/template/config/boot.rb

@@ -6,17 +6,17 @@ Bundler.require(:default, Kimurai.env)
 require 'dotenv/load'
 
 # require initializers
-Dir.glob(File.join("./config/initializers", "*.rb"), &method(:require))
+Dir.glob(File.join('./config/initializers', '*.rb'), &method(:require))
 
 # require helpers
-Dir.glob(File.join("./helpers", "*.rb"), &method(:require))
+Dir.glob(File.join('./helpers', '*.rb'), &method(:require))
 
 # require pipelines
-Dir.glob(File.join("./pipelines", "*.rb"), &method(:require))
+Dir.glob(File.join('./pipelines', '*.rb'), &method(:require))
 
 # require spiders recursively in the `spiders/` folder
 require_relative '../spiders/application_spider'
-require_all "spiders"
+require_all 'spiders'
 
 # require Kimurai configuration
 require_relative 'application'

data/lib/kimurai/template/config/schedule.rb

@@ -2,7 +2,7 @@
 require 'tzinfo'
 
 # Export current PATH to the cron
-env :PATH, ENV["PATH"]
+env :PATH, ENV['PATH']
 
 # Use 24 hour format when using `at:` option
 set :chronic_options, hours24: true
@@ -19,34 +19,34 @@ def local_to_utc(time_string, zone:)
   TZInfo::Timezone.get(zone).local_to_utc(Time.parse(time_string))
 end
 
-# Note: by default Whenever exports cron commands with :environment == "production".
+# NOTE: by default Whenever exports cron commands with :environment == "production".
 # Note: Whenever can only append log data to a log file (>>). If you want
 # to overwrite (>) log file before each run, pass lambda:
 # crawl "google_spider.com", output: -> { "> log/google_spider.com.log 2>&1" }
 
 # Project job types
-job_type :crawl,  "cd :path && KIMURAI_ENV=:environment bundle exec kimurai crawl :task :output"
-job_type :runner, "cd :path && KIMURAI_ENV=:environment bundle exec kimurai runner --jobs :task :output"
+job_type :crawl,  'cd :path && KIMURAI_ENV=:environment bundle exec kimurai crawl :task :output'
+job_type :runner, 'cd :path && KIMURAI_ENV=:environment bundle exec kimurai runner --jobs :task :output'
 
 # Single file job type
-job_type :single, "cd :path && KIMURAI_ENV=:environment ruby :task :output"
+job_type :single, 'cd :path && KIMURAI_ENV=:environment ruby :task :output'
 # Single with bundle exec
-job_type :single_bundle, "cd :path && KIMURAI_ENV=:environment bundle exec ruby :task :output"
+job_type :single_bundle, 'cd :path && KIMURAI_ENV=:environment bundle exec ruby :task :output'
 
 ### Schedule ###
 # Usage (check examples here https://github.com/javan/whenever#example-schedulerb-file):
 # every 1.day do
-  # Example to schedule a single spider in the project:
-  # crawl "google_spider.com", output: "log/google_spider.com.log"
+# Example to schedule a single spider in the project:
+# crawl "google_spider.com", output: "log/google_spider.com.log"
 
-  # Example to schedule all spiders in the project using runner. Each spider will write
-  # it's own output to the `log/spider_name.log` file (handled by a runner itself).
-  # Runner output will be written to log/runner.log file.
-  # Argument number it's a count of concurrent jobs:
-  # runner 3, output:"log/runner.log"
+# Example to schedule all spiders in the project using runner. Each spider will write
+# it's own output to the `log/spider_name.log` file (handled by a runner itself).
+# Runner output will be written to log/runner.log file.
+# Argument number it's a count of concurrent jobs:
+# runner 3, output:"log/runner.log"
 
-  # Example to schedule single spider (without project):
-  # single "single_spider.rb", output: "single_spider.log"
+# Example to schedule single spider (without project):
+# single "single_spider.rb", output: "single_spider.log"
 # end
 
 ### How to set a cron schedule ###

data/lib/kimurai/template/spiders/application_spider.rb

@@ -5,19 +5,18 @@
 class ApplicationSpider < Kimurai::Base
   include ApplicationHelper
 
-  # Default engine for spiders (available engines: :mechanize, :poltergeist_phantomjs,
-  # :selenium_firefox, :selenium_chrome)
-  @engine = :poltergeist_phantomjs
+  # Default engine for spiders (available engines: :mechanize, :selenium_firefox, :selenium_chrome)
+  @engine = :selenium_chrome
 
   # Pipelines list, by order.
   # To process item through pipelines pass item to the `send_item` method
-  @pipelines = [:validator, :saver]
+  @pipelines = %i[validator saver]
 
   # Default config. Set here options which are default for all spiders inherited
   # from ApplicationSpider. Child's class config will be deep merged with this one
   @config = {
     # Custom headers, format: hash. Example: { "some header" => "some value", "another header" => "another value" }
-    # Works only for :mechanize and :poltergeist_phantomjs engines (Selenium doesn't allow to set/get headers)
+    # Works for :mechanize engine. Selenium doesn't allow to set/get headers.
     # headers: {},
 
     # Custom User Agent, format: string or lambda.
@@ -49,7 +48,7 @@ class ApplicationSpider < Kimurai::Base
     # window_size: [1366, 768],
 
     # Skip images downloading if true, works for all engines
-    disable_images: true,
+    # disable_images: true,
 
     # Selenium engines only: headless mode, `:native` or `:virtual_display` (default is :native)
     # Although native mode has a better performance, virtual display mode
@@ -61,14 +60,9 @@ class ApplicationSpider < Kimurai::Base
     # Format: array of strings. Works only for :selenium_firefox and selenium_chrome
     # proxy_bypass_list: [],
 
-    # Option to provide custom SSL certificate. Works only for :poltergeist_phantomjs and :mechanize
+    # Option to provide custom SSL certificate. Works only for :mechanize
     # ssl_cert_path: "path/to/ssl_cert",
 
-    # Inject some JavaScript code to the browser.
-    # Format: array of strings, where each string is a path to JS file.
-    # Works only for poltergeist_phantomjs engine (Selenium doesn't support JS code injection)
-    # extensions: ["lib/code_to_inject.js"],
-
     # Automatically skip duplicated (already visited) urls when using `request_to` method.
     # Possible values: `true` or `hash` with options.
     # In case of `true`, all visited urls will be added to the storage's scope `:requests_urls`
@@ -100,6 +94,12 @@ class ApplicationSpider < Kimurai::Base
     # Format: same like for `skip_request_errors` option.
     # retry_request_errors: [Net::ReadTimeout],
 
+    # Handle page encoding while parsing html response using Nokogiri. There are two modes:
+    # Auto (`:auto`) (try to fetch correct encoding from <meta http-equiv="Content-Type"> or <meta charset> tags)
+    # Set required encoding manually, example: `encoding: "GB2312"` (Set required encoding manually)
+    # Default this option is unset.
+    # encoding: nil,
+
     # Restart browser if one of the options is true:
     restart_if: {
       # Restart browser if provided memory limit (in kilobytes) is exceeded (works for all engines)
@@ -112,12 +112,12 @@ class ApplicationSpider < Kimurai::Base
     # Perform several actions before each request:
     before_request: {
       # Change proxy before each request. The `proxy:` option above should be presented
-      # and has lambda format. Works only for poltergeist and mechanize engines
+      # and has lambda format. Works for :mechanize engine.
       # (Selenium doesn't support proxy rotation).
       # change_proxy: true,
 
       # Change user agent before each request. The `user_agent:` option above should be presented
-      # and has lambda format. Works only for poltergeist and mechanize engines
+      # and has lambda format. Works for :mechanize engine.
       # (selenium doesn't support to get/set headers).
       # change_user_agent: true,
 

data/lib/kimurai/version.rb

@@ -1,3 +1,3 @@
 module Kimurai
-  VERSION = "1.3.2"
+  VERSION = '2.0.0'.freeze
 end

data/lib/kimurai.rb

@@ -1,6 +1,8 @@
 require 'ostruct'
 require 'logger'
 require 'json'
+require 'uri'
+
 require 'active_support'
 require 'active_support/core_ext'
 require 'rbcat'
@@ -28,26 +30,28 @@ module Kimurai
     end
 
     def env
-      ENV.fetch("KIMURAI_ENV") { "development" }
+      ENV.fetch('KIMURAI_ENV', 'development')
     end
 
     def time_zone
-      ENV["TZ"]
+      ENV['TZ']
     end
 
     def time_zone=(value)
-      ENV.store("TZ", value)
+      ENV.store('TZ', value)
     end
 
     def list
       Base.descendants.map do |klass|
         next unless klass.name
+
         [klass.name, klass]
       end.compact.to_h
     end
 
     def find_by_name(name)
       return unless name
+
       Base.descendants.find { |klass| klass.name == name }
     end
   end