kimurai_dynamic 1.4.1

data/lib/kimurai/capybara_ext/session.rb

@@ -0,0 +1,249 @@
+require 'capybara'
+require 'nokogiri'
+require 'json'
+require_relative 'session/config'
+
+module Capybara
+  class Session
+    attr_accessor :spider
+
+    alias_method :original_visit, :visit
+    def visit(visit_uri, delay: config.before_request[:delay], skip_request_options: false, max_retries: 3)
+      if spider
+        process_delay(delay) if delay
+        retries, sleep_interval = 0, 0
+
+        begin
+          check_request_options(visit_uri) unless skip_request_options
+          driver.requests += 1 and logger.info "Browser: started get request to: #{visit_uri}"
+          spider.class.update(:visits, :requests) if spider.with_info
+
+          original_visit(visit_uri)
+        rescue => e
+          if match_error?(e, type: :to_skip)
+            logger.error "Browser: skip request error: #{e.inspect}, url: #{visit_uri}"
+            spider.add_event(:requests_errors, e.inspect) if spider.with_info
+            false
+          elsif match_error?(e, type: :to_retry)
+            logger.error "Browser: retry request error: #{e.inspect}, url: #{visit_uri}"
+            spider.add_event(:requests_errors, e.inspect) if spider.with_info
+
+            if (retries += 1) <= max_retries
+              logger.info "Browser: sleep #{(sleep_interval += 15)} seconds and process retry № #{retries} to the url: #{visit_uri}"
+              sleep sleep_interval and retry
+            else
+              logger.error "Browser: all retries (#{retries - 1}) to the url #{visit_uri} are gone"
+              raise e unless skip_error_on_failure?(e)
+            end
+          else
+            raise e
+          end
+        else
+          driver.responses += 1 and logger.info "Browser: finished get request to: #{visit_uri}"
+          spider.class.update(:visits, :responses) if spider.with_info
+          driver.visited = true unless driver.visited
+          true
+        ensure
+          if spider.with_info
+            logger.info "Info: visits: requests: #{spider.class.visits[:requests]}, responses: #{spider.class.visits[:responses]}"
+          end
+
+          if memory = driver.current_memory
+            logger.debug "Browser: driver.current_memory: #{memory}"
+          end
+        end
+      else
+        original_visit(visit_uri)
+      end
+    end
+
+    def destroy_driver!
+      if @driver
+        begin
+          @driver.quit
+        # handle Net::ReadTimeout error for Selenium like drivers
+        rescue Net::ReadTimeout => e
+          @driver.quit
+        end
+
+        @driver = nil
+        logger.info "Browser: driver #{mode} has been destroyed"
+      else
+        logger.warn "Browser: driver #{mode} is not present"
+      end
+    end
+
+    def restart!
+      if mode.match?(/poltergeist/)
+        @driver.browser.restart
+        @driver.requests, @driver.responses = 0, 0
+      else
+        destroy_driver!
+        driver
+      end
+
+      logger.info "Browser: driver has been restarted: name: #{mode}, pid: #{driver.pid}, port: #{driver.port}"
+    end
+
+    def current_response(response_type = :html)
+      case response_type
+      when :html
+        if config.encoding
+          if config.encoding == :auto
+            charset = body.force_encoding("ISO-8859-1").encode("UTF-8")[/<meta.*?charset=["]?([\w+\d+\-]*)/i, 1]
+            Nokogiri::HTML(body, nil, charset)
+          else
+            Nokogiri::HTML(body, nil, config.encoding)
+          end
+        else
+          Nokogiri::HTML(body)
+        end
+      when :json
+        JSON.parse(body)
+      end
+    end
+
+    ###
+
+    # Handy method to perform some processing in the new tab within block and then automatically close this tab:
+    # Usage (url):
+    # browser.within_new_window_by(url: "https://google.com") do
+      # do some stuff and then automatically close this tab and return back to the first tab
+    # end
+    # Usage (action) (when new tab opening by some action, for example by clicking
+    # on a particular element):
+    # action = -> { browser.find("//some/element/path").click }
+    # browser.within_new_window_by(action: action) do
+      # do some stuff and then automatically close this tab and return back to the first tab
+    # end
+    def within_new_window_by(action: nil, url: nil)
+      case
+      when action
+        opened_window = window_opened_by { action.call }
+        within_window(opened_window) do
+          yield
+          current_window.close
+        end
+      when url
+        within_window(open_new_window) do
+          visit(url)
+
+          yield
+          current_window.close
+        end
+      end
+    end
+
+    ###
+
+    def scroll_to_bottom
+      execute_script("window.scrollBy(0,10000)")
+    end
+
+    private
+
+    def skip_error_on_failure?(e)
+      config.retry_request_errors.any? do |error|
+        error[:skip_on_failure] && e.class.ancestors.include?(error[:error]) if error.kind_of?(Hash)
+      end
+    end
+
+    def match_error?(e, type:)
+      errors =
+        case type
+        when :to_retry then config.retry_request_errors
+        when :to_skip then config.skip_request_errors
+        end
+
+      errors.any? do |error|
+        if error.kind_of?(Hash)
+          match_class = e.class.ancestors.include?(error[:error])
+          if error[:message].present?
+            if error[:message].kind_of?(Regexp)
+              e.message&.match?(error[:message])
+            else
+              e.message&.include?(error[:message])
+            end && match_class
+          else
+            match_class
+          end
+        else
+          e.class.ancestors.include?(error)
+        end
+      end
+    end
+
+    def process_delay(delay)
+      interval = (delay.class == Range ? rand(delay) : delay)
+      logger.debug "Browser: sleep #{interval.round(2)} #{'second'.pluralize(interval)} before request..."
+      sleep interval
+    end
+
+    def check_request_options(url_to_visit)
+      # restart_if
+      if memory_limit = config.restart_if[:memory_limit]
+        memory = driver.current_memory
+        if memory && memory >= memory_limit
+          logger.warn "Browser: memory_limit #{memory_limit} of driver.current_memory (#{memory}) is exceeded (engine: #{mode})"
+          restart!
+        end
+      end
+
+      if requests_limit = config.restart_if[:requests_limit]
+        requests = driver.requests
+        if requests >= requests_limit
+          logger.warn "Browser: requests_limit #{requests_limit} of driver.requests (#{requests}) is exceeded (engine: #{mode})"
+          restart!
+        end
+      end
+
+      # cookies
+      # (Selenium only) if config.cookies present and browser was just created,
+      # visit url_to_visit first and only then set cookies:
+      if driver.visited.nil? && config.cookies && mode.match?(/selenium/)
+        visit(url_to_visit, skip_request_options: true)
+        config.cookies.each do |cookie|
+          driver.set_cookie(cookie[:name], cookie[:value], cookie)
+        end
+      end
+
+      if config.before_request[:clear_cookies]
+        driver.clear_cookies
+        logger.debug "Browser: cleared cookies before request"
+      end
+
+      if config.before_request[:clear_and_set_cookies]
+        driver.clear_cookies
+
+        # (Selenium only) if browser is not visited yet any page, visit url_to_visit
+        # first and then set cookies (needs after browser restart):
+        if driver.visited.nil? && mode.match?(/selenium/)
+          visit(url_to_visit, skip_request_options: true)
+        end
+
+        config.cookies.each do |cookie|
+          driver.set_cookie(cookie[:name], cookie[:value], cookie)
+        end
+
+        logger.debug "Browser: cleared and set cookies before request"
+      end
+
+      # user_agent
+      if config.before_request[:change_user_agent]
+        driver.add_header("User-Agent", config.user_agent.call)
+        logger.debug "Browser: changed user_agent before request"
+      end
+
+      # proxy
+      if config.before_request[:change_proxy]
+        proxy_string = config.proxy.call
+        driver.set_proxy(*proxy_string.split(":"))
+        logger.debug "Browser: changed proxy before request"
+      end
+    end
+
+    def logger
+      spider.logger
+    end
+  end
+end

data/lib/kimurai/cli/ansible_command_builder.rb

@@ -0,0 +1,71 @@
+require 'cliver'
+
+module Kimurai
+  class CLI
+    class AnsibleCommandBuilder
+      def initialize(user_host, options, playbook:, vars: {})
+        @user_host = user_host
+        @options = options
+        @playbook = playbook
+        @vars = vars
+      end
+
+      def get
+        unless Cliver.detect("ansible-playbook")
+          raise "Can't find `ansible-playbook` executable, to install: " \
+            "Mac OS X: `$ brew install ansible`, Ubuntu: `$ sudo apt install ansible`"
+        end
+
+        user = @user_host[/(.*?)\@/, 1]
+        host = @user_host[/\@(.+)/, 1] || @user_host
+        inventory = @options["port"] ? "#{host}:#{@options['port']}," : "#{host},"
+
+        gem_dir = Gem::Specification.find_by_name("kimurai").gem_dir
+        playbook_path = gem_dir + "/lib/kimurai/automation/" + "#{@playbook}.yml"
+
+        command = [
+          "ansible-playbook", playbook_path,
+          "--inventory", inventory,
+          "--ssh-extra-args", "-oForwardAgent=yes",
+          "--connection", @options["local"] ? "local" : "smart",
+          "--extra-vars", "ansible_python_interpreter=/usr/bin/python3"
+        ]
+
+        if File.exists? "config/automation.yml"
+          require 'yaml'
+          if config = YAML.load_file("config/automation.yml").dig(@playbook)
+            config.each { |key, value| @vars[key] = value unless @vars[key] }
+          end
+        end
+
+        @vars.each do |key, value|
+          next unless value.present?
+          command.push "--extra-vars", "#{key}=#{value}"
+        end
+
+        if user
+          command.push "--user", user
+        end
+
+        if @options["ask-sudo"]
+          command.push "--ask-become-pass"
+        end
+
+        if @options["ask-auth-pass"]
+          unless Cliver.detect("sshpass")
+            raise "Can't find `sshpass` executable for password authentication, to install: " \
+              "Mac OS X: `$ brew install http://git.io/sshpass.rb`, Ubuntu: `$ sudo apt install sshpass`"
+          end
+
+          command.push "--ask-pass"
+        end
+
+        if ssh_key_path = @options["ssh-key-path"]
+          command.push "--private-key", ssh_key_path
+        end
+
+        command
+      end
+    end
+  end
+end

data/lib/kimurai/cli/generator.rb

@@ -0,0 +1,57 @@
+module Kimurai
+  class CLI
+    class Generator < Thor::Group
+      include Thor::Actions
+
+      def self.source_root
+        File.dirname(File.expand_path('..', __FILE__))
+      end
+
+      def generate_project(project_name)
+        directory "template", project_name
+        inside(project_name) do
+          run "bundle install"
+          run "git init"
+        end
+      end
+
+      def generate_spider(spider_name, in_project:)
+        spider_path = in_project ? "spiders/#{spider_name}.rb" : "./#{spider_name}.rb"
+        raise "Spider #{spider_path} already exists" if File.exists? spider_path
+
+        spider_class = to_spider_class(spider_name)
+        create_file spider_path do
+          <<~RUBY
+            class #{spider_class} < #{in_project ? 'ApplicationSpider' : 'Kimurai::Base'}
+              @name = "#{spider_name}"
+              @start_urls = []
+              @config = {}
+
+              def parse(response, url:, data: {})
+              end
+            end
+          RUBY
+        end
+
+        unless in_project
+          insert_into_file spider_path, "  @engine = :mechanize\n", after: "@name = \"#{spider_name}\"\n"
+          prepend_to_file spider_path, "require 'kimurai'\n\n"
+          append_to_file spider_path, "\n#{spider_class}.crawl!"
+        end
+      end
+
+      def generate_schedule
+        copy_file "template/config/schedule.rb", "./schedule.rb"
+      end
+
+      private
+
+      def to_spider_class(string)
+        string.sub(/^./) { $&.capitalize }
+          .gsub(/(?:_|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }
+          .gsub(/(?:-|(\/))([a-z\d]*)/) { "Dash#{$2.capitalize}" }
+          .gsub(/(?:\.|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }
+      end
+    end
+  end
+end

data/lib/kimurai/cli.rb

@@ -0,0 +1,183 @@
+require 'thor'
+
+module Kimurai
+  class CLI < Thor
+    map %w[--version -v] => :__print_version
+
+    desc "generate", "Generator, available types: project, spider, schedule"
+    def generate(generator_type, *args)
+      case generator_type
+      when "project"
+        project_name = args.shift
+        raise "Provide project name to generate a new project" unless project_name.present?
+        Generator.new.generate_project(project_name)
+      when "spider"
+        spider_name = args.shift
+        raise "Provide spider name to generate a spider" unless spider_name.present?
+        Generator.new.generate_spider(spider_name, in_project: inside_project?)
+      when "schedule"
+        Generator.new.generate_schedule
+      else
+        raise "Don't know this generator type: #{generator_type}"
+      end
+    end
+
+    ###
+
+    desc "setup", "Setup server"
+    option :port, aliases: :p, type: :string, banner: "Port for ssh connection"
+    option "ask-sudo", type: :boolean, banner: "Provide sudo password for a user to install system-wide packages"
+    option "ask-auth-pass", type: :boolean, banner: "Auth using password"
+    option "ssh-key-path", type: :string, banner: "Auth using ssh key"
+    option :local, type: :boolean, banner: "Run setup on a local machine (Ubuntu only)"
+    def setup(user_host)
+      command = AnsibleCommandBuilder.new(user_host, options, playbook: "setup").get
+
+      pid = spawn *command
+      Process.wait pid
+    end
+
+    desc "deploy", "Deploy project to the server and update cron schedule"
+    option :port, aliases: :p, type: :string, banner: "Port for ssh connection"
+    option "ask-auth-pass", type: :boolean, banner: "Auth using password"
+    option "ssh-key-path", type: :string, banner: "Auth using ssh key"
+    option "repo-url", type: :string, banner: "Repo url"
+    option "repo-key-path", type: :string, banner: "SSH key for a git repo"
+    option "skip-check", type: :boolean, default: false, banner: "Skip git repository checks"
+    def deploy(user_host)
+      unless options["skip-check"]
+        if !`git status --short`.empty?
+          raise "Deploy: Please commit your changes first"
+        elsif `git remote`.empty?
+          raise "Deploy: Please add remote origin repository to your repo first"
+        elsif !`git rev-list master...origin/master`.empty?
+          raise "Deploy: Please push your commits to the remote origin repo first"
+        end
+      end
+
+      repo_url = options["repo-url"] ? options["repo-url"] : `git remote get-url origin`.strip
+      repo_name = repo_url[/\/([^\/]*)\.git/i, 1]
+
+      command = AnsibleCommandBuilder.new(user_host, options, playbook: "deploy",
+        vars: { repo_url: repo_url, repo_name: repo_name, repo_key_path: options["repo-key-path"] }
+      ).get
+
+      pid = spawn *command
+      Process.wait pid
+    end
+
+    ###
+
+    desc "crawl", "Run a particular spider by it's name"
+    def crawl(spider_name)
+      raise "Can't find Kimurai project" unless inside_project?
+      require './config/boot'
+
+      unless klass = Kimurai.find_by_name(spider_name)
+        raise "Can't find spider with name `#{spider_name}` in the project. " \
+          "To list all available spiders, run: `$ bundle exec kimurai list`"
+      end
+
+      # Set time_zone if exists
+      if time_zone = Kimurai.configuration.time_zone
+        Kimurai.time_zone = time_zone
+      end
+
+      klass.crawl!
+    end
+
+    desc "parse", "Parse url in the particular spider method"
+    option :url, type: :string, required: true, banner: "Url to pass to the method"
+    def parse(spider_name, method_name)
+      raise "Can't find Kimurai project" unless inside_project?
+      require './config/boot'
+
+      unless klass = Kimurai.find_by_name(spider_name)
+        raise "Can't find spider with name `#{spider_name}` in the project. " \
+          "To list all available spiders, run: `$ bundle exec kimurai list`"
+      end
+
+      klass.parse!(method_name, url: options["url"])
+    end
+
+    desc "console", "Start Kimurai console"
+    option :engine, type: :string, banner: "Engine to use"
+    option :url, type: :string, banner: "Url to process"
+    def console(spider_name = nil)
+      require 'pry'
+      require './config/boot' if inside_project?
+
+      if spider_name
+        raise "Can't find Kimurai project" unless inside_project?
+
+        unless klass = Kimurai.find_by_name(spider_name)
+          raise "Can't find spider with name `#{spider_name}` in the project. " \
+            "To list all available spiders, run: `$ bundle exec kimurai list`"
+        end
+      else
+        klass = inside_project? ? ApplicationSpider : ::Kimurai::Base
+      end
+
+      engine = options["engine"]&.delete(":")&.to_sym
+      if url = options["url"]
+        klass.new(engine).request_to(:console, url: options["url"])
+      else
+        klass.new(engine).public_send(:console)
+      end
+    end
+
+    desc "list", "List all available spiders in the current project"
+    def list
+      raise "Can't find Kimurai project" unless inside_project?
+      require './config/boot'
+
+      Kimurai.list.keys.sort.each { |name| puts name }
+    end
+
+    desc "runner", "Run all spiders in the project in queue"
+    option :include, type: :array, default: [], banner: "List of spiders to run"
+    option :exclude, type: :array, default: [], banner: "List of spiders to exclude from run"
+    option :jobs, aliases: :j, type: :numeric, default: 1, banner: "The number of concurrent jobs"
+    def runner
+      raise "Can't find Kimurai project" unless inside_project?
+
+      jobs = options["jobs"]
+      raise "Jobs count can't be 0" if jobs == 0
+
+      require './config/boot'
+      require 'kimurai/runner'
+
+      spiders = options["include"].presence || Kimurai.list.keys
+      spiders -= options["exclude"]
+
+      Runner.new(spiders, jobs).run!
+    end
+
+    desc "--version, -v", "Print the version"
+    def __print_version
+      puts VERSION
+    end
+
+    desc "dashboard", "Run dashboard"
+    def dashboard
+      raise "Can't find Kimurai project" unless inside_project?
+
+      require './config/boot'
+      if Object.const_defined?("Kimurai::Dashboard")
+        require 'kimurai/dashboard/app'
+        Kimurai::Dashboard::App.run!
+      else
+        raise "Kimurai::Dashboard is not defined"
+      end
+    end
+
+    private
+
+    def inside_project?
+      Dir.exists?("spiders") && File.exists?("./config/boot.rb")
+    end
+  end
+end
+
+require_relative 'cli/generator'
+require_relative 'cli/ansible_command_builder'

data/lib/kimurai/core_ext/array.rb

@@ -0,0 +1,14 @@
+class Array
+  def in_sorted_groups(number, fill_width = nil)
+    sorted_groups = Array.new(number) { |a| a = [] }
+
+    self.in_groups_of(number, fill_width).each do |group|
+      number.times do |i|
+        group.fetch(i) rescue next
+        sorted_groups[i] << group[i]
+      end
+    end
+
+    sorted_groups
+  end
+end

data/lib/kimurai/core_ext/hash.rb

@@ -0,0 +1,5 @@
+class Hash
+  def deep_merge_excl(second, exclude)
+    self.merge(second.slice(*exclude)).deep_merge(second.except(*exclude))
+  end
+end

data/lib/kimurai/core_ext/numeric.rb

@@ -0,0 +1,19 @@
+class Numeric
+  # https://stackoverflow.com/a/1679963
+  def duration
+    secs  = self.to_int
+    mins  = secs / 60
+    hours = mins / 60
+    days  = hours / 24
+
+    if days > 0
+      "#{days}d, #{hours % 24}h"
+    elsif hours > 0
+      "#{hours}h, #{mins % 60}m"
+    elsif mins > 0
+      "#{mins}m, #{secs % 60}s"
+    elsif secs >= 0
+      "#{secs}s"
+    end
+  end
+end

data/lib/kimurai/core_ext/string.rb

@@ -0,0 +1,7 @@
+require 'murmurhash3'
+
+class String
+  def to_id
+    MurmurHash3::V32.str_hash(self)
+  end
+end

data/lib/kimurai/pipeline.rb

@@ -0,0 +1,33 @@
+module Kimurai
+  class Pipeline
+    class DropItemError < StandardError; end
+    def self.name
+      self.to_s.sub(/.*?::/, "").underscore.to_sym
+    end
+
+    include BaseHelper
+    attr_accessor :spider
+
+    def name
+      self.class.name
+    end
+
+    ###
+
+    def storage
+      spider.storage
+    end
+
+    def unique?(scope, value)
+      spider.unique?(scope, value)
+    end
+
+    def save_to(path, item, format:, position: true, append: false)
+      spider.save_to(path, item, format: format, position: position, append: append)
+    end
+
+    def logger
+      spider.logger
+    end
+  end
+end