kimurai 1.0.0

data/lib/kimurai/capybara_configuration.rb

@@ -0,0 +1,10 @@
+require 'capybara'
+
+Capybara.configure do |config|
+  config.run_server = false
+  config.default_selector = :xpath
+  config.save_path = "tmp"
+  config.default_max_wait_time = 10
+  config.ignore_hidden_elements = false
+  config.threadsafe = true
+end

data/lib/kimurai/capybara_ext/driver/base.rb

@@ -0,0 +1,62 @@
+require 'pathname'
+
+class Capybara::Driver::Base
+  attr_accessor :visited
+  attr_writer :requests, :responses
+
+  def requests
+    @requests ||= 0
+  end
+
+  def responses
+    @responses ||= 0
+  end
+
+  def current_memory
+    driver_pid = pid
+
+    all = (get_descendant_processes(driver_pid) << driver_pid).uniq
+    all.map { |pid| get_process_memory(pid) }.sum
+  end
+
+  private
+
+  def get_descendant_processes(base)
+    descendants = Hash.new { |ht, k| ht[k] = [k] }
+    Hash[*`ps -eo pid,ppid`.scan(/\d+/).map(&:to_i)].each do |pid, ppid|
+      descendants[ppid] << descendants[pid]
+    end
+
+    descendants[base].flatten - [base]
+  end
+
+  # https://github.com/schneems/get_process_mem
+  # Note: for Linux takes PSS (not RSS) memory (I think PSS better fits in this case)
+  def get_process_memory(pid)
+    case @platform ||= Gem::Platform.local.os
+    when "linux"
+      begin
+        file = Pathname.new "/proc/#{pid}/smaps"
+        return 0 unless file.exist?
+
+        lines = file.each_line.select { |line| line.match(/^Pss/) }
+        return 0 if lines.empty?
+
+        lines.reduce(0) do |sum, line|
+          line.match(/(?<value>(\d*\.{0,1}\d+))\s+(?<unit>\w\w)/) do |m|
+            sum += m[:value].to_i
+          end
+
+          sum
+        end
+      rescue Errno::EACCES
+        0
+      end
+    when "darwin"
+      mem = `ps -o rss= -p #{pid}`.strip
+      mem.empty? ? 0 : mem.to_i
+    else
+      raise "Can't check process memory, wrong type of platform: #{@platform}"
+    end
+  end
+end

data/lib/kimurai/capybara_ext/mechanize/driver.rb

@@ -0,0 +1,55 @@
+require 'mechanize'
+require_relative '../driver/base'
+
+class Capybara::Mechanize::Driver
+  # Extend capybara-mechnize to support Poltergeist-like methods
+  # https://www.rubydoc.info/gems/poltergeist/Capybara/Poltergeist/Driver
+
+  def set_proxy(ip, port, type, user, password)
+    # type is always "http", "socks" is not supported (yet)
+    browser.agent.set_proxy(ip, port, user, password)
+  end
+
+  def headers
+    browser.agent.request_headers
+  end
+
+  def headers=(headers)
+    browser.agent.request_headers = headers
+  end
+
+  def add_header(name, value)
+    browser.agent.request_headers[name] = value
+  end
+
+  def set_cookie(name, value, options = {})
+    options[:name]  ||= name
+    options[:value] ||= value
+
+    cookie = Mechanize::Cookie.new(options.merge path: "/")
+    browser.agent.cookie_jar << cookie
+  end
+
+  def clear_cookies
+    browser.agent.cookie_jar.clear!
+  end
+
+  def quit
+    browser.agent.shutdown
+  end
+
+  ###
+
+  # Reset parent method `current_memory` for mechanize (we can't measure memory of Mechanize driver)
+  def current_memory
+    nil
+  end
+
+  def pid
+    nil
+  end
+
+  def port
+    nil
+  end
+end

data/lib/kimurai/capybara_ext/poltergeist/driver.rb

@@ -0,0 +1,13 @@
+require_relative '../driver/base'
+
+module Capybara::Poltergeist
+  class Driver
+    def pid
+      client_pid
+    end
+
+    def port
+      server.port
+    end
+  end
+end

data/lib/kimurai/capybara_ext/selenium/driver.rb

@@ -0,0 +1,24 @@
+require_relative '../driver/base'
+
+class Capybara::Selenium::Driver
+  def set_cookie(name, value, options = {})
+    options[:name]  ||= name
+    options[:value] ||= value
+
+    browser.manage.add_cookie(options)
+  end
+
+  def clear_cookies
+    browser.manage.delete_all_cookies
+  end
+
+  ###
+
+  def pid
+    @pid ||= `lsof -i tcp:#{port} -t`.strip.to_i
+  end
+
+  def port
+    @port ||= browser.send(:bridge).instance_variable_get("@http").instance_variable_get("@server_url").port
+  end
+end

data/lib/kimurai/capybara_ext/session.rb

@@ -0,0 +1,150 @@
+require 'capybara'
+require 'nokogiri'
+require_relative 'session/config'
+
+module Capybara
+  class Session
+    attr_accessor :spider
+
+    def current_response
+      Nokogiri::HTML(body)
+    end
+
+    alias_method :original_visit, :visit
+    def visit(visit_uri, delay: config.before_request[:delay], skip_request_options: false, max_retries: 3)
+      if spider
+        process_delay(delay) if delay
+        retries, sleep_interval = 0, 0
+
+        begin
+          check_request_options(visit_uri) unless skip_request_options
+          driver.requests += 1 and logger.info "Browser: started get request to: #{visit_uri}"
+          spider.class.update(:visits, :requests) if spider.with_info
+
+          original_visit(visit_uri)
+        rescue *config.retry_request_errors => e
+          logger.error "Browser: request visit error: #{e.inspect}, url: #{visit_uri}"
+
+          if (retries += 1) < max_retries
+            logger.info "Browser: sleep #{(sleep_interval += 15)} seconds and process retry № #{retries} to the url: #{visit_uri}"
+            sleep sleep_interval and retry
+          else
+            logger.error "Browser: all retries (#{retries}) to the url `#{visit_uri}` are gone"
+            raise e
+          end
+        else
+          driver.responses += 1 and logger.info "Browser: finished get request to: #{visit_uri}"
+          spider.class.update(:visits, :responses) if spider.with_info
+          driver.visited = true unless driver.visited
+        ensure
+          if spider.with_info
+            logger.info "Info: visits: requests: #{spider.class.visits[:requests]}, responses: #{spider.class.visits[:responses]}"
+          end
+
+          if memory = driver.current_memory
+            logger.debug "Browser: driver.current_memory: #{memory}"
+          end
+        end
+      else
+        original_visit(visit_uri)
+      end
+    end
+
+    def destroy_driver!
+      if @driver
+        @driver.quit
+        @driver = nil
+        logger.info "Browser: driver #{mode} has been destroyed"
+      else
+        logger.warn "Browser: driver #{mode} is not present"
+      end
+    end
+
+    def restart!
+      if mode.match?(/poltergeist/)
+        @driver.browser.restart
+        @driver.requests, @driver.responses = 0, 0
+      else
+        destroy_driver!
+        driver
+      end
+
+      logger.info "Browser: driver has been restarted: name: #{mode}, pid: #{driver.pid}, port: #{driver.port}"
+    end
+
+    private
+
+    def process_delay(delay)
+      interval = (delay.class == Range ? rand(delay) : delay)
+      logger.debug "Browser: sleep #{interval.round(2)} #{'second'.pluralize(interval)} before request..."
+      sleep interval
+    end
+
+    def check_request_options(url_to_visit)
+      # restart_if
+      if memory_limit = config.restart_if[:memory_limit]
+        memory = driver.current_memory
+        if memory && memory >= memory_limit
+          logger.warn "Browser: memory_limit #{memory_limit} of driver.current_memory (#{memory}) is exceeded (engine: #{mode})"
+          restart!
+        end
+      end
+
+      if requests_limit = config.restart_if[:requests_limit]
+        requests = driver.requests
+        if requests >= requests_limit
+          logger.warn "Browser: requests_limit #{requests_limit} of driver.requests (#{requests}) is exceeded (engine: #{mode})"
+          restart!
+        end
+      end
+
+      # cookies
+      # (Selenium only) if config.cookies present and browser was just created,
+      # visit url_to_visit first and only then set cookies:
+      if driver.visited.nil? && config.cookies && mode.match?(/selenium/)
+        visit(url_to_visit, skip_request_options: true)
+        config.cookies.each do |cookie|
+          driver.set_cookie(cookie[:name], cookie[:value], cookie)
+        end
+      end
+
+      if config.before_request[:clear_cookies]
+        driver.clear_cookies
+        logger.debug "Browser: cleared cookies before request"
+      end
+
+      if config.before_request[:clear_and_set_cookies]
+        driver.clear_cookies
+
+        # (Selenium only) if browser is not visited yet any page, visit url_to_visit
+        # first and then set cookies (needs after browser restart):
+        if driver.visited.nil? && mode.match?(/selenium/)
+          visit(url_to_visit, skip_request_options: true)
+        end
+
+        config.cookies.each do |cookie|
+          driver.set_cookie(cookie[:name], cookie[:value], cookie)
+        end
+
+        logger.debug "Browser: cleared and set cookies before request"
+      end
+
+      # user_agent
+      if config.before_request[:change_user_agent]
+        driver.add_header("User-Agent", config.user_agent.call)
+        logger.debug "Browser: changed user_agent before request"
+      end
+
+      # proxy
+      if config.before_request[:change_proxy]
+        proxy_string = config.proxy.call
+        driver.set_proxy(*proxy_string.split(":"))
+        logger.debug "Browser: changed proxy before request"
+      end
+    end
+
+    def logger
+      spider.logger
+    end
+  end
+end

data/lib/kimurai/capybara_ext/session/config.rb

@@ -0,0 +1,18 @@
+module Capybara
+  class SessionConfig
+    attr_accessor :cookies, :proxy, :user_agent
+    attr_writer :retry_request_errors
+
+    def retry_request_errors
+      @retry_request_errors ||= []
+    end
+
+    def restart_if
+      @restart_if ||= {}
+    end
+
+    def before_request
+      @before_request ||= {}
+    end
+  end
+end

data/lib/kimurai/cli.rb

@@ -0,0 +1,157 @@
+require 'thor'
+
+module Kimurai
+  class CLI < Thor
+    map %w[--version -v] => :__print_version
+
+    desc "generate", "Generator, available types: project, spider, schedule"
+    def generate(generator_type, *args)
+      case generator_type
+      when "project"
+        project_name = args.shift
+        raise "Provide project name to generate a new project" unless project_name.present?
+        Generator.new.generate_project(project_name)
+      when "spider"
+        spider_name = args.shift
+        raise "Provide spider name to generate a spider" unless spider_name.present?
+        Generator.new.generate_spider(spider_name, in_project: inside_project?)
+      when "schedule"
+        Generator.new.generate_schedule
+      else
+        raise "Don't know this generator type: #{generator_type}"
+      end
+    end
+
+    ###
+
+    desc "setup", "Setup server"
+    option :port, aliases: :p, type: :string, banner: "Port for ssh connection"
+    option "ask-sudo", type: :boolean, banner: "Provide sudo password for a user to install system-wide packages"
+    option "ask-auth-pass", type: :boolean, banner: "Auth using password"
+    option "ssh-key-path", type: :string, banner: "Auth using ssh key"
+    option :local, type: :boolean, banner: "Run setup on a local machine (Ubuntu only)"
+    def setup(user_host)
+      command = AnsibleCommandBuilder.new(user_host, options, playbook: "setup").get
+
+      pid = spawn *command
+      Process.wait pid
+    end
+
+    desc "deploy", "Deploy project to the server and update cron schedule"
+    option :port, aliases: :p, type: :string, banner: "Port for ssh connection"
+    option "ask-auth-pass", type: :boolean, banner: "Auth using password"
+    option "ssh-key-path", type: :string, banner: "Auth using ssh key"
+    option "repo-url", type: :string, banner: "Repo url"
+    option "repo-key-path", type: :string, banner: "SSH key for a git repo"
+    def deploy(user_host)
+      if !`git status --short`.empty?
+        raise "Deploy: Please commit your changes first"
+      elsif `git remote`.empty?
+        raise "Deploy: Please add remote origin repository to your repo first"
+      elsif !`git rev-list master...origin/master`.empty?
+        raise "Deploy: Please push your commits to the remote origin repo first"
+      end
+
+      repo_url = options["repo-url"] ? options["repo-url"] : `git remote get-url origin`.strip
+      repo_name = repo_url[/\/([^\/]*)\.git/i, 1]
+
+      command = AnsibleCommandBuilder.new(user_host, options, playbook: "deploy",
+        vars: { repo_url: repo_url, repo_name: repo_name, repo_key_path: options["repo-key-path"] }
+      ).get
+
+      pid = spawn *command
+      Process.wait pid
+    end
+
+    ###
+
+    desc "crawl", "Run a particular spider by it's name"
+    def crawl(spider_name)
+      raise "Can't find Kimurai project" unless inside_project?
+      require './config/boot'
+
+      unless klass = Kimurai.find_by_name(spider_name)
+        raise "Can't find spider with name `#{spider_name}` in the project. " \
+          "To list all available spiders, run: `$ bundle exec kimurai list`"
+      end
+
+      # Set time_zone if exists
+      if time_zone = Kimurai.configuration.time_zone
+        Kimurai.time_zone = time_zone
+      end
+
+      klass.crawl!
+    end
+
+    desc "parse", "Parse url in the particular spider method"
+    option :url, type: :string, required: true, banner: "Url to pass to the method"
+    def parse(spider_name, method_name)
+      raise "Can't find Kimurai project" unless inside_project?
+      require './config/boot'
+
+      unless klass = Kimurai.find_by_name(spider_name)
+        raise "Can't find spider with name `#{spider_name}` in the project. " \
+          "To list all available spiders, run: `$ bundle exec kimurai list`"
+      end
+
+      klass.parse!(method_name, url: options["url"])
+    end
+
+    desc "console", "Start Kimurai console"
+    option :engine, type: :string, banner: "Engine to use"
+    option :url, type: :string, banner: "Url to process"
+    def console(spider_name = nil)
+      require 'pry'
+      require './config/boot' if inside_project?
+
+      if spider_name
+        raise "Can't find Kimurai project" unless inside_project?
+
+        unless klass = Kimurai.find_by_name(spider_name)
+          raise "Can't find spider with name `#{spider_name}` in the project. " \
+            "To list all available spiders, run: `$ bundle exec kimurai list`"
+        end
+      else
+        klass = inside_project? ? ApplicationSpider : ::Kimurai::Base
+      end
+
+      engine = options["engine"]&.delete(":")&.to_sym
+      klass.parse!(:console, engine, url: options["url"])
+    end
+
+    desc "list", "List all available spiders in the current project"
+    def list
+      raise "Can't find Kimurai project" unless inside_project?
+      require './config/boot'
+
+      Kimurai.list.keys.each { |name| puts name }
+    end
+
+    desc "runner", "Run all spiders in the project in queue"
+    option :jobs, aliases: :j, type: :numeric, default: 1, banner: "The number of concurrent jobs"
+    def runner
+      raise "Can't find Kimurai project" unless inside_project?
+
+      jobs = options["jobs"]
+      raise "Jobs count can't be 0" if jobs == 0
+
+      require './config/boot'
+      require 'kimurai/runner'
+      Runner.new(parallel_jobs: jobs).run!
+    end
+
+    desc "--version, -v", "Print the version"
+    def __print_version
+      puts VERSION
+    end
+
+    private
+
+    def inside_project?
+      Dir.exists? "spiders"
+    end
+  end
+end
+
+require_relative 'cli/generator'
+require_relative 'cli/ansible_command_builder'