tanakai 1.5.0

data/lib/tanakai/capybara_ext/driver/base.rb

@@ -0,0 +1,62 @@
+require 'pathname'
+
+class Capybara::Driver::Base
+  attr_accessor :visited
+  attr_writer :requests, :responses
+
+  def requests
+    @requests ||= 0
+  end
+
+  def responses
+    @responses ||= 0
+  end
+
+  def current_memory
+    driver_pid = pid
+
+    all = (get_descendant_processes(driver_pid) << driver_pid).uniq
+    all.map { |pid| get_process_memory(pid) }.sum
+  end
+
+  private
+
+  def get_descendant_processes(base)
+    descendants = Hash.new { |ht, k| ht[k] = [k] }
+    Hash[*`ps -eo pid,ppid`.scan(/\d+/).map(&:to_i)].each do |pid, ppid|
+      descendants[ppid] << descendants[pid]
+    end
+
+    descendants[base].flatten - [base]
+  end
+
+  # https://github.com/schneems/get_process_mem
+  # Note: for Linux takes PSS (not RSS) memory (I think PSS better fits in this case)
+  def get_process_memory(pid)
+    case @platform ||= Gem::Platform.local.os
+    when "linux"
+      begin
+        file = Pathname.new "/proc/#{pid}/smaps"
+        return 0 unless file.exist?
+
+        lines = file.each_line.select { |line| line.match(/^Pss/) }
+        return 0 if lines.empty?
+
+        lines.reduce(0) do |sum, line|
+          line.match(/(?<value>(\d*\.{0,1}\d+))\s+(?<unit>\w\w)/) do |m|
+            sum += m[:value].to_i
+          end
+
+          sum
+        end
+      rescue Errno::EACCES
+        0
+      end
+    when "darwin"
+      mem = `ps -o rss= -p #{pid}`.strip
+      mem.empty? ? 0 : mem.to_i
+    else
+      raise "Can't check process memory, wrong type of platform: #{@platform}"
+    end
+  end
+end

data/lib/tanakai/capybara_ext/mechanize/driver.rb

@@ -0,0 +1,71 @@
+require 'mechanize'
+require_relative '../driver/base'
+
+class Capybara::Mechanize::Driver
+  # Extend capybara-mechnize to support Poltergeist-like methods
+  # https://www.rubydoc.info/gems/poltergeist/Capybara/Poltergeist/Driver
+
+  def set_proxy(ip, port, type, user = nil, password = nil)
+    # type is always "http", "socks" is not supported (yet)
+    browser.agent.set_proxy(ip, port, user, password)
+  end
+
+  ###
+
+  def headers
+    browser.agent.request_headers
+  end
+
+  def headers=(headers)
+    browser.agent.request_headers = headers
+  end
+
+  def add_header(name, value)
+    browser.agent.request_headers[name] = value
+  end
+
+  ###
+
+  def get_cookies
+    browser.agent.cookies
+  end
+
+  def set_cookie(name, value, options = {})
+    options[:name]  ||= name
+    options[:value] ||= value
+
+    cookie = Mechanize::Cookie.new(options.merge path: "/")
+    browser.agent.cookie_jar << cookie
+  end
+
+  def set_cookies(cookies)
+    cookies.each do |cookie|
+      set_cookie(cookie[:name], cookie[:value], cookie)
+    end
+  end
+
+  def clear_cookies
+    browser.agent.cookie_jar.clear!
+  end
+
+  ###
+
+  def quit
+    browser.agent.shutdown
+  end
+
+  ###
+
+  # Reset parent method `current_memory` for mechanize (we can't measure memory of Mechanize driver)
+  def current_memory
+    nil
+  end
+
+  def pid
+    nil
+  end
+
+  def port
+    nil
+  end
+end

data/lib/tanakai/capybara_ext/poltergeist/driver.rb

@@ -0,0 +1,13 @@
+require_relative '../driver/base'
+
+module Capybara::Poltergeist
+  class Driver
+    def pid
+      client_pid
+    end
+
+    def port
+      server.port
+    end
+  end
+end

data/lib/tanakai/capybara_ext/selenium/driver.rb

@@ -0,0 +1,34 @@
+require_relative '../driver/base'
+
+class Capybara::Selenium::Driver
+  def get_cookies
+    browser.manage.all_cookies
+  end
+
+  def set_cookie(name, value, options = {})
+    options[:name]  ||= name
+    options[:value] ||= value
+
+    browser.manage.add_cookie(options)
+  end
+
+  def set_cookies(cookies)
+    cookies.each do |cookie|
+      set_cookie(cookie[:name], cookie[:value], cookie)
+    end
+  end
+
+  def clear_cookies
+    browser.manage.delete_all_cookies
+  end
+
+  ###
+
+  def pid
+    @pid ||= `lsof -i tcp:#{port} -t`.strip.to_i
+  end
+
+  def port
+    @port ||= browser.send(:bridge).instance_variable_get("@http").instance_variable_get("@server_url").port
+  end
+end

data/lib/tanakai/capybara_ext/session/config.rb

@@ -0,0 +1,22 @@
+module Capybara
+  class SessionConfig
+    attr_accessor :cookies, :proxy, :user_agent, :encoding
+    attr_writer :retry_request_errors, :skip_request_errors
+
+    def retry_request_errors
+      @retry_request_errors ||= []
+    end
+
+    def skip_request_errors
+      @skip_request_errors ||= []
+    end
+
+    def restart_if
+      @restart_if ||= {}
+    end
+
+    def before_request
+      @before_request ||= {}
+    end
+  end
+end

data/lib/tanakai/capybara_ext/session.rb

@@ -0,0 +1,249 @@
+require 'capybara'
+require 'nokogiri'
+require 'json'
+require_relative 'session/config'
+
+module Capybara
+  class Session
+    attr_accessor :spider
+
+    alias_method :original_visit, :visit
+    def visit(visit_uri, delay: config.before_request[:delay], skip_request_options: false, max_retries: 3)
+      if spider
+        process_delay(delay) if delay
+        retries, sleep_interval = 0, 0
+
+        begin
+          check_request_options(visit_uri) unless skip_request_options
+          driver.requests += 1 and logger.info "Browser: started get request to: #{visit_uri}"
+          spider.class.update(:visits, :requests) if spider.with_info
+
+          original_visit(visit_uri)
+        rescue => e
+          if match_error?(e, type: :to_skip)
+            logger.error "Browser: skip request error: #{e.inspect}, url: #{visit_uri}"
+            spider.add_event(:requests_errors, e.inspect) if spider.with_info
+            false
+          elsif match_error?(e, type: :to_retry)
+            logger.error "Browser: retry request error: #{e.inspect}, url: #{visit_uri}"
+            spider.add_event(:requests_errors, e.inspect) if spider.with_info
+
+            if (retries += 1) <= max_retries
+              logger.info "Browser: sleep #{(sleep_interval += 15)} seconds and process retry № #{retries} to the url: #{visit_uri}"
+              sleep sleep_interval and retry
+            else
+              logger.error "Browser: all retries (#{retries - 1}) to the url #{visit_uri} are gone"
+              raise e unless skip_error_on_failure?(e)
+            end
+          else
+            raise e
+          end
+        else
+          driver.responses += 1 and logger.info "Browser: finished get request to: #{visit_uri}"
+          spider.class.update(:visits, :responses) if spider.with_info
+          driver.visited = true unless driver.visited
+          true
+        ensure
+          if spider.with_info
+            logger.info "Info: visits: requests: #{spider.class.visits[:requests]}, responses: #{spider.class.visits[:responses]}"
+          end
+
+          if memory = driver.current_memory
+            logger.debug "Browser: driver.current_memory: #{memory}"
+          end
+        end
+      else
+        original_visit(visit_uri)
+      end
+    end
+
+    def destroy_driver!
+      if @driver
+        begin
+          @driver.quit
+        # handle Net::ReadTimeout error for Selenium like drivers
+        rescue Net::ReadTimeout => e
+          @driver.quit
+        end
+
+        @driver = nil
+        logger.info "Browser: driver #{mode} has been destroyed"
+      else
+        logger.warn "Browser: driver #{mode} is not present"
+      end
+    end
+
+    def restart!
+      if mode.match?(/poltergeist/)
+        @driver.browser.restart
+        @driver.requests, @driver.responses = 0, 0
+      else
+        destroy_driver!
+        driver
+      end
+
+      logger.info "Browser: driver has been restarted: name: #{mode}, pid: #{driver.pid}, port: #{driver.port}"
+    end
+
+    def current_response(response_type = :html)
+      case response_type
+      when :html
+        if config.encoding
+          if config.encoding == :auto
+            charset = body.force_encoding("ISO-8859-1").encode("UTF-8")[/<meta.*?charset=["]?([\w+\d+\-]*)/i, 1]
+            Nokogiri::HTML(body, nil, charset)
+          else
+            Nokogiri::HTML(body, nil, config.encoding)
+          end
+        else
+          Nokogiri::HTML(body)
+        end
+      when :json
+        JSON.parse(body)
+      end
+    end
+
+    ###
+
+    # Handy method to perform some processing in the new tab within block and then automatically close this tab:
+    # Usage (url):
+    # browser.within_new_window_by(url: "https://google.com") do
+      # do some stuff and then automatically close this tab and return back to the first tab
+    # end
+    # Usage (action) (when new tab opening by some action, for example by clicking
+    # on a particular element):
+    # action = -> { browser.find("//some/element/path").click }
+    # browser.within_new_window_by(action: action) do
+      # do some stuff and then automatically close this tab and return back to the first tab
+    # end
+    def within_new_window_by(action: nil, url: nil)
+      case
+      when action
+        opened_window = window_opened_by { action.call }
+        within_window(opened_window) do
+          yield
+          current_window.close
+        end
+      when url
+        within_window(open_new_window) do
+          visit(url)
+
+          yield
+          current_window.close
+        end
+      end
+    end
+
+    ###
+
+    def scroll_to_bottom
+      execute_script("window.scrollBy(0,10000)")
+    end
+
+    private
+
+    def skip_error_on_failure?(e)
+      config.retry_request_errors.any? do |error|
+        error[:skip_on_failure] && e.class.ancestors.include?(error[:error]) if error.kind_of?(Hash)
+      end
+    end
+
+    def match_error?(e, type:)
+      errors =
+        case type
+        when :to_retry then config.retry_request_errors
+        when :to_skip then config.skip_request_errors
+        end
+
+      errors.any? do |error|
+        if error.kind_of?(Hash)
+          match_class = e.class.ancestors.include?(error[:error])
+          if error[:message].present?
+            if error[:message].kind_of?(Regexp)
+              e.message&.match?(error[:message])
+            else
+              e.message&.include?(error[:message])
+            end && match_class
+          else
+            match_class
+          end
+        else
+          e.class.ancestors.include?(error)
+        end
+      end
+    end
+
+    def process_delay(delay)
+      interval = (delay.class == Range ? rand(delay) : delay)
+      logger.debug "Browser: sleep #{interval.round(2)} #{'second'.pluralize(interval)} before request..."
+      sleep interval
+    end
+
+    def check_request_options(url_to_visit)
+      # restart_if
+      if memory_limit = config.restart_if[:memory_limit]
+        memory = driver.current_memory
+        if memory && memory >= memory_limit
+          logger.warn "Browser: memory_limit #{memory_limit} of driver.current_memory (#{memory}) is exceeded (engine: #{mode})"
+          restart!
+        end
+      end
+
+      if requests_limit = config.restart_if[:requests_limit]
+        requests = driver.requests
+        if requests >= requests_limit
+          logger.warn "Browser: requests_limit #{requests_limit} of driver.requests (#{requests}) is exceeded (engine: #{mode})"
+          restart!
+        end
+      end
+
+      # cookies
+      # (Selenium only) if config.cookies present and browser was just created,
+      # visit url_to_visit first and only then set cookies:
+      if driver.visited.nil? && config.cookies && mode.match?(/selenium/)
+        visit(url_to_visit, skip_request_options: true)
+        config.cookies.each do |cookie|
+          driver.set_cookie(cookie[:name], cookie[:value], cookie)
+        end
+      end
+
+      if config.before_request[:clear_cookies]
+        driver.clear_cookies
+        logger.debug "Browser: cleared cookies before request"
+      end
+
+      if config.before_request[:clear_and_set_cookies]
+        driver.clear_cookies
+
+        # (Selenium only) if browser is not visited yet any page, visit url_to_visit
+        # first and then set cookies (needs after browser restart):
+        if driver.visited.nil? && mode.match?(/selenium/)
+          visit(url_to_visit, skip_request_options: true)
+        end
+
+        config.cookies.each do |cookie|
+          driver.set_cookie(cookie[:name], cookie[:value], cookie)
+        end
+
+        logger.debug "Browser: cleared and set cookies before request"
+      end
+
+      # user_agent
+      if config.before_request[:change_user_agent]
+        driver.add_header("User-Agent", config.user_agent.call)
+        logger.debug "Browser: changed user_agent before request"
+      end
+
+      # proxy
+      if config.before_request[:change_proxy]
+        proxy_string = config.proxy.call
+        driver.set_proxy(*proxy_string.split(":"))
+        logger.debug "Browser: changed proxy before request"
+      end
+    end
+
+    def logger
+      spider.logger
+    end
+  end
+end

data/lib/tanakai/cli/ansible_command_builder.rb

@@ -0,0 +1,71 @@
+require 'cliver'
+
+module Tanakai
+  class CLI
+    class AnsibleCommandBuilder
+      def initialize(user_host, options, playbook:, vars: {})
+        @user_host = user_host
+        @options = options
+        @playbook = playbook
+        @vars = vars
+      end
+
+      def get
+        unless Cliver.detect("ansible-playbook")
+          raise "Can't find `ansible-playbook` executable, to install: " \
+            "Mac OS X: `$ brew install ansible`, Ubuntu: `$ sudo apt install ansible`"
+        end
+
+        user = @user_host[/(.*?)\@/, 1]
+        host = @user_host[/\@(.+)/, 1] || @user_host
+        inventory = @options["port"] ? "#{host}:#{@options['port']}," : "#{host},"
+
+        gem_dir = Gem::Specification.find_by_name("tanakai").gem_dir
+        playbook_path = gem_dir + "/lib/tanakai/automation/" + "#{@playbook}.yml"
+
+        command = [
+          "ansible-playbook", playbook_path,
+          "--inventory", inventory,
+          "--ssh-extra-args", "-oForwardAgent=yes",
+          "--connection", @options["local"] ? "local" : "smart",
+          "--extra-vars", "ansible_python_interpreter=/usr/bin/python3"
+        ]
+
+        if File.exists? "config/automation.yml"
+          require 'yaml'
+          if config = YAML.load_file("config/automation.yml").dig(@playbook)
+            config.each { |key, value| @vars[key] = value unless @vars[key] }
+          end
+        end
+
+        @vars.each do |key, value|
+          next unless value.present?
+          command.push "--extra-vars", "#{key}=#{value}"
+        end
+
+        if user
+          command.push "--user", user
+        end
+
+        if @options["ask-sudo"]
+          command.push "--ask-become-pass"
+        end
+
+        if @options["ask-auth-pass"]
+          unless Cliver.detect("sshpass")
+            raise "Can't find `sshpass` executable for password authentication, to install: " \
+              "Mac OS X: `$ brew install http://git.io/sshpass.rb`, Ubuntu: `$ sudo apt install sshpass`"
+          end
+
+          command.push "--ask-pass"
+        end
+
+        if ssh_key_path = @options["ssh-key-path"]
+          command.push "--private-key", ssh_key_path
+        end
+
+        command
+      end
+    end
+  end
+end

data/lib/tanakai/cli/generator.rb

@@ -0,0 +1,57 @@
+module Tanakai
+  class CLI
+    class Generator < Thor::Group
+      include Thor::Actions
+
+      def self.source_root
+        File.dirname(File.expand_path('..', __FILE__))
+      end
+
+      def generate_project(project_name)
+        directory "template", project_name
+        inside(project_name) do
+          run "bundle install"
+          run "git init"
+        end
+      end
+
+      def generate_spider(spider_name, in_project:)
+        spider_path = in_project ? "spiders/#{spider_name}.rb" : "./#{spider_name}.rb"
+        raise "Spider #{spider_path} already exists" if File.exists? spider_path
+
+        spider_class = to_spider_class(spider_name)
+        create_file spider_path do
+          <<~RUBY
+            class #{spider_class} < #{in_project ? 'ApplicationSpider' : 'Tanakai::Base'}
+              @name = "#{spider_name}"
+              @start_urls = []
+              @config = {}
+
+              def parse(response, url:, data: {})
+              end
+            end
+          RUBY
+        end
+
+        unless in_project
+          insert_into_file spider_path, "  @engine = :mechanize\n", after: "@name = \"#{spider_name}\"\n"
+          prepend_to_file spider_path, "require 'tanakai'\n\n"
+          append_to_file spider_path, "\n#{spider_class}.crawl!"
+        end
+      end
+
+      def generate_schedule
+        copy_file "template/config/schedule.rb", "./schedule.rb"
+      end
+
+      private
+
+      def to_spider_class(string)
+        string.sub(/^./) { $&.capitalize }
+          .gsub(/(?:_|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }
+          .gsub(/(?:-|(\/))([a-z\d]*)/) { "Dash#{$2.capitalize}" }
+          .gsub(/(?:\.|(\/))([a-z\d]*)/) { "#{$1}#{$2.capitalize}" }
+      end
+    end
+  end
+end