kimurai 1.4.0 → 2.0.0

data/lib/kimurai/automation/setup/ruby_environment.yml

@@ -1,124 +0,0 @@
----
-- name: Install dependencies for ruby-build
-  become: true
-  apt:
-    pkg: "{{ item }}"
-    state: present
-  with_items:
-    - zlib1g-dev
-    - build-essential
-    - libssl-dev
-    - libreadline-dev
-    - libreadline6-dev
-    - libyaml-dev
-    - libxml2-dev
-    - libxslt1-dev
-    - libcurl4-openssl-dev
-    - libffi-dev
-
-- name: Clone Rbenv repository to the {{ ansible_user_id }} user directory
-  git:
-    repo: https://github.com/sstephenson/rbenv.git
-    dest: "{{ rbenv_root_path }}"
-
-- name: Clone ruby-build repo to the {{ ansible_user_id }} user directory
-  git:
-    repo: https://github.com/sstephenson/ruby-build.git
-    dest: "{{ rbenv_root_path }}/plugins/ruby-build"
-
-- name: Add Rbenv path to the .bashrc
-  lineinfile:
-    dest: ~/.bashrc
-    regexp: '^export PATH="\$HOME\/\.rbenv'
-    line: export PATH="$HOME/.rbenv/bin:$PATH"
-    state: present
-
-- name: Add Rbenv init to the .bashrc
-  lineinfile:
-    dest: ~/.bashrc
-    regexp: '^eval "\$\(rbenv'
-    line: eval "$(rbenv init -)"
-    state: present
-
-- name: Check if desired Ruby version already installed
-  stat:
-    path: "{{ ruby_versions_path }}/{{ ruby }}"
-  register: ruby_present
-
-- name: Install desired Ruby version using ruby-build (this can take a while)
-  command: rbenv install {{ ruby }}
-  when: not ruby_present.stat.exists
-  environment:
-    CONFIGURE_OPTS: "--disable-install-doc"
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-- name: Get current Ruby version
-  command: "ruby -v"
-  register: current_ruby_version
-  changed_when: false
-  ignore_errors: true
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-- name: Set desired Ruby version as a global version
-  command: "rbenv global {{ ruby }}"
-  when: ruby not in current_ruby_version.stdout
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-  register: set_ruby
-
-- name: Execute `rbenv rehash` command
-  command: rbenv rehash
-  when: set_ruby.changed
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-- name: Create ~/.gemrc file to skip docs
-  copy:
-    dest: ~/.gemrc
-    content: "gem: --no-ri --no-rdoc"
-
-- name: Create ~/.bundle directory
-  file:
-    dest: ~/.bundle
-    state: directory
-
-- name: Create ~/.bundle/config file with default settings `BUNDLE_GIT__ALLOW_INSECURE true` and `BUNDLE_JOBS 4`
-  copy:
-    dest: ~/.bundle/config
-    content: |
-      BUNDLE_GIT__ALLOW_INSECURE: "true"
-      BUNDLE_JOBS: "4"
-
-- name: Check if Bundler gem installed
-  stat:
-    path: "{{ ruby_versions_path }}/{{ ruby }}/bin/bundler"
-  register: bundler_gem_present
-
-- name: Install Bundler gem
-  command: gem install bundler
-  when: not bundler_gem_present.stat.exists
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-- name: Check if Whenever gem installed
-  stat:
-    path: "{{ ruby_versions_path }}/{{ ruby }}/bin/whenever"
-  register: whenever_gem_present
-
-- name: Install Whenever gem
-  command: gem install whenever
-  when: not whenever_gem_present.stat.exists
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-- name: Check if Kimurai gem installed
-  stat:
-    path: "{{ ruby_versions_path }}/{{ ruby }}/bin/kimurai"
-  register: kimurai_gem_present
-
-- name: Install Kimurai gem
-  command: gem install kimurai
-  when: not kimurai_gem_present.stat.exists
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"

data/lib/kimurai/automation/setup.yml

@@ -1,44 +0,0 @@
----
-- hosts: all
-  vars:
-    ruby: 2.5.3
-    rbenv_root_path: /home/{{ ansible_user_id }}/.rbenv
-    rbenv_shims_path: "{{ rbenv_root_path }}/shims"
-    ruby_versions_path: "{{ rbenv_root_path }}/versions"
-    # check latest here http://phantomjs.org/download.html
-    phantomjs: 2.1.1
-    # check latest here https://github.com/mozilla/geckodriver/releases/
-    geckodriver: 0.23.0
-    # check latest here https://sites.google.com/a/chromium.org/chromedriver/downloads
-    chromedriver: 2.44
-
-  tasks:
-    - name: Update apt cache
-      become: true
-      apt: update_cache=yes cache_valid_time=86400
-
-    - name: Install base packages
-      become: true
-      apt:
-        pkg: "{{ item }}"
-        state: present
-      with_items:
-        - xvfb
-        - libsqlite3-dev
-        - sqlite3
-        - mongodb-clients
-        - mysql-client
-        - libmysqlclient-dev
-        - postgresql-client
-        - libpq-dev
-
-    - import_tasks: setup/ruby_environment.yml
-
-    - import_tasks: setup/phantomjs.yml
-      become: true
-
-    - import_tasks: setup/firefox_geckodriver.yml
-      become: true
-
-    - import_tasks: setup/chromium_chromedriver.yml
-      become: true

data/lib/kimurai/browser_builder/poltergeist_phantomjs_builder.rb

@@ -1,175 +0,0 @@
-require 'capybara'
-require 'capybara/poltergeist'
-require_relative '../capybara_configuration'
-require_relative '../capybara_ext/poltergeist/driver'
-require_relative '../capybara_ext/session'
-
-module Kimurai::BrowserBuilder
-  class PoltergeistPhantomjsBuilder
-    attr_reader :logger, :spider
-
-    def initialize(config, spider:)
-      @config = config
-      @spider = spider
-      @logger = spider.logger
-    end
-
-    def build
-      # Register driver
-      Capybara.register_driver :poltergeist_phantomjs do |app|
-        # Create driver options
-        driver_options = {
-          js_errors: false, debug: false, inspector: false, phantomjs_options: []
-        }
-
-        if extensions = @config[:extensions].presence
-          driver_options[:extensions] = extensions
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled extensions"
-        end
-
-        # Window size
-        if size = @config[:window_size].presence
-          driver_options[:window_size] = size
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled window_size"
-        end
-
-        # SSL
-        if ssl_cert_path = @config[:ssl_cert_path].presence
-          driver_options[:phantomjs_options] << "--ssl-certificates-path=#{ssl_cert_path}"
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom ssl_cert"
-        end
-
-        if @config[:ignore_ssl_errors].present?
-          driver_options[:phantomjs_options].push("--ignore-ssl-errors=yes", "--ssl-protocol=any")
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled ignore_ssl_errors"
-        end
-
-        # Disable images
-        if @config[:disable_images].present?
-          driver_options[:phantomjs_options] << "--load-images=no"
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled disable_images"
-        end
-
-        Capybara::Poltergeist::Driver.new(app, driver_options)
-      end
-
-      # Create browser instance (Capybara session)
-      @browser = Capybara::Session.new(:poltergeist_phantomjs)
-      @browser.spider = spider
-      logger.debug "BrowserBuilder (poltergeist_phantomjs): created browser instance"
-
-      # Proxy
-      if proxy = @config[:proxy].presence
-        proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
-        ip, port, type = proxy_string.split(":")
-
-        if %w(http socks5).include?(type)
-          @browser.driver.set_proxy(*proxy_string.split(":"))
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled #{type} proxy, ip: #{ip}, port: #{port}"
-        else
-          logger.error "BrowserBuilder (poltergeist_phantomjs): wrong type of proxy: #{type}, skipped"
-        end
-      end
-
-      # Headers
-      if headers = @config[:headers].presence
-        @browser.driver.headers = headers
-        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom headers"
-      end
-
-      if user_agent = @config[:user_agent].presence
-        user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
-
-        @browser.driver.add_header("User-Agent", user_agent_string)
-        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom user_agent"
-      end
-
-      # Cookies
-      if cookies = @config[:cookies].presence
-        cookies.each do |cookie|
-          @browser.driver.set_cookie(cookie[:name], cookie[:value], cookie)
-        end
-
-        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom cookies"
-      end
-
-      # Browser instance options
-      # skip_request_errors
-      if skip_errors = @config[:skip_request_errors].presence
-        @browser.config.skip_request_errors = skip_errors
-        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled skip_request_errors"
-      end
-
-      # retry_request_errors
-      if retry_errors = @config[:retry_request_errors].presence
-        @browser.config.retry_request_errors = retry_errors
-        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled retry_request_errors"
-      end
-
-      # restart_if
-      if requests_limit = @config.dig(:restart_if, :requests_limit).presence
-        @browser.config.restart_if[:requests_limit] = requests_limit
-        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled restart_if.requests_limit >= #{requests_limit}"
-      end
-
-      if memory_limit = @config.dig(:restart_if, :memory_limit).presence
-        @browser.config.restart_if[:memory_limit] = memory_limit
-        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled restart_if.memory_limit >= #{memory_limit}"
-      end
-
-      # before_request clear_cookies
-      if @config.dig(:before_request, :clear_cookies)
-        @browser.config.before_request[:clear_cookies] = true
-        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.clear_cookies"
-      end
-
-      # before_request clear_and_set_cookies
-      if @config.dig(:before_request, :clear_and_set_cookies)
-        if cookies = @config[:cookies].presence
-          @browser.config.cookies = cookies
-          @browser.config.before_request[:clear_and_set_cookies] = true
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.clear_and_set_cookies"
-        else
-          logger.error "BrowserBuilder (poltergeist_phantomjs): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
-        end
-      end
-
-      # before_request change_user_agent
-      if @config.dig(:before_request, :change_user_agent)
-        if @config[:user_agent].present? && @config[:user_agent].class == Proc
-          @browser.config.user_agent = @config[:user_agent]
-          @browser.config.before_request[:change_user_agent] = true
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.change_user_agent"
-        else
-          logger.error "BrowserBuilder (poltergeist_phantomjs): user_agent should be present and has lambda format to enable before_request.change_user_agent, skipped"
-        end
-      end
-
-      # before_request change_proxy
-      if @config.dig(:before_request, :change_proxy)
-        if @config[:proxy].present? && @config[:proxy].class == Proc
-          @browser.config.proxy = @config[:proxy]
-          @browser.config.before_request[:change_proxy] = true
-          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.change_proxy"
-        else
-          logger.error "BrowserBuilder (poltergeist_phantomjs): proxy should be present and has lambda format to enable before_request.change_proxy, skipped"
-        end
-      end
-
-      # before_request delay
-      if delay = @config.dig(:before_request, :delay).presence
-        @browser.config.before_request[:delay] = delay
-        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.delay"
-      end
-
-      # encoding
-      if encoding = @config[:encoding]
-        @browser.config.encoding = encoding
-        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled encoding: #{encoding}"
-      end
-
-      # return Capybara session instance
-      @browser
-    end
-  end
-end

data/lib/kimurai/capybara_ext/poltergeist/driver.rb

@@ -1,13 +0,0 @@
-require_relative '../driver/base'
-
-module Capybara::Poltergeist
-  class Driver
-    def pid
-      client_pid
-    end
-
-    def port
-      server.port
-    end
-  end
-end

data/lib/kimurai/cli/ansible_command_builder.rb

@@ -1,71 +0,0 @@
-require 'cliver'
-
-module Kimurai
-  class CLI
-    class AnsibleCommandBuilder
-      def initialize(user_host, options, playbook:, vars: {})
-        @user_host = user_host
-        @options = options
-        @playbook = playbook
-        @vars = vars
-      end
-
-      def get
-        unless Cliver.detect("ansible-playbook")
-          raise "Can't find `ansible-playbook` executable, to install: " \
-            "Mac OS X: `$ brew install ansible`, Ubuntu: `$ sudo apt install ansible`"
-        end
-
-        user = @user_host[/(.*?)\@/, 1]
-        host = @user_host[/\@(.+)/, 1] || @user_host
-        inventory = @options["port"] ? "#{host}:#{@options['port']}," : "#{host},"
-
-        gem_dir = Gem::Specification.find_by_name("kimurai").gem_dir
-        playbook_path = gem_dir + "/lib/kimurai/automation/" + "#{@playbook}.yml"
-
-        command = [
-          "ansible-playbook", playbook_path,
-          "--inventory", inventory,
-          "--ssh-extra-args", "-oForwardAgent=yes",
-          "--connection", @options["local"] ? "local" : "smart",
-          "--extra-vars", "ansible_python_interpreter=/usr/bin/python3"
-        ]
-
-        if File.exists? "config/automation.yml"
-          require 'yaml'
-          if config = YAML.load_file("config/automation.yml").dig(@playbook)
-            config.each { |key, value| @vars[key] = value unless @vars[key] }
-          end
-        end
-
-        @vars.each do |key, value|
-          next unless value.present?
-          command.push "--extra-vars", "#{key}=#{value}"
-        end
-
-        if user
-          command.push "--user", user
-        end
-
-        if @options["ask-sudo"]
-          command.push "--ask-become-pass"
-        end
-
-        if @options["ask-auth-pass"]
-          unless Cliver.detect("sshpass")
-            raise "Can't find `sshpass` executable for password authentication, to install: " \
-              "Mac OS X: `$ brew install http://git.io/sshpass.rb`, Ubuntu: `$ sudo apt install sshpass`"
-          end
-
-          command.push "--ask-pass"
-        end
-
-        if ssh_key_path = @options["ssh-key-path"]
-          command.push "--private-key", ssh_key_path
-        end
-
-        command
-      end
-    end
-  end
-end

data/lib/kimurai/template/config/automation.yml

@@ -1,13 +0,0 @@
-# software versions to install for `setup` command
-setup:
-  ruby: 2.5.1
-  # check latest here http://phantomjs.org/download.html
-  phantomjs: 2.1.1
-  # check latest here https://github.com/mozilla/geckodriver/releases/
-  geckodriver: 0.21.0
-  # check latest here https://sites.google.com/a/chromium.org/chromedriver/downloads
-  chromedriver: 2.39
-# settings for deploy command, you can use cli options as well (--repo-url, --git-key-path)
-deploy:
-  # repo_url: git@bitbucket.org:username/repo_name.git
-  # repo_key_path: ~/.ssh/id_rsa