kimurai 1.3.2 → 2.0.0

metadata

@@ -1,17 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: kimurai
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 2.0.0
 platform: ruby
 authors:
 - Victor Afanasev
-autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-08 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +38,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,27 +80,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: capybara
+  name: ostruct
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.15'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.15'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: capybara-mechanize
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -115,7 +108,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: poltergeist
+  name: mutex_m
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -129,7 +122,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: selenium-webdriver
+  name: nkf
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -143,7 +136,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: headless
+  name: reline
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -157,49 +150,49 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pmap
+  name: capybara
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.40'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.40'
 - !ruby/object:Gem::Dependency
-  name: whenever
+  name: capybara-mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.13'
 - !ruby/object:Gem::Dependency
-  name: rbcat
+  name: selenium-webdriver
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '4.27'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '4.27'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: headless
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -213,48 +206,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: pmap
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: whenever
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rbcat
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
+        version: '1.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
-description: 
+        version: '1.0'
 email:
 - vicfreefly@gmail.com
 executables:
@@ -263,7 +269,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".travis.yml"
+- ".rubocop.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -274,30 +280,21 @@ files:
 - exe/kimurai
 - kimurai.gemspec
 - lib/kimurai.rb
-- lib/kimurai/automation/deploy.yml
-- lib/kimurai/automation/setup.yml
-- lib/kimurai/automation/setup/chromium_chromedriver.yml
-- lib/kimurai/automation/setup/firefox_geckodriver.yml
-- lib/kimurai/automation/setup/phantomjs.yml
-- lib/kimurai/automation/setup/ruby_environment.yml
 - lib/kimurai/base.rb
 - lib/kimurai/base/saver.rb
 - lib/kimurai/base/storage.rb
 - lib/kimurai/base_helper.rb
 - lib/kimurai/browser_builder.rb
 - lib/kimurai/browser_builder/mechanize_builder.rb
-- lib/kimurai/browser_builder/poltergeist_phantomjs_builder.rb
 - lib/kimurai/browser_builder/selenium_chrome_builder.rb
 - lib/kimurai/browser_builder/selenium_firefox_builder.rb
 - lib/kimurai/capybara_configuration.rb
 - lib/kimurai/capybara_ext/driver/base.rb
 - lib/kimurai/capybara_ext/mechanize/driver.rb
-- lib/kimurai/capybara_ext/poltergeist/driver.rb
 - lib/kimurai/capybara_ext/selenium/driver.rb
 - lib/kimurai/capybara_ext/session.rb
 - lib/kimurai/capybara_ext/session/config.rb
 - lib/kimurai/cli.rb
-- lib/kimurai/cli/ansible_command_builder.rb
 - lib/kimurai/cli/generator.rb
 - lib/kimurai/core_ext/array.rb
 - lib/kimurai/core_ext/hash.rb
@@ -309,7 +306,6 @@ files:
 - lib/kimurai/template/Gemfile
 - lib/kimurai/template/README.md
 - lib/kimurai/template/config/application.rb
-- lib/kimurai/template/config/automation.yml
 - lib/kimurai/template/config/boot.rb
 - lib/kimurai/template/config/initializers/.keep
 - lib/kimurai/template/config/schedule.rb
@@ -326,7 +322,6 @@ homepage: https://github.com/vifreefly/kimuraframework
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -334,16 +329,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 4.0.1
 specification_version: 4
 summary: Modern web scraping framework written in Ruby and based on Capybara/Nokogiri
 test_files: []

data/.travis.yml

@@ -1,5 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.5.1
-before_install: gem install bundler -v 1.16.2

data/lib/kimurai/automation/deploy.yml

@@ -1,54 +0,0 @@
----
-- hosts: all
-  vars:
-    rbenv_root_path: /home/{{ ansible_user_id }}/.rbenv
-    rbenv_shims_path: "{{ rbenv_root_path }}/shims"
-    repo_url:
-    repo_name:
-    repo_key_path:
-
-  tasks:
-    - name: Copy custom git ssh key to /tmp/private_key (if provided)
-      when: repo_key_path is not none
-      copy:
-        src: "{{ repo_key_path }}"
-        dest: /tmp/private_key
-        mode: 0600
-
-    - name: Clone/pull project repo to ~/{{ repo_name }} user directory (using ssh-agent forwarding or https)
-      when: repo_key_path is none
-      git:
-        repo: "{{ repo_url }}"
-        dest: "~/{{ repo_name }}"
-        force: true
-        accept_hostkey: true
-
-    - name: Clone/pull project repo to ~/{{ repo_name }} user directory (using custom git ssh key)
-      when: repo_key_path is not none
-      git:
-        repo: "{{ repo_url }}"
-        dest: "~/{{ repo_name }}"
-        force: true
-        accept_hostkey: true
-        key_file: /tmp/private_key
-
-    - name: Delete custom git ssh key from /tmp/private_key (if provided)
-      when: repo_key_path is not none
-      file:
-        state: absent
-        path: /tmp/private_key
-
-    - name: Run bundle install
-      command: bundle install
-      args:
-        chdir: ~/{{ repo_name }}
-      environment:
-        PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-    - name: Run whenever to update crontab
-      command: whenever --update-crontab
-      args:
-        chdir: ~/{{ repo_name }}
-      environment:
-        PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-

data/lib/kimurai/automation/setup/chromium_chromedriver.yml

@@ -1,26 +0,0 @@
----
-- name: Install chromium browser
-  apt:
-    pkg: chromium-browser
-    state: present
-
-- name: Get current chromedriver version
-  shell: chromedriver --version
-  args:
-    executable: /bin/bash
-  register: current_chromedriver_version
-  changed_when: false
-  ignore_errors: true
-
-- name: Install unzip tool to unarchive chromedriver archive
-  apt:
-    pkg: unzip
-    state: present
-
-- name: Download chromedriver binary archive and unarchive it to /usr/local/bin
-  unarchive:
-    src: https://chromedriver.storage.googleapis.com/{{ chromedriver }}/chromedriver_linux64.zip
-    dest: /usr/local/bin
-    remote_src: true
-    mode: a+x
-  when: chromedriver not in current_chromedriver_version.stdout_lines

data/lib/kimurai/automation/setup/firefox_geckodriver.yml

@@ -1,20 +0,0 @@
----
-- name: Install firefox
-  apt:
-    pkg: firefox
-    state: present
-
-- name: Get current geckodriver version
-  shell: geckodriver --version
-  args:
-    executable: /bin/bash
-  register: current_geckodriver_version
-  changed_when: false
-  ignore_errors: true
-
-- name: Download geckodriver binary archive and unarchive it to /usr/local/bin
-  unarchive:
-    src: https://github.com/mozilla/geckodriver/releases/download/v{{ geckodriver }}/geckodriver-v{{ geckodriver }}-linux64.tar.gz
-    dest: /usr/local/bin
-    remote_src: true
-  when: geckodriver not in current_geckodriver_version.stdout

data/lib/kimurai/automation/setup/phantomjs.yml

@@ -1,33 +0,0 @@
----
-- name: Install dependencies for PhantomJS
-  apt:
-    pkg: "{{ item }}"
-    state: present
-  with_items:
-    - chrpath
-    - libxft-dev
-    - libfreetype6
-    - libfreetype6-dev
-    - libfontconfig1
-    - libfontconfig1-dev
-
-- name: Get current phantomjs version
-  shell: phantomjs -v
-  args:
-    executable: /bin/bash
-  register: current_phantomjs_version
-  changed_when: false
-  ignore_errors: true
-
-- name: Download phantomJS archive and unarchive it to /usr/local/lib
-  unarchive:
-    src: https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-{{ phantomjs }}-linux-x86_64.tar.bz2
-    dest: /usr/local/lib
-    remote_src: true
-  when: phantomjs not in current_phantomjs_version.stdout
-
-- name: Link PhantomJS binary to /usr/local/bin/phantomjs
-  file:
-    src: /usr/local/lib/phantomjs-{{ phantomjs }}-linux-x86_64/bin/phantomjs
-    dest: /usr/local/bin/phantomjs
-    state: link

data/lib/kimurai/automation/setup/ruby_environment.yml

@@ -1,124 +0,0 @@
----
-- name: Install dependencies for ruby-build
-  become: true
-  apt:
-    pkg: "{{ item }}"
-    state: present
-  with_items:
-    - zlib1g-dev
-    - build-essential
-    - libssl-dev
-    - libreadline-dev
-    - libreadline6-dev
-    - libyaml-dev
-    - libxml2-dev
-    - libxslt1-dev
-    - libcurl4-openssl-dev
-    - libffi-dev
-
-- name: Clone Rbenv repository to the {{ ansible_user_id }} user directory
-  git:
-    repo: https://github.com/sstephenson/rbenv.git
-    dest: "{{ rbenv_root_path }}"
-
-- name: Clone ruby-build repo to the {{ ansible_user_id }} user directory
-  git:
-    repo: https://github.com/sstephenson/ruby-build.git
-    dest: "{{ rbenv_root_path }}/plugins/ruby-build"
-
-- name: Add Rbenv path to the .bashrc
-  lineinfile:
-    dest: ~/.bashrc
-    regexp: '^export PATH="\$HOME\/\.rbenv'
-    line: export PATH="$HOME/.rbenv/bin:$PATH"
-    state: present
-
-- name: Add Rbenv init to the .bashrc
-  lineinfile:
-    dest: ~/.bashrc
-    regexp: '^eval "\$\(rbenv'
-    line: eval "$(rbenv init -)"
-    state: present
-
-- name: Check if desired Ruby version already installed
-  stat:
-    path: "{{ ruby_versions_path }}/{{ ruby }}"
-  register: ruby_present
-
-- name: Install desired Ruby version using ruby-build (this can take a while)
-  command: rbenv install {{ ruby }}
-  when: not ruby_present.stat.exists
-  environment:
-    CONFIGURE_OPTS: "--disable-install-doc"
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-- name: Get current Ruby version
-  command: "ruby -v"
-  register: current_ruby_version
-  changed_when: false
-  ignore_errors: true
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-- name: Set desired Ruby version as a global version
-  command: "rbenv global {{ ruby }}"
-  when: ruby not in current_ruby_version.stdout
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-  register: set_ruby
-
-- name: Execute `rbenv rehash` command
-  command: rbenv rehash
-  when: set_ruby.changed
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-- name: Create ~/.gemrc file to skip docs
-  copy:
-    dest: ~/.gemrc
-    content: "gem: --no-ri --no-rdoc"
-
-- name: Create ~/.bundle directory
-  file:
-    dest: ~/.bundle
-    state: directory
-
-- name: Create ~/.bundle/config file with default settings `BUNDLE_GIT__ALLOW_INSECURE true` and `BUNDLE_JOBS 4`
-  copy:
-    dest: ~/.bundle/config
-    content: |
-      BUNDLE_GIT__ALLOW_INSECURE: "true"
-      BUNDLE_JOBS: "4"
-
-- name: Check if Bundler gem installed
-  stat:
-    path: "{{ ruby_versions_path }}/{{ ruby }}/bin/bundler"
-  register: bundler_gem_present
-
-- name: Install Bundler gem
-  command: gem install bundler
-  when: not bundler_gem_present.stat.exists
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-- name: Check if Whenever gem installed
-  stat:
-    path: "{{ ruby_versions_path }}/{{ ruby }}/bin/whenever"
-  register: whenever_gem_present
-
-- name: Install Whenever gem
-  command: gem install whenever
-  when: not whenever_gem_present.stat.exists
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-- name: Check if Kimurai gem installed
-  stat:
-    path: "{{ ruby_versions_path }}/{{ ruby }}/bin/kimurai"
-  register: kimurai_gem_present
-
-- name: Install Kimurai gem
-  command: gem install kimurai
-  when: not kimurai_gem_present.stat.exists
-  environment:
-    PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"

data/lib/kimurai/automation/setup.yml

@@ -1,44 +0,0 @@
----
-- hosts: all
-  vars:
-    ruby: 2.5.3
-    rbenv_root_path: /home/{{ ansible_user_id }}/.rbenv
-    rbenv_shims_path: "{{ rbenv_root_path }}/shims"
-    ruby_versions_path: "{{ rbenv_root_path }}/versions"
-    # check latest here http://phantomjs.org/download.html
-    phantomjs: 2.1.1
-    # check latest here https://github.com/mozilla/geckodriver/releases/
-    geckodriver: 0.23.0
-    # check latest here https://sites.google.com/a/chromium.org/chromedriver/downloads
-    chromedriver: 2.44
-
-  tasks:
-    - name: Update apt cache
-      become: true
-      apt: update_cache=yes cache_valid_time=86400
-
-    - name: Install base packages
-      become: true
-      apt:
-        pkg: "{{ item }}"
-        state: present
-      with_items:
-        - xvfb
-        - libsqlite3-dev
-        - sqlite3
-        - mongodb-clients
-        - mysql-client
-        - libmysqlclient-dev
-        - postgresql-client
-        - libpq-dev
-
-    - import_tasks: setup/ruby_environment.yml
-
-    - import_tasks: setup/phantomjs.yml
-      become: true
-
-    - import_tasks: setup/firefox_geckodriver.yml
-      become: true
-
-    - import_tasks: setup/chromium_chromedriver.yml
-      become: true