kimurai 1.4.0 → 2.0.0

data/lib/kimurai/template/config/boot.rb

@@ -6,17 +6,17 @@ Bundler.require(:default, Kimurai.env)
 require 'dotenv/load'
 
 # require initializers
-Dir.glob(File.join("./config/initializers", "*.rb"), &method(:require))
+Dir.glob(File.join('./config/initializers', '*.rb'), &method(:require))
 
 # require helpers
-Dir.glob(File.join("./helpers", "*.rb"), &method(:require))
+Dir.glob(File.join('./helpers', '*.rb'), &method(:require))
 
 # require pipelines
-Dir.glob(File.join("./pipelines", "*.rb"), &method(:require))
+Dir.glob(File.join('./pipelines', '*.rb'), &method(:require))
 
 # require spiders recursively in the `spiders/` folder
 require_relative '../spiders/application_spider'
-require_all "spiders"
+require_all 'spiders'
 
 # require Kimurai configuration
 require_relative 'application'

data/lib/kimurai/template/config/schedule.rb

@@ -2,7 +2,7 @@
 require 'tzinfo'
 
 # Export current PATH to the cron
-env :PATH, ENV["PATH"]
+env :PATH, ENV['PATH']
 
 # Use 24 hour format when using `at:` option
 set :chronic_options, hours24: true
@@ -19,34 +19,34 @@ def local_to_utc(time_string, zone:)
   TZInfo::Timezone.get(zone).local_to_utc(Time.parse(time_string))
 end
 
-# Note: by default Whenever exports cron commands with :environment == "production".
+# NOTE: by default Whenever exports cron commands with :environment == "production".
 # Note: Whenever can only append log data to a log file (>>). If you want
 # to overwrite (>) log file before each run, pass lambda:
 # crawl "google_spider.com", output: -> { "> log/google_spider.com.log 2>&1" }
 
 # Project job types
-job_type :crawl,  "cd :path && KIMURAI_ENV=:environment bundle exec kimurai crawl :task :output"
-job_type :runner, "cd :path && KIMURAI_ENV=:environment bundle exec kimurai runner --jobs :task :output"
+job_type :crawl,  'cd :path && KIMURAI_ENV=:environment bundle exec kimurai crawl :task :output'
+job_type :runner, 'cd :path && KIMURAI_ENV=:environment bundle exec kimurai runner --jobs :task :output'
 
 # Single file job type
-job_type :single, "cd :path && KIMURAI_ENV=:environment ruby :task :output"
+job_type :single, 'cd :path && KIMURAI_ENV=:environment ruby :task :output'
 # Single with bundle exec
-job_type :single_bundle, "cd :path && KIMURAI_ENV=:environment bundle exec ruby :task :output"
+job_type :single_bundle, 'cd :path && KIMURAI_ENV=:environment bundle exec ruby :task :output'
 
 ### Schedule ###
 # Usage (check examples here https://github.com/javan/whenever#example-schedulerb-file):
 # every 1.day do
-  # Example to schedule a single spider in the project:
-  # crawl "google_spider.com", output: "log/google_spider.com.log"
+# Example to schedule a single spider in the project:
+# crawl "google_spider.com", output: "log/google_spider.com.log"
 
-  # Example to schedule all spiders in the project using runner. Each spider will write
-  # it's own output to the `log/spider_name.log` file (handled by a runner itself).
-  # Runner output will be written to log/runner.log file.
-  # Argument number it's a count of concurrent jobs:
-  # runner 3, output:"log/runner.log"
+# Example to schedule all spiders in the project using runner. Each spider will write
+# it's own output to the `log/spider_name.log` file (handled by a runner itself).
+# Runner output will be written to log/runner.log file.
+# Argument number it's a count of concurrent jobs:
+# runner 3, output:"log/runner.log"
 
-  # Example to schedule single spider (without project):
-  # single "single_spider.rb", output: "single_spider.log"
+# Example to schedule single spider (without project):
+# single "single_spider.rb", output: "single_spider.log"
 # end
 
 ### How to set a cron schedule ###

data/lib/kimurai/template/spiders/application_spider.rb

@@ -5,19 +5,18 @@
 class ApplicationSpider < Kimurai::Base
   include ApplicationHelper
 
-  # Default engine for spiders (available engines: :mechanize, :poltergeist_phantomjs,
-  # :selenium_firefox, :selenium_chrome)
-  @engine = :poltergeist_phantomjs
+  # Default engine for spiders (available engines: :mechanize, :selenium_firefox, :selenium_chrome)
+  @engine = :selenium_chrome
 
   # Pipelines list, by order.
   # To process item through pipelines pass item to the `send_item` method
-  @pipelines = [:validator, :saver]
+  @pipelines = %i[validator saver]
 
   # Default config. Set here options which are default for all spiders inherited
   # from ApplicationSpider. Child's class config will be deep merged with this one
   @config = {
     # Custom headers, format: hash. Example: { "some header" => "some value", "another header" => "another value" }
-    # Works only for :mechanize and :poltergeist_phantomjs engines (Selenium doesn't allow to set/get headers)
+    # Works for :mechanize engine. Selenium doesn't allow to set/get headers.
     # headers: {},
 
     # Custom User Agent, format: string or lambda.
@@ -49,7 +48,7 @@ class ApplicationSpider < Kimurai::Base
     # window_size: [1366, 768],
 
     # Skip images downloading if true, works for all engines
-    disable_images: true,
+    # disable_images: true,
 
     # Selenium engines only: headless mode, `:native` or `:virtual_display` (default is :native)
     # Although native mode has a better performance, virtual display mode
@@ -61,14 +60,9 @@ class ApplicationSpider < Kimurai::Base
     # Format: array of strings. Works only for :selenium_firefox and selenium_chrome
     # proxy_bypass_list: [],
 
-    # Option to provide custom SSL certificate. Works only for :poltergeist_phantomjs and :mechanize
+    # Option to provide custom SSL certificate. Works only for :mechanize
     # ssl_cert_path: "path/to/ssl_cert",
 
-    # Inject some JavaScript code to the browser.
-    # Format: array of strings, where each string is a path to JS file.
-    # Works only for poltergeist_phantomjs engine (Selenium doesn't support JS code injection)
-    # extensions: ["lib/code_to_inject.js"],
-
     # Automatically skip duplicated (already visited) urls when using `request_to` method.
     # Possible values: `true` or `hash` with options.
     # In case of `true`, all visited urls will be added to the storage's scope `:requests_urls`
@@ -118,12 +112,12 @@ class ApplicationSpider < Kimurai::Base
     # Perform several actions before each request:
     before_request: {
       # Change proxy before each request. The `proxy:` option above should be presented
-      # and has lambda format. Works only for poltergeist and mechanize engines
+      # and has lambda format. Works for :mechanize engine.
       # (Selenium doesn't support proxy rotation).
       # change_proxy: true,
 
       # Change user agent before each request. The `user_agent:` option above should be presented
-      # and has lambda format. Works only for poltergeist and mechanize engines
+      # and has lambda format. Works for :mechanize engine.
       # (selenium doesn't support to get/set headers).
       # change_user_agent: true,
 

data/lib/kimurai/version.rb

@@ -1,3 +1,3 @@
 module Kimurai
-  VERSION = "1.4.0"
+  VERSION = '2.0.0'.freeze
 end

data/lib/kimurai.rb

@@ -1,6 +1,8 @@
 require 'ostruct'
 require 'logger'
 require 'json'
+require 'uri'
+
 require 'active_support'
 require 'active_support/core_ext'
 require 'rbcat'
@@ -28,26 +30,28 @@ module Kimurai
     end
 
     def env
-      ENV.fetch("KIMURAI_ENV") { "development" }
+      ENV.fetch('KIMURAI_ENV', 'development')
     end
 
     def time_zone
-      ENV["TZ"]
+      ENV['TZ']
     end
 
     def time_zone=(value)
-      ENV.store("TZ", value)
+      ENV.store('TZ', value)
     end
 
     def list
       Base.descendants.map do |klass|
         next unless klass.name
+
         [klass.name, klass]
       end.compact.to_h
     end
 
     def find_by_name(name)
       return unless name
+
       Base.descendants.find { |klass| klass.name == name }
     end
   end

metadata

@@ -1,17 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: kimurai
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Victor Afanasev
-autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-30 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: thor
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -39,7 +38,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: activesupport
+  name: csv
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,27 +80,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: capybara
+  name: ostruct
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.15'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.15'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: capybara-mechanize
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -115,7 +108,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: poltergeist
+  name: mutex_m
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -129,7 +122,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: selenium-webdriver
+  name: nkf
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -143,7 +136,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: headless
+  name: reline
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -157,49 +150,49 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pmap
+  name: capybara
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.40'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '3.40'
 - !ruby/object:Gem::Dependency
-  name: whenever
+  name: capybara-mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.13'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.13'
 - !ruby/object:Gem::Dependency
-  name: rbcat
+  name: selenium-webdriver
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '4.27'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '4.27'
 - !ruby/object:Gem::Dependency
-  name: pry
+  name: headless
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -213,48 +206,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: pmap
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: whenever
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
+        version: '0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
+  name: rbcat
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
+        version: '1.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
-description: 
+        version: '1.0'
 email:
 - vicfreefly@gmail.com
 executables:
@@ -263,7 +269,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".travis.yml"
+- ".rubocop.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -274,30 +280,21 @@ files:
 - exe/kimurai
 - kimurai.gemspec
 - lib/kimurai.rb
-- lib/kimurai/automation/deploy.yml
-- lib/kimurai/automation/setup.yml
-- lib/kimurai/automation/setup/chromium_chromedriver.yml
-- lib/kimurai/automation/setup/firefox_geckodriver.yml
-- lib/kimurai/automation/setup/phantomjs.yml
-- lib/kimurai/automation/setup/ruby_environment.yml
 - lib/kimurai/base.rb
 - lib/kimurai/base/saver.rb
 - lib/kimurai/base/storage.rb
 - lib/kimurai/base_helper.rb
 - lib/kimurai/browser_builder.rb
 - lib/kimurai/browser_builder/mechanize_builder.rb
-- lib/kimurai/browser_builder/poltergeist_phantomjs_builder.rb
 - lib/kimurai/browser_builder/selenium_chrome_builder.rb
 - lib/kimurai/browser_builder/selenium_firefox_builder.rb
 - lib/kimurai/capybara_configuration.rb
 - lib/kimurai/capybara_ext/driver/base.rb
 - lib/kimurai/capybara_ext/mechanize/driver.rb
-- lib/kimurai/capybara_ext/poltergeist/driver.rb
 - lib/kimurai/capybara_ext/selenium/driver.rb
 - lib/kimurai/capybara_ext/session.rb
 - lib/kimurai/capybara_ext/session/config.rb
 - lib/kimurai/cli.rb
-- lib/kimurai/cli/ansible_command_builder.rb
 - lib/kimurai/cli/generator.rb
 - lib/kimurai/core_ext/array.rb
 - lib/kimurai/core_ext/hash.rb
@@ -309,7 +306,6 @@ files:
 - lib/kimurai/template/Gemfile
 - lib/kimurai/template/README.md
 - lib/kimurai/template/config/application.rb
-- lib/kimurai/template/config/automation.yml
 - lib/kimurai/template/config/boot.rb
 - lib/kimurai/template/config/initializers/.keep
 - lib/kimurai/template/config/schedule.rb
@@ -326,7 +322,6 @@ homepage: https://github.com/vifreefly/kimuraframework
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -334,16 +329,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 3.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
-signing_key: 
+rubygems_version: 4.0.1
 specification_version: 4
 summary: Modern web scraping framework written in Ruby and based on Capybara/Nokogiri
 test_files: []

data/.travis.yml

@@ -1,5 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.5.1
-before_install: gem install bundler -v 1.16.2

data/lib/kimurai/automation/deploy.yml

@@ -1,54 +0,0 @@
----
-- hosts: all
-  vars:
-    rbenv_root_path: /home/{{ ansible_user_id }}/.rbenv
-    rbenv_shims_path: "{{ rbenv_root_path }}/shims"
-    repo_url:
-    repo_name:
-    repo_key_path:
-
-  tasks:
-    - name: Copy custom git ssh key to /tmp/private_key (if provided)
-      when: repo_key_path is not none
-      copy:
-        src: "{{ repo_key_path }}"
-        dest: /tmp/private_key
-        mode: 0600
-
-    - name: Clone/pull project repo to ~/{{ repo_name }} user directory (using ssh-agent forwarding or https)
-      when: repo_key_path is none
-      git:
-        repo: "{{ repo_url }}"
-        dest: "~/{{ repo_name }}"
-        force: true
-        accept_hostkey: true
-
-    - name: Clone/pull project repo to ~/{{ repo_name }} user directory (using custom git ssh key)
-      when: repo_key_path is not none
-      git:
-        repo: "{{ repo_url }}"
-        dest: "~/{{ repo_name }}"
-        force: true
-        accept_hostkey: true
-        key_file: /tmp/private_key
-
-    - name: Delete custom git ssh key from /tmp/private_key (if provided)
-      when: repo_key_path is not none
-      file:
-        state: absent
-        path: /tmp/private_key
-
-    - name: Run bundle install
-      command: bundle install
-      args:
-        chdir: ~/{{ repo_name }}
-      environment:
-        PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-
-    - name: Run whenever to update crontab
-      command: whenever --update-crontab
-      args:
-        chdir: ~/{{ repo_name }}
-      environment:
-        PATH: "{{ rbenv_root_path }}/bin:{{ rbenv_root_path }}/shims:{{ ansible_env.PATH }}"
-

data/lib/kimurai/automation/setup/chromium_chromedriver.yml

@@ -1,26 +0,0 @@
----
-- name: Install chromium browser
-  apt:
-    pkg: chromium-browser
-    state: present
-
-- name: Get current chromedriver version
-  shell: chromedriver --version
-  args:
-    executable: /bin/bash
-  register: current_chromedriver_version
-  changed_when: false
-  ignore_errors: true
-
-- name: Install unzip tool to unarchive chromedriver archive
-  apt:
-    pkg: unzip
-    state: present
-
-- name: Download chromedriver binary archive and unarchive it to /usr/local/bin
-  unarchive:
-    src: https://chromedriver.storage.googleapis.com/{{ chromedriver }}/chromedriver_linux64.zip
-    dest: /usr/local/bin
-    remote_src: true
-    mode: a+x
-  when: chromedriver not in current_chromedriver_version.stdout_lines

data/lib/kimurai/automation/setup/firefox_geckodriver.yml

@@ -1,20 +0,0 @@
----
-- name: Install firefox
-  apt:
-    pkg: firefox
-    state: present
-
-- name: Get current geckodriver version
-  shell: geckodriver --version
-  args:
-    executable: /bin/bash
-  register: current_geckodriver_version
-  changed_when: false
-  ignore_errors: true
-
-- name: Download geckodriver binary archive and unarchive it to /usr/local/bin
-  unarchive:
-    src: https://github.com/mozilla/geckodriver/releases/download/v{{ geckodriver }}/geckodriver-v{{ geckodriver }}-linux64.tar.gz
-    dest: /usr/local/bin
-    remote_src: true
-  when: geckodriver not in current_geckodriver_version.stdout

data/lib/kimurai/automation/setup/phantomjs.yml

@@ -1,33 +0,0 @@
----
-- name: Install dependencies for PhantomJS
-  apt:
-    pkg: "{{ item }}"
-    state: present
-  with_items:
-    - chrpath
-    - libxft-dev
-    - libfreetype6
-    - libfreetype6-dev
-    - libfontconfig1
-    - libfontconfig1-dev
-
-- name: Get current phantomjs version
-  shell: phantomjs -v
-  args:
-    executable: /bin/bash
-  register: current_phantomjs_version
-  changed_when: false
-  ignore_errors: true
-
-- name: Download phantomJS archive and unarchive it to /usr/local/lib
-  unarchive:
-    src: https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-{{ phantomjs }}-linux-x86_64.tar.bz2
-    dest: /usr/local/lib
-    remote_src: true
-  when: phantomjs not in current_phantomjs_version.stdout
-
-- name: Link PhantomJS binary to /usr/local/bin/phantomjs
-  file:
-    src: /usr/local/lib/phantomjs-{{ phantomjs }}-linux-x86_64/bin/phantomjs
-    dest: /usr/local/bin/phantomjs
-    state: link