kimurai 1.0.1 → 1.1.0

data/lib/kimurai/browser_builder/selenium_firefox_builder.rb

@@ -82,7 +82,7 @@ module Kimurai
           if user_agent = @config[:user_agent].presence
             user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
             driver_options.profile["general.useragent.override"] = user_agent_string
-            logger.debug "BrowserBuilder (selenium_firefox): enabled custom user-agent"
+            logger.debug "BrowserBuilder (selenium_firefox): enabled custom user_agent"
           end
 
           # Headless mode
@@ -114,6 +114,10 @@ module Kimurai
         @browser.spider = spider
         logger.debug "BrowserBuilder (selenium_firefox): created browser instance"
 
+        if @config[:extensions].present?
+          logger.error "BrowserBuilder (selenium_firefox): `extensions` option not supported by Selenium, skipped"
+        end
+
         # Window size
         if size = @config[:window_size].presence
           @browser.current_window.resize_to(*size)
@@ -128,53 +132,53 @@ module Kimurai
 
         # Browser instance options
         # retry_request_errors
-        if errors = @config.dig(:browser, :retry_request_errors).presence
+        if errors = @config[:retry_request_errors].presence
           @browser.config.retry_request_errors = errors
-          logger.debug "BrowserBuilder (selenium_firefox): enabled `browser retry_request_errors`"
+          logger.debug "BrowserBuilder (selenium_firefox): enabled retry_request_errors"
         end
 
         # restart_if
-        if requests_limit = @config.dig(:browser, :restart_if, :requests_limit).presence
+        if requests_limit = @config.dig(:restart_if, :requests_limit).presence
           @browser.config.restart_if[:requests_limit] = requests_limit
-          logger.debug "BrowserBuilder (selenium_firefox): enabled `browser restart_if requests_limit` >= #{requests_limit}"
+          logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.requests_limit >= #{requests_limit}"
         end
 
-        if memory_limit = @config.dig(:browser, :restart_if, :memory_limit).presence
+        if memory_limit = @config.dig(:restart_if, :memory_limit).presence
           @browser.config.restart_if[:memory_limit] = memory_limit
-          logger.debug "BrowserBuilder (selenium_firefox): enabled `browser restart_if memory_limit` >= #{memory_limit}"
+          logger.debug "BrowserBuilder (selenium_firefox): enabled restart_if.memory_limit >= #{memory_limit}"
         end
 
         # before_request clear_cookies
-        if @config.dig(:browser, :before_request, :clear_cookies)
+        if @config.dig(:before_request, :clear_cookies)
           @browser.config.before_request[:clear_cookies] = true
-          logger.debug "BrowserBuilder (selenium_firefox): enabled `browser before_request clear_cookies`"
+          logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_cookies"
         end
 
         # before_request clear_and_set_cookies
-        if @config.dig(:browser, :before_request, :clear_and_set_cookies)
+        if @config.dig(:before_request, :clear_and_set_cookies)
           if cookies = @config[:cookies].presence
             @browser.config.cookies = cookies
             @browser.config.before_request[:clear_and_set_cookies] = true
-            logger.debug "BrowserBuilder (selenium_firefox): enabled `browser before_request clear_and_set_cookies`"
+            logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.clear_and_set_cookies"
           else
-            logger.error "BrowserBuilder (selenium_firefox): `cookies` should be present to enable `browser before_request clear_and_set_cookies`, skipped"
+            logger.error "BrowserBuilder (selenium_firefox): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
           end
         end
 
         # before_request change_user_agent
-        if @config.dig(:browser, :before_request, :change_user_agent)
-          logger.error "BrowserBuilder (selenium_firefox): `browser before_request change_user_agent` option not supported by Selenium, skipped"
+        if @config.dig(:before_request, :change_user_agent)
+          logger.error "BrowserBuilder (selenium_firefox): before_request.change_user_agent option not supported by Selenium, skipped"
         end
 
         # before_request change_proxy
-        if @config.dig(:browser, :before_request, :change_proxy)
-          logger.error "BrowserBuilder (selenium_firefox): `browser before_request change_proxy` option not supported by Selenium, skipped"
+        if @config.dig(:before_request, :change_proxy)
+          logger.error "BrowserBuilder (selenium_firefox): before_request.change_proxy option not supported by Selenium, skipped"
         end
 
         # before_request delay
-        if delay = @config.dig(:browser, :before_request, :delay).presence
+        if delay = @config.dig(:before_request, :delay).presence
           @browser.config.before_request[:delay] = delay
-          logger.debug "BrowserBuilder (selenium_firefox): enabled `browser before_request delay`"
+          logger.debug "BrowserBuilder (selenium_firefox): enabled before_request.delay"
         end
 
         # return Capybara session instance

data/lib/kimurai/capybara_ext/mechanize/driver.rb

@@ -5,7 +5,7 @@ class Capybara::Mechanize::Driver
   # Extend capybara-mechnize to support Poltergeist-like methods
   # https://www.rubydoc.info/gems/poltergeist/Capybara/Poltergeist/Driver
 
-  def set_proxy(ip, port, type, user, password)
+  def set_proxy(ip, port, type, user = nil, password = nil)
     # type is always "http", "socks" is not supported (yet)
     browser.agent.set_proxy(ip, port, user, password)
   end

data/lib/kimurai/capybara_ext/session.rb

@@ -6,10 +6,6 @@ module Capybara
   class Session
     attr_accessor :spider
 
-    def current_response
-      Nokogiri::HTML(body)
-    end
-
     alias_method :original_visit, :visit
     def visit(visit_uri, delay: config.before_request[:delay], skip_request_options: false, max_retries: 3)
       if spider
@@ -24,12 +20,13 @@ module Capybara
           original_visit(visit_uri)
         rescue *config.retry_request_errors => e
           logger.error "Browser: request visit error: #{e.inspect}, url: #{visit_uri}"
+          spider.add_event(:requests_errors, e.inspect) if spider.with_info
 
-          if (retries += 1) < max_retries
+          if (retries += 1) <= max_retries
             logger.info "Browser: sleep #{(sleep_interval += 15)} seconds and process retry № #{retries} to the url: #{visit_uri}"
             sleep sleep_interval and retry
           else
-            logger.error "Browser: all retries (#{retries}) to the url `#{visit_uri}` are gone"
+            logger.error "Browser: all retries (#{retries - 1}) to the url `#{visit_uri}` are gone"
             raise e
           end
         else
@@ -52,7 +49,13 @@ module Capybara
 
     def destroy_driver!
       if @driver
-        @driver.quit
+        begin
+          @driver.quit
+        # handle Net::ReadTimeout error for Selenium like drivers
+        rescue Net::ReadTimeout => e
+          @driver.quit
+        end
+
         @driver = nil
         logger.info "Browser: driver #{mode} has been destroyed"
       else
@@ -72,6 +75,43 @@ module Capybara
       logger.info "Browser: driver has been restarted: name: #{mode}, pid: #{driver.pid}, port: #{driver.port}"
     end
 
+    def current_response
+      Nokogiri::HTML(body)
+    end
+
+    ###
+
+    # Handy method to perform some processing in the new tab within block and then automatically close this tab:
+    # Usage (url):
+    # browser.within_new_window_by(url: "https://google.com") do
+      # do some stuff and then automatically close this tab and return back to the first tab
+    # end
+    # Usage (action) (when new tab opening by some action, for example by clicking
+    # on a particular element):
+    # action = -> { browser.find("//some/element/path").click }
+    # browser.within_new_window_by(action: action) do
+      # do some stuff and then automatically close this tab and return back to the first tab
+    # end
+    def within_new_window_by(action: nil, url: nil)
+      case
+      when action
+        opened_window = window_opened_by { action.call }
+        within_window(opened_window) do
+          yield
+          current_window.close
+        end
+      when url
+        within_window(open_new_window) do
+          visit(url)
+
+          yield
+          current_window.close
+        end
+      end
+    end
+
+    ###
+
     private
 
     def process_delay(delay)

data/lib/kimurai/cli.rb

@@ -66,6 +66,7 @@ module Kimurai
     ###
 
     desc "crawl", "Run a particular spider by it's name"
+    option :continue, aliases: :c, type: :boolean, default: false, banner: "Continue previous crawling"
     def crawl(spider_name)
       raise "Can't find Kimurai project" unless inside_project?
       require './config/boot'
@@ -80,7 +81,7 @@ module Kimurai
         Kimurai.time_zone = time_zone
       end
 
-      klass.crawl!
+      klass.crawl!(continue: options["continue"])
     end
 
     desc "parse", "Parse url in the particular spider method"

data/lib/kimurai/pipeline.rb

@@ -14,12 +14,16 @@ module Kimurai
 
     ###
 
+    def storage
+      spider.storage
+    end
+
     def unique?(scope, value)
       spider.unique?(scope, value)
     end
 
-    def save_to(path, item, format:, position: true)
-      spider.save_to(path, item, format: format, position: position)
+    def save_to(path, item, format:, position: true, append: false)
+      spider.save_to(path, item, format: format, position: position, append: append)
     end
 
     def logger

data/lib/kimurai/template/Gemfile

@@ -18,3 +18,11 @@ group :development do
   gem 'pry'
 end
 
+# If you want to save items to the database, require one of these gems:
+# gem 'sqlite3'
+# gem 'pg'
+# gem 'mysql2'
+
+# And use your preferred ORM/database connector:
+# gem 'activerecord', require: 'active_record'
+# gem 'sequel'

data/lib/kimurai/template/spiders/application_spider.rb

@@ -64,41 +64,56 @@ class ApplicationSpider < Kimurai::Base
     # Option to provide custom SSL certificate. Works only for :poltergeist_phantomjs and :mechanize
     # ssl_cert_path: "path/to/ssl_cert",
 
-    # Browser (Capybara session instance) options:
-    browser: {
-      # Array of errors to retry while processing a request
-      # retry_request_errors: [Net::ReadTimeout],
-      # Restart browser if one of the options is true:
-      restart_if: {
-        # Restart browser if provided memory limit (in kilobytes) is exceeded (works for all engines)
-        # memory_limit: 350_000,
-
-        # Restart browser if provided requests limit is exceeded (works for all engines)
-        # requests_limit: 100
-      },
-      before_request: {
-        # Change proxy before each request. The `proxy:` option above should be presented
-        # and has lambda format. Works only for poltergeist and mechanize engines
-        # (Selenium doesn't support proxy rotation).
-        # change_proxy: true,
-
-        # Change user agent before each request. The `user_agent:` option above should be presented
-        # and has lambda format. Works only for poltergeist and mechanize engines
-        # (selenium doesn't support to get/set headers).
-        # change_user_agent: true,
-
-        # Clear all cookies before each request, works for all engines
-        # clear_cookies: true,
-
-        # If you want to clear all cookies + set custom cookies (`cookies:` option above should be presented)
-        # use this option instead (works for all engines)
-        # clear_and_set_cookies: true,
-
-        # Global option to set delay between requests.
-        # Delay can be `Integer`, `Float` or `Range` (`2..5`). In case of a range,
-        # delay number will be chosen randomly for each request: `rand (2..5) # => 3`
-        # delay: 1..3
-      }
+    # Inject some JavaScript code to the browser.
+    # Format: array of strings, where each string is a path to JS file.
+    # Works only for poltergeist_phantomjs engine (Selenium doesn't support JS code injection)
+    # extensions: ["lib/code_to_inject.js"],
+
+    # Automatically skip duplicated (already visited) urls when using `request_to` method.
+    # Possible values: `true` or `hash` with options.
+    # In case of `true`, all visited urls will be added to the storage's scope `:requests_urls`
+    # and if url already contains in this scope, request will be skipped.
+    # You can configure this setting by providing additional options as hash:
+    # `skip_duplicate_requests: { scope: :custom_scope, check_only: true }`, where:
+    # `scope:` - use custom scope than `:requests_urls`
+    # `check_only:` - if true, then scope will be only checked for url, url will not
+    # be added to the scope if scope doesn't contains it.
+    # works for all drivers
+    # skip_duplicate_requests: true,
+
+    # Array of errors to retry while processing a request
+    # retry_request_errors: [Net::ReadTimeout],
+
+    # Restart browser if one of the options is true:
+    restart_if: {
+      # Restart browser if provided memory limit (in kilobytes) is exceeded (works for all engines)
+      # memory_limit: 350_000,
+
+      # Restart browser if provided requests limit is exceeded (works for all engines)
+      # requests_limit: 100
+    },
+    before_request: {
+      # Change proxy before each request. The `proxy:` option above should be presented
+      # and has lambda format. Works only for poltergeist and mechanize engines
+      # (Selenium doesn't support proxy rotation).
+      # change_proxy: true,
+
+      # Change user agent before each request. The `user_agent:` option above should be presented
+      # and has lambda format. Works only for poltergeist and mechanize engines
+      # (selenium doesn't support to get/set headers).
+      # change_user_agent: true,
+
+      # Clear all cookies before each request, works for all engines
+      # clear_cookies: true,
+
+      # If you want to clear all cookies + set custom cookies (`cookies:` option above should be presented)
+      # use this option instead (works for all engines)
+      # clear_and_set_cookies: true,
+
+      # Global option to set delay between requests.
+      # Delay can be `Integer`, `Float` or `Range` (`2..5`). In case of a range,
+      # delay number will be chosen randomly for each request: `rand (2..5) # => 3`
+      # delay: 1..3
     }
   }
 end

data/lib/kimurai/version.rb

@@ -1,3 +1,3 @@
 module Kimurai
-  VERSION = "1.0.1"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kimurai
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - Victor Afanasev
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-27 00:00:00.000000000 Z
+date: 2018-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -282,8 +282,8 @@ files:
 - lib/kimurai/automation/setup/phantomjs.yml
 - lib/kimurai/automation/setup/ruby_environment.yml
 - lib/kimurai/base.rb
-- lib/kimurai/base/simple_saver.rb
-- lib/kimurai/base/uniq_checker.rb
+- lib/kimurai/base/saver.rb
+- lib/kimurai/base/storage.rb
 - lib/kimurai/base_helper.rb
 - lib/kimurai/browser_builder.rb
 - lib/kimurai/browser_builder/mechanize_builder.rb
@@ -323,7 +323,7 @@ files:
 - lib/kimurai/template/spiders/application_spider.rb
 - lib/kimurai/template/tmp/.keep
 - lib/kimurai/version.rb
-homepage: https://github.com/vifreefly/kimurai
+homepage: https://github.com/vifreefly/kimuraframework
 licenses:
 - MIT
 metadata: {}

data/lib/kimurai/base/uniq_checker.rb

@@ -1,22 +0,0 @@
-module Kimurai
-  class Base
-    class UniqChecker
-      def initialize
-        @database = {}
-        @mutex = Mutex.new
-      end
-
-      def unique?(scope, value)
-        @mutex.synchronize do
-          @database[scope] ||= []
-          if @database[scope].include?(value)
-            false
-          else
-            @database[scope].push(value)
-            true
-          end
-        end
-      end
-    end
-  end
-end