kimurai_dynamic 1.4.1

data/lib/kimurai/base.rb

@@ -0,0 +1,330 @@
+require_relative 'base/saver'
+require_relative 'base/storage'
+
+module Kimurai
+  class Base
+    class InvalidUrlError < StandardError; end
+
+    # don't deep merge config's headers hash option
+    DMERGE_EXCLUDE = [:headers]
+
+    LoggerFormatter = proc do |severity, datetime, progname, msg|
+      current_thread_id = Thread.current.object_id
+      thread_type = Thread.main == Thread.current ? "M" : "C"
+      output = "%s, [%s#%d] [%s: %s] %5s -- %s: %s\n"
+        .freeze % [severity[0..0], datetime, $$, thread_type, current_thread_id, severity, progname, msg]
+
+      if Kimurai.configuration.colorize_logger != false && Kimurai.env == "development"
+        Rbcat.colorize(output, predefined: [:jsonhash, :logger])
+      else
+        output
+      end
+    end
+
+    include BaseHelper
+
+    ###
+
+    class << self
+      attr_reader :run_info, :savers, :storage
+    end
+
+    def self.running?
+      @run_info && @run_info[:status] == :running
+    end
+
+    def self.completed?
+      @run_info && @run_info[:status] == :completed
+    end
+
+    def self.failed?
+      @run_info && @run_info[:status] == :failed
+    end
+
+    def self.visits
+      @run_info && @run_info[:visits]
+    end
+
+    def self.items
+      @run_info && @run_info[:items]
+    end
+
+    def self.update(type, subtype)
+      return unless @run_info
+      @update_mutex.synchronize { @run_info[type][subtype] += 1 }
+    end
+
+    def self.add_event(scope, event)
+      return unless @run_info
+      @update_mutex.synchronize { @run_info[:events][scope][event] += 1 }
+    end
+
+    ###
+
+    @engine = :mechanize
+    @pipelines = []
+    @config = {}
+
+    def self.name
+      @name
+    end
+
+    def self.engine
+      @engine ||= superclass.engine
+    end
+
+    def self.pipelines
+      @pipelines ||= superclass.pipelines
+    end
+
+    def self.start_urls
+      @start_urls
+    end
+
+    def self.config
+      if superclass.equal?(::Object)
+        @config
+      else
+        superclass.config.deep_merge_excl(@config || {}, DMERGE_EXCLUDE)
+      end
+    end
+
+    ###
+
+    def self.logger
+      @logger ||= Kimurai.configuration.logger || begin
+        log_level = (ENV["LOG_LEVEL"] || Kimurai.configuration.log_level || "DEBUG").to_s.upcase
+        log_level = "Logger::#{log_level}".constantize
+        Logger.new(STDOUT, formatter: LoggerFormatter, level: log_level, progname: name)
+      end
+    end
+
+    def self.crawl!(exception_on_fail: true, url: nil)
+
+      logger.error "Spider: already running: #{name}" and return false if running?
+      if !url.nil?
+        start_urls = [url]
+      end
+
+      @storage = Storage.new
+      @savers = {}
+      @update_mutex = Mutex.new
+
+      @run_info = {
+        spider_name: name, status: :running, error: nil, environment: Kimurai.env,
+        start_time: Time.new, stop_time: nil, running_time: nil,
+        visits: { requests: 0, responses: 0 }, items: { sent: 0, processed: 0 },
+        events: { requests_errors: Hash.new(0), drop_items_errors: Hash.new(0), custom: Hash.new(0) }
+      }
+
+      ###
+
+      logger.info "Spider: started: #{name}"
+      open_spider if self.respond_to? :open_spider
+
+      spider = self.new
+      spider.with_info = true
+      if start_urls
+        start_urls.each do |start_url|
+          if start_url.class == Hash
+            spider.request_to(:parse, start_url)
+          else
+            spider.request_to(:parse, url: start_url)
+          end
+        end
+      else
+        spider.parse
+      end
+    rescue StandardError, SignalException, SystemExit => e
+      @run_info.merge!(status: :failed, error: e.inspect)
+      exception_on_fail ? raise(e) : [@run_info, e]
+    else
+      @run_info.merge!(status: :completed)
+    ensure
+      if spider
+        spider.browser.destroy_driver! if spider.instance_variable_get("@browser")
+
+        stop_time  = Time.now
+        total_time = (stop_time - @run_info[:start_time]).round(3)
+        @run_info.merge!(stop_time: stop_time, running_time: total_time)
+
+        close_spider if self.respond_to? :close_spider
+
+        message = "Spider: stopped: #{@run_info.merge(running_time: @run_info[:running_time]&.duration)}"
+        failed? ? logger.fatal(message) : logger.info(message)
+
+        @run_info, @storage, @savers, @update_mutex = nil
+      end
+    end
+
+    def self.parse!(handler, *args, **request)
+      spider = self.new
+
+      if args.present?
+        spider.public_send(handler, *args)
+      elsif request.present?
+        spider.request_to(handler, request)
+      else
+        spider.public_send(handler)
+      end
+    ensure
+      spider.browser.destroy_driver! if spider.instance_variable_get("@browser")
+    end
+
+    ###
+
+    attr_reader :logger
+    attr_accessor :with_info
+
+    def initialize(engine = self.class.engine, config: {})
+      @engine = engine || self.class.engine
+      @config = self.class.config.deep_merge_excl(config, DMERGE_EXCLUDE)
+      @pipelines = self.class.pipelines.map do |pipeline_name|
+        klass = Pipeline.descendants.find { |kl| kl.name == pipeline_name }
+        instance = klass.new
+        instance.spider = self
+        [pipeline_name, instance]
+      end.to_h
+
+      @logger = self.class.logger
+      @savers = {}
+    end
+
+    def browser
+      @browser ||= BrowserBuilder.build(@engine, @config, spider: self)
+    end
+
+    def request_to(handler, delay = nil, url:, data: {}, response_type: :html)
+      raise InvalidUrlError, "Requested url is invalid: #{url}" unless URI.parse(url).kind_of?(URI::HTTP)
+
+      if @config[:skip_duplicate_requests] && !unique_request?(url)
+        add_event(:duplicate_requests) if self.with_info
+        logger.warn "Spider: request_to: not unique url: #{url}, skipped" and return
+      end
+
+      visited = delay ? browser.visit(url, delay: delay) : browser.visit(url)
+      return unless visited
+
+      public_send(handler, browser.current_response(response_type), { url: url, data: data })
+    end
+
+    def console(response = nil, url: nil, data: {})
+      binding.pry
+    end
+
+    ###
+
+    def storage
+      # Note: for `.crawl!` uses shared thread safe Storage instance,
+      # otherwise, each spider instance will have it's own Storage
+      @storage ||= self.with_info ? self.class.storage : Storage.new
+    end
+
+    def unique?(scope, value)
+      storage.unique?(scope, value)
+    end
+
+    def save_to(path, item, format:, position: true, append: false)
+      @savers[path] ||= begin
+        options = { format: format, position: position, append: append }
+        if self.with_info
+          self.class.savers[path] ||= Saver.new(path, options)
+        else
+          Saver.new(path, options)
+        end
+      end
+
+      @savers[path].save(item)
+    end
+
+    ###
+
+    def add_event(scope = :custom, event)
+      if self.with_info
+        self.class.add_event(scope, event)
+      end
+
+      logger.info "Spider: new event (scope: #{scope}): #{event}" if scope == :custom
+    end
+
+    ###
+
+    private
+
+    def create_browser(engine, config = {})
+      Kimurai::BrowserBuilder.build(engine, config, spider: self)
+    end
+
+    def unique_request?(url)
+      options = @config[:skip_duplicate_requests]
+      if options.class == Hash
+        scope = options[:scope] || :requests_urls
+        if options[:check_only]
+          storage.include?(scope, url) ? false : true
+        else
+          storage.unique?(scope, url) ? true : false
+        end
+      else
+        storage.unique?(:requests_urls, url) ? true : false
+      end
+    end
+
+    def send_item(item, options = {})
+      logger.debug "Pipeline: starting processing item through #{@pipelines.size} #{'pipeline'.pluralize(@pipelines.size)}..."
+      self.class.update(:items, :sent) if self.with_info
+
+      @pipelines.each do |name, instance|
+        item = options[name] ? instance.process_item(item, options: options[name]) : instance.process_item(item)
+      end
+    rescue => e
+      logger.error "Pipeline: dropped: #{e.inspect} (#{e.backtrace.first}), item: #{item}"
+      add_event(:drop_items_errors, e.inspect) if self.with_info
+      false
+    else
+      self.class.update(:items, :processed) if self.with_info
+      logger.info "Pipeline: processed: #{JSON.generate(item)}"
+      true
+    ensure
+      if self.with_info
+        logger.info "Info: items: sent: #{self.class.items[:sent]}, processed: #{self.class.items[:processed]}"
+      end
+    end
+
+    def in_parallel(handler, urls, threads:, data: {}, delay: nil, engine: @engine, config: {})
+      parts = urls.in_sorted_groups(threads, false)
+      urls_count = urls.size
+
+      all = []
+      start_time = Time.now
+      logger.info "Spider: in_parallel: starting processing #{urls_count} urls within #{threads} threads"
+
+      parts.each do |part|
+        all << Thread.new(part) do |part|
+          Thread.current.abort_on_exception = true
+
+          spider = self.class.new(engine, config: @config.deep_merge_excl(config, DMERGE_EXCLUDE))
+          spider.with_info = true if self.with_info
+
+          part.each do |url_data|
+            if url_data.class == Hash
+              if url_data[:url].present? && url_data[:data].present?
+                spider.request_to(handler, delay, url_data)
+              else
+                spider.public_send(handler, url_data)
+              end
+            else
+              spider.request_to(handler, delay, url: url_data, data: data)
+            end
+          end
+        ensure
+          spider.browser.destroy_driver! if spider.instance_variable_get("@browser")
+        end
+
+        sleep 0.5
+      end
+
+      all.each(&:join)
+      logger.info "Spider: in_parallel: stopped processing #{urls_count} urls within #{threads} threads, total time: #{(Time.now - start_time).duration}"
+    end
+  end
+end

data/lib/kimurai/base_helper.rb

@@ -0,0 +1,22 @@
+module Kimurai
+  module BaseHelper
+    private
+
+    def absolute_url(url, base:)
+      return unless url
+      URI.join(base, URI.escape(url)).to_s
+    end
+
+    def escape_url(url)
+      uri = URI.parse(url)
+    rescue URI::InvalidURIError => e
+      URI.parse(URI.escape url).to_s rescue url
+    else
+      url
+    end
+
+    def normalize_url(url, base:)
+      escape_url(absolute_url(url, base: base))
+    end
+  end
+end

data/lib/kimurai/browser_builder/mechanize_builder.rb

@@ -0,0 +1,154 @@
+require 'capybara'
+require 'capybara/mechanize'
+require_relative '../capybara_configuration'
+require_relative '../capybara_ext/mechanize/driver'
+require_relative '../capybara_ext/session'
+
+module Kimurai::BrowserBuilder
+  class MechanizeBuilder
+    attr_reader :logger, :spider
+
+    def initialize(config, spider:)
+      @config = config
+      @spider = spider
+      @logger = spider.logger
+    end
+
+    def build
+      # Register driver
+      Capybara.register_driver :mechanize do |app|
+        driver = Capybara::Mechanize::Driver.new("app")
+        # keep the history as small as possible (by default it's unlimited)
+        driver.configure { |a| a.history.max_size = 2 }
+        driver
+      end
+
+      # Create browser instance (Capybara session)
+      @browser = Capybara::Session.new(:mechanize)
+      @browser.spider = spider
+      logger.debug "BrowserBuilder (mechanize): created browser instance"
+
+      if @config[:extensions].present?
+        logger.error "BrowserBuilder (mechanize): `extensions` option not supported, skipped"
+      end
+
+      # Proxy
+      if proxy = @config[:proxy].presence
+        proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
+        ip, port, type = proxy_string.split(":")
+
+        if type == "http"
+          @browser.driver.set_proxy(*proxy_string.split(":"))
+          logger.debug "BrowserBuilder (mechanize): enabled http proxy, ip: #{ip}, port: #{port}"
+        else
+          logger.error "BrowserBuilder (mechanize): can't set #{type} proxy (not supported), skipped"
+        end
+      end
+
+      # SSL
+      if ssl_cert_path = @config[:ssl_cert_path].presence
+        @browser.driver.browser.agent.http.ca_file = ssl_cert_path
+        logger.debug "BrowserBuilder (mechanize): enabled custom ssl_cert"
+      end
+
+      if @config[:ignore_ssl_errors].present?
+        @browser.driver.browser.agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        logger.debug "BrowserBuilder (mechanize): enabled ignore_ssl_errors"
+      end
+
+      # Headers
+      if headers = @config[:headers].presence
+        @browser.driver.headers = headers
+        logger.debug "BrowserBuilder (mechanize): enabled custom headers"
+      end
+
+      if user_agent = @config[:user_agent].presence
+        user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
+
+        @browser.driver.add_header("User-Agent", user_agent_string)
+        logger.debug "BrowserBuilder (mechanize): enabled custom user_agent"
+      end
+
+      # Cookies
+      if cookies = @config[:cookies].presence
+        cookies.each do |cookie|
+          @browser.driver.set_cookie(cookie[:name], cookie[:value], cookie)
+        end
+
+        logger.debug "BrowserBuilder (mechanize): enabled custom cookies"
+      end
+
+      # Browser instance options
+      # skip_request_errors
+      if skip_errors = @config[:skip_request_errors].presence
+        @browser.config.skip_request_errors = skip_errors
+        logger.debug "BrowserBuilder (mechanize): enabled skip_request_errors"
+      end
+
+      # retry_request_errors
+      if retry_errors = @config[:retry_request_errors].presence
+        @browser.config.retry_request_errors = retry_errors
+        logger.debug "BrowserBuilder (mechanize): enabled retry_request_errors"
+      end
+
+      # restart_if
+      if @config[:restart_if].present?
+        logger.warn "BrowserBuilder (mechanize): restart_if options not supported by Mechanize, skipped"
+      end
+
+      # before_request clear_cookies
+      if @config.dig(:before_request, :clear_cookies)
+        @browser.config.before_request[:clear_cookies] = true
+        logger.debug "BrowserBuilder (mechanize): enabled before_request.clear_cookies"
+      end
+
+      # before_request clear_and_set_cookies
+      if @config.dig(:before_request, :clear_and_set_cookies)
+        if cookies = @config[:cookies].presence
+          @browser.config.cookies = cookies
+          @browser.config.before_request[:clear_and_set_cookies] = true
+          logger.debug "BrowserBuilder (mechanize): enabled before_request.clear_and_set_cookies"
+        else
+          logger.error "BrowserBuilder (mechanize): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
+        end
+      end
+
+      # before_request change_user_agent
+      if @config.dig(:before_request, :change_user_agent)
+        if @config[:user_agent].present? && @config[:user_agent].class == Proc
+          @browser.config.user_agent = @config[:user_agent]
+          @browser.config.before_request[:change_user_agent] = true
+          logger.debug "BrowserBuilder (mechanize): enabled before_request.change_user_agent"
+        else
+          logger.error "BrowserBuilder (mechanize): user_agent should be present and has lambda format to enable before_request.change_user_agent, skipped"
+        end
+      end
+
+      # before_request change_proxy
+      if @config.dig(:before_request, :change_proxy)
+        if @config[:proxy].present? && @config[:proxy].class == Proc
+          @browser.config.proxy = @config[:proxy]
+          @browser.config.before_request[:change_proxy] = true
+          logger.debug "BrowserBuilder (mechanize): enabled before_request.change_proxy"
+        else
+          logger.error "BrowserBuilder (mechanize): proxy should be present and has lambda format to enable before_request.change_proxy, skipped"
+        end
+      end
+
+      # before_request delay
+      if delay = @config.dig(:before_request, :delay).presence
+        @browser.config.before_request[:delay] = delay
+        logger.debug "BrowserBuilder (mechanize): enabled before_request.delay"
+      end
+
+      # encoding
+      if encoding = @config[:encoding]
+        @browser.config.encoding = encoding
+        logger.debug "BrowserBuilder (mechanize): enabled encoding: #{encoding}"
+      end
+
+      # return Capybara session instance
+      @browser
+    end
+  end
+end

data/lib/kimurai/browser_builder/poltergeist_phantomjs_builder.rb

@@ -0,0 +1,175 @@
+require 'capybara'
+require 'capybara/poltergeist'
+require_relative '../capybara_configuration'
+require_relative '../capybara_ext/poltergeist/driver'
+require_relative '../capybara_ext/session'
+
+module Kimurai::BrowserBuilder
+  class PoltergeistPhantomjsBuilder
+    attr_reader :logger, :spider
+
+    def initialize(config, spider:)
+      @config = config
+      @spider = spider
+      @logger = spider.logger
+    end
+
+    def build
+      # Register driver
+      Capybara.register_driver :poltergeist_phantomjs do |app|
+        # Create driver options
+        driver_options = {
+          js_errors: false, debug: false, inspector: false, phantomjs_options: []
+        }
+
+        if extensions = @config[:extensions].presence
+          driver_options[:extensions] = extensions
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled extensions"
+        end
+
+        # Window size
+        if size = @config[:window_size].presence
+          driver_options[:window_size] = size
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled window_size"
+        end
+
+        # SSL
+        if ssl_cert_path = @config[:ssl_cert_path].presence
+          driver_options[:phantomjs_options] << "--ssl-certificates-path=#{ssl_cert_path}"
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom ssl_cert"
+        end
+
+        if @config[:ignore_ssl_errors].present?
+          driver_options[:phantomjs_options].push("--ignore-ssl-errors=yes", "--ssl-protocol=any")
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled ignore_ssl_errors"
+        end
+
+        # Disable images
+        if @config[:disable_images].present?
+          driver_options[:phantomjs_options] << "--load-images=no"
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled disable_images"
+        end
+
+        Capybara::Poltergeist::Driver.new(app, driver_options)
+      end
+
+      # Create browser instance (Capybara session)
+      @browser = Capybara::Session.new(:poltergeist_phantomjs)
+      @browser.spider = spider
+      logger.debug "BrowserBuilder (poltergeist_phantomjs): created browser instance"
+
+      # Proxy
+      if proxy = @config[:proxy].presence
+        proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
+        ip, port, type = proxy_string.split(":")
+
+        if %w(http socks5).include?(type)
+          @browser.driver.set_proxy(*proxy_string.split(":"))
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled #{type} proxy, ip: #{ip}, port: #{port}"
+        else
+          logger.error "BrowserBuilder (poltergeist_phantomjs): wrong type of proxy: #{type}, skipped"
+        end
+      end
+
+      # Headers
+      if headers = @config[:headers].presence
+        @browser.driver.headers = headers
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom headers"
+      end
+
+      if user_agent = @config[:user_agent].presence
+        user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
+
+        @browser.driver.add_header("User-Agent", user_agent_string)
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom user_agent"
+      end
+
+      # Cookies
+      if cookies = @config[:cookies].presence
+        cookies.each do |cookie|
+          @browser.driver.set_cookie(cookie[:name], cookie[:value], cookie)
+        end
+
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled custom cookies"
+      end
+
+      # Browser instance options
+      # skip_request_errors
+      if skip_errors = @config[:skip_request_errors].presence
+        @browser.config.skip_request_errors = skip_errors
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled skip_request_errors"
+      end
+
+      # retry_request_errors
+      if retry_errors = @config[:retry_request_errors].presence
+        @browser.config.retry_request_errors = retry_errors
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled retry_request_errors"
+      end
+
+      # restart_if
+      if requests_limit = @config.dig(:restart_if, :requests_limit).presence
+        @browser.config.restart_if[:requests_limit] = requests_limit
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled restart_if.requests_limit >= #{requests_limit}"
+      end
+
+      if memory_limit = @config.dig(:restart_if, :memory_limit).presence
+        @browser.config.restart_if[:memory_limit] = memory_limit
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled restart_if.memory_limit >= #{memory_limit}"
+      end
+
+      # before_request clear_cookies
+      if @config.dig(:before_request, :clear_cookies)
+        @browser.config.before_request[:clear_cookies] = true
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.clear_cookies"
+      end
+
+      # before_request clear_and_set_cookies
+      if @config.dig(:before_request, :clear_and_set_cookies)
+        if cookies = @config[:cookies].presence
+          @browser.config.cookies = cookies
+          @browser.config.before_request[:clear_and_set_cookies] = true
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.clear_and_set_cookies"
+        else
+          logger.error "BrowserBuilder (poltergeist_phantomjs): cookies should be present to enable before_request.clear_and_set_cookies, skipped"
+        end
+      end
+
+      # before_request change_user_agent
+      if @config.dig(:before_request, :change_user_agent)
+        if @config[:user_agent].present? && @config[:user_agent].class == Proc
+          @browser.config.user_agent = @config[:user_agent]
+          @browser.config.before_request[:change_user_agent] = true
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.change_user_agent"
+        else
+          logger.error "BrowserBuilder (poltergeist_phantomjs): user_agent should be present and has lambda format to enable before_request.change_user_agent, skipped"
+        end
+      end
+
+      # before_request change_proxy
+      if @config.dig(:before_request, :change_proxy)
+        if @config[:proxy].present? && @config[:proxy].class == Proc
+          @browser.config.proxy = @config[:proxy]
+          @browser.config.before_request[:change_proxy] = true
+          logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.change_proxy"
+        else
+          logger.error "BrowserBuilder (poltergeist_phantomjs): proxy should be present and has lambda format to enable before_request.change_proxy, skipped"
+        end
+      end
+
+      # before_request delay
+      if delay = @config.dig(:before_request, :delay).presence
+        @browser.config.before_request[:delay] = delay
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled before_request.delay"
+      end
+
+      # encoding
+      if encoding = @config[:encoding]
+        @browser.config.encoding = encoding
+        logger.debug "BrowserBuilder (poltergeist_phantomjs): enabled encoding: #{encoding}"
+      end
+
+      # return Capybara session instance
+      @browser
+    end
+  end
+end