kimurai 1.0.0

data/lib/kimurai/base.rb

@@ -0,0 +1,249 @@
+require_relative 'base/simple_saver'
+require_relative 'base/uniq_checker'
+
+module Kimurai
+  class Base
+    LoggerFormatter = proc do |severity, datetime, progname, msg|
+      current_thread_id = Thread.current.object_id
+      thread_type = Thread.main == Thread.current ? "M" : "C"
+      output = "%s, [%s#%d] [%s: %s] %5s -- %s: %s\n"
+        .freeze % [severity[0..0], datetime, $$, thread_type, current_thread_id, severity, progname, msg]
+
+      if Kimurai.configuration.colorize_logger != false && Kimurai.env == "development"
+        Rbcat.colorize(output, predefined: [:jsonhash, :logger])
+      else
+        output
+      end
+    end
+
+    include BaseHelper
+
+    ###
+
+    class << self
+      attr_reader :run_info
+    end
+
+    def self.running?
+      @run_info && @run_info[:status] == :running
+    end
+
+    def self.completed?
+      @run_info && @run_info[:status] == :completed
+    end
+
+    def self.failed?
+      @run_info && @run_info[:status] == :failed
+    end
+
+    def self.visits
+      @run_info && @run_info[:visits]
+    end
+
+    def self.items
+      @run_info && @run_info[:items]
+    end
+
+    def self.update(type, subtype)
+      return unless @run_info
+
+      (@update_mutex ||= Mutex.new).synchronize do
+        @run_info[type][subtype] += 1
+      end
+    end
+
+    ###
+
+    @engine = :mechanize
+    @pipelines = []
+    @config = {}
+
+    ###
+
+    def self.name
+      @name
+    end
+
+    def self.engine
+      @engine ||= superclass.engine
+    end
+
+    def self.pipelines
+      @pipelines ||= superclass.pipelines
+    end
+
+    def self.start_urls
+      @start_urls
+    end
+
+    def self.config
+      superclass.equal?(::Object) ? @config : superclass.config.deep_merge(@config || {})
+    end
+
+    ###
+
+    def self.logger
+      @logger ||= Kimurai.configuration.logger || begin
+        log_level = (ENV["LOG_LEVEL"] || Kimurai.configuration.log_level || "DEBUG").to_s.upcase
+        log_level = "Logger::#{log_level}".constantize
+        Logger.new(STDOUT, formatter: LoggerFormatter, level: log_level, progname: name)
+      end
+    end
+
+    ###
+
+    def self.checker
+      @checker ||= UniqChecker.new
+    end
+
+    def unique?(scope, value)
+      self.class.checker.unique?(scope, value)
+    end
+
+    def self.saver
+      @saver ||= SimpleSaver.new
+    end
+
+    def save_to(path, item, format:, position: true)
+      self.class.saver.save(path, item, format: format, position: position)
+    end
+
+    ###
+
+    def self.crawl!
+      logger.error "Spider: already running: #{name}" and return false if running?
+      @run_info = {
+        spider_name: name, status: :running, environment: Kimurai.env,
+        start_time: Time.new, stop_time: nil, running_time: nil,
+        visits: { requests: 0, responses: 0 }, items: { sent: 0, processed: 0 }, error: nil
+      }
+
+      logger.info "Spider: started: #{name}"
+      open_spider if self.respond_to? :open_spider
+
+      spider = self.new
+      spider.with_info = true
+      if start_urls
+        start_urls.each do |start_url|
+          spider.request_to(:parse, url: start_url)
+        end
+      else
+        spider.parse
+      end
+    rescue StandardError, SignalException => e
+      @run_info.merge!(status: :failed, error: e.inspect)
+      raise e
+    else
+      @run_info[:status] = :completed
+      @run_info
+    ensure
+      if spider
+        spider.browser.destroy_driver!
+
+        stop_time  = Time.now
+        total_time = (stop_time - @run_info[:start_time]).round(3)
+        @run_info.merge!(stop_time: stop_time, running_time: total_time)
+
+        close_spider if self.respond_to? :close_spider
+        message = "Spider: stopped: #{@run_info.merge(running_time: @run_info[:running_time]&.duration)}"
+        failed? ? @logger.fatal(message) : @logger.info(message)
+
+        @run_info, @checker, @saver = nil
+      end
+    end
+
+    def self.parse!(handler, engine = nil, url: nil, data: {})
+      spider = engine ? self.new(engine) : self.new
+      url.present? ? spider.request_to(handler, url: url, data: data) : spider.public_send(handler)
+    ensure
+      spider.browser.destroy_driver!
+    end
+
+    ###
+
+    attr_reader :logger
+    attr_accessor :with_info
+
+    def initialize(engine = self.class.engine, config: {})
+      @engine = engine
+      @config = self.class.config.deep_merge(config)
+      @pipelines = self.class.pipelines.map do |pipeline_name|
+        klass = Pipeline.descendants.find { |kl| kl.name == pipeline_name }
+        instance = klass.new
+        instance.spider = self
+        [pipeline_name, instance]
+      end.to_h
+
+      @logger = self.class.logger
+    end
+
+    def browser
+      @browser ||= BrowserBuilder.build(@engine, @config, spider: self)
+    end
+
+    def request_to(handler, delay = nil, url:, data: {})
+      request_data = { url: url, data: data }
+      delay ? browser.visit(url, delay: delay) : browser.visit(url)
+      public_send(handler, browser.current_response, request_data)
+    end
+
+    def console(response = nil, url: nil, data: {})
+      binding.pry
+    end
+
+    private
+
+    def send_item(item, options = {})
+      logger.debug "Pipeline: starting processing item through #{@pipelines.size} #{'pipeline'.pluralize(@pipelines.size)}..."
+      self.class.update(:items, :sent) if self.with_info
+
+      @pipelines.each do |name, instance|
+        item = options[name] ? instance.process_item(item, options: options[name]) : instance.process_item(item)
+      end
+    rescue => e
+      logger.error "Pipeline: dropped: #{e.inspect}, item: #{item}"
+      false
+    else
+      self.class.update(:items, :processed) if self.with_info
+      logger.info "Pipeline: processed: #{JSON.generate(item)}"
+      true
+    ensure
+      if self.with_info
+        logger.info "Info: items: sent: #{self.class.items[:sent]}, processed: #{self.class.items[:processed]}"
+      end
+    end
+
+    def in_parallel(handler, urls, threads:, data: {}, delay: nil, engine: @engine, config: {})
+      parts = urls.in_sorted_groups(threads, false)
+      urls_count = urls.size
+
+      all = []
+      start_time = Time.now
+      logger.info "Spider: in_parallel: starting processing #{urls_count} urls within #{threads} threads"
+
+      parts.each do |part|
+        all << Thread.new(part) do |part|
+          Thread.current.abort_on_exception = true
+
+          spider = self.class.new(engine, config: config)
+          spider.with_info = true if self.with_info
+
+          part.each do |url_data|
+            if url_data.class == Hash
+              spider.request_to(handler, delay, url_data)
+            else
+              spider.request_to(handler, delay, url: url_data, data: data)
+            end
+          end
+        ensure
+          spider.browser.destroy_driver!
+        end
+
+        sleep 0.5
+      end
+
+      all.each(&:join)
+      logger.info "Spider: in_parallel: stopped processing #{urls_count} urls within #{threads} threads, total time: #{(Time.now - start_time).duration}"
+    end
+  end
+end

data/lib/kimurai/base/simple_saver.rb

@@ -0,0 +1,98 @@
+require 'json'
+require 'csv'
+
+module Kimurai
+  class Base
+    class SimpleSaver
+      def initialize
+        @index = 0
+        @mutex = Mutex.new
+      end
+
+      def save(path, item, format:, position:)
+        @mutex.synchronize do
+          @index += 1
+          item[:position] = @index if position
+
+          case format
+          when :json
+            save_to_json(item, path)
+          when :pretty_json
+            save_to_pretty_json(item, path)
+          when :jsonlines
+            save_to_jsonlines(item, path)
+          when :csv
+            save_to_csv(item, path)
+          else
+            raise "SimpleSaver: wrong type of format: #{format}"
+          end
+        end
+      end
+
+      private
+
+      def save_to_json(item, path)
+        data = JSON.generate([item])
+
+        if @index > 1
+          file_content = File.read(path).sub(/\}\]\Z/, "\}\,")
+          File.open(path, "w") do |f|
+            f.write(file_content + data.sub(/\A\[/, ""))
+          end
+        else
+          File.open(path, "w") { |f| f.write(data) }
+        end
+      end
+
+      def save_to_pretty_json(item, path)
+        data = JSON.pretty_generate([item])
+
+        if @index > 1
+          file_content = File.read(path).sub(/\}\n\]\Z/, "\}\,\n")
+          File.open(path, "w") do |f|
+            f.write(file_content + data.sub(/\A\[\n/, ""))
+          end
+        else
+          File.open(path, "w") { |f| f.write(data) }
+        end
+      end
+
+      def save_to_jsonlines(item, path)
+        data = JSON.generate(item)
+
+        if @index > 1
+          File.open(path, "a") { |file| file.write("\n" + data) }
+        else
+          File.open(path, "w") { |file| file.write(data) }
+        end
+      end
+
+      def save_to_csv(item, path)
+        data = flatten_hash(item)
+
+        if @index > 1
+          CSV.open(path, "a+", force_quotes: true) do |csv|
+            csv << data.values
+          end
+        else
+          CSV.open(path, "w", force_quotes: true) do |csv|
+            csv << data.keys
+            csv << data.values
+          end
+        end
+      end
+
+      def flatten_hash(hash)
+        hash.each_with_object({}) do |(k, v), h|
+          if v.is_a? Hash
+            flatten_hash(v).map { |h_k, h_v| h["#{k}.#{h_k}"] = h_v }
+          else
+            h[k&.to_s] = v
+          end
+        end
+      end
+    end
+  end
+end
+
+

data/lib/kimurai/base/uniq_checker.rb

@@ -0,0 +1,22 @@
+module Kimurai
+  class Base
+    class UniqChecker
+      def initialize
+        @database = {}
+        @mutex = Mutex.new
+      end
+
+      def unique?(scope, value)
+        @mutex.synchronize do
+          @database[scope] ||= []
+          if @database[scope].include?(value)
+            false
+          else
+            @database[scope].push(value)
+            true
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kimurai/base_helper.rb

@@ -0,0 +1,22 @@
+module Kimurai
+  module BaseHelper
+    private
+
+    def absolute_url(url, base:)
+      return unless url
+      URI.join(base, URI.escape(url)).to_s
+    end
+
+    def escape_url(url)
+      uri = URI.parse(url)
+    rescue URI::InvalidURIError => e
+      URI.parse(URI.escape url).to_s rescue url
+    else
+      url
+    end
+
+    def normalize_url(url, base:)
+      escape_url(absolute_url(url, base: base))
+    end
+  end
+end

data/lib/kimurai/browser_builder.rb

@@ -0,0 +1,32 @@
+module Kimurai
+  class BrowserBuilder
+    AVAILABLE_ENGINES = [
+      :mechanize,
+      :mechanize_standalone,
+      :poltergeist_phantomjs,
+      :selenium_firefox,
+      :selenium_chrome
+    ]
+
+    def self.build(engine, config = {}, spider:)
+      unless AVAILABLE_ENGINES.include? engine
+        raise "BrowserBuilder: wrong name of engine, available engines: #{AVAILABLE_ENGINES.join(', ')}"
+      end
+
+      case engine
+      when :mechanize
+        require_relative 'browser_builder/mechanize_builder'
+        MechanizeBuilder.new(config, spider: spider).build
+      when :selenium_chrome
+        require_relative 'browser_builder/selenium_chrome_builder'
+        SeleniumChromeBuilder.new(config, spider: spider).build
+      when :poltergeist_phantomjs
+        require_relative 'browser_builder/poltergeist_phantomjs_builder'
+        PoltergeistPhantomJSBuilder.new(config, spider: spider).build
+      when :selenium_firefox
+        require_relative 'browser_builder/selenium_firefox_builder'
+        SeleniumFirefoxBuilder.new(config, spider: spider).build
+      end
+    end
+  end
+end

data/lib/kimurai/browser_builder/mechanize_builder.rb

@@ -0,0 +1,140 @@
+require 'capybara'
+require 'capybara/mechanize'
+require_relative '../capybara_configuration'
+require_relative '../capybara_ext/mechanize/driver'
+require_relative '../capybara_ext/session'
+
+module Kimurai
+  class BrowserBuilder
+    class MechanizeBuilder
+      attr_reader :logger, :spider
+
+      def initialize(config, spider:)
+        @config = config
+        @spider = spider
+        @logger = spider.logger
+      end
+
+      def build
+        # Register driver
+        Capybara.register_driver :mechanize do |app|
+          driver = Capybara::Mechanize::Driver.new("app")
+          # keep the history as small as possible (by default it's unlimited)
+          driver.configure { |a| a.history.max_size = 2 }
+          driver
+        end
+
+        # Create browser instance (Capybara session)
+        @browser = Capybara::Session.new(:mechanize)
+        @browser.spider = spider
+        logger.debug "BrowserBuilder (mechanize): created browser instance"
+
+        # Proxy
+        if proxy = @config[:proxy].presence
+          proxy_string = (proxy.class == Proc ? proxy.call : proxy).strip
+          ip, port, type = proxy_string.split(":")
+
+          if type == "socks5"
+            logger.error "BrowserBuilder (mechanize): can't set socks5 proxy (not supported), skipped"
+          else
+            @browser.set_proxy(*proxy_string.split(":"))
+            logger.debug "BrowserBuilder (mechanize): enabled #{type} proxy, ip: #{ip}, port: #{port}"
+          end
+        end
+
+        # SSL
+        if ssl_cert_path = @config[:ssl_cert_path].presence
+          @browser.driver.browser.agent.http.ca_file = ssl_cert_path
+          logger.debug "BrowserBuilder (mechanize): enabled custom ssl_cert"
+        end
+
+        if @config[:ignore_ssl_errors].present?
+          @browser.driver.browser.agent.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          logger.debug "BrowserBuilder (mechanize): enabled ignore_ssl_errors"
+        end
+
+        # Headers
+        if headers = @config[:headers].presence
+          @browser.driver.headers = headers
+          logger.debug "BrowserBuilder (mechanize): enabled custom headers"
+        end
+
+        if user_agent = @config[:user_agent].presence
+          user_agent_string = (user_agent.class == Proc ? user_agent.call : user_agent).strip
+
+          @browser.driver.add_header("User-Agent", user_agent_string)
+          logger.debug "BrowserBuilder (mechanize): enabled custom user-agent"
+        end
+
+        # Cookies
+        if cookies = @config[:cookies].presence
+          cookies.each do |cookie|
+            @browser.driver.set_cookie(cookie[:name], cookie[:value], cookie)
+          end
+
+          logger.debug "BrowserBuilder (mechanize): enabled custom cookies"
+        end
+
+        # Browser instance options
+        # retry_request_errors
+        if errors = @config.dig(:browser, :retry_request_errors).presence
+          @browser.config.retry_request_errors = errors
+          logger.debug "BrowserBuilder (mechanize): enabled `browser retry_request_errors`"
+        end
+
+        # restart_if
+        if @config.dig(:browser, :restart_if).present?
+          logger.error "BrowserBuilder (mechanize): `browser restart_if` options not supported by Mechanize, skipped"
+        end
+
+        # before_request clear_cookies
+        if @config.dig(:browser, :before_request, :clear_cookies)
+          @browser.config.before_request[:clear_cookies] = true
+          logger.debug "BrowserBuilder (mechanize): enabled `browser before_request clear_cookies`"
+        end
+
+        # before_request clear_and_set_cookies
+        if @config.dig(:browser, :before_request, :clear_and_set_cookies)
+          if cookies = @config[:cookies].presence
+            @browser.config.cookies = cookies
+            @browser.config.before_request[:clear_and_set_cookies] = true
+            logger.debug "BrowserBuilder (mechanize): enabled `browser before_request clear_and_set_cookies`"
+          else
+            logger.error "BrowserBuilder (mechanize): `cookies` should be present to enable `browser before_request clear_and_set_cookies`, skipped"
+          end
+        end
+
+        # before_request change_user_agent
+        if @config.dig(:browser, :before_request, :change_user_agent)
+          if @config[:user_agent].present? && @config[:user_agent].class == Proc
+            @browser.config.user_agent = @config[:user_agent]
+            @browser.config.before_request[:change_user_agent] = true
+            logger.debug "BrowserBuilder (mechanize): enabled `browser before_request change_user_agent`"
+          else
+            logger.error "BrowserBuilder (mechanize): `user_agent` should be present and has lambda format to enable `browser before_request change_user_agent`, skipped"
+          end
+        end
+
+        # before_request change_proxy
+        if @config.dig(:browser, :before_request, :change_proxy)
+          if @config[:proxy].present? && @config[:proxy].class == Proc
+            @browser.config.proxy = @config[:proxy]
+            @browser.config.before_request[:change_proxy] = true
+            logger.debug "BrowserBuilder (mechanize): enabled `browser before_request change_proxy`"
+          else
+            logger.error "BrowserBuilder (mechanize): `proxy` should be present and has lambda format to enable `browser before_request change_proxy`, skipped"
+          end
+        end
+
+        # before_request delay
+        if delay = @config.dig(:browser, :before_request, :delay).presence
+          @browser.config.before_request[:delay] = delay
+          logger.debug "BrowserBuilder (mechanize): enabled `browser before_request delay`"
+        end
+
+        # return Capybara session instance
+        @browser
+      end
+    end
+  end
+end