scruber 0.1.1

data/lib/scruber/core/page_format.rb

@@ -0,0 +1,33 @@
+module Scruber
+  module Core
+    module PageFormat
+      class << self
+        def process(page, page_format)
+          if page_format.nil?
+            nil
+          elsif _registered_formats.keys.include?(page_format.to_sym)
+            _registered_formats[page_format.to_sym].process(page)
+          else
+            raise "Unsupported format"
+          end
+        end
+
+        def add(label, claz)
+          unless claz.respond_to?(:process)
+            raise NoMethodError, "process is not declared in the #{claz.inspect}"
+          end
+
+          _registered_formats[label] = claz
+        end
+
+        def [](label)
+          _registered_formats[label]
+        end
+
+        def _registered_formats
+          @registered_formats ||= {}
+        end
+      end
+    end
+  end
+end

data/lib/scruber/fetcher.rb

@@ -0,0 +1,34 @@
+module Scruber
+  module Fetcher
+    class << self
+      attr_writer :adapter
+
+      def new(options={})
+        adapter.new(::Scruber.configuration.fetcher_options.merge(options))
+      end
+
+      def adapter
+        unless @adapter
+          @adapter = ::Scruber.configuration.fetcher_adapter || _adapters.keys.first
+        end
+        raise Scruber::ArgumentError.new("Adapter not found") unless @adapter
+        _adapters[@adapter]
+      end
+
+      def add_adapter(label, claz)
+        unless claz.method_defined?(:run)
+          raise NoMethodError, "run is not declared in the #{label.inspect}"
+        end
+        _adapters[label] = claz
+      end
+
+      def [](label)
+        _adapters[label]
+      end
+
+      def _adapters
+        @_adapters ||= {}
+      end
+    end
+  end
+end

data/lib/scruber/fetcher_adapters/abstract_adapter.rb

@@ -0,0 +1,119 @@
+module Scruber
+  module FetcherAdapters
+    class AbstractAdapter
+      attr_accessor :options, # all passed options
+                    :max_concurrency,
+                    :max_retry_times,
+                    :retry_delays,
+                    :followlocation,
+                    :request_timeout
+
+      def initialize(options={})
+        @options = options
+        @max_concurrency = options.fetch(:max_concurrency) { 1 }
+        @max_retry_times = options.fetch(:max_retry_times) { 5 }
+        @retry_delays = options.fetch(:retry_delays) { [1,2,2,4,4] }
+        @followlocation = options.fetch(:followlocation) { false }
+        @request_timeout = options.fetch(:request_timeout) { 15 }
+      end
+
+      def run(queue)
+        raise NotImplementedError
+      end
+
+      def before_request_callback(page)
+        page
+      end
+
+      def after_request_callback(page)
+        if bad_response?(page)
+          page.retry_at = determine_retry_at(page)
+          page.retry_count += 1
+          if page.max_retry_times.nil?
+            page.max_retry_times = @max_retry_times
+          end
+        else
+          page.fetched_at = Time.now.to_i
+        end
+        page
+      end
+
+      def headers_for(page)
+        if page.fetcher_agent
+          headers = page.fetcher_agent.headers
+        else
+          headers = page.headers
+        end
+        headers = {} unless headers.is_a?(Hash)
+        headers["User-Agent"] = user_agent_for(page)
+        cookie = cookie_for(page)
+        if cookie
+          headers["Cookie"] = cookie
+        end
+        headers
+      end
+
+      def cookie_for(page)
+        if page.fetcher_agent
+          cookie = page.fetcher_agent.cookie_for(page.url)
+          cookie.blank? ? nil : cookie
+        else
+          nil
+        end
+      end
+
+      def user_agent_for(page)
+        if page.user_agent
+          page.user_agent
+        elsif page.fetcher_agent && page.fetcher_agent.user_agent
+          page.fetcher_agent.user_agent
+        else
+          Scruber::Helpers::UserAgentRotator.next
+        end
+      end
+
+      def proxy_for(page)
+        if page.proxy
+          page.proxy
+        elsif page.fetcher_agent && page.fetcher_agent.proxy
+          page.fetcher_agent.proxy
+        else
+          Scruber::Helpers::ProxyRotator.next
+        end
+      end
+
+      def determine_retry_at(page)
+        delay = @retry_delays[page.retry_count] || @retry_delays.last
+        Time.now.to_i + delay
+      end
+
+      def bad_response?(page)
+        case page.response_code
+        when 0
+          true
+        when 1
+          true
+        when 100..199
+          true
+        when 200
+          false
+        when 201..299
+          false
+        when 300..399
+          @options.fetch(:followlocation) { false }
+        when 404
+          false
+        when 407
+          raise "RejectedByProxyError"
+        when 400..499
+          true
+        when 500..599
+          true
+        else
+          true
+        end
+      end
+
+    end
+  end
+end

data/lib/scruber/fetcher_adapters/typhoeus_fetcher.rb

@@ -0,0 +1,78 @@
+require 'typhoeus'
+module Scruber
+  module FetcherAdapters
+    class TyphoeusFetcher < AbstractAdapter
+      attr_accessor :ssl_verifypeer,
+                    :ssl_verifyhost
+
+      def initialize(options={})
+        super(options)
+        @ssl_verifypeer = options.fetch(:ssl_verifypeer) { false }
+        @ssl_verifyhost = options.fetch(:ssl_verifyhost) { 0 }
+        @max_requests = options.fetch(:max_requests) { @max_concurrency * 10 }
+      end
+
+      def run(queue)
+        queue.fetch_pending(@max_requests).each do |page|
+          request = build_request(page)
+
+          hydra.queue(request)
+        end
+        if hydra.queued_requests.count > 0
+          hydra.run
+        else
+          sleep 1
+        end
+      end
+
+      def build_request(page)
+        page = before_request_callback(page)
+        request_options = {
+          method: page[:method],
+          body: page[:body],
+          # params: page[:params],
+          headers: headers_for(page),
+          accept_encoding: 'gzip',
+          forbid_reuse: true,
+          followlocation: page.options.fetch(:followlocation){ @followlocation },
+          ssl_verifypeer: page.options.fetch(:ssl_verifypeer){ @ssl_verifypeer },
+          ssl_verifyhost: page.options.fetch(:ssl_verifyhost){ @ssl_verifyhost },
+          timeout: @request_timeout
+        }
+        
+        proxy = proxy_for(page)
+        request_options.merge!({proxy: proxy.http? ? proxy.address :  "socks://#{proxy.address}"}) if proxy
+        request_options.merge!({proxyuserpwd: proxy.proxyuserpwd}) if proxy && proxy.proxyuserpwd.present?
+
+        request = Typhoeus::Request.new(page[:url], request_options)
+
+        request.on_complete do |response|
+          on_complete_callback(page, response)
+        end
+
+        request
+      end
+
+      def hydra
+        @hydra ||= Typhoeus::Hydra.new(max_concurrency: @max_concurrency)
+      end
+
+      def on_complete_callback(page, response)
+        page.response_code = response.code
+        page.response_body = response.body
+        page.response_headers = response.response_headers
+        page.response_total_time = response.total_time
+        
+        if response.timed_out?
+          page[:response_code] = 1
+        end
+
+        page = after_request_callback(page)
+        page.save
+      end
+
+    end
+  end
+end
+
+Scruber::Fetcher.add_adapter(:typhoeus_fetcher, Scruber::FetcherAdapters::TyphoeusFetcher)

data/lib/scruber/helpers/dictionary_reader/csv.rb

@@ -0,0 +1,27 @@
+module Scruber
+  module Helpers
+    module DictionaryReader
+      class Csv
+        def initialize(file_path)
+          @file_path = file_path
+        end
+
+        def read(options={})
+          col_sep = options.delete(:col_sep) || ';'
+
+          CSV.foreach(@file_path, col_sep: col_sep, headers: true, encoding: 'utf-8') do |csv_row|
+            if options.blank?
+              yield csv_row
+            else
+              if options.all?{|(k,v)| csv_row[k.to_s] == v }
+                yield csv_row
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+Scruber::Helpers::DictionaryReader.add(:csv, Scruber::Helpers::DictionaryReader::Csv)

data/lib/scruber/helpers/dictionary_reader/xml.rb

@@ -0,0 +1,23 @@
+module Scruber
+  module Helpers
+    module DictionaryReader
+      class Xml
+        def initialize(file_path)
+          @xml = Nokogiri.parse(File.open(file_path).read)
+        end
+
+        def read(options={})
+          selector = options.delete(:selector) || 'item'
+          options.each do |k,v|
+            selector = "#{selector}[#{k}=\"#{v}\"]"
+          end
+          @xml.search(selector).each do |item|
+            yield item.to_h
+          end
+        end
+      end
+    end
+  end
+end
+
+Scruber::Helpers::DictionaryReader.add(:xml, Scruber::Helpers::DictionaryReader::Xml)

data/lib/scruber/helpers/dictionary_reader.rb

@@ -0,0 +1,33 @@
+module Scruber
+  module Helpers
+    module DictionaryReader
+      class << self 
+        def read(file_path, file_type, options)
+          if _registered_types.keys.include?(file_type.to_sym)
+            _registered_types[file_type.to_sym].new(file_path).read(options) do |obj|
+              yield obj
+            end
+          else
+            raise "Unsupported type, supported types #{_registered_types.keys}"
+          end
+        end
+
+        def add(label, claz)
+          unless claz.instance_methods.include?(:read)
+            raise NoMethodError, "read is not declared in the #{claz.inspect}"
+          end
+
+          _registered_types[label] = claz
+        end
+
+        def [](label)
+          _registered_types[label]
+        end
+
+        def _registered_types
+          @registered_types ||= {}
+        end
+      end
+    end
+  end
+end

data/lib/scruber/helpers/fetcher_agent.rb

@@ -0,0 +1,40 @@
+module Scruber
+  module Helpers
+    module FetcherAgent
+      class << self
+        attr_writer :adapter
+
+        def new(options={})
+          adapter.new(::Scruber.configuration.fetcher_agent_options.merge(options))
+        end
+
+        def find(id)
+          adapter.find(id)
+        end
+
+        def adapter
+          unless @adapter
+            @adapter = ::Scruber.configuration.fetcher_agent_adapter || _adapters.keys.first
+          end
+          raise Scruber::ArgumentError.new("Adapter not found") unless @adapter
+          _adapters[@adapter]
+        end
+
+        def add_adapter(label, claz)
+          # unless claz.method_defined?(:run)
+          #   raise NoMethodError, "run is not declared in the #{label.inspect}"
+          # end
+          _adapters[label] = claz
+        end
+
+        def [](label)
+          _adapters[label]
+        end
+
+        def _adapters
+          @_adapters ||= {}
+        end
+      end
+    end
+  end
+end

data/lib/scruber/helpers/fetcher_agent_adapters/abstract_adapter.rb

@@ -0,0 +1,69 @@
+module Scruber
+  module Helpers
+    module FetcherAgentAdapters
+      class AbstractAdapter
+        attr_accessor :id, :user_agent, :proxy_id, :headers, :cookie_jar, :updated_at, :created_at, :disable_proxy
+        attr_reader :jar
+
+        def initialize(options={})
+          @id = options.fetch(:id) { nil }
+          @user_agent = options.fetch(:user_agent) { nil }
+          @proxy_id = options.fetch(:proxy_id) { nil }
+          @headers = options.fetch(:headers) { {} }
+          @cookie_jar = options.fetch(:cookie_jar) { {} }
+          @disable_proxy = options.fetch(:disable_proxy) { false }
+          @updated_at = options.fetch(:updated_at) { Time.now }
+          @created_at = options.fetch(:created_at) { Time.now }
+          @jar = HTTP::CookieJar.new
+          if @cookie_jar.is_a?(String)
+            @jar.load(StringIO.new(@cookie_jar))
+          end
+          @_proxy = false
+        end
+
+        def proxy
+          if @_proxy == false
+            @_proxy = (@proxy_id ? Scruber::Helpers::ProxyRotator.find(@proxy_id) : nil)
+          else
+            @_proxy
+          end
+        end
+
+        def parse_cookies_from_page!(page)
+          cookies = page.response_cookies
+          cookies.each do |cookie|
+            @jar.parse(cookie, URI(page.url))
+          end
+        end
+
+        def serialize_cookies
+          io = StringIO.new
+          @jar.save(io)
+          @cookie_jar = io.string
+        end
+
+        def cookie_for(uri_or_url)
+          if uri_or_url.is_a?(String)
+            uri_or_url = URI(uri_or_url)
+          end
+          HTTP::Cookie.cookie_value(@jar.cookies(uri_or_url))
+        end
+
+        def save
+          raise NotImplementedError
+        end
+
+        def delete
+          raise NotImplementedError
+        end
+
+        class << self
+          def find(id)
+            raise NotImplementedError
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/scruber/helpers/fetcher_agent_adapters/memory.rb

@@ -0,0 +1,41 @@
+module Scruber
+  module Helpers
+    module FetcherAgentAdapters
+      class Memory < AbstractAdapter
+        def initialize(options={})
+          super(options)
+          @id = Time.now.to_i.to_s+'_'+rand(1_000..999_999).to_s if @id.nil?
+        end
+
+        def save
+          Scruber::Helpers::FetcherAgentAdapters::Memory.store(self)
+        end
+
+        def delete
+          Scruber::Helpers::FetcherAgentAdapters::Memory.delete(self)
+        end
+
+        class << self
+          def find(id)
+            _collection[id]
+          end
+
+          def _collection
+            @_collection ||= {}
+          end
+
+          def store(fetcher_agent)
+            _collection[fetcher_agent.id] = fetcher_agent
+          end
+
+          def delete(fetcher_agent)
+            _collection.delete fetcher_agent.id
+          end
+        end
+
+      end
+    end
+  end
+end
+
+Scruber::Helpers::FetcherAgent.add_adapter(:memory, Scruber::Helpers::FetcherAgentAdapters::Memory)

data/lib/scruber/helpers/proxy_rotator.rb

@@ -0,0 +1,125 @@
+module Scruber
+  module Helpers
+    class ProxyRotator
+
+      class Proxy
+        attr_accessor :host, :port, :user, :password, :probability, :type
+
+        def initialize(proxy, options={})
+          @host = proxy.split(':', 2).first
+          raise Scruber::ArgumentError.new("You need to specify proxy address") if @host.blank?
+          @port = options.fetch(:port) { proxy.split(':', 2)[1] }.to_i rescue nil
+          raise Scruber::ArgumentError.new("You need to specify :port for this proxy or pass full proxy address like 127.0.0.1:100") if @port.nil? || @port.zero?
+          @type = options.fetch(:type) { 'http' }
+          @user = options.fetch(:user) { nil }
+          @password = options.fetch(:password) { nil }
+          @probability = options.fetch(:probability) { 1 }
+        end
+
+        def id
+          (@host + ':' + @port.to_s)
+        end
+
+        def address
+          @host + ':' + @port.to_s
+        end
+
+        def http?
+          @type == 'http'
+        end
+
+        def proxyuserpwd
+          if @user.blank?
+            nil
+          else
+            "#{@user}:#{@password}"
+          end
+        end
+      end
+
+      class Configuration
+        include Scruber::Core::Extensions::Loop::CoreMethods
+
+        AVAILABLE_MODES=[:random, :round_robin]
+
+        attr_reader :mode, :proxies, :proxy_keys, :pickup
+
+        def initialize
+          @mode = :round_robin
+          @proxies = {}
+          @proxy_keys = []
+          @pickup = nil
+        end
+
+        def configure(&block)
+          instance_eval &block
+          rebuild_caches
+        end
+
+        def clean
+          @proxies = {}
+        end
+
+        def add(proxy_address, options={})
+          proxy = Proxy.new(proxy_address, options)
+          @proxies[proxy.id] = proxy
+        end
+
+        def set_mode(mode)
+          if AVAILABLE_MODES.include?(mode)
+            @mode = mode
+          else
+            raise Scruber::ArgumentError.new("Wrong mode. Available modes: #{AVAILABLE_MODES}")
+          end
+        end
+
+        private
+
+          def rebuild_caches
+            if @mode == :random
+              @pickup = Pickup.new(@proxies.inject({}){ |acc,(k,p)| acc[p] = p.probability; acc })
+            else
+              @proxy_keys = @proxies.keys
+            end
+          end
+      end
+
+      class << self
+        attr_writer :configuration
+        attr_accessor :cursor
+
+        def configuration
+          @configuration ||= Configuration.new
+        end
+
+        def configure(&block)
+          configuration.configure(&block)
+        end
+
+        def next(options={})
+          # raise Scruber::ArgumentError.new("Proxy rotator not configured") if @configuration.nil?
+          return nil if @configuration.nil?
+          if @configuration.mode == :random
+            @configuration.pickup.pick
+          else
+            if @cursor.nil? || @cursor >= @configuration.proxy_keys.count-1
+              @cursor = 0
+            else
+              @cursor += 1
+            end
+            @configuration.proxies[@configuration.proxy_keys[@cursor]]
+          end
+        end
+        alias_method :random, :next
+
+        def find(id)
+          @configuration.proxies[id] rescue nil
+        end
+
+        def configured?
+          !@configuration.nil? && !@configuration.proxies.blank?
+        end
+      end
+    end
+  end
+end