scraypa 0.1.1

data/lib/scraypa/configuration.rb

@@ -0,0 +1,20 @@
+module Scraypa
+  class Configuration
+    attr_accessor :use_capybara, :driver, :driver_options, :tor, :tor_options,
+                  :user_agent, :eye_tor_config_template, :throttle_seconds,
+                  :headless_chromium, :reset_driver_every_n_requests
+
+    def initialize
+      @use_capybara = nil
+      @tor = nil
+      @tor_options = nil
+      @user_agent = nil
+      @driver = nil
+      @driver_options = nil
+      @eye_tor_config_template = nil
+      @throttle_seconds = nil
+      @headless_chromium = nil
+      @reset_driver_every_n_requests = 5
+    end
+  end
+end

data/lib/scraypa/driver_resetter.rb

@@ -0,0 +1,37 @@
+module Scraypa
+  include Capybara::DSL
+
+  class DriverResetter
+    attr_accessor :requests_since_last_reset
+
+    def initialize every_n_requests
+      @every_n_requests = every_n_requests
+      @requests_since_last_reset = 0
+    end
+
+    def reset_if_nth_request
+      @requests_since_last_reset += 1
+      if @requests_since_last_reset >= @every_n_requests
+        Capybara.current_driver == :poltergeist ?
+          reset_poltergeist_driver : reset_headless_chromium_driver
+        @requests_since_last_reset = 0
+      end
+    end
+
+    private
+
+    def reset_poltergeist_driver
+      Capybara.reset_sessions!
+      Capybara.send(:session_pool).each do |session_name, session|
+        session.driver.restart if session_name.include?('poltergeist')
+      end
+    end
+
+    def reset_headless_chromium_driver
+      Capybara.reset_sessions!
+      Capybara.send(:session_pool).each do |session_name, session|
+        session.driver.quit if session_name.include?('headless_chromium')
+      end
+    end
+  end
+end

data/lib/scraypa/throttle.rb

@@ -0,0 +1,27 @@
+require 'rest-client'
+
+module Scraypa
+  class Throttle
+    attr_accessor :last_request_time
+    attr_reader :seconds
+
+    def initialize params={}
+      @seconds = params.fetch(:seconds, nil)
+    end
+
+    def throttle
+      @seconds && @last_request_time ? (@seconds.is_a?(Hash) ?
+          sleep_from_last_request_time_for(
+              Random.new.rand(@seconds[:from]..@seconds[:to])) :
+          sleep_from_last_request_time_for(@seconds)) : nil
+    end
+
+    private
+
+    def sleep_from_last_request_time_for seconds
+      sleep_time = @last_request_time ?
+          seconds - (Time.now - @last_request_time) : seconds
+      sleep(sleep_time) if sleep_time > 0
+    end
+  end
+end

data/lib/scraypa/user_agent/user_agent_abstract.rb

@@ -0,0 +1,15 @@
+module Scraypa
+  class UserAgentAbstract
+    def initialize(*args)
+
+    end
+
+    def user_agent
+      raise NotImplementedError, 'user_agent action not implemented.'
+    end
+
+    def list
+      raise NotImplementedError, 'list action not implemented'
+    end
+  end
+end

data/lib/scraypa/user_agent/user_agent_common_aliases_lists.rb

@@ -0,0 +1,27 @@
+module Scraypa
+  USER_AGENT_LIST = {
+      'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0',
+      'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)',
+      'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
+      'Mac Firefox' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0',
+      'Mac Mozilla' => 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.4a) Gecko/20030401',
+      'Mac Safari 4' => 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_2; de-at) AppleWebKit/531.21.8 (KHTML, like Gecko) Version/4.0.4 Safari/531.21.10',
+      'Mac Safari' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9',
+      'Windows Chrome' => 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.125 Safari/537.36',
+      'Windows IE 6' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)',
+      'Windows IE 7' => 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
+      'Windows IE 8' => 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
+      'Windows IE 9' => 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)',
+      'Windows IE 10' => 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)',
+      'Windows IE 11' => 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko',
+      'Windows Edge' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586',
+      'Windows Mozilla' => 'Mozilla/5.0 (Windows; U; Windows NT 5.0; en-US; rv:1.4b) Gecko/20030516 Mozilla Firebird/0.6',
+      'Windows Firefox' => 'Mozilla/5.0 (Windows NT 6.3; WOW64; rv:43.0) Gecko/20100101 Firefox/43.0'
+  }
+
+  USER_AGENT_MOBILE_LIST = {
+      'iPhone' => 'Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B5110e Safari/601.1',
+      'iPad' => 'Mozilla/5.0 (iPad; CPU OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1',
+      'Android' => 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.76 Safari/537.36'
+  }
+end

data/lib/scraypa/user_agent/user_agent_factory.rb

@@ -0,0 +1,19 @@
+module Scraypa
+  class UserAgentFactory
+    def self.build(*args)
+      #{
+      #    method: :common_aliases, :randomizer
+      #    list: :common_aliases,
+      #    strategy: :randomize,
+      #    change_after_n_requests: 2
+      #}
+
+      case args[0] && args[0][:method]
+        when :randomizer
+          UserAgentRandom.new(*args)
+        else
+          UserAgentIterator.new(*args)
+      end
+    end
+  end
+end

data/lib/scraypa/user_agent/user_agent_iterator.rb

@@ -0,0 +1,97 @@
+module Scraypa
+  class UserAgentIterator < UserAgentAbstract
+    attr_reader :current_user_agent
+
+    def initialize *args
+      super(*args)
+      @config = args[0] || {}
+      @change_after_n_requests = @config.fetch(:change_after_n_requests, 0)
+      @list_limit = @config.fetch(:list_limit, 0).to_i
+      @strategy = @config.fetch(:strategy, :roundrobin)
+      @list = limit_list to_array@config.fetch(:list, USER_AGENT_LIST)
+      @reducing_list = @list.clone
+      @current_user_agent = nil
+      @current_user_agent_requests = 0
+    end
+
+    def user_agent
+      get_a_new_user_agent? ? (
+        @current_user_agent_requests = 0
+        select_user_agent_using_strategy
+      ) : (
+        @current_user_agent_requests += 1
+        @current_user_agent
+      )
+    end
+
+    def list
+      @list
+    end
+
+    private
+
+    def to_array variable
+      case variable
+        when Array
+          variable
+        when Hash
+          variable.values
+        else
+          [variable]
+      end
+    end
+
+    def limit_list list
+      @list_limit <= 0 || @list_limit >= list.length ?
+          list :
+          @strategy == :randomize ?
+              limit_list_randomly(list) :
+              list[0..@list_limit-1]
+    end
+
+    def limit_list_randomly list
+      random_list = []
+      loop do
+        sample = list.sample
+        if list.include? sample
+          random_list << sample
+          list.delete(sample)
+        end
+        break if random_list.length >= @list_limit
+      end
+      random_list
+    end
+
+    def get_a_new_user_agent?
+      !@current_user_agent ||
+          @current_user_agent_requests >= @change_after_n_requests
+    end
+
+    def select_user_agent_using_strategy
+      @strategy == :randomize ?
+          random_user_agent_from_list :
+          next_user_agent_from_list
+    end
+
+    def random_user_agent_from_list
+      @current_user_agent_requests += 1
+      @current_user_agent = ensure_a_new_random_user_agent
+    end
+
+    def ensure_a_new_random_user_agent
+      return @list.first if @list.length == 1
+      random_user_agent = nil
+      loop do
+        random_user_agent = @list.sample
+        break unless random_user_agent == @current_user_agent
+      end
+      random_user_agent
+    end
+
+    def next_user_agent_from_list
+      @reducing_list = @list.clone if @reducing_list.empty?
+      @current_user_agent_requests += 1
+      @current_user_agent = @reducing_list.shift
+    end
+  end
+end

data/lib/scraypa/user_agent/user_agent_random.rb

@@ -0,0 +1,66 @@
+require 'useragents'
+
+module Scraypa
+  class UserAgentRandom < UserAgentAbstract
+    attr_reader :current_user_agent
+
+    def initialize *args
+      super(*args)
+      @config = args[0] || {}
+      @change_after_n_requests = @config.fetch(:change_after_n_requests, 0)
+      @list_limit = @config.fetch(:list_limit, 0).to_i
+      random_limited_list if @list_limit > 0
+      @current_user_agent = nil
+      @current_user_agent_requests = 0
+    end
+
+    def user_agent
+      get_a_new_user_agent? ? (
+        @current_user_agent_requests = 0
+        @list ?
+            next_user_agent_from_list :
+            select_user_agent_using_randomizer
+      ) : (
+        @current_user_agent_requests += 1
+        @current_user_agent
+      )
+    end
+
+    private
+
+    def random_limited_list
+      @list = []
+      loop do
+        random_ua = UserAgents.rand()
+        @list << random_ua unless @list.include? random_ua
+        break if @list.length >= @list_limit
+      end
+      @reducing_list = @list.clone
+    end
+
+    def get_a_new_user_agent?
+      !@current_user_agent ||
+          @current_user_agent_requests >= @change_after_n_requests
+    end
+
+    def select_user_agent_using_randomizer
+      @current_user_agent_requests += 1
+      @current_user_agent = ensure_a_new_random_user_agent
+    end
+
+    def ensure_a_new_random_user_agent
+      random_user_agent = nil
+      loop do
+        random_user_agent = UserAgents.rand()
+        break unless random_user_agent == @current_user_agent
+      end
+      random_user_agent
+    end
+
+    def next_user_agent_from_list
+      @reducing_list = @list.clone if @reducing_list.empty?
+      @current_user_agent_requests += 1
+      @current_user_agent = @reducing_list.shift
+    end
+  end
+end

data/lib/scraypa/version.rb

@@ -0,0 +1,3 @@
+module Scraypa
+  VERSION = "0.1.1"
+end

data/lib/scraypa/visit/visit_capabara_headless_chromium.rb

@@ -0,0 +1,120 @@
+module Scraypa
+  include Capybara::DSL
+
+  class VisitCapybaraHeadlessChromium < VisitInterface
+    def initialize params={}
+      super(params)
+      @config = params[:config]
+      @driver_resetter = params[:driver_resetter]
+      @user_agent_retriever = params[:user_agent_retriever]
+      reset_and_setup_driver
+    end
+
+    def execute params={}
+      visit_get_response params
+    end
+
+    private
+
+    def visit_get_response params={}
+      update_user_agent_if_changed if @has_visited
+      @has_visited = true
+      Capybara.visit params[:url]
+      @driver_resetter.reset_if_nth_request if @driver_resetter
+      Capybara.page
+    end
+
+    def update_user_agent_if_changed
+      if @user_agent_retriever
+        new_user_agent = @user_agent_retriever.user_agent
+        update_user_agent_and_setup_driver new_user_agent if
+            @current_user_agent != new_user_agent
+      end
+    end
+
+    def update_user_agent_and_setup_driver new_user_agent
+      @current_user_agent = new_user_agent
+      @user_agents << @current_user_agent unless
+          @user_agents.include? @current_user_agent
+      setup_headless_chromium_driver
+    end
+
+    def reset_and_setup_driver
+      case @config.driver
+        when :headless_chromium
+          reset_headless_chromium_drivers
+          update_user_agent_if_changed
+          setup_headless_chromium_driver
+        when :selenium_chrome_billy
+          setup_billy_driver
+        else
+          raise CapybaraDriverUnsupported,
+                "Currently no support for capybara driver: #{@config.driver}"
+      end
+    end
+
+    def reset_headless_chromium_drivers
+      clear_capybara_session_pool
+      Capybara.drivers.delete_if{|k,v|
+        ![:poltergeist_billy,
+          :selenium_chrome_billy].include?(k)
+      }
+      @current_user_agent = nil
+      @user_agents = []
+    end
+
+    def clear_capybara_session_pool
+      Capybara.reset_sessions!
+      Capybara.send(:session_pool).each do |session_name, session|
+        session.driver.quit if session_name.include?('headless_chromium')
+      end
+      Capybara.send(:session_pool).delete_if{true}
+    end
+
+    def setup_billy_driver
+      Capybara.javascript_driver = @config.driver
+    end
+
+    def setup_headless_chromium_driver
+      driver_name = driver_name_from_config
+      Capybara.register_driver driver_name do |app|
+        Capybara::Selenium::Driver.new(app,
+                                       build_driver_options_from_config)
+      end unless Capybara.drivers.keys.include? driver_name
+      Capybara.default_driver = driver_name
+    end
+
+    def driver_name_from_config
+      (@config.driver.to_s +
+          (@config.tor ? "tor#{@config.tor_options[:tor_port]}" : "") +
+          (@current_user_agent ?
+              "ua#{@user_agents.index(@current_user_agent)}" : "")).to_sym
+    end
+
+    def build_driver_options_from_config
+      driver_options = {browser: @config.headless_chromium[:browser] || :chrome}
+      if @config.headless_chromium[:chromeOptions] || @current_user_agent
+        driver_options[:desired_capabilities] =
+            Selenium::WebDriver::Remote::Capabilities.chrome(
+                :chromeOptions =>
+                    merge_user_agent_with_chrome_options
+            )
+      end
+      driver_options[:args] = @config.headless_chromium[:args] if
+          @config.headless_chromium[:args]
+      driver_options
+    end
+
+    def merge_user_agent_with_chrome_options
+      chrome_options =
+          @config.headless_chromium[:chromeOptions] || {args: []}
+      if @current_user_agent &&
+          (chrome_options[:args] || chrome_options['args'])
+        args_key = chrome_options[:args] ? :args : 'args'
+        chrome_options[args_key].delete_if {|d| d.include?("user-agent=")}
+        chrome_options[args_key] << "--user-agent=#{@current_user_agent}"
+      end
+      chrome_options
+    end
+  end
+end

data/lib/scraypa/visit/visit_capabara_poltergeist.rb

@@ -0,0 +1,76 @@
+require 'capybara'
+require 'capybara/poltergeist'
+require 'phantomjs'
+
+module Scraypa
+  include Capybara::DSL
+
+  class VisitCapybaraPoltergeist < VisitInterface
+    def initialize params={}
+      super(params)
+      @config = params[:config]
+      @tor_proxy = params[:tor_proxy]
+      @driver_resetter = params[:driver_resetter]
+      @user_agent_retriever = params[:user_agent_retriever]
+      setup_driver
+      @current_user_agent = nil
+    end
+
+    def execute params={}
+      @config.tor && @tor_proxy ?
+        visit_get_response_through_tor(params) :
+        visit_get_response(params)
+    end
+
+    private
+
+    def visit_get_response_through_tor params={}
+      @tor_proxy.proxy do
+        return visit_get_response params
+      end
+    end
+
+    def visit_get_response params={}
+      update_user_agent_if_changed
+      Capybara.visit params[:url]
+      @driver_resetter.reset_if_nth_request if @driver_resetter
+      Capybara.page
+    end
+
+    def update_user_agent_if_changed
+      if @user_agent_retriever
+        new_user_agent = @user_agent_retriever.user_agent
+        if @current_user_agent != new_user_agent
+          @current_user_agent = new_user_agent
+          Capybara.page.driver.add_headers(
+              "User-Agent" => @current_user_agent)
+        end
+      end
+    end
+
+    def setup_driver
+      case @config.driver
+        when :poltergeist
+          setup_poltergeist_driver
+        when :poltergeist_billy
+          setup_billy_driver
+        else
+          raise CapybaraDriverUnsupported,
+                "Currently no support for capybara driver: #{@config.driver}"
+      end
+    end
+
+    def setup_poltergeist_driver
+      driver_name = (@config.driver.to_s +
+          (@config.tor ? "tor#{@config.tor_options[:tor_port]}" : '')).to_sym
+      Capybara.default_driver = driver_name
+      Capybara.register_driver driver_name do |app|
+        Capybara::Poltergeist::Driver.new(app, @config.driver_options || {})
+      end
+    end
+
+    def setup_billy_driver
+      Capybara.javascript_driver = @config.driver
+    end
+  end
+end