scraper_clients 9.0.0

data/lib/clients/http_client/response.rb

@@ -0,0 +1,57 @@
+require "http"
+require "nokogiri"
+
+module Clients
+  class HttpClient
+    class Response < SimpleDelegator
+      alias_method :object, :__getobj__
+
+      DEFAULT_ENCODING = Encoding::UTF_8
+
+      def success?
+        object.status.success?
+      end
+
+      def fail?
+        !success?
+      end
+
+      def to_s(force_utf8: false)
+        response = object.to_s
+        return response unless force_utf8
+
+        if object.charset
+          response
+            .encode(DEFAULT_ENCODING)
+            .scrub("_")
+        else
+          response
+            .force_encoding(DEFAULT_ENCODING)
+            .scrub("_")
+        end
+      end
+
+      def to_html(**kargs)
+        Nokogiri::HTML.parse to_s(**kargs)
+      end
+
+      def to_xml(**kargs)
+        Nokogiri::XML.parse to_s(**kargs)
+      end
+
+      def to_json(**kargs)
+        JSON.parse to_s(**kargs), symbolize_names: true
+      end
+
+      def to_io
+        StringIO.new(to_s)
+      end
+
+      def stream(size = HTTP::Connection::BUFFER_SIZE)
+        while (chunk = object.body.readpartial(size))
+          yield chunk
+        end
+      end
+    end
+  end
+end

data/lib/clients/proxy6_client.rb

@@ -0,0 +1,70 @@
+require "http"
+require "clients/errors"
+
+module Clients
+  class Proxy6Client
+    API_URL = "https://proxy6.net/api".freeze
+
+    attr_reader :ip_version
+
+    def initialize(
+      api_url: API_URL,
+      api_key: ENV["PROXY6_KEY"],
+      ip_version: "4"
+    )
+      @api_url = api_url
+      @api_key = api_key
+      @ip_version = ip_version.to_s
+      @proxy = fetch_proxy
+    end
+
+    def host
+      @proxy["host"]
+    end
+
+    def port
+      @proxy["port"].to_i
+    end
+
+    def user
+      @proxy["user"]
+    end
+
+    def password
+      @proxy["pass"]
+    end
+
+    def to_s
+      [host, port, user, password].compact.join(":")
+    end
+
+    def reset!
+      @proxy = fetch_proxy
+    end
+
+    private
+
+    def fetch_proxy
+      response = HTTP.accept(:json).get(api_url, params: { state: "active" })
+      json = JSON.parse(response.to_s) if response.status.success?
+
+      fail_on_invalid_list(response) if !response.status.success? || !json.key?("list")
+
+      proxies = json.fetch("list")
+      fail_on_invalid_list(response) if proxies.is_a?(Array)
+
+      proxies = proxies.values.select { |h| h["version"] == @ip_version }
+      fail_on_invalid_list(response) if proxies.empty?
+
+      proxies.sample
+    end
+
+    def api_url
+      [@api_url, @api_key, "getproxy/"].join("/")
+    end
+
+    def fail_on_invalid_list(response)
+      fail ProxyClientError, "Invalid proxy list: #{response.status} #{response.to_s}"
+    end
+  end
+end

data/lib/clients/proxy_client.rb

@@ -0,0 +1,14 @@
+require "clients/errors"
+require "clients/proxy6_client"
+require "clients/proxy_list_client"
+
+module Clients
+  class ProxyClient
+    def self.from_env
+      case ENV["CLIENTS_PROXY_CLIENT"]
+      when "proxy6" then Proxy6Client.new
+      when "list" then ProxyListClient.new
+      end
+    end
+  end
+end

data/lib/clients/proxy_list_client.rb

@@ -0,0 +1,38 @@
+module Clients
+  class ProxyListClient
+    DEFAULT_LIST_PATH = "/tmp/clients_proxy_list.txt".freeze
+
+    attr_reader :host, :user, :password
+
+    def self.cache_list(list_url, list_path = DEFAULT_LIST_PATH)
+      response = HTTP.get(list_url)
+      fail "Invalid list response: #{response.status}" unless response.status.success?
+      File.open(list_path, "w") { |f| f << response.to_s }
+    end
+
+    def initialize(path = DEFAULT_LIST_PATH)
+      @path = path
+      select_proxy_from_list
+    end
+
+    def port
+      @port.to_i
+    end
+
+    def to_s
+      [host, port, user, password].compact.join(":")
+    end
+
+    def reset!
+      select_proxy_from_list
+    end
+
+    private
+
+    def select_proxy_from_list
+      proxies = File.readlines(@path)
+      address = proxies.sample
+      @host, @port, @user, @password = address.split(":").map(&:strip)
+    end
+  end
+end

data/lib/clients/recaptcha/client.rb

@@ -0,0 +1,48 @@
+require "clients/http_client"
+require "clients/recaptcha/solver"
+
+module Clients
+  module Recaptcha
+    class Client < Clients::HttpClient
+      attr_writer :solver
+
+      def get(url, **options, &block)
+        response = super
+        response = bypass_captcha(url, response) if captcha_protected?(response)
+        response
+      end
+
+      def get_without_bypass(url, **options, &block)
+        request :get, url, **options, &block
+      end
+
+      def solver
+        @solver ||= Solver.new(self)
+      end
+
+      private
+
+      def captcha_protected?(response)
+        response.status == 403 &&
+          response.to_s.include?("g-recaptcha-response")
+      end
+
+      def bypass_captcha(url, response)
+        if has_cookies?
+          reset_cookies
+          response = get_without_bypass(url)
+        end
+
+        fail "captcha with empty cookie" if response.cookies.empty?
+
+        solved_cookies = solver.solve(url, response)
+        store_cookies solved_cookies
+
+        solved_response = get_without_bypass(url, follow_redirects: false)
+        reset_cookies
+
+        solved_response
+      end
+    end
+  end
+end

data/lib/clients/recaptcha/response.rb

@@ -0,0 +1,15 @@
+module Clients
+  module Recaptcha
+    class Response < SimpleDelegator
+      alias_method :object, :__getobj__
+
+      def success?
+        to_s[0..1] == "OK"
+      end
+
+      def data
+        to_s[3..-1]
+      end
+    end
+  end
+end

data/lib/clients/recaptcha/solver.rb

@@ -0,0 +1,115 @@
+require "clients/errors"
+require "clients/recaptcha/response"
+
+module Clients
+  module Recaptcha
+    class Solver
+      INIT_URL = Addressable::URI.parse("http://2captcha.com/in.php").freeze
+      SOLVE_STATUS_URL = Addressable::URI.parse("http://2captcha.com/res.php").freeze
+
+      attr_reader :client, :captcha_key, :sleep_duration
+
+      def initialize(client, captcha_key: ENV["CAPTCHA_SOLVER_KEY"], sleep_duration: 5)
+        @client = client
+        @captcha_key = captcha_key
+        @sleep_duration = sleep_duration
+      end
+
+      def solve(banned_url, response)
+        site_key = find_site_key(response)
+        solved_path = find_solve_url(response)
+
+        id = init_solver banned_url, site_key
+        token = get_solution id
+
+        solved_response = solve_captcha banned_url, solved_path, token
+        solved_response.cookies
+      end
+
+      private
+
+      def init_url(banned_url, site_key)
+        url = INIT_URL.dup
+        url.query_values = {
+          key: captcha_key,
+          method: "userrecaptcha",
+          googlekey: site_key,
+          url: banned_url
+        }
+        url
+      end
+
+      def status_url(id, tries = 0)
+        url = SOLVE_STATUS_URL.dup
+        url.query_values = {
+          key: captcha_key,
+          action: "get",
+          id: id,
+          try: tries
+        }
+        url
+      end
+
+      def solved_url(banned_url, solved_path, token)
+        url = Addressable::URI.parse(banned_url)
+        url.path = solved_path
+        url.query_values = { "g-recaptcha-response": token }
+        url
+      end
+
+      def find_site_key(response)
+        key = response.to_s.match(/data-sitekey=\"(.+?)\"/) { |m| m[1] }
+        fail RecaptchaError, "Empty sitekey in recaptcha form" unless key
+        key
+      end
+
+      def find_solve_url(response)
+        url = response.to_s.match(/action=\"(.+?)\"/) { |m| m[1] }
+        fail RecaptchaError, "Empty action in recaptcha form" unless url
+        url
+      end
+
+      def init_solver(banned_url, site_key)
+        url = init_url banned_url, site_key
+        response = wrap_response client.get_without_bypass(url)
+        fail RecaptchaError, response.to_s unless response.success?
+        response.data
+      end
+
+      def get_solution(id)
+        tries = 0
+        response = nil
+
+        until response
+          fail RecaptchaError, "Solve timeout after 10 tries" if tries > 10
+
+          sleep sleep_duration
+          response = check_status id, tries
+          tries += 1
+        end
+
+        response.data
+      end
+
+      def check_status(id, tries)
+        url = status_url id, tries
+        response = wrap_response client.get_without_bypass(url)
+        response if response.success?
+      end
+
+      def solve_captcha(banned_url, solved_path, token)
+        url = solved_url banned_url, solved_path, token
+        response = client.get_without_bypass(url, follow_redirects: false) do |request|
+          request.headers(referer: banned_url)
+        end
+
+        fail RecaptchaError, "Unable to solve recaptcha" if response.status.redirect?
+        response
+      end
+
+      def wrap_response(response)
+        Recaptcha::Response.new response
+      end
+    end
+  end
+end

data/lib/clients/tor_client.rb

@@ -0,0 +1,146 @@
+require "net/telnet"
+require "ostruct"
+
+# Proxy to Tor
+#
+# From http://martincik.com/?p=402
+module Clients
+  class TorClient
+    DEFAULT_PORT = 9050
+    DEFAULT_CONTROL_PORT = 9051
+    DEFAULT_HTTP_PORT = 8080
+    OK_STATUS = "250 OK\n".freeze
+
+    extend Forwardable
+
+    attr_reader :config, :threshold, :pool_num
+
+    def_delegators :@config,
+      :tor_host, :tor_port
+
+    # rubocop:disable Metrics/MethodLength
+    def initialize(options = {})
+      options = {
+        tor_host:        "localhost",
+        tor_port:        (ENV["TOR_PORT"] || DEFAULT_PORT).to_i,
+        control_port:    (ENV["TOR_CONTROL_PORT"] || DEFAULT_CONTROL_PORT).to_i,
+        host:            "localhost",
+        port:            (ENV["HTTP_TOR_PORT"] || DEFAULT_HTTP_PORT).to_i,
+        circuit_timeout: 10,
+        throttle_by:     10, # .seconds implied
+        pool_num:        nil
+      }.merge(options)
+
+      @pool_num = options.delete(:pool_num)
+      @config = OpenStruct.new options
+
+      setup_pool
+    end
+
+    def host
+      @config[:host]
+    end
+
+    def port
+      @config[:port]
+    end
+
+    def user
+      nil
+    end
+
+    def password
+      nil
+    end
+
+    def to_s
+      "#{host}:#{port}"
+    end
+
+    def switch_identity
+      throttle do
+        client = Net::Telnet.new(
+          "Host"    => config.tor_host,
+          "Port"    => config.control_port,
+          "Timeout" => config.circuit_timeout,
+          "Prompt"  => Regexp.new(OK_STATUS)
+        )
+
+        authenticate client
+        new_route client
+
+        client.close
+      end
+    end
+    alias_method :reset!, :switch_identity
+
+    private
+
+    def throttle
+      scheduled = if check_threshold
+                    update_threshold
+                    :now
+                  else
+                    schedule_switch
+                  end
+
+      if scheduled == :now
+        yield
+      else
+        sleep until_next_switch_time
+
+        if scheduled
+          update_threshold
+          yield
+        end
+      end
+    end
+
+    def authenticate(client)
+      client.cmd("AUTHENTICATE") do |c|
+        fail "cannot authenticate to Tor!" unless c == OK_STATUS
+      end
+    end
+
+    def new_route(client)
+      client.cmd("SIGNAL NEWNYM") do |c|
+        fail "cannot switch Tor to new route!" unless c == OK_STATUS
+      end
+    end
+
+    def setup_pool
+      return unless pool_num
+
+      config.tor_port += 2 * pool_num
+      config.control_port += 2 * pool_num
+      config.port += 2 * pool_num
+    end
+
+    def check_threshold
+      !threshold || (next_switch_time < Time.now)
+    end
+
+    def next_switch_time
+      threshold + config.throttle_by
+    end
+
+    def until_next_switch_time
+      diff = next_switch_time - Time.now
+      diff < 0 ? 0 : diff
+    end
+
+    def schedule_switch
+      return if @scheduled
+      @scheduled = true
+    end
+
+    def update_threshold
+      @threshold = Time.now
+      @scheduled = false
+    end
+
+    def reset_threshold
+      @threshold = nil
+    end
+  end
+end