RubyGems - scraper_clients - Versions diffs - 9.0.0 - Mend

scraper_clients 9.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (24) hide show

checksums.yaml +7 -0
data/README.md +26 -0
data/bin/pry +17 -0
data/bin/rspec +17 -0
data/data/user_agents.txt +204 -0
data/lib/clients.rb +24 -0
data/lib/clients/errors.rb +16 -0
data/lib/clients/ftp_client.rb +17 -0
data/lib/clients/http_client.rb +152 -0
data/lib/clients/http_client/response.rb +57 -0
data/lib/clients/proxy6_client.rb +70 -0
data/lib/clients/proxy_client.rb +14 -0
data/lib/clients/proxy_list_client.rb +38 -0
data/lib/clients/recaptcha/client.rb +48 -0
data/lib/clients/recaptcha/response.rb +15 -0
data/lib/clients/recaptcha/solver.rb +115 -0
data/lib/clients/tor_client.rb +146 -0
data/lib/clients/url_decoder.rb +8 -0
data/lib/clients/version.rb +3 -0
data/spec/lib/clients/http_client/response_spec.rb +197 -0
data/spec/lib/clients/http_client_spec.rb +221 -0
data/spec/lib/clients/tor_client_spec.rb +34 -0
data/spec/spec_helper.rb +66 -0
metadata +168 -0

data/lib/clients/url_decoder.rb ADDED

@@ -0,0 +1,8 @@
+module Clients
+  module UrlDecoder
+    def self.decode(url)
+      current_url, url = url, URI.decode(url) until url == current_url
+      url
+    end
+  end
+end

data/lib/clients/version.rb ADDED

@@ -0,0 +1,3 @@
+module Clients
+  VERSION = "9.0.0"
+end

data/spec/lib/clients/http_client/response_spec.rb ADDED

@@ -0,0 +1,197 @@
+require "spec_helper"
+module Clients
+  class HttpClient
+    RSpec.describe Response do
+      let(:headers) { {} }
+      let(:body) { "BODY" }
+      let(:status) { 200 }
+      let(:response) {
+        HTTP::Response.new(
+          status: status,
+          version: "1.1",
+          headers: headers,
+          body: body
+        )
+      }
+      subject { described_class.new(response) }
+      describe "#success?" do
+        context "when response has succeeded" do
+          let(:status) { 200 }
+          it "returns true" do
+            is_expected.to be_success
+          end
+        end
+        context "when response has failed" do
+          let(:status) { 502 }
+          it "returns false" do
+            is_expected.not_to be_success
+          end
+        end
+      end
+      describe "#fail?" do
+        context "when response has failed" do
+          let(:status) { 400 }
+          it "returns true" do
+            is_expected.to be_fail
+          end
+        end
+        context "when response has succeeded" do
+          let(:status) { 201 }
+          it "returns false" do
+            is_expected.not_to be_fail
+          end
+        end
+      end
+      describe "#to_s" do
+        it "returns response body" do
+          expect(subject.to_s).to eq("BODY")
+        end
+        context "when force_utf8 hasn't been provided" do
+          let(:body) { "\x89PNG\r\n\x1A\n\x00\x00\x00" }
+          it "sets force_utf8 to FALSE by default" do
+            expect(subject.to_s).to eq(body)
+          end
+        end
+        context "when force_utf8 is FALSE" do
+          shared_examples "unmodified body" do
+            it "returns unmodified response body" do
+              expect(subject.to_s(force_utf8: false)).to eq(body)
+            end
+          end
+          context "when response doesn't have valid charset" do
+            let(:body) { "Correct Ответ".force_encoding Encoding::CP1251 }
+            include_examples "unmodified body"
+          end
+          context "when response is binary" do
+            let(:body) { "\x89PNG\r\n\x1A\n\x00\x00\x00" }
+            include_examples "unmodified body"
+          end
+        end
+        context "when force_utf8 is TRUE" do
+          context "when response doesn't have valid charset" do
+            let(:body) { "Correct Ответ".force_encoding Encoding::CP1251 }
+            it "returns response body in UTF-8 encoding" do
+              response = subject.to_s force_utf8: true
+              expect(response.encoding).to eq(Encoding::UTF_8)
+              expect(response).to eq("Correct Ответ")
+            end
+          end
+          context "when response have valid charset - windows-1251" do
+            let(:headers) {
+              {
+                "Content-Type" => "text/html; charset=windows-1251"
+              }
+            }
+            let(:body) { "Correct Ответ".encode Encoding::CP1251 }
+            it "returns response body in UTF-8 encoding" do
+              response = subject.to_s force_utf8: true
+              expect(response.encoding).to eq(Encoding::UTF_8)
+              expect(response).to eq("Correct Ответ")
+            end
+          end
+        end
+      end
+      describe "#to_html" do
+        it "returns parsed response body" do
+          html = subject.to_html
+          expect(html).to be_an_instance_of(Nokogiri::HTML::Document)
+          expect(html.to_s).to include("<body><p>BODY</p></body>")
+        end
+        context "when force_utf8 is TRUE" do
+          let(:body) { "Correct Ответ".force_encoding Encoding::CP1251 }
+          it "returns parsed response body in valid UTF_8 encodin" do
+            html = subject.to_html(force_utf8: true)
+            expect(html).to be_an_instance_of(Nokogiri::HTML::Document)
+            expect(html.to_s).to include("<body><p>Correct Ответ</p></body>")
+          end
+        end
+      end
+      describe "#to_xml" do
+        let(:body) { "<node><![CDATA[ Brazil ]]></node>" }
+        it "returns parsed response body" do
+          xml = subject.to_xml
+          expect(xml).to be_an_instance_of(Nokogiri::XML::Document)
+          expect(xml.to_s).to include("<node><![CDATA[ Brazil ]]></node>")
+        end
+        context "when force_utf8 is TRUE", skip: true do
+          let(:body) { "<node><![CDATA[ Бразилия ]]></node>".force_encoding Encoding::CP1251 }
+          it "returns parsed response body in valid UTF_8 encodin" do
+            xml = subject.to_xml(force_utf8: true)
+            expect(xml).to be_an_instance_of(Nokogiri::XML::Document)
+            expect(xml.to_s).to include("<node><![CDATA[ Бразилия ]]></node>")
+          end
+        end
+      end
+      describe "#to_json" do
+        let(:body) { "[{\"brand\":\"Фирма ZANUSSI\",\"product_code\":\"91460370200\"}]" }
+        let(:parsed_body) { [{brand: "Фирма ZANUSSI", product_code: "91460370200"}] }
+        it "returns parsed json body" do
+          expect(subject.to_json).to eq parsed_body
+        end
+        context "when force_utf8 is TRUE" do
+          let(:body) { super().force_encoding Encoding::CP1251 }
+          it "returns parsed json body" do
+            expect(subject.to_json(force_utf8: true)).to eq parsed_body
+          end
+        end
+      end
+      describe "#to_io" do
+        let(:body) { "IO BODY" }
+        it "returns response as StringIO" do
+          io = subject.to_io
+          expect(io).to be_an_instance_of(StringIO)
+          expect(io.read).to eq("IO BODY")
+        end
+      end
+      describe "#stream" do
+        let(:url) { "http://example.com" }
+        let(:response) { Clients::HttpClient.new.get(url) }
+        before do
+          stub_request(:get, url).and_return(body: body)
+        end
+        it "streams response body" do
+          expect { |b| subject.stream(1, &b) }.to yield_successive_args("B", "O", "D", "Y")
+        end
+        context "buffer size is not specified" do
+          it "streams response body" do
+            expect { |b| subject.stream(&b) }.to yield_successive_args("BODY")
+          end
+        end
+      end
+    end
+  end
+end

data/spec/lib/clients/http_client_spec.rb ADDED

@@ -0,0 +1,221 @@
+require "spec_helper"
+module Clients
+  RSpec.describe HttpClient do
+    subject { described_class.new }
+    describe "#get" do
+      let(:url) { "http://ya.ru/index.html" }
+      let(:response) { subject.get(url) }
+      before do
+        stub_request(:get, url).and_return(
+          status: 202,
+          body: "RESPONSE",
+          headers: {
+            content_type: "image/png; charset=UTF-8"
+          }
+        )
+      end
+      it "makes a request to given url" do
+        response
+        expect(WebMock).to have_requested(:get, url)
+      end
+      shared_examples "follow_redirects" do
+        let(:url) { "http://ya.ru/redirect" }
+        let(:redirect_to_url) { "http://ya.ru/final" }
+        before do
+          stub_request(:get, url).and_return(
+            status: 301,
+            headers: { location: redirect_to_url }
+          )
+          stub_request(:get, redirect_to_url).and_return(
+            status: 200,
+            body: "RESPONSE"
+          )
+        end
+        it "follows redirects" do
+          expect(response.to_s).to eq("RESPONSE")
+          expect(WebMock).to have_requested(:get, url).once
+          expect(WebMock).to have_requested(:get, redirect_to_url).once
+        end
+      end
+      context "when follow_redirects is true" do
+        let(:response) { subject.get(url, follow_redirects: true) }
+        include_examples "follow_redirects"
+      end
+      context "when follow_redirects is nil" do
+        let(:response) { subject.get(url) }
+        include_examples "follow_redirects"
+      end
+      context "when follow_redirects is false" do
+        let(:url) { "http://ya.ru/redirect" }
+        let(:redirect_to_url) { "http://ya.ru/final" }
+        let(:response) { subject.get(url, follow_redirects: false) }
+        before do
+          stub_request(:get, url).and_return(
+            status: 301,
+            headers: { location: redirect_to_url }
+          )
+        end
+        it "DOESN'T follow redirects" do
+          expect(response.status).to eq(301)
+          expect(response.headers["Location"]).to eq(redirect_to_url)
+          expect(WebMock).to have_requested(:get, url).once
+          expect(WebMock).not_to have_requested(:get, redirect_to_url)
+        end
+      end
+      it "modifies request from the block" do
+        subject.get(url) do |request|
+          request.headers(cookie: "was_here=1;")
+        end
+        expect(WebMock).to have_requested(:get, url).with(headers: { "Cookie" => "was_here=1;" }).once
+      end
+      it "returns wrapped response" do
+        expect(response).to be_an_instance_of(HttpClient::Response)
+        expect(response.to_s).to eq("RESPONSE")
+        expect(response.code).to eq(202)
+        expect(response.mime_type).to eq("image/png")
+      end
+    end
+    describe "#head" do
+      let(:url) { "http://ya.ru/index.html" }
+      let(:response) { subject.head(url) }
+      before do
+        stub_request(:head, url).and_return(status: 202)
+      end
+      it "makes a request to given url" do
+        response
+        expect(WebMock).to have_requested(:head, url)
+      end
+      it "returns wrapped response" do
+        expect(response).to be_an_instance_of(HttpClient::Response)
+        expect(response.code).to eq(202)
+        expect(response.uri.to_s).to eq(url)
+      end
+    end
+    describe "#proxy?" do
+      context "when proxy has been used" do
+        let(:proxy) { instance_spy("Clients::TorProxy") }
+        subject { described_class.new proxy: proxy }
+        it "returns true" do
+          expect(subject.proxy?).to eq(true)
+        end
+      end
+      context "when proxy has NOT been used" do
+        it "returns false" do
+          expect(subject.proxy?).to eq(false)
+        end
+      end
+    end
+    describe "#has_cookies?" do
+      context "when client has cookies" do
+        let(:cookie) { HTTP::Cookie.new("group", "admin", domain: "example.com", path: "/") }
+        it "returns true" do
+          subject.cookies << cookie
+          expect(subject).to have_cookies
+        end
+      end
+      context "when proxy has NOT been used" do
+        it "returns false" do
+          expect(subject).not_to have_cookies
+        end
+      end
+    end
+    describe "#store_cookies" do
+      let(:old_cookie) { HTTP::Cookie.new("group", "admin", domain: "example.com", path: "/") }
+      let(:new_cookie) { HTTP::Cookie.new("uid", "u12345", domain: "ya.ru", path: "/admin") }
+      let(:cookies) { HTTP::CookieJar.new }
+      before do
+        subject.cookies << old_cookie
+        cookies << new_cookie
+      end
+      it "adds given cookies from the response" do
+        subject.store_cookies cookies
+        expect(subject.cookies.to_a).to contain_exactly(old_cookie, new_cookie)
+      end
+      it "sents new and old cookies with the new request" do
+        url = "https://placeholder.com"
+        stub_request(:get, url).and_return(status: 200)
+        subject.store_cookies cookies
+        subject.get(url)
+        expect(WebMock).to have_requested(:get, url)
+          .with(headers: { "Cookie" => "group=admin; uid=u12345" })
+          .once
+      end
+    end
+    describe "#reset_cookies" do
+      let(:cookie) { HTTP::Cookie.new("group", "admin", domain: "example.com", path: "/") }
+      it "reset client cookies" do
+        subject.cookies << cookie
+        subject.reset_cookies
+        expect(subject.cookies).to be_empty
+      end
+    end
+    describe "#reset_user_agent" do
+      it "reset client user agent" do
+        subject.user_agent
+        expect(subject.user_agent).not_to be_empty
+        # Need to stub sample, because it's not deterministic
+        allow(subject).to receive(:sample_user_agent).and_return("UA")
+        subject.reset_user_agent
+        expect(subject.user_agent).to eq("UA")
+      end
+    end
+    describe "#reset_proxy" do
+      context "when proxy has been used" do
+        let(:proxy) { instance_spy("Clients::TorProxy") }
+        subject { described_class.new proxy: proxy }
+        it "calls reset on proxy" do
+          subject.reset_proxy
+          expect(proxy).to have_received(:reset!).once
+        end
+      end
+      context "when proxy has NOT been used" do
+        subject { described_class.new }
+        it "does nothing" do
+          expect { subject.reset_proxy }.not_to raise_error
+        end
+      end
+    end
+  end
+end

data/spec/lib/clients/tor_client_spec.rb ADDED

@@ -0,0 +1,34 @@
+require "spec_helper"
+module Clients
+  RSpec.describe TorClient do
+    let(:ok_prompt) { "250 OK\n" }
+    let(:new_route_signal) { "SIGNAL NEWNYM" }
+    let(:localhost) { double("telnet") }
+    subject { described_class.new }
+    describe "#switch_identity" do
+      before do
+        allow(Net::Telnet).to receive(:new).and_return localhost
+        allow(localhost).to receive("cmd").and_return(ok_prompt)
+        allow(localhost).to receive("close")
+        allow(subject).to receive(:sleep).and_return(0)
+      end
+      it "throttles tor switch route command by 10 seconds", skip: true do
+        time = Time.now
+        Timecop.freeze(time)      { subject.switch_identity }
+        Timecop.freeze(time + 2)  { subject.switch_identity }
+        Timecop.freeze(time + 3)  { subject.switch_identity }
+        Timecop.freeze(time + 5)  { subject.switch_identity }
+        Timecop.freeze(time + 11) { subject.switch_identity }
+        Timecop.freeze(time + 15) { subject.switch_identity }
+        expect(subject).to have_received(:sleep).exactly(4)
+        expect(localhost).to have_received("cmd").with(new_route_signal).exactly(4)
+      end
+    end
+  end
+end