sunbro 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 90a10b3e643f709e46b6a0cefe53e09755a333df
-  data.tar.gz: 124e852db53309e2284fb3daf372b61b30af669e
+  metadata.gz: d1de54ea6b7adab57393a8a220e6ce2715a1875d
+  data.tar.gz: 6d1b4fa82d615a15320b9a2281dba34c44f6b4f3
 SHA512:
-  metadata.gz: 5294a0e2819a7ca39c3e59f73952c7b5ae73569a6337bc24128ef77e45d905de18ec2796648e53cfe53d26012a38523b815fe0c37e81a33d33302da591e0dfc5
-  data.tar.gz: f4596d13a476704c094b1661e5c1a7297f0aa487984c7363c1eeb558853f7d30833dd238539546902ae178e6b0ebaa7c9f127400438100c34a639807bb263277
+  metadata.gz: d658cb22b7b57abd492ffac8f2b0b4173c6f806ed93a4d28fa699cbce6178bd97fa87af64778e88eb841bd1f59c1efe0c0e3912377046709104ddc85a48a6ba1
+  data.tar.gz: e96e191c25f0145f1e540237fc2c59dc8dc390871c23df01466186d15d539bb1d76ef8becc113bce6726fa627799886718f1da0bb36480925fae92ae36ea312e

data/lib/sunbro.rb

@@ -1,7 +1,9 @@
 require 'nokogiri'
 require 'capybara/poltergeist'
-require 'net/http/persistent'
+require 'rest-client'
 require 'webrick/cookie'
+require 'active_support/all'
+require 'retryable'
 
 %w(
   sunbro/version
@@ -9,6 +11,7 @@ require 'webrick/cookie'
   sunbro/dynamic_http
   sunbro/http
   sunbro/page
+  sunbro/initialize
 ).each do |f|
   require f
 end

data/lib/sunbro/dynamic_http.rb

@@ -1,22 +1,26 @@
 module Sunbro
   class DynamicHTTP
+
     attr_reader :session
 
     def initialize(opts = {})
       @opts = opts
-      new_session
+      Retryable.retryable { new_session }
     end
 
     def close
       @session.driver.quit
+    rescue IOError
+      nil
     end
 
     def new_session
       Capybara.register_driver :poltergeist do |app|
         Capybara::Poltergeist::Driver.new(
           app,
+          timeout: 10,
           js_errors: false,
-          phantomjs_options: ['--load-images=no', '--ignore-ssl-errors=yes']
+          phantomjs_options: phantomjs_options,
         )
       end
       Capybara.default_driver = :poltergeist
@@ -29,13 +33,27 @@ module Sunbro
       @session
     end
 
+    def phantomjs_options
+      @phantomjs_options ||= begin
+        opts = [ '--load-images=no', '--ignore-ssl-errors=yes' ]
+        if Sunbro::Settings.proxy_host
+          if Sunbro::Settings.proxy_port
+            opts << "--proxy=#{Sunbro::Settings.proxy_host}:#{Sunbro::Settings.proxy_port}"
+          else
+            opts << "--proxy=#{Sunbro::Settings.proxy_host}"
+          end
+        end
+        opts
+      end
+    end
+
     def user_agent
       @opts[:agent] || Settings.phantomjs_user_agent
     end
 
     def restart_session
       close
-      new_session
+      Retryable.retryable { new_session }
     end
 
     #
@@ -46,7 +64,7 @@ module Sunbro
       begin
         tries ||= 5
         get_page(url, opts)
-      rescue Capybara::Poltergeist::DeadClient, Errno::EPIPE, NoMethodError, Capybara::Poltergeist::BrowserError => e
+      rescue IOError, Capybara::Poltergeist::DeadClient, Errno::EPIPE, NoMethodError, Capybara::Poltergeist::BrowserError => e
         restart_session
         retry unless (tries -= 1).zero?
         close

data/lib/sunbro/http.rb

@@ -3,17 +3,23 @@ module Sunbro
     # Maximum number of redirects to follow on each get_response
     REDIRECT_LIMIT = 5
 
+    class RestResponse < Struct.new(:body, :headers, :code, :location)
+      def clean!
+        body.present?
+      rescue ArgumentError
+        body.encode!('UTF-16', 'UTF-8', :invalid => :replace, :replace => '')
+        body.encode!('UTF-8', 'UTF-16')
+      end
+    end
+
     def initialize(opts = {})
       @connections = {}
       @opts = opts
     end
 
     def close
-      @connections.each do |host, ports|
-        ports.each do |port, connection|
-          connection.finish
-        end
-      end
+      # Deprecated with move to RestClient
+      true
     end
 
     #
@@ -44,15 +50,15 @@ module Sunbro
       begin
         url = convert_to_uri(url) unless url.is_a?(URI)
         pages = []
-        get(url, referer) do |response, code, location, redirect_to, response_time|
-          pages << Page.new(location, :body => response.body.dup,
-                                      :code => code,
-                                      :headers => response.to_hash,
-                                      :referer => referer,
-                                      :depth => depth,
-                                      :redirect_to => redirect_to,
+        get(url) do |response, code, location, redirect_to, response_time|
+          pages << Page.new(location, :body          => response.body.dup,
+                                      :code          => code,
+                                      :headers       => response.headers.stringify_keys,
+                                      :referer       => referer,
+                                      :depth         => depth,
+                                      :redirect_to   => redirect_to,
                                       :response_time => response_time,
-                                      :force_format => force_format)
+                                      :force_format  => force_format)
         end
 
         return pages
@@ -124,7 +130,7 @@ module Sunbro
     # Yields the response object, response code, and URI location
     # for each response.
     #
-    def get(url, referer = nil)
+    def get(url)
       limit = redirect_limit
       loc = url
       begin
@@ -132,9 +138,9 @@ module Sunbro
           # request url
           loc = url.merge(loc) if loc.relative?
 
-          response, response_time = get_response(loc, referer)
+          response, response_time = get_response(loc)
           code = Integer(response.code)
-          redirect_to = response.is_a?(Net::HTTPRedirection) ? URI(response['location']).normalize : nil
+          redirect_to = 300.upto(307).include?(response['code']) ? URI(response['location']).normalize : nil
           yield response, code, loc, redirect_to, response_time
           limit -= 1
       end while (loc = redirect_to) && allowed?(redirect_to, url) && limit > 0
@@ -147,19 +153,27 @@ module Sunbro
       full_path = url.query.nil? ? url.path : "#{url.path}?#{url.query}"
 
       opts = {}
-      opts['User-Agent'] = user_agent if user_agent
-      opts['Referer'] = referer.to_s if referer
+      opts[:headers] = {
+          user_agent: user_agent
+      } if user_agent
 
       retries = 0
       begin
         start = Time.now()
-        # format request
-        req = Net::HTTP::Get.new(full_path, opts)
-        # HTTP Basic authentication
-        req.basic_auth url.user, url.password if url.user
-        response = connection(url).request(req)
+        response = RestResponse.new
+
+        # This causes RestClient to skip following the redirect automatically
+        connection(url)[full_path].get(opts) do |res, request, result|
+          response.body     = res.body
+          response.headers  = res.headers
+          response.code     = res.code
+          response.location = res.headers[:location]
+        end
+
         finish = Time.now()
         response_time = ((finish - start) * 1000).round
+        response.clean!
+
         return response, response_time
       rescue Timeout::Error, Net::HTTPBadResponse, EOFError => e
         puts e.inspect if verbose?
@@ -180,16 +194,12 @@ module Sunbro
     end
 
     def refresh_connection(url)
-      http = Net::HTTP.new(url.host, url.port, proxy_host, proxy_port)
-
-      http.read_timeout = read_timeout if !!read_timeout
-
-      if url.scheme == 'https'
-        http.use_ssl = true
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      end
+      @connections[url.host][url.port] = RestClient::Resource.new(
+          "#{url.scheme}://#{url.host}",
+          timeout:    read_timeout || 5,
+          verify_ssl: OpenSSL::SSL::VERIFY_NONE
+      )
 
-      @connections[url.host][url.port] = http.start
     end
 
     def verbose?

data/lib/sunbro/initialize.rb

@@ -0,0 +1,4 @@
+if Sunbro::Settings.proxy_url
+  puts "## Setting RestClient proxy to #{Sunbro::Settings.proxy_url}"
+  RestClient.proxy = Sunbro::Settings.proxy_url
+end

data/lib/sunbro/page.rb

@@ -98,7 +98,8 @@ module Sunbro
     # The content-type returned by the HTTP request for this page
     #
     def content_type
-      headers['content-type'].first
+      return headers['content-type'].first if headers['content-type'].first.present?
+      headers['content_type']
     end
 
     #

data/lib/sunbro/settings.rb

@@ -1,5 +1,3 @@
-require 'hashie'
-
 module Sunbro
   module Settings
 
@@ -8,29 +6,62 @@ module Sunbro
       phantomjs_user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X)",
       page_format:          :auto
     }
+    
+    class SettingsData < Struct.new(:user_agent, :proxy_url, :proxy_host, :proxy_port, :phantomjs_user_agent, :page_format);end
 
     def self.configure
-      $sunbro_configuration ||= Hashie::Mash.new
-      yield $sunbro_configuration
+      @configuration ||= SettingsData.new
+      yield @configuration
+    end
+
+    def self.proxy_url
+      return unless configured?
+      if @configuration.proxy_url
+        @configuration.proxy_url
+      elsif @configuration.proxy_host
+        if @configuration.proxy_port
+          "http://#{@configuration.proxy_host}:#{@configuration.proxy_port}/"
+        else
+          "http://#{@configuration.proxy_host}/"
+        end
+      end
+    end
+
+    def self.proxy_host
+      return unless configured?
+      if @configuration.proxy_url
+        @configuration.proxy_host = URI.parse(proxy_url).host
+      else
+        @configuration.proxy_host
+      end
+    end
+
+    def self.proxy_port
+      return unless configured?
+      if @configuration.proxy_url
+        @configuration.proxy_port = URI.parse(proxy_url).port
+      else
+        @configuration.proxy_port
+      end
     end
 
     def self.user_agent
-      return DEFAULTS[:user_agent] unless configured?
-      $sunbro_configuration.user_agent
+      return DEFAULTS[:user_agent] unless configured? && @configuration.user_agent
+      @configuration.user_agent
     end
 
     def self.phantomjs_user_agent
-      return DEFAULTS[:phantomjs_user_agent] unless configured?
-      $sunbro_configuration.phantomjs_user_agent
+      return DEFAULTS[:phantomjs_user_agent] unless configured? && @configuration.phantomjs_user_agent
+      @configuration.phantomjs_user_agent
     end
 
     def self.page_format
-      return DEFAULTS[:page_format] unless configured?
-      $sunbro_configuration.page_format
+      return DEFAULTS[:page_format] unless configured? && @configuration.page_format
+      @configuration.page_format
     end
 
     def self.configured?
-      !!$sunbro_configuration
+      !!@configuration
     end
   end
 end

data/lib/sunbro/version.rb

@@ -1,3 +1,3 @@
 module Sunbro
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/spec/page_spec.rb

@@ -1,23 +1,33 @@
 require 'spec_helper'
-require 'mocktra'
+require 'open-uri'
 
 describe Sunbro::Page do
 
   before :each do
     @http = Sunbro::HTTP.new(verbose: true)
+    @body = "<html><head><title>Title</title></head><body><p>Body text</p></body></html>"
 
     Mocktra("www.retailer.com") do
       get '/1.html' do
         "<html><head><title>Title</title></head><body><p>Body text</p></body></html>"
       end
+
+      get '/invalid.html' do
+        "<html><head><title>Title</title></head><body><p>Body text</p></body></html>\255".force_encoding('UTF-8')
+      end
     end
   end
 
   describe "#initialize" do
-    it "it scrubs invalid UTF-8 from @body by converting to UTF-16, then back again" do
-      # See http://stackoverflow.com/a/8873922/1169868
-      pending "Example"
-      fail
+    it "it scrubs invalid UTF-8 from @body" do
+      url = "http://www.retailer.com/invalid.html"
+
+      page = @http.fetch_page(url)
+      expect(page.body.present?).to eq(true)
+      expect(page.url.to_s).to eq(url)
+      expect(page.redirect_to).to be_nil
+      expect(page.redirect_from).to be_nil
+
     end
   end
 
@@ -26,6 +36,7 @@ describe Sunbro::Page do
       url = "http://www.retailer.com/1.html"
 
       page = @http.fetch_page(url)
+      expect(page.body).to eq(@body)
       expect(page.url.to_s).to eq(url)
       expect(page.redirect_to).to be_nil
       expect(page.redirect_from).to be_nil

data/spec/settings_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+require 'open-uri'
+
+describe Sunbro::Settings do
+
+  before :each do
+    @proxy_url = 'http://proxy1.oddwonout.com:8888/'
+    Sunbro::Settings.configure {|c| c.proxy_url = @proxy_url}
+  end
+
+  describe '::proxy_url' do
+    it 'returns the proxy url' do
+      expect(Sunbro::Settings.proxy_url).to eq(@proxy_url)
+    end
+  end
+
+  describe '::proxy_host' do
+    it 'returns the proxy host' do
+      expect(Sunbro::Settings.proxy_host).to eq('proxy1.oddwonout.com')
+    end
+  end
+
+  describe '::proxy_port' do
+    it 'returns the proxy port' do
+      expect(Sunbro::Settings.proxy_port).to eq(8888)
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -1,6 +1,5 @@
-require 'rubygems'
-require 'bundler/setup'
-require 'active_support'
+require 'rspec'
+require 'mocktra'
 require 'sunbro'
 
 RSpec.configure do |config|

data/sunbro.gemspec

@@ -21,9 +21,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "nokogiri"
   spec.add_dependency "capybara"
   spec.add_dependency "poltergeist"
-  spec.add_dependency "net-http-persistent"
+  spec.add_dependency "rest-client"
   spec.add_dependency "activesupport"
-  spec.add_dependency "hashie"
+  spec.add_dependency "retryable"
 
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sunbro
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Jon Stokes
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-09 00:00:00.000000000 Z
+date: 2015-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: net-http-persistent
+  name: rest-client
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
@@ -86,7 +86,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: hashie
+  name: retryable
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
@@ -166,10 +166,12 @@ files:
 - lib/sunbro/connection.rb
 - lib/sunbro/dynamic_http.rb
 - lib/sunbro/http.rb
+- lib/sunbro/initialize.rb
 - lib/sunbro/page.rb
 - lib/sunbro/settings.rb
 - lib/sunbro/version.rb
 - spec/page_spec.rb
+- spec/settings_spec.rb
 - spec/spec_helper.rb
 - sunbro.gemspec
 - tasks/rspec.rake
@@ -193,10 +195,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Some code that I use to crawl the web at scale. Shared in the spirit of jolly cooperation.
 test_files:
 - spec/page_spec.rb
+- spec/settings_spec.rb
 - spec/spec_helper.rb