sunbro 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d1de54ea6b7adab57393a8a220e6ce2715a1875d
-  data.tar.gz: 6d1b4fa82d615a15320b9a2281dba34c44f6b4f3
+  metadata.gz: 0d7f760133a4fab073cd7a7dd5c064ed1744f025
+  data.tar.gz: 68b98b70f0e591ca9056b1db73238ea1a578418c
 SHA512:
-  metadata.gz: d658cb22b7b57abd492ffac8f2b0b4173c6f806ed93a4d28fa699cbce6178bd97fa87af64778e88eb841bd1f59c1efe0c0e3912377046709104ddc85a48a6ba1
-  data.tar.gz: e96e191c25f0145f1e540237fc2c59dc8dc390871c23df01466186d15d539bb1d76ef8becc113bce6726fa627799886718f1da0bb36480925fae92ae36ea312e
+  metadata.gz: fd0844f9a0157b3dfe7f38f460fe44d023985fbb6cd62d99738f6007fef896499f1fc5439881b85bc37e49a45cdb704af176350f766eab7e3d5d0309f7afdf7f
+  data.tar.gz: cef9770fe754b0829452adc5ebb20cc0c5119e819b01a339cb807f3f034e982c78873b9e8414fe480510f99643d7d57945e0a0b5b4198ce5b9c415dc0efed499

data/lib/sunbro.rb

@@ -11,43 +11,10 @@ require 'retryable'
   sunbro/dynamic_http
   sunbro/http
   sunbro/page
-  sunbro/initialize
+  sunbro/connection
 ).each do |f|
   require f
 end
 
 module Sunbro
-  MAX_RETRIES = 5
-
-  def get_page(link, opts={})
-    fetch_with_connection(http, link, opts)
-  end
-
-  def render_page(link, opts={})
-    fetch_with_connection(dhttp, link, opts)
-  end
-
-  def fetch_with_connection(conn, link, opts)
-    page, tries = nil, MAX_RETRIES
-    begin
-      page = conn.fetch_page(link, opts)
-      sleep 1
-    end until page.try(:present?) || (tries -= 1).zero?
-    page.discard_doc! unless page.is_valid?
-    page
-  end
-
-  def http
-    @http ||= HTTP.new
-  end
-
-  def dhttp
-    @dhttp ||= DynamicHTTP.new
-  end
-
-  def close_http_connections
-    @http.close if @http
-    @dhttp.close if @dhttp
-  rescue IOError
-  end
 end

data/lib/sunbro/connection.rb

@@ -1,9 +1,48 @@
-class Connection
-  extend Sunbro
-  attr_reader :http, :dhttp
+module Sunbro
+  class Connection
+    attr_reader :http, :dhttp
 
-  def close
-    close_http_connections
+    MAX_RETRIES = 3
+
+    def fetch_page(link, opts={})
+      conn = opts[:force_format] == (:dhtml || 'dhtml') ? dhttp : http
+      tries = opts[:tries] || MAX_RETRIES
+      sleep_interval = opts[:sleep] || 1
+
+      page = Retryable.retryable(sleep: sleep_interval, tries: tries) do
+        web_retry(opts) do
+          conn.fetch_page(link, opts)
+        end
+      end
+      page.discard_doc! unless page.valid?
+      page
+    end
+
+    def session
+      @dhttp.try(:session)
+    end
+
+    def http
+      @http ||= HTTP.new
+    end
+
+    def dhttp
+      @dhttp ||= DynamicHTTP.new
+    end
+
+    def close
+      @http.try(:close)
+      @dhttp.try(:close)
+    rescue IOError
+    end
+
+    def web_retry(opts)
+      page, tries, sleep_interval = nil, opts[:tries], opts[:sleep]
+      begin
+        page = yield
+        sleep(sleep_interval) unless page.valid?
+      end until page.valid? || (tries -= 1).zero?
+      page
+    end
   end
 end
-

data/lib/sunbro/dynamic_http.rb

@@ -74,8 +74,10 @@ module Sunbro
 
     def get_page(url, opts)
       reset = opts.fetch(:reset) rescue true
+      start = Time.current.to_i
       session.visit(url.to_s)
       page = create_page_from_session(url, session, opts)
+      page.response_time = ((Time.now - start) * 1000).round
       session.reset! if reset
       page
     rescue Capybara::Poltergeist::TimeoutError => e

data/lib/sunbro/http.rb

@@ -13,6 +13,7 @@ module Sunbro
     end
 
     def initialize(opts = {})
+      RestClient.proxy ||= Sunbro::Settings.proxy_url
       @connections = {}
       @opts = opts
     end

data/lib/sunbro/page.rb

@@ -64,7 +64,7 @@ module Sunbro
       end
     end
 
-    def is_valid?
+    def valid?
       (url != "about:blank") && !not_found? && present?
     end
 
@@ -151,7 +151,7 @@ module Sunbro
         href = doc.search('//head/base/@href')
         URI(href.to_s) unless href.nil? rescue nil
       end unless @base
-      
+
       return nil if @base && @base.to_s().empty?
       @base
     end

data/lib/sunbro/version.rb

@@ -1,3 +1,3 @@
 module Sunbro
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,153 +1,153 @@
 --- !ruby/object:Gem::Specification
 name: sunbro
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Jon Stokes
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-25 00:00:00.000000000 Z
+date: 2015-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: nokogiri
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: capybara
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: capybara
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: poltergeist
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: poltergeist
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rest-client
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: activesupport
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: retryable
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: retryable
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
-  name: bundler
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.5'
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: rspec
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
+  name: mocktra
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: mocktra
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Requires phantomjs.
@@ -157,7 +157,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -166,7 +166,6 @@ files:
 - lib/sunbro/connection.rb
 - lib/sunbro/dynamic_http.rb
 - lib/sunbro/http.rb
-- lib/sunbro/initialize.rb
 - lib/sunbro/page.rb
 - lib/sunbro/settings.rb
 - lib/sunbro/version.rb
@@ -179,26 +178,27 @@ homepage: ''
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.1.9
-signing_key:
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
 specification_version: 4
-summary: Some code that I use to crawl the web at scale. Shared in the spirit of jolly cooperation.
+summary: Some code that I use to crawl the web at scale. Shared in the spirit of jolly
+  cooperation.
 test_files:
 - spec/page_spec.rb
 - spec/settings_spec.rb

data/lib/sunbro/initialize.rb

@@ -1,4 +0,0 @@
-if Sunbro::Settings.proxy_url
-  puts "## Setting RestClient proxy to #{Sunbro::Settings.proxy_url}"
-  RestClient.proxy = Sunbro::Settings.proxy_url
-end