dubdubdub 0.2.7 → 0.3.0

data/Gemfile

@@ -1,6 +1,6 @@
 source "http://rubygems.org"
 
-gem 'rest-client'
+gem 'rest-client', git: 'https://github.com/daveola/rest-client'
 gem 'nokogiri'
 gem 'mechanize'
 

data/Gemfile.lock

@@ -1,3 +1,11 @@
+GIT
+  remote: https://github.com/daveola/rest-client
+  revision: 9bbe538aa2003e172f818eeff5c2e2e6828a8453
+  specs:
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
+      netrc
+
 GEM
   remote: http://rubygems.org/
   specs:
@@ -25,6 +33,7 @@ GEM
     mime-types (1.19)
     net-http-digest_auth (1.2.1)
     net-http-persistent (2.8)
+    netrc (0.7.7)
     nokogiri (1.5.5)
     ntlm-http (0.1.1)
     pry (0.9.10)
@@ -34,8 +43,6 @@ GEM
     rake (10.0.2)
     rdoc (3.12)
       json (~> 1.4)
-    rest-client (1.6.7)
-      mime-types (>= 1.16)
     rspec (2.8.0)
       rspec-core (~> 2.8.0)
       rspec-expectations (~> 2.8.0)
@@ -61,6 +68,6 @@ DEPENDENCIES
   mechanize
   nokogiri
   pry
-  rest-client
+  rest-client!
   rspec (~> 2.8.0)
   vcr (~> 2.3.0)

data/dubdubdub.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = "dubdubdub"
-  s.version = "0.2.7"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["James Hu"]
@@ -33,17 +33,17 @@ Gem::Specification.new do |s|
     "spec/support/vcr.rb",
     "spec/vcr/browse/basic.yml",
     "spec/vcr/crawl/basic.yml",
-    "spec/vcr/follow_url/alias_link.yml",
-    "spec/vcr/follow_url/base.yml",
-    "spec/vcr/follow_url/block_base_url.yml",
-    "spec/vcr/follow_url/eoferror.yml",
-    "spec/vcr/follow_url/https.yml",
-    "spec/vcr/follow_url/pass_block.yml",
-    "spec/vcr/follow_url/pass_block_iteration.yml",
-    "spec/vcr/follow_url/proxied.yml",
-    "spec/vcr/follow_url/proxy.yml",
-    "spec/vcr/follow_url/proxy_forbidden.yml",
-    "spec/vcr/follow_url/relative_redirects.yml",
+    "spec/vcr/follow/alias_link.yml",
+    "spec/vcr/follow/all_the_way.yml",
+    "spec/vcr/follow/base.yml",
+    "spec/vcr/follow/eoferror.yml",
+    "spec/vcr/follow/https.yml",
+    "spec/vcr/follow/pass_block.yml",
+    "spec/vcr/follow/pass_block_iteration.yml",
+    "spec/vcr/follow/proxy.yml",
+    "spec/vcr/follow/proxy_forbidden.yml",
+    "spec/vcr/follow/relative_redirects.yml",
+    "spec/vcr/follow/uri_error.yml",
     "spec/vcr/get/basic.yml",
     "spec/vcr/get/params.yml",
     "spec/vcr/get/proxy.yml"

data/lib/dubdubdub.rb

@@ -1,6 +1,6 @@
 class DubDubDub
   # Version
-  VERSION = "0.2.7"
+  VERSION = "0.3.0"
 
   attr_accessor :client
 

data/lib/dubdubdub/client.rb

@@ -19,7 +19,7 @@ class DubDubDub::Client
         unless DubDubDub.configuration.ignore_proxy?
           proxy = DubDubDub.configuration.proxy
 
-          raise DubDubDub::Exception, "No proxy has been configured or provided!" if proxy.nil?
+          raise ArgumentError, "No proxy has been configured or provided!" if proxy.nil?
 
           self.proxy = proxy
         end
@@ -43,36 +43,19 @@ class DubDubDub::Client
   end
 
   def proxy
-    "#{proxy_host}:#{proxy_port}"
+    "#{proxy_host}:#{proxy_port}" if proxy_host and proxy_port
   end
 
   def proxy?
-    return false if DubDubDub.configuration.ignore_proxy
+    return false if DubDubDub.configuration.ignore_proxy?
 
     !!proxy
   end
 
-  # Returns a Net::HTTP object
-  def net_http(uri)
-    raise ArgumentError, "A URI must be provided!" unless uri.kind_of? URI::Generic
-
-    net_http_class = if proxy?
-      Net::HTTP.Proxy(proxy_host, proxy_port, proxy_user, proxy_password)
-    else
-      Net::HTTP
-    end
-
-    http = net_http_class.new(uri.host, uri.port)
-    http.verify_mode = OpenSSL::SSL::VERIFY_NONE  # ssl certificate doesn't need to be verified, otherwise a OpenSSL::SSL::SSLError might get thrown
-    http.use_ssl = true if uri.scheme == "https"
-
-    http
-  end
-
   # Returns a RestClient::Resource
   def rest_client_resource(url)
     options = {}
-    options[:proxy] = proxy if proxy?
+    options[:proxy] = "http://#{proxy}" if proxy?
 
     RestClient::Resource.new(url, options)
   end
@@ -109,93 +92,34 @@ class DubDubDub::Client
 
   # Helper method to browse by using a GET request via Mechanize
   def browse(url, *args)
-    mechanize.get(url, *args)
-  end
-
-  # Follow a url to the end until it can no longer go any further
-  # Even if it times out, it will return the url that it times out on!
-  def follow_url(url, options = {}, &block)
-    default_options = { limit: 20, attempts: 5, timeout: 5 }
-    options = default_options.merge(options)
-
-    at_base = false
-    previous_uri = nil  # Keep track of previous uri for relative path redirects
-    response = nil
-    user_agents = [
-      'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.79 Safari/535.11',
-      ''
-    ]
-    urls = []   # the url history
-
-    raise ArgumentError if options[:until] and !options[:until].is_a?(Proc)
-
-    # before we begin, let's yield the initial url if a block was given
-    yield(url) if block_given?
-
-    options[:limit].downto(1).each do |i|
-      begin
-        at_base = true if options[:until] and options[:until].call(url)
-
-        uri = URI.parse(url)
-        net_http = net_http(uri)
-        at_base = true unless uri.respond_to?(:request_uri)   # make sure its a proper url
-
-        unless at_base
-          request = Net::HTTP::Get.new(uri.request_uri)
-          request_attempts = 0
-
-          # we make a certain amount of attempts in case we timeout
-          while request_attempts < options[:attempts]
-            begin
-              request_attempts += 1
-
-              # Don't let the request take too long
-              response = Timeout::timeout(options[:timeout]) do
-                net_http.request(request)
-              end
-
-              break   # if it reaches this, that means the request was successful do break out!
-            # If any of these exceptions are thrown, it has timed out, so keep trying depending on how many attempts we have
-            rescue Timeout::Error, Errno::ETIMEDOUT, Errno::EHOSTUNREACH
-              # do another attempt if we are allowed one, or stop
-              at_base = true and break if request_attempts == options[:attempts]
-            rescue SocketError  # doesn't exist
-              at_base = true and break
-            end
-          end
-
-          case response
-          when Net::HTTPSuccess then at_base = true
-          when Net::HTTPRedirection then url = response['location']
-          when Net::HTTPForbidden then raise DubDubDub::Forbidden
-          # Couldn't resolve, just return url
-          else at_base = true
-          end if response
-        end
-
-      # If any of these exceptions get thrown, return the current url
-      rescue SocketError, EOFError
-        at_base = true
-      rescue URI::InvalidURIError
-        return url  # Just return it
+    handle_net_http_exceptions do
+      handle_mechanize_exceptions do
+        mechanize.get(url, *args)
       end
-
-      urls << url
-
-      break if at_base
-
-      previous_uri = uri   # Keep track of previous uri
-      yield(url) if block_given?
     end
+  end
 
-    end_uri = URI.parse(url)
+  # Follow a URL to the end
+  def follow(url)
+    browse(url).uri.to_s
+  end
 
-    # If there is no host, it's due to a relative 301 redirect. Use previous uri's host, port, etc
-    if !end_uri.host and previous_uri
-      end_uri = previous_uri
-      end_uri.path = url
+  private
+  def handle_net_http_exceptions(&block)
+    begin
+      yield
+    rescue Timeout::Error, Errno::ETIMEDOUT, Errno::EHOSTUNREACH
+      raise DubDubDub::ResponseError.new(e, 408)  # Timeout
+    rescue SocketError, EOFError => e
+      raise DubDubDub::ResponseError.new(e, 404)  # Not found
     end
+  end
 
-    end_uri.to_s
+  def handle_mechanize_exceptions(&block)
+    begin
+      yield
+    rescue Mechanize::ResponseCodeError => e
+      raise DubDubDub::ResponseError.new(e, e.response_code)
+    end
   end
 end

data/lib/dubdubdub/exceptions.rb

@@ -1,2 +1,88 @@
-class DubDubDub::Exception < RuntimeError; end
-class DubDubDub::Forbidden < DubDubDub::Exception; end
+class DubDubDub::Error < RuntimeError; end
+
+# URL not properly formatted
+class DubDubDub::URLFormatError < DubDubDub::Error; end
+
+# There was an unhandled response. Contains
+# a reference to the originating error.
+class DubDubDub::ResponseError < DubDubDub::Error
+  attr_reader :error
+  attr_reader :code
+
+  # Pulled from rest-client, thanks!
+  STATUSES = {
+    100 => 'Continue',
+    101 => 'Switching Protocols',
+    102 => 'Processing', #WebDAV
+
+    200 => 'OK',
+    201 => 'Created',
+    202 => 'Accepted',
+    203 => 'Non-Authoritative Information', # http/1.1
+    204 => 'No Content',
+    205 => 'Reset Content',
+    206 => 'Partial Content',
+    207 => 'Multi-Status', #WebDAV
+
+    300 => 'Multiple Choices',
+    301 => 'Moved Permanently',
+    302 => 'Found',
+    303 => 'See Other', # http/1.1
+    304 => 'Not Modified',
+    305 => 'Use Proxy', # http/1.1
+    306 => 'Switch Proxy', # no longer used
+    307 => 'Temporary Redirect', # http/1.1
+
+    400 => 'Bad Request',
+    401 => 'Unauthorized',
+    402 => 'Payment Required',
+    403 => 'Forbidden',
+    404 => 'Resource Not Found',
+    405 => 'Method Not Allowed',
+    406 => 'Not Acceptable',
+    407 => 'Proxy Authentication Required',
+    408 => 'Request Timeout',
+    409 => 'Conflict',
+    410 => 'Gone',
+    411 => 'Length Required',
+    412 => 'Precondition Failed',
+    413 => 'Request Entity Too Large',
+    414 => 'Request-URI Too Long',
+    415 => 'Unsupported Media Type',
+    416 => 'Requested Range Not Satisfiable',
+    417 => 'Expectation Failed',
+    418 => 'I\'m A Teapot',
+    421 => 'Too Many Connections From This IP',
+    422 => 'Unprocessable Entity', #WebDAV
+    423 => 'Locked', #WebDAV
+    424 => 'Failed Dependency', #WebDAV
+    425 => 'Unordered Collection', #WebDAV
+    426 => 'Upgrade Required',
+    449 => 'Retry With', #Microsoft
+    450 => 'Blocked By Windows Parental Controls', #Microsoft
+
+    500 => 'Internal Server Error',
+    501 => 'Not Implemented',
+    502 => 'Bad Gateway',
+    503 => 'Service Unavailable',
+    504 => 'Gateway Timeout',
+    505 => 'HTTP Version Not Supported',
+    506 => 'Variant Also Negotiates',
+    507 => 'Insufficient Storage', #WebDAV
+    509 => 'Bandwidth Limit Exceeded', #Apache
+    510 => 'Not Extended'
+  }
+
+  def initialize(error, code)
+    @error = error
+    @code = code.to_i
+  end
+
+  def message
+    STATUSES[code.to_i]
+  end
+
+  def to_s
+    "#{code}: #{message} => #{error.class.name}: #{error.message}"
+  end
+end

data/spec/dubdubdub_spec.rb

@@ -57,7 +57,7 @@ describe DubDubDub do
         config.proxy = nil
       end
 
-      lambda { DubDubDub.new(proxy: true) }.should raise_error(DubDubDub::Exception)
+      lambda { DubDubDub.new(proxy: true) }.should raise_error(ArgumentError)
     end
 
     it "doesn't raise an error if configured to ignore proxies and we have specified to use a global proxy that hasn't been set" do
@@ -66,7 +66,7 @@ describe DubDubDub do
         config.proxy = nil
       end
 
-      lambda { DubDubDub.new(proxy: true) }.should_not raise_error(DubDubDub::Exception)
+      lambda { DubDubDub.new(proxy: true) }.should_not raise_error(ArgumentError)
     end
 
     it "does not pass the method to client if that method doesn't exist within the client" do
@@ -168,8 +168,11 @@ describe DubDubDub do
     end
 
     it "works with a proxy", vcr: { cassette_name: "get/proxy", record: :once } do
-      www.proxy = "203.131.212.166"
-      response = www.get "http://www.google.com"
+      www.proxy = "173.234.181.64:8800"
+      response = www.get "http://www.whatismyipaddress.com"
+      html = Nokogiri::HTML(response)
+
+      html.css('.ip').text.strip.should == "173.234.181.64"
     end
   end
 
@@ -187,78 +190,62 @@ describe DubDubDub do
     end
   end
 
-  describe '#follow_url' do
-    it "follows url to the end", vcr: { cassette_name: "follow_url/base", record: :once } do
-      www.follow_url("http://say.ly/TCc1CEp").should == "http://www.whosay.com/TomHanks/photos/148406"
-      www.follow_url("http://t.co//qbJx26r").should == "http://twitter.com/twitter/status/76360760606986241/photo/1"
-      www.follow_url("http://mypict.me/mMgLU").should == "http://mypict.me/mobile.php?id=336583610"
+  describe '#follow' do
+    it "follows url to the end", vcr: { cassette_name: "follow/base", record: :once } do
+      www.follow("http://say.ly/TCc1CEp").should == "http://www.whosay.com/TomHanks/photos/148406"
+      www.follow("http://t.co//qbJx26r").should == "http://twitter.com/twitter/status/76360760606986241/photo/1"
+      www.follow("http://mypict.me/mMgLU").should == "http://mypict.me/mobile.php?id=336583610"
     end
 
-    it "returns the base url if it meets a passed in block", vcr: { cassette_name: "follow_url/block_base_url", record: :once } do
-      www.follow_url("http://ow.ly/9Rp7p", until: lambda { |url| url =~ /ow\.ly/ }).should == "http://ow.ly/9Rp7p"
-      www.follow_url("http://ow.ly/9Rp7p", until: lambda { |url| url =~ /bit\.ly/ }).should == "http://bit.ly/GMx5lu"
-      www.follow_url("http://ow.ly/9Rp7p", until: lambda { |url| url =~ /bit\.lyyy/ }).should == "http://instagram.com/p/IbhSB6EKRQ/"
+    it "handles invalid uris", vcr: { cassette_name: "follow/invalid_uris", record: :once } do
+      lambda { www.follow("http://rank.1new.biz/sharp-紙パック式クリーナー-床用吸い込み口タイプ-オ/") }.should_not raise_error(DubDubDub::URLFormatError)
     end
 
-    it "can pass in a block to get the url every step of the way", vcr: { cassette_name: "follow_url/pass_block_iteration", record: :once } do
-      urls = []
-
-      www.follow_url("http://ow.ly/9Rp7p") do |url|
-        urls << url
-      end
-
-      urls.first.should == "http://ow.ly/9Rp7p"   # first url should be the initial one
-      urls.count.should == 4
+    it "handles https", vcr: { cassette_name: "follow/https", record: :once } do
+      lambda { www.follow("https://www.youtube.com/watch?v=DM58Zdk7el0&feature=youtube_gdata_player") }.should_not raise_error(EOFError)
     end
 
-    it "can pass in a block with the last url being the base url", vcr: { cassette_name: "follow_url/pass_block", record: :once } do
-      urls = []
+    it "raises an exception if doesn't exist", vcr: { cassette_name: "follow/doesnt_exist", record: :once } do
+      lambda { www.follow("http://cnnsadasdasdasdasdasd.com/asd") }.should raise_error(DubDubDub::ResponseError)
 
-      www.follow_url("http://twitpic.com/92a2p5") do |url|
-        urls << url
+      begin
+        www.follow("http://cnnsadasdasdasdasdasd.com/asd")
+      rescue DubDubDub::ResponseError => e
+        e.code.should == 404
+        e.error.should_not be_nil
+        e.message.should_not be_nil
       end
-
-      urls.count.should == 1
-      urls.last.should == "http://twitpic.com/92a2p5"
-    end
-
-    it "handles invalid uris", vcr: { cassette_name: "follow_url/invalid_uris", record: :once } do
-      lambda { www.follow_url("http://rank.1new.biz/sharp-紙パック式クリーナー-床用吸い込み口タイプ-オ/") }.should_not raise_error(URI::InvalidURIError)
-      www.follow_url("http://rank.1new.biz/sharp-紙パック式クリーナー-床用吸い込み口タイプ-オ/").should == "http://rank.1new.biz/sharp-紙パック式クリーナー-床用吸い込み口タイプ-オ/"
     end
 
-    it "handles https", vcr: { cassette_name: "follow_url/https", record: :once } do
-      lambda { www.follow_url("https://www.youtube.com/watch?v=DM58Zdk7el0&feature=youtube_gdata_player") }.should_not raise_error(EOFError)
+    it "returns actual asset link for an alias link", vcr: { cassette_name: "follow/alias_link", record: :once } do
+      www.follow("http://yfrog.us/evlb0z:medium").should == "http://img535.imageshack.us/img535/9845/lb0.mp4"
     end
 
-    it "returns the same url if the name or service doesn't exist", vcr: { cassette_name: "follow_url/doesnt_exist", record: :once } do
-      www.follow_url("http://cnnsadasdasdasdasdasd.com/asd").should == "http://cnnsadasdasdasdasdasd.com/asd"
+    it "does not raise a EOFError", vcr: { cassette_name: "follow/eoferror", record: :once } do
+      lambda { www.follow("http://www.soulpancake.com/post/1607/whats-your-beautiful-mess.html") }.should_not raise_error
     end
 
-    it "returns actual asset link for an alias link", vcr: { cassette_name: "follow_url/alias_link", record: :once } do
-      www.follow_url("http://yfrog.us/evlb0z:medium").should == "http://img535.imageshack.us/img535/9845/lb0.mp4"
+    it 'works with a proxy', vcr: { cassette_name: "follow/proxy", record: :once } do
+      www.proxy = "198.154.114.100:8080"
+      www.follow("http://yfrog.us/evlb0z:medium").should == "http://img535.imageshack.us/img535/9845/lb0.mp4"
     end
 
-    it "does not raise a EOFError", vcr: { cassette_name: "follow_url/eoferror", record: :once } do
-      lambda { www.follow_url("http://www.soulpancake.com/post/1607/whats-your-beautiful-mess.html") }.should_not raise_error
+    it "works with relative path redirects", vcr: { cassette_name: "follow/relative_redirects", record: :once } do
+      www.follow("http://www.retailmenot.com/out/4223117").should == "http://www.papajohns.com/index.html"
     end
 
-    it 'works with a proxy', vcr: { cassette_name: "follow_url/proxy", record: :once } do
-      www.proxy = "198.154.114.100:8080"
-      www.follow_url("http://yfrog.us/evlb0z:medium").should == "http://img535.imageshack.us/img535/9845/lb0.mp4"
-    end
-
-    it 'works for domains', vcr: { cassette_name: "follow_url/domains", record: :once } do
-      www.follow_url("google.com").should == "google.com"
+    it "raises response error on a bad proxy", vcr: { cassette_name: "follow/proxy_forbidden", record: :once } do
+      www.proxy = "190.202.116.101:3128"
+      lambda { www.follow("http://yfrog.us/evlb0z:medium").should }.should raise_error(DubDubDub::ResponseError)
     end
 
-    it "works with relative path redirects", vcr: { cassette_name: "follow_url/relative_redirects", record: :once } do
-      www.follow_url("http://www.retailmenot.com/out/4223117").should == "http://www.papajohns.com/index.html"
+    it "follows to the end for some types of urls", vcr: { cassette_name: "follow/all_the_way", record: :once } do
+      www.follow("http://www.apmebf.com/fo122tenm4/elq/32A39898/4432424/2/2/2").should == "http://www.bedbathandbeyond.com/default.asp?utm_source=WhaleShark+Media%3A+RetailMeNot%2Ecom&utm_medium=affiliate&utm_term=&utm_campaign=Bed+Bath+and+Beyond+Product+Catalog&aid=10817676&pid=2210202&sid=&"
     end
 
-    it "raises forbidden properly on a bad proxy", vcr: { cassette_name: "follow_url/proxy_forbidden", record: :once } do
-      www.proxy = "190.202.116.101:3128"
-      lambda { www.follow_url("http://yfrog.us/evlb0z:medium").should }.should raise_error(DubDubDub::Forbidden)
+    it "handles doesn't error out due to URI", vcr: { cassette_name: "follow/uri_error", record: :once } do
+      url = www.follow "http://retailmenot.com/out/4231224"
+      url.should == "http://www.toysrus.com/category/index.jsp?categoryId=3999911"
     end
   end
 end