unwind 0.2.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 22070510ea3066c9103a8efe3d75cc4106642a9b
+  data.tar.gz: a55418df179313c6ad2c60dfbf8f8dd4600b4c18
+SHA512:
+  metadata.gz: 2b40822f4063259cbfad43246342d55f13fcdf6291f85e3b67decd4186e565b1ed373fd0647b0e995cd83ddf7cec88c4c13aef0959d6600de4e9dccc8f16e2e6
+  data.tar.gz: b6f9a718d3890d9c0570ee4be065727c7479ce9b342161241fec08ec0274241fde6ba7d025aa779c908b66d27f53cd039b7b0451d27a4ff0f0667af1ed137f48

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.rvmrc

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in unwind.gemspec
+gemspec

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.pattern = "test/*_test.rb"
+end
+
+task :default => :test

data/Readme.md

@@ -0,0 +1,25 @@
+# Description
+
+Enables following a series of redirects (shortened urls)
+
+# Prerequisites
+
+Tested on Ruby 1.9.3 & 2.0 (likely still works on 1.8.7, but you are on your own.)
+
+# Example Code
+  
+	require 'unwind'
+	
+	follower = Unwind::RedirectFollower.new('http://j.mp/xZVND1')
+	follower.resolve
+	assert_equal 'http://ow.ly/i/s1O0', follower.final_url 
+	assert_equal 'http://j.mp/xZVND1', follower.original_url
+	assert_equal 2, follower.redirects.count
+	
+# Hat tip
+
+Most of the code is based on John Nunemaker's blog post [Following Redirects with Net/HTTP](http://railstips.org/blog/archives/2009/03/04/following-redirects-with-nethttp/).
+
+# License 
+
+Provided under the Do Whatever You Want With This Code License.

data/example/sample.rb

@@ -0,0 +1,9 @@
+require './lib/unwind'
+
+follower = Unwind::RedirectFollower.new('http://j.mp/xZVND1')
+follower.resolve
+follower.redirects.each {|url| puts "redirects to #{url}"}
+puts follower.original_url
+puts follower.final_url
+
+

data/lib/unwind/version.rb

@@ -0,0 +1,3 @@
+module Unwind
+  VERSION = "0.2.1"
+end

data/lib/unwind.rb

@@ -0,0 +1,188 @@
+require "unwind/version"
+require 'net/http'
+require 'addressable/uri'
+
+module Unwind
+
+  class TooManyRedirects < StandardError; end
+  class MissingRedirectLocation < StandardError; end
+
+  class RedirectFollower
+
+    attr_reader :final_url,  :original_url, :redirect_limit, :response, :redirects
+
+    def initialize(original_url, limit=5)
+      @original_url, @redirect_limit = original_url, limit
+      @redirects = []
+    end
+
+    def redirected?
+      !(self.final_url == self.original_url)
+    end
+
+    def resolve(current_url=nil, options={})
+      ok_to_continue?
+
+      current_url ||= self.original_url
+      #adding this header because we really only care about resolving the url
+      headers = (options || {}).merge({"accept-encoding" => "none"})
+
+      url = URI.parse(current_url)
+
+      request = Net::HTTP::Get.new(url)
+
+      headers.each do |header, value|
+        request.add_field(header, value)
+      end
+
+      response = Net::HTTP.start(
+        url.host,
+        url.port,
+        :use_ssl => url.scheme == 'https'
+      ) do |http|
+        http.request(request)
+      end
+
+      if is_response_redirect?(response)
+        handle_redirect(redirect_url(response), current_url, response, headers)
+      elsif meta_uri = meta_refresh?(current_url, response)
+        handle_redirect(meta_uri, current_url, response, headers)
+      else
+        handle_final_response(current_url, response)
+      end
+
+      self
+    end
+
+    def self.resolve(original_url, limit=5)
+      new(original_url, limit).resolve
+    end
+
+  private
+
+    def record_redirect(url)
+      @redirects << url.to_s
+      @redirect_limit -= 1
+    end
+
+    def is_response_redirect?(response)
+      Net::HTTPRedirection === response
+    end
+
+    def handle_redirect(uri_to_redirect, url, response, headers)
+      record_redirect url
+      resolve(uri_to_redirect.normalize, apply_cookie(response, headers))
+    end
+
+    def handle_final_response(current_url, response)
+      current_url = current_url.dup.to_s
+      if Net::HTTPSuccess === response && canonical = canonical_link?(response)
+        @redirects << current_url
+        if Addressable::URI.parse(canonical).relative?
+          @final_url = make_url_absolute(current_url, Addressable::URI.parse(canonical)).to_s
+        else
+          @final_url = canonical
+        end
+
+      else
+        @final_url = current_url
+      end
+      @response = response
+    end
+
+    def ok_to_continue?
+      raise TooManyRedirects if redirect_limit < 0
+    end
+
+    def redirect_url(response)
+      if response['location'].nil?
+        body_match = (response.body || "").match(/<a href=\"([^>]+)\">/i)
+        raise MissingRedirectLocation unless body_match
+        Addressable::URI.parse(body_match[0])
+      else
+        redirect_uri = Addressable::URI.parse(response['location'])
+        redirect_uri.relative? ? Addressable::URI.parse(response.uri).join(response['location']) : redirect_uri
+      end
+    end
+
+    def meta_refresh?(current_url, response)
+      if Net::HTTPSuccess === response
+        body_match = response.body.match(/<meta http-equiv=\"refresh\" content=\"0; URL=(.*?)\"\s*\/*>/i)
+        if body_match
+          uri = Addressable::URI.parse(body_match[1])
+          make_url_absolute(current_url, uri)
+        end
+      end
+    end
+
+    def canonical_link?(response)
+      body_match = response.body.match(/<link rel=[\'\"]canonical[\'\"] href=[\'\"](.*?)[\'\"]/i)
+      body_match ? Addressable::URI.parse(body_match[1]).to_s : false
+    end
+
+    def apply_cookie(response, headers)
+      if response.code.to_i == 302 && response['set-cookie']
+        headers.merge("cookie" => CookieHash.to_cookie_string(response['set-cookie']))
+      else
+        #todo: should we delete the cookie at this point if it exists?
+        headers
+      end
+    end
+
+    def make_url_absolute(current_url, relative_url)
+      current_uri = Addressable::URI.parse(current_url)
+      if (relative_url.relative?)
+        url = Addressable::URI.new(
+          :scheme => current_uri.scheme,
+          :user => current_uri.user,
+          :password => current_uri.password,
+          :host => current_uri.host,
+          :port => current_uri.port,
+          :path => relative_url.path,
+          :query => relative_url.query,
+          :fragment => relative_url.fragment)
+      else
+        relative_url
+      end
+    end
+
+  end
+
+  #borrowed (stolen) from HTTParty with minor updates
+  #to handle all cookies existing in a single string
+  class CookieHash < Hash
+
+    CLIENT_COOKIES = %w{path expires domain path secure httponly}
+
+    def add_cookies(value)
+      case value
+      when Hash
+        merge!(value)
+      when String
+        value = value.gsub(/expires=[\w,\s\-\:]+;/i, '')
+        value = value.gsub(/httponly[\,\;]*/i, '')
+        value.split(/[;,]\s/).each do |cookie|
+          array = cookie.split('=')
+          self[array[0].strip.to_sym] = array[1]
+        end
+      else
+        raise "add_cookies only takes a Hash or a String"
+      end
+    end
+
+    def to_cookie_string
+      delete_if { |k, v| CLIENT_COOKIES.include?(k.to_s.downcase) }.collect { |k, v| "#{k}=#{v}" }.join("; ")
+    end
+
+    def self.to_cookie_string(*cookie_strings)
+      h = CookieHash.new
+      cookie_strings.each do |cs|
+        h.add_cookies(cs)
+      end
+
+      h.to_cookie_string
+    end
+  end
+
+
+end

data/test/redirect_follower_test.rb

@@ -0,0 +1,129 @@
+require 'minitest'
+require 'minitest/autorun'
+require 'vcr'
+require './lib/unwind'
+
+VCR.configure do |c|
+  c.hook_into :webmock
+  c.cassette_library_dir = 'vcr_cassettes'
+end
+
+describe Unwind::RedirectFollower do
+
+  # needs to be regenerated to properly test...need to stop that :(
+  it 'should handle url with cookie requirement' do
+    VCR.use_cassette('with cookie') do
+      follower = Unwind::RedirectFollower.resolve('http://ow.ly/1hf37P')
+      assert_equal 200,  follower.response.code.to_i
+      assert follower.redirected?
+    end
+  end
+
+  it 'should resolve the url' do
+    VCR.use_cassette('xZVND1') do
+      follower = Unwind::RedirectFollower.resolve('http://j.mp/xZVND1')
+      assert_equal 'http://ow.ly/i/s1O0', follower.final_url
+      assert_equal 'http://j.mp/xZVND1', follower.original_url
+      assert_equal 2, follower.redirects.count
+      assert follower.redirected?
+    end
+  end
+
+  it 'should handle relative redirects' do
+    VCR.use_cassette('relative stackoverflow') do
+      follower = Unwind::RedirectFollower.resolve('http://stackoverflow.com/q/9277007/871617?stw=1')
+      assert follower.redirected?
+      assert_equal 'http://stackoverflow.com/questions/9277007/gitlabhq-w-denied-for-rails', follower.final_url
+    end
+  end
+
+  it 'should still handine relative redirects' do
+    # http://bit.ly/A4H3a2
+    VCR.use_cassette('relative stackoverflow 2') do
+      follower = Unwind::RedirectFollower.resolve('http://bit.ly/A4H3a2')
+      assert follower.redirected?
+    end
+  end
+
+  it 'should handle redirects to pdfs' do
+    VCR.use_cassette('pdf') do
+      follower = Unwind::RedirectFollower.resolve('http://binged.it/wVSFs5')
+      assert follower.redirected?
+      assert_equal 'https://microsoft.promo.eprize.com/bingtwitter/public/fulfillment/rules.pdf', follower.final_url
+    end
+  end
+
+  it 'should handle the lame amazon spaces' do
+    VCR.use_cassette('amazon') do
+      follower = Unwind::RedirectFollower.resolve('http://amzn.to/xrHQWS')
+      assert follower.redirected?
+    end
+  end
+
+  #http://amzn.to/xrHQWS
+
+  it 'should handle a https redirect' do
+    VCR.use_cassette('ssl tpope') do
+      follower = Unwind::RedirectFollower.resolve('http://github.com/tpope/vim-rails')
+      assert follower.redirected?
+      assert_equal 'https://github.com/tpope/vim-rails', follower.final_url
+    end
+  end
+
+  it 'should not be redirected' do
+    VCR.use_cassette('no redirect') do
+      follower  = Unwind::RedirectFollower.resolve('https://flippa.com')
+      assert !follower.redirected?
+    end
+  end
+
+  it 'should set the final url as being the canonical url and treat it as s redirect' do
+    VCR.use_cassette('canonical url', :preserve_exact_body_bytes => true) do
+      follower  = Unwind::RedirectFollower.resolve('http://www.scottw.com?test=abc')
+      assert  follower.redirected?
+      assert 'http://www.scottw.com', follower.final_url
+      assert 'http://www.scottw?test=abc', follower.redirects[0]
+    end
+  end
+
+  it 'should raise TooManyRedirects' do
+    VCR.use_cassette('xZVND1') do
+      follower = Unwind::RedirectFollower.new('http://j.mp/xZVND1', 1)
+      too_many_redirects = lambda {follower.resolve}
+      too_many_redirects.must_raise Unwind::TooManyRedirects
+    end
+  end
+
+  it 'should raise MissingRedirectLocation' do
+    VCR.use_cassette('missing redirect') do
+      follower = Unwind::RedirectFollower.new('http://tinyurl.com/6oqzkff')
+      missing_redirect_location = lambda{follower.resolve}
+      missing_redirect_location.must_raise Unwind::MissingRedirectLocation
+    end
+  end
+
+  it 'should handle a meta-refresh' do
+    VCR.use_cassette('meta refresh') do
+      follower = Unwind::RedirectFollower.resolve('http://www.nullrefer.com/?www.google.com')
+      assert follower.redirected?
+      assert_equal "www.google.com.au", URI(follower.final_url).host
+    end
+  end
+
+  it 'should handle a relative meta-refresh' do
+    VCR.use_cassette('relative meta refresh') do
+      follower = Unwind::RedirectFollower.resolve('http://fb.me/2JYu23acx')
+      assert follower.redirected?
+      assert_equal 'https://www.facebook.com/londonswf/posts/696389650411604?_fb_noscript=1', follower.final_url
+    end
+  end
+
+  it 'should handle a relative canonical url' do
+    VCR.use_cassette('relative canonical url') do
+      follower = Unwind::RedirectFollower.resolve('http://youtu.be/hPJ0oLahGDg')
+      assert follower.redirected?
+      assert_equal 'http://www.youtube.com/watch?v=hPJ0oLahGDg', follower.final_url
+    end
+  end
+
+end

data/unwind.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "unwind/version"
+
+Gem::Specification.new do |s|
+  s.name        = "unwind"
+  s.version     = Unwind::VERSION
+  s.authors     = ["Scott Watermasysk"]
+  s.email       = ["scottwater@gmail.com"]
+  s.homepage    = "http://www.scottw.com/unwind"
+  s.licenses    = ['MIT']
+  s.summary     = %q{Follows a chain redirects.}
+  s.description = <<-description 
+										Follows a chain of redirects and reports back on all the steps. 
+										Heavily inspired by John Nunemaker's blog post.
+										http://railstips.org/blog/archives/2009/03/04/following-redirects-with-nethttp/
+									description
+
+  s.rubyforge_project = "unwind"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency 'rake', '~> 10.1'
+  s.add_development_dependency 'minitest', '~> 5.2'
+  s.add_development_dependency 'vcr', '~> 2.8'
+  s.add_development_dependency 'webmock', '~> 1.17'
+  s.add_runtime_dependency 'addressable', '~> 2.2'
+end