gogetter 0.1.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,9 @@
+source "http://rubygems.org"
+
+group :development, :test do
+  gem "rspec", "~> 2.6.0"
+  gem "yard", "~> 0.7.2"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "fakeweb", "~> 1.3.0"
+end

data/Gemfile.lock

@@ -0,0 +1,30 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.2)
+    fakeweb (1.3.0)
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.9.2)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    yard (0.7.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  fakeweb (~> 1.3.0)
+  jeweler (~> 1.6.4)
+  rspec (~> 2.6.0)
+  yard (~> 0.7.2)

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Elad Kehat
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,28 @@
+= GoGetter
+
+Easily send get requests, with a little more sophistication than Net::HTTP.get
+
+Unlike HTTParty, which is a great gem if your class works against a specific website, GoGetter fits the use case where
+you need to send a bunch of HTTP GETs to several domains, and don't want to wrap each one in its own class.
+
+Think of it as an alternative to open-uri that doesn't create any temporary files.
+
+It handles proxies, basic authentication, and HTTP redirects.
+
+Before releasing this code I used it extensively in a proprietary web crawler that sent around a billion GET requests
+so far, so you could say that it's quite robust :)
+
+== Contributing to gogetter
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise
+necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Elad Kehat. See LICENSE.txt for further details.

data/Rakefile

@@ -0,0 +1,35 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  gem.name = "gogetter"
+  gem.homepage = "http://github.com/eladkehat/gogetter"
+  gem.license = "MIT"
+  gem.summary = %Q{Go get something over HTTP}
+  gem.email = "eladkehat@gmail.com"
+  gem.authors = ["Elad Kehat"]
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+  spec.rspec_opts = ['--options', 'spec/rspec.opts']
+end
+
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/gogetter.gemspec

@@ -0,0 +1,69 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{gogetter}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Elad Kehat"]
+  s.date = %q{2011-07-25}
+  s.email = %q{eladkehat@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "gogetter.gemspec",
+    "lib/go_getter.rb",
+    "lib/go_getter/go_getter.rb",
+    "lib/go_getter/response.rb",
+    "lib/go_getter/utils.rb",
+    "lib/gogetter.rb",
+    "spec/fixtures/google.html",
+    "spec/fixtures/google.redirect.html",
+    "spec/go_getter/go_getter_spec.rb",
+    "spec/rspec.opts",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/eladkehat/gogetter}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.7.2}
+  s.summary = %q{Go get something over HTTP}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, ["~> 2.6.0"])
+      s.add_development_dependency(%q<yard>, ["~> 0.7.2"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<fakeweb>, ["~> 1.3.0"])
+    else
+      s.add_dependency(%q<rspec>, ["~> 2.6.0"])
+      s.add_dependency(%q<yard>, ["~> 0.7.2"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<fakeweb>, ["~> 1.3.0"])
+    end
+  else
+    s.add_dependency(%q<rspec>, ["~> 2.6.0"])
+    s.add_dependency(%q<yard>, ["~> 0.7.2"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<fakeweb>, ["~> 1.3.0"])
+  end
+end
+

data/lib/go_getter/go_getter.rb

@@ -0,0 +1,76 @@
+# Monkey-patch Net::HTTPResponse
+# Add a final_url attribute, which we use in handling HTTP redirections to determine
+# the ultimate URI that the response was retrieve from
+class Net::HTTPResponse
+  attr_accessor :final_uri
+end
+
+
+module GoGetter
+
+  def GoGetter.get(uri, http_headers = {}, params = {})
+    uri = parse_url(uri.to_s) unless uri.is_a? URI
+    path = uri.path
+    path << "?#{uri.query}" if uri.query
+    request = Net::HTTP::Get.new(path)
+    http_headers.each {|key, value| request.add_field key, value }
+
+    # basic authentication
+    request.basic_auth(params[:auth_user], params[:auth_pass]) if params[:auth_user] and params[:auth_pass]
+
+    # proxy
+    klass = (params[:proxy_host] and params[:proxy_port]) ?
+      Net::HTTP::Proxy(params[:proxy_host], params[:proxy_port], params[:proxy_user], params[:proxy_pass]) : Net::HTTP
+
+    response = klass.start(uri.host, uri.port) do |http|
+      http.read_timeout = params.fetch(:read_timeout, 600)
+      http.request(request)
+    end
+
+    if response.is_a?(Net::HTTPRedirection) # Redirect
+      # allow for a single redirection by default
+      params[:max_redirects] = 1 unless params.has_key?(:max_redirects)
+      response = handle_redirection(uri, response, http_headers, params)
+    else
+      response.final_uri = uri
+    end
+
+    return response
+  end
+
+  # Given a URL, which may not be formatted properly, parse a URI
+  def GoGetter.parse_url(url)
+    unless (url =~ %r{^https?://}mi) == 0
+      url = "http://#{url}"
+    end
+    uri = URI.parse url
+    if uri.path.length == 0 and uri.query.nil?
+      uri.path = "/"
+    end
+    uri
+  end
+
+  def GoGetter.handle_redirection(from_uri, response, http_headers, params)
+    if params.fetch(:max_redirects, 0) > 0
+      params[:uris_seen] = Set.new unless params[:uris_seen]
+      if params[:uris_seen].size < params.fetch(:max_redirects, 0) && response['Location']
+        params[:uris_seen] << from_uri
+        new_uri = URI.parse(response['Location'])
+        # new uri may be just the path, w/o host and port; if so, copy from old
+        unless new_uri.host
+          new_uri.host = from_uri.host
+          new_uri.port = from_uri.port
+        end
+        new_uri.scheme = from_uri.scheme unless new_uri.scheme
+        # avoid infinite redirect loops
+        unless params[:uris_seen].member? new_uri
+          # request the new location just as we did the old one.
+          params[:max_redirects] -= 1
+          response = GoGetter.get(new_uri, http_headers, params)
+        end
+      end
+    end
+    response
+  end
+
+end

data/lib/go_getter/response.rb

@@ -0,0 +1,16 @@
+# Monkey-patch for Net::HTTPResponse
+# This file isn't required by the gem by default, so require it in your code
+class Net::HTTPResponse
+
+  alias :body_asis :body
+  # New version of #body unzips a gzipped body before returning it
+  # Call GoGetter.get with the following in http_headers: "Accept-Encoding" => "gzip")
+  def body
+    if key?("Content-Encoding") and fetch("Content-Encoding") == "gzip"
+      body_io = StringIO.new(body_asis)
+      Zlib::GzipReader.new(body_io).read
+    else
+      body_asis
+    end
+  end
+end

data/lib/go_getter/utils.rb

@@ -0,0 +1,17 @@
+module GoGetter
+
+  # Some user agents for use with websites that change their behavior according to your browser
+  # Set by adding to http_headers: "User-Agent" => USER_AGENTS[:chrome10_linux]
+  # Use http://www.useragentstring.com/pages/useragentstring.php to find more user agent strings
+  USER_AGENTS = {
+    :chrome10_win => "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.638.0 Safari/534.16",
+    :chrome10_linux => "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.648.0 Chrome/10.0.648.0 Safari/534.16",
+    :firefox36_win => "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8 ( .NET CLR 3.5.30729; .NET4.0C)",
+    :firefox36_linux => "Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.8) Gecko/20100804 Gentoo Firefox/3.6.8",
+    :ie8 => "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)",
+    :ie7 => "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)",
+    :opera11_win => "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00",
+    :safari5_mac => "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_3; en-us) AppleWebKit/534.1+ (KHTML, like Gecko) Version/5.0 Safari/533.16",
+  }
+
+end

data/lib/go_getter.rb

@@ -0,0 +1,3 @@
+require 'net/http'
+require 'set'
+require 'go_getter/go_getter'

data/lib/gogetter.rb

@@ -0,0 +1,2 @@
+# So you can require "gogetter" instead of "go_getter". Picked this up from fakeweb.
+require 'go_getter'

data/spec/fixtures/google.html

@@ -0,0 +1,3 @@
+<html><head><meta http-equiv="content-type" content="text/html; charset=ISO-8859-1"><title>Google</title><style>body,td,a,p,.h{font-family:arial,sans-serif}.h{color:#36c;font-size:20px}.q{color:#00c}.ts td{padding:0}.ts{border-collapse:collapse}#gbar{height:22px;padding-left:2px}.gbh,.gbd{border-top:1px solid #c9d7f1;font-size:1px}.gbh{height:0;position:absolute;top:24px;width:100%}#gbi,#gbs{background:#fff;left:0;position:absolute;top:24px;visibility:hidden;z-index:1000}#gbi{border:1px solid;border-color:#c9d7f1 #36c #36c #a2bae7;z-index:1001}#guser{padding-bottom:7px !important}#gbar,#guser{font-size:13px;padding-top:1px !important}@media all{.gb1,.gb3{height:22px;margin-right:.73em;vertical-align:top}#gbar{float:left}}.gb2{display:block;padding:.2em .5em}a.gb1,a.gb2,a.gb3{color:#00c !important}.gb2,.gb3{text-decoration:none}a.gb2:hover{background:#36c;color:#fff !important}</style><script>window.google={kEI:"Zuk6ScOkLKHCMrrttckF",kEXPI:"17259,19124,19314",kHL:"en"};
+google.y={};google.x=function(e,g){google.y[e.id]=[e,g];return false};function sf(){document.f.q.focus()}
+window.gbar={};(function(){var b=window.gbar,f,h;b.qs=function(a){var c=window.encodeURIComponent&&(document.forms[0].q||"").value;if(c)a.href=a.href.replace(/([?&])q=[^&]*|$/,function(i,g){return(g||"&")+"q="+encodeURIComponent(c)})};function j(a,c){a.visibility=h?"hidden":"visible";a.left=c+"px"}b.tg=function(a){a=a||window.event;var c=0,i,g=window.navExtra,d=document.getElementById("gbi"),e=a.target||a.srcElement;a.cancelBubble=true;if(!f){f=document.createElement(Array.every||window.createPopup?"iframe":"div");f.frameBorder="0";f.src="#";d.parentNode.appendChild(f).id="gbs";if(g)for(i in g)d.insertBefore(g[i],d.firstChild).className="gb2";document.onclick=b.close}if(e.className!="gb3")e=e.parentNode;do c+=e.offsetLeft;while(e=e.offsetParent);j(d.style,c);f.style.width=d.offsetWidth+"px";f.style.height=d.offsetHeight+"px";j(f.style,c);h=!h};b.close=function(a){h&&b.tg(a)}})();</script></head><body bgcolor=#ffffff text=#000000 link=#0000cc vlink=#551a8b alink=#ff0000 onload="sf();if(document.images)new Image().src='/images/nav_logo3.png'" topmargin=3 marginheight=3><div id=gbar><nobr><b class=gb1>Web</b> <a href="http://images.google.com/imghp?hl=en&tab=wi" onclick=gbar.qs(this) class=gb1>Images</a> <a href="http://maps.google.com/maps?hl=en&tab=wl" onclick=gbar.qs(this) class=gb1>Maps</a> <a href="http://news.google.com/nwshp?hl=en&tab=wn" onclick=gbar.qs(this) class=gb1>News</a> <a href="http://www.google.com/prdhp?hl=en&tab=wf" onclick=gbar.qs(this) class=gb1>Shopping</a> <a href="http://mail.google.com/mail/?hl=en&tab=wm" class=gb1>Gmail</a> <a href="http://www.google.com/intl/en/options/" onclick="this.blur();gbar.tg(event);return !1" class=gb3><u>more</u> <small>&#9660;</small></a><div id=gbi> <a href="http://video.google.com/?hl=en&tab=wv" onclick=gbar.qs(this) class=gb2>Video</a> <a href="http://groups.google.com/grphp?hl=en&tab=wg" onclick=gbar.qs(this) class=gb2>Groups</a> <a href="http://books.google.com/bkshp?hl=en&tab=wp" onclick=gbar.qs(this) class=gb2>Books</a> <a href="http://scholar.google.com/schhp?hl=en&tab=ws" onclick=gbar.qs(this) class=gb2>Scholar</a> <a href="http://finance.google.com/finance?hl=en&tab=we" onclick=gbar.qs(this) class=gb2>Finance</a> <a href="http://blogsearch.google.com/?hl=en&tab=wb" onclick=gbar.qs(this) class=gb2>Blogs</a> <div class=gb2><div class=gbd></div></div> <a href="http://www.youtube.com/?hl=en&tab=w1" onclick=gbar.qs(this) class=gb2>YouTube</a> <a href="http://www.google.com/calendar/render?hl=en&tab=wc" class=gb2>Calendar</a> <a href="http://picasaweb.google.com/home?hl=en&tab=wq" onclick=gbar.qs(this) class=gb2>Photos</a> <a href="http://docs.google.com/?hl=en&tab=wo" class=gb2>Documents</a> <a href="http://www.google.com/reader/view/?hl=en&tab=wy" class=gb2>Reader</a> <a href="http://sites.google.com/?hl=en&tab=w3" class=gb2>Sites</a> <div class=gb2><div class=gbd></div></div> <a href="http://www.google.com/intl/en/options/" class=gb2>even more &raquo;</a></div> </nobr></div><div class=gbh style=left:0></div><div class=gbh style=right:0></div><div align=right id=guser style="font-size:84%;padding:0 0 4px" width=100%><nobr><a href="/url?sa=p&pref=ig&pval=3&q=http://www.google.com/ig%3Fhl%3Den%26source%3Diglk&usg=AFQjCNFA18XPfgb7dKnXfKz7x7g1GDH1tg">iGoogle</a> | <a href="https://www.google.com/accounts/Login?continue=http://www.google.com/&hl=en">Sign in</a></nobr></div><center><br clear=all id=lgpd><img alt="Google" height=110 src="/intl/en_ALL/images/logo.gif" width=276><br><br><form action="/search" name=f><table cellpadding=0 cellspacing=0><tr valign=top><td width=25%>&nbsp;</td><td align=center nowrap><input name=hl type=hidden value=en><input type=hidden name=ie value="ISO-8859-1"><input autocomplete="off" maxlength=2048 name=q size=55 title="Google Search" value=""><br><input name=btnG type=submit value="Google Search"><input name=btnI type=submit value="I'm Feeling Lucky"></td><td nowrap width=25%><font size=-2>&nbsp;&nbsp;<a href=/advanced_search?hl=en>Advanced Search</a><br>&nbsp;&nbsp;<a href=/preferences?hl=en>Preferences</a><br>&nbsp;&nbsp;<a href=/language_tools?hl=en>Language Tools</a></font></td></tr></table></form><br><br><font size=-1><a href="/intl/en/ads/">Advertising&nbsp;Programs</a> - <a href="/services/">Business Solutions</a> - <a href="/intl/en/about.html">About Google</a></font><p><font size=-2>&copy;2008 - <a href="/intl/en/privacy.html">Privacy</a></font></p></center></body><script>if(google.y)google.y.first=[];window.setTimeout(function(){var xjs=document.createElement('script');xjs.src='/extern_js/f/CgJlbhICdXMgACswCjgMLCswDjgCLCswGDgDLA/8MIofMT_4o8.js';document.getElementsByTagName('head')[0].appendChild(xjs)},0);google.y.first.push(function(){google.ac.i(document.f,document.f.q,'','')})</script></html>

data/spec/fixtures/google.redirect.html

@@ -0,0 +1,6 @@
+<HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">
+<TITLE>302 Moved</TITLE></HEAD><BODY>
+<H1>302 Moved</H1>
+The document has moved
+<A HREF="http://www.google.co.il/">here</A>.
+</BODY></HTML>

data/spec/go_getter/go_getter_spec.rb

@@ -0,0 +1,151 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe "GoGetter" do
+
+  describe "#parse_url" do
+    it "returns a URI instance, given a URL string" do
+      uri = GoGetter.parse_url('http://www.google.com/')
+      uri.scheme.should == 'http'
+      uri.host.should == 'www.google.com'
+      uri.port.should == 80
+      uri.path.should == '/'
+    end
+
+    context "when the URL has no scheme" do
+      it "prepends http" do
+        uri = GoGetter.parse_url('www.google.com')
+        uri.scheme.should == 'http'
+      end
+    end
+
+    context "when the URL has no path" do
+      it "appends '/'" do
+        uri = GoGetter.parse_url('www.google.com')
+        uri.path.should == '/'
+      end
+    end
+
+    context "when the URL has a path" do
+      it "retains the path" do
+        uri = GoGetter.parse_url('www.google.com/search')
+        uri.path.should == '/search'
+      end
+    end
+
+    context "when the URL has a query part" do
+      it "retains the query" do
+        uri = GoGetter.parse_url('www.google.com/search?q=gogetter&hl=en')
+        uri.query.should == 'q=gogetter&hl=en'
+      end
+    end
+  end
+
+  describe "#handle_redirection" do
+    context "when the max_redirects param is 0" do
+      it "should not send another request" do
+        GoGetter.should_not_receive(:get)
+        response = Net::HTTPRedirection.new('1.1', '302', 'Found')
+        response['Location'] = 'http://www.google.co.il/'
+        GoGetter.handle_redirection(URI.parse('http://www.google.com/'), response, {}, {max_redirects: 0}).should == response
+      end
+    end
+
+    context "when the max_redirects param > 0" do
+      it "should send another get request to the new location" do
+        response = Net::HTTPRedirection.new('1.1', '302', 'Found')
+        response['Location'] = 'http://www.google.co.il/'
+        GoGetter.should_receive(:get).with(URI.parse('http://www.google.co.il'), {}, hash_including(:max_redirects=>0))
+        GoGetter.handle_redirection(URI.parse('http://www.google.com/'), response, {}, {max_redirects: 1})
+      end
+    end
+
+    context "when the response location is only a path (no host)" do
+      it "uses the host from the original URI" do
+        response = Net::HTTPRedirection.new('1.1', '302', 'Found')
+        response['Location'] = '/new_target'
+        exp_uri = URI.parse('/new_target')
+        exp_uri.host = 'www.google.com'; exp_uri.port = 80; exp_uri.scheme = 'http'
+        GoGetter.should_receive(:get).with(exp_uri, {}, hash_including(:max_redirects=>0))
+        GoGetter.handle_redirection(URI.parse('http://www.google.com/'), response, {}, {max_redirects: 1})
+      end
+    end
+
+    context "when the response redirects to a location that had already been got in the recursive chain" do
+      it "should not send another request" do
+        GoGetter.should_not_receive(:get)
+        response = Net::HTTPRedirection.new('1.1', '302', 'Found')
+        response['Location'] = 'http://www.google.co.il/'
+        GoGetter.handle_redirection(
+            URI.parse('http://www.google.com/'),
+            response, {},
+            {:max_redirects=>4,
+             :uris_seen => Set.new([URI.parse('http://www.google.co.il/'), URI.parse('http://www.yahoo.com/')])}
+        ).should == response
+      end
+    end
+  end
+
+  describe "#get" do
+    context "when given a URL" do
+      it "should get it" do
+        url = "http://google.html/"
+        body = file_fixture('google.html')
+        FakeWeb.register_uri(:get, url, :status => ['200', 'OK'], :body => body)
+        response = GoGetter.get url
+        response.should be_a(Net::HTTPOK)
+        response.body.should == body
+      end
+    end
+
+    context "when given a URI" do
+      it "should get that too" do
+        url = "http://google.html/"
+        body = file_fixture('google.html')
+        FakeWeb.register_uri(:get, url, :status => ['200', 'OK'], :body => body)
+        response = GoGetter.get URI.parse(url)
+        response.should be_a(Net::HTTPOK)
+        response.body.should == body
+      end
+    end
+
+    context "when given basic auth params" do
+      it "should do basic authentication" do
+        url = 'http://example.com/secret'
+        url_auth = 'http://user:pass@example.com/secret'
+        FakeWeb.register_uri(:get, url, :body => "Unauthorized", :status => ["401", "Unauthorized"])
+        FakeWeb.register_uri(:get, url_auth, :status => ['200', 'OK'], :body => "Authorized")
+        GoGetter.get(url).should be_a(Net::HTTPUnauthorized)
+        GoGetter.get(url, {}, {:auth_user => 'user', :auth_pass => 'pass'}).should be_a(Net::HTTPOK)
+      end
+    end
+
+    context "when given proxy params" do
+      it "should use a proxy" do
+        url = "http://google.html/"
+        #proxy:
+        host = 'proxy.example.com'; port = '8080'
+        user = 'user'; pass = 'pass'
+        #proxy_class = Net::HTTP::Proxy(proxy_host, proxy_port, proxy_user, proxy_pass)
+        FakeWeb.register_uri(:get, url, :status => ['200', 'OK'], :body => file_fixture('google.html'))
+        Net::HTTP.should_receive(:Proxy).with(host, port, user, pass)
+        # I was unable to mock proxy behavior properly and #get keeps raising errors
+        # However, this code still tests that a proxy class is created, which is the whole point
+        expect {
+          GoGetter.get(url, {}, {proxy_host: host,proxy_port: port,proxy_user: user,proxy_pass: pass})
+        }.to raise_error
+      end
+    end
+
+    context "when the response is a redirect" do
+      it "does redirection" do
+        url1 = "http://www.google.com/"
+        url2 = "http://www.google.co.il/"
+        body = file_fixture('google.redirect.html')
+        FakeWeb.register_uri(:get, url1, :status => ['302','Found'], :headers => {'Location'=>url2},:body => body)
+        params = {max_redirects: 1}
+        GoGetter.should_receive(:handle_redirection).with(URI.parse(url1), an_instance_of(Net::HTTPFound), {}, params)
+        GoGetter.get(url1, {}, params)
+      end
+    end
+  end
+end

data/spec/rspec.opts

@@ -0,0 +1,3 @@
+--colour
+--format documentation
+--backtrace

data/spec/spec_helper.rb

@@ -0,0 +1,21 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'gogetter'
+require 'fakeweb'
+
+def file_fixture(filename)
+  open(File.join(File.dirname(__FILE__), 'fixtures', "#{filename}")).read
+end
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    FakeWeb.allow_net_connect = false
+  end
+  config.after(:suite) do
+    FakeWeb.allow_net_connect = true
+  end
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: gogetter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Elad Kehat
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-07-25 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &74419330 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: *74419330
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: &74418990 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.2
+  type: :development
+  prerelease: false
+  version_requirements: *74418990
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &74418500 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *74418500
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &74417980 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.4
+  type: :development
+  prerelease: false
+  version_requirements: *74417980
+- !ruby/object:Gem::Dependency
+  name: fakeweb
+  requirement: &74417470 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :development
+  prerelease: false
+  version_requirements: *74417470
+description: 
+email: eladkehat@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- .document
+- .rspec
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- gogetter.gemspec
+- lib/go_getter.rb
+- lib/go_getter/go_getter.rb
+- lib/go_getter/response.rb
+- lib/go_getter/utils.rb
+- lib/gogetter.rb
+- spec/fixtures/google.html
+- spec/fixtures/google.redirect.html
+- spec/go_getter/go_getter_spec.rb
+- spec/rspec.opts
+- spec/spec_helper.rb
+homepage: http://github.com/eladkehat/gogetter
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 740502813
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: Go get something over HTTP
+test_files: []