itrigga-net_helper 0.2.0 → 0.3.0

data/Gemfile

@@ -5,13 +5,17 @@ source "http://rubygems.org"
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
+gem "rest-client"
+gem "hpricot"
+gem 'itrigga-core_ext', :require=>'core_ext'
+
 group :development do
   gem "hpricot"
   gem "bundler", "~> 1.0.0"
   gem "jeweler", "~> 1.6.4"
+  gem "rspec", "1.3.0"
+  gem "rspec-rails", "1.3.2"
   gem "rcov", ">= 0"
-  gem 'rspec'
   gem 'json_pure'
-  gem 'itrigga-core_ext', ">= 0"
   gem "fastercsv", ">= 1.5.4"
 end

data/README.rdoc

@@ -1,6 +1,41 @@
 = net_helper
 
-Description goes here.
+Provides a wrapper around HTTP GET requests. Uses Net::HTTP (under RestClient) by default, but will use Typhoeus if available
+Will return the raw body (unparsed) for each request.
+
+== Usage
+Rails2
+  gem.config 'itrigga-net_helper', :lib => 'net_helper'
+
+Rails3 (bundler)
+  gem 'itrigga-net_helper', :require => 'net_helper'
+
+=== Basic example
+  Itrigga::NetHelper.get :url => "http://www.google.com"
+
+=== Adding params
+  Itrigga::NetHelper.get :url => "http://www.google.com", :params => {:p1 => "123"}
+Gives url http://www.google.com?p1=123
+
+=== Variables
+
+timeout - in seconds (default 5)
+retries_on_timeout - how many times to try again (default 5)
+max_redirects - number of redirects to follow (default 3)
+
+The SystemTimer gem will be used for timeout is available.
+
+  Itrigga::NetHelper.get :url => "http://www.google.com", :timeout => 300, :max_redirects => 1
+
+== Using Typhoeus
+The Typhoeus gem needs to be installed (see https://github.com/dbalatero/typhoeus)
+Rails2 - gem.config 'typhoeus'
+Rails3 - gem 'typhoeus'
+
+To enable for all requests set the constant ITNH_HTTP_ENGINE = "Typhoeus"
+To use on a per request basis
+
+  Itrigga::NetHelper.get :url => "http://www.google.com", :http_engine => "Typhoeus"
 
 == Contributing to net_helper
  

data/Rakefile

@@ -26,22 +26,27 @@ Jeweler::Tasks.new do |gem|
 end
 Jeweler::RubygemsDotOrgTasks.new
 
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
+require 'spec/version'
+require 'spec/rake/spectask'
+require 'spec/ruby'
 
-require 'rcov/rcovtask'
-Rcov::RcovTask.new do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-  test.rcov_opts << '--exclude "gems/*"'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  RAILS_ENV = "test"
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+  spec.spec_opts = ['--options', 'spec/spec.opts']
 end
 
-task :default => :test
+task :default => :spec
+
+desc "Run all specs with rcov"
+Spec::Rake::SpecTask.new(:rcov) do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.spec_opts = ['--options', 'spec/spec.opts']
+  t.rcov = true
+  t.rcov_dir = 'coverage'
+  t.rcov_opts = ['--exclude', "features,kernel,load-diff-lcs\.rb,instance_exec\.rb,lib/spec.rb,lib/spec/runner.rb,^spec/*,bin/spec,examples,/gems,/Library/Ruby,\.autotest,#{ENV['GEM_HOME']}"]
+  t.rcov_opts << '--sort coverage --text-summary --aggregate coverage.data'
+end
 
 require 'rake/rdoctask'
 Rake::RDocTask.new do |rdoc|

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

data/lib/itrigga/net_helper.rb

@@ -1,3 +1,7 @@
+require 'rest-client'
+require 'itrigga/core_ext'
+Dir[File.join(File.expand_path(File.dirname(__FILE__)),"net_helper","*.rb")].each{|f| require f}
+
 require 'uri'
 require 'hpricot'
 require 'json'
@@ -10,171 +14,74 @@ module Itrigga
   
     module_function
     
-    def typhoeus_present?
-      if defined?(Typhoeus)
-        true
-      else
-        false
-      end
+    # gets a list of engines available (each is a different class in this module that implements 'get' method)
+    def available_http_engines
+      Itrigga::NetHelper.constants
     end
-  
-    def do_get(url, time_out=5, retries_on_timeout=5, max_redirects = 3)
-      retrycount = 0
-      resp = nil
-      begin
-        resp = get_with_timeout( url, time_out )
-      
-        handle_response( url, resp, retries_on_timeout,  max_redirects )
-    
-      rescue Timeout::Error      
-        if(retrycount.to_i < retries_on_timeout.to_i)
-          retrycount+=1
-          retry
-        else
-          raise IOError.new( "HTTP request timed out #{retrycount} times" )
-        end
-      end
     
+    def default_http_engine
+      "RestClient"
     end
     
-    def typhoeus_request( opts={} )
-      opts[:timeout] ||= 30
-      opts[:timeout] *= 1000 # Typhoeus does its timeouts in ms
-      opts[:follow_location] ||= true
-      opts[:disable_ssl_peer_verification] = true if opts[:disable_ssl_peer_verification].nil?
-      
-      request = Typhoeus::Request.new(opts[:url], opts)
-      
-      request.on_complete do |response|
-        if response.success?
-          return response.body
-        elsif response.timed_out?
-          # aw hell no
-          raise IOError.new("Timed out request to #{opts[:url]} after #{opts[:timeout]}ms") 
-        elsif response.code == 0
-          # Could not get an http response, something's wrong.
-          raise IOError.new(response.curl_error_message)
-        else
-          # Received a non-successful http response.
-          raise IOError.new("HTTP request failed: " + response.code.to_s)
-        end
-      end
-      
-      hydra = Typhoeus::Hydra.new
-      hydra.queue(request)
-      hydra.run
-    end
     
-    def get_response( url, timeout=nil )
-      if typhoeus_present?
-        typhoeus_request( :url=>url, :timeout=>timeout )
-      else
-        Net::HTTP.get_response(URI.parse(url))
-      end        
-    end
-  
-    def get_with_timeout( url, time_out)
-      resp = nil
-      if defined?(SystemTimer)
-        resp = SystemTimer.timeout_after(time_out) do
-          get_response(url, time_out)
-        end
-      else
-        resp = timeout(time_out) do
-          get_response(url, time_out)
-        end
-      end
-      resp
+    # wrapper method for backwards compat.
+    def do_get(url, timeout=5, retries_on_timeout=5, max_redirects = 3)
+      get :url => url, :timeout => timeout, :retries_on_timeout => retries_on_timeout, :max_redirects => max_redirects
     end
+    
+    
+    def get(options = {})
+      opts = { :timeout => 5, :retries_on_timeout => 5, :max_redirects => 3, :headers => {} }.merge(options)
+      raise ArgumentError.new(":url is required" ) unless opts[:url]
       
-  
-  
-    def handle_response( url, resp, retries_on_timeout=5, max_redirects = 3 )
-      if resp.is_a? Net::HTTPSuccess then resp.body
-      elsif resp.is_a? String then resp
-      elsif resp.is_a? Net::HTTPRedirection
-          if max_redirects > 0
-            do_get( URI.parse(url).merge(resp['location']).to_s, retries_on_timeout, max_redirects - 1 )
-          else
-            raise IOError.new("too many redirects!")
-          end
-      else
-        resp.error!
-      end
+      engine_klass = get_engine opts
+      with_timeout(opts) { engine_klass.get(opts) }
     end
-  
- 
-    def get( options = {} )
-      opts = {:timeout=>5, :retries_on_timeout=>5, :max_redirects => 3, :headers=>{} }.merge(options)
-      raise ArgumentError.new(":url is required" ) unless opts[:url]
     
-      if (opts[:username] || opts[:headers]).to_s.empty?
-        do_get(opts[:url], opts[:timeout], opts[:retries_on_timeout], opts[:max_redirects])
-      else
-      
-        retrycount = 0
-        resp = begin
-          if defined?(SystemTimer)
-            SystemTimer.timeout_after(opts[:timeout]) do
-              raw_get(opts)
-            end
-          else
-            timeout( opts[:timeout] ) do
-              raw_get(opts)
-            end
+    
+    
+    def with_timeout(opts, &block)
+      retrycount = 0
+      resp = begin
+        if defined?(SystemTimer)
+          SystemTimer.timeout_after(opts[:timeout]) do
+            yield
           end
-        rescue TimeoutError
-          if(retrycount < opts[:retries_on_timeout])
-            retrycount+=1
-            retry
-          else
-            raise IOError.new( "HTTP request timed out #{retrycount} times" )
+        else
+          timeout( opts[:timeout] ) do
+            yield
           end
         end
-        resp
+      rescue ::TimeoutError, ::RestClient::RequestTimeout
+        if(retrycount < opts[:retries_on_timeout])
+          retrycount+=1
+          retry
+        else
+          raise IOError.new( "HTTP request timed out #{retrycount} times" )
+        end
       end
+      resp  
     end
-  
-    def raw_get(opts) 
+    
+    
+    
+    def get_engine(opts = {})
+      # default to rest_client if not given an engine
+      opts[:http_engine] ||= ( defined?(ITNH_HTTP_ENGINE) ? ITNH_HTTP_ENGINE : default_http_engine )
       
-      if typhoeus_present?
-        return typhoeus_request(opts)  
-      end
+      # only use typhoeus if it is actually been required
+      opts[:http_engine] = default_http_engine if opts[:http_engine] == "Typhoeus" && defined?(::Typhoeus) == nil
       
-      resp = nil
-      establish_session_if_needed(opts)
-    
-      if opts[:username]
-        resp = get_with_auth(opts)
-        retries = 0
-        while resp.is_a? Net::HTTPRedirection do
-          retries += 1
-          raise IOError.new( "HTTP request timed out #{retries} times" ) if retries > (opts[:max_redirects] || 3)
-        
-          resp = get_with_auth(opts.merge(:parsed_url=>URI.parse(resp['location'])))
-        end
-        
-        resp.body
-        
+      if available_http_engines.include?(opts[:http_engine])
+        Itrigga::NetHelper.const_get opts[:http_engine]
       else
-        response = opts[:http_session].request_get(opts[:parsed_url].path, opts[:headers])
-        response.body
+        puts "[NetHelper] Could not find http_engine '#{opts[:http_engine]}'. Available engines are: #{available_http_engines.join(',')}. Defaulting to Net::HTTP"
+        Itrigga::NetHelper.const_get default_http_engine
       end
+      
     end
-  
-    def get_with_auth( opts )
-      establish_session_if_needed(opts)
     
-      req = Net::HTTP::Get.new(opts[:parsed_url].path)
-      req.basic_auth( opts[:username], opts[:password] ) if opts[:username]
-      resp = opts[:http_session].request(req, opts[:headers])
-    end
-  
-    def establish_session_if_needed(opts)
-      opts[:parsed_url] ||= URI.parse(opts[:url])
-      opts[:http_session] ||= Net::HTTP.new(opts[:parsed_url].host, opts[:parsed_url].port)
-      opts[:http_session].use_ssl = true if opts[:parsed_url].scheme == 'https'
-    end
+    
   
     def query_string( h, opts={:encode_values=>false, :skip_empty=>false} )
       params = []

data/lib/itrigga/net_helper/rest_client.rb

@@ -0,0 +1,19 @@
+module Itrigga
+  module NetHelper
+    class RestClient
+   
+      def self.get( opts={} )
+        raise ArgumentError.new(":url is required") unless opts[:url]
+        
+        # rest_client expects :user and not :username
+        opts[:user] ||= opts.delete(:username) if opts[:username]
+        
+        opts[:headers] = {:params => opts.delete(:params)} if opts[:params]
+        
+        response = ::RestClient::Request.new(opts.merge(:method => :get)).execute
+        response.to_str
+      end
+  
+    end
+  end
+end

data/lib/itrigga/net_helper/typhoeus.rb

@@ -0,0 +1,35 @@
+module Itrigga
+  module NetHelper
+    class Typhoeus
+   
+      def self.get( opts={} )
+        opts[:timeout] ||= 30
+        opts[:timeout] *= 1000 # Typhoeus does its timeouts in ms
+        opts[:follow_location] ||= true
+        opts[:disable_ssl_peer_verification] = true if opts[:disable_ssl_peer_verification].nil?
+        
+        request = ::Typhoeus::Request.new(opts[:url], opts)
+        
+        request.on_complete do |response|
+          if response.success?
+            return response.body
+          elsif response.timed_out?
+            # aw hell no
+            raise ::TimeoutError.new("Timed out request to #{opts[:url]} after #{opts[:timeout]}ms") 
+          elsif response.code == 0
+            # Could not get an http response, something's wrong.
+            raise IOError.new(response.curl_error_message)
+          else
+            # Received a non-successful http response.
+            raise IOError.new("HTTP request failed: " + response.code.to_s)
+          end
+        end
+        
+        hydra = ::Typhoeus::Hydra.new
+        hydra.queue(request)
+        hydra.run
+      end
+  
+    end
+  end
+end

data/spec/net_helper/rest_client_spec.rb

@@ -0,0 +1,32 @@
+require File.join(File.dirname(__FILE__), '..', 'spec_helper' )
+
+describe "Itrigga::NetHelper::RestClient" do
+  before do
+    @mock_response = mock("Response", :to_str => "stuff")
+    @client = mock("RestClient::Request")
+    @client.stub!(:execute).and_return(@mock_response)
+    RestClient::Request.stub!(:new).and_return(@client)
+  end
+  
+  it "should raise error if no url" do
+    lambda { Itrigga::NetHelper::RestClient.get }.should raise_error(ArgumentError,":url is required")
+  end
+  
+  it "should call execute on new Request" do
+    RestClient::Request.should_receive(:new).with(hash_including(:method => :get, :url => "abc", :param1 => "blart")).and_return(@client)
+    Itrigga::NetHelper::RestClient.get :url => "abc", :param1 => "blart"
+  end
+  
+  it "should return the correct value" do
+    @mock_response.should_receive(:to_str).and_return("stuff")
+    Itrigga::NetHelper::RestClient.get(:url => "abc").should == "stuff"
+  end
+  
+  context "when given a :username" do
+    it "should convert it to :user" do
+      RestClient::Request.should_receive(:new).with(hash_including(:method => :get, :url => "abc", :user => "blart")).and_return(@client)
+      Itrigga::NetHelper::RestClient.get :url => "abc", :username => "blart"
+    end
+  end
+  
+end

data/spec/net_helper/typhoeus_spec.rb

@@ -0,0 +1,90 @@
+require File.join(File.dirname(__FILE__), '..', 'spec_helper' )
+
+
+
+describe "Itrigga::NetHelper::Typhoeus" do
+  before do
+    # define Typhoeus stubs as not installed as gem dependancy
+    Object.class_eval %{
+      module Typhoeus
+        class Request
+          attr_accessor :on_complete
+          def initialize(url,opts = {}); end
+          def on_complete(&block); @on_complete = block; end
+        end
+        
+        class Hydra
+          attr_accessor :request
+          def queue(request); end
+          def run; end
+        end
+      end
+    }
+    
+    @opts = {:url => "blart"}
+    @response = mock("Response", :success? => true, :body => "stuff")
+    @request = Typhoeus::Request.new("blart", @opts)
+  #  @request.stub!(:on_complete).and_yield(@response)
+    Typhoeus::Request.stub!(:new).and_return(@request)
+    @hydra = Typhoeus::Hydra.new
+    Typhoeus::Hydra.stub!(:new).and_return(@hydra)
+  end
+  
+  after do
+    Object.send(:remove_const, :Typhoeus)
+  end
+  
+  it "should create a Request object" do
+    Typhoeus::Request.should_receive(:new).with("blart",@opts).and_return(@request)
+    Itrigga::NetHelper::Typhoeus.get @opts
+  end
+  
+  it "should call queue on hydra" do
+    @hydra.should_receive(:queue).with(@request)
+    Itrigga::NetHelper::Typhoeus.get(@opts)
+  end
+  
+  it "should call run on hydra" do
+    @hydra.should_receive(:run)
+    Itrigga::NetHelper::Typhoeus.get(@opts)
+  end
+  
+  it "should return the response body" do
+    @request.stub!(:on_complete).and_yield(@response)
+    Itrigga::NetHelper::Typhoeus.get(@opts).should == "stuff"
+  end
+  
+  context "a timeout" do
+    before do
+      @response = mock("Response", :success? => false, :timed_out? => true, :body => "stuff")
+      @request.stub!(:on_complete).and_yield(@response)
+    end
+    
+    it "should raise a TimeoutError" do
+      lambda { Itrigga::NetHelper::Typhoeus.get(@opts) }.should raise_error(TimeoutError)
+    end
+  end
+  
+  context "response code == 0" do
+    before do
+      @response = mock("Response", :success? => false, :timed_out? => false, :code => 0, :body => "stuff", :curl_error_message => "blart")
+      @request.stub!(:on_complete).and_yield(@response)
+    end
+    
+    it "should raise a TimeoutError" do
+      lambda { Itrigga::NetHelper::Typhoeus.get(@opts) }.should raise_error(IOError)
+    end
+  end
+  
+  context "any other http status code" do
+    before do
+      @response = mock("Response", :success? => false, :timed_out? => false, :code => 500, :body => "stuff")
+      @request.stub!(:on_complete).and_yield(@response)
+    end
+    
+    it "should raise a TimeoutError" do
+      lambda { Itrigga::NetHelper::Typhoeus.get(@opts) }.should raise_error(IOError)
+    end
+  end
+  
+end