downspout 0.2.7 → 0.2.8

data/VERSION

@@ -1 +1 @@
-0.2.7
+0.2.8

data/lib/downspout/base.rb

@@ -15,17 +15,16 @@ module Downspout
   end
 
 =begin rdoc
-  Download a file from a given URL to a given Path on the local system
-  The path is optional and will default to a generated temporary file
+  Download a file from a given URL. The path is optional and will default to a generated temporary file.
 =end
-  def self.download_url_to_path( some_url, some_path = nil )
-    $logger.debug("downspout | download_url_to_path |  URL : #{some_url}")
-    $logger.debug("downspout | download_url_to_path |  Download Path : #{some_path}")
+  def self.fetch_url( some_url, some_path = nil )
+    $logger.debug("downspout | fetch_url |  URL : #{some_url}")
+    $logger.debug("downspout | fetch_url |  Download Path : #{some_path}")
 
     begin
       d = Downspout::Downloader.new( :url => some_url, :path => some_path )
     rescue Exception => e
-      $logger.error("downspout | download_url_to_path | Exception : '#{e}'")
+      $logger.error("downspout | fetch_url | Exception : '#{e}'")
       return nil if e.class == Downspout::UnsupportedScheme
       raise e
     end
@@ -33,19 +32,16 @@ module Downspout
     fetched = d.download!
 
     if !(fetched) then
-      $logger.error("downspout | download_url_to_path |  Fetch Failed : #{d.url} ")
+      $logger.error("downspout | fetch_url |  Fetch Failed : #{d.url} ")
       return nil
     end
 
-    $logger.debug("downspout | download_url_to_path |  Local File : #{d.path} ")
+    $logger.debug("downspout | fetch_url |  Local File : #{d.path} ")
     return d
   end
 
-=begin rdoc
-  Convenience method for downloading a file from an URL without specifying a path for storage.
-=end
-  def self.fetch_url( the_url )
-    return self.download_url_to_path( the_url )
+  def self.download_url_to_path( the_url, the_path ) #:nodoc:
+    return self.fetch_url( the_url, the_path )
   end
 
 =begin rdoc

data/lib/downspout/config.rb

@@ -8,6 +8,8 @@ module Downspout
     @@curb_allowed = true
     @@curb_enabled = true
     @@prefix = 'downspout'
+    @@max_redirects = 2
+    @@ssl_verification = true
 
     def self.tmp_dir
       return @@tmp_dir
@@ -29,6 +31,14 @@ module Downspout
       return @@credentials
     end
 
+    def self.max_redirects
+      @@max_redirects
+    end
+
+    def self.max_redirects=( num )
+      @@max_redirects = num
+    end
+
     def self.network_enabled?
       return @@network_enabled
     end
@@ -42,6 +52,10 @@ module Downspout
       @@network_enabled = true
     end
 
+    def self.ssl_verification?
+      @@ssl_verification
+    end
+
     def self.curb_available?
       begin
         require 'curb'

data/lib/downspout/downloader.rb

@@ -23,6 +23,7 @@ module Downspout
       @response_headers = {}
       @started_at = nil
       @finished_at = nil
+      @redirected_url = nil
       
       if options.respond_to?(:keys) then
         options.each do |key, value|
@@ -58,7 +59,7 @@ module Downspout
       if !(@path.nil?) then
         @basename = File.basename( @path )
       else
-        if !(@uri.path.nil? || @uri.path.empty? || uri.path == '/')
+        if !(@uri.path.nil? || @uri.path.empty? || @uri.path == '/')
           @basename = File.basename( @uri.path ) 
         else
           $logger.debug("downspout | downloader | basename | Bad URI path") 
@@ -197,7 +198,7 @@ module Downspout
       $logger.debug("downspout | downloader | net_http_download | Downloading #{@url} ...")
 
       begin
-        response = net_http_fetch( @url , 1)
+        response = net_http_fetch( @url )
         open( @path, "wb" ) do |file|
           file.write(response.body)
         end
@@ -230,27 +231,38 @@ module Downspout
       return true
     end
 
-    def net_http_fetch( url_str, limit = 10 )
-      $logger.debug("downspout | downloader | net_http_fetch | URL: #{url_str}, Redirects: #{limit}.")
+    def net_http_fetch( url_str, redirects = 0 )
+      $logger.debug("downspout | downloader | net_http_fetch | URL: #{url_str}, Redirects: #{redirects}.")
+
       raise Downspout::BadURL, 'URL is missing' if url_str.nil?
-      raise Downspout::ExcessiveRedirects, 'HTTP redirect too deep' if limit == 0
+
+      if redirects > Downspout::Config.max_redirects then
+        raise Downspout::ExcessiveRedirects, 'HTTP redirect too deep'
+      end
 
       u = URI.parse( url_str )
+      
+      http = Net::HTTP.new( u.host, u.port )
+        
+      if (u.scheme == "https") then
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE if !(Downspout::Config.ssl_verification?)
+      end
 
-      my_request = Net::HTTP::Get.new( "#{u.path}?#{u.query}" )
+      my_request = Net::HTTP::Get.new( u.request_uri )
 
       # TODO : implement credentials for downloads via net_http_fetch
       my_request.basic_auth 'account', 'p4ssw0rd'
 
-      @response = Net::HTTP.start( u.host, u.port ) do |http|
-        http.request( my_request )
-      end
+      @response = http.request( my_request )
 
       case @response
       when Net::HTTPSuccess
         @response
       when Net::HTTPRedirection
-        net_http_fetch( @response['location'], limit - 1 )
+        @redirected_url = @response['location']
+        # TODO : use the new location to update the file name / extension when unknown
+        net_http_fetch( @redirected_url, redirects + 1 )
       else
         $logger.error("downspout | downloader | net_http_fetch | Response : #{@response}")
         @response.error!
@@ -261,10 +273,10 @@ module Downspout
       $logger.debug("downspout | downloader | curb_http_download | Downloading #{@url} ...")
 
       begin
-        curb = Curl::Easy.download( @url, @path) {|c| c.follow_location=true; c.max_redirects=1;}
-      rescue Curl::Err::HostResolutionError
+        curb = Curl::Easy.download( @url, @path) {|c| c.follow_location=true; c.max_redirects = Downspout::Config.max_redirects;}
+      rescue Curl::Err::HostResolutionError => dns_err
         $logger.error("downspout | downloader | curb_http_download | Curb/Curl DNS Error | #{@uri.host}")
-        return false
+        raise dns_err
       end
       
       $logger.debug("downspout | downloader | curb_http_download | Response Code : #{curb.response_code}")
@@ -277,6 +289,12 @@ module Downspout
       # populate the response headers from curb header string
       parse_headers_from_string!( curb.header_str )
 
+      ultimate_url = curb_last_location( curb.header_str )
+      if !( ultimate_url == @url ) then
+        # re-directed
+        @redirected_url = ultimate_url
+      end
+
       # populate a 'proxy' HTTPResponse object with the Curb data...
       hr_klass = Net::HTTPResponse.send('response_class', curb.response_code.to_s)
       $logger.debug("downspout | downloader | curb_http_download | Response Type : #{hr_klass.name}")
@@ -293,22 +311,42 @@ module Downspout
       return true
     end
 
+    def curb_last_location( header_string )
+      matches = header_string.scan(/Location\:\s?(.*)\W/)
+
+      return nil if matches.nil?
+      return nil if (matches.class == Array) && (matches.last.nil?)
+
+      result = matches.last.first.strip
+      $logger.error("downspout | downloader | curb_last_location | #{result}")
+      return result
+    end
+
     def parse_headers_from_string!( header_str )
+      #  $logger.debug("downspout | downloader | parse_headers_from_string! | Header String : #{header_str}")
       header_hash = {}
-      http_hash = {}
+
       headers = header_str.split("\r\n")
-  
       http_info = headers[0]
+
+      http_hash = {}
       http_hash[:header] = http_info
       http_hash[:version] = http_info.split(" ")[0].match("HTTP/([0-9\.]+)")[1]
       http_hash[:code] = (http_info.split("\r\n")[0].split(" ")[1]).to_i
       http_hash[:message] = http_info.split("\r\n")[0].split(" ")[2]
       
       header_hash["HTTP"] = http_hash
-      
-      headers[1..-1].each do |line|
-        header_name, header_value = line.match(/([\w\-\s]+)\:\s?(.*)/)[1..2]
-        header_hash[header_name] = header_value
+
+      headers[1..-1].each do |line|      
+        next if line.nil? || line.empty?
+        begin
+          matches = line.match(/([\w\-\s]+)\:\s?(.*)/)
+          next if matches.nil? || matches.size < 3
+          header_name, header_value = matches[1..2]
+          header_hash[header_name] = header_value
+        rescue Exception => e
+          $logger.warn("downspout | downloader | parse_headers_from_string! | #{line}, Exception : #{e}")
+        end
       end
 
       @response_headers = header_hash
@@ -318,20 +356,22 @@ module Downspout
       result = nil
 
       result = file_name_from_content_disposition
+      result = file_name_from_redirect if result.nil?
       result = file_name_from_content_type if result.nil?
 
       return result
     end
 
-
     # Extracts filename from Content-Disposition Header per RFC 2183
     # "http://tools.ietf.org/html/rfc2183"
     def file_name_from_content_disposition
-    
       file_name = nil
 
-      cd_key = response_headers.keys.select{|k| k =~ /content-disposition/i }.first
+      cd_key = response_headers.keys.select{|k| k =~ /content-disposition/i }.first # TODO: better to use the last?
 
+      $logger.debug("downspout | downloader | file_name_from_content_disposition | cd key : #{cd_key}")
+      return nil if cd_key.nil?
+      
       if cd_key then
         disposition = @response_headers[cd_key]
         if disposition then
@@ -357,6 +397,19 @@ module Downspout
       $logger.debug("downspout | downloader | file_name_from_content_type | #{file_name}")
       return file_name
     end
+
+    def file_name_from_redirect
+      return nil if @redirected_url.nil?
+
+      my_uri = URI::parse( @redirected_url )
+
+      if !(my_uri.path.nil? || my_uri.path.empty? || my_uri.path == '/')
+        return File.basename( my_uri.path )
+      else
+        $logger.debug("downspout | downloader | basename | Bad URI path") 
+        return nil
+      end
+    end
     
   end
   

data/lib/downspout/logger.rb

@@ -1,10 +1,10 @@
 if !(defined?( $logger )) then
-  if defined?( RAILS_DEFAULT_LOGGER ) then
-    $logger = RAILS_DEFAULT_LOGGER
+  if ( defined?( Rails ) && !(Rails.logger.nil?) ) then
+    $logger = Rails.logger
   else
     require 'logger'
     $logger = Logger.new( STDERR )
-    $logger.level = Logger::INFO
+    $logger.level = Logger::WARN
   end
 end
 

data/lib/downspout.rb

@@ -4,6 +4,7 @@ require 'rubygems'
 require 'fileutils'
 require 'uri'
 require 'net/http'
+require 'net/https'
 require 'net/ftp'
 
 # add this directory to the path...

data/test/downspout_test.rb

@@ -17,26 +17,40 @@ class DownspoutTest < Test::Unit::TestCase
 
     $test_ws.mount(TestServlet.path, TestServlet)
 
+    n_deep_proc = Proc.new { |req, resp|
+      resp.body = '3-deep redirector proc mounted on #{req.script_name}'
+
+      num = req.query['n'].to_i
+
+      if num > 1 then
+        resp.set_redirect( HTTPStatus::MovedPermanently, "/deep/?n=#{num - 1}") 
+      else
+        resp.set_redirect( HTTPStatus::MovedPermanently, '/READ_ME.rdoc')
+      end
+    }
+
+    $test_ws.mount('/deep', HTTPServlet::ProcHandler.new(n_deep_proc) )
+
     two_deep_proc = Proc.new { |req, resp|
       resp.body = '2-deep redirector proc mounted on #{req.script_name}'
-      resp.set_redirect( HTTPStatus::MovedPermanently, '/one/deep')
+      resp.set_redirect( HTTPStatus::MovedPermanently, '/deep/one')
     }
 
-    $test_ws.mount('/two/deep/', HTTPServlet::ProcHandler.new(two_deep_proc) )
+    $test_ws.mount('/deep/two', HTTPServlet::ProcHandler.new(two_deep_proc) )
 
     redir_proc = Proc.new { |req, resp|
       resp.body = 'redirector proc mounted on #{req.script_name}'
       resp.set_redirect( HTTPStatus::MovedPermanently, '/READ_ME.rdoc')
     }
 
-    $test_ws.mount('/one/deep/', HTTPServlet::ProcHandler.new(redir_proc) )
+    $test_ws.mount('/deep/one', HTTPServlet::ProcHandler.new(redir_proc) )
 
     $test_ws.mount("/images", HTTPServlet::FileHandler,
       File.join( Test::App.root, "test", "fixtures"), {:FancyIndexing => true} )
 
     $test_ws_thread = Thread.new { $test_ws.start }
   end
-      
+
   def test_download_rdoc_from_servlet
     some_url = $test_read_me_url
 
@@ -69,10 +83,12 @@ class DownspoutTest < Test::Unit::TestCase
     end
 
     should "fail with Curb error in case of excessive redirects" do
-      two_deep_url = "http://127.0.0.1:8899/two/deep?curby"
+      assert_equal 2, Downspout::Config.max_redirects
+
+      too_deep_url = "http://127.0.0.1:8899/deep?n=3&test=curb"
 
       assert_raise Curl::Err::TooManyRedirectsError do
-        dl = Downspout.fetch_url( two_deep_url )
+        dl = Downspout.fetch_url( too_deep_url )
       end
     end
 
@@ -82,12 +98,13 @@ class DownspoutTest < Test::Unit::TestCase
       end
 
       should "fail with Downspout error in case of excessive redirects" do
-        two_deep_url = "http://127.0.0.1:8899/two/deep?no-curb"
+        assert_equal 2, Downspout::Config.max_redirects
+
+        too_deep_url = "http://127.0.0.1:8899/deep/?n=3&test=no-curb"
   
         assert_raise Downspout::ExcessiveRedirects do
-          dl = Downspout.fetch_url( two_deep_url )
+          dl = Downspout.fetch_url( too_deep_url )
         end
-
       end
 
       teardown do
@@ -102,10 +119,12 @@ class DownspoutTest < Test::Unit::TestCase
   end
 
   should "fail due to excessive redirects" do
-    @obj = Downspout::Downloader.new( :url => "http://127.0.0.1:8899/two/deep?over-draft" )
+    @obj = Downspout::Downloader.new( :url => "http://127.0.0.1:8899/deep/?n=2&test=over-draft" )
 
     assert_raise Downspout::ExcessiveRedirects do
-     @obj.send('net_http_fetch', @obj.url, 1 )  # uses send to bypass Curb and force Net::HTTP
+      # use send to bypass Curb and force Net::HTTP
+      # also pass 1 so that the second redirect is too much
+      @obj.send('net_http_fetch', @obj.url, 1 )
     end
   end
 
@@ -113,7 +132,8 @@ class DownspoutTest < Test::Unit::TestCase
     @obj = Downspout::Downloader.new( :url => "http://fu.man.chu/deep/nested/resource?over-draft" )
 
     assert_raise SocketError do
-     @obj.send('net_http_fetch', @obj.url) # uses send to bypass Curb and force Net::HTTP
+      # uses send to bypass Curb and force Net::HTTP
+      @obj.send('net_http_fetch', @obj.url)
     end
   end
 
@@ -125,3 +145,4 @@ class DownspoutTest < Test::Unit::TestCase
   end
 
 end
+

data/test/focused_test.rb

@@ -0,0 +1,54 @@
+require 'test_helper'
+
+class FocusedTest < Test::Unit::TestCase
+  context "Downspout" do
+    context "Focus" do
+      setup do
+        @expected_url = "https://download.github.com/rails-rails-v2.3.11-0-gb0c3d45.tar.gz"
+        @header_str = "HTTP/1.1 302 Found
+Server: nginx/0.7.67
+Date: Mon, 18 Apr 2011 01:53:16 GMT
+Content-Type: text/html; charset=utf-8
+Connection: keep-alive
+Status: 302 Found
+Location: https://nodeload.github.com/rails/rails/tarball/v2.3.11
+X-Runtime: 3ms
+Content-Length: 121
+Set-Cookie: _gh_sess=BAh7BzoRbG9jYWxlX2d1ZXNzMCIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNoSGFzaHsABjoKQHVzZWR7AA%3D%3D--719def2ab2d99283383e374327fcdbb716c45c12; path=/; expires=Fri, 01 Jan 2021 00:00:00 GMT; secure; HttpOnly
+Cache-Control: no-cache
+Strict-Transport-Security: max-age=2592000
+
+HTTP/1.1 302 Moved Temporarily
+Server: nginx/0.7.67
+Date: Mon, 18 Apr 2011 01:53:17 GMT
+Transfer-Encoding: chunked
+Connection: keep-alive
+Location: https://download.github.com/rails-rails-v2.3.11-0-gb0c3d45.tar.gz
+
+HTTP/1.1 200 OK
+Server: nginx/0.7.67
+Date: Mon, 18 Apr 2011 01:53:17 GMT
+Content-Type: application/octet-stream
+Content-Length: 3416081
+Last-Modified: Mon, 18 Apr 2011 01:45:25 GMT
+Connection: keep-alive
+Accept-Ranges: bytes"
+      end
+
+      should "grep last location header" do
+        last_location = @header_str.scan(/Location\:\s?(.*)\W/).last.first
+
+        assert_equal @expected_url, last_location
+      end
+
+      should "match last_location method" do
+        @obj = Downspout::Downloader.new( :url => "http://bad.url/path" )
+        last_location = @obj.send('curb_last_location', @header_str )
+
+        assert_equal String, last_location.class
+        assert_equal @expected_url, last_location
+      end
+
+    end
+  end
+end

data/test/unit/config_test.rb

@@ -4,28 +4,56 @@ class ConfigTest < Test::Unit::TestCase
   context "Downspout" do
     context "Config" do
 
-      should "respond to tmp_dir" do
-        assert Downspout::Config.respond_to?(:tmp_dir)
+      should "respond to max_redirects" do
+        assert Downspout::Config.respond_to?(:max_redirects)
       end
 
-      should "default to '/tmp/downloads/' for Downspout::Config#tmp_dir" do
-        assert_equal "/tmp/downloads/", Downspout::Config.tmp_dir
+      should "respond to tmp_dir" do
+        assert Downspout::Config.respond_to?(:tmp_dir)
       end
 
       should "respond to Network Enabled" do
         assert Downspout::Config.respond_to?("network_enabled?")
       end
 
-      should "default to Network Enabled" do
-        assert Downspout::Config.network_enabled?
-      end
-
       should "respond to Disable Networking" do
         assert Downspout::Config.respond_to?("disable_networking!")
       end
 
+      should "respond to SSL Verification" do
+        assert Downspout::Config.respond_to?("ssl_verification?")
+      end
+
+      context "defaults" do
+  
+        should "use '/tmp/downloads/' for Downspout::Config#tmp_dir" do
+          assert_equal "/tmp/downloads/", Downspout::Config.tmp_dir
+        end
+
+        should "be 2 redirect max" do
+          assert_equal 2, Downspout::Config.max_redirects
+        end
+
+        should "be Network Enabled" do
+          assert Downspout::Config.network_enabled?
+        end
+
+        should "use 'downspout' Default Prefix" do
+          assert_equal "downspout", Downspout::Config.default_prefix
+        end
+
+        should "use Curb library if available" do
+          assert Downspout::Config.curb_available?
+          assert Downspout::Config.enable_curb!
+        end
+
+        should "use SSL Verification" do
+          assert Downspout::Config.ssl_verification?
+        end
+
+      end
+      
       should "allow setting Default Prefix" do
-        assert_equal "downspout", Downspout::Config.default_prefix
         Downspout::Config.default_prefix = "fu-manchu"
         assert_equal "fu-manchu", Downspout::Config.default_prefix
       end
@@ -44,17 +72,19 @@ class ConfigTest < Test::Unit::TestCase
         assert Downspout::Config.curb_available?
       end
 
-      should "enable Curb if library is available" do
-        assert Downspout::Config.curb_available?
-        assert Downspout::Config.enable_curb!
-      end
-
       should "support enabling network operations" do
         assert !(Downspout::Config.network_enabled?)
         assert Downspout::Config.enable_networking!
         assert Downspout::Config.network_enabled?
       end
 
+      should "allow customization of redirect maximum" do
+        Downspout::Config.max_redirects = 3
+        assert_equal 3, Downspout::Config.max_redirects
+        Downspout::Config.max_redirects = 2
+        assert_equal 2, Downspout::Config.max_redirects
+      end
+
       context "Host-Based Credentials" do
         should "respond to Credentials" do
           assert Downspout::Config.respond_to?(:credentials)

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: downspout
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 7
   prerelease: 
   segments: 
   - 0
   - 2
-  - 7
-  version: 0.2.7
+  - 8
+  version: 0.2.8
 platform: ruby
 authors: 
 - Phi.Sanders
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-15 00:00:00 -04:00
+date: 2011-04-17 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -122,6 +122,7 @@ files:
 - test/downspout_test.rb
 - test/fixtures/faux_headers.txt
 - test/fixtures/ruby.png
+- test/focused_test.rb
 - test/servlet.rb
 - test/test_helper.rb
 - test/test_logger.rb
@@ -168,6 +169,7 @@ specification_version: 3
 summary: Downspout is an easy-to-use ruby library for downloading files from URLs.
 test_files: 
 - test/downspout_test.rb
+- test/focused_test.rb
 - test/servlet.rb
 - test/test_helper.rb
 - test/test_logger.rb