downspout 0.2.2

data/lib/downspout/downloader.rb

@@ -0,0 +1,344 @@
+module Downspout
+# The object returned by a call to fetch_url() or download_url_to_disk().
+  class Downloader < Base
+
+    # returns the path to the downloaded file
+    attr_accessor :path
+
+    # returns the remote response as the appropriate Net::HTTPResponse
+    attr_reader :response
+
+    # returns the headers parsed from the remote response
+    attr_reader :response_headers
+
+    # returns the URI parsed from the URL
+    attr_reader :uri
+
+    # returns the URL initially given
+    attr_accessor :url
+
+    def initialize( options=nil ) #:nodoc:
+      @basename = nil
+      @curb_enabled = Downspout::Config.use_curb?
+      @response_headers = {}
+      @started_at = nil
+      @finished_at = nil
+      
+      if options.respond_to?(:keys) then
+        options.each do |key, value|
+          self.send("#{key}=", value)
+        end
+      end
+
+      @uri = URI.parse( @url ) unless @url.nil?
+    end
+
+    def to_s #:nodoc:
+      return @path
+    end
+
+    #  returns the protocol or 'scheme' of the URL
+    def scheme
+      return @uri.scheme unless @uri.nil?
+      return nil
+    end
+
+    #  returns the time taken to download the file
+    def duration
+      return nil unless @started_at
+      return nil unless @finished_at
+      
+      return @finished_at - @started_at
+    end
+
+    #  Extracts the file name from the URL or uses a default name based on the content-type header
+    def basename
+      return @basename unless @basename.nil?
+      
+      if !(@path.nil?) then
+        @basename = File.basename( @path )
+      else
+        if !(@uri.path.nil? || @uri.path.empty? || uri.path == '/')
+          @basename = File.basename( @uri.path ) 
+        else
+          $logger.debug("downspout | downloader | basename | Bad URI path") 
+          @basename = 'file.downspout'
+        end
+      end
+            
+      $logger.debug("downspout | downloader | basename | #{@basename} ")
+
+      return @basename
+    end
+
+    #  will this download use the Curb library?
+    def use_curb?
+      return @curb_enabled
+    end
+
+    #  will this download use the default Net/HTTP library?
+    def use_net_http?
+      return false if use_curb?
+      return true
+    end
+
+    #  configure this download to use the Curb library (will fail if Curb is unavailable.)
+    def enable_curb!
+      @curb_enabled = true if Downspout::Config.curb_available?
+
+      return @curb_enabled
+    end
+
+    #  configure this download NOT to use the Curb library
+    def disable_curb!
+      @curb_enabled = false
+    end
+
+    def download! #:nodoc:
+      $logger.debug("downspout | downloader | download! | URL : #{@url} ")
+      @started_at = Time.now
+
+      raise UnsupportedScheme if @uri.nil?
+      raise UnsupportedScheme unless Downspout.supported_protocol?( @uri.scheme )
+
+      if @path.nil? then
+        tf = Downspout::Tmpfile.new( :name => self.basename )
+        @path = tf.path
+      end
+      $logger.debug("downspout | downloader | download! | Path : #{@path} ")
+
+      remove_file_at_target_path
+
+      if Downspout::Config.network_enabled? then
+        case self.scheme
+        when /ftp/
+          net_ftp_download
+        when /http[s]?/
+          if use_curb? then
+            curb_http_download
+          else
+            net_http_download
+          end
+        else
+          $logger.error("downspout | downloader | download! | Unknown URL Scheme : '#{self.scheme}'")
+          raise UnsupportedScheme
+        end
+      else
+        $logger.warn("downspout | downloader | download! | >>>>>   Networking Disabled   <<<<<")
+      end
+
+      downloaded = File.exist?( @path )
+
+      $logger.debug("downspout | downloader | download! | #{self.basename} downloaded? : #{downloaded} ")
+      @finished_at = Time.now
+      
+      if (tf && @basename == 'file.downspout') then
+        # rename file more appropriately
+        new_name = generate_file_name
+        if !(new_name.nil?) then
+          $logger.debug("downspout | downloader | download! | Renaming #{@basename} to #{new_name} ...")
+          new_path = File.join( File.dirname( tf.path ), new_name)
+          FileUtils.mv( tf.path, new_path )
+          @path = new_path
+        end
+      end
+
+      $logger.debug("downspout | downloader | download! | Started: #{@started_at.utc}, Finished: #{@finished_at.utc}, Duration: #{duration}")
+      
+      return downloaded
+    end
+
+    private
+
+    def remove_file_at_target_path
+      if File.exist?( @path ) then
+        $logger.debug("downspout | downloader | remove_file_at_target_path | Removing #{@path} ... ")
+        FileUtils.rm( @path )
+      end
+    end
+
+    def net_ftp_download
+      $logger.debug("downspout | downloader | net_ftp_download | Downloading #{@url} ...")
+
+      # look up the credentials for this host
+      cred = Downspout::Config.credentials.select{|c| c.scheme == 'ftp' }.select{ |c| c.host == @uri.host }.first
+      if cred.nil? then
+        $logger.warn("downspout | downloader | net_ftp_download | No credentials found for '#{@uri.host}'.")
+        # proceed anyway - slight possibility it's an un-authorized FTP account...
+      else
+        $logger.debug("downspout | downloader | net_ftp_download | Loaded credentials for #{cred.host} ...")
+      end
+      
+      begin
+        ftp = Net::FTP.open( @uri.host ) do |ftp|
+          ftp.login( cred.user_name, cred.pass_word ) unless cred.nil?
+          ftp.passive
+          ftp.chdir( File.dirname( @uri.path ) )
+          
+          $logger.debug("downspout | downloader | net_ftp_download | Local Path : #{@path} ...")
+          ftp.getbinaryfile( self.basename, @path )
+        end
+      rescue Exception => e
+        $logger.error("downspout | downloader | net_ftp_download | Exception : #{e}")
+        raise e
+      end
+
+      done = File.exist?( @path )
+      $logger.debug("downspout | downloader | net_ftp_download | #{basename} downloaded? : #{done}.")
+
+      return done
+    end
+
+    def net_http_download
+      $logger.debug("downspout | downloader | net_http_download | Downloading #{@url} ...")
+
+      begin
+        response = net_http_fetch( @url , 1)
+        open( @path, "wb" ) do |file|
+        
+          file.write(response.body)
+        end
+
+       $logger.debug("downspout | downloader | net_http_download | Response Body : #{response.body[0..5].strip}")
+
+      rescue SocketError => dns_err
+        $logger.error("downspout | downloader | net_http_download | Net/HTTP DNS Error | #{@uri.host} | #{dns_err.inspect}")
+        remove_file_at_target_path
+        raise dns_err
+      end
+      
+      $logger.debug("downspout | downloader | net_http_download | Response Code : #{response.code}")
+
+      # populate the response headers from net/http headers...
+      new_header_str = "HTTP/1.1 #{@response.code} #{@response.message}\r\n"
+      @response.each_header do |k,v|
+        new_header_str += "#{k}: #{v}\r\n"
+      end
+      @response_headers = parse_headers_from_string( new_header_str )
+
+  
+      if ((response.code.to_i != 200) and (response.code.to_i != 202)) then
+        # missing file, failed download - delete the response body [if downloaded]
+        remove_file_at_target_path
+        return false
+      end
+
+      $logger.debug("downspout | downloader | net_http_download | Headers : #{response.header}")
+    
+      if !( File.exist?( @path ) ) then
+        $logger.error("downspout | downloader | net_http_download | Missing File at download path : #{@path}")
+        return false
+      end
+      
+      $logger.debug("downspout | downloader | net_http_download | Successful.")
+      return true
+    end
+
+    def net_http_fetch( url_str, limit = 10 )
+      $logger.debug("downspout | downloader | net_http_fetch | URL: #{url_str}, Redirects: #{limit}.")
+      raise Downspout::BadURL, 'URL is missing' if url_str.nil?
+      raise Downspout::ExcessiveRedirects, 'HTTP redirect too deep' if limit == 0
+
+      u = URI.parse( url_str )
+
+      my_request = Net::HTTP::Get.new( "#{u.path}?#{u.query}" )
+
+      # TODO : implement credentials for downloads via net_http_fetch
+      my_request.basic_auth 'account', 'p4ssw0rd'
+
+      $logger.debug("downspout | downloader | net_http_fetch | Firing...")
+      @response = Net::HTTP.start( u.host, u.port ) do |http|
+        http.request( my_request )
+      end
+
+      $logger.debug("downspout | downloader | net_http_fetch | Response : #{@response}")
+      
+      case @response
+      when Net::HTTPSuccess
+        @response
+      when Net::HTTPRedirection
+        net_http_fetch( @response['location'], limit - 1 )
+      else
+        @response.error!
+      end
+    end
+    
+    def curb_http_download
+      $logger.debug("downspout | downloader | curb_http_download | Downloading #{@url} ...")
+
+      begin
+        curb = Curl::Easy.download( @url, @path) {|c| c.follow_location=true; c.max_redirects=1;}
+      rescue Curl::Err::HostResolutionError
+        $logger.error("downspout | downloader | curb_http_download | Curb/Curl DNS Error | #{@uri.host}")
+        return false
+      end
+      
+      $logger.debug("downspout | downloader | curb_http_download | Response Code : #{curb.response_code}")
+
+      if ((curb.response_code != 200) and (curb.response_code != 202)) then
+        # missing file, failed download - delete the response body [if downloaded]
+        remove_file_at_target_path
+      end
+
+      $logger.debug("downspout | downloader | curb_http_download | Headers : #{curb.header_str}")
+
+      # populate the response headers from curb header string
+      @response_headers = parse_headers_from_string( curb.header_str )
+
+      # populate a 'proxy' HTTPResponse object with the Curb data...
+      hr_klass = Net::HTTPResponse.send('response_class', curb.response_code.to_s)
+      $logger.debug("downspout | downloader | curb_http_download | Response Type : #{hr_klass.name}")
+      
+      @response = hr_klass.new( @response_headers["HTTP"][:version],
+        curb.response_code,
+        @response_headers["HTTP"][:message] )
+        
+      $logger.debug("downspout | downloader | curb_http_download | Response : #{@response.inspect}")
+
+      if !( File.exist?( @path ) ) then
+        $logger.error("downspout | downloader | curb_http_download | Missing File at download path : #{@path}")
+        return false
+      end
+      
+      $logger.debug("downspout | downloader | curb_http_download | Successful.")
+      return true
+    end
+
+    def parse_headers_from_string( header_str )
+      $logger.debug("downspout | downloader | parse_headers_from_string | String : #{header_str}")
+      header_hash = {}
+      http_hash = {}
+      headers = header_str.split("\r\n")
+  
+      http_info = headers[0]
+      http_hash[:header] = http_info
+      http_hash[:version] = http_info.split(" ")[0].match("HTTP/([0-9\.]+)")[1]
+      http_hash[:code] = (http_info.split("\r\n")[0].split(" ")[1]).to_i
+      http_hash[:message] = http_info.split("\r\n")[0].split(" ")[2]
+      
+      $logger.debug("downspout | downloader | parse_headers_from_string | Response : #{http_hash[:version]}, #{http_hash[:code]}, #{http_hash[:message]}")
+      header_hash["HTTP"] = http_hash
+      
+      headers[1..-1].each do |line|
+        header_name, header_value = line.match(/([\w\-\s]+)\:\s?(.*)/)[1..2]
+        header_hash[header_name] = header_value
+      end
+  
+      return header_hash
+    end
+
+    def generate_file_name
+      file_type = @response_headers['Content-Type'] if use_curb?
+      file_type = @response_headers['Content-Type'] if use_net_http?
+      return nil unless file_type
+
+      return "default.html" if file_type =~ /text\/html/
+      # TODO : smarter file name generation
+
+      return nil
+    end
+    
+  end
+
+  
+end

data/lib/downspout/logger.rb

@@ -0,0 +1,11 @@
+if !(defined?( $logger )) then
+  if defined?( RAILS_DEFAULT_LOGGER ) then
+    $logger = RAILS_DEFAULT_LOGGER
+  else
+    require 'logger'
+    $logger = Logger.new( STDERR )
+    $logger.level = Logger::INFO
+  end
+end
+
+$logger.debug "initialized logging facility..."

data/lib/downspout/tmp_file.rb

@@ -0,0 +1,101 @@
+require 'fileutils'
+require 'tempfile'
+
+module Downspout
+
+  class Tmpfile < File
+
+    # acepts an options hash which can include either or both of :name and :prefix
+    # then creates a Tempfile with the optionally given name in a unique sub-folder
+    # of the configured directory, optionally named with the prefix string. 
+    # The unique folder name includes the prefix, a sortable date, the PID of
+    # the download process, and a randomly generated sequence of characters.
+    #
+    #   => "/tmp/downloads/my-app-20110203-59488-1run8k2-0/desired-file-name.txt"
+    #
+    # call-seq:
+    # Downspout::Tmpfile.new( :name => 'desired-file-name.txt', :prefix => 'my-app' )
+    #
+    def initialize( options = nil )
+      # make sure the configured directory exists
+      FileUtils.mkdir_p( Downspout::Config.tmp_dir )
+
+      defaults = {:prefix => 'downspout', :name => "downloaded_file.tmp"}
+
+      # overwrite defaults with given options
+      defaults.merge!( options ) unless options.nil?
+
+      # create a unique file path from the given options
+      unique_path = File.join( Downspout::Config.tmp_dir, tmp_dir_name( defaults[:prefix] ), defaults[:name] )
+
+      # make sure the unique directory exists
+      $logger.debug("downspout | tmpfile | initialize | Creating unique directory : #{File.dirname(unique_path)}")
+      FileUtils.mkdir_p( File.dirname( unique_path ) )
+      raise "MakeDir Error" unless File.exist?( File.dirname( unique_path ) )
+      
+      super( unique_path, File::CREAT, 0644 )
+
+    end
+
+    def self.clean_dir( dir_path, delay=30 ) #:nodoc:
+      # remove files older than DELAY (in minutes) from configured folder
+      delay = 30 unless (delay.class == Fixnum)
+      t0 = Time.now - ( delay * 60 )
+
+      return false unless File.exist?( dir_path )
+      the_dir = Dir.new( dir_path )
+    
+      $logger.debug( "downspout | tmpfile | clean_dir | start | Entries : #{the_dir.entries.size}" )
+    
+      the_dir.entries.each do |item|
+        next if item == "."
+        next if item == ".."
+      
+        $logger.debug( "downspout | tmpfile | clean_dir | sub item : #{item}" )
+      
+        item_path = File.join( dir_path, item )
+      
+        tx = File.mtime( item_path ).utc
+      
+        # skip files with modtime changed less than sixty minutes ago
+        next unless (tx < t0)
+
+        if File.directory?( item_path ) then
+          $logger.debug( "downspout | tmpfile | clean_dir | Removing Directory : #{item}/*" )
+          
+          clean_dir( item_path, delay )
+          
+          begin
+            FileUtils.rmdir( item_path )
+          rescue Exception => e
+            $logger.debug( "downspout | tmpfile | clean_dir | Exception : #{e}" )
+            return false
+          end
+        else
+          $logger.debug( "downspout | tmpfile | clean_dir | Removing Item : #{item}" )
+
+          FileUtils.rm( item_path )
+        end
+      end
+
+      $logger.debug( "downspout | tmpfile | clean_download_dir | finish | Entries : #{the_dir.entries.size}" )
+      return true
+    end
+    
+    private
+    
+    def tmp_dir_name( prefix, n=rand(9) )
+      t = Time.now.strftime("%Y%m%d")
+      path = "#{prefix}-#{t}-#{$$}-#{rand(0x100000000).to_s(36)}-#{n}"
+    end
+
+  end
+
+
+  # Utility method for periodically removing download files from the configured directory.
+  # Expects an integer as the number of minutes 'old' a file should be before removal.
+  def self.clean_download_dir( delay=30 )
+    Tmpfile.clean_dir( Downspout::Config.tmp_dir, delay )
+  end
+
+end

data/lib/downspout.rb

@@ -0,0 +1,23 @@
+require 'rubygems'
+
+# common dependencies
+require 'fileutils'
+require 'uri'
+require 'net/http'
+require 'net/ftp'
+
+# unusual dependencies
+require 'curb'
+
+# add this directory to the path...
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+# customized logger
+require 'downspout/logger'
+
+# required components
+require 'downspout/base'
+require 'downspout/config'
+require 'downspout/credential'
+require 'downspout/tmp_file'
+require 'downspout/downloader'

data/test/downspout_test.rb

@@ -0,0 +1,127 @@
+require 'test_helper'
+require 'test_servlet'
+
+class DownspoutTest < Test::Unit::TestCase
+
+  def setup
+    $test_ws_root ||= Test::App.root
+    $test_read_me_url = "http://127.0.0.1:8899/READ_ME.rdoc"
+    $test_image_url = "http://127.0.0.1:8899/images/ruby.png"
+
+    $test_ws ||= WEBrick::HTTPServer.new(:Port => 8899,
+     :DocumentRoot => @test_ws_root,
+#     :Logger => Log.new(nil, BasicLog::WARN), # TODO : Use Log/BasicLog from WEBrick to reduce spam in tests
+     :Logger => $logger,
+     :AccessLog => [])
+
+
+    $test_ws.mount(TestServlet.path, TestServlet)
+
+    two_deep_proc = Proc.new { |req, resp|
+      resp.body = '2-deep redirector proc mounted on #{req.script_name}'
+      resp.set_redirect( HTTPStatus::MovedPermanently, '/one/deep')
+    }
+
+    $test_ws.mount('/two/deep/', HTTPServlet::ProcHandler.new(two_deep_proc) )
+
+    redir_proc = Proc.new { |req, resp|
+      resp.body = 'redirector proc mounted on #{req.script_name}'
+      resp.set_redirect( HTTPStatus::MovedPermanently, '/READ_ME.rdoc')
+    }
+
+    $test_ws.mount('/one/deep/', HTTPServlet::ProcHandler.new(redir_proc) )
+
+    $test_ws.mount("/images", HTTPServlet::FileHandler,
+      File.join( Test::App.root, "test", "fixtures"), {:FancyIndexing => true} )
+
+    $test_ws_thread = Thread.new { $test_ws.start }
+  end
+      
+  def test_download_rdoc_from_servlet
+    some_url = $test_read_me_url
+
+    dl = Downspout.fetch_url( some_url )
+
+    assert_not_nil dl
+
+    assert File.exist?( dl.path )
+  end
+
+  def test_by_fetching_image_from_w3c
+    some_url = "http://www.w3.org/wiki/images/2/2e/Ruby01.png"
+
+    dl = Downspout.fetch_url( some_url )
+
+    assert_not_nil dl
+
+    assert File.exist?( dl.path )
+  end
+
+  context "Downspout" do
+    should "define the test URL" do
+      assert_not_nil $test_read_me_url
+    end
+
+    context "for HTTP URLs" do
+      should "download file via HTTP from local TestServlet" do
+        assert Downspout.fetch_url( $test_read_me_url )
+      end
+    end
+
+    should "fail with Curb error in case of excessive redirects" do
+      two_deep_url = "http://127.0.0.1:8899/two/deep?curby"
+
+      assert_raise Curl::Err::TooManyRedirectsError do
+        dl = Downspout.fetch_url( two_deep_url )
+      end
+    end
+
+    context "with Curb disabled" do
+      setup do
+        Downspout::Config.disable_curb!
+      end
+
+      should "fail with Downspout error in case of excessive redirects" do
+        two_deep_url = "http://127.0.0.1:8899/two/deep?no-curb"
+  
+        assert_raise Downspout::ExcessiveRedirects do
+          dl = Downspout.fetch_url( two_deep_url )
+        end
+
+      end
+
+      teardown do
+        Downspout::Config.enable_curb!
+      end
+    end
+
+    teardown do
+     Downspout.clean_download_dir( 0 )
+    end
+
+  end
+
+  should "fail due to excessive redirects" do
+    @obj = Downspout::Downloader.new( :url => "http://127.0.0.1:8899/two/deep?over-draft" )
+
+    assert_raise Downspout::ExcessiveRedirects do
+     @obj.send('net_http_fetch', @obj.url, 1 )  # uses send to bypass Curb and force Net::HTTP
+    end
+  end
+
+  should "fail due to DNS error" do
+    @obj = Downspout::Downloader.new( :url => "http://fu.man.chu/deep/nested/resource?over-draft" )
+
+    assert_raise SocketError do
+     @obj.send('net_http_fetch', @obj.url) # uses send to bypass Curb and force Net::HTTP
+    end
+  end
+
+  should "download to custom path" do
+    gfx_path = File.join( Test::App.root, "tmp", "download-test", "image.png" )
+    FileUtils.mkdir_p( File.dirname( gfx_path ) )
+    
+    dl = Downspout.download_url_to_path( $test_image_url, gfx_path )
+  end
+
+end

data/test/servlet.rb

@@ -0,0 +1,32 @@
+require 'test_servlet'
+
+ws_root = File.expand_path( File.dirname( File.dirname( __FILE__ ) ) )
+
+ws_app = WEBrick::HTTPServer.new(:Port => 8899,
+     :DocumentRoot => @test_ws_root,
+     :Logger => Log.new(nil, BasicLog::WARN), # Log/BasicLog from WEBrick - reduces spam in tests
+#     :Logger => $logger,
+     :AccessLog => [])
+
+ws_app.mount(TestServlet.path, TestServlet)
+
+two_deep_proc = Proc.new { |req, resp|
+  resp.body = '2-deep redirector proc mounted on #{req.script_name}'
+  resp.set_redirect( HTTPStatus::MovedPermanently, '/one/deep')
+}
+
+ws_app.mount('/two/deep/', HTTPServlet::ProcHandler.new(two_deep_proc) )
+
+redir_proc = Proc.new { |req, resp|
+  resp.body = 'redirector proc mounted on #{req.script_name}'
+  resp.set_redirect( HTTPStatus::MovedPermanently, '/READ_ME.rdoc')
+}
+
+ws_app.mount('/one/deep/', HTTPServlet::ProcHandler.new(redir_proc) )
+
+ws_thread = Thread.new { ws_app.start }    
+    
+read_me_url = "#{TestServlet.url}/READ_ME.doc"
+
+puts "Request #{read_me_url}"
+

data/test/test_helper.rb

@@ -0,0 +1,31 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'mocha'
+
+begin
+  require 'redgreen'
+rescue LoadError
+  # nice to have, aesthetic, not functional
+end
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+class Test::Unit::TestCase
+end
+
+module Test
+  module App
+    def self.root
+      return File.expand_path( File.dirname( File.dirname( __FILE__ ) ) )
+    end
+  end
+end
+
+# test_logger must be loaded before downspout
+require 'test_logger'
+require 'test_servlet'
+
+# The object of our affections
+require 'downspout'

data/test/test_logger.rb

@@ -0,0 +1,28 @@
+require 'logger'
+
+if !defined?( RAILS_DEFAULT_LOGGER ) then
+  root_dir = File.dirname( File.dirname(__FILE__) )
+  test_log_path = File.join( root_dir, 'tmp', 'log', 'test.log')
+
+  begin
+    require 'fileutils'
+    # creates the log directory if possible
+    FileUtils.mkdir_p( File.dirname( test_log_path ) )
+  
+    # touching the file ensures it is writable
+    FileUtils.touch( test_log_path )
+  
+    if File.exist?( File.dirname( test_log_path ) ) then
+      $logger = Logger.new( test_log_path )
+    else
+      $logger = Logger.new( STDERR )
+    end
+  
+  rescue Exception => e
+    # ignore the error and try to carry on...  
+    $logger = Logger.new( STDERR )
+    $logger.warn "Failed to create log file due to exception : #{e}"
+  end
+end
+
+$logger.level = Logger::DEBUG