downspout 0.2.2

data/test/test_servlet.rb

@@ -0,0 +1,42 @@
+require 'webrick'
+include WEBrick
+
+class TestServlet < WEBrick::HTTPServlet::AbstractServlet
+  #
+  # Simple server for integration tests
+  #
+
+  def self.port
+    8899
+  end
+
+  def self.path
+    '/'
+  end
+
+  def self.url
+    "http://127.0.0.1:#{port}#{path}"
+  end
+
+  def respond_with(method,req,res)
+    res.body = method.to_s
+    res['Content-Type'] = "text/plain"
+  end
+
+  def do_GET(req,res)
+    respond_with(:GET,req,res)
+  end
+
+  def do_POST(req,res)
+    respond_with(:POST,req,res)
+  end
+
+  def do_PUT(req,res)
+    respond_with(:PUT,req,res)
+  end
+
+  def do_DELETE(req,res)
+    respond_with(:DELETE,req,res)
+  end
+
+end

data/test/unit/base_test.rb

@@ -0,0 +1,124 @@
+require 'test_helper'
+
+class DownspoutTest < Test::Unit::TestCase
+  context "Downspout" do
+    context "Base" do
+
+      should "respond to supported_protocol?" do
+        assert Downspout.respond_to?("supported_protocol?")
+      end
+
+      should "respond to supported_protocols" do
+        assert Downspout.respond_to?("supported_protocols")
+      end
+
+      should "respond to viable_url?" do
+        assert Downspout.respond_to?("viable_url?")
+      end
+
+      should "respond to download_url_to_path for backwards compatibility" do
+        assert Downspout.respond_to?(:download_url_to_path)
+      end
+
+      should "respond to fetch_url" do
+        assert Downspout.respond_to?(:download_url_to_path)
+      end
+    end
+    
+    context "protocols" do
+      
+      should "not support SFTP" do
+        assert !( Downspout.supported_protocol?( "sftp" ) )
+      end
+
+      should "not support AFP" do
+        assert !( Downspout.supported_protocol?( "afp" ) )
+      end
+
+      should "not support SCP" do
+        assert !( Downspout.supported_protocol?( "scp" ) )
+      end
+      
+      should "support all secret herbs and spices" do
+        assert_not_nil Downspout.supported_protocols
+        assert_equal Array, Downspout.supported_protocols.class
+        assert_equal 3, Downspout.supported_protocols.size
+        $logger.info "List of supported protocols : #{Downspout.supported_protocols.join(", ")}"
+      end
+
+      should "support HTTP" do
+        assert Downspout.supported_protocol?( "http" )
+      end
+
+      should "support HTTPS" do
+        assert Downspout.supported_protocol?( "https" )
+      end
+
+      should "support URI::HTTP" do
+        assert Downspout.supported_protocol?( URI::HTTP )
+      end
+
+      should "support URI::HTTPS" do
+        assert Downspout.supported_protocol?( URI::HTTPS )
+      end
+
+      should "support FTP" do
+        assert Downspout.supported_protocol?( "ftp" )
+      end
+
+      should "support URI::FTP" do
+        assert Downspout.supported_protocol?( URI::FTP )
+      end
+
+    end
+
+    context "URLs" do
+      context "which are bad" do
+        should "be rejected with unknown protocols" do
+          assert !( Downspout.viable_url?( "protocol://host.domain.tld/resource/file.format" ) )
+        end
+
+        should "be rejected with SCP protocol" do
+          assert !( Downspout.viable_url?( "scp://host.domain.tld/resource/file.format" ) )
+        end
+
+        should "be rejected with SFTP protocol" do
+          assert !( Downspout.viable_url?( "sftp://host.domain.tld/resource/file.format" ) )
+        end
+
+        should "be rejected with FTPS protocol" do
+          assert !( Downspout.viable_url?( "ftps://host.domain.tld/resource/file.format" ) )
+        end
+
+      end
+
+      context "which are good" do
+        should "be accepted with HTTP protocol" do
+          assert Downspout.viable_url?( "http://host.domain.tld/resource/file.format" )
+        end
+
+        should "be accepted with HTTPS protocol" do
+          assert Downspout.viable_url?( "https://host.domain.tld/resource/file.format" )
+        end
+
+        should "be accepted to PDF file" do
+          assert Downspout.viable_url?( "http://host.domain.tld/resource/file.pdf" )
+        end
+
+        should "be accepted to PHP file with arbitrary parameters" do
+          assert Downspout.viable_url?( "http://host.domain.tld/resource/file.php?doc_id=A1B2C3&vendor=xyz" )
+        end
+
+        should "be accepted with FTP protocol" do
+          assert Downspout.viable_url?( "ftp://host.domain.tld/resource/file.format" )
+        end
+
+        should "be accepted with FTP protocol containing user & password" do
+          assert Downspout.viable_url?( "ftp://uzer:passw0rd@host.domain.tld/resource/file.format" )
+        end
+
+      end
+    end
+
+  end
+end

data/test/unit/config_test.rb

@@ -0,0 +1,101 @@
+require 'test_helper'
+
+class ConfigTest < Test::Unit::TestCase
+  context "Downspout" do
+    context "Config" do
+
+      should "respond to tmp_dir" do
+        assert Downspout::Config.respond_to?(:tmp_dir)
+      end
+
+      should "default to '/tmp/downloads/' for Downspout::Config#tmp_dir" do
+        assert_equal "/tmp/downloads/", Downspout::Config.tmp_dir
+      end
+
+      should "respond to Network Enabled" do
+        assert Downspout::Config.respond_to?("network_enabled?")
+      end
+
+      should "default to Network Enabled" do
+        assert Downspout::Config.network_enabled?
+      end
+
+      should "respond to Disable Networking" do
+        assert Downspout::Config.respond_to?("disable_networking!")
+      end
+
+      should "support disabling network operations" do
+        assert Downspout::Config.network_enabled?
+        assert Downspout::Config.disable_networking!
+        assert !(Downspout::Config.network_enabled?)
+      end
+
+      should "respond to Enable Networking" do
+        assert Downspout::Config.respond_to?("enable_networking!")
+      end
+
+      should "detect whether Curb is available" do
+        assert Downspout::Config.curb_available?
+      end
+
+      should "enable Curb if library is available" do
+        assert Downspout::Config.curb_available?
+        assert Downspout::Config.enable_curb!
+      end
+
+      should "support enabling network operations" do
+        assert !(Downspout::Config.network_enabled?)
+        assert Downspout::Config.enable_networking!
+        assert Downspout::Config.network_enabled?
+      end
+
+      context "Host-Based Credentials" do
+        should "respond to Credentials" do
+          assert Downspout::Config.respond_to?(:credentials)
+          assert_not_nil Downspout::Config.credentials
+        end
+  
+        should "support adding Credentials" do
+          assert Downspout::Config.respond_to?("add_credential")        
+        end
+
+        should "add FTP Credential" do
+          assert_equal 0, Downspout::Config.credentials.size
+
+          ftp_url = "ftp://ftp-intake.vitalbook.com/hammer.rb"
+          ftp_host = URI.parse( ftp_url ).host
+          
+          Downspout::Config.add_credential( :scheme => 'ftp', 
+            :host => ftp_host,
+            :user_name => "deploy",
+            :pass_word => "m@keitRUUN!"
+            )
+
+          assert_equal 1, Downspout::Config.credentials.size
+
+          assert_equal ftp_host, Downspout::Config.credentials.first.host
+        end
+
+      end
+
+    end
+  end
+
+  context "with stubbed-out curb detection" do
+    setup do
+      Downspout::Config.stubs('curb_available?').returns(false)    
+    end
+
+    should "fail to enable Curb when library is unavailable" do
+      assert !(Downspout::Config.curb_available?)
+      assert !(Downspout::Config.enable_curb!)
+    end
+
+    teardown do
+      # this is pretty heinous...
+      Downspout::Config.stubs('curb_available?').returns(true)
+      Downspout::Config.enable_curb!
+    end
+  end
+
+end

data/test/unit/credential_test.rb

@@ -0,0 +1,31 @@
+require 'test_helper'
+
+class CredentialTest < Test::Unit::TestCase
+  context "Downspout" do
+    context "Credential" do
+
+      should "respond to scheme" do
+        assert Downspout::Credential.new.respond_to?(:scheme)
+      end
+
+      should "respond to host" do
+        assert Downspout::Credential.new.respond_to?(:host)
+      end
+
+      should "respond to port" do
+        assert Downspout::Credential.new.respond_to?(:port)
+      end
+
+      should "respond to user_name" do
+        assert Downspout::Credential.new.respond_to?(:user_name)
+      end
+
+      should "respond to pass_word" do
+        assert Downspout::Credential.new.respond_to?(:pass_word)
+      end
+
+    end
+
+  end
+
+end

data/test/unit/downloader_test.rb

@@ -0,0 +1,211 @@
+require 'test_helper'
+
+class DownloaderTest < Test::Unit::TestCase
+
+  context "Downspout" do
+    context "Downloader" do
+
+      should "respond to URL" do
+        assert Downspout::Downloader.new.respond_to?(:url)
+      end
+
+      should "respond to URI" do
+        assert Downspout::Downloader.new.respond_to?(:uri)
+      end
+
+      should "respond to Scheme" do
+        assert Downspout::Downloader.new.respond_to?(:scheme)
+      end
+
+      should "respond to Path" do
+        assert Downspout::Downloader.new.respond_to?(:path)
+      end
+
+      should "respond to Basename" do
+        assert Downspout::Downloader.new.respond_to?(:basename)
+      end
+
+      should "respond to Response" do
+        assert Downspout::Downloader.new.respond_to?(:response)
+      end
+
+      should "respond to Response Headers" do
+        assert Downspout::Downloader.new.respond_to?(:response_headers)
+      end
+
+      should "respond to download!" do
+        assert Downspout::Downloader.new.respond_to?("download!")
+      end
+
+      should "support explicitly using Net:::HTTP for downloads" do
+        assert Downspout::Config.curb_available?
+
+        dlx = Downspout::Downloader.new()
+        assert dlx.use_curb?
+
+        dlx.disable_curb!
+
+        assert !(dlx.use_curb?)
+        assert dlx.use_net_http?
+      end
+
+      should "default to using Curb for downloads when available" do
+        assert Downspout::Config.curb_available?
+
+        dlx = Downspout::Downloader.new
+        assert dlx.use_curb?
+        assert !(dlx.use_net_http?)
+      end
+
+      should "support configuration to switch default to Net:::HTTP for downloads" do
+        assert Downspout::Config.curb_available?
+        assert Downspout::Config.use_curb?
+        begin
+          Downspout::Config.disable_curb!
+
+          assert !(Downspout::Config.use_curb?)
+          
+          dlx = Downspout::Downloader.new()
+          assert !(dlx.use_curb?)
+          assert dlx.use_net_http?
+        ensure
+          Downspout::Config.enable_curb!
+        end
+      end
+      
+      context "with HTTP URL" do
+        setup do
+          @obj = Downspout::Downloader.new( :url => "http://machine.local/downspout.test" )
+        end
+
+        should "populate URL" do
+          assert_not_nil @obj.url
+        end
+
+        should "automatically parse URI from given URL" do
+          assert_not_nil @obj.uri
+        end
+
+        should "determine download scheme" do
+          assert_not_nil @obj.scheme
+          assert_equal "http", @obj.scheme
+        end
+
+        should "populate basename" do
+          assert_not_nil @obj.basename
+          assert_equal "downspout.test", @obj.basename
+        end
+
+      end
+
+      context "with URL for unsupported scheme (protocol)" do
+        should "fail on invalid scheme (Gopher)" do
+          assert_raise Downspout::UnsupportedScheme do
+            @obj = Downspout::Downloader.new( :url => "gopher://umn.edu/" )
+            @obj.download!
+          end
+        end
+        should "fail on invalid scheme (SMB)" do
+          assert_raise Downspout::UnsupportedScheme do
+            @obj = Downspout::Downloader.new( :url => "smb://windows.microsoft.com/" )
+            @obj.download!
+          end
+        end
+      end
+
+      context "over-writing an existing file" do
+        setup do
+          dts = Time.now.utc.strftime("%Y%m%d_%H%M%S")
+          @tmp_conflict_path = File.join( Test::App.root , 'tmp', "#{dts}.tmp" )
+
+          FileUtils.touch( @tmp_conflict_path )
+          
+          @ds_url = "http://machine.local/folder/file.txt"
+
+          Downspout::Config.disable_networking!
+        end
+
+        should "remove the conflict file before the download" do
+          assert File.exist?( @tmp_conflict_path )
+          
+          @df = Downspout.download_url_to_path( @ds_url, @tmp_conflict_path )
+          # note: due to disabled networking, the download is not actually performed,
+          # but the method otherwise operates as normal
+
+          assert !( File.exist?( @tmp_conflict_path ) )
+        end
+        
+        teardown do
+          FileUtils.rm( @tmp_conflict_path) if (@tmp_conflict_path && File.exist?( @tmp_conflict_path ))
+          Downspout::Config.enable_networking!
+        end
+      end
+
+    end
+
+    context "faking net/http response" do
+      should "map 302 code to Found" do
+        assert_equal Net::HTTPFound, Net::HTTPResponse.send('response_class','302')
+      end
+      should "map 301 code to Moved" do
+        assert_equal Net::HTTPMovedPermanently, Net::HTTPResponse.send('response_class','301')
+      end
+    end
+    
+    context "with FTP URL" do
+      setup do
+        @obj = Downspout::Downloader.new( :url => "ftp://ftp.machine.local/directory/image.jpg" )
+      end
+
+      should "populate URL" do
+        assert_not_nil @obj.url
+      end
+
+      should "automatically parse URI from given URL" do
+        assert_not_nil @obj.uri
+      end
+
+      should "determine download scheme" do
+        assert_not_nil @obj.scheme
+        assert_equal "ftp", @obj.scheme
+      end
+
+      should "populate basename" do
+        assert_not_nil @obj.basename
+        assert_equal "image.jpg", @obj.basename
+      end
+
+    end
+
+  end
+
+  should 'download google index page via curb' do
+    g = Downspout.fetch_url( 'http://www.google.com/' )
+
+    assert_not_nil g
+    assert File.exist?( g.path )
+    assert_equal Downspout::Downloader, g.class
+    assert g.response.is_a?(Net::HTTPResponse)
+    assert_equal Net::HTTPOK, g.response.class
+    assert g.response_headers.keys.include?('Content-Type')
+  end
+
+  should 'download google index page via net/http' do
+    g = Downspout::Downloader.new( :url => 'http://www.google.com/' )
+    assert g.use_curb?
+
+    g.disable_curb!
+
+    assert g.use_net_http?
+
+    g.download!
+
+    assert_not_nil g
+    assert File.exist?( g.path )
+    assert_equal Downspout::Downloader, g.class
+    assert g.response.is_a?(Net::HTTPResponse)
+    assert_equal Net::HTTPOK, g.response.class
+    assert g.response_headers.keys.include?('content-type') # Note case difference for Net/HTTP vs Curb
+  end
+
+end

data/test/unit/tmp_file_test.rb

@@ -0,0 +1,32 @@
+require 'test_helper'
+
+class TmpFileTest < Test::Unit::TestCase
+  context "Downspout" do
+    context "Tmpfile" do
+      setup do
+        @explicit_name = "explicit.txt"
+        @explicit_path = File.join( Downspout::Config.tmp_dir, @explicit_name )
+        FileUtils.rm( @explicit_path ) if File.exist?( @explicit_path )
+      end
+
+      should "work similar to Ruby tempfile" do
+        jltf = Downspout::Tmpfile.new()
+        assert jltf
+        assert jltf.path =~ /#{Downspout::Config.tmp_dir}/
+      end
+      
+      should "create tempfile in configured directory" do
+        assert Downspout::Tmpfile.new.path =~ /#{Downspout::Config.tmp_dir}/
+      end
+
+      should "create file with given basename in configured directory" do       
+        dstf = Downspout::Tmpfile.new( :name => @explicit_name )
+
+        assert_equal @explicit_name, File.basename( dstf.path )
+
+        assert dstf.path =~ /#{Downspout::Config.tmp_dir}/
+      end
+      
+    end
+  end
+end

data/test/watchr.rb

@@ -0,0 +1,17 @@
+# Watchr 'Auto-Test' Config
+
+$module_name = 'downspout'
+
+def lib_trigger(match)
+  test_name = match.gsub('lib/','test/test_')
+  system("ruby -I'test' #{test_name}")
+end
+
+def unit_trigger(match)
+  test_name = match.gsub('lib','test').gsub( $module_name, 'unit' ).gsub('.rb','_test.rb')
+  system("ruby -I'test' #{test_name}")
+end
+
+watch( 'lib/.*\.rb' )  {|md| lib_trigger( md[0] ) }
+watch( "lib/#{$module_name }/.*\.rb" )  {|md| unit_trigger( md[0] ) }
+watch( 'test/.*_test\.rb' )  {|md| system("ruby -Itest #{md[0]}") }

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification 
+name: downspout
+version: !ruby/object:Gem::Version 
+  hash: 19
+  prerelease: 
+  segments: 
+  - 0
+  - 2
+  - 2
+  version: 0.2.2
+platform: ruby
+authors: 
+- Phi.Sanders
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-29 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: jeweler
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 1
+        - 5
+        - 2
+        version: 1.5.2
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: shoulda
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rcov
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+description: Downspout is an easy-to-use library for downloading files from given URLs. HTTP downloads can use either Net::HTTP, or libcurl (via the Curb gem)
+email: phi.sanders@sciphi.me
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.rdoc
+files: 
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/downspout.rb
+- lib/downspout/base.rb
+- lib/downspout/config.rb
+- lib/downspout/credential.rb
+- lib/downspout/downloader.rb
+- lib/downspout/logger.rb
+- lib/downspout/tmp_file.rb
+- test/downspout_test.rb
+- test/fixtures/ruby.png
+- test/servlet.rb
+- test/test_helper.rb
+- test/test_logger.rb
+- test/test_servlet.rb
+- test/unit/base_test.rb
+- test/unit/config_test.rb
+- test/unit/credential_test.rb
+- test/unit/downloader_test.rb
+- test/unit/tmp_file_test.rb
+- test/watchr.rb
+has_rdoc: true
+homepage: http://github.com/sci-phi/downspout
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.4.2
+signing_key: 
+specification_version: 3
+summary: Downspout is an easy-to-use library for downloading files from given URLs.
+test_files: 
+- test/downspout_test.rb
+- test/servlet.rb
+- test/test_helper.rb
+- test/test_logger.rb
+- test/test_servlet.rb
+- test/unit/base_test.rb
+- test/unit/config_test.rb
+- test/unit/credential_test.rb
+- test/unit/downloader_test.rb
+- test/unit/tmp_file_test.rb
+- test/watchr.rb