downspout 0.2.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+Copyright (c) <2011> <Phi.Sanders>
+
+Licensed under the MIT license
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,89 @@
+= Downspout
+
+Downspout is an easy-to-use ruby library for downloading files from URLs, supporting both HTTP & FTP protocols. 
+
+= Examples
+== HTTP
+
+=== Fetch files with ease :
+
+The 'fetch_url' method accepts an URL and on a successful download returns an Downloader object. that includes the path method to the resulting temporary file.
+
+  some_url = "http://www-host.domain.tld/folder/some_image_file.gif"
+
+  dl = Downspout.fetch_url( some_url )
+
+  dl.path
+  => "/tmp/downloads/downspout-20110203-59488-1run8k2-0/some_image_file.gif"
+
+=== Download to a Specified Path
+
+By default, Downspout stores the download in an automatically generated temp file, but you can also specify the path you wish the file to go to :
+
+  some_url = "http://www-host.domain.tld/folder/hipster_hacker.png"
+
+  some_path = "/my/custom/downloads/folder/my_hero.png"
+
+  Downspout.download_url_to_path( some_url, some_path )
+
+  => "/my/custom/downloads/folder/my_hero.png"
+
+  Of course, you are responsible for ensuring the specified path is usable!
+
+== FTP
+
+Downspout also supports FTP, but that usually requires authorization, so you should do a small amount of configuration, to map URLs by host.
+
+  Downspout::Config.add_credential( :host => "ftp-host.domain.tld",
+    :user_name => "luser",
+    :pass_word => "pAzw0rd",
+    :scheme => 'ftp'
+    )
+              
+  dl = Downspout.fetch_url("ftp://ftp-host.domain.tld/folder/path/archive.zip")
+
+  dl.path  
+  => "/tmp/downloads/downspout-20110203-59843-109cmu0-7/archive.zip"
+
+Downspout will now attempt to create a credential on the fly, in the case where user info is embedded in the FTP URL, as in :
+
+  dl = Downspout.fetch_url("ftp://account:s3cr3t@ftp-host.domain.tld/file.zip")
+
+  dl.path
+  => "/tmp/downloads/downspout-20110203-62579-109cmu0-7/file.zip"
+
+The main advantage of setting up credentials is to avoid repeatedly communicating the secret information from system to system, especially permitting the download system to receive secure URLs via service calls or text files.
+
+*Caveat* : Credentials are only stored in-memory at this time, so should be defined via application initializers. One strategy is to use an obfuscated yaml file for the sensitive info.
+
+== Storage and Clean-up
+
+By default, files are stored in a "downloads" directory under "/tmp" but this can (and should) be changed via Config :
+
+
+  Downspout::Config.tmp_dir = "/home/luser/downloads/"
+  
+You should clean up this folder periodically, by calling 
+
+  Downspout.clean_download_dir( minutes )
+
+which takes an integer for how many minutes old a file should be in order to be removed. The default is 30. The adequate delay for your application depends on the volume of downloads and your disk capacity.
+
+
+== Contributing to downspout
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Phi.Sanders. See LICENSE.txt for further details.
+
+== Thanks
+
+Thanks to my employer, VitalSource Technologies (http://www.vitalsource.com), for permission to release as open source.

data/Rakefile

@@ -0,0 +1,51 @@
+# Rakefile
+require 'rubygems'
+
+$LOAD_PATH.unshift('lib')
+
+require 'rake'
+require 'jeweler'
+
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... 
+  # see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "downspout"
+  gem.homepage = "http://github.com/sci-phi/downspout"
+  gem.license = "MIT"
+  gem.summary = %Q{Downspout is an easy-to-use library for downloading files from given URLs.}
+  gem.description = %Q{Downspout is an easy-to-use library for downloading files from given URLs. HTTP downloads can use either Net::HTTP, or libcurl (via the Curb gem)}
+  gem.email = "phi.sanders@sciphi.me"
+  gem.authors = ["Phi.Sanders"]
+  gem.files = FileList['lib/**/*.rb', '[A-Z]*', 'test/**/*'].to_a
+  gem.add_development_dependency "jeweler", "~> 1.5.2"
+  gem.add_development_dependency "shoulda", ">= 0"
+  gem.add_development_dependency "rcov", ">= 0"
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rdoc'
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "downspout #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/VERSION

@@ -0,0 +1 @@
+0.2.2

data/lib/downspout/base.rb

@@ -0,0 +1,131 @@
+module Downspout
+
+  class Base
+    class << self
+    end
+  end
+
+  class UnsupportedScheme < StandardError
+  end
+
+  class ExcessiveRedirects < StandardError
+  end
+
+  class BadURL < StandardError
+  end
+
+=begin rdoc
+  Download a file from a given URL to a given Path on the local system
+  The path is optional and will default to a generated temporary file
+=end
+  def self.download_url_to_path( some_url, some_path = nil )
+    $logger.debug("downspout | download_url_to_path |  URL : #{some_url}")
+    $logger.debug("downspout | download_url_to_path |  Download Path : #{some_path}")
+
+    begin
+      d = Downspout::Downloader.new( :url => some_url, :path => some_path )
+    rescue Exception => e
+      $logger.error("downspout | download_url_to_path | Exception : '#{e}'")
+      return nil if e.class == Downspout::UnsupportedScheme
+      raise e
+    end
+
+    fetched = d.download!
+
+    if !(fetched) then
+      $logger.error("downspout | download_url_to_path |  Fetch Failed : #{d.url} ")
+      return nil
+    end
+
+    $logger.debug("downspout | download_url_to_path |  Local File : #{d.path} ")
+    return d
+  end
+
+=begin rdoc
+  Convenience method for downloading a file from an URL without specifying a path for storage.
+=end
+  def self.fetch_url( the_url )
+    return self.download_url_to_path( the_url )
+  end
+
+=begin rdoc
+  Utility method for validating a URL without initiating a download
+=end
+  def self.viable_url?( url_string )
+    $logger.info("downspout | supported_protocol? |  URL : #{url_string} ")
+
+    # remove user/password prefix if provided
+    clean_url = self.extract_credentials_from_url!( url_string )
+
+    begin
+      uri = URI.parse( clean_url )
+    rescue URI::InvalidURIError
+      $logger.warn("downspout | supported_protocol? | The format of the url is not valid : #{url_string}")
+      return false
+    end
+
+    return false unless self.supported_protocol?( uri.scheme )
+
+    # TODO : do more in-depth checks on URL validity
+
+    return true
+  end
+
+=begin rdoc
+  Utility method for checking the support for URLs of the given network protocol or 'scheme'
+=end
+  def self.supported_protocol?( some_protocol )
+    $logger.debug("downspout | supported_protocol? |  protocol : #{some_protocol} ")
+
+    protocol_string = some_protocol.to_s.upcase
+
+    return true if self.supported_protocols.include?( protocol_string )
+
+    case protocol_string
+    when "HTTP"
+      return true
+    when "URI::HTTP"
+      return true
+    when "HTTPS"
+      return true
+    when "URI::HTTPS"
+      return true
+    when "FTP"
+      return true
+    when "URI::FTP"
+      return true
+    else
+      $logger.warn("downspout | supported_protocol? | #{protocol_string} is not supported by Downspout.")
+    end
+
+    return false
+  end
+
+  private
+
+  def self.supported_protocols
+    return ["HTTP", "HTTPS", "FTP"]
+  end
+  
+  def self.extract_credentials_from_url!( some_url )
+    the_uri = URI.parse( some_url )
+
+    if the_uri.userinfo.nil? then
+      return some_url
+    end
+
+    begin
+      Downspout::Config.add_credential( :scheme => the_uri.scheme,
+        :host => the_uri.host,
+        :user_name => the_uri.user,
+        :pass_word => the_uri.password
+        )
+    ensure
+      the_uri.user = nil
+      the_uri.password = nil
+    end
+    
+    return the_uri.to_s
+  end
+
+end

data/lib/downspout/config.rb

@@ -0,0 +1,77 @@
+module Downspout
+
+  class Config
+    # Default Values
+    @@tmp_dir = "/tmp/downloads/"
+    @@network_enabled = true
+    @@credentials = []
+    @@curb_allowed = true
+    @@curb_enabled = true
+
+    def self.tmp_dir
+      return @@tmp_dir
+    end
+
+    def self.tmp_dir=( some_path )
+      @@tmp_dir = some_path
+    end
+
+    def self.credentials
+      return @@credentials
+    end
+
+    def self.network_enabled?
+      return @@network_enabled
+    end
+
+    def self.disable_networking!
+      @@network_enabled = false
+      return !(@@network_enabled)
+    end
+
+    def self.enable_networking!
+      @@network_enabled = true
+    end
+
+    def self.curb_available?
+      begin
+        require 'curb'
+        return true
+      rescue LoadError
+        return false
+      end
+    end
+
+    def self.use_curb?
+      @@curb_enabled
+    end
+
+    def self.enable_curb!
+      if self.curb_available? then
+        @@curb_enabled = true
+      else
+        @@curb_enabled = false
+      end
+    end
+
+    def self.disable_curb!
+      $logger.debug("downspout | config | disable_curb! | will fall back to Net/HTTP.")
+      @@curb_enabled = false
+    end
+
+    def self.add_credential( options = nil )
+      return nil unless options && options.respond_to?(:keys)
+      options = {:scheme => 'ftp'}.merge!( options )
+
+      c = Credential.new( options )
+
+      $logger.debug("downspout | config | add_credential | #{c.host}, #{c.user_name}, #{c.scheme} ")
+
+      @@credentials << c      
+
+      return c
+    end
+
+  end
+
+end

data/lib/downspout/credential.rb

@@ -0,0 +1,20 @@
+module Downspout
+
+  class Credential
+    attr_accessor :scheme
+    attr_accessor :host
+    attr_accessor :port
+    attr_accessor :user_name
+    attr_accessor :pass_word
+
+    def initialize( options = nil )
+      if options && options.respond_to?(:keys) then
+        options.each do |key, value|
+          self.send("#{key}=", value) if self.respond_to?("#{key}=")
+        end
+      end
+    end
+
+  end
+
+end