schiphol 0.9

data/LICENSE

@@ -0,0 +1,16 @@
+Schiphol - A file downloader script for Ruby.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Author: Louis-Antoine Mullie (louis.mullie@gmail.com). Copyright 2011-12.

data/README.md

@@ -0,0 +1,45 @@
+## Schiphol
+
+... is a smart downloader script for Ruby, with
+
+- Automatic file type resolution w/ MIMEs.
+- Progress bar for downloads, with ETA.
+- Downloads and extracts ZIP archives.
+- Automatically retry downloads N times.
+
+## Install
+
+    gem install schiphol
+
+## Usage
+
+**Basic**
+
+It can't get simpler.
+
+```ruby
+require 'schiphol'
+
+Schiphol.download('http://www.url.com/path/to/file.html')
+```
+
+**Advanced**
+
+The options shown are the default values.
+
+```ruby
+require 'schiphol'
+
+Schiphol.download(
+  'http://www.url.com/path/to/file.html',
+  :download_folder => './my_downloads',
+  :target_directory => '',
+  :show_progress => true,
+  :rectify_extensions => false,
+  :max_tries => 3
+)
+```
+
+## License
+
+This software is released under the GPL.

data/lib/schiphol/mimes.rb

@@ -0,0 +1,3 @@
+class Schiphol
+    MIMETypes = {"application/pdf"=>:pdf, "application/x-pdf"=>:pdf, "application/acrobat"=>:pdf, "applications/vnd.pdf"=>:pdf, "text/pdf"=>:pdf, "text/x-pdf"=>:pdf, "application/msword,\n    application/doc"=>:doc, "appl/text"=>:doc, "application/vnd.msword"=>:doc, "application/vnd.ms-word"=>:doc, "application/winword"=>:doc, "application/word"=>:doc, "application/x-msw6"=>:doc, "application/x-msword"=>:doc, "text/plain"=>:txt, "text/html"=>:html, "application/xhtml+xml"=>:html, "text/xml"=>:xml, "application/xml"=>:xml, "application/x-xml"=>:xml, "application/abiword"=>:abw, "image/gif"=>:gif, "image/x-xbitmap"=>:gif, "image/gi_"=>:gif, "image/jpeg"=>:jpeg, "image/jpg"=>:jpeg, "image/jpe_"=>:jpeg, "image/pjpeg"=>:jpeg, "image/vnd.swiftview-jpeg"=>:jpeg, "image/png"=>:png, "application/png"=>:png, "application/x-png"=>:png }
+end

data/lib/schiphol.rb

@@ -0,0 +1,241 @@
+# **Schiphol: downloader script for Ruby.**
+
+# - Automatic file type resolution w/ MIMEs.
+# - Progress bar for downloads, with ETA.
+# - Downloads and extracts ZIP archives.
+# - Automatically retry downloads N times.
+class Schiphol
+
+  VERSION = '0.9'
+  
+  #* External dependencies *#
+  
+  # Require net/http to download files.
+  require 'net/http'
+  # Require fileutils to move files around.
+  require 'fileutils'
+  # Require rubyzip to unzip packages.
+  require 'zip/zip'
+  # Require progressbar to track progress.
+  require 'progressbar'
+  
+  #* Internal dependencies *#
+  require 'schiphol/mimes'
+  
+  #* Default runtime options *#
+  
+  DefaultOptions = {
+    # The main folder for downloaded files.
+    :download_folder => './downloads',
+    # A directory inside self.downloads in
+    # which the file will be downloaded.
+    # When empty, files will be downloaded
+    # directly into self.downloads.
+    :target_directory => '',
+    # Whether to show a progress bar or not.
+    :show_progress => true,
+    # If set to true, downloaded files bear
+    # the appropriate extension for their 
+    # MIME type rather than the original
+    # web file's extension.
+    :rectify_extensions => false,
+    # Maximal number of times to try.
+    :max_tries => 3
+  }
+  
+  #* Public static methods *#
+  
+  # Download a file into destination, and return
+  # the path to the downloaded file. If the filename 
+  # is nil, it will set the default filename to 'top'.
+  def self.download(url, options = {})
+    
+    # Get the default options that aren't set.
+    options = DefaultOptions.merge(options)
+   
+    # Get the folder we're downloading to.
+    path = get_or_create_path(options)
+    
+    # Get the parsed URI from the URL.
+    uri = ::URI.parse(url)
+    
+    # Get the filename.
+    dname, fname = get_dname_fname(uri)
+
+    # Globalize number of tries for this download.
+    tries = 0
+    
+    # Globalize the scope of our file handler.
+    file = nil
+    
+    begin
+
+      # Start an HTTP server to download.
+      Net::HTTP.start(uri.host) do |http|
+        
+        # Use SSL if appropriate based on the scheme.
+        http.use_ssl = true if uri.scheme == 'https'
+        
+        # Start a GET request to the server.
+        http.request_get(uri.path) do |response|
+          
+          # Get filename and rectify extension.
+          if options[:rectify_extensions]
+            fname = rectify_extensions(
+            uri.path, response.content_type)
+          end
+          
+          # Setup progress bar.
+          if options[:show_progress]
+            bar = create_bar(url, 
+            response.content_length)
+          end
+          
+          # Check response code was OK.
+          check_response_code(response.code)
+          
+          # Open a file to write to.
+          file = File.open("#{path}/#{fname}", 'w')
+      
+          # Write the downloaded file.
+          response.read_body do |segment|
+            # Increment the progress bar.
+            bar.inc(segment.length) if bar
+            # Write the read segment.
+            file.write(segment)
+          end
+
+          # Terminate the progresss bar.
+          bar.finish if bar
+
+        end
+
+      end
+
+      # Return the path to the download.
+      file.path.to_s
+      
+    # Attempt to retry N times.
+    rescue Exception => error
+      
+      # Retry if more tries available.
+      retry if (tries += 1) > options[:max_tries]
+      
+      # Raise exception if can't retry.
+      raise "Couldn't download #{url} " +
+      "(Max number of attempts reached). " +
+      "Error: (#{error.message})"
+      # Delete the file opened for writing.
+      file.delete
+    
+    # Ensure the file handler is closed. 
+    ensure
+      file.close unless file.nil?
+    end
+
+  end
+  
+  #* Private methods *#
+  
+  private
+  
+  # Create a progress bar w/ length.
+  def self.create_bar(url, length)
+    
+    unless length
+      warn 'Unknown file size; ETR unknown.'
+      length = 10000
+    end
+    
+    ProgressBar.new(url, length)
+    
+  end
+  
+  # Get or create download folder.
+  def self.get_or_create_path(options)
+    
+    # Path is download folder [+ directory].
+    path = File.join(
+      options[:download_folder], 
+      options[:target_directory]
+    )
+    
+    # Create path if non-existent.
+    unless FileTest.directory?(path)
+      FileUtils.mkdir(path)
+    end
+    
+    path
+    
+  end
+  
+  # Parse the directory and filename
+  # out of the path.
+  def self.get_dname_fname(uri)
+    
+    split = uri.path.split('/')
+    
+    if split.size == 1
+      return '/', split[0]
+    else
+      return File.join(
+      *split[0..-2]), split[-1]
+    end
+    
+  end
+  
+  # Rectify extension based on MIME type.
+  def self.rectify_extensions(file, t)
+    
+    fn = File.basename(file, '.*')
+    
+    ext = MIMETypes[t].to_s
+    
+    unless ext
+      raise "Don't know how to handle MIME type #{t}."
+    end
+      
+    fn + '.' + ext
+    
+  end
+  
+  # Check that response code is OK.
+  def self.check_response_code(code)
+    unless code == '200'
+      raise "Response code was not 200 , but #{code}."
+    end
+  end
+  
+  #* Unimplemented methods *#
+  
+  def download_and_extract(url, options = {})
+    raise 'Not implemented yet.'
+    unzip(download(url, options), 
+    options[:destination])
+  end
+  
+  # Decompress a ZIP archive; result will 
+  # be stored in same folder as downloaded 
+  # ZIP file, under a directory bearing the
+  # same name as the ZIP archive.
+  def self.unzip(file, options)
+    
+    raise 'Not implemented yet.'
+    f_path = ''
+    
+    Zip::ZipFile.open(file) do |zip_file|
+      zip_file.each do |f|
+        f_path = File.join(destination, f.name)
+        FileUtils.mkdir_p(File.absolute_path(File.dirname(f_path)))
+        zip_file.extract(f, f_path) unless File.exist?(f_path)
+      end
+    end
+    
+    mac_remove = File.join(dest, '__MACOSX')
+    if File.readable?(mac_remove)
+      FileUtils.rm_rf(mac_remove)
+    end
+    
+  end
+  
+end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: schiphol
+version: !ruby/object:Gem::Version
+  version: '0.9'
+  prerelease: 
+platform: ruby
+authors:
+- Louis Mullie
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-06-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.6.1
+- !ruby/object:Gem::Dependency
+  name: progressbar
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+description: ! ' Schiphol is a smart file downloader for Ruby, with automatic file
+  type resolution by MIME header, progress bar with ETR, extraction of downloaded
+  ZIP archives, and auto-retries for a preset number of times. '
+email:
+- louis.mullie@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/schiphol/mimes.rb
+- lib/schiphol.rb
+- README.md
+- LICENSE
+homepage: https://github.com/louismullie/schiphol
+licenses: []
+post_install_message: ! '********************************************************************************
+
+
+  Thank you for installing Schiphol!
+
+
+  ********************************************************************************
+
+
+'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: ! 'Schiphol: a smart file downloader for Ruby.'
+test_files: []