fetcher 0.0.1 → 0.1.0

data/Manifest.txt

@@ -5,3 +5,4 @@ Rakefile
 bin/fetch
 lib/fetcher.rb
 lib/fetcher/runner.rb
+lib/fetcher/worker.rb

data/README.markdown

@@ -10,7 +10,15 @@ TBD
 
     require 'fetcher'
     
-    Fetcher.new.copy( '/tmp/hoe.html', 'http://geraldb.github.com/rubybook/hoe.html' )
+    Fetcher.copy( 'http://geraldb.github.com/rubybook/hoe.html', '/tmp/hoe.html' )
+
+or
+
+    logger = Logger.new( STDOUT )
+    worker = Fetcher::Worker.new( logger )
+    worker.copy( 'http://geraldb.github.com/rubybook/hoe.html', '/tmp/hoe.html' )
+
+
 
 ## Install
 

data/lib/fetcher.rb

@@ -11,16 +11,23 @@ require 'pp'
 require 'logger'
 require 'optparse'
 require 'fileutils'
+require 'uri'
+require 'net/http'
+require 'net/https'
+require 'ostruct'
+require 'date'
+require 'cgi'
 
 
 # our own code
 
 require 'fetcher/runner'
+require 'fetcher/worker'
 
 
 module Fetcher
 
-  VERSION = '0.0.1'
+  VERSION = '0.1.0'
 
   # version string for generator meta tag (includes ruby version)
   def self.banner

data/lib/fetcher/runner.rb

@@ -1,13 +1,101 @@
 
 module Fetcher
 
+   def self.copy( src, dest )
+      Worker.new( Logger.new(STDOUT) ).copy( src, dest )
+   end
+
+
+  class Opts
+  
+    def initialize
+      @hash = {}
+    end
+    
+    def put( key, value )
+      @hash[ key.to_s ] = value
+    end    
+  
+    def output_path
+      @hash.fetch( 'output', '.' )
+    end
+
+  end # class Opts
+
+
   class Runner
     
-    def run(args)
-      puts "hello from Runner.run"
+    attr_reader :logger
+    attr_reader :opts
+    
+    def initialize
+      @logger       = Logger.new(STDOUT)
+      @logger.level = Logger::INFO
+      @opts         = Opts.new
     end
     
-  end
+    def run( args )
+      opt=OptionParser.new do |cmd|
+    
+        cmd.banner = "Usage: fetch [options] uri"
+    
+        cmd.on( '-o', '--output PATH', 'Output Path' ) { |s| opts.put( 'output', s ) }
+
+        # todo: find different letter for debug trace switch (use v for version?)
+        cmd.on( "-v", "--verbose", "Show debug trace" )  do
+           logger.datetime_format = "%H:%H:%S"
+           logger.level = Logger::DEBUG
+        end
+
+      usage =<<EOS
+ 
+fetch - Lets you fetch text documents or binary blobs via HTTP, HTTPS.
+
+#{cmd.help}
+
+Examples:
+  fetch http://geraldb.github.com/rubybook/hoe.html
+  fetch -o downloads http://geraldb.github.com/rubybook/hoe.html
+
+Further information:
+  http://geraldb.github.com/fetcher
+
+EOS
+ 
+        cmd.on_tail( "-h", "--help", "Show this message" ) do
+           puts usage
+           exit
+        end
+      end
+
+      opt.parse!( args )
+  
+      puts Fetcher.banner
+          
+      args.each do |arg|
+        
+        src = arg
+        uri = URI.parse( src )
+        
+        logger.debug "uri.host=<#{uri.host}>, uri.path=<#{uri.path}>"
+        
+        if uri.path == '/' || uri.path == ''
+          dest = "#{uri.host}"
+        else
+          dest = "#{uri.host}@#{uri.path.gsub( /[ \-]/, '_').gsub( /[\/\\]/, '-')}"
+        end
+        
+        ## todo: use output path option
+        
+        Worker.new( logger ).copy( src, dest )
+
+      end # each arg
+
+      puts "Done."
+      
+    end   # method run
+    
+  end # class Runner
 
 end  # module Fetcher
 

data/lib/fetcher/worker.rb

@@ -0,0 +1,96 @@
+module Fetcher
+
+  class Worker
+
+    attr_reader :logger
+
+    def initialize( logger )
+      @logger = logger
+    end
+
+
+## todo: add file protocol 
+
+    def copy( src, dest )
+      logger.debug "fetch( src: #{src}, dest: #{dest} )"
+
+      uri = URI.parse( src )
+  
+      # new code: honor proxy env variable HTTP_PROXY
+      proxy = ENV['HTTP_PROXY']
+      proxy = ENV['http_proxy'] if proxy.nil?   # try possible lower/case env variable (for *nix systems) is this necessary??
+    
+      if proxy
+        proxy = URI.parse( proxy )
+        logger.debug "using net http proxy: proxy.host=#{proxy.host}, proxy.port=#{proxy.port}"
+        if proxy.user && proxy.password
+          logger.debug "  using credentials: proxy.user=#{proxy.user}, proxy.password=****"
+        else
+          logger.debug "  using no credentials"
+        end
+      else
+        logger.debug "using direct net http access; no proxy configured"
+        proxy = OpenStruct.new   # all fields return nil (e.g. proxy.host, etc.)
+      end
+      
+      http_proxy = Net::HTTP::Proxy( proxy.host, proxy.port, proxy.user, proxy.password )
+
+      redirect_limit = 4
+      response = nil
+
+      until false
+        raise ArgumentError, 'HTTP redirect too deep' if redirect_limit == 0
+        redirect_limit -= 1
+      
+        http = http_proxy.new( uri.host, uri.port )
+    
+        logger.debug "GET #{uri.request_uri} uri=#{uri}, redirect_limit=#{redirect_limit}"
+    
+        request    = Net::HTTP::Get.new( uri.request_uri, { 'User-Agent'=> 'slideshow'} )
+        if uri.instance_of? URI::HTTPS
+          http.use_ssl = true
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+        end
+    
+        response   = http.request( request )  
+  
+        if response.code == '200'
+          logger.debug "#{response.code} #{response.message}"
+          break
+        elsif (response.code == '301' || response.code == '302' || response.code == '303' || response.code == '307' )
+          # 301 = moved permanently
+          # 302 = found
+          # 303 = see other
+          # 307 = temporary redirect
+          logger.debug "#{response.code} #{response.message} location=#{response.header['location']}"
+          newuri = URI.parse( response.header['location'] )
+          if newuri.relative?
+            logger.debug "url relative; try to make it absolute"
+            newuri = uri + response.header['location']
+          end
+          uri = newuri
+        else
+          msg = "#{response.code} #{response.message}" 
+          puts "*** error: #{msg}"
+          return   # todo: throw StandardException?  
+        end
+      end
+
+      logger.debug "  content_type: #{response.content_type}, content_length: #{response.content_length}"
+  
+      # check for content type; use 'wb' for images
+      if response.content_type =~ /image/
+        logger.debug '  switching to binary'
+        flags = 'wb'
+      else
+        flags = 'w'
+      end
+  
+      File.open( dest, flags ) do |f|
+        f.write( response.body )
+      end
+    end # method copy
+
+  end # class Worker
+
+end  # module Fetcher

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: fetcher
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: 
   segments: 
   - 0
-  - 0
   - 1
-  version: 0.0.1
+  - 0
+  version: 0.1.0
 platform: ruby
 authors: 
 - Gerald Bauer
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-06-02 00:00:00 Z
+date: 2012-06-03 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rdoc
@@ -63,6 +63,7 @@ files:
 - bin/fetch
 - lib/fetcher.rb
 - lib/fetcher/runner.rb
+- lib/fetcher/worker.rb
 homepage: http://geraldb.github.com/fetcher
 licenses: []