fetcher 0.4.0 → 0.4.1

data/README.markdown

@@ -1,7 +1,7 @@
 # fetcher - Fetch Text Documents or Binary Blobs via HTTP, HTTPS
 
-* home  :: [github.com/geraldb/fetcher](https://github.com/geraldb/fetcher)
-* bugs  :: [github.com/geraldb/fetcher/issues](https://github.com/geraldb/fetcher/issues)
+* home  :: [github.com/rubylibs/fetcher](https://github.com/rubylibs/fetcher)
+* bugs  :: [github.com/rubylibs/fetcher/issues](https://github.com/rubylibs/fetcher/issues)
 * gem   :: [rubygems.org/gems/fetcher](https://rubygems.org/gems/fetcher)
 * rdoc  :: [rubydoc.info/gems/fetcher](http://rubydoc.info/gems/fetcher)
 * forum :: [groups.google.com/group/webslideshow](https://groups.google.com/group/webslideshow)
@@ -56,7 +56,7 @@ Note: The method `get` will return a `Net::HTTPResponse` object
 ## Command Line
 
 ~~~
-fetch version 0.7.0 - Lets you fetch text documents or binary blobs via HTTP, HTTPS.
+fetch version 0.5.0 - Lets you fetch text documents or binary blobs via HTTP, HTTPS.
 
 Usage: fetch [options] URI
     -o, --output PATH                Output Path (default is '.')
@@ -83,12 +83,12 @@ that lets you create slide shows
 and author slides in plain text using a wiki-style markup language that's easy-to-write and easy-to-read
 ships with the `fetcher` gem.
 
-The [`sportdb`](https://github.com/geraldb/sport.db.ruby) gem that lets you read football (soccer) fixtures
-and more in plain text
+The [`pluto`](https://github.com/feedreader) gem that lets you build web pages
+from published web feeds
 ships with the `fetcher` gem.
 
-The [`pluto`](https://github.com/geraldb/pluto) gem that lets you build web pages
-from published web feeds
+The [`sportdb`](https://github.com/geraldb/sport.db.ruby) gem that lets you read football (soccer) fixtures
+and more in plain text
 ships with the `fetcher` gem.
 
 

data/Rakefile

@@ -8,8 +8,8 @@ Hoe.spec 'fetcher' do
   self.summary = 'fetcher - Fetch Text Documents or Binary Blobs via HTTP, HTTPS'
   self.description = summary
 
-  self.urls    = ['https://github.com/geraldb/fetcher']
-  
+  self.urls    = ['https://github.com/rubylibs/fetcher']
+
   self.author  = 'Gerald Bauer'
   self.email   = 'webslideshow@googlegroups.com'
     

data/lib/fetcher/version.rb

@@ -1,7 +1,7 @@
 
 module Fetcher
 
-  VERSION = '0.4.0'
+  VERSION = '0.4.1'
 
 end  # module Fetcher
 

data/lib/fetcher/worker.rb

@@ -14,8 +14,19 @@ module Fetcher
       if old_logger_do_not_use != nil
          puts "*** depreciated API call [Fetcher.initialize] - do NOT pass in logger; no longer required/needed; logger arg will get removed"
       end
+      
+      ### cache for conditional get (e.g. etags and last-modified headers/checks)
+      @cache = {}
+      @use_cache = false
     end
 
+    ## note: use cache[ uri ] = hash for headers+plus body+plus code(410,etc.)
+    #            cache[ uri ]
+    def clear_cache()              @cache = {};              end
+    def cache()                    @cache;                   end
+    def use_cache=(true_or_false)  @use_cache=true_or_false; end  # true|false
+    def use_cache?()               @use_cache;               end
+
 
     def get( src )
       # return HTTPResponse (code,message,body,etc.)
@@ -95,19 +106,41 @@ module Fetcher
     
         logger.debug "GET #{uri.request_uri} uri=#{uri}, redirect_limit=#{redirect_limit}"
     
-        request    = Net::HTTP::Get.new( uri.request_uri, { 'User-Agent' => "fetcher gem v#{VERSION}" } )
+        headers = { 'User-Agent' => "fetcher gem v#{VERSION}" }
+
+        if use_cache?
+          ## check for existing cache entry in cache store (lookup by uri)
+          ## todo/fix: normalize uri!!!! - how?
+          cache_entry = cache[ uri.request_uri ]
+          if cache_entry
+            logger.info "found cache entry for >#{uri.request_uri}<"
+            if cache_entry['etag']
+              logger.info "adding header If-None-Match (etag) >#{cache_entry['etag']}< for conditional GET"
+              headers['If-None-Match'] = cache_entry['etag']
+            end
+            if cache_entry['last-modified']
+              logger.info "adding header If-Modified-Since (last-modified) >#{cache_entry['last-modified']}< for conditional GET"
+              headers['If-Modified-Since'] = cache_entry['last-modified']
+            end
+          end
+        end
+
+        request = Net::HTTP::Get.new( uri.request_uri, headers )
         if uri.instance_of? URI::HTTPS
           http.use_ssl = true
           http.verify_mode = OpenSSL::SSL::VERIFY_NONE
         end
-    
+
         response   = http.request( request )  
-  
+
         if response.code == '200'
           logger.debug "#{response.code} #{response.message}"
           logger.debug "  content_type: #{response.content_type}, content_length: #{response.content_length}"
           break  # will return response
-        elsif (response.code == '301' || response.code == '302' || response.code == '303' || response.code == '307' )
+        elsif( response.code == '304' ) # -- Not Modified - for conditional GETs (using etag,last-modified)
+          logger.debug "#{response.code} #{response.message}"
+          break  # will return response
+        elsif( response.code == '301' || response.code == '302' || response.code == '303' || response.code == '307' )
           # 301 = moved permanently
           # 302 = found
           # 303 = see other

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fetcher
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-14 00:00:00.000000000 Z
+date: 2013-10-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &75519830 !ruby/object:Gem::Requirement
+  requirement: &75656580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0.6'
   type: :runtime
   prerelease: false
-  version_requirements: *75519830
+  version_requirements: *75656580
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &75537890 !ruby/object:Gem::Requirement
+  requirement: &75656270 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *75537890
+  version_requirements: *75656270
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &75536950 !ruby/object:Gem::Requirement
+  requirement: &75656000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: '3.3'
   type: :development
   prerelease: false
-  version_requirements: *75536950
+  version_requirements: *75656000
 description: fetcher - Fetch Text Documents or Binary Blobs via HTTP, HTTPS
 email: webslideshow@googlegroups.com
 executables:
@@ -62,7 +62,7 @@ files:
 - lib/fetcher/cli/runner.rb
 - lib/fetcher/version.rb
 - lib/fetcher/worker.rb
-homepage: https://github.com/geraldb/fetcher
+homepage: https://github.com/rubylibs/fetcher
 licenses:
 - Public Domain
 post_install_message: