pluto 0.8.9 → 0.8.10

data/lib/pluto.rb

@@ -13,6 +13,7 @@ require 'optparse'
 require 'fileutils'
 require 'logger'
 require 'date'
+require 'digest/md5'
 
 
 # 3rd party ruby gems/libs

data/lib/pluto/fetcher.rb

@@ -98,10 +98,10 @@ class Fetcher
 
     if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
       puts "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
+      puts "no change; request returns not modified (304); skipping parsing feed"
       return nil   # no updates available; nothing to do
     end
 
-
     feed_fetched = Time.now
 
     if response.code != '200'   # note Net::HTTP response.code is a string in ruby
@@ -113,13 +113,13 @@ class Fetcher
         http_etag:          nil,
         http_last_modified: nil,
         body:               nil,
+        md5:                nil,
         fetched:            feed_fetched 
       }
       feed_rec.update_attributes!( feed_attribs )
       return nil  #  sorry; no feed for parsing available
     end
-    
-    
+
     puts "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
 
     feed_xml = response.body
@@ -137,12 +137,28 @@ class Fetcher
     # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
     feed_xml = feed_xml.force_encoding( Encoding::UTF_8 )
     logger.debug "feed_xml.encoding.name (after): #{feed_xml.encoding.name}"
+
+    ## check for md5 hash for response.body
+
+    last_feed_md5 = feed_rec.md5
+    feed_md5 = Digest::MD5.hexdigest( feed_xml )
+    
+    if last_feed_md5 && last_feed_md5 == feed_md5
+      # not all servers handle conditional gets, so while not much can be
+      # done about the bandwidth, but if the response body is identical
+      # the downstream processing (parsing, caching, ...) can be avoided.
+      #  - thanks to planet mars -fido.rb for the idea, cheers.
       
+      puts "no change; md5 digests match; skipping parsing feed"
+      return nil   # no updates available; nothing to do
+    end
+
     feed_attribs = {
       http_code:          response.code.to_i,
       http_etag:          response.header[ 'etag' ],
       http_last_modified: response.header[ 'last-modified' ], ## note: last_modified header gets stored as plain text (not datetime)
       body:               feed_xml,
+      md5:                feed_md5,
       fetched:            feed_fetched
     }
 

data/lib/pluto/refresher.rb

@@ -25,11 +25,14 @@ class Refresher
     Action.create!( title: 'update feeds' )
     
     feeds_fetched = Time.now
-    Site.all.each do |site|
+    
+    #### - hack - use order(:id) instead of .all - avoids rails/activerecord 4 warnings
+    
+    Site.order(:id).each do |site|
       site.update_attributes!( fetched: feeds_fetched )
     end
 
-    Feed.all.each do |feed|
+    Feed.order(:id).each do |feed|
       update_feed_worker( feed )
     end
   end

data/lib/pluto/schema.rb

@@ -62,7 +62,8 @@ class CreateDb < ActiveRecord::Migration
       ## note: save last-modified header as text (not datetime) - pass through as is
       t.string   :http_last_modified   # last http header last-modified - note: save header as plain text!!! pass along in next request as-is
 
-      t.text    :body    # last http response body (complete feed!)
+      t.string   :md5       # md5 hash of body
+      t.text     :body      # last http response body (complete feed!)
 
       t.datetime :fetched    # last fetched/checked date
 

data/lib/pluto/version.rb

@@ -1,4 +1,4 @@
 
 module Pluto
-  VERSION = '0.8.9'
+  VERSION = '0.8.10'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pluto
 version: !ruby/object:Gem::Version
-  version: 0.8.9
+  version: 0.8.10
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-05 00:00:00.000000000 Z
+date: 2013-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pakman
-  requirement: &77270490 !ruby/object:Gem::Requirement
+  requirement: &74547830 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0.5'
   type: :runtime
   prerelease: false
-  version_requirements: *77270490
+  version_requirements: *74547830
 - !ruby/object:Gem::Dependency
   name: fetcher
-  requirement: &77270080 !ruby/object:Gem::Requirement
+  requirement: &74547550 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 0.4.1
   type: :runtime
   prerelease: false
-  version_requirements: *77270080
+  version_requirements: *74547550
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &77269570 !ruby/object:Gem::Requirement
+  requirement: &74547300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0.6'
   type: :runtime
   prerelease: false
-  version_requirements: *77269570
+  version_requirements: *74547300
 - !ruby/object:Gem::Dependency
   name: feedutils
-  requirement: &77268950 !ruby/object:Gem::Requirement
+  requirement: &74547000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.3.2
   type: :runtime
   prerelease: false
-  version_requirements: *77268950
+  version_requirements: *74547000
 - !ruby/object:Gem::Dependency
   name: props
-  requirement: &77268430 !ruby/object:Gem::Requirement
+  requirement: &74546740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *77268430
+  version_requirements: *74546740
 - !ruby/object:Gem::Dependency
   name: textutils
-  requirement: &77268060 !ruby/object:Gem::Requirement
+  requirement: &74546440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: 0.6.8
   type: :runtime
   prerelease: false
-  version_requirements: *77268060
+  version_requirements: *74546440
 - !ruby/object:Gem::Dependency
   name: gli
-  requirement: &77267780 !ruby/object:Gem::Requirement
+  requirement: &74546190 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 2.5.6
   type: :runtime
   prerelease: false
-  version_requirements: *77267780
+  version_requirements: *74546190
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &77267490 !ruby/object:Gem::Requirement
+  requirement: &74545900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,10 +98,10 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *77267490
+  version_requirements: *74545900
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &77267190 !ruby/object:Gem::Requirement
+  requirement: &74545650 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -109,7 +109,7 @@ dependencies:
         version: '3.3'
   type: :development
   prerelease: false
-  version_requirements: *77267190
+  version_requirements: *74545650
 description: pluto - Another Planet Generator (Lets You Build Web Pages from Published
   Web Feeds)
 email: feedreader@googlegroups.com