pluto-update 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d631c3eb45ac43efb9e2d0186270a5ff6adc0e43
-  data.tar.gz: bb4774dcb0d97bc05b3f19f05956f608d94bf18d
+  metadata.gz: 49adc91fcbc3b7df6d0ee5aed048fa0df4abb186
+  data.tar.gz: 16b22d7a9993c287f592176acd9991ac5d42e3c5
 SHA512:
-  metadata.gz: 858c5e290c621b417ab5773e3c307daed4a0c0174d00ef88729ac9a235deafc5e0c42027eaaf86220017415b9538be04c6661a7f989fedd5c87c5143c97e2fe9
-  data.tar.gz: 228ffdf0626f7455c73bc108bdc4639c5f2ff42d468238d2d6dcd308a8e0df3aead6f0f2cecf0d374a36b378b3babba0353ad10f26d5e8ca81ef4d102117302b
+  metadata.gz: c0db8f9b7dcb72009df75b4cc001fe522e579f4b55bc8292b7ab5d19f4166aead75084b0da9356444dddd00b999a372e9972abaedf9230497706db138d766f80
+  data.tar.gz: 0fc2ba83226d96ca4b9a71829079c7232322997963230a99cda0948ae4591dc4071725ddc287a2485dcdb083ec8fc1015c0399a50238ff78901f0dd39d908d00

data/Manifest.txt

@@ -3,8 +3,12 @@ Manifest.txt
 README.md
 Rakefile
 lib/pluto/update.rb
-lib/pluto/update/fetcher.rb
-lib/pluto/update/refresher.rb
-lib/pluto/update/subscriber.rb
-lib/pluto/update/updater.rb
+lib/pluto/update/feed_refresher.rb
+lib/pluto/update/site_fetcher.rb
+lib/pluto/update/site_refresher.rb
+lib/pluto/update/site_updater.rb
 lib/pluto/update/version.rb
+test/data/ruby.ini
+test/helper.rb
+test/test_refresh.rb
+test/test_site.rb

data/README.md

@@ -17,10 +17,10 @@
 ```
 title = Planet Ruby
 
-[rubyflow]
-  title  = Ruby Flow
-  link   = http://rubyflow.com
-  feed   = http://feeds.feedburner.com/Rubyflow?format=xml
+[rubylang]
+  title = Ruby Lang News
+  link  = http://www.ruby-lang.org/en/news
+  feed  = http://www.ruby-lang.org/en/feeds/news.rss
 
 [rubyonrails]
   title = Ruby on Rails Blog

data/Rakefile

@@ -18,9 +18,10 @@ Hoe.spec 'pluto-update' do
   self.history_file = 'HISTORY.md'
 
   self.extra_deps = [
-    ['pluto-models',  '>= 1.3.2'],
-    ['fetcher',       '>= 0.4.4'],
-    ['preproc',       '>= 0.1.0'],
+    ['pluto-models',      '>= 1.3.2'],
+    ['pluto-feedfetcher', '>= 0.1.0'],
+    ['fetcher',           '>= 0.4.4'],
+    ['preproc',           '>= 0.1.0'],
   ]
 
   self.licenses = ['Public Domain']

data/lib/pluto/update.rb

@@ -2,6 +2,7 @@
 
 
 require 'pluto/models'
+require 'pluto/feedfetcher'
 
 
 # more 3rd party gems
@@ -11,26 +12,30 @@ require 'preproc'    # include preprocessor
 
 # our own code
 require 'pluto/update/version'   # Note: let version always go first
-require 'pluto/update/fetcher'
-require 'pluto/update/refresher'
-require 'pluto/update/subscriber'
-require 'pluto/update/updater'
+require 'pluto/update/feed_refresher'
+require 'pluto/update/site_refresher'
+require 'pluto/update/site_fetcher'
+require 'pluto/update/site_updater'
 
 
 
 module Pluto
 
-  # todo: add alias update_site( config ) ??
-  def self.update_subscriptions( config )
-    Subscriber.new.update_subscriptions( config )
+  def self.refresh_feeds   ## refresh == fetch+parse+update
+    FeedRefresher.new.refresh_feeds
   end
 
+  def self.refresh_sites  ## refresh == fetch+parse+update
+    SiteRefresher.new.refresh_sites
+  end
+
+### convenience alias w/ update_  -- use refresh (only) - why? why not??
   def self.update_feeds
-    Refresher.new.update_feeds
+    FeedRefresher.new.refresh_feeds
   end
 
   def self.update_sites
-    Refresher.new.update_sites
+    SiteRefresher.new.refresh_sites
   end
 
 end  # module Pluto

data/lib/pluto/update/{refresher.rb → feed_refresher.rb}

@@ -3,41 +3,26 @@
 
 module Pluto
 
-class Refresher
+#######
+# note: refresh
+#  refresh will fetch feeds, parse feeds and than update feeds
+#    (e.g. update is just one operation of refresh)
 
-  include LogUtils::Logging
+class FeedRefresher
 
+  include LogUtils::Logging
   include Models
 
   def initialize
-    @worker  = Fetcher.new
+    ## @worker = FeedFetcherBasic.new  ## -- simple fetch (strategy); no cache, no cond get etc. 
+    @worker  = FeedFetcherCondGetWithCache.new
   end
 
   def debug=(value)  @debug = value;   end
   def debug?()       @debug || false;  end
 
 
-  def update_sites( opts={} )  # update all site configs
-    if debug?
-      ## turn on logging for sql too
-      ActiveRecord::Base.logger = Logger.new( STDOUT )
-      @worker.debug = true   # also pass along worker debug flag if set
-    end
-
-    start_time = Time.now
-    Activity.create!( text: "start update sites (#{Site.count})" )
-
-    #### - hack - use order(:id) instead of .all - avoids rails/activerecord 4 warnings
-    Site.order(:id).each do |site|
-      update_site_worker( site )  if site.url.present?  # note: only update if (source) url present
-    end
-
-    total_secs = Time.now - start_time
-    Activity.create!( text: "done update sites (#{Site.count}) in #{total_secs}s" )
-  end
-
-
-  def update_feeds( opts={} )  # update all feeds
+  def refresh_feeds( opts={} )  # refresh (fetch+parse+update) all feeds
     if debug?
       ## turn on logging for sql too
       ActiveRecord::Base.logger = Logger.new( STDOUT )
@@ -49,16 +34,16 @@ class Refresher
 
     #### - hack - use order(:id) instead of .all - avoids rails/activerecord 4 warnings
     Feed.order(:id).each do |feed|
-      update_feed_worker( feed )
+      refresh_feed_worker( feed )
       ### todo/fix: add catch exception in loop and log to activity log and continue w/ next feed
     end
 
     total_secs = Time.now - start_time
-    Activity.create!( text: "done update feeds (#{Site.count}) in #{total_secs}s" )
+    Activity.create!( text: "done update feeds (#{Feed.count}) in #{total_secs}s" )
   end
 
 
-  def update_feeds_for( site_key, opts={} )
+  def refresh_feeds_for( site_key, opts={} ) # refresh (fetch+parse+update) feeds for site
     if debug?
       ## turn on logging for sql too
       ActiveRecord::Base.logger = Logger.new( STDOUT )
@@ -71,35 +56,23 @@ class Refresher
     site = Site.find_by_key!( site_key )
 
     site.feeds.each do |feed|
-      update_feed_worker( feed )
+      refresh_feed_worker( feed )
     end
 
-  end # method update_feeds
+  end # method refresh_feeds_for
 
 
 private
-  def update_site_worker( site_rec )
-    site_config = @worker.site_by_rec_if_modified( site_rec )
-  
-    # on error or if http-not modified etc. skip update/processing
-    return  if site_config.nil?
-
-    subscriber = Subscriber.new
-    subscriber.debug = debug? ? true : false    # pass along debug flag
+  def refresh_feed_worker( feed_rec )
+    feed_xml = @worker.fetch( feed_rec )
 
-    subscriber.update_subscriptions_for( site_rec.key, site_config )
-  end
-
-
-  def update_feed_worker( feed_rec )
-    feed = @worker.feed_by_rec_if_modified( feed_rec )
-      
     # on error or if http-not modified etc. skip update/processing
-    return  if feed.nil?
+    return  if feed_xml.nil?
+
+    feed = FeedUtils::Parser.parse( feed_xml )
 
     ## fix/todo: reload feed_red   - fetched date updated etc.
     ##  check if needed for access to fetched date
-      
 
     ## todo/check: move feed_rec update to the end (after item updates??)
 
@@ -107,7 +80,7 @@ private
     #    generator
     #    published_at,built_at,touched_at,fetched_at
     #    summary,title2
-      
+
     ## fix:
     ## weird rss exception error on windows w/ dates
     #  e.g. /lib/ruby/1.9.1/rss/rss.rb:37:in `w3cdtf': wrong number of arguments (1 for 0) (ArgumentError)
@@ -117,9 +90,10 @@ private
 
 
     feed_rec.debug = debug? ? true : false    # pass along debug flag
-    ## fix/todo: pass debug flag as opts - debug: true|false !!!!!!
-    feed_rec.save_from_struct!( feed )  # todo: find a better name - why? why not??
 
+    ## fix/todo: pass debug flag as opts - debug: true|false !!!!!!
+    # fix/todo: find a better name - why? why not?? => use update_from_struct!
+    feed_rec.save_from_struct!( feed )  
 
     #  update  cached value last published for item
     last_item_rec = feed_rec.items.latest.limit(1).first  # note limit(1) will return relation/arrar - use first to get first element or nil from ary
@@ -130,8 +104,8 @@ private
         feed_rec.update_attributes!( last_published: last_item_rec.touched )
       end
     end
-  end  # method update_feed_worker
+  end  # method refresh_feed_worker
 
-end # class Refresher
+end # class FeedRefresher
 
 end # module Pluto

data/lib/pluto/update/site_fetcher.rb

@@ -0,0 +1,134 @@
+# encoding: utf-8
+
+
+module Pluto
+
+class SiteFetcher
+
+  include LogUtils::Logging
+  include Models   # for easy convenience access for Activity etc.
+
+  def initialize
+    @worker  = Fetcher::Worker.new
+  end
+
+  def debug=(value)  @debug = value;   end
+  def debug?()       @debug || false;  end
+
+  def fetch( site_rec )
+    ####################################################
+    # try smart http update; will update db records
+
+    site_url = site_rec.url
+    site_key = site_rec.key
+
+    ### todo/fix: normalize/unifiy feed_url
+    ##  - same in fetcher - use shared utitlity method or similar
+
+    @worker.use_cache = true
+    @worker.cache[ site_url ] = {
+      'etag'          => site_rec.http_etag,
+      'last-modified' => site_rec.http_last_modified
+    }
+
+    response = @worker.get( site_url )
+    @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
+
+    site_fetched = Time.now
+
+    ###
+    # Note: Net::HTTP will NOT set encoding UTF-8 etc.
+    # will be set to ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+    # thus, set/force encoding to utf-8
+    site_text = response.body.to_s
+    site_text = site_text.force_encoding( Encoding::UTF_8 )
+
+    if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
+
+      if site_text.index('@include')
+        ## note: if the site_text includes @include
+        ##  we must revalidate complete file hierachy(tree) for now
+        ### continue;
+        ##
+        ##  fix/todo: use ahead-of-time preprocessor ?? in the future to simplify???
+      else
+        puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
+        puts "no change; request returns not modified (304); skipping parsing site config"
+        return nil   # no updates available; nothing to do
+      end
+
+    elsif response.code != '200'   # note Net::HTTP response.code is a string in ruby
+
+      puts "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
+
+      site_attribs = {
+        http_code:          response.code.to_i,
+        http_server:        response.header[ 'server' ],
+        http_etag:          nil,
+        http_last_modified: nil,
+        fetched:            site_fetched 
+      }
+      site_rec.update_attributes!( site_attribs )
+      
+      ## add log error activity -- in future add to error log - better - why? why not?
+      Activity.create!( text: "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}" )
+
+      return nil  #  sorry; no feed for parsing available
+    else
+      # assume 200; continue w/ processing
+    end
+
+    puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
+
+    site_attribs = {
+      http_code:          response.code.to_i,
+      http_server:        response.header[ 'server' ],
+      http_etag:          response.header[ 'etag' ],
+      http_last_modified: response.header[ 'last-modified' ], ## note: last_modified header gets stored as plain text (not datetime)
+      fetched:            site_fetched
+    }
+
+    ## if debug?
+      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
+      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
+      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
+    ## end
+
+    site_rec.update_attributes!( site_attribs )
+
+
+    #################
+    ### fix: add support for http_etag cache etc.  - how??
+    ###
+    ###   use from_text( text, base: base )  !!!!!!!!
+    ###    do NOT reissue first request
+    ##
+    ##  fix: use special case/method for update_with_includes!!!
+    ##    keep it simple w/o includes (do NOT mix in one method)
+    ##  split into two methods!!!
+
+    ## retry w/ preprocesser
+    ## refetch if @include found w/ all includes included
+    if site_text.index('@include')
+      site_text = InclPreproc.from_url( site_url ).read
+    end
+
+    ## logger.debug "site_text:"
+    ## logger.debug site_text[ 0..300 ] # get first 300 chars
+
+    site_text
+
+    ###
+    ## todo/fix:
+    ### move INI.load out of this method!! - return site_text or nil
+    ##
+    ## puts "Before parsing site config >#{site_key}<..."
+    ##
+    # assume ini format for now
+    ## site_config = INI.load( site_text )
+    ## site_config
+  end
+
+end # class SiteFetcher
+
+end # module Pluto

data/lib/pluto/update/site_refresher.rb

@@ -0,0 +1,67 @@
+# encoding: utf-8
+
+
+module Pluto
+
+#######
+# note: refresh
+#  refresh will fetch site subscriptions, parse and than update the site subscriptions
+#    (e.g. update is just one operation of refresh)
+
+
+class SiteRefresher
+
+  include LogUtils::Logging
+  include Models
+
+  def initialize
+    @worker  = SiteFetcher.new
+  end
+
+  def debug=(value)  @debug = value;   end
+  def debug?()       @debug || false;  end
+
+  def refresh_sites( opts={} )  # refresh (fetch+parse+update) all site configs
+    if debug?
+      ## turn on logging for sql too
+      ActiveRecord::Base.logger = Logger.new( STDOUT )
+      @worker.debug = true   # also pass along worker debug flag if set
+    end
+
+    start_time = Time.now
+    Activity.create!( text: "start update sites (#{Site.count})" )
+
+    #### - hack - use order(:id) instead of .all - avoids rails/activerecord 4 warnings
+    Site.order(:id).each do |site|
+      refresh_site_worker( site )  if site.url.present?  # note: only update if (source) url present
+    end
+
+    total_secs = Time.now - start_time
+    Activity.create!( text: "done update sites (#{Site.count}) in #{total_secs}s" )
+  end
+
+
+private
+  def refresh_site_worker( site_rec )
+    site_text = @worker.fetch( site_rec )
+
+    # on error or if http-not modified etc. skip update/processing
+    return  if site_text.nil?
+
+    site_config = INI.load( site_text )
+
+    site_updater = SiteUpdater.new
+    site_updater.debug = debug? ? true : false    # pass along debug flag
+
+    ### todo/fix:
+    ##   allow passing in as first arg database rec!!!
+    ##   - if passed in database rec - do NOT lookup record by site_config.key!!!
+    ##       use existing key (lets you change/update key without creating a new duplicate site entry, for example)
+    ##   - or use a new method (instead of overloading arg) ?? - why? why not??
+    site_updater.update_subscriptions_for( site_rec.key, site_config )
+  end
+
+end # class SiteRefresher
+
+end # module Pluto
+

data/lib/pluto/update/{subscriber.rb → site_updater.rb}

@@ -3,10 +3,9 @@
 
 module Pluto
 
-class Subscriber
+class SiteUpdater
 
   include LogUtils::Logging
-
   include Models
 
   def debug=(value)  @debug = value;   end
@@ -156,8 +155,8 @@ class Subscriber
       site_rec.subscriptions.create!( feed_id: feed_rec.id )   
     end
 
-  end # method update_subscriptions
+  end # method update_subscriptions_for
 
-end # class Subscriber
+end # class SiteUpdater
 
 end # module Pluto

data/lib/pluto/update/version.rb

@@ -4,7 +4,7 @@
 module PlutoUpdate
 
   MAJOR = 1
-  MINOR = 4
+  MINOR = 5
   PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 

data/test/data/ruby.ini

@@ -0,0 +1,18 @@
+title  = Planet Ruby
+source = https://github.com/feedreader/pluto.update/raw/master/test/data/ruby.ini
+
+[rubylang]
+  title = Ruby Lang News
+  link  = http://www.ruby-lang.org/en/news
+  feed  = http://www.ruby-lang.org/en/feeds/news.rss
+
+[rubyonrails]
+  title = Ruby on Rails News
+  link  = http://weblog.rubyonrails.org
+  feed  = http://weblog.rubyonrails.org/feed/atom.xml
+
+[viennarb]
+  title = Vienna.rb News
+  link  = http://vienna-rb.at
+  feed  = http://vienna-rb.at/atom.xml
+

data/test/helper.rb

@@ -0,0 +1,25 @@
+## $:.unshift(File.dirname(__FILE__))
+
+
+## minitest setup
+
+require 'minitest/autorun'
+
+## our own code
+require 'pluto/update'
+
+
+LogUtils::Logger.root.level = :debug
+
+
+## some shortcuts
+Log         = LogDb::Model::Log
+Prop        = ConfDb::Model::Prop
+
+Site         = Pluto::Model::Site
+Feed         = Pluto::Model::Feed
+Item         = Pluto::Model::Item
+Subscription = Pluto::Model::Subscription
+
+
+Pluto.setup_in_memory_db

data/test/test_refresh.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_refresh.rb
+#  or better
+#     rake test
+
+require 'helper'
+
+class TestRefresh < MiniTest::Test
+
+  def setup
+    Site.delete_all
+    Feed.delete_all
+    Item.delete_all
+    Subscription.delete_all
+
+    site_text   = File.read( "#{PlutoUpdate.root}/test/data/ruby.ini")
+    site_config = INI.load( site_text )
+
+    site_updater = Pluto::SiteUpdater.new
+    site_updater.update_subscriptions_for( 'ruby', site_config )
+  end
+
+
+  def test_refresh_sites
+    Pluto.refresh_sites
+    assert true
+  end
+
+  def test_refresh_feeds
+    Pluto.refresh_feeds
+    assert true
+  end
+
+end # class TestRefresh
+

data/test/test_site.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_site.rb
+#  or better
+#     rake test
+
+require 'helper'
+
+class TestSite < MiniTest::Test
+
+  def setup
+    Site.delete_all
+    Feed.delete_all
+    Item.delete_all
+    Subscription.delete_all
+  end
+
+
+  def test_site_updater
+    site_text   = File.read( "#{PlutoUpdate.root}/test/data/ruby.ini")
+    site_config = INI.load( site_text )
+    pp site_config
+
+    assert_equal 0, Site.count
+    assert_equal 0, Feed.count
+
+    site_updater = Pluto::SiteUpdater.new
+    site_updater.update_subscriptions_for( 'ruby', site_config )
+
+    assert_equal 1, Site.count
+    assert_equal 3, Feed.count
+
+    ruby = Site.find_by_key!( 'ruby' )
+    assert_equal 'Planet Ruby', ruby.title
+    assert_equal 3, ruby.subscriptions.count
+    assert_equal 3, ruby.feeds.count
+
+    rubylang = Feed.find_by_key!( 'rubylang' )
+    assert_equal 'Ruby Lang News', rubylang.title
+    assert_equal 'http://www.ruby-lang.org/en/news', rubylang.url
+    assert_equal 'http://www.ruby-lang.org/en/feeds/news.rss', rubylang.feed_url
+  end
+
+end # class TestSite
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pluto-update
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-09 00:00:00.000000000 Z
+date: 2015-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pluto-models
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.2
+- !ruby/object:Gem::Dependency
+  name: pluto-feedfetcher
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
 - !ruby/object:Gem::Dependency
   name: fetcher
   requirement: !ruby/object:Gem::Requirement
@@ -89,16 +103,21 @@ extra_rdoc_files:
 - Manifest.txt
 - README.md
 files:
+- ".gemtest"
 - HISTORY.md
 - Manifest.txt
 - README.md
 - Rakefile
 - lib/pluto/update.rb
-- lib/pluto/update/fetcher.rb
-- lib/pluto/update/refresher.rb
-- lib/pluto/update/subscriber.rb
-- lib/pluto/update/updater.rb
+- lib/pluto/update/feed_refresher.rb
+- lib/pluto/update/site_fetcher.rb
+- lib/pluto/update/site_refresher.rb
+- lib/pluto/update/site_updater.rb
 - lib/pluto/update/version.rb
+- test/data/ruby.ini
+- test/helper.rb
+- test/test_refresh.rb
+- test/test_site.rb
 homepage: https://github.com/feedreader/pluto.update
 licenses:
 - Public Domain
@@ -125,4 +144,6 @@ rubygems_version: 2.4.2
 signing_key: 
 specification_version: 4
 summary: pluto-update - planet feed 'n' subscription updater
-test_files: []
+test_files:
+- test/test_site.rb
+- test/test_refresh.rb

data/lib/pluto/update/fetcher.rb

@@ -1,357 +0,0 @@
-# encoding: utf-8
-
-
-module Pluto
-
-class Fetcher
-
-  include LogUtils::Logging
-
-  include Models   # for easy convenience access for Activity etc.
-
-  def initialize
-    @worker  = ::Fetcher::Worker.new
-  end
-
-  def debug=(value)  @debug = value;   end
-  def debug?()       @debug || false;  end
-
-
-  def fetch_feed( url )
-    response = @worker.get( url )
-
-    ## if debug?
-      puts "http status #{response.code} #{response.message}"
-
-      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
-      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
-      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
-    ## end
-
-    xml = response.body
-
-    ###
-    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
-    # will mostly be ASCII
-    # - try to change encoding to UTF-8 ourselves
-    logger.debug "xml.encoding.name (before): #{xml.encoding.name}"
-
-    #####
-    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-
-    ## NB:
-    # for now "hardcoded" to utf8 - what else can we do?
-    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    xml = xml.force_encoding( Encoding::UTF_8 )
-    logger.debug "xml.encoding.name (after): #{xml.encoding.name}"      
-
-    xml
-  end
-
-
-  def feed_by_rec( feed_rec )
-    # simple feed fetcher; use for debugging (only/mostly)
-    #  -- will NOT change db records in any way
-
-    feed_url = feed_rec.feed_url
-    feed_key = feed_rec.key
-
-    feed_xml = fetch_feed( feed_url )
-
-    logger.debug "feed_xml:"
-    logger.debug feed_xml[ 0..500 ] # get first 500 chars
-
-    #  if opts.verbose?  # also write a copy to disk
-    if debug?
-      logger.debug "saving feed to >./#{feed_key}.xml<..."
-      File.open( "./#{feed_key}.xml", 'w' ) do |f|
-        f.write( feed_xml )
-      end
-    end
-      
-    puts "Before parsing feed >#{feed_key}<..."
-
-    ## fix/todo: check for feed.nil?   -> error parsing!!!
-    #    or throw exception
-    feed = FeedUtils::Parser.parse( feed_xml )
-    feed
-  end
-
-
-  def feed_by_rec_if_modified( feed_rec )   # try smart http update; will update db records
-    feed_url = feed_rec.feed_url
-    feed_key = feed_rec.key
-
-    ### todo/fix: normalize/unifiy feed_url
-    ##  - same in fetcher - use shared utitlity method or similar
-
-    @worker.use_cache = true
-    @worker.cache[ feed_url ] = {
-      'etag'          => feed_rec.http_etag,
-      'last-modified' => feed_rec.http_last_modified
-    }
-
-    begin
-      response = @worker.get( feed_url )
-    rescue SocketError => e
-      ## catch socket error for unknown domain names (e.g. pragdave.blogs.pragprog.com)
-      ###  will result in SocketError -- getaddrinfo: Name or service not known
-      puts "*** error: fetching feed '#{feed_key}' - #{e.to_s}"
-      Activity.create!( text: "*** error: fetching feed '#{feed_key}' - #{e.to_s}" )
-
-      ### todo/fix: update feed rec in db
-      @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
-      return nil
-    end
-
-    @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
-
-    if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
-      puts "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
-      puts "no change; request returns not modified (304); skipping parsing feed"
-      return nil   # no updates available; nothing to do
-    end
-
-    feed_fetched = Time.now
-
-    if response.code != '200'   # note Net::HTTP response.code is a string in ruby
-
-      puts "*** error: fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
-      
-      feed_attribs = {
-        http_code:          response.code.to_i,
-        http_server:        response.header[ 'server' ],
-        http_etag:          nil,
-        http_last_modified: nil,
-        body:               nil,
-        md5:                nil,
-        fetched:            feed_fetched 
-      }
-      feed_rec.update_attributes!( feed_attribs )
-      
-      ## add log error activity -- in future add to error log - better - why? why not?
-      Activity.create!( text: "*** error: fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}" )
-
-      return nil  #  sorry; no feed for parsing available
-    end
-
-    puts "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
-
-    feed_xml = response.body
-    ###
-    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
-    # will mostly be ASCII
-    # - try to change encoding to UTF-8 ourselves
-    logger.debug "feed_xml.encoding.name (before): #{feed_xml.encoding.name}"
-
-
-    #####
-    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-
-    # try Converting ASCII-8BIT to UTF-8 based domain-specific guesses
-    begin
-      # Try it as UTF-8 directly
-      #   Note: make a copy/dup - otherwise convert fails (because string is already changed/corrupted)
-      feed_xml_cleaned = feed_xml.dup.force_encoding( Encoding::UTF_8 )
-      unless feed_xml_cleaned.valid_encoding?
-
-         puts "*** warn: feed '#{feed_key}' charset encoding not valid utf8 - trying latin1"
-         Activity.create!( text: "*** warn: feed '#{feed_key}' charset encoding not valid utf8 - trying latin1" )
-         # Some of it might be old Windows code page
-         # -- (Windows Code Page CP1252 is ISO_8859_1 is Latin-1 - check ??)
-
-         # tell ruby the encoding
-         # encode to utf-8
-         ## use all in code encode ?? e.g. feed_xml_cleaned = feed_xml.encode( Encoding::UTF_8, Encoding::ISO_8859_1 )
-         feed_xml_cleaned = feed_xml.dup.force_encoding( Encoding::ISO_8859_1 ).encode( Encoding::UTF_8 ) 
-      end
-      feed_xml = feed_xml_cleaned
-    rescue EncodingError => e
-      puts "*** warn: feed '#{feed_key}' charset encoding to utf8 failed; throwing out invalid bits - #{e.to_s}"
-      Activity.create!( text: "*** warn: feed '#{feed_key}' charset encoding to utf8 failed; throwing out invalid bits - #{e.to_s}" )
-
-      # Force it to UTF-8, throwing out invalid bits
-      ## todo: check options - add ?? or something to mark invalid chars ???
-      feed_xml.encode!( Encoding::UTF_8, :invalid => :replace, :undef => :replace )
-    end
-
-    ## NB:
-    # for now "hardcoded" to utf8 - what else can we do?
-    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    ### old "simple" version
-    ## feed_xml = feed_xml.force_encoding( Encoding::UTF_8 )
-
-
-    logger.debug "feed_xml.encoding.name (after): #{feed_xml.encoding.name}"
-
-    ## check for md5 hash for response.body
-
-    last_feed_md5 = feed_rec.md5
-    feed_md5 = Digest::MD5.hexdigest( feed_xml )
-    
-    if last_feed_md5 && last_feed_md5 == feed_md5
-      # not all servers handle conditional gets, so while not much can be
-      # done about the bandwidth, but if the response body is identical
-      # the downstream processing (parsing, caching, ...) can be avoided.
-      #  - thanks to planet mars -fido.rb for the idea, cheers.
-      
-      puts "no change; md5 digests match; skipping parsing feed"
-      return nil   # no updates available; nothing to do
-    end
-
-    feed_attribs = {
-      http_code:          response.code.to_i,
-      http_server:        response.header[ 'server' ],
-      http_etag:          response.header[ 'etag' ],
-      http_last_modified: response.header[ 'last-modified' ], ## note: last_modified header gets stored as plain text (not datetime)
-      body:               feed_xml,
-      md5:                feed_md5,
-      fetched:            feed_fetched
-    }
-
-    ## if debug?
-      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
-      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
-      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
-    ## end
-
-    ### note: might crash w/ encoding errors when saving in postgress
-    ##  e.g. PG::CharacterNotInRepertoire: ERROR: ...
-    ##  catch error, log it and stop for now
-    #
-    #  in the future check for different charset than utf-8 ?? possible?? how to deal with non-utf8 charsets??
-
-    begin
-      feed_rec.update_attributes!( feed_attribs )
-    rescue Exception => e
-      # log db error; and continue
-      puts "*** error: updating feed database record '#{feed_key}' - #{e.to_s}"
-      Activity.create!( text: "*** error: updating feed database record '#{feed_key}' - #{e.to_s}" )
-      return nil  #  sorry; corrupt feed; parsing not possible; fix char encoding - make it an option in config??
-    end
-
-
-    logger.debug "feed_xml:"
-    logger.debug feed_xml[ 0..300 ] # get first 300 chars
-      
-    puts "Before parsing feed >#{feed_key}<..."
-
-    ### move to feedutils
-    ### logger.debug "using stdlib RSS::VERSION #{RSS::VERSION}"
-
-    ## fix/todo: check for feed.nil?   -> error parsing!!!
-    #    or throw exception
-    feed = FeedUtils::Parser.parse( feed_xml )
-    feed
-  end
-
-
-  def site_by_rec_if_modified( site_rec )   # try smart http update; will update db records
-    site_url = site_rec.url
-    site_key = site_rec.key
-
-    ### todo/fix: normalize/unifiy feed_url
-    ##  - same in fetcher - use shared utitlity method or similar
-
-    @worker.use_cache = true
-    @worker.cache[ site_url ] = {
-      'etag'          => site_rec.http_etag,
-      'last-modified' => site_rec.http_last_modified
-    }
-
-    response = @worker.get( site_url )
-    @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
-
-    site_fetched = Time.now
-
-    ###
-    # Note: Net::HTTP will NOT set encoding UTF-8 etc.
-    # will be set to ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-    # thus, set/force encoding to utf-8
-    site_text = response.body.to_s
-    site_text = site_text.force_encoding( Encoding::UTF_8 )
-
-    if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
-
-      if site_text.index('@include')
-        ## note: if the site_text includes @include
-        ##  we must revalidate complete file hierachy(tree) for now
-        ### continue;
-        ##
-        ##  fix/todo: use ahead-of-time preprocessor ?? in the future to simplify???
-      else
-        puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
-        puts "no change; request returns not modified (304); skipping parsing site config"
-        return nil   # no updates available; nothing to do
-      end
-
-    elsif response.code != '200'   # note Net::HTTP response.code is a string in ruby
-
-      puts "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
-
-      site_attribs = {
-        http_code:          response.code.to_i,
-        http_server:        response.header[ 'server' ],
-        http_etag:          nil,
-        http_last_modified: nil,
-        fetched:            site_fetched 
-      }
-      site_rec.update_attributes!( site_attribs )
-      
-      ## add log error activity -- in future add to error log - better - why? why not?
-      Activity.create!( text: "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}" )
-
-      return nil  #  sorry; no feed for parsing available
-    else
-      # assume 200; continue w/ processing
-    end
-
-    puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
-
-    site_attribs = {
-      http_code:          response.code.to_i,
-      http_server:        response.header[ 'server' ],
-      http_etag:          response.header[ 'etag' ],
-      http_last_modified: response.header[ 'last-modified' ], ## note: last_modified header gets stored as plain text (not datetime)
-      fetched:            site_fetched
-    }
-
-    ## if debug?
-      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
-      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
-      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
-    ## end
-
-    site_rec.update_attributes!( site_attribs )
-
-
-    #################
-    ### fix: add support for http_etag cache etc.  - how??
-    ###
-    ###   use from_text( text, base: base )  !!!!!!!!
-    ###    do NOT reissue first request
-    ##
-    ##  fix: use special case/method for update_with_includes!!!
-    ##    keep it simple w/o includes (do NOT mix in one method)
-    ##  split into two methods!!!
-
-    ## retry w/ preprocesser
-    ## refetch if @include found w/ all includes included
-    if site_text.index('@include')
-      site_text = InclPreproc.from_url( site_url ).read
-    end
-
-    ## logger.debug "site_text:"
-    ## logger.debug site_text[ 0..300 ] # get first 300 chars
-
-
-    puts "Before parsing site config >#{site_key}<..."
-    
-    # assume ini format for now
-    site_config = INI.load( site_text )
-    site_config
-  end
-
-end # class Fetcher
-
-end # module Pluto

data/lib/pluto/update/updater.rb

@@ -1,62 +0,0 @@
-# encoding: utf-8
-
-
-module Pluto
-
-class Updater
-
-  include LogUtils::Logging
-
-  ### fix!!!!!: change config    to text   - yes/no - why? why not??
-  #   or pass along struct
-  #      - with hash and text and format(e.g. ini) as fields???
-  #
-  #   - why? - we need to get handle on md5 digest/hash plus on plain text, ideally to store in db
-  ##  - pass along unparsed text!! - not hash struct
-  #      - will get saved in db plus we need to generate md5 hash
-  #    - add filename e.g. ruby.ini|ruby.conf as opt ??
-  #           or add config format as opt e.g. ini?
-
-  def initialize( opts, config )
-    @opts    = opts
-    @config  = config
-  end
-
-  attr_reader :opts, :config
-
-  def run( arg )
-    arg = arg.downcase.gsub('.ini','')  # remove file extension if present
-
-    update_for( arg )
-  end
-
-  def update_for( name )
-
-    ## note: allow (optional) config of site key too
-    site_key = config['key'] || config['slug']
-    if site_key.nil?
-      ## if no key configured; use (file)name; remove -_ chars
-      ##   e.g. jekyll-meta becomes jekyllmeta etc.
-      site_key = name.downcase.gsub( /[\-_]/, '' )
-    end
-
-    ###################
-    # step 1) update subscriptions
-    subscriber = Subscriber.new
-
-    # pass along debug/verbose setting/switch
-    subscriber.debug = true    if opts.verbose?
-    subscriber.update_subscriptions_for( site_key, config )
-
-    ##############################
-    # step 2) update feeds
-    refresher = Refresher.new
-
-    # pass along debug/verbose setting/switch
-    refresher.debug = true    if opts.verbose?
-    refresher.update_feeds_for( site_key )
-  end # method run
-
-end # class Updater
-
-end # module Pluto