pluto 0.9.1 → 0.9.2

data/README.md

@@ -76,8 +76,8 @@ COMMAND OPTIONS
     -n, --dbname=NAME       - Database name (default: <PLANET>.db e.g. ruby.db)
 
 EXAMPLE
-    pluto build ruby.yml
-    pluto build ruby.yml --template news
+    pluto build ruby.ini
+    pluto build ruby.ini --template news
     pluto b ruby
     pluto b ruby -t news
     pluto b            # will use pluto.ini|pluto.yml|planet.ini|planet.yml if present
@@ -127,7 +127,7 @@ SYNOPSIS
     pluto [global options] update FILE
 
 EXAMPLE
-    pluto update ruby.yml
+    pluto update ruby.ini
     pluto u ruby
 ~~~
 
@@ -148,8 +148,8 @@ COMMAND OPTIONS
     -n, --dbname=NAME       - Database name (default: <PLANET>.db e.g. ruby.db)
 
 EXAMPLE
-    pluto merge ruby.yml
-    pluto merge ruby.yml --template news
+    pluto merge ruby.ini
+    pluto merge ruby.ini --template news
     pluto m ruby
     pluto m ruby -t news
 ~~~

data/lib/pluto/fetcher.rb

@@ -77,11 +77,6 @@ class Fetcher
     feed_url = feed_rec.feed_url
     feed_key = feed_rec.key
 
-    ### todo/fix:
-    ##  add if available http_etag  machinery for smarter updates
-    ##    and http_last_modified headers
-    ##    and brute force body_old == body_new   etc.
-
     ### todo/fix: normalize/unifiy feed_url
     ##  - same in fetcher - use shared utitlity method or similar
 
@@ -91,15 +86,6 @@ class Fetcher
       'last-modified' => feed_rec.http_last_modified
     }
 
-    ### fix bug in fetcher - do NOT use request_uri use uri.to
-    ## - add request_uri entry to (e.g. w/o host etc.)
-    ## - remove code here once fixed in fetcher
-    @worker.cache[ URI.parse( feed_url ).request_uri ] = {
-      'etag'          => feed_rec.http_etag,
-      'last-modified' => feed_rec.http_last_modified
-    }
-
-
     response = @worker.get( feed_url )
     @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
 
@@ -125,6 +111,10 @@ class Fetcher
         fetched:            feed_fetched 
       }
       feed_rec.update_attributes!( feed_attribs )
+      
+      ## add log error activity -- in future add to error log - better - why? why not?
+      Activity.create!( text: "*** error: fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}" )
+
       return nil  #  sorry; no feed for parsing available
     end
 
@@ -191,7 +181,98 @@ class Fetcher
     #    or throw exception
     feed = FeedUtils::Parser.parse( feed_xml )
   end
-  
+
+
+  def site_by_rec_if_modified( site_rec )   # try smart http update; will update db records
+    site_url = site_rec.url
+    site_key = site_rec.key
+
+    ### todo/fix: normalize/unifiy feed_url
+    ##  - same in fetcher - use shared utitlity method or similar
+
+    @worker.use_cache = true
+    @worker.cache[ site_url ] = {
+      'etag'          => site_rec.http_etag,
+      'last-modified' => site_rec.http_last_modified
+    }
+
+    response = @worker.get( site_url )
+    @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
+
+    if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
+      puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
+      puts "no change; request returns not modified (304); skipping parsing site config"
+      return nil   # no updates available; nothing to do
+    end
+
+    site_fetched = Time.now
+
+    if response.code != '200'   # note Net::HTTP response.code is a string in ruby
+
+      puts "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
+
+      site_attribs = {
+        http_code:          response.code.to_i,
+        http_server:        response.header[ 'server' ],
+        http_etag:          nil,
+        http_last_modified: nil,
+        body:               nil,
+        md5:                nil,
+        fetched:            feed_fetched 
+      }
+      site_rec.update_attributes!( site_attribs )
+      
+      ## add log error activity -- in future add to error log - better - why? why not?
+      Activity.create!( text: "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}" )
+
+      return nil  #  sorry; no feed for parsing available
+    end
+
+    puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
+
+    site_text = response.body
+    
+    ###
+    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
+    # will mostly be ASCII
+    # - try to change encoding to UTF-8 ourselves
+    logger.debug "site_text.encoding.name (before): #{site_text.encoding.name}"
+
+    #####
+    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+
+    ## NB:
+    # for now "hardcoded" to utf8 - what else can we do?
+    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    site_text = site_text.force_encoding( Encoding::UTF_8 )
+    logger.debug "site_text.encoding.name (after): #{site_text.encoding.name}"
+
+    site_attribs = {
+      http_code:          response.code.to_i,
+      http_server:        response.header[ 'server' ],
+      http_etag:          response.header[ 'etag' ],
+      http_last_modified: response.header[ 'last-modified' ], ## note: last_modified header gets stored as plain text (not datetime)
+      fetched:            site_fetched
+    }
+
+    ## if debug?
+      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
+      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
+      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
+    ## end
+
+    site_rec.update_attributes!( site_attribs )
+
+    ## logger.debug "site_text:"
+    ## logger.debug site_text[ 0..300 ] # get first 300 chars
+
+
+    puts "Before parsing site config >#{site_key}<..."
+    
+    # assume ini format for now
+    site_config = INI.load( site_text )
+  end
+
 end # class Fetcher
 
 end # module Pluto

data/lib/pluto/refresher.rb

@@ -14,7 +14,7 @@ class Refresher
   def debug?()       @debug || false;  end
 
 
-  def update_feeds( opts={} )  # update all feeds
+  def update_sites( opts={} )  # update all site configs
     if debug?
       ## turn on logging for sql too
       ActiveRecord::Base.logger = Logger.new( STDOUT )
@@ -22,16 +22,28 @@ class Refresher
     end
 
     # -- log update activity
-    Activity.create!( text: 'update feeds' )
-    
-    feeds_fetched = Time.now
-    
+    Activity.create!( text: 'update sites' )
+
     #### - hack - use order(:id) instead of .all - avoids rails/activerecord 4 warnings
-    
+
     Site.order(:id).each do |site|
-      site.update_attributes!( fetched: feeds_fetched )
+      update_site_worker( site )  if site.url.present?  # note: only update if (source) url present
+    end
+  end
+
+
+  def update_feeds( opts={} )  # update all feeds
+    if debug?
+      ## turn on logging for sql too
+      ActiveRecord::Base.logger = Logger.new( STDOUT )
+      @worker.debug = true   # also pass along worker debug flag if set
     end
 
+    # -- log update activity
+    Activity.create!( text: 'update feeds' )
+
+    #### - hack - use order(:id) instead of .all - avoids rails/activerecord 4 warnings
+
     Feed.order(:id).each do |feed|
       update_feed_worker( feed )
     end
@@ -48,12 +60,7 @@ class Refresher
     # -- log update activity
     Activity.create!( text: "update feeds >#{site_key}<" )
 
-    #####
-    # -- update fetched  timestamps for all sites
-    feeds_fetched = Time.now
-
     site = Site.find_by_key!( site_key )
-    site.update_attributes!( fetched: feeds_fetched )
 
     site.feeds.each do |feed|
       update_feed_worker( feed )
@@ -61,7 +68,22 @@ class Refresher
 
   end # method update_feeds
 
+
 private
+  def update_site_worker( site_rec )
+    site_config = @worker.site_by_rec_if_modified( site_rec )
+  
+    # on error or if http-not modified etc. skip update/processing
+    return  if site_config.nil?
+
+    subscriber = Subscriber.new
+    subscriber.debug = debug? ? true : false    # pass along debug flag
+
+    site_key = site_rec.key
+    subscriber.update_subscriptions_for( site_key, site_config )
+  end
+
+
   def update_feed_worker( feed_rec )
     feed = @worker.feed_by_rec_if_modified( feed_rec )
       

data/lib/pluto/schema.rb

@@ -2,18 +2,38 @@
 module Pluto
 
 class CreateDb < ActiveRecord::Migration
-    
+
   def up
     create_table :sites do |t|
       t.string   :title,     :null => false    # e.g Planet Ruby, Planet JavaScript, etc.
       t.string   :key,       :null => false    # e.g. ruby, js, etc.
-      t.datetime :fetched   #  last fetched/checked date -- make not null ??
 
       ############
       # filters (site-wide)
       t.string   :includes  # regex
       t.string   :excludes  # regex
 
+      ######################
+      # for auto-update of feed list/site config
+
+      t.string   :url    # source url for auto-update (optional)
+
+      ## note: make sure to use same fields for update check as feed 
+
+      t.datetime :fetched   #  last fetched/checked date -- make not null ??
+      t.integer  :http_code   # last http status code e.g. 200,404,etc.
+      t.string   :http_etag    # last http header etag
+      ## note: save last-modified header as text (not datetime) - pass through as is
+      t.string   :http_last_modified   # last http header last-modified - note: save header as plain text!!! pass along in next request as-is
+      t.string   :http_server    # last http server header if present
+
+      # note: do NOT store body content (that is, text) and md5 digest
+      #   use git! and github! commit will be http_etag!!
+
+      t.datetime :fetched    # last fetched/checked date
+
+      #############
+      # more fields
 
       t.timestamps  # created_at, updated_at
     end

data/lib/pluto/subscriber.rb

@@ -12,16 +12,17 @@ class Subscriber
 
   def update_subscriptions( config, opts={} )
     # !!!! -- depreciated API - remove - do NOT use anymore
-    puts "warn - [Pluto::Subscriber] depreciated API -- use update_subscriptions_for( site_key)"
+    puts "*** warn - [Pluto::Subscriber] depreciated API -- use update_subscriptions_for( site_key )"
     update_subscriptions_for( 'planet', config, opts )  # default to planet site_key
   end
 
 
   def update_subscriptions_for( site_key, config, opts={} )
     site_attribs = {
-      title: config['title'] || config['name']   # support either title or name
+      title:  config['title'] || config['name'],   # support either title or name
+      url:    config['source'] || config['url']    # support source or url   for source url for auto-update (optional)
     }
-    
+
     logger.debug "site_attribs: #{site_attribs.inspect}"
 
     site_rec = Site.find_by_key( site_key )
@@ -49,6 +50,7 @@ class Subscriber
 
       # skip "top-level" feed keys e.g. title, etc. or planet planet sections (e.g. planet,defaults)
       next if ['title','title2','name',
+               'source', 'url',
                'include','includes','exclude','excludes',
                'feeds',
                'planet','defaults'].include?( key )

data/lib/pluto/updater.rb

@@ -4,6 +4,16 @@ class Updater
 
   include LogUtils::Logging
 
+  ### fix!!!!!: change config    to text   - yes/no - why? why not??
+  #   or pass along struct
+  #      - with hash and text and format(e.g. ini/yml) as fields???
+  #
+  #   - why? - we need to get handle on md5 digest/hash plus on plain text, ideally to store in db
+  ##  - pass along unparsed text!! - not hash struct
+  #      - will get saved in db plus we need to generate md5 hash
+  #    - add filename e.g. ruby.ini|ruby.conf|ruby.yml as opt ??
+  #           or add config format as opt e.g. ini or yml?
+
   def initialize( opts, config )
     @opts    = opts
     @config  = config

data/lib/pluto/version.rb

@@ -1,4 +1,4 @@
 
 module Pluto
-  VERSION = '0.9.1'
+  VERSION = '0.9.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pluto
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.2
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pakman
-  requirement: &81472690 !ruby/object:Gem::Requirement
+  requirement: &75633210 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0.5'
   type: :runtime
   prerelease: false
-  version_requirements: *81472690
+  version_requirements: *75633210
 - !ruby/object:Gem::Dependency
   name: fetcher
-  requirement: &81472130 !ruby/object:Gem::Requirement
+  requirement: &75632900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: 0.4.1
   type: :runtime
   prerelease: false
-  version_requirements: *81472130
+  version_requirements: *75632900
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &81494060 !ruby/object:Gem::Requirement
+  requirement: &75632580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0.6'
   type: :runtime
   prerelease: false
-  version_requirements: *81494060
+  version_requirements: *75632580
 - !ruby/object:Gem::Dependency
   name: feedutils
-  requirement: &81493690 !ruby/object:Gem::Requirement
+  requirement: &75632320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: 0.4.0
   type: :runtime
   prerelease: false
-  version_requirements: *81493690
+  version_requirements: *75632320
 - !ruby/object:Gem::Dependency
   name: props
-  requirement: &81493150 !ruby/object:Gem::Requirement
+  requirement: &75632050 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: 1.0.3
   type: :runtime
   prerelease: false
-  version_requirements: *81493150
+  version_requirements: *75632050
 - !ruby/object:Gem::Dependency
   name: textutils
-  requirement: &81492890 !ruby/object:Gem::Requirement
+  requirement: &75631720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0.7'
   type: :runtime
   prerelease: false
-  version_requirements: *81492890
+  version_requirements: *75631720
 - !ruby/object:Gem::Dependency
   name: activityutils
-  requirement: &81492660 !ruby/object:Gem::Requirement
+  requirement: &75631420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 0.1.0
   type: :runtime
   prerelease: false
-  version_requirements: *81492660
+  version_requirements: *75631420
 - !ruby/object:Gem::Dependency
   name: gli
-  requirement: &81492380 !ruby/object:Gem::Requirement
+  requirement: &75631140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: 2.5.6
   type: :runtime
   prerelease: false
-  version_requirements: *81492380
+  version_requirements: *75631140
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &81492080 !ruby/object:Gem::Requirement
+  requirement: &75653690 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -109,10 +109,10 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *81492080
+  version_requirements: *75653690
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &81491610 !ruby/object:Gem::Requirement
+  requirement: &75653420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -120,7 +120,7 @@ dependencies:
         version: '3.3'
   type: :development
   prerelease: false
-  version_requirements: *81491610
+  version_requirements: *75653420
 description: pluto - Another Planet Generator (Lets You Build Web Pages from Published
   Web Feeds)
 email: feedreader@googlegroups.com