pluto 0.8.2 → 0.8.3

data/lib/pluto/updater.rb

@@ -4,246 +4,31 @@ class Updater
 
   include LogUtils::Logging
 
-  include Models
-
-  def initialize
-    @worker  = ::Fetcher::Worker.new
-  end
-
-  attr_reader :worker
-
-  def debug=(value)
-    @debug = value
-    ### logger.debug "[Updater] setting debug flag - debug? #{debug?}"
-  end
-
-  def debug?
-    @debug || false
+  def initialize( opts, config )
+    @opts    = opts
+    @config  = config
   end
 
-  def fetch_feed( url )
-    
-    ### fix: use worker.get( url )  # check http response code etc.
-    
-    xml = worker.read( url )
-
-    ###
-    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
-    # will mostly be ASCII
-    # - try to change encoding to UTF-8 ourselves
-    logger.debug "xml.encoding.name (before): #{xml.encoding.name}"
-
-    #####
-    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-
-    ## NB:
-    # for now "hardcoded" to utf8 - what else can we do?
-    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    xml = xml.force_encoding( Encoding::UTF_8 )
-    logger.debug "xml.encoding.name (after): #{xml.encoding.name}"      
-    xml
-  end
-
-
-  def update_subscriptions( config, opts={} )
-
-    ## for now - use single site w/ key planet  -- fix!! allow multiple sites (planets)
-    
-    site_attribs = {
-      title: config[ 'title' ]
-    }
-    
-    site_key = 'planet'
-    site_rec = Site.find_by_key( site_key )
-    if site_rec.nil?
-      site_rec             = Site.new
-      site_attribs[ :key ] = site_key
-
-      ## use object_id: site.id and object_type: Site
-      ## change - model/table/schema!!!
-      Action.create!( title: 'new site', object: site_attribs[ :title ] )
-    end
-    site_rec.update_attributes!( site_attribs )
-
-    # -- log update action
-    Action.create!( title: 'update subscriptions' )
-
-
-    config.each do |key, value|
-      
-      next if ['title','feeds'].include?( key )   # skip "top-level" feed keys e.g. title, etc.
-
-      ### todo/check:
-      ##   check value - must be hash
-      #     check if url or feed_url present
-      #      that is, check for required props/key-value pairs
-
-      feed_key   = key.to_s.dup
-      feed_hash  = value
-
-      feed_attribs = {
-        feed_url: feed_hash[ 'feed_url' ],
-        url:      feed_hash[ 'url'      ],
-        title:    feed_hash[ 'title'    ]   # todo: use title from feed?
-      }
-
-      puts "Updating feed subscription >#{feed_key}< - >#{feed_attribs[:feed_url]}<..."
-
-      feed_rec = Feed.find_by_key( feed_key )
-      if feed_rec.nil?
-        feed_rec             = Feed.new
-        feed_attribs[ :key ] = feed_key
-
-        ## use object_id: feed.id and object_type: Feed
-        ## change - model/table/schema!!!
-        ## todo: add parent_action_id - why? why not?
-        Action.create!( title: 'new feed', object: feed_attribs[ :title ] )
-      end
-      
-      feed_rec.update_attributes!( feed_attribs )
-      
-      ## todo:
-      #  add subscription records  (feed,site)  - how? 
-    end
-
-  end # method update_subscriptions
-
-
-  def update_feeds( opts={} )
-
-    if debug?
-      ## turn on logging for sql too
-      ActiveRecord::Base.logger = Logger.new( STDOUT )
-    end
-
-    ### move to feedutils
-    ### logger.debug "using stdlib RSS::VERSION #{RSS::VERSION}"
-
-    # -- log update action
-    Action.create!( title: 'update feeds' )
-
-    #####
-    # -- update fetched_at  timestamps for all sites
-    feeds_fetched_at = Time.now
-    Site.all.each do |site|
-      site.fetched_at = feeds_fetched_at
-      site.save!
-    end
-
-    Feed.all.each do |feed_rec|
-
-      feed_key = feed_rec.key
-      feed_url = feed_rec.feed_url
-      
-      feed_xml = fetch_feed( feed_url )
-
-      logger.debug "feed_xml:"
-      logger.debug feed_xml[ 0..300 ] # get first 300 chars
-
-      #  if opts.verbose?  # also write a copy to disk
-      if debug?
-        logger.debug "saving feed to >./#{feed_key}.xml<..."
-        File.open( "./#{feed_key}.xml", 'w' ) do |f|
-          f.write( feed_xml )
-        end
-      end
-      
-      puts "Before parsing feed >#{feed_key}<..."
-
-      ## fix/todo: check for feed.nil?   -> error parsing!!!
-      #    or throw exception
-      feed = FeedUtils::Parser.parse( feed_xml )
-
-      feed_fetched_at = Time.now
-
-      ## todo/check: move feed_rec update to the end (after item updates??)
-
-      # update feed attribs e.g.
-      #    generator
-      #    published_at,built_at,touched_at,fetched_at
-      #    summary,title2
-      
-      ## fix:
-      ## weird rss exception error on windows w/ dates
-      #  e.g. /lib/ruby/1.9.1/rss/rss.rb:37:in `w3cdtf': wrong number of arguments (1 for 0) (ArgumentError)
-      #
-      #  move to_datetime to feedutils!! if it works
-      
-      
-      feed_attribs = {
-        fetched_at:   feed_fetched_at,
-        format:       feed.format,
-        published_at: feed.published? ? feed.published.to_datetime : nil,
-        touched_at:   feed.updated?   ? feed.updated.to_datetime   : nil,
-        built_at:     feed.built?     ? feed.built.to_datetime     : nil,
-        summary:      feed.summary?   ? feed.summary   : nil,
-        title2:       feed.title2?    ? feed.title2    : nil,
-        generator:    feed.generator
-      }
-
-      if debug?
-        ## puts "*** dump feed_attribs:"
-        ## pp feed_attribs
-        puts "*** dump feed_attribs w/ class types:"
-        feed_attribs.each do |key,value|
-          puts "  #{key}: >#{value}< : #{value.class.name}"
-        end
-      end
-
-      feed_rec.update_attributes!( feed_attribs )
-
-
-      feed.items.each do |item|
-
-        item_attribs = {
-          fetched_at:   feed_fetched_at,
-          title:        item.title,
-          url:          item.url,
-          summary:      item.summary?   ? item.summary   : nil,
-          content:      item.content?   ? item.content   : nil,
-          published_at: item.published? ? item.published.to_datetime : nil,
-          touched_at:   item.updated?   ? item.updated.to_datetime   : nil,
-          feed_id:      feed_rec.id    # add feed_id fk_ref
-        }
-
-        if debug?
-          puts "*** dump item_attribs w/ class types:"
-          item_attribs.each do |key,value|
-            next if [:summary,:content].include?( key )   # skip summary n content
-            puts "  #{key}: >#{value}< : #{value.class.name}"
-          end
-        end
-
-
-        rec = Item.find_by_guid( item.guid )
-        if rec.nil?
-          rec      = Item.new
-          item_attribs[ :guid ] = item.guid
-          puts "** NEW | #{item.title}"
-        else
-          ## todo: check if any attribs changed
-          puts "UPDATE | #{item.title}"
-        end
+  attr_reader :opts, :config
 
-        rec.update_attributes!( item_attribs )
-      end  # each item
+  def run
+    ###################
+    # step 1) update subscriptions
+    subscriber = Subscriber.new
 
-      #  update  cached value latest published_at for item
-      item_recs = feed_rec.items.latest.limit(1).all
-      unless item_recs.empty?
-        if item_recs[0].published_at?
-          feed_rec.latest_published_at = item_recs[0].published_at
-        else # try touched_at
-          feed_rec.latest_published_at = item_recs[0].touched_at
-        end
-        feed_rec.save!
-      end
+    # pass along debug/verbose setting/switch
+    subscriber.debug = true    if opts.verbose?
+    subscriber.update_subscriptions( config )
 
-    end # each feed
+    ##############################
+    # step 2) update feeds
+    refresher = Refresher.new
 
+    # pass along debug/verbose setting/switch
+    refresher.debug = true    if opts.verbose?
+    refresher.update_feeds
   end # method run
 
- 
-end # class Fetcher
+end # class Updater
 
 end # module Pluto

data/lib/pluto/version.rb

@@ -1,5 +1,5 @@
 
 module Pluto
-  VERSION = '0.8.2'
+  VERSION = '0.8.3'
 end
 

data/test/helper.rb

@@ -0,0 +1,18 @@
+## $:.unshift(File.dirname(__FILE__))
+
+
+## minitest setup
+
+# require 'minitest/unit'
+require 'minitest/autorun'
+
+# include MiniTest::Unit  # lets us use TestCase instead of MiniTest::Unit::TestCase
+
+
+## our own code
+
+require 'pluto'
+
+
+LogUtils::Logger.root.level = :debug
+

data/test/test_helpers.rb

@@ -0,0 +1,75 @@
+# encoding: UTF-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_rss.rb
+#  or better
+#     rake test
+
+require 'helper'
+
+class TestHelper < MiniTest::Unit::TestCase
+
+  include Pluto::TemplateHelper   #  lets us use textify, etc.
+
+  def test_textify
+    hyin =<<EOS
+<p><img style="float:left; margin-right:4px" src="http://photos1.meetupstatic.com/photos/event/7/c/b/2/event_244651922.jpeg" alt="photo" class="photo" />vienna.rb</p>
+<p>
+  <p>The cool guys from <a href="http://platogo.com/">Platogo</a> will sponsor (y)our drinks. Which is awesome.</p>
+  <p><strong>Talks</strong>*</p>
+  <p>Jakob Sommerhuber - sponsor talk</p>
+  <p>Martin Schürrer - Erlang/OTP in production for highly-available, scalable systems</p>
+  <p>Markus Prinz - How to improve your code</p>
+  <p>Gerald Bauer - working with Sinatra</p>
+  <p>Kathrin Folkendt - 'Chapter one' (lightning talk on getting started with Rails, and building her first web app)</p>
+  <p><em>*preliminary program</em></p>
+</p>
+<p>Vienna   - Austria</p>
+<p>Friday, October 11 at 6:00 PM</p>
+<p>Attending: 21</p>
+<p>Details: http://www.meetup.com/vienna-rb/events/130346182/</p>
+EOS
+
+    hystep1 = <<EOS
+‹p›♦vienna.rb‹/p›
+‹p›
+  ‹p›The cool guys from Platogo will sponsor (y)our drinks. Which is awesome.‹/p›
+  ‹p›Talks*‹/p›
+  ‹p›Jakob Sommerhuber - sponsor talk‹/p›
+  ‹p›Martin Schürrer - Erlang/OTP in production for highly-available, scalable systems‹/p›
+  ‹p›Markus Prinz - How to improve your code‹/p›
+  ‹p›Gerald Bauer - working with Sinatra‹/p›
+  ‹p›Kathrin Folkendt - 'Chapter one' (lightning talk on getting started with Rails, and building her first web app)‹/p›
+  ‹p›*preliminary program‹/p›
+‹/p›
+‹p›Vienna   - Austria‹/p›
+‹p›Friday, October 11 at 6:00 PM‹/p›
+‹p›Attending: 21‹/p›
+‹p›Details: www.meetup.com/vienna-rb/events/130346182/‹/p›
+EOS
+
+    hyout = <<EOS
+<p>♦vienna.rb</p>
+<p>
+  <p>The cool guys from Platogo will sponsor (y)our drinks. Which is awesome.</p>
+  <p>Talks*</p>
+  <p>Jakob Sommerhuber - sponsor talk</p>
+  <p>Martin Schürrer - Erlang/OTP in production for highly-available, scalable systems</p>
+  <p>Markus Prinz - How to improve your code</p>
+  <p>Gerald Bauer - working with Sinatra</p>
+  <p>Kathrin Folkendt - 'Chapter one' (lightning talk on getting started with Rails, and building her first web app)</p>
+  <p>*preliminary program</p>
+</p>
+<p>Vienna   - Austria</p>
+<p>Friday, October 11 at 6:00 PM</p>
+<p>Attending: 21</p>
+<p>Details: www.meetup.com/vienna-rb/events/130346182/</p>
+EOS
+
+    assert_equal( hyout, textify( hyin ) )
+
+    assert_equal( hystep1, textify( hyin, skip_restore: true ) )
+  end
+
+end # class TestHelper

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pluto
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-24 00:00:00.000000000 Z
+date: 2013-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pakman
-  requirement: &83915310 !ruby/object:Gem::Requirement
+  requirement: &75684250 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0.5'
   type: :runtime
   prerelease: false
-  version_requirements: *83915310
+  version_requirements: *75684250
 - !ruby/object:Gem::Dependency
   name: fetcher
-  requirement: &83914830 !ruby/object:Gem::Requirement
+  requirement: &75684020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0.3'
   type: :runtime
   prerelease: false
-  version_requirements: *83914830
+  version_requirements: *75684020
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &83914450 !ruby/object:Gem::Requirement
+  requirement: &75683730 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,32 +43,32 @@ dependencies:
         version: '0.6'
   type: :runtime
   prerelease: false
-  version_requirements: *83914450
+  version_requirements: *75683730
 - !ruby/object:Gem::Dependency
   name: feedutils
-  requirement: &83914160 !ruby/object:Gem::Requirement
+  requirement: &75683490 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: 0.3.2
   type: :runtime
   prerelease: false
-  version_requirements: *83914160
+  version_requirements: *75683490
 - !ruby/object:Gem::Dependency
   name: props
-  requirement: &83913880 !ruby/object:Gem::Requirement
+  requirement: &75683190 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *83913880
+  version_requirements: *75683190
 - !ruby/object:Gem::Dependency
   name: textutils
-  requirement: &83913580 !ruby/object:Gem::Requirement
+  requirement: &75682950 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: 0.6.8
   type: :runtime
   prerelease: false
-  version_requirements: *83913580
+  version_requirements: *75682950
 - !ruby/object:Gem::Dependency
   name: gli
-  requirement: &83913340 !ruby/object:Gem::Requirement
+  requirement: &75682640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: 2.5.6
   type: :runtime
   prerelease: false
-  version_requirements: *83913340
+  version_requirements: *75682640
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &83913010 !ruby/object:Gem::Requirement
+  requirement: &75698770 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -98,10 +98,10 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *83913010
+  version_requirements: *75698770
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &83912770 !ruby/object:Gem::Requirement
+  requirement: &75698470 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -109,10 +109,10 @@ dependencies:
         version: '3.3'
   type: :development
   prerelease: false
-  version_requirements: *83912770
+  version_requirements: *75698470
 description: pluto - Another Planet Generator (Lets You Build Web Pages from Published
   Web Feeds)
-email: webslideshow@googlegroups.com
+email: feedreader@googlegroups.com
 executables:
 - pluto
 extensions: []
@@ -135,10 +135,15 @@ files:
 - lib/pluto/lister.rb
 - lib/pluto/manifest_helpers.rb
 - lib/pluto/models.rb
+- lib/pluto/refresher.rb
 - lib/pluto/schema.rb
+- lib/pluto/subscriber.rb
 - lib/pluto/template_helpers.rb
 - lib/pluto/updater.rb
 - lib/pluto/version.rb
+- test/helper.rb
+- test/test_helpers.rb
+- .gemtest
 homepage: https://github.com/feedreader/pluto
 licenses:
 - Public Domain
@@ -166,4 +171,5 @@ rubygems_version: 1.8.17
 signing_key: 
 specification_version: 3
 summary: pluto - Another Planet Generator
-test_files: []
+test_files:
+- test/test_helpers.rb