pluto 0.8.2 → 0.8.3

data/lib/pluto/formatter.rb

@@ -14,15 +14,12 @@ class Formatter
     @config  = config
   end
 
-  attr_reader :opts
-
-  def site
-    ### fix !!!!!!!!!!
-    ## fix/todo: change to db record for site
-    @config
-  end
+  attr_reader :opts, :config, :site
 
   def run( arg )
+    ## fix: change arg to planet_key or just key or similar
+    #  todo: rename run to some less generic  - merge/build/etc. ??
+    
     manifest_name = opts.manifest
     manifest_name = manifest_name.downcase.gsub('.txt', '' )  # remove .txt if present
     
@@ -51,9 +48,20 @@ class Formatter
 
     manifestsrc = manifests[0][1]
     pakpath     = opts.output_path
-    
-        
+
     name = arg
+
+    ## for now - use single site w/ key planet
+    ##-- fix!! allow multiple sites (planets)
+
+    site_key = 'planet'
+    @site = Site.find_by_key( site_key )
+    if @site.nil?
+      puts "*** warn: no site with key '#{site_key}' found; using untitled site record"
+      @site = Site.new
+      @site.title = 'Planet Untitled'
+    end
+
     Pakman::Templater.new.merge_pak( manifestsrc, pakpath, binding, name )
   end
 

data/lib/pluto/refresher.rb

@@ -0,0 +1,186 @@
+module Pluto
+
+class Refresher
+
+  include LogUtils::Logging
+
+  include Models
+
+  def initialize
+    @worker  = ::Fetcher::Worker.new
+  end
+
+  attr_reader :worker
+
+  def debug=(value)
+    @debug = value
+    ### logger.debug "[Updater] setting debug flag - debug? #{debug?}"
+  end
+
+  def debug?
+    @debug || false
+  end
+
+  def fetch_feed( url )
+    
+    ### fix: use worker.get( url )  # check http response code etc.
+    
+    xml = worker.read( url )
+
+    ###
+    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
+    # will mostly be ASCII
+    # - try to change encoding to UTF-8 ourselves
+    logger.debug "xml.encoding.name (before): #{xml.encoding.name}"
+
+    #####
+    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+
+    ## NB:
+    # for now "hardcoded" to utf8 - what else can we do?
+    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    xml = xml.force_encoding( Encoding::UTF_8 )
+    logger.debug "xml.encoding.name (after): #{xml.encoding.name}"      
+    xml
+  end
+
+
+  def update_feeds( opts={} )
+
+    if debug?
+      ## turn on logging for sql too
+      ActiveRecord::Base.logger = Logger.new( STDOUT )
+    end
+
+    ### move to feedutils
+    ### logger.debug "using stdlib RSS::VERSION #{RSS::VERSION}"
+
+    # -- log update action
+    Action.create!( title: 'update feeds' )
+
+    #####
+    # -- update fetched_at  timestamps for all sites
+    feeds_fetched_at = Time.now
+    Site.all.each do |site|
+      site.fetched_at = feeds_fetched_at
+      site.save!
+    end
+
+    Feed.all.each do |feed_rec|
+
+      feed_key = feed_rec.key
+      feed_url = feed_rec.feed_url
+      
+      feed_xml = fetch_feed( feed_url )
+
+      logger.debug "feed_xml:"
+      logger.debug feed_xml[ 0..300 ] # get first 300 chars
+
+      #  if opts.verbose?  # also write a copy to disk
+      if debug?
+        logger.debug "saving feed to >./#{feed_key}.xml<..."
+        File.open( "./#{feed_key}.xml", 'w' ) do |f|
+          f.write( feed_xml )
+        end
+      end
+      
+      puts "Before parsing feed >#{feed_key}<..."
+
+      ## fix/todo: check for feed.nil?   -> error parsing!!!
+      #    or throw exception
+      feed = FeedUtils::Parser.parse( feed_xml )
+
+      feed_fetched_at = Time.now
+
+      ## todo/check: move feed_rec update to the end (after item updates??)
+
+      # update feed attribs e.g.
+      #    generator
+      #    published_at,built_at,touched_at,fetched_at
+      #    summary,title2
+      
+      ## fix:
+      ## weird rss exception error on windows w/ dates
+      #  e.g. /lib/ruby/1.9.1/rss/rss.rb:37:in `w3cdtf': wrong number of arguments (1 for 0) (ArgumentError)
+      #
+      #  move to_datetime to feedutils!! if it works
+      ##   todo: move this comments to feedutils??
+      
+      
+      feed_attribs = {
+        fetched_at:   feed_fetched_at,
+        format:       feed.format,
+        published_at: feed.published? ? feed.published : nil,
+        touched_at:   feed.updated?   ? feed.updated   : nil,
+        built_at:     feed.built?     ? feed.built     : nil,
+        summary:      feed.summary?   ? feed.summary   : nil,
+        title2:       feed.title2?    ? feed.title2    : nil,
+        generator:    feed.generator
+      }
+
+      if debug?
+        ## puts "*** dump feed_attribs:"
+        ## pp feed_attribs
+        puts "*** dump feed_attribs w/ class types:"
+        feed_attribs.each do |key,value|
+          puts "  #{key}: >#{value}< : #{value.class.name}"
+        end
+      end
+
+      feed_rec.update_attributes!( feed_attribs )
+
+
+      feed.items.each do |item|
+
+        item_attribs = {
+          fetched_at:   feed_fetched_at,
+          title:        item.title,
+          url:          item.url,
+          summary:      item.summary?   ? item.summary   : nil,
+          content:      item.content?   ? item.content   : nil,
+          published_at: item.published? ? item.published : nil,
+          touched_at:   item.updated?   ? item.updated   : nil,
+          feed_id:      feed_rec.id    # add feed_id fk_ref
+        }
+
+        if debug?
+          puts "*** dump item_attribs w/ class types:"
+          item_attribs.each do |key,value|
+            next if [:summary,:content].include?( key )   # skip summary n content
+            puts "  #{key}: >#{value}< : #{value.class.name}"
+          end
+        end
+
+
+        rec = Item.find_by_guid( item.guid )
+        if rec.nil?
+          rec      = Item.new
+          item_attribs[ :guid ] = item.guid
+          puts "** NEW | #{item.title}"
+        else
+          ## todo: check if any attribs changed
+          puts "UPDATE | #{item.title}"
+        end
+
+        rec.update_attributes!( item_attribs )
+      end  # each item
+
+      #  update  cached value latest published_at for item
+      item_recs = feed_rec.items.latest.limit(1).all
+      unless item_recs.empty?
+        if item_recs[0].published_at?
+          feed_rec.latest_published_at = item_recs[0].published_at
+        else # try touched_at
+          feed_rec.latest_published_at = item_recs[0].touched_at
+        end
+        feed_rec.save!
+      end
+
+    end # each feed
+
+  end # method update_feeds
+
+ 
+end # class Refresher
+
+end # module Pluto

data/lib/pluto/subscriber.rb

@@ -0,0 +1,85 @@
+module Pluto
+
+class Subscriber
+
+  include LogUtils::Logging
+
+  include Models
+
+  def debug=(value)
+    @debug = value
+    ### logger.debug "[Updater] setting debug flag - debug? #{debug?}"
+  end
+
+  def debug?
+    @debug || false
+  end
+
+  def update_subscriptions( config, opts={} )
+
+    site_attribs = {
+      title: config[ 'title' ] || config[ 'name' ]   # support either title or name
+    }
+
+    ## for now - use single site w/ key planet
+    ##  -- fix!! allow multiple sites (planets)
+    
+    site_key = 'planet'
+    site_rec = Site.find_by_key( site_key )
+    if site_rec.nil?
+      site_rec             = Site.new
+      site_attribs[ :key ] = site_key
+
+      ## use object_id: site.id and object_type: Site
+      ## change - model/table/schema!!!
+      Action.create!( title: 'new site', object: site_attribs[ :title ] )
+    end
+    site_rec.update_attributes!( site_attribs )
+
+    # -- log update action
+    Action.create!( title: 'update subscriptions' )
+
+
+    config.each do |key, value|
+      
+      next if ['title','name','feeds'].include?( key )   # skip "top-level" feed keys e.g. title, etc.
+
+      ### todo/check:
+      ##   check value - must be hash
+      #     check if url or feed_url present
+      #      that is, check for required props/key-value pairs
+
+      feed_key   = key.to_s.dup
+      feed_hash  = value
+
+      # todo: use title from feed?
+      feed_attribs = {
+        feed_url: feed_hash[ 'feed' ]  || feed_hash[ 'feed_url' ],
+        url:      feed_hash[ 'link' ]  || feed_hash[ 'site' ] || feed_hash[ 'url' ],
+        title:    feed_hash[ 'title' ] || feed_hash[ 'name' ] || feed_hash[ 'author' ]
+      }
+
+      puts "Updating feed subscription >#{feed_key}< - >#{feed_attribs[:feed_url]}<..."
+
+      feed_rec = Feed.find_by_key( feed_key )
+      if feed_rec.nil?
+        feed_rec             = Feed.new
+        feed_attribs[ :key ] = feed_key
+
+        ## use object_id: feed.id and object_type: Feed
+        ## change - model/table/schema!!!
+        ## todo: add parent_action_id - why? why not?
+        Action.create!( title: 'new feed', object: feed_attribs[ :title ] )
+      end
+      
+      feed_rec.update_attributes!( feed_attribs )
+      
+      ## todo:
+      #  add subscription records  (feed,site)  - how? 
+    end
+
+  end # method update_subscriptions
+
+end # class Subscriber
+
+end # module Pluto

data/lib/pluto/template_helpers.rb

@@ -1,17 +1,107 @@
+# encoding: UTF-8
+
 module Pluto
 
-####
-# fix: rename to DateHelper
 
 module TemplateHelper
 
-  def strip_tags( hypertext )
+  def strip_tags( hy )
     ### tobe done
     ## strip markup tags; return plain text
-    hypertext.gsub( /<[^>]+>/, '' )
+    hy.gsub( /<[^>]+>/, '' )
   end
 
 
+  def whitelist( hy, tags, opts={} )
+
+    # note: assumes properly escaped <> in hy/hypertext
+
+    ###############################################
+    # step one - save whitelisted tags use ‹tag›
+    tags.each do |tag|
+      # note: we strip all attribues
+      # note: match all tags case insensitive e.g. allow a,A or br,BR,bR etc.
+      #   downcase all tags
+
+      # convert xml-style empty tags to simple html emtpty tags
+      #  e.g. <br/> or <br /> becomses <br>
+      hy = hy.gsub( /<(#{tag})\s*\/>/i )       { |_| "‹#{$1.downcase}›" }   # eg. <br /> or <br/> becomes ‹br›
+
+      # make sure we won't swall <br> for <b> for example, thus use \s+ before [^>]
+      hy = hy.gsub( /<(#{tag})(\s+[^>]*)?>/i ) { |_| "‹#{$1.downcase}›" }   # opening tag <p>
+      hy = hy.gsub( /<\/(#{tag})\s*>/i )       { |_| "‹/#{$1.downcase}›" }  # closing tag e.g. </p>
+    end
+
+    ############################
+    # step two - clean tags
+
+    #   strip images - special treatment for debugging
+    hy = hy.gsub( /<img[^>]*>/i, '♦' )   # for debugging use black diamond e.g. ♦
+    hy = hy.gsub( /<\/img>/i, '' )   # should not exists
+
+    # strip all remaining tags
+    hy = hy.gsub( /<[^>]+>/, '' )
+    
+    pp hy  # fix: debugging indo - remove
+    
+    ############################################
+    # step three - restore whitelisted tags
+
+    return hy if opts[:skip_restore].present?   # skip step 3 for debugging
+
+    tags.each do |tag|
+#      hy = hy.gsub( /‹(#{tag})›/, "<\1>" )  # opening tag e.g. <p>
+#      hy = hy.gsub( /‹\/(#{tag})›/, "<\/\1>" )  # closing tag e.g. </p>
+      hy = hy.gsub( /‹(#{tag})›/ )   { |_| "<#{$1}>" }
+      hy = hy.gsub( /‹\/(#{tag})›/ ) { |_| "<\/#{$1}>" }  # closing tag e.g. </p>
+    end
+
+    hy
+  end  # method whitelist
+
+
+  ##  change to simple_hypertext or
+  #     hypertext_simple or
+  #     sanitize ???
+  
+  def textify( hy, opts={} )   # hy -> hypertext
+    ## turn into text
+    # todo: add options for
+    #   keep links, images, lists (?too), code, codeblocks
+
+    hy = whitelist( hy, [:br, :p, :ul, :ol, :li, :pre, :code], opts )
+
+   # strip bold
+#    hy = hy.gsub( /<b[^>]*>/, '**' )  # fix: will also swallow bxxx tags - add b space
+#    hy = hy.gsub( /<\/b>/, '**' )
+
+    # strip em
+#   hy = hy.gsub( /<em[^>]*>/, '__' )
+#   hy = hy.gsub( /<\/em>/, '__' )
+
+    # clean (prettify) literal urls (strip protocoll) 
+    hy = hy.gsub( /(http|https):\/\//, '' )
+
+#    hy = hy.gsub( /&nbsp;/, ' ' )
+
+#    # try to cleanup whitespaces
+#    # -- keep no more than two spaces
+#    hy = hy.gsub( /[ \t]{3,}/, '  ' )
+#    # -- keep no more than two new lines
+#    hy = hy.gsub( /\n{2,}/m, "\n\n" ) 
+#    # -- remove all trailing spaces
+#    hy = hy.gsub( /[ \t\n]+$/m, '' )
+#    # -- remove all leading spaces
+#    hy = hy.gsub( /^[ \t\n]+/m, '' ) 
+
+    hy
+  end
+
+
+####
+# fix: move to DateHelper  ??
+
+
   def time_ago_in_words( from_time )
     from_time = from_time.to_time
     to_time   = Time.now
@@ -27,23 +117,26 @@ module TemplateHelper
     case distance_in_minutes
       when 0..1             then  "just now"
       when 2...45           then  "%d minutes ago" % distance_in_minutes
-      when 45...90          then  "about 1 hour ago"   ## use one instead of 1 ?? why? why not?
+      when 45...90          then  "an hour ago"   ## use one instead of 1 ?? why? why not?
       # 90 mins up to 24 hours
-      when 90...1440        then  "about %d hours ago" % (distance_in_minutes.to_f / 60.0).round
+      when 90...1440        then  "%d hours ago" % (distance_in_minutes.to_f / 60.0).round
       # 24 hours up to 42 hours
-      when 1440...2520      then "1 day ago"   ## use one day ago - why? why not?
+      when 1440...2520      then "a day ago"   ## use one day ago - why? why not?
       # 42 hours up to 30 days
       when 2520...43200     then "%d days ago" % (distance_in_minutes.to_f / 1440.0).round
       # 30 days up to 60 days
-      #  fix: use pluralize for months
-      when 43200...86400    then "about %d months ago" % (distance_in_minutes.to_f / 43200.0).round
+      #  fix: use pluralize for months  - fix: change when - use just for a month ago
+      when 43200...86400    then "%d months ago" % (distance_in_minutes.to_f / 43200.0).round
       # 60 days up to 365 days
       when 86400...525600   then "%d months ago" % (distance_in_minutes.to_f / 43200.0).round
       ## fix - add number of years ago
-      else                       "about a year ago"  #todo: use over a year ago???
+      else                       "over a year ago"  #todo: use over a year ago???
+                                                    # fix: split into two - use
+                                                    #  1) a year ago
+                                                    #  2) (x) years ago
     end
   end
 
 
 end # module TemplateHelper
-end # module Pluto
+end # module Pluto