pluto 0.5.0 → 0.6.0

data/Manifest.txt

@@ -6,6 +6,7 @@ bin/pluto
 lib/pluto.rb
 lib/pluto/cli/opts.rb
 lib/pluto/cli/runner.rb
+lib/pluto/connecter.rb
 lib/pluto/fetcher.rb
 lib/pluto/formatter.rb
 lib/pluto/models.rb
@@ -17,6 +18,7 @@ lib/pluto/server/views/_version.erb
 lib/pluto/server/views/debug.erb
 lib/pluto/server/views/index.erb
 lib/pluto/server/views/layout.erb
+lib/pluto/updater.rb
 lib/pluto/version.rb
 templates/blank.html.erb
 templates/blank.top.html.erb

data/lib/pluto.rb

@@ -6,11 +6,15 @@
 # core and stlibs
 
 require 'yaml'
+require 'json'
+require 'uri'
 require 'pp'
 require 'logger'
 require 'optparse'
 require 'fileutils'
 
+
+
 require 'rss'
 
 # rubygems
@@ -28,6 +32,8 @@ require 'pakman'    # template pack manager
 require 'pluto/version'   # let version always get first
 require 'pluto/schema'
 require 'pluto/models'
+require 'pluto/connecter'
+require 'pluto/updater'
 require 'pluto/fetcher'
 require 'pluto/formatter'
 
@@ -44,6 +50,19 @@ module Pluto
     "#{File.expand_path( File.dirname(File.dirname(__FILE__)) )}"
   end
 
+  def self.connect!( config=nil )  # convenience shortcut
+    Connecter.new.connect!( config )
+  end
+
+  def self.update_subscriptions( config )
+    Updater.new.update_subscriptions( config )
+  end
+
+  def self.update_feeds
+    Updater.new.update_feeds
+  end
+
+
   def self.main
     Runner.new.run(ARGV)
   end
@@ -51,4 +70,8 @@ module Pluto
 end  # module Pluto
 
 
-Pluto.main if __FILE__ == $0
+if __FILE__ == $0
+  Pluto.main
+else
+  puts Pluto.banner    # say hello
+end

data/lib/pluto/cli/opts.rb

@@ -11,7 +11,7 @@ class Opts
   end
 
   def verbose=(value)
-    @verbose = true  # note: assumes true; default is false
+    @verbose = true  # note: always assumes true for now; default is false
   end
 
   def verbose?

data/lib/pluto/cli/runner.rb

@@ -5,7 +5,7 @@ class Runner
 
   include LogUtils::Logging
   
-  include Models  # e.g. Feed,Item,etc.
+  include Models  # e.g. Feed,Item,Site,etc.
 
   def initialize
    @opts = Opts.new
@@ -37,6 +37,7 @@ class Runner
 
       cmd.on( "--verbose", "Show debug trace" )  do
         LogUtils::Logger.root.level = :debug
+        opts.verbose = true
       end
 
       ## todo: add/allow -?  too
@@ -68,10 +69,10 @@ EOS
  
       db_config = {
         adapter:  'sqlite3',
-        database: "#{opts.output_path}/#{name}.sqlite"
+        database: "#{opts.output_path}/#{name}.db"
       }
  
-      setup_db( db_config )
+      Connecter.new.connect!( db_config )
 
       config_path = arg.dup   # add .yml file extension if missing (for convenience)
       config_path << '.yml'  unless config_path.ends_with?( '.yml' )
@@ -91,18 +92,5 @@ EOS
   end   # method run
 
 
-private
-
-  def setup_db( db_config )
-    puts 'db settings:'
-    pp db_config
-
-    ActiveRecord::Base.establish_connection( db_config )
-    
-    unless Feed.table_exists?
-       CreateDb.new.up  # run db migratation, that is, create db tables 
-    end
-  end # method setup_db
-
 end # class Runner
 end # module Pakman

data/lib/pluto/connecter.rb

@@ -0,0 +1,70 @@
+module Pluto
+
+
+# DB Connecter / Connection Manager
+#   lets you establish connection 
+
+class Connecter
+
+  include LogUtils::Logging
+
+  def initialize
+    # do nothing for now
+  end
+
+
+  def debug=(value)
+    @debug = value
+  end
+
+  def debug?
+    @debug || false
+  end
+
+
+  def connect!( config = nil )
+
+    if config.nil?   # use DATABASE_URL
+
+      logger.debug "ENV['DATBASE_URL'] - >#{ENV['DATABASE_URL']}<"
+
+      db = URI.parse( ENV['DATABASE_URL'] || 'sqlite3:///pluto.db' )
+
+      if db.scheme == 'postgres'
+        config = {
+          adapter:  'postgresql',
+          host:     db.host,
+          port:     db.port,
+          username: db.user,
+          password: db.password,
+          database: db.path[1..-1],
+          encoding: 'utf8'
+        }
+      else 
+        config = {
+          adapter:  db.scheme,       # sqlite3
+          database: db.path[1..-1]   # pluto.db (NB: cut off leading /, thus 1..-1)
+        }
+      end
+    end  # if config.nil?
+
+    puts 'db settings:'
+    pp config
+
+    # for debugging - disable for production use
+    if debug?
+      ActiveRecord::Base.logger = Logger.new( STDOUT ) 
+    end
+
+    ActiveRecord::Base.establish_connection( config )
+    
+    # first time? - auto-run db migratation, that is, create db tables
+    unless Models::Feed.table_exists?
+       CreateDb.new.up  
+    end
+  end # method connect!
+
+
+end # class Connecter
+
+end # module Pluto

data/lib/pluto/fetcher.rb

@@ -5,184 +5,27 @@ class Fetcher
 
   include LogUtils::Logging
 
-  include Models
-
   def initialize( opts, config )
     @opts    = opts
     @config  = config
-    @worker  = ::Fetcher::Worker.new
   end
 
-  attr_reader :opts, :config, :worker
-
-
-  def fetch_feed( url )
-    xml = worker.read( url )
-
-    ###
-    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
-    # will mostly be ASCII
-    # - try to change encoding to UTF-8 ourselves
-    logger.debug "xml.encoding.name (before): #{xml.encoding.name}"
-
-    #####
-    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
-
-    ## NB:
-    # for now "hardcoded" to utf8 - what else can we do?
-    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
-    xml = xml.force_encoding( Encoding::UTF_8 )
-    logger.debug "xml.encoding.name (after): #{xml.encoding.name}"      
-    xml
-  end
-
-
-  def parse_feed( xml )
-    parser = RSS::Parser.new( xml )
-    parser.do_validate            = false
-    parser.ignore_unknown_element = true
-
-    puts "Parsing feed..."
-    feed = parser.parse
-      
-    puts "  feed.class=#{feed.class.name}"
-    feed
-  end
+  attr_reader :opts, :config
 
 
   def run
-    logger.debug "RSS::VERSION #{RSS::VERSION}"
-
-    config[ 'feeds' ].each do |feed_key|
-      
-      feed_hash  = config[ feed_key ]
-      feed_url   = feed_hash[ 'feed_url' ]
-      
-      puts "Fetching feed >#{feed_key}< using >#{feed_url}<..."
-
-      feed_rec = Feed.find_by_key( feed_key )
-      if feed_rec.nil?
-        feed_rec      = Feed.new
-        feed_rec.key  = feed_key
-      end
-      feed_rec.feed_url = feed_url
-      feed_rec.url      = feed_hash[ 'url' ]
-      feed_rec.title    = feed_hash[ 'title' ]    # todo: use title from feed?
-      feed_rec.save!
-
-      feed_xml = fetch_feed( feed_url )
-
-#      if opts.verbose?  # also write a copy to disk
-#        ## fix: use just File.write instead of fetching again
-#        worker.copy( feed_url, "./#{feed_key}.xml" )
-#      end
-
-      # xml = File.read( "./#{feed_key}.xml" )
-
-      puts "Before parsing feed >#{feed_key}<..."
 
-      feed = parse_feed( feed_xml )
+    updater = Updater.new
 
-      if feed.class == RSS::Atom::Feed
-        puts "== #{feed.title.content} =="
-      else  ## assume RSS::Rss::Feed
-        puts "==  #{feed.channel.title} =="
-      end
+    # pass along debug/verbose setting/switch
+    updater.debug = true    if opts.verbose?
 
-      feed.items.each do |item|
-        if feed.class == RSS::Atom::Feed
-          item_attribs = handle_feed_item_atom( item )
-        else  ## assume RSS::Rss::Feed
-          item_attribs = handle_feed_item_rss( item )
-        end
-
-        # add feed_id fk_ref
-        item_attribs[ :feed_id ] = feed_rec.id
-
-        rec = Item.find_by_guid( item_attribs[ :guid ] )
-        if rec.nil?
-          rec      = Item.new
-          puts "** NEW"
-        else
-          puts "UPDATE"
-        end
-                
-        rec.update_attributes!( item_attribs )
-      end  # each item
-
-    end # each feed
+    updater.update_subscriptions( config )
+    updater.update_feeds
 
   end # method run
 
 
-  def handle_feed_item_atom( item )
-
-        ## todo: if content.content empty use summary for example
-        item_attribs = {
-          title:        item.title.content,
-          url:          item.link.href,
-          published_at: item.updated.content.utc.strftime( "%Y-%m-%d %H:%M" ),
-          # content:   item.content.content,
-        }
-
-        item_attribs[ :guid ] = item.id.content
-
-        if item.summary
-          item_attribs[ :content ] = item.summary.content
-        else
-          if item.content
-            text  = item.content.content.dup
-            ## strip all html tags
-            text = text.gsub( /<[^>]+>/, '' )
-            text = text[ 0..400 ] # get first 400 chars
-            ## todo: check for length if > 400 add ... at the end???
-            item_attribs[ :content ] = text
-          end
-        end
-
-        puts "- #{item.title.content}"
-        puts "  link >#{item.link.href}<"
-        puts "  id (~guid) >#{item.id.content}<"
-        
-        ### todo: use/try published first? why? why not?
-        puts "  updated (~pubDate) >#{item.updated.content}< >#{item.updated.content.utc.strftime( "%Y-%m-%d %H:%M" )}< : #{item.updated.content.class.name}"
-        puts
-        
-        # puts "*** dump item:"
-        # pp item
-
-        item_attribs
-  end
-
-  def handle_feed_item_rss( item )
-
-       item_attribs = {
-          title:        item.title,
-          url:          item.link,
-          published_at: item.pubDate.utc.strftime( "%Y-%m-%d %H:%M" ),
-          # content:  item.content_encoded,
-        }
-        
-        # if item.content_encoded.nil?
-          # puts " using description for content"
-          item_attribs[ :content ] = item.description
-        # end
-        
-        item_attribs[ :guid ] = item.guid.content
-        
-        puts "- #{item.title}"
-        puts "  link (#{item.link})"
-        puts "  guid (#{item.guid.content})"
-        puts "  pubDate >#{item.pubDate}< >#{item.pubDate.utc.strftime( "%Y-%m-%d %H:%M" )}< : #{item.pubDate.class.name}"
-        puts
-
-        # puts "*** dump item:"
-        # pp item
-
-        item_attribs
-  end
-
-
 end # class Fetcher
 
 end # module Pluto

data/lib/pluto/models.rb

@@ -5,6 +5,8 @@ class Feed < ActiveRecord::Base
   self.table_name = 'feeds'
   
   has_many :items
+  has_many :subscriptions
+  has_many :sites, :through => :subscriptions
 end
 
 class Item < ActiveRecord::Base
@@ -13,9 +15,24 @@ class Item < ActiveRecord::Base
   belongs_to :feed
   
   def self.latest
-    self.order( 'published_at desc' )
+    order( 'published_at desc' )
   end
 end
 
+class Site < ActiveRecord::Base
+  self.table_name = 'sites'
+  
+  has_many :subscriptions
+  has_many :feeds, :through => :subscriptions
+end
+
+class Subscription < ActiveRecord::Base
+  self.table_name = 'subscriptions'
+  
+  belongs_to :site
+  belongs_to :feed
+end
+
+
   end # module Models
 end # module Pluto

data/lib/pluto/schema.rb

@@ -4,6 +4,18 @@ module Pluto
 class CreateDb < ActiveRecord::Migration
     
   def up
+    create_table :sites do |t|
+      t.string  :title,    :null => false    # e.g Planet Ruby, Planet JavaScript, etc.
+      t.string  :key,      :null => false    # e.g. ruby, js, etc.
+      t.timestamps
+    end
+
+    create_table :subscriptions do |t|   # has_many join table (sites/feeds)
+      t.references :site, :null => false
+      t.references :feed, :null => false
+      t.timestamps
+    end
+
     create_table :feeds do |t|
       t.string  :title,    :null => false
       t.string  :url,      :null => false

data/lib/pluto/server.rb

@@ -31,19 +31,29 @@ class Server < Sinatra::Base
   set :static, true   # set up static file routing
 
 
-  set :site_config, {}   # empty hash by default; use site_config
-
   #######################
   # Models
   
-  include Models   # e.g. Feed, Item, etc.
-
+  include Models   # e.g. Feed, Item, Site, etc.
+
+  #################
+  # Utilities 
+
+  def site_config_hash
+    h = {}
+    site = Site.first      # FIX: for now assume one planet per DB (fix later; allow planet key or similar)
+    if site.present?
+      h['title']  = site.title
+    else
+      h['title']  = 'Planet Untitled'
+    end
+  end
 
   ##############################################
   # Controllers / Routing / Request Handlers
 
   get '/' do
-    erb :index, locals: { site: settings.site_config }
+    erb :index, locals: { site: site_config_hash }
   end
 
   # todo/fix: make a generic route for erb w /regex

data/lib/pluto/updater.rb

@@ -0,0 +1,224 @@
+module Pluto
+
+class Updater
+
+  include LogUtils::Logging
+
+  include Models
+
+  def initialize
+    @worker  = ::Fetcher::Worker.new
+  end
+
+  attr_reader :worker
+
+  def debug=(value)
+    @debug = value
+    ### logger.debug "[Updater] setting debug flag - debug? #{debug?}"
+  end
+
+  def debug?
+    @debug || false
+  end
+
+  def fetch_feed( url )
+    xml = worker.read( url )
+
+    ###
+    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
+    # will mostly be ASCII
+    # - try to change encoding to UTF-8 ourselves
+    logger.debug "xml.encoding.name (before): #{xml.encoding.name}"
+
+    #####
+    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+
+    ## NB:
+    # for now "hardcoded" to utf8 - what else can we do?
+    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    xml = xml.force_encoding( Encoding::UTF_8 )
+    logger.debug "xml.encoding.name (after): #{xml.encoding.name}"      
+    xml
+  end
+
+
+  def parse_feed( xml )
+    parser = RSS::Parser.new( xml )
+    parser.do_validate            = false
+    parser.ignore_unknown_element = true
+
+    puts "Parsing feed..."
+    feed = parser.parse
+      
+    puts "  feed.class=#{feed.class.name}"
+    feed
+  end
+
+
+  def update_subscriptions( config, opts={} )
+
+    ## for now - use single site w/ key planet  -- fix!! allow multiple sites (planets)
+    
+    site_key = 'planet'
+    site_rec = Site.find_by_key( site_key )
+    if site_rec.nil?
+      site_rec      = Site.new
+      site_rec.key  = site_key
+    end
+    site_rec.title  = config[ 'title' ]
+    site_rec.save!
+
+
+    config[ 'feeds' ].each do |feed_key|
+      
+      feed_hash  = config[ feed_key ]
+      feed_url   = feed_hash[ 'feed_url' ]
+      
+      puts "Updating feed subscription >#{feed_key}< - >#{feed_url}<..."
+
+      feed_rec = Feed.find_by_key( feed_key )
+      if feed_rec.nil?
+        feed_rec      = Feed.new
+        feed_rec.key  = feed_key
+      end
+      feed_rec.feed_url = feed_url
+      feed_rec.url      = feed_hash[ 'url' ]
+      feed_rec.title    = feed_hash[ 'title' ]    # todo: use title from feed?
+      feed_rec.save!
+      
+      ## todo:
+      #  add subscription records  (feed,site)  - how? 
+    end
+
+  end # method update_subscriptions
+  
+  
+  def update_feeds( opts={} )
+
+    logger.debug "using stdlib RSS::VERSION #{RSS::VERSION}"
+
+    Feed.all.each do |feed_rec|
+
+      feed_key = feed_rec.key
+      feed_url = feed_rec.feed_url
+      
+      feed_xml = fetch_feed( feed_url )
+
+      logger.debug "feed_xml:"
+      logger.debug feed_xml[ 0..300 ] # get first 300 chars
+
+      #  if opts.verbose?  # also write a copy to disk
+      if debug?
+        logger.debug "saving feed to >./#{feed_key}.xml<..."
+        File.open( "./#{feed_key}.xml", 'w' ) do |f|
+          f.write( feed_xml )
+        end
+      end
+      
+      puts "Before parsing feed >#{feed_key}<..."
+
+      feed = parse_feed( feed_xml )
+
+      if feed.class == RSS::Atom::Feed
+        puts "== #{feed.title.content} =="
+      else  ## assume RSS::Rss::Feed
+        puts "==  #{feed.channel.title} =="
+      end
+
+      feed.items.each do |item|
+        if feed.class == RSS::Atom::Feed
+          item_attribs = handle_feed_item_atom( item )
+        else  ## assume RSS::Rss::Feed
+          item_attribs = handle_feed_item_rss( item )
+        end
+
+        # add feed_id fk_ref
+        item_attribs[ :feed_id ] = feed_rec.id
+
+        rec = Item.find_by_guid( item_attribs[ :guid ] )
+        if rec.nil?
+          rec      = Item.new
+          puts "** NEW"
+        else
+          puts "UPDATE"
+        end
+                
+        rec.update_attributes!( item_attribs )
+      end  # each item
+
+    end # each feed
+
+  end # method run
+
+
+  def handle_feed_item_atom( item )
+
+        ## todo: if content.content empty use summary for example
+        item_attribs = {
+          title:        item.title.content,
+          url:          item.link.href,
+          published_at: item.updated.content.utc.strftime( "%Y-%m-%d %H:%M" ),
+          # content:   item.content.content,
+        }
+
+        item_attribs[ :guid ] = item.id.content
+
+        if item.summary
+          item_attribs[ :content ] = item.summary.content
+        else
+          if item.content
+            text  = item.content.content.dup
+            ## strip all html tags
+            text = text.gsub( /<[^>]+>/, '' )
+            text = text[ 0..400 ] # get first 400 chars
+            ## todo: check for length if > 400 add ... at the end???
+            item_attribs[ :content ] = text
+          end
+        end
+
+        puts "- #{item.title.content}"
+        puts "  link >#{item.link.href}<"
+        puts "  id (~guid) >#{item.id.content}<"
+        
+        ### todo: use/try published first? why? why not?
+        puts "  updated (~pubDate) >#{item.updated.content}< >#{item.updated.content.utc.strftime( "%Y-%m-%d %H:%M" )}< : #{item.updated.content.class.name}"
+        puts
+        
+        # puts "*** dump item:"
+        # pp item
+
+        item_attribs
+  end
+
+  def handle_feed_item_rss( item )
+
+       item_attribs = {
+          title:        item.title,
+          url:          item.link,
+          published_at: item.pubDate.utc.strftime( "%Y-%m-%d %H:%M" ),
+          # content:  item.content_encoded,
+        }
+        
+        # if item.content_encoded.nil?
+          # puts " using description for content"
+          item_attribs[ :content ] = item.description
+        # end
+        
+        item_attribs[ :guid ] = item.guid.content
+        
+        puts "- #{item.title}"
+        puts "  link (#{item.link})"
+        puts "  guid (#{item.guid.content})"
+        puts "  pubDate >#{item.pubDate}< >#{item.pubDate.utc.strftime( "%Y-%m-%d %H:%M" )}< : #{item.pubDate.class.name}"
+        puts
+
+        # puts "*** dump item:"
+        # pp item
+
+        item_attribs
+  end
+
+
+end # class Fetcher
+
+end # module Pluto

data/lib/pluto/version.rb

@@ -1,6 +1,6 @@
 
 module Pluto
-  VERSION = '0.5.0'
+  VERSION = '0.6.0'
 end
 
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pluto
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2013-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pakman
-  requirement: &79513810 !ruby/object:Gem::Requirement
+  requirement: &83822120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0.5'
   type: :runtime
   prerelease: false
-  version_requirements: *79513810
+  version_requirements: *83822120
 - !ruby/object:Gem::Dependency
   name: fetcher
-  requirement: &79512200 !ruby/object:Gem::Requirement
+  requirement: &83821440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0.3'
   type: :runtime
   prerelease: false
-  version_requirements: *79512200
+  version_requirements: *83821440
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &79510570 !ruby/object:Gem::Requirement
+  requirement: &83818890 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0.6'
   type: :runtime
   prerelease: false
-  version_requirements: *79510570
+  version_requirements: *83818890
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &79509720 !ruby/object:Gem::Requirement
+  requirement: &83844650 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -54,10 +54,10 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *79509720
+  version_requirements: *83844650
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &79508820 !ruby/object:Gem::Requirement
+  requirement: &83842540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -65,7 +65,7 @@ dependencies:
         version: '3.3'
   type: :development
   prerelease: false
-  version_requirements: *79508820
+  version_requirements: *83842540
 description: pluto - Another Planet Generator (Lets You Build Web Pages from Published
   Web Feeds)
 email: webslideshow@googlegroups.com
@@ -84,6 +84,7 @@ files:
 - lib/pluto.rb
 - lib/pluto/cli/opts.rb
 - lib/pluto/cli/runner.rb
+- lib/pluto/connecter.rb
 - lib/pluto/fetcher.rb
 - lib/pluto/formatter.rb
 - lib/pluto/models.rb
@@ -95,6 +96,7 @@ files:
 - lib/pluto/server/views/debug.erb
 - lib/pluto/server/views/index.erb
 - lib/pluto/server/views/layout.erb
+- lib/pluto/updater.rb
 - lib/pluto/version.rb
 - templates/blank.html.erb
 - templates/blank.top.html.erb