pluto-models 1.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7d114af161602287c52ed90e78691eb9e4b04b74
+  data.tar.gz: 009c9f806a64391c97cee02f2d0e3f5e8a48b094
+SHA512:
+  metadata.gz: d07c395a1a7e88beab19c3f6b5b058fd99ae4c0a48ded6021b2265186ba3560b92ec514e056ed8c9ff32c0ce0f1ca0bcb2808c8967e5e8e98fa86dceaddf236e
+  data.tar.gz: 401d00c0da7059d0eefd9adec5f13a18e74446796e5cf92364c61c2547c886312c26b8b65eaa5510d46d423cdc27556c43778e090843f806cd7dd1a9384b4b95

data/HISTORY.md

@@ -0,0 +1,4 @@
+
+### 0.0.1 / 2012-07-14
+
+* Everything is new. First release

data/Manifest.txt

@@ -0,0 +1,30 @@
+HISTORY.md
+Manifest.txt
+README.md
+Rakefile
+config/pluto.index.yml
+lib/pluto/activerecord.rb
+lib/pluto/connecter.rb
+lib/pluto/fetcher.rb
+lib/pluto/formatter.rb
+lib/pluto/installer.rb
+lib/pluto/lister.rb
+lib/pluto/manifest_helpers.rb
+lib/pluto/models.rb
+lib/pluto/models/activity.rb
+lib/pluto/models/feed.rb
+lib/pluto/models/item.rb
+lib/pluto/models/site.rb
+lib/pluto/models/subscription.rb
+lib/pluto/models/utils.rb
+lib/pluto/refresher.rb
+lib/pluto/schema.rb
+lib/pluto/subscriber.rb
+lib/pluto/tasks/env.rake
+lib/pluto/tasks/setup.rake
+lib/pluto/tasks/stats.rake
+lib/pluto/tasks/update.rake
+lib/pluto/updater.rb
+lib/pluto/version.rb
+test/helper.rb
+test/test_helpers.rb

data/README.md

@@ -0,0 +1,82 @@
+# pluto-models gem - planet models and generator machinery for easy (re)use
+
+* home  :: [github.com/feedreader/pluto-models](https://github.com/feedreader/pluto-models)
+* bugs  :: [github.com/feedreader/pluto-models/issues](https://github.com/feedreader/pluto-models/issues)
+* gem   :: [rubygems.org/gems/pluto-models](https://rubygems.org/gems/pluto-models)
+* rdoc  :: [rubydoc.info/gems/pluto-models](http://rubydoc.info/gems/pluto-models)
+* forum :: [groups.google.com/group/feedreader](http://groups.google.com/group/feedreader)
+
+
+
+## Usage
+
+### Planet Configuration Sample 
+
+`ruby.ini`:
+
+```
+title = Planet Ruby
+
+[rubyflow]
+  title  = Ruby Flow
+  link   = http://rubyflow.com
+  feed   = http://feeds.feedburner.com/Rubyflow?format=xml
+
+[rubyonrails]
+  title = Ruby on Rails Blog
+  link  = http://weblog.rubyonrails.org
+  feed  = http://weblog.rubyonrails.org/feed/atom.xml
+
+[viennarb]
+  title = vienna.rb Blog
+  link  = http://vienna-rb.at
+  feed  = http://vienna-rb.at/atom.xml
+```
+
+or `ruby.yml`:
+
+```
+title: Planet Ruby
+
+
+rubyflow:
+  title: Ruby Flow
+  link:  http://rubyflow.com
+  feed:  http://feeds.feedburner.com/Rubyflow?format=xml
+
+rubyonrails:
+  title: Ruby on Rails Blog
+  link:  http://weblog.rubyonrails.org
+  feed:  http://weblog.rubyonrails.org/feed/atom.xml
+
+viennarb:
+  title: vienna.rb Blog
+  link:  http://vienna-rb.at
+  feed:  http://vienna-rb.at/atom.xml
+```
+
+For more samples, see [`nytimes.ini`](https://github.com/feedreader/pluto.samples/blob/master/nytimes.ini),
+[`js.ini`](https://github.com/feedreader/pluto.samples/blob/master/js.ini),
+[`dart.ini`](https://github.com/feedreader/pluto.samples/blob/master/dart.ini),
+[`haskell.ini`](https://github.com/feedreader/pluto.samples/blob/master/haskell.ini),
+[`viennarb.ini`](https://github.com/feedreader/pluto.samples/blob/master/viennarb.ini),
+[`beer.ini`](https://github.com/feedreader/pluto.samples/blob/master/beer.ini),
+[`football.ini`](https://github.com/feedreader/pluto.samples/blob/master/football.ini).
+
+
+## Real World Usage
+
+[`pluto`](https://github.com/feedreader/pluto) - planet generator command line tool using the pluto-models gem
+[`pluto.live`](https://github.com/feedreader/pluto.live) - sample planet site; sinatra web app/starter template in ruby using the pluto-models gem
+
+
+
+## License
+
+The `pluto-models` scripts are dedicated to the public domain.
+Use it as you please with no restrictions whatsoever.
+
+## Questions? Comments?
+
+Send them along to the [Planet Pluto and Friends Forum/Mailing List](http://groups.google.com/group/feedreader).
+Thanks!

data/Rakefile

@@ -0,0 +1,41 @@
+require 'hoe'
+require './lib/pluto/version.rb'
+
+Hoe.spec 'pluto-models' do
+
+  self.version = Pluto::VERSION
+
+  self.summary = 'pluto-models - planet models and generator machinery for easy (re)use'
+  self.description = summary
+
+  self.urls    = ['https://github.com/feedreader/pluto-models']
+
+  self.author  = 'Gerald Bauer'
+  self.email   = 'feedreader@googlegroups.com'
+
+  # switch extension to .markdown for gihub formatting
+  self.readme_file  = 'README.md'
+  self.history_file = 'HISTORY.md'
+
+  self.extra_deps = [
+    ['pakman',        '>= 0.5.0'], 
+    ['fetcher',       '>= 0.4.4'],
+    ['logutils',      '>= 0.6.1'],
+    ['feedutils',     '>= 0.4.0'],
+    ['props',         '>= 1.1.2'],
+    ['textutils',     '>= 0.10.0'],
+    ['gli',           '>= 2.12.2'],
+    ['activerecord'],
+    ['logutils-activerecord', '>= 0.2.0'],
+    ['props-activerecord', '0.1.0'],
+    ['activityutils', '>= 0.1.0' ],
+  ]
+
+
+  self.licenses = ['Public Domain']
+
+  self.spec_extras = {
+    required_ruby_version: '>= 1.9.2'
+  }
+
+end

data/config/pluto.index.yml

@@ -0,0 +1,23 @@
+#########################################
+# Pluto Update Index for Template Packs
+
+
+#############
+# shortcuts for fetching template packs
+#   maps shortcut to URI
+
+blank: https://raw.github.com/planet-templates/planet-blank/master/blank.txt
+
+top: https://raw.github.com/planet-templates/planet-top/master/top.txt
+
+news: https://raw.github.com/planet-templates/planet-news/master/news.txt
+
+feeds: https://raw.github.com/planet-templates/planet-feeds/master/feeds.txt
+
+classic: https://raw.github.com/planet-templates/planet-classic/master/classic.txt
+
+
+####
+# all:
+#   - add why? why not??
+#

data/lib/pluto/activerecord.rb

@@ -0,0 +1,18 @@
+module Pluto
+
+module ActiveRecordMethods
+
+  def read_attribute_w_fallbacks( *keys )
+    ### todo: use a different name e.g.:
+    ##   read_attribute_cascade ??  - does anything like this exists already?
+    ## why? why not?
+    keys.each do |key|
+      value = read_attribute( key )
+      return value unless value.nil?
+    end
+    value # fallthrough? return latest value (will be nil)
+  end
+
+
+end # module ActiveRecordMethods
+end # module Pluto

data/lib/pluto/connecter.rb

@@ -0,0 +1,96 @@
+module Pluto
+
+
+# DB Connecter / Connection Manager
+#   lets you establish connection 
+
+class Connecter
+
+  include LogUtils::Logging
+
+  def initialize
+    # do nothing for now
+  end
+
+
+  def debug=(value)
+    @debug = value
+  end
+
+  def debug?
+    @debug || false
+  end
+
+
+  def connect!( config = nil )
+
+    if config.nil?   # use/try DATABASE_URL from environment
+
+      logger.debug "ENV['DATBASE_URL'] - >#{ENV['DATABASE_URL']}<"
+
+      db = URI.parse( ENV['DATABASE_URL'] || 'sqlite3:///pluto.db' )
+
+      if db.scheme == 'postgres'
+        config = {
+          adapter:  'postgresql',
+          host:     db.host,
+          port:     db.port,
+          username: db.user,
+          password: db.password,
+          database: db.path[1..-1],
+          encoding: 'utf8'
+        }
+      else   # assume sqlite3
+        config = {
+          adapter:  db.scheme,       # sqlite3
+          database: db.path[1..-1]   # pluto.db (NB: cut off leading /, thus 1..-1)
+        }
+      end
+    end  # if config.nil?
+
+    puts 'db settings:'
+    pp config
+
+    ### for dbbrowser and other tools add to ActiveRecord
+    
+    if ActiveRecord::Base.configurations.nil?   # todo/check: can this ever happen? remove?
+       puts "ActiveRecord configurations nil - set to empty hash"
+       ActiveRecord::Base.configurations = {}  # make it an empty hash
+    end
+
+    if debug?
+      puts 'ar configurations (before):'
+      pp ActiveRecord::Base.configurations
+    end
+    
+    # note: for now always use pluto key for config storage
+    ActiveRecord::Base.configurations['pluto'] = config
+
+    if debug?
+      puts 'ar configurations (after):'
+      pp ActiveRecord::Base.configurations
+    end
+
+
+    # for debugging - disable for production use
+    if debug?
+      ActiveRecord::Base.logger = Logger.new( STDOUT ) 
+    end
+
+
+    ActiveRecord::Base.establish_connection( config )
+
+    # first time? - auto-run db migratation, that is, create db tables
+    unless ActivityDb::Models::Activity.table_exists?
+      ActivityDb::CreateDb.new.up
+    end
+
+    unless Models::Feed.table_exists?
+      CreateDb.new.up  
+    end
+  end # method connect!
+
+
+end # class Connecter
+
+end # module Pluto

data/lib/pluto/fetcher.rb

@@ -0,0 +1,292 @@
+module Pluto
+
+
+class Fetcher
+
+  include LogUtils::Logging
+
+  include Models   # for easy convenience access for Activity etc.
+
+  def initialize
+    @worker  = ::Fetcher::Worker.new
+  end
+
+  def debug=(value)  @debug = value;   end
+  def debug?()       @debug || false;  end
+
+
+  def fetch_feed( url )
+    response = @worker.get( url )
+
+    ## if debug?
+      puts "http status #{response.code} #{response.message}"
+
+      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
+      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
+      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
+    ## end
+
+    xml = response.body
+
+    ###
+    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
+    # will mostly be ASCII
+    # - try to change encoding to UTF-8 ourselves
+    logger.debug "xml.encoding.name (before): #{xml.encoding.name}"
+
+    #####
+    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+
+    ## NB:
+    # for now "hardcoded" to utf8 - what else can we do?
+    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    xml = xml.force_encoding( Encoding::UTF_8 )
+    logger.debug "xml.encoding.name (after): #{xml.encoding.name}"      
+
+    xml
+  end
+
+
+  def feed_by_rec( feed_rec )
+    # simple feed fetcher; use for debugging (only/mostly)
+    #  -- will NOT change db records in any way
+
+    feed_url = feed_rec.feed_url
+    feed_key = feed_rec.key
+
+    feed_xml = fetch_feed( feed_url )
+
+    logger.debug "feed_xml:"
+    logger.debug feed_xml[ 0..500 ] # get first 500 chars
+
+    #  if opts.verbose?  # also write a copy to disk
+    if debug?
+      logger.debug "saving feed to >./#{feed_key}.xml<..."
+      File.open( "./#{feed_key}.xml", 'w' ) do |f|
+        f.write( feed_xml )
+      end
+    end
+      
+    puts "Before parsing feed >#{feed_key}<..."
+
+    ## fix/todo: check for feed.nil?   -> error parsing!!!
+    #    or throw exception
+    feed = FeedUtils::Parser.parse( feed_xml )
+  end
+
+
+  def feed_by_rec_if_modified( feed_rec )   # try smart http update; will update db records
+    feed_url = feed_rec.feed_url
+    feed_key = feed_rec.key
+
+    ### todo/fix: normalize/unifiy feed_url
+    ##  - same in fetcher - use shared utitlity method or similar
+
+    @worker.use_cache = true
+    @worker.cache[ feed_url ] = {
+      'etag'          => feed_rec.http_etag,
+      'last-modified' => feed_rec.http_last_modified
+    }
+
+    begin
+      response = @worker.get( feed_url )
+    rescue SocketError => e
+      ## catch socket error for unknown domain names (e.g. pragdave.blogs.pragprog.com)
+      ###  will result in SocketError -- getaddrinfo: Name or service not known
+      puts "*** error: fetching feed '#{feed_key}' - #{e.to_s}"
+      Activity.create!( text: "*** error: fetching feed '#{feed_key}' - #{e.to_s}" )
+
+      ### todo/fix: update feed rec in db
+      @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
+      return nil
+    end
+
+    @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
+
+    if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
+      puts "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
+      puts "no change; request returns not modified (304); skipping parsing feed"
+      return nil   # no updates available; nothing to do
+    end
+
+    feed_fetched = Time.now
+
+    if response.code != '200'   # note Net::HTTP response.code is a string in ruby
+
+      puts "*** error: fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
+      
+      feed_attribs = {
+        http_code:          response.code.to_i,
+        http_server:        response.header[ 'server' ],
+        http_etag:          nil,
+        http_last_modified: nil,
+        body:               nil,
+        md5:                nil,
+        fetched:            feed_fetched 
+      }
+      feed_rec.update_attributes!( feed_attribs )
+      
+      ## add log error activity -- in future add to error log - better - why? why not?
+      Activity.create!( text: "*** error: fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}" )
+
+      return nil  #  sorry; no feed for parsing available
+    end
+
+    puts "OK - fetching feed '#{feed_key}' - HTTP status #{response.code} #{response.message}"
+
+    feed_xml = response.body
+    ###
+    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
+    # will mostly be ASCII
+    # - try to change encoding to UTF-8 ourselves
+    logger.debug "feed_xml.encoding.name (before): #{feed_xml.encoding.name}"
+
+    #####
+    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+
+    ## NB:
+    # for now "hardcoded" to utf8 - what else can we do?
+    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    feed_xml = feed_xml.force_encoding( Encoding::UTF_8 )
+    logger.debug "feed_xml.encoding.name (after): #{feed_xml.encoding.name}"
+
+    ## check for md5 hash for response.body
+
+    last_feed_md5 = feed_rec.md5
+    feed_md5 = Digest::MD5.hexdigest( feed_xml )
+    
+    if last_feed_md5 && last_feed_md5 == feed_md5
+      # not all servers handle conditional gets, so while not much can be
+      # done about the bandwidth, but if the response body is identical
+      # the downstream processing (parsing, caching, ...) can be avoided.
+      #  - thanks to planet mars -fido.rb for the idea, cheers.
+      
+      puts "no change; md5 digests match; skipping parsing feed"
+      return nil   # no updates available; nothing to do
+    end
+
+    feed_attribs = {
+      http_code:          response.code.to_i,
+      http_server:        response.header[ 'server' ],
+      http_etag:          response.header[ 'etag' ],
+      http_last_modified: response.header[ 'last-modified' ], ## note: last_modified header gets stored as plain text (not datetime)
+      body:               feed_xml,
+      md5:                feed_md5,
+      fetched:            feed_fetched
+    }
+
+    ## if debug?
+      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
+      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
+      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
+    ## end
+
+    feed_rec.update_attributes!( feed_attribs )
+
+    logger.debug "feed_xml:"
+    logger.debug feed_xml[ 0..300 ] # get first 300 chars
+      
+    puts "Before parsing feed >#{feed_key}<..."
+
+    ### move to feedutils
+    ### logger.debug "using stdlib RSS::VERSION #{RSS::VERSION}"
+
+    ## fix/todo: check for feed.nil?   -> error parsing!!!
+    #    or throw exception
+    feed = FeedUtils::Parser.parse( feed_xml )
+  end
+
+
+  def site_by_rec_if_modified( site_rec )   # try smart http update; will update db records
+    site_url = site_rec.url
+    site_key = site_rec.key
+
+    ### todo/fix: normalize/unifiy feed_url
+    ##  - same in fetcher - use shared utitlity method or similar
+
+    @worker.use_cache = true
+    @worker.cache[ site_url ] = {
+      'etag'          => site_rec.http_etag,
+      'last-modified' => site_rec.http_last_modified
+    }
+
+    response = @worker.get( site_url )
+    @worker.use_cache = false   # fix/todo: restore old use_cache setting instead of false
+
+    if response.code == '304'  # not modified (conditional GET - e.g. using etag/last-modified)
+      puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
+      puts "no change; request returns not modified (304); skipping parsing site config"
+      return nil   # no updates available; nothing to do
+    end
+
+    site_fetched = Time.now
+
+    if response.code != '200'   # note Net::HTTP response.code is a string in ruby
+
+      puts "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
+
+      site_attribs = {
+        http_code:          response.code.to_i,
+        http_server:        response.header[ 'server' ],
+        http_etag:          nil,
+        http_last_modified: nil,
+        body:               nil,
+        md5:                nil,
+        fetched:            feed_fetched 
+      }
+      site_rec.update_attributes!( site_attribs )
+      
+      ## add log error activity -- in future add to error log - better - why? why not?
+      Activity.create!( text: "*** error: fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}" )
+
+      return nil  #  sorry; no feed for parsing available
+    end
+
+    puts "OK - fetching site '#{site_key}' - HTTP status #{response.code} #{response.message}"
+
+    site_text = response.body
+    
+    ###
+    # NB: Net::HTTP will NOT set encoding UTF-8 etc.
+    # will mostly be ASCII
+    # - try to change encoding to UTF-8 ourselves
+    logger.debug "site_text.encoding.name (before): #{site_text.encoding.name}"
+
+    #####
+    # NB: ASCII-8BIT == BINARY == Encoding Unknown; Raw Bytes Here
+
+    ## NB:
+    # for now "hardcoded" to utf8 - what else can we do?
+    # - note: force_encoding will NOT change the chars only change the assumed encoding w/o translation
+    site_text = site_text.force_encoding( Encoding::UTF_8 )
+    logger.debug "site_text.encoding.name (after): #{site_text.encoding.name}"
+
+    site_attribs = {
+      http_code:          response.code.to_i,
+      http_server:        response.header[ 'server' ],
+      http_etag:          response.header[ 'etag' ],
+      http_last_modified: response.header[ 'last-modified' ], ## note: last_modified header gets stored as plain text (not datetime)
+      fetched:            site_fetched
+    }
+
+    ## if debug?
+      puts "http header - server: #{response.header['server']} - #{response.header['server'].class.name}"
+      puts "http header - etag: #{response.header['etag']} - #{response.header['etag'].class.name}"
+      puts "http header - last-modified: #{response.header['last-modified']} - #{response.header['last-modified'].class.name}"
+    ## end
+
+    site_rec.update_attributes!( site_attribs )
+
+    ## logger.debug "site_text:"
+    ## logger.debug site_text[ 0..300 ] # get first 300 chars
+
+
+    puts "Before parsing site config >#{site_key}<..."
+    
+    # assume ini format for now
+    site_config = INI.load( site_text )
+  end
+
+end # class Fetcher
+
+end # module Pluto