pluto-models 1.3.2 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b32361390e52c8909c4c2ff6a49ffa07df29b7d8
-  data.tar.gz: 0335b356a6de669eed82424fc08a080d2f158ffd
+  metadata.gz: 2adea6bc1a5429eea27ded5314e603e339ff2595
+  data.tar.gz: 50f68d6d907924a99745c74a5ba71545cafcf746
 SHA512:
-  metadata.gz: cfa986ebd8dc839d7259724c7fa739535fa0adb3923bf30e88ad4c47679c3917ac2b62a4521613c266e1260cd76f0136360bce20cd540f7ac9d9f0b63fdf5c22
-  data.tar.gz: 87868c583b0bfa102457197e4893c22108d809ec9ac257d677f314d0366bd5e4e0c12ea155f13c3d3ce91cd9ac5d88b748152dc3dac19b4ddf9f6faf4694d0f2
+  metadata.gz: 81b50daf9bfc77c04abedece3600bca9e9355f46f3e2df182ef6b22676fbdeed6529b6511142707cc3dc666ff82384ce86b71f24b04a421282f6f74357229844
+  data.tar.gz: cd4cd39e1a282a1ca242d9b8a743672bf37db241890485a520355310b8cdc5904c8e147ceae0d277b3163d8b74524468463fc921e50f8be63f5b5a2e4090ff5c

data/Manifest.txt

@@ -12,6 +12,8 @@ lib/pluto/models/subscription.rb
 lib/pluto/models/utils.rb
 lib/pluto/schema.rb
 lib/pluto/version.rb
+test/data/ruby.ini
 test/helper.rb
 test/test_filter.rb
 test/test_helpers.rb
+test/test_site.rb

data/README.md

@@ -1,7 +1,7 @@
 # pluto-models gem - planet schema 'n' models for easy (re)use
 
-* home  :: [github.com/feedreader/pluto-models](https://github.com/feedreader/pluto-models)
-* bugs  :: [github.com/feedreader/pluto-models/issues](https://github.com/feedreader/pluto-models/issues)
+* home  :: [github.com/feedreader/pluto.models](https://github.com/feedreader/pluto.models)
+* bugs  :: [github.com/feedreader/pluto.models/issues](https://github.com/feedreader/pluto.models/issues)
 * gem   :: [rubygems.org/gems/pluto-models](https://rubygems.org/gems/pluto-models)
 * rdoc  :: [rubydoc.info/gems/pluto-models](http://rubydoc.info/gems/pluto-models)
 * forum :: [groups.google.com/group/feedreader](http://groups.google.com/group/feedreader)
@@ -85,7 +85,8 @@ end
 ## Real World Usage
 
 - [`pluto`](https://github.com/feedreader/pluto) - planet generator command line tool using the pluto-models gem
-- [`pluto.live`](https://github.com/feedreader/pluto.live) - sample planet site; sinatra web app/starter template in ruby using the pluto-models gem
+- [`pluto.live.starter`](https://github.com/feedreader/pluto.live.starter) - sample planet site; sinatra web app starter template in ruby using the pluto-models gem
+- [`pluto.live`](https://github.com/feedreader/pluto.live) - sample planet site; rails web app in ruby using the pluto-models gem
 
 
 

data/Rakefile

@@ -8,7 +8,7 @@ Hoe.spec 'pluto-models' do
   self.summary = "pluto-models - planet schema 'n' models for easy (re)use"
   self.description = summary
 
-  self.urls    = ['https://github.com/feedreader/pluto-models']
+  self.urls    = ['https://github.com/feedreader/pluto.models']
 
   self.author  = 'Gerald Bauer'
   self.email   = 'feedreader@googlegroups.com'
@@ -20,8 +20,9 @@ Hoe.spec 'pluto-models' do
   self.extra_deps = [
     ['props',         '>= 1.1.2'],
     ['logutils',      '>= 0.6.1'],
-    ['feedutils',     '>= 0.4.0'],
-    ['textutils',     '>= 0.10.0'],
+    ['feedparser',    '>= 1.0.0'],
+    ['feedfilter',    '>= 1.1.1'],
+    ['textutils',     '>= 1.0.1'],
     ['activerecord'],
     ['logutils-activerecord', '>= 0.2.0'],
     ['props-activerecord',    '>= 0.1.0'],

data/lib/pluto/models.rb

@@ -19,8 +19,10 @@ require 'active_record'
 
 require 'props'     # manage settings/env
 require 'logutils'
-require 'feedutils'
 require 'textutils'
+require 'feedparser'
+require 'feedfilter'
+
 
 ## add more activerecords addons/utils
 require 'activerecord/utils'     # add macros e.g. read_attr_w_fallbacks etc.
@@ -108,45 +110,5 @@ end  # module Pluto
 
 
 
-######
-# todo - move to ext/array.rb or similar
-
-class Array
-
-  ## todo: check if there's already a builtin method for this
-  #
-  #  note:
-  #   in rails ary.in_groups(3)  results in
-  #          top-to-bottom, left-to-right.
-  #  and not left-to-right first and than top-to-bottom.
-  #
-  #  rename to in_groups_vertical(3) ???
-
-  def in_columns( cols )  # alias for convenience for chunks - needed? why? why not?
-    chunks( cols )
-  end
-
-  def chunks( number_of_chunks )
-    ## NB: use chunks - columns might be in use by ActiveRecord! 
-    ###
-    # e.g.
-    #  [1,2,3,4,5,6,7,8,9,10].columns(3)
-    #   becomes:
-    #  [[1,4,7,10],
-    #   [2,5,8],
-    #   [3,6,9]]
-
-    ## check/todo: make a copy of the array first??
-    #  for now reference to original items get added to columns
-    chunks = (1..number_of_chunks).collect { [] }
-    each_with_index do |item,index|
-      chunks[ index % number_of_chunks ] << item
-    end
-    chunks
-  end
-
-end
-
-
 # say hello
 puts Pluto.banner   if $DEBUG || (defined?($RUBYLIBS_DEBUG) && $RUBYLIBS_DEBUG) 

data/lib/pluto/models/feed.rb

@@ -21,9 +21,9 @@ class Feed < ActiveRecord::Base
     # note: order by first non-null datetime field
     #   coalesce - supported by sqlite (yes), postgres (yes)
 
-    # note: if not published, touched or built use hardcoded 1971-01-01 for now
-    ## order( "coalesce(published,touched,built,'1971-01-01') desc" )
-    order( "coalesce(feeds.last_published,'1971-01-01') desc" )
+    # note: if not updated or published use hardcoded 1971-01-01 for now
+    ## order( "coalesce(updated,published,'1971-01-01') desc" )
+    order( "coalesce(feeds.items_last_updated,'1971-01-01') desc" )
   end
 
   ##################################
@@ -34,6 +34,7 @@ class Feed < ActiveRecord::Base
   def name()        title;    end  # alias    for title
   def description() summary;  end  # alias    for summary
   def desc()        summary;  end  # alias(2) for summary
+  def subtitle()    summary;  end  # alias(3) for summary
   def link()        url;      end  # alias    for url
   def feed()        feed_url; end  # alias    for feed_url
 
@@ -46,28 +47,28 @@ class Feed < ActiveRecord::Base
 
   def url?()           read_attribute(:url).present?;       end
   def title?()         read_attribute(:title).present?;     end
-  def title2?()        read_attribute(:title2).present?;    end
   def feed_url?()      read_attribute(:feed_url).present?;  end
 
   def url()      read_attribute_w_fallbacks( :url,      :auto_url );      end
   def title()    read_attribute_w_fallbacks( :title,    :auto_title );    end
-  def title2()   read_attribute_w_fallbacks( :title2,   :auto_title2 );   end
   def feed_url() read_attribute_w_fallbacks( :feed_url, :auto_feed_url ); end
 
+  def summary?()      read_attribute(:summary).present?;  end
 
+
+  def updated?()    read_attribute(:updated).present?;    end
   def published?()  read_attribute(:published).present?;  end
-  def touched?()    read_attribute(:touched).present?;    end
 
+  def updated
+    ## todo/fix: use a new name - do NOT squeeze convenience lookup into existing
+    #    db backed attribute
+    read_attribute_w_fallbacks( :updated, :published )
+  end
 
   def published
     ## todo/fix: use a new name - do NOT squeeze convenience lookup into existing
     #    db backed attribute
-
-    read_attribute_w_fallbacks(
-       :published,  
-       :touched,     # try touched (aka updated (ATOM))
-       :built        # try build (aka lastBuildDate (RSS))
-    )
+    read_attribute_w_fallbacks( :published, :updated )
   end
 
 
@@ -75,44 +76,23 @@ class Feed < ActiveRecord::Base
   def debug?()       @debug || false;  end
 
 
-  def match_terms?( terms, text )   ### make helper method private - why? why not??
-    return false  if text.blank?     ## allow/guard against nil and empty string
+  def deep_update_from_struct!( data )
 
-    terms.each do |term|
-      if /#{term}/i =~ text      ## Note: lets ignore case (use i regex option) 
-        return true
-      end
+    ######
+    ## check for filters (includes/excludes) if present
+    ##  for now just check for includes
+    ##
+    if includes.present?
+      includesFilter = FeedFilter::IncludeFilters.new( includes )
+    else
+      includesFilter = nil
     end
 
-    false  # no term match found
-  end
-
-
-
-  def save_from_struct!( data )
-
-    update_from_struct!( data )
-    
     data.items.each do |item|
-
-      ######
-      ## check for filters (includes/excludes) if present
-      ##  for now just check for includes
-      ##
-      if includes.present?
-        ## split terms (allow comma,pipe) - do NOT use space; allows e.g. terms such as github pages
-        terms = includes.split( /\s*[,|]\s*/ )
-        ## remove leading and trailing white spaces - check - still required when using \s* ??
-        terms = terms.map { |term| term.strip }
-        match = match_terms?( terms, item.title  ) ||
-                match_terms?( terms, item.summary) ||
-                match_terms?( terms, item.content)
-        
-        if match == false
-          puts "** SKIPPING | #{item.title}"
-          puts "  no include terms match: #{terms.join('|')}"
-          next   ## skip to next item
-        end
+      if includesFilter && includesFilter.match_item?( item ) == false
+        puts "** SKIPPING | #{item.title}"
+        puts "  no include terms match: #{includes}"
+        next   ## skip to next item
       end
 
       item_rec = Item.find_by_guid( item.guid )
@@ -123,17 +103,37 @@ class Feed < ActiveRecord::Base
         ## todo: check if any attribs changed
         puts "UPDATE | #{item.title}"
       end
-      
+
       item_rec.debug = debug? ? true : false  # pass along debug flag
-      item_rec.update_from_struct!( self, item )
+
+      item_rec.feed_id = id        # feed_rec.id - add feed_id fk_ref
+      item_rec.fetched = fetched   # feed_rec.fetched
+
+      item_rec.update_from_struct!( item )
 
     end  # each item
-  end
+
+
+    #  update  cached value last published for item
+    ##  todo/check: force reload of items - why? why not??
+    last_item_rec = items.latest.limit(1).first  # note limit(1) will return relation/arrar - use first to get first element or nil from ary
+    if last_item_rec.present?
+      if last_item_rec.updated?
+        self.items_last_updated = last_item_rec.updated
+        ## save!  ## note: will get save w/ update_from_struct!  - why? why not??
+      else # try published
+        self.items_last_updated = last_item_rec.published
+        ## save!  ## note: will get save w/ update_from_struct!  - why? why not??
+      end
+    end
+
+    update_from_struct!( data )
+  end # method deep_update_from_struct!
 
 
   def update_from_struct!( data )
 
-    ## todo: move to FeedUtils::Feed ??? why? why not??
+    ## todo: move to FeedParser::Feed ??? why? why not??
     if data.generator
       generator_full = ''
       generator_full << data.generator
@@ -145,9 +145,9 @@ class Feed < ActiveRecord::Base
 
 ##
 # todo:
-##  strip all tags from title2
+##  strip all tags from summary (subtitle)
 ##  limit to 255 chars
-## e.g. title2 such as this exist
+## e.g. summary (subtitle) such as this exist
 ##  This is a low-traffic announce-only list for people interested
 ##  in hearing news about Polymer (<a href="http://polymer-project.org">http://polymer-project.org</a>).
 ## The higher-traffic mailing list for all kinds of discussion is
@@ -156,16 +156,14 @@ class Feed < ActiveRecord::Base
 
     feed_attribs = {
         format:       data.format,
+        updated:      data.updated,
         published:    data.published,
-        touched:      data.updated,
-        built:        data.built,
         summary:      data.summary,
+        generator:    generator_full,
         ### todo/fix: add/use
         # auto_title:     ???,
         # auto_url:       ???,
         # auto_feed_url:  ???,
-        auto_title2:  data.title2 ? strip_tags(data.title2)[0...255] : data.title2,   # limit to 255 chars; strip tags
-        generator:    generator_full
       }
 
     if debug?

data/lib/pluto/models/item.rb

@@ -11,6 +11,7 @@ class Item < ActiveRecord::Base
   ## todo/fix:
   ##  use a module ref or something; do NOT include all methods - why? why not? 
   include TextUtils::HypertextHelper   ## e.g. lets us use strip_tags( ht )
+  include FeedFilter::AdsFilter        ## e.g. lets us use strip_ads( ht )
 
 
   ##################################
@@ -25,41 +26,45 @@ class Item < ActiveRecord::Base
     # note: order by first non-null datetime field
     #   coalesce - supported by sqlite (yes), postgres (yes)
 
-    # note: if not published,touched or built_at use hardcoded 1971-01-01 for now
-    order( "coalesce(items.published,items.touched,'1971-01-01') desc" )
+    # note: if not updated,published use hardcoded 1971-01-01 for now
+    order( "coalesce(items.updated,items.published,'1971-01-01') desc" )
   end
 
-  def published?()  read_attribute(:published).present?;  end
+  def updated?()    read_attribute(:updated).present?;  end
+  def published?()  read_attribute(:published).present?;  end   # note: published is basically an alias for created
 
-  def published
+  def updated
     ## todo/fix: use a new name - do NOT squeeze convenience lookup into existing
     #    db backed attribute
-
-    read_attribute_w_fallbacks(
-      :published,
-      :touched       # try touched (aka updated RSS/ATOM)
-    )
+    read_attribute_w_fallbacks( :updated, :published )
   end
 
+  def published
+    ## todo/fix: use a new name - do NOT squeeze convenience lookup into existing
+    #    db backed attribute
+    read_attribute_w_fallbacks( :published, :updated )
+  end
 
 
   def debug=(value)  @debug = value;   end
   def debug?()       @debug || false;  end
 
-  def update_from_struct!( feed_rec, data )
+
+  def update_from_struct!( data )
     ## check: new item/record?  not saved?  add guid
     #   otherwise do not add guid  - why? why not?
 
+    ## note: for now also strip ads in summary
+    ##  fix/todo: summary (in the future) is supposed to be only plain vanilla text
+
     item_attribs = {
       guid:         data.guid,   # todo: only add for new records???
       title:        data.title ? strip_tags(data.title)[0...255] : data.title,   ## limit to 255 chars; strip tags
       url:          data.url,
-      summary:      data.summary,
-      content:      data.content,
+      summary:      data.summary.blank? ? data.summary : strip_ads( data.summary ).strip,
+      content:      data.content.blank? ? data.content : strip_ads( data.content ).strip,
+      updated:      data.updated,
       published:    data.published,
-      touched:      data.updated,
-      feed_id:      feed_rec.id,    # add feed_id fk_ref
-      fetched:      feed_rec.fetched
     }
 
     if debug?

data/lib/pluto/models/site.rb

@@ -13,7 +13,6 @@ class Site < ActiveRecord::Base
   ##################################
   # attribute reader aliases
   def name()        title;    end  # alias for title
-  def fetched_at()  fetched;  end  # - legacy attrib reader -- remove!!!
 
   def owner_name()  author;  end  # alias    for author
   def owner()       author;  end  # alias(2) for author
@@ -22,8 +21,179 @@ class Site < ActiveRecord::Base
   def owner_email()  email;  end  # alias    for email
   def author_email() email;  end  # alias(2) for email
 
+
+
+  def self.deep_create_or_update_from_hash!( name, config, opts={} )
+
+    ## note: allow (optional) config of site key too
+    site_key = config['key'] || config['slug']
+    if site_key.nil?
+      ## if no key configured; use (file)name; remove -_ chars
+      ##   e.g. jekyll-meta becomes jekyllmeta etc.
+      site_key = name.downcase.gsub( /[\-_]/, '' )
+    end
+
+    site_rec = Site.find_by_key( site_key )
+    if site_rec.nil?
+      site_rec        = Site.new
+      site_rec.key    = site_key
+    end
+
+    site_rec.deep_update_from_hash!( config, opts )
+    site_rec
+  end
+
+
+  def deep_update_from_hash!( config, opts={} )
+
+    site_attribs = {
+      title:   config['title']  || config['name'],   # support either title or name
+      url:     config['source'] || config['url'],    # support source or url   for source url for auto-update (optional)
+      author:  config['author'] || config['owner'],
+      email:   config['email'],
+      updated: Time.now,   ## track last_update via pluto (w/ update_subscription_for fn)
+    }
+
+    ## note: allow (optional) config of site key too
+    site_key = config['key'] || config['slug']
+    site_attribs[:key] = site_key   if site_key
+
+
+    logger = LogUtils::Logger.root
+    logger.debug "site_attribs: #{site_attribs.inspect}"
+
+    if new_record?
+      ## use object_id: site.id and object_type: Site
+      ## change - model/table/schema!!!
+      Activity.create!( text: "new site >#{key}< - #{title}" )
+    end
+    
+    update_attributes!( site_attribs )
+
+
+    # -- log update activity
+    ##  Activity.create!( text: "update subscriptions for site >#{key}<" )
+
+    #### todo/fix:
+    ##  double check - how to handle delete
+    ##    feeds might get referenced by other sites
+    ##   cannot just delete feeds; only save to delete join table (subscriptions)
+    ##   check if feed "lingers" on with no reference (to site)???
+
+    # clean out subscriptions and add again
+    logger.debug "before site.subscriptions.delete_all - count: #{subscriptions.count}"
+    # note: use destroy_all NOT delete_all (delete_all tries by default only nullify)
+    subscriptions.destroy_all
+    logger.debug "after site.subscriptions.delete_all - count: #{subscriptions.count}"
+
+
+    config.each do |k, v|
+
+      ## todo: downcase key - why ??? why not???
+
+      # skip "top-level" feed keys e.g. title, etc. or planet planet sections (e.g. planet,defaults)
+      next if ['key','slug',
+               'title','name','name2','title2','subtitle',
+               'source', 'url',
+               'include','includes','exclude','excludes',
+               'feeds',
+               'author', 'owner', 'email',
+               'planet','defaults'].include?( k )
+
+      ### todo/check:
+      ##   check value - must be hash
+      #     check if url or feed_url present
+      #      that is, check for required props/key-value pairs
+
+      feed_key   = k.to_s.dup
+      feed_hash  = v
+
+      # todo/fix: use title from feed?
+      #  e.g. fill up auto_title, auto_url, etc.
+
+      feed_attribs = {
+        feed_url: feed_hash[ 'feed' ]  || feed_hash[ 'feed_url' ] || feed_hash[ 'xml_url' ],
+        url:      feed_hash[ 'link' ]  || feed_hash[ 'url' ] || feed_hash[ 'html_url' ],
+        title:    feed_hash[ 'title' ] || feed_hash[ 'name' ],
+## note: title2 no longer supported; use summary or subtitle?
+###     title2:   feed_hash[ 'title2' ] || feed_hash[ 'name2' ] || feed_hash[ 'subtitle'],
+        includes: feed_hash[ 'includes' ] || feed_hash[ 'include' ],
+        excludes: feed_hash[ 'excludes' ] || feed_hash[ 'exclude' ],
+        author:   feed_hash[ 'author' ] || feed_hash[ 'owner' ],
+        email:    feed_hash[ 'email' ],
+        avatar:   feed_hash[ 'avatar'   ] || feed_hash[ 'face'],
+        location: feed_hash[ 'location' ],
+        github:   feed_hash[ 'github'  ],
+        twitter:  feed_hash[ 'twitter' ],
+        rubygems: feed_hash[ 'rubygems' ],
+        meetup:   feed_hash[ 'meetup' ],   ### -- remove from schema - virtual attrib ?? - why? why not??
+      }
+
+      feed_attribs[:encoding] = feed_hash['encoding']||feed_hash['charset']    if feed_hash['encoding']||feed_hash['charset']
+
+#####
+##
+# auto-fill; convenience helpers
+
+    if feed_hash['meetup']
+##  link/url      = http://www.meetup.com/vienna-rb
+##  feed/feed_url = http://www.meetup.com/vienna-rb/events/rss/vienna.rb/
+
+      feed_attribs[:url]      = "http://www.meetup.com/#{feed_hash['meetup']}"    if feed_attribs[:url].nil?
+      feed_attribs[:feed_url] = "http://www.meetup.com/#{feed_hash['meetup']}/events/rss/#{feed_hash['meetup']}/"    if feed_attribs[:feed_url].nil?
+    end
+
+    if feed_hash['googlegroups']
+##  link/url      = https://groups.google.com/group/beerdb or
+##                  https://groups.google.com/forum/#!forum/beerdb
+##  feed/feed_url = https://groups.google.com/forum/feed/beerdb/topics/atom.xml?num=15
+
+      feed_attribs[:url]      = "https://groups.google.com/group/#{feed_hash['googlegroups']}"    if feed_attribs[:url].nil?
+      feed_attribs[:feed_url] = "https://groups.google.com/forum/feed//#{feed_hash['googlegroups']}/topics/atom.xml?num=15"    if feed_attribs[:feed_url].nil?
+    end
+
+    if feed_hash['github'] && feed_hash['github'].index('/')  ## e.g. jekyll/jekyll
+## link/url      = https://github.com/jekyll/jekyll
+## feed/feed_url = https://github.com/jekyll/jekyll/commits/master.atom
+
+      feed_attribs[:url]      = "https://github.com/#{feed_hash['github']}"    if feed_attribs[:url].nil?
+      feed_attribs[:feed_url] = "https://github.com/#{feed_hash['github']}/commits/master.atom"    if feed_attribs[:feed_url].nil?
+    end
+
+    if feed_hash['rubygems'] && feed_attribs[:url].nil? && feed_attribs[:feed_url].nil?
+## link/url      = http://rubygems.org/gems/jekyll
+## feed/feed_url = http://rubygems.org/gems/jekyll/versions.atom
+
+      feed_attribs[:url]      = "http://rubygems.org/gems/#{feed_hash['rubygems']}"    if feed_attribs[:url].nil?
+      feed_attribs[:feed_url] = "http://rubygems.org/gems/#{feed_hash['rubygems']}/versions.atom"    if feed_attribs[:feed_url].nil?
+    end
+
+
+      puts "Updating feed subscription >#{feed_key}< - >#{feed_attribs[:feed_url]}<..."
+
+      feed_rec = Feed.find_by_key( feed_key )
+      if feed_rec.nil?
+        feed_rec           = Feed.new
+        feed_attribs[:key] = feed_key
+
+        ## use object_id: feed.id and object_type: Feed
+        ## change - model/table/schema!!!
+        ## todo: add parent_action_id - why? why not?
+        Activity.create!( text: "new feed >#{feed_key}< - #{feed_attribs[:title]}" )
+      end
+
+      feed_rec.update_attributes!( feed_attribs )
+
+      #  add subscription record
+      #   note: subscriptions get cleaned out on update first (see above)
+      subscriptions.create!( feed_id: feed_rec.id )   
+    end
+
+  end # method deep_update_from_hash!
+
 end # class Site
 
 
   end # module Model
 end # module Pluto
+

data/lib/pluto/models/utils.rb

@@ -10,16 +10,16 @@ class ItemCursor
   end
 
   def each
-    last_published = Time.local( 1971, 1, 1 )
+    last_updated      = Time.local( 1971, 1, 1 )
     last_feed_id      = -1     ## todo: use feed_key instead of id?? why? why not??
-    
+
     @items.each do |item|
 
-      item_published = item.published  # cache published value ref
+      item_updated = item.updated  # cache updated value ref
 
-      if last_published.year   == item_published.year  &&
-         last_published.month  == item_published.month &&
-         last_published.day    == item_published.day 
+      if last_updated.year   == item_updated.year  &&
+         last_updated.month  == item_updated.month &&
+         last_updated.day    == item_updated.day
         new_date = false
       else
         new_date = true
@@ -37,7 +37,7 @@ class ItemCursor
 
       yield( item, new_date, new_feed )
 
-      last_published = item.published
+      last_updated   = item.updated
       last_feed_id   = item.feed.id
     end
   end # method each

data/lib/pluto/schema.rb

@@ -35,6 +35,7 @@ class CreateDb < ActiveRecord::Migration
 
       # note: do NOT store body content (that is, text) and md5 digest
       #   use git! and github! commit will be http_etag!!
+      t.string   :md5       # md5 hash of body
 
 
       #############
@@ -53,37 +54,35 @@ class CreateDb < ActiveRecord::Migration
     create_table :feeds do |t|
       t.string  :key,       null: false
       t.string  :encoding,  null: false, default: 'utf8'  # charset encoding; default to utf8
-      t.string  :format      # e.g. atom (1.0), rss 2.0, rss 0.7 etc.
-
-      t.string  :title        # user supplied title
-      t.string  :auto_title   # "fallback" - auto(fill) title from feed
-
-      t.string  :title2        # user supplied title2
-      t.string  :auto_title2   # "fallback" - auto(fill) title2 from feed e.g. subtitle (atom)
-
-      t.string  :url          # user supplied site url
-      t.string  :auto_url     # "fallback" - auto(fill) url from feed
+      t.string  :format      # e.g. atom (1.0), rss 2.0, etc.
 
+      t.string  :title          # user supplied title
+      t.string  :url            # user supplied site url
       t.string  :feed_url       # user supplied feed url
+
+      t.string  :auto_title     # "fallback" - auto(fill) title from feed
+      t.string  :auto_url       # "fallback" - auto(fill) url from feed
       t.string  :auto_feed_url  # "fallback" - auto discovery feed url from (site) url
 
-      t.text    :summary    # e.g. description (rss)
+      t.text    :summary    # e.g. description (rss), subtitle (atom)
+      ## todo: add auto_summary  - why? why not?
 
       t.string  :generator   # feed generator (e.g. wordpress, etc.)  from feed
-      
-      t.datetime :published  # from feed published(atom)+ pubDate(rss)
-      t.datetime :built      # from feed lastBuiltDate(rss)
-      t.datetime :touched    # from feed updated(atom)
+
+      t.datetime :updated    # from feed updated(atom) + lastBuildDate(rss)
+      t.datetime :published  # from feed published(atom) + pubDate(rss) - note: published basically an alias for created
 
 
       ### extras (move to array for custom fields or similar??)
-      t.string   :author   # author_name, owner_name
-      t.string   :email    # author_email, owner_email
-      t.string   :avatar   # gravator or hackergotchi handle (optional)
+      t.string   :author    # author_name, owner_name
+      t.string   :email     # author_email, owner_email
+      t.string   :avatar    # gravator or hackergotchi handle (optional)
+      t.string   :location  # e.g. Vienna > Austria,  Bamberg > Germany etc. (optional)
 
-      t.string   :github   # github handle  (optional)
-      t.string   :twitter  # twitter handle (optional)
-      t.string   :meetup   # meetup handle (optional)
+      t.string   :github     # github handle  (optional)
+      t.string   :rubygems   # rubygems handle (optional)
+      t.string   :twitter    # twitter handle (optional)
+      t.string   :meetup     # meetup handle (optional)
 
 
       ### add class/kind field e.g.
@@ -100,7 +99,7 @@ class CreateDb < ActiveRecord::Migration
       # todo: add generic filter list e.g. t.string :filters  (comma,pipe or space separated method names?)
 
       # -- our own (meta) fields
-      t.datetime :last_published  # cache last (latest) published for items - e.g. latest date from publisehd item
+      t.datetime :items_last_updated  # cache last (latest) updated for items - e.g. latest date from updated item
       t.datetime :fetched    # last fetched date via pluto
 
       t.integer  :http_code   # last http status code e.g. 200,404,etc.
@@ -123,15 +122,15 @@ class CreateDb < ActiveRecord::Migration
 
       ## note: title may contain more than 255 chars!!
       ## e.g. Rails Girls blog has massive titles in feed
-      ## cut-off/limit to 255
+      ## cut-off/limit to 255 - why?? why not??
       ##  also strip tags in titles - why? why not?? - see feed.title2/auto_title2
 
-      t.string   :title   # todo: add some :null => false ??
+      t.text     :title   # todo: add some :null => false ??
       t.text     :summary  # e.g. description (rss), summary (atom)
       t.text     :content
 
-      t.datetime :published   # from feed (published)  + pubDate(rss)
-      t.datetime :touched     # from feed updated (atom)
+      t.datetime :updated     # from feed updated (atom) + pubDate(rss)
+      t.datetime :published   # from feed published (atom) -- note: published is basically an alias for created
 
       ## todo: add :last_updated_at ??  (NOTE: updated_at already take by auto-timestamps)
       t.references :feed, null: false

data/lib/pluto/version.rb

@@ -3,8 +3,8 @@
 module Pluto
 
   MAJOR = 1
-  MINOR = 3
-  PATCH = 2
+  MINOR = 4
+  PATCH = 0
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
   def self.version

data/test/data/ruby.ini

@@ -0,0 +1,18 @@
+title  = Planet Ruby
+source = https://github.com/feedreader/pluto.models/raw/master/test/data/ruby.ini
+
+[rubylang]
+  title = Ruby Lang News
+  link  = http://www.ruby-lang.org/en/news
+  feed  = http://www.ruby-lang.org/en/feeds/news.rss
+
+[rubyonrails]
+  title = Ruby on Rails News
+  link  = http://weblog.rubyonrails.org
+  feed  = http://weblog.rubyonrails.org/feed/atom.xml
+
+[viennarb]
+  title = Vienna.rb News
+  link  = http://vienna-rb.at
+  feed  = http://vienna-rb.at/atom.xml
+

data/test/test_filter.rb

@@ -23,34 +23,37 @@ class TestFilter < MiniTest::Test
       title: 'Test'
     )
 
-    feed_data = FeedUtils::Feed.new
+    feed_data = FeedParser::Feed.new
     feed_data.title = 'Test'
     feed_data.items = []
     
-    item_data = FeedUtils::Item.new
+    item_data = FeedParser::Item.new
     item_data.title = 'Test #1'
     item_data.summary = 'Test'
     item_data.content = 'Test'
+    item_data.updated = Date.today
 
     feed_data.items << item_data
 
-    item_data = FeedUtils::Item.new
+    item_data = FeedParser::Item.new
     item_data.title = 'Test #2'
     item_data.summary = "Test\nTest\nTest1"
     item_data.content = 'Test'
+    item_data.updated = Date.today
 
     feed_data.items << item_data
 
 
-    item_data = FeedUtils::Item.new
+    item_data = FeedParser::Item.new
     item_data.title = 'Test #3'
     item_data.summary = "Test\nTest\nTest"
     item_data.content = 'Test\nTest\nGitHub Pages'
+    item_data.updated = Date.today
 
     feed_data.items << item_data
 
-    feed1.save_from_struct!( feed_data )   ## check w/ includes
-    ## feed2.save_from_struct!( feed_data )   ## check w/o includes
+    feed1.deep_update_from_struct!( feed_data )   ## check w/ includes
+    feed2.deep_update_from_struct!( feed_data )   ## check w/o includes
 
     assert true    ## if we get here it should workd
   end

data/test/test_helpers.rb

@@ -10,7 +10,13 @@ require 'helper'
 
 class TestHelper < MiniTest::Test
 
-  ## add some tests; to be done
+  def setup
+    Log.delete_all
+    Site.delete_all
+    Feed.delete_all
+    Subscription.delete_all
+    Item.delete_all
+  end
 
   def test_banner
     puts Pluto.banner

data/test/test_site.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+
+###
+#  to run use
+#     ruby -I ./lib -I ./test test/test_site.rb
+#  or better
+#     rake test
+
+require 'helper'
+
+class TestSite < MiniTest::Test
+
+  def setup
+    Log.delete_all
+    Site.delete_all
+    Feed.delete_all
+    Subscription.delete_all
+    Item.delete_all
+  end
+
+
+  def test_site_create
+    site_text   = File.read( "#{Pluto.root}/test/data/ruby.ini")
+    site_config = INI.load( site_text )
+    ## pp site_config
+
+    assert_equal 0, Site.count
+    assert_equal 0, Feed.count
+
+    Site.deep_create_or_update_from_hash!( 'ruby', site_config )
+
+    assert_equal 1, Site.count
+    assert_equal 3, Feed.count
+
+    ruby = Site.find_by_key!( 'ruby' )
+    assert_equal 'Planet Ruby', ruby.title
+    assert_equal 3, ruby.subscriptions.count
+    assert_equal 3, ruby.feeds.count
+
+    rubylang = Feed.find_by_key!( 'rubylang' )
+    assert_equal 'Ruby Lang News', rubylang.title
+    assert_equal 'http://www.ruby-lang.org/en/news', rubylang.url
+    assert_equal 'http://www.ruby-lang.org/en/feeds/news.rss', rubylang.feed_url
+  end
+
+  def test_site_update
+    site_text   = File.read( "#{Pluto.root}/test/data/ruby.ini")
+    site_config = INI.load( site_text )
+    ## pp site_config
+
+    assert_equal 0, Site.count
+    assert_equal 0, Feed.count
+
+    ## note: call twice (first time for create, second time for update)
+    Site.deep_create_or_update_from_hash!( 'ruby', site_config )
+    Site.deep_create_or_update_from_hash!( 'ruby', site_config )
+
+    assert_equal 1, Site.count
+    assert_equal 3, Feed.count
+
+    ruby = Site.find_by_key!( 'ruby' )
+    assert_equal 'Planet Ruby', ruby.title
+    assert_equal 3, ruby.subscriptions.count
+    assert_equal 3, ruby.feeds.count
+
+    rubylang = Feed.find_by_key!( 'rubylang' )
+    assert_equal 'Ruby Lang News', rubylang.title
+    assert_equal 'http://www.ruby-lang.org/en/news', rubylang.url
+    assert_equal 'http://www.ruby-lang.org/en/feeds/news.rss', rubylang.feed_url
+  end
+
+
+end # class TestSite
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pluto-models
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.4.0
 platform: ruby
 authors:
 - Gerald Bauer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-26 00:00:00.000000000 Z
+date: 2015-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: props
@@ -39,33 +39,47 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.6.1
 - !ruby/object:Gem::Dependency
-  name: feedutils
+  name: feedparser
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: feedfilter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.1.1
 - !ruby/object:Gem::Dependency
   name: textutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -188,10 +202,12 @@ files:
 - lib/pluto/models/utils.rb
 - lib/pluto/schema.rb
 - lib/pluto/version.rb
+- test/data/ruby.ini
 - test/helper.rb
 - test/test_filter.rb
 - test/test_helpers.rb
-homepage: https://github.com/feedreader/pluto-models
+- test/test_site.rb
+homepage: https://github.com/feedreader/pluto.models
 licenses:
 - Public Domain
 metadata: {}
@@ -219,4 +235,5 @@ specification_version: 4
 summary: pluto-models - planet schema 'n' models for easy (re)use
 test_files:
 - test/test_filter.rb
+- test/test_site.rb
 - test/test_helpers.rb